From 179ec2dc348f5fb5966d8a5913925573bfeb74bb Mon Sep 17 00:00:00 2001 From: madjin <32600939+madjin@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:10:22 -0400 Subject: [PATCH 1/8] update docs --- docs/{docs => api}/_category_.yml | 0 docs/{docs => api}/classes/BgentRuntime.md | 0 docs/{docs => api}/classes/DatabaseAdapter.md | 0 docs/{docs => api}/classes/MemoryManager.md | 0 .../classes/SqlJsDatabaseAdapter.md | 0 .../classes/SqliteDatabaseAdapter.md | 0 .../classes/SupabaseDatabaseAdapter.md | 0 docs/{docs => api}/classes/_category_.yml | 0 docs/{docs => api}/concepts.md | 0 docs/{docs => api}/docs/_category_.yml | 0 docs/{docs => api}/docs/index.md | 0 docs/{docs => api}/docs/modules.md | 0 docs/{docs => api}/enums/GoalStatus.md | 0 docs/{docs => api}/enums/_category_.yml | 0 docs/{docs => api}/functions/_category_.yml | 0 docs/{docs => api}/functions/addHeader.md | 0 docs/{docs => api}/functions/addLore.md | 0 .../functions/composeActionExamples.md | 0 .../{docs => api}/functions/composeContext.md | 0 docs/{docs => api}/functions/createGoal.md | 0 .../functions/createRelationship.md | 0 .../functions/formatActionConditions.md | 0 .../functions/formatActionNames.md | 0 docs/{docs => api}/functions/formatActions.md | 0 docs/{docs => api}/functions/formatActors.md | 0 .../functions/formatEvaluatorConditions.md | 0 .../formatEvaluatorExampleConditions.md | 0 .../formatEvaluatorExampleDescriptions.md | 0 .../functions/formatEvaluatorExamples.md | 0 .../functions/formatEvaluatorNames.md | 0 .../functions/formatEvaluators.md | 0 .../functions/formatGoalsAsString.md | 0 docs/{docs => api}/functions/formatLore.md | 0 .../{docs => api}/functions/formatMessages.md | 0 .../functions/formatRelationships.md | 0 .../functions/getActorDetails.md | 0 .../functions/getFormattedActions.md | 0 docs/{docs => api}/functions/getGoals.md | 0 docs/{docs => api}/functions/getLore.md | 0 docs/{docs => api}/functions/getProviders.md | 0 .../functions/getRelationship.md | 0 .../functions/getRelationships.md | 0 .../functions/parseJSONObjectFromText.md | 0 .../functions/parseJsonArrayFromText.md | 0 docs/{docs => api}/functions/updateGoal.md | 0 docs/api/guides/advanced.md | 0 docs/api/guides/basic-usage.md | 0 docs/api/guides/configuration.md | 0 docs/{docs => api}/index.md | 0 docs/api/installation.md | 0 docs/{docs => api}/interfaces/Account.md | 0 docs/{docs => api}/interfaces/Action.md | 0 .../{docs => api}/interfaces/ActionExample.md | 0 docs/{docs => api}/interfaces/Actor.md | 0 docs/{docs => api}/interfaces/Content.md | 0 .../interfaces/ConversationExample.md | 0 .../interfaces/EvaluationExample.md | 0 docs/{docs => api}/interfaces/Evaluator.md | 0 docs/{docs => api}/interfaces/Goal.md | 0 docs/{docs => api}/interfaces/Memory.md | 0 docs/{docs => api}/interfaces/Message.md | 0 .../interfaces/MessageExample.md | 0 docs/{docs => api}/interfaces/Objective.md | 0 docs/{docs => api}/interfaces/Participant.md | 0 docs/{docs => api}/interfaces/Provider.md | 0 docs/{docs => api}/interfaces/Relationship.md | 0 docs/{docs => api}/interfaces/Room.md | 0 docs/{docs => api}/interfaces/State.md | 0 docs/{docs => api}/interfaces/_category_.yml | 0 docs/api/intro.md | 0 docs/{docs => api}/modules.md | 0 docs/api/quickstart.md | 0 docs/{docs => api}/rooms.md | 0 docs/{docs => api}/types/Handler.md | 0 docs/{docs => api}/types/Validator.md | 0 docs/{docs => api}/types/_category_.yml | 0 docs/{docs => api}/variables/_category_.yml | 0 .../{docs => api}/variables/defaultActions.md | 0 .../variables/defaultEvaluators.md | 0 .../variables/defaultProviders.md | 0 .../variables/embeddingDimension.md | 0 .../variables/embeddingZeroVector.md | 0 .../variables/evaluationTemplate.md | 0 .../variables/messageHandlerTemplate.md | 0 docs/backup_docusaurus.config.js | 131 +++ docs/backup_sidebars.js | 20 + docs/docs/guides/advanced.md | 0 docs/docs/guides/basic-usage.md | 0 docs/docs/guides/characterfile.md | 173 ++++ docs/docs/guides/configuration.md | 0 docs/docs/installation.md | 126 +++ docs/docs/intro.md | 20 + docs/docs/quickstart.md | 98 +++ docs/docusaurus.config.js | 119 +-- docs/package-lock.json | 826 +++++++++++++++++- docs/package.json | 1 + docs/sidebars.api.js | 11 + docs/sidebars.js | 36 +- docs/{ => static/img}/eliza_banner.jpg | Bin 99 files changed, 1468 insertions(+), 93 deletions(-) rename docs/{docs => api}/_category_.yml (100%) rename docs/{docs => api}/classes/BgentRuntime.md (100%) rename docs/{docs => api}/classes/DatabaseAdapter.md (100%) rename docs/{docs => api}/classes/MemoryManager.md (100%) rename docs/{docs => api}/classes/SqlJsDatabaseAdapter.md (100%) rename docs/{docs => api}/classes/SqliteDatabaseAdapter.md (100%) rename docs/{docs => api}/classes/SupabaseDatabaseAdapter.md (100%) rename docs/{docs => api}/classes/_category_.yml (100%) rename docs/{docs => api}/concepts.md (100%) rename docs/{docs => api}/docs/_category_.yml (100%) rename docs/{docs => api}/docs/index.md (100%) rename docs/{docs => api}/docs/modules.md (100%) rename docs/{docs => api}/enums/GoalStatus.md (100%) rename docs/{docs => api}/enums/_category_.yml (100%) rename docs/{docs => api}/functions/_category_.yml (100%) rename docs/{docs => api}/functions/addHeader.md (100%) rename docs/{docs => api}/functions/addLore.md (100%) rename docs/{docs => api}/functions/composeActionExamples.md (100%) rename docs/{docs => api}/functions/composeContext.md (100%) rename docs/{docs => api}/functions/createGoal.md (100%) rename docs/{docs => api}/functions/createRelationship.md (100%) rename docs/{docs => api}/functions/formatActionConditions.md (100%) rename docs/{docs => api}/functions/formatActionNames.md (100%) rename docs/{docs => api}/functions/formatActions.md (100%) rename docs/{docs => api}/functions/formatActors.md (100%) rename docs/{docs => api}/functions/formatEvaluatorConditions.md (100%) rename docs/{docs => api}/functions/formatEvaluatorExampleConditions.md (100%) rename docs/{docs => api}/functions/formatEvaluatorExampleDescriptions.md (100%) rename docs/{docs => api}/functions/formatEvaluatorExamples.md (100%) rename docs/{docs => api}/functions/formatEvaluatorNames.md (100%) rename docs/{docs => api}/functions/formatEvaluators.md (100%) rename docs/{docs => api}/functions/formatGoalsAsString.md (100%) rename docs/{docs => api}/functions/formatLore.md (100%) rename docs/{docs => api}/functions/formatMessages.md (100%) rename docs/{docs => api}/functions/formatRelationships.md (100%) rename docs/{docs => api}/functions/getActorDetails.md (100%) rename docs/{docs => api}/functions/getFormattedActions.md (100%) rename docs/{docs => api}/functions/getGoals.md (100%) rename docs/{docs => api}/functions/getLore.md (100%) rename docs/{docs => api}/functions/getProviders.md (100%) rename docs/{docs => api}/functions/getRelationship.md (100%) rename docs/{docs => api}/functions/getRelationships.md (100%) rename docs/{docs => api}/functions/parseJSONObjectFromText.md (100%) rename docs/{docs => api}/functions/parseJsonArrayFromText.md (100%) rename docs/{docs => api}/functions/updateGoal.md (100%) create mode 100644 docs/api/guides/advanced.md create mode 100644 docs/api/guides/basic-usage.md create mode 100644 docs/api/guides/configuration.md rename docs/{docs => api}/index.md (100%) create mode 100644 docs/api/installation.md rename docs/{docs => api}/interfaces/Account.md (100%) rename docs/{docs => api}/interfaces/Action.md (100%) rename docs/{docs => api}/interfaces/ActionExample.md (100%) rename docs/{docs => api}/interfaces/Actor.md (100%) rename docs/{docs => api}/interfaces/Content.md (100%) rename docs/{docs => api}/interfaces/ConversationExample.md (100%) rename docs/{docs => api}/interfaces/EvaluationExample.md (100%) rename docs/{docs => api}/interfaces/Evaluator.md (100%) rename docs/{docs => api}/interfaces/Goal.md (100%) rename docs/{docs => api}/interfaces/Memory.md (100%) rename docs/{docs => api}/interfaces/Message.md (100%) rename docs/{docs => api}/interfaces/MessageExample.md (100%) rename docs/{docs => api}/interfaces/Objective.md (100%) rename docs/{docs => api}/interfaces/Participant.md (100%) rename docs/{docs => api}/interfaces/Provider.md (100%) rename docs/{docs => api}/interfaces/Relationship.md (100%) rename docs/{docs => api}/interfaces/Room.md (100%) rename docs/{docs => api}/interfaces/State.md (100%) rename docs/{docs => api}/interfaces/_category_.yml (100%) create mode 100644 docs/api/intro.md rename docs/{docs => api}/modules.md (100%) create mode 100644 docs/api/quickstart.md rename docs/{docs => api}/rooms.md (100%) rename docs/{docs => api}/types/Handler.md (100%) rename docs/{docs => api}/types/Validator.md (100%) rename docs/{docs => api}/types/_category_.yml (100%) rename docs/{docs => api}/variables/_category_.yml (100%) rename docs/{docs => api}/variables/defaultActions.md (100%) rename docs/{docs => api}/variables/defaultEvaluators.md (100%) rename docs/{docs => api}/variables/defaultProviders.md (100%) rename docs/{docs => api}/variables/embeddingDimension.md (100%) rename docs/{docs => api}/variables/embeddingZeroVector.md (100%) rename docs/{docs => api}/variables/evaluationTemplate.md (100%) rename docs/{docs => api}/variables/messageHandlerTemplate.md (100%) create mode 100644 docs/backup_docusaurus.config.js create mode 100644 docs/backup_sidebars.js create mode 100644 docs/docs/guides/advanced.md create mode 100644 docs/docs/guides/basic-usage.md create mode 100644 docs/docs/guides/characterfile.md create mode 100644 docs/docs/guides/configuration.md create mode 100644 docs/docs/installation.md create mode 100644 docs/docs/intro.md create mode 100644 docs/docs/quickstart.md create mode 100644 docs/sidebars.api.js rename docs/{ => static/img}/eliza_banner.jpg (100%) diff --git a/docs/docs/_category_.yml b/docs/api/_category_.yml similarity index 100% rename from docs/docs/_category_.yml rename to docs/api/_category_.yml diff --git a/docs/docs/classes/BgentRuntime.md b/docs/api/classes/BgentRuntime.md similarity index 100% rename from docs/docs/classes/BgentRuntime.md rename to docs/api/classes/BgentRuntime.md diff --git a/docs/docs/classes/DatabaseAdapter.md b/docs/api/classes/DatabaseAdapter.md similarity index 100% rename from docs/docs/classes/DatabaseAdapter.md rename to docs/api/classes/DatabaseAdapter.md diff --git a/docs/docs/classes/MemoryManager.md b/docs/api/classes/MemoryManager.md similarity index 100% rename from docs/docs/classes/MemoryManager.md rename to docs/api/classes/MemoryManager.md diff --git a/docs/docs/classes/SqlJsDatabaseAdapter.md b/docs/api/classes/SqlJsDatabaseAdapter.md similarity index 100% rename from docs/docs/classes/SqlJsDatabaseAdapter.md rename to docs/api/classes/SqlJsDatabaseAdapter.md diff --git a/docs/docs/classes/SqliteDatabaseAdapter.md b/docs/api/classes/SqliteDatabaseAdapter.md similarity index 100% rename from docs/docs/classes/SqliteDatabaseAdapter.md rename to docs/api/classes/SqliteDatabaseAdapter.md diff --git a/docs/docs/classes/SupabaseDatabaseAdapter.md b/docs/api/classes/SupabaseDatabaseAdapter.md similarity index 100% rename from docs/docs/classes/SupabaseDatabaseAdapter.md rename to docs/api/classes/SupabaseDatabaseAdapter.md diff --git a/docs/docs/classes/_category_.yml b/docs/api/classes/_category_.yml similarity index 100% rename from docs/docs/classes/_category_.yml rename to docs/api/classes/_category_.yml diff --git a/docs/docs/concepts.md b/docs/api/concepts.md similarity index 100% rename from docs/docs/concepts.md rename to docs/api/concepts.md diff --git a/docs/docs/docs/_category_.yml b/docs/api/docs/_category_.yml similarity index 100% rename from docs/docs/docs/_category_.yml rename to docs/api/docs/_category_.yml diff --git a/docs/docs/docs/index.md b/docs/api/docs/index.md similarity index 100% rename from docs/docs/docs/index.md rename to docs/api/docs/index.md diff --git a/docs/docs/docs/modules.md b/docs/api/docs/modules.md similarity index 100% rename from docs/docs/docs/modules.md rename to docs/api/docs/modules.md diff --git a/docs/docs/enums/GoalStatus.md b/docs/api/enums/GoalStatus.md similarity index 100% rename from docs/docs/enums/GoalStatus.md rename to docs/api/enums/GoalStatus.md diff --git a/docs/docs/enums/_category_.yml b/docs/api/enums/_category_.yml similarity index 100% rename from docs/docs/enums/_category_.yml rename to docs/api/enums/_category_.yml diff --git a/docs/docs/functions/_category_.yml b/docs/api/functions/_category_.yml similarity index 100% rename from docs/docs/functions/_category_.yml rename to docs/api/functions/_category_.yml diff --git a/docs/docs/functions/addHeader.md b/docs/api/functions/addHeader.md similarity index 100% rename from docs/docs/functions/addHeader.md rename to docs/api/functions/addHeader.md diff --git a/docs/docs/functions/addLore.md b/docs/api/functions/addLore.md similarity index 100% rename from docs/docs/functions/addLore.md rename to docs/api/functions/addLore.md diff --git a/docs/docs/functions/composeActionExamples.md b/docs/api/functions/composeActionExamples.md similarity index 100% rename from docs/docs/functions/composeActionExamples.md rename to docs/api/functions/composeActionExamples.md diff --git a/docs/docs/functions/composeContext.md b/docs/api/functions/composeContext.md similarity index 100% rename from docs/docs/functions/composeContext.md rename to docs/api/functions/composeContext.md diff --git a/docs/docs/functions/createGoal.md b/docs/api/functions/createGoal.md similarity index 100% rename from docs/docs/functions/createGoal.md rename to docs/api/functions/createGoal.md diff --git a/docs/docs/functions/createRelationship.md b/docs/api/functions/createRelationship.md similarity index 100% rename from docs/docs/functions/createRelationship.md rename to docs/api/functions/createRelationship.md diff --git a/docs/docs/functions/formatActionConditions.md b/docs/api/functions/formatActionConditions.md similarity index 100% rename from docs/docs/functions/formatActionConditions.md rename to docs/api/functions/formatActionConditions.md diff --git a/docs/docs/functions/formatActionNames.md b/docs/api/functions/formatActionNames.md similarity index 100% rename from docs/docs/functions/formatActionNames.md rename to docs/api/functions/formatActionNames.md diff --git a/docs/docs/functions/formatActions.md b/docs/api/functions/formatActions.md similarity index 100% rename from docs/docs/functions/formatActions.md rename to docs/api/functions/formatActions.md diff --git a/docs/docs/functions/formatActors.md b/docs/api/functions/formatActors.md similarity index 100% rename from docs/docs/functions/formatActors.md rename to docs/api/functions/formatActors.md diff --git a/docs/docs/functions/formatEvaluatorConditions.md b/docs/api/functions/formatEvaluatorConditions.md similarity index 100% rename from docs/docs/functions/formatEvaluatorConditions.md rename to docs/api/functions/formatEvaluatorConditions.md diff --git a/docs/docs/functions/formatEvaluatorExampleConditions.md b/docs/api/functions/formatEvaluatorExampleConditions.md similarity index 100% rename from docs/docs/functions/formatEvaluatorExampleConditions.md rename to docs/api/functions/formatEvaluatorExampleConditions.md diff --git a/docs/docs/functions/formatEvaluatorExampleDescriptions.md b/docs/api/functions/formatEvaluatorExampleDescriptions.md similarity index 100% rename from docs/docs/functions/formatEvaluatorExampleDescriptions.md rename to docs/api/functions/formatEvaluatorExampleDescriptions.md diff --git a/docs/docs/functions/formatEvaluatorExamples.md b/docs/api/functions/formatEvaluatorExamples.md similarity index 100% rename from docs/docs/functions/formatEvaluatorExamples.md rename to docs/api/functions/formatEvaluatorExamples.md diff --git a/docs/docs/functions/formatEvaluatorNames.md b/docs/api/functions/formatEvaluatorNames.md similarity index 100% rename from docs/docs/functions/formatEvaluatorNames.md rename to docs/api/functions/formatEvaluatorNames.md diff --git a/docs/docs/functions/formatEvaluators.md b/docs/api/functions/formatEvaluators.md similarity index 100% rename from docs/docs/functions/formatEvaluators.md rename to docs/api/functions/formatEvaluators.md diff --git a/docs/docs/functions/formatGoalsAsString.md b/docs/api/functions/formatGoalsAsString.md similarity index 100% rename from docs/docs/functions/formatGoalsAsString.md rename to docs/api/functions/formatGoalsAsString.md diff --git a/docs/docs/functions/formatLore.md b/docs/api/functions/formatLore.md similarity index 100% rename from docs/docs/functions/formatLore.md rename to docs/api/functions/formatLore.md diff --git a/docs/docs/functions/formatMessages.md b/docs/api/functions/formatMessages.md similarity index 100% rename from docs/docs/functions/formatMessages.md rename to docs/api/functions/formatMessages.md diff --git a/docs/docs/functions/formatRelationships.md b/docs/api/functions/formatRelationships.md similarity index 100% rename from docs/docs/functions/formatRelationships.md rename to docs/api/functions/formatRelationships.md diff --git a/docs/docs/functions/getActorDetails.md b/docs/api/functions/getActorDetails.md similarity index 100% rename from docs/docs/functions/getActorDetails.md rename to docs/api/functions/getActorDetails.md diff --git a/docs/docs/functions/getFormattedActions.md b/docs/api/functions/getFormattedActions.md similarity index 100% rename from docs/docs/functions/getFormattedActions.md rename to docs/api/functions/getFormattedActions.md diff --git a/docs/docs/functions/getGoals.md b/docs/api/functions/getGoals.md similarity index 100% rename from docs/docs/functions/getGoals.md rename to docs/api/functions/getGoals.md diff --git a/docs/docs/functions/getLore.md b/docs/api/functions/getLore.md similarity index 100% rename from docs/docs/functions/getLore.md rename to docs/api/functions/getLore.md diff --git a/docs/docs/functions/getProviders.md b/docs/api/functions/getProviders.md similarity index 100% rename from docs/docs/functions/getProviders.md rename to docs/api/functions/getProviders.md diff --git a/docs/docs/functions/getRelationship.md b/docs/api/functions/getRelationship.md similarity index 100% rename from docs/docs/functions/getRelationship.md rename to docs/api/functions/getRelationship.md diff --git a/docs/docs/functions/getRelationships.md b/docs/api/functions/getRelationships.md similarity index 100% rename from docs/docs/functions/getRelationships.md rename to docs/api/functions/getRelationships.md diff --git a/docs/docs/functions/parseJSONObjectFromText.md b/docs/api/functions/parseJSONObjectFromText.md similarity index 100% rename from docs/docs/functions/parseJSONObjectFromText.md rename to docs/api/functions/parseJSONObjectFromText.md diff --git a/docs/docs/functions/parseJsonArrayFromText.md b/docs/api/functions/parseJsonArrayFromText.md similarity index 100% rename from docs/docs/functions/parseJsonArrayFromText.md rename to docs/api/functions/parseJsonArrayFromText.md diff --git a/docs/docs/functions/updateGoal.md b/docs/api/functions/updateGoal.md similarity index 100% rename from docs/docs/functions/updateGoal.md rename to docs/api/functions/updateGoal.md diff --git a/docs/api/guides/advanced.md b/docs/api/guides/advanced.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/api/guides/basic-usage.md b/docs/api/guides/basic-usage.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/api/guides/configuration.md b/docs/api/guides/configuration.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/index.md b/docs/api/index.md similarity index 100% rename from docs/docs/index.md rename to docs/api/index.md diff --git a/docs/api/installation.md b/docs/api/installation.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/interfaces/Account.md b/docs/api/interfaces/Account.md similarity index 100% rename from docs/docs/interfaces/Account.md rename to docs/api/interfaces/Account.md diff --git a/docs/docs/interfaces/Action.md b/docs/api/interfaces/Action.md similarity index 100% rename from docs/docs/interfaces/Action.md rename to docs/api/interfaces/Action.md diff --git a/docs/docs/interfaces/ActionExample.md b/docs/api/interfaces/ActionExample.md similarity index 100% rename from docs/docs/interfaces/ActionExample.md rename to docs/api/interfaces/ActionExample.md diff --git a/docs/docs/interfaces/Actor.md b/docs/api/interfaces/Actor.md similarity index 100% rename from docs/docs/interfaces/Actor.md rename to docs/api/interfaces/Actor.md diff --git a/docs/docs/interfaces/Content.md b/docs/api/interfaces/Content.md similarity index 100% rename from docs/docs/interfaces/Content.md rename to docs/api/interfaces/Content.md diff --git a/docs/docs/interfaces/ConversationExample.md b/docs/api/interfaces/ConversationExample.md similarity index 100% rename from docs/docs/interfaces/ConversationExample.md rename to docs/api/interfaces/ConversationExample.md diff --git a/docs/docs/interfaces/EvaluationExample.md b/docs/api/interfaces/EvaluationExample.md similarity index 100% rename from docs/docs/interfaces/EvaluationExample.md rename to docs/api/interfaces/EvaluationExample.md diff --git a/docs/docs/interfaces/Evaluator.md b/docs/api/interfaces/Evaluator.md similarity index 100% rename from docs/docs/interfaces/Evaluator.md rename to docs/api/interfaces/Evaluator.md diff --git a/docs/docs/interfaces/Goal.md b/docs/api/interfaces/Goal.md similarity index 100% rename from docs/docs/interfaces/Goal.md rename to docs/api/interfaces/Goal.md diff --git a/docs/docs/interfaces/Memory.md b/docs/api/interfaces/Memory.md similarity index 100% rename from docs/docs/interfaces/Memory.md rename to docs/api/interfaces/Memory.md diff --git a/docs/docs/interfaces/Message.md b/docs/api/interfaces/Message.md similarity index 100% rename from docs/docs/interfaces/Message.md rename to docs/api/interfaces/Message.md diff --git a/docs/docs/interfaces/MessageExample.md b/docs/api/interfaces/MessageExample.md similarity index 100% rename from docs/docs/interfaces/MessageExample.md rename to docs/api/interfaces/MessageExample.md diff --git a/docs/docs/interfaces/Objective.md b/docs/api/interfaces/Objective.md similarity index 100% rename from docs/docs/interfaces/Objective.md rename to docs/api/interfaces/Objective.md diff --git a/docs/docs/interfaces/Participant.md b/docs/api/interfaces/Participant.md similarity index 100% rename from docs/docs/interfaces/Participant.md rename to docs/api/interfaces/Participant.md diff --git a/docs/docs/interfaces/Provider.md b/docs/api/interfaces/Provider.md similarity index 100% rename from docs/docs/interfaces/Provider.md rename to docs/api/interfaces/Provider.md diff --git a/docs/docs/interfaces/Relationship.md b/docs/api/interfaces/Relationship.md similarity index 100% rename from docs/docs/interfaces/Relationship.md rename to docs/api/interfaces/Relationship.md diff --git a/docs/docs/interfaces/Room.md b/docs/api/interfaces/Room.md similarity index 100% rename from docs/docs/interfaces/Room.md rename to docs/api/interfaces/Room.md diff --git a/docs/docs/interfaces/State.md b/docs/api/interfaces/State.md similarity index 100% rename from docs/docs/interfaces/State.md rename to docs/api/interfaces/State.md diff --git a/docs/docs/interfaces/_category_.yml b/docs/api/interfaces/_category_.yml similarity index 100% rename from docs/docs/interfaces/_category_.yml rename to docs/api/interfaces/_category_.yml diff --git a/docs/api/intro.md b/docs/api/intro.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/modules.md b/docs/api/modules.md similarity index 100% rename from docs/docs/modules.md rename to docs/api/modules.md diff --git a/docs/api/quickstart.md b/docs/api/quickstart.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/rooms.md b/docs/api/rooms.md similarity index 100% rename from docs/docs/rooms.md rename to docs/api/rooms.md diff --git a/docs/docs/types/Handler.md b/docs/api/types/Handler.md similarity index 100% rename from docs/docs/types/Handler.md rename to docs/api/types/Handler.md diff --git a/docs/docs/types/Validator.md b/docs/api/types/Validator.md similarity index 100% rename from docs/docs/types/Validator.md rename to docs/api/types/Validator.md diff --git a/docs/docs/types/_category_.yml b/docs/api/types/_category_.yml similarity index 100% rename from docs/docs/types/_category_.yml rename to docs/api/types/_category_.yml diff --git a/docs/docs/variables/_category_.yml b/docs/api/variables/_category_.yml similarity index 100% rename from docs/docs/variables/_category_.yml rename to docs/api/variables/_category_.yml diff --git a/docs/docs/variables/defaultActions.md b/docs/api/variables/defaultActions.md similarity index 100% rename from docs/docs/variables/defaultActions.md rename to docs/api/variables/defaultActions.md diff --git a/docs/docs/variables/defaultEvaluators.md b/docs/api/variables/defaultEvaluators.md similarity index 100% rename from docs/docs/variables/defaultEvaluators.md rename to docs/api/variables/defaultEvaluators.md diff --git a/docs/docs/variables/defaultProviders.md b/docs/api/variables/defaultProviders.md similarity index 100% rename from docs/docs/variables/defaultProviders.md rename to docs/api/variables/defaultProviders.md diff --git a/docs/docs/variables/embeddingDimension.md b/docs/api/variables/embeddingDimension.md similarity index 100% rename from docs/docs/variables/embeddingDimension.md rename to docs/api/variables/embeddingDimension.md diff --git a/docs/docs/variables/embeddingZeroVector.md b/docs/api/variables/embeddingZeroVector.md similarity index 100% rename from docs/docs/variables/embeddingZeroVector.md rename to docs/api/variables/embeddingZeroVector.md diff --git a/docs/docs/variables/evaluationTemplate.md b/docs/api/variables/evaluationTemplate.md similarity index 100% rename from docs/docs/variables/evaluationTemplate.md rename to docs/api/variables/evaluationTemplate.md diff --git a/docs/docs/variables/messageHandlerTemplate.md b/docs/api/variables/messageHandlerTemplate.md similarity index 100% rename from docs/docs/variables/messageHandlerTemplate.md rename to docs/api/variables/messageHandlerTemplate.md diff --git a/docs/backup_docusaurus.config.js b/docs/backup_docusaurus.config.js new file mode 100644 index 0000000000..df68a49028 --- /dev/null +++ b/docs/backup_docusaurus.config.js @@ -0,0 +1,131 @@ +// @ts-check +// `@type` JSDoc annotations allow editor autocompletion and type checking +// (when paired with `@ts-check`). +// There are various equivalent ways to declare your Docusaurus config. +// See: https://docusaurus.io/docs/api/docusaurus-config + +import { themes as prismThemes } from 'prism-react-renderer' + +/** @type {import('@docusaurus/types').Config} */ +const config = { + title: 'eliza', + tagline: 'The flexible, scalable AI agent for everyone', + favicon: 'img/favicon.ico', + + // Set the production url of your site here + url: 'https://docs.ai16z.ai', + // Set the // pathname under which your site is served + // For GitHub pages deployment, it is often '//' + baseUrl: '/', + + // GitHub pages deployment config. + // If you aren't using GitHub pages, you don't need these. + organizationName: 'ai16z', // Usually your GitHub org/user name. + projectName: 'eliza', // Usually your repo name. + + onBrokenLinks: 'throw', + onBrokenMarkdownLinks: 'warn', + + // Even if you don't use internationalization, you can use this field to set + // useful metadata like html lang. For example, if your site is Chinese, you + // may want to replace "en" with "zh-Hans". + i18n: { + defaultLocale: 'en', + locales: ['en'] + }, + + plugins: [ + [ + 'docusaurus-plugin-typedoc', + + // Plugin / TypeDoc options + { + entryPoints: ['../src/index.ts'], + tsconfig: '../tsconfig.json', + out: "./docs", + }, + ], + ], + + presets: [ + [ + 'classic', + /** @type {import('@docusaurus/preset-classic').Options} */ + ({ + docs: { + sidebarPath: './sidebars.js', + // Please change this to your repo. + // Remove this to remove the "edit this page" links. + editUrl: + 'https://github.com/ai16z/eliza/tree/main/docs/' + }, + theme: { + customCss: './src/css/custom.css' + } + }) + ] + ], + + themeConfig: + /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ + ({ + // Replace with your project's social card + image: 'img/docusaurus-social-card.jpg', + navbar: { + title: 'eliza', + items: [ + { + type: 'docSidebar', + sidebarId: 'tutorialSidebar', + position: 'left', + label: 'API Documentation' + }, + { + href: 'https://github.com/ai16z/eliza', + label: 'GitHub', + position: 'right' + } + ] + }, + footer: { + style: 'dark', + links: [ + { + title: 'Docs', + items: [ + ] + }, + + { + title: 'Community', + items: [ + { + label: 'Discord', + href: 'https://discord.gg/xe7HtWru' + }, + { + label: 'Twitter', + href: 'https://twitter.com/pmairca' + } + ] + }, + { + title: 'More', + items: [ + { + label: 'GitHub', + href: 'https://github.com/ai16z/eliza' + } + ] + } + ], + copyright: `Copyright © ${new Date().getFullYear()} ai16z.ai` + }, + prism: { + theme: prismThemes.github, + darkTheme: prismThemes.dracula + } + }) +} + +export default config diff --git a/docs/backup_sidebars.js b/docs/backup_sidebars.js new file mode 100644 index 0000000000..14554ca1d4 --- /dev/null +++ b/docs/backup_sidebars.js @@ -0,0 +1,20 @@ +/** + * Creating a sidebar enables you to: + - create an ordered group of docs + - render a sidebar for each doc of that group + - provide next/previous navigation + + The sidebars can be generated from the filesystem, or explicitly defined here. + + Create as many sidebars as you want. + */ + +// @ts-check + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + // By default, Docusaurus generates a sidebar from the docs folder structure + tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], +}; + +export default sidebars; diff --git a/docs/docs/guides/advanced.md b/docs/docs/guides/advanced.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/guides/basic-usage.md b/docs/docs/guides/basic-usage.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/guides/characterfile.md b/docs/docs/guides/characterfile.md new file mode 100644 index 0000000000..4002058d27 --- /dev/null +++ b/docs/docs/guides/characterfile.md @@ -0,0 +1,173 @@ +--- +sidebar_position: 4 +--- + + +# Character Files + +Character files are JSON-formatted configurations that define an AI character's personality, knowledge, and behavior patterns. This guide explains how to create effective character files for use with LLM agents. + +## Structure Overview + +A character file contains several key sections that work together to define the character's personality and behavior: + +```json +{ + "name": "character_name", + "bio": [], + "lore": [], + "knowledge": [], + "messageExamples": [], + "postExamples": [], + "topics": [], + "style": {}, + "adjectives": [] +} +``` + +## Core Components + +### Bio Array +- Contains biographical information about the character +- Can be a single comprehensive biography or multiple shorter statements +- Multiple statements are randomized to create variety in responses +- Example: +```json +"bio": [ + "Mark Andreessen is an American entrepreneur and investor", + "Co-founder of Netscape and Andreessen Horowitz", + "Pioneer of the early web, created NCSA Mosaic" +] +``` + +### Lore Array +- Contains interesting facts and details about the character +- Helps define personality and unique traits +- Gets randomly sampled during conversations +- Example: +```json +"lore": [ + "Believes strongly in the power of software to transform industries", + "Known for saying 'Software is eating the world'", + "Early investor in Facebook, Twitter, and other tech giants" +] +``` + +### Knowledge Array +- Used for RAG (Retrieval Augmented Generation) +- Can contain chunks of text from articles, books, or other sources +- Helps ground the character's responses in factual information +- Can be generated from PDFs or other documents using provided tools + +### Message Examples +- Sample conversations between users and the character +- Helps establish the character's conversational style +- Should cover various topics and scenarios +- Example: +```json +"messageExamples": [ + [ + {"user": "user1", "content": {"text": "What's your view on AI?"}}, + {"user": "character", "content": {"text": "AI is transforming every industry..."}} + ] +] +``` + +### Style Object +Contains three key sections: +1. `all`: General style instructions for all interactions +2. `chat`: Specific instructions for chat interactions +3. `post`: Specific instructions for social media posts + +Each section can contain multiple instructions that guide the character's communication style. + +### Topics Array +- List of subjects the character is interested in or knowledgeable about +- Used to guide conversations and generate relevant content +- Helps maintain character consistency + +### Adjectives Array +- Words that describe the character's traits and personality +- Used for generating responses with consistent tone +- Can be used in "Mad Libs" style content generation + +## Best Practices + +1. **Randomization for Variety** + - Break bio and lore into smaller chunks + - This creates more natural, varied responses + - Prevents repetitive or predictable behavior + +2. **Knowledge Management** + - Use the provided tools to convert documents into knowledge: + ```bash + npx folder2knowledge + npx knowledge2character + ``` + +3. **Style Instructions** + - Be specific about communication patterns + - Include both dos and don'ts + - Consider platform-specific behavior (chat vs posts) + +4. **Message Examples** + - Include diverse scenarios + - Show character-specific responses + - Demonstrate typical interaction patterns + +## Tools and Utilities + +1. **Generate from Twitter** +```bash +npx tweets2character +``` + +2. **Convert Documents to Knowledge** +```bash +npx folder2knowledge +``` + +3. **Add Knowledge to Character** +```bash +npx knowledge2character +``` + +## Context Length Considerations + +- Modern LLMs support longer contexts (128k tokens) +- No strict limits on section lengths +- Focus on quality and relevance rather than size +- Consider randomization for large collections of information + +## Validation + +You can validate your character file against the schema using provided tools: +```bash +# Python +python examples/validate.py + +# JavaScript +node examples/validate.mjs +``` + +## Tips for Quality + +1. **Bio and Lore** + - Mix factual and personality-defining information + - Include both historical and current details + - Break into modular, reusable pieces + +2. **Style Instructions** + - Be specific about tone and mannerisms + - Include platform-specific guidance + - Define clear boundaries and limitations + +3. **Examples** + - Cover common scenarios + - Show character-specific reactions + - Demonstrate proper tone and style + +4. **Knowledge** + - Focus on relevant information + - Organize in digestible chunks + - Update regularly to maintain relevance diff --git a/docs/docs/guides/configuration.md b/docs/docs/guides/configuration.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/installation.md b/docs/docs/installation.md new file mode 100644 index 0000000000..81b1c952bb --- /dev/null +++ b/docs/docs/installation.md @@ -0,0 +1,126 @@ +--- +sidebar_position: 3 +--- + +# Installation + +Currently eliza is dependent on Supabase for local development. You can install it with the following command: + +`npm install eliza` + +## Select your database adapter + +``` +npm install sqlite-vss better-sqlite3 # for sqlite (simple, for local development) + +npm install @supabase/supabase-js # for supabase (more complicated but can be deployed at scale) +``` + + +### Set up environment variables + +You will need a Supbase account, as well as an OpenAI developer account. + +Copy and paste the .dev.vars.example to .dev.vars and fill in the environment variables: + +``` +SUPABASE_URL="https://your-supabase-url.supabase.co" +SUPABASE_SERVICE_API_KEY="your-supabase-service-api-key" +OPENAI_API_KEY="your-openai-api-key" +``` + +### SQLite Local Setup (Easiest) +You can use SQLite for local development. This is the easiest way to get started with eliza. + +``` +import { BgentRuntime, SqliteDatabaseAdapter } from "eliza"; +import { Database } from "sqlite3"; +const sqliteDatabaseAdapter = new SqliteDatabaseAdapter(new Database(":memory:")); + +const runtime = new BgentRuntime({ + serverUrl: "https://api.openai.com/v1", + token: process.env.OPENAI_API_KEY, // Can be an API key or JWT token for your AI services + databaseAdapter: sqliteDatabaseAdapter, + // ... other options +}); +``` + +### Supabase Local Setup +First, you will need to install the Supabase CLI. You can install it using the instructions here. + +Once you have the CLI installed, you can run the following commands to set up a local Supabase instance: + + +``` +supabase init +supabase start +``` + +You can now start the eliza project with `npm run dev` and it will connect to the local Supabase instance by default. + +NOTE: You will need Docker installed for this to work. If that is an issue for you, use the Supabase Cloud Setup instructions instead below). + +### Supabase Cloud Setup +This library uses Supabase as a database. You can set up a free account at supabase.io and create a new project. + +* Step 1: On the Subase All Projects Dashboard, select “New Project”. +* Step 2: Select the organization to store the new project in, assign a database name, password and region. +* Step 3: Select “Create New Project”. +* Step 4: Wait for the database to setup. This will take a few minutes as supabase setups various directories. +* Step 5: Select the “SQL Editor” tab from the left navigation menu. +* Step 6: Copy in your own SQL dump file or optionally use the provided file in the eliza directory at: "src/supabase/db.sql". Note: You can use the command "supabase db dump" if you have a pre-exisiting supabase database to generate the SQL dump file. +* Step 7: Paste the SQL code into the SQL Editor and hit run in the bottom right. +* Step 8: Select the “Databases” tab from the left navigation menu to verify all of the tables have been added properly. + +Once you've set up your Supabase project, you can find your API key by going to the "Settings" tab and then "API". You will need to set the` SUPABASE_URL and SUPABASE_SERVICE_API_KEY` environment variables in your `.dev.vars` file. + +### Local Model Setup + +While eliza uses ChatGPT 3.5 by default, you can use a local model by setting the serverUrl to a local endpoint. The LocalAI project is a great way to run a local model with a compatible API endpoint. + + +``` +const runtime = new BgentRuntime({ + serverUrl: process.env.LOCALAI_URL, + token: process.env.LOCALAI_TOKEN, // Can be an API key or JWT token for your AI service + // ... other options +}); +``` + +### Development + +``` +npm run dev # start the server + +npm run shell # start the shell in another terminal to talk to the default agent +``` + +### Usage + +``` + import { BgentRuntime, SupabaseDatabaseAdapter, SqliteDatabaseAdapter } from "eliza"; + + const sqliteDatabaseAdapter = new SqliteDatabaseAdapter(new Database(":memory:")); + + + // You can also use Supabase like this + // const supabaseDatabaseAdapter = new SupabaseDatabaseAdapter( + // process.env.SUPABASE_URL, + // process.env.SUPABASE_SERVICE_API_KEY) + // ; + + + const runtime = new BgentRuntime({ + serverUrl: "https://api.openai.com/v1", + token: process.env.OPENAI_API_KEY, // Can be an API key or JWT token for your AI services + databaseAdapter: sqliteDatabaseAdapter, + actions: [ + /* your custom actions */ + ], + evaluators: [ + /* your custom evaluators */ + ], + model: "gpt-3.5-turbo", // whatever model you want to use + embeddingModel: "text-embedding-3-small", // whatever model you want to use + }); +``` diff --git a/docs/docs/intro.md b/docs/docs/intro.md new file mode 100644 index 0000000000..3034c5ece5 --- /dev/null +++ b/docs/docs/intro.md @@ -0,0 +1,20 @@ +--- +sidebar_position: 1 +--- + +# Introduction + +![](/img/eliza_banner.jpg) + +*As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)* + +- Multi-agent simulation framework +- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) +- Full-featured Discord and Twitter connectors, with Discord voice channel support +- Full conversational and document RAG memory +- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more +- Highly extensible - create your own actions and clients to extend Eliza's capabilities +- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) +- Supports OpenAI for cloud inference on a light-weight device +- "Ask Claude" mode for calling Claude on more complex queries +- 100% Typescript diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md new file mode 100644 index 0000000000..3d3d43ca3c --- /dev/null +++ b/docs/docs/quickstart.md @@ -0,0 +1,98 @@ +--- +sidebar_position: 2 +--- + +# Quickstart + +## Install Node.js +https://docs.npmjs.com/downloading-and-installing-node-js-and-npm + +## Using pnpm +We use pnpm to manage our dependencies. It is faster and more efficient than npm, and it supports workspaces. +https://pnpm.io/installation + +## Edit the .env file +- Copy .env.example to .env and fill in the appropriate values +- Edit the TWITTER environment variables to add your bot's username and password + +## Edit the character file +- Check out the file `src/core/defaultCharacter.ts` - you can modify this +- You can also load characters with the `node --loader ts-node/esm src/index.ts --characters="path/to/your/character.json"` and run multiple bots at the same time. + +### Run with Llama +You can run Llama 70B or 405B models by setting the `XAI_MODEL` environment variable to `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` or `meta-llama/Meta-Llama-3.1-405B-Instruct` + +### Run with Grok +You can run Grok models by setting the `XAI_MODEL` environment variable to `grok-beta` + +### Run with OpenAI +You can run OpenAI models by setting the `XAI_MODEL` environment variable to `gpt-4o-mini` or `gpt-4o` + +# Requires Node 20+ +If you are getting strange issues when starting up, make sure you're using Node 20+. Some APIs are not compatible with previous versions. You can check your node version with `node -v`. If you need to install a new version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm). + +## Additional Requirements +You may need to install Sharp. If you see an error when starting up, try installing it with the following command: +``` +pnpm install --include=optional sharp +``` + +# Environment Setup + +You will need to add environment variables to your .env file to connect to various platforms: +``` +# Required environment variables +# Start Discord +DISCORD_APPLICATION_ID= +DISCORD_API_TOKEN= # Bot token + +# Start Twitter +TWITTER_USERNAME= # Account username +TWITTER_PASSWORD= # Account password +TWITTER_EMAIL= # Account email +TWITTER_COOKIES= # Account cookies +``` + +# Local Setup + +## CUDA Setup + +If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. +``` +pnpm install +npx --no node-llama-cpp source download --gpu cuda +``` + +Make sure that you've installed the CUDA Toolkit, including cuDNN and cuBLAS. + +## Running locally +Add XAI_MODEL and set it to one of the above options from [Run with +Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it +downloads the model from huggingface and queries it locally + +# Cloud Setup (with OpenAI) + +In addition to the environment variables above, you will need to add the following: +``` +# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. +OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk- + +# The agent can also ask Claude for help if you have an API key +ANTHROPIC_API_KEY= + +# For Elevenlabs voice generation on Discord voice +ELEVENLABS_XI_API_KEY= # API key from elevenlabs + +# ELEVENLABS SETTINGS +ELEVENLABS_MODEL_ID=eleven_multilingual_v2 +ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM +ELEVENLABS_VOICE_STABILITY=0.5 +ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 +ELEVENLABS_VOICE_STYLE=0.66 +ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false +ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 +ELEVENLABS_OUTPUT_FORMAT=pcm_16000 +``` + +# Discord Bot +For help with setting up your Discord Bot, check out here: https://discordjs.guide/preparations/setting-up-a-bot-application.html diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index c16e74d74b..aa434ef18c 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -1,48 +1,43 @@ // @ts-check -// `@type` JSDoc annotations allow editor autocompletion and type checking -// (when paired with `@ts-check`). -// There are various equivalent ways to declare your Docusaurus config. -// See: https://docusaurus.io/docs/api/docusaurus-config - -import { themes as prismThemes } from 'prism-react-renderer' +import { themes as prismThemes } from 'prism-react-renderer'; /** @type {import('@docusaurus/types').Config} */ const config = { title: 'eliza', tagline: 'The flexible, scalable AI agent for everyone', favicon: 'img/favicon.ico', - - // Set the production url of your site here - url: 'https://docs.ai16z.vc', - // Set the // pathname under which your site is served - // For GitHub pages deployment, it is often '//' + url: 'https://docs.ai16z.ai', baseUrl: '/', - - // GitHub pages deployment config. - // If you aren't using GitHub pages, you don't need these. - organizationName: 'ai16z', // Usually your GitHub org/user name. - projectName: 'eliza', // Usually your repo name. - + organizationName: 'ai16z', + projectName: 'eliza', onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'warn', - // Even if you don't use internationalization, you can use this field to set - // useful metadata like html lang. For example, if your site is Chinese, you - // may want to replace "en" with "zh-Hans". i18n: { defaultLocale: 'en', - locales: ['en'] + locales: ['en'], }, plugins: [ + // TypeDoc plugin for API documentation [ 'docusaurus-plugin-typedoc', - - // Plugin / TypeDoc options { entryPoints: ['../src/index.ts'], tsconfig: '../tsconfig.json', - out: "./docs", + out: './api', // Changed to output directly to api folder + }, + ], + // Search functionality + require.resolve('docusaurus-lunr-search'), + // Separate API docs plugin instance + [ + '@docusaurus/plugin-content-docs', + { + id: 'api', + path: 'api', + routeBasePath: 'api', + sidebarPath: './sidebars.api.js', }, ], ], @@ -54,78 +49,48 @@ const config = { ({ docs: { sidebarPath: './sidebars.js', - // Please change this to your repo. - // Remove this to remove the "edit this page" links. - editUrl: - 'https://github.com/ai16z/eliza/tree/main/docs/' + editUrl: 'https://github.com/ai16z/eliza/tree/main/docs/', + routeBasePath: 'docs', }, theme: { - customCss: './src/css/custom.css' - } - }) - ] + customCss: './src/css/custom.css', + }, + }), + ], ], themeConfig: /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({ - // Replace with your project's social card - image: 'img/docusaurus-social-card.jpg', navbar: { title: 'eliza', + logo: { + alt: 'Eliza Logo', + src: 'img/favicon.ico', + }, items: [ { type: 'docSidebar', sidebarId: 'tutorialSidebar', position: 'left', - label: 'API Documentation' + label: 'Documentation', }, { - href: 'https://github.com/ai16z/eliza', - label: 'GitHub', - position: 'right' - } - ] - }, - footer: { - style: 'dark', - links: [ - { - title: 'Docs', - items: [ - ] + type: 'doc', + docsPluginId: 'api', + position: 'left', + label: 'API', + docId: 'index', }, - { - title: 'Community', - items: [ - { - label: 'Discord', - href: 'https://discord.gg/xe7HtWru' - }, - { - label: 'Twitter', - href: 'https://twitter.com/pmairca' - } - ] + href: 'https://github.com/ai16z/eliza', + label: 'GitHub', + position: 'right', }, - { - title: 'More', - items: [ - { - label: 'GitHub', - href: 'https://github.com/ai16z/eliza' - } - ] - } ], - copyright: `Copyright © ${new Date().getFullYear()} ai16z.vc` }, - prism: { - theme: prismThemes.github, - darkTheme: prismThemes.dracula - } - }) -} + // ... rest of themeConfig remains the same + }), +}; -export default config +export default config; diff --git a/docs/package-lock.json b/docs/package-lock.json index 647fdb209b..cdec4cecd2 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -12,6 +12,7 @@ "@docusaurus/preset-classic": "3.5.2", "@mdx-js/react": "3.0.1", "clsx": "2.1.0", + "docusaurus-lunr-search": "^3.5.0", "prism-react-renderer": "2.3.1", "react": "18.2.0", "react-dom": "18.2.0", @@ -4043,6 +4044,12 @@ "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", "license": "MIT" }, + "node_modules/@types/parse5": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-5.0.3.tgz", + "integrity": "sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==", + "license": "MIT" + }, "node_modules/@types/prismjs": { "version": "1.26.5", "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.5.tgz", @@ -4357,6 +4364,12 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "license": "Apache-2.0" }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -4659,6 +4672,12 @@ "node": ">= 8" } }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "license": "ISC" + }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -4704,6 +4723,15 @@ "node": ">= 4.0.0" } }, + "node_modules/autocomplete.js": { + "version": "0.37.1", + "resolved": "https://registry.npmjs.org/autocomplete.js/-/autocomplete.js-0.37.1.tgz", + "integrity": "sha512-PgSe9fHYhZEsm/9jggbjtVsGXJkPLvd+9mC7gZJ662vVL5CRWEtm/mIrrzCx0MrNxHVwxD5d00UOn6NsmL2LUQ==", + "license": "MIT", + "dependencies": { + "immediate": "^3.2.3" + } + }, "node_modules/autoprefixer": { "version": "10.4.20", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", @@ -4837,6 +4865,16 @@ "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "license": "MIT" }, + "node_modules/bcp-47-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bcp-47-match/-/bcp-47-match-1.0.3.tgz", + "integrity": "sha512-LggQ4YTdjWQSKELZF5JwchnBa1u0pIQSZf5lSdOHEdbVP55h0qICA/FUp3+W99q0xqxYa1ZQizTUH87gecII5w==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -5411,6 +5449,15 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "bin": { + "color-support": "bin.js" + } + }, "node_modules/colord": { "version": "2.9.3", "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", @@ -5570,6 +5617,12 @@ "integrity": "sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==", "license": "MIT" }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "license": "ISC" + }, "node_modules/content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -5903,6 +5956,12 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/css-selector-parser": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-1.4.1.tgz", + "integrity": "sha512-HYPSb7y/Z7BNDCOrakL4raGO2zltZkbeXyAd6Tg9obzix6QhzxCotdBl6VT0Dv4vZfJGVz3WL/xaEI9Ly3ul0g==", + "license": "MIT" + }, "node_modules/css-tree": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz", @@ -6351,6 +6410,19 @@ "node": ">=8" } }, + "node_modules/direction": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/direction/-/direction-1.0.4.tgz", + "integrity": "sha512-GYqKi1aH7PJXxdhTeZBFrg8vUBeKXi+cNprXsC1kpJcbcVnV9wBsrOu1cQEdG0WeQwlfHiy3XvnKfIrJ2R0NzQ==", + "license": "MIT", + "bin": { + "direction": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/dns-packet": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", @@ -6363,6 +6435,151 @@ "node": ">=6" } }, + "node_modules/docusaurus-lunr-search": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/docusaurus-lunr-search/-/docusaurus-lunr-search-3.5.0.tgz", + "integrity": "sha512-k3zN4jYMi/prWInJILGKOxE+BVcgYinwj9+gcECsYm52tS+4ZKzXQzbPnVJAEXmvKOfFMcDFvS3MSmm6cEaxIQ==", + "license": "MIT", + "dependencies": { + "autocomplete.js": "^0.37.0", + "clsx": "^1.2.1", + "gauge": "^3.0.0", + "hast-util-select": "^4.0.0", + "hast-util-to-text": "^2.0.0", + "hogan.js": "^3.0.2", + "lunr": "^2.3.8", + "lunr-languages": "^1.4.0", + "mark.js": "^8.11.1", + "minimatch": "^3.0.4", + "rehype-parse": "^7.0.1", + "to-vfile": "^6.1.0", + "unified": "^9.0.0", + "unist-util-is": "^4.0.2" + }, + "engines": { + "node": ">= 8.10.0" + }, + "peerDependencies": { + "@docusaurus/core": "^2.0.0-alpha.60 || ^2.0.0 || ^3.0.0", + "react": "^16.8.4 || ^17 || ^18", + "react-dom": "^16.8.4 || ^17 || ^18" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/docusaurus-lunr-search/node_modules/bail": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/bail/-/bail-1.0.5.tgz", + "integrity": "sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/clsx": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz", + "integrity": "sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/trough": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/trough/-/trough-1.0.5.tgz", + "integrity": "sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/unified": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz", + "integrity": "sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==", + "license": "MIT", + "dependencies": { + "bail": "^1.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^2.0.0", + "trough": "^1.0.0", + "vfile": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/unist-util-stringify-position": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz", + "integrity": "sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/vfile": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-4.2.1.tgz", + "integrity": "sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^2.0.0", + "vfile-message": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/docusaurus-lunr-search/node_modules/vfile-message": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz", + "integrity": "sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/docusaurus-plugin-typedoc": { "version": "0.22.0", "resolved": "https://registry.npmjs.org/docusaurus-plugin-typedoc/-/docusaurus-plugin-typedoc-0.22.0.tgz", @@ -7498,6 +7715,47 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/gauge/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/gauge/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -7870,6 +8128,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "license": "ISC" + }, "node_modules/has-yarn": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-3.0.0.tgz", @@ -7914,6 +8178,26 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-has-property": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-1.0.4.tgz", + "integrity": "sha512-ghHup2voGfgFoHMGnaLHOjbYFACKrRh9KFttdCzMCbFoBMJXiNi2+XTrPP8+q6cDJM/RSqlCfVWrjp1H201rZg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-1.1.0.tgz", + "integrity": "sha512-oUmNua0bFbdrD/ELDSSEadRVtWZOf3iF6Lbv81naqsIV99RnSCieTbWuWCY8BAeEfKJTKl0gRdokv+dELutHGQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-parse-selector": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", @@ -7952,6 +8236,130 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-select": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/hast-util-select/-/hast-util-select-4.0.2.tgz", + "integrity": "sha512-8EEG2//bN5rrzboPWD2HdS3ugLijNioS1pqOTIolXNf67xxShYw4SQEmVXd3imiBG+U2bC2nVTySr/iRAA7Cjg==", + "license": "MIT", + "dependencies": { + "bcp-47-match": "^1.0.0", + "comma-separated-tokens": "^1.0.0", + "css-selector-parser": "^1.0.0", + "direction": "^1.0.0", + "hast-util-has-property": "^1.0.0", + "hast-util-is-element": "^1.0.0", + "hast-util-to-string": "^1.0.0", + "hast-util-whitespace": "^1.0.0", + "not": "^0.1.0", + "nth-check": "^2.0.0", + "property-information": "^5.0.0", + "space-separated-tokens": "^1.0.0", + "unist-util-visit": "^2.0.0", + "zwitch": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/hast-util-select/node_modules/comma-separated-tokens": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", + "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-select/node_modules/hast-util-whitespace": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-1.0.4.tgz", + "integrity": "sha512-I5GTdSfhYfAPNztx2xJRQpG8cuDSNt599/7YUn7Gx/WxNMsG+a835k97TDkFgk123cwjfwINaZknkKkphx/f2A==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select/node_modules/property-information": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", + "integrity": "sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-select/node_modules/space-separated-tokens": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz", + "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-select/node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select/node_modules/unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select/node_modules/unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-select/node_modules/zwitch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-1.0.5.tgz", + "integrity": "sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/hast-util-to-estree": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.0.tgz", @@ -8041,6 +8449,31 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-string": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz", + "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-text": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-2.0.1.tgz", + "integrity": "sha512-8nsgCARfs6VkwH2jJU9b8LNTuR4700na+0h3PqCaEk4MAnMDeu5P0tP8mjk9LLNGxIeQRLbiDbZVw6rku+pYsQ==", + "license": "MIT", + "dependencies": { + "hast-util-is-element": "^1.0.0", + "repeat-string": "^1.0.0", + "unist-util-find-after": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-whitespace": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", @@ -8094,6 +8527,18 @@ "value-equal": "^1.0.1" } }, + "node_modules/hogan.js": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/hogan.js/-/hogan.js-3.0.2.tgz", + "integrity": "sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==", + "dependencies": { + "mkdirp": "0.3.0", + "nopt": "1.0.10" + }, + "bin": { + "hulk": "bin/hulk" + } + }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -8460,6 +8905,12 @@ "node": ">=16.x" } }, + "node_modules/immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==", + "license": "MIT" + }, "node_modules/immer": { "version": "9.0.21", "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.21.tgz", @@ -8620,6 +9071,29 @@ "node": ">=8" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/is-ci": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz", @@ -9232,7 +9706,18 @@ "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", - "dev": true, + "license": "MIT" + }, + "node_modules/lunr-languages": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/lunr-languages/-/lunr-languages-1.14.0.tgz", + "integrity": "sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==", + "license": "MPL-1.1" + }, + "node_modules/mark.js": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", + "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", "license": "MIT" }, "node_modules/markdown-extensions": { @@ -11631,6 +12116,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mkdirp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", + "integrity": "sha512-OHsdUcVAQ6pOtg5JYWpCBo9W/GySVuwvP9hueRMW7UqshC0tbfzLv8wjySTPm3tfUZ/21CE9E1pJagOA91Pxew==", + "deprecated": "Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.)", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/mrmime": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", @@ -11732,6 +12227,21 @@ "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", "license": "MIT" }, + "node_modules/nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==", + "license": "MIT", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "*" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -11762,6 +12272,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/not": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/not/-/not-0.1.0.tgz", + "integrity": "sha512-5PDmaAsVfnWUgTUbJ3ERwn7u79Z0dYxN9ErxCpVJJqe2RK0PJ3z+iFUxuqjwtlDDegXvtWoxD/3Fzxox7tFGWA==" + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -13659,6 +14174,182 @@ "regjsparser": "bin/parser" } }, + "node_modules/rehype-parse": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-7.0.1.tgz", + "integrity": "sha512-fOiR9a9xH+Le19i4fGzIEowAbwG7idy2Jzs4mOrFWBSJ0sNUgy0ev871dwWnbOo371SjgjG4pwzrbgSVrKxecw==", + "license": "MIT", + "dependencies": { + "hast-util-from-parse5": "^6.0.0", + "parse5": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/@types/hast": { + "version": "2.3.10", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.10.tgz", + "integrity": "sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2" + } + }, + "node_modules/rehype-parse/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/rehype-parse/node_modules/comma-separated-tokens": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", + "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/rehype-parse/node_modules/hast-util-from-parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-6.0.1.tgz", + "integrity": "sha512-jeJUWiN5pSxW12Rh01smtVkZgZr33wBokLzKLwinYOUfSzm1Nl/c3GUGebDyOKjdsRgMvoVbV0VpAcpjF4NrJA==", + "license": "MIT", + "dependencies": { + "@types/parse5": "^5.0.0", + "hastscript": "^6.0.0", + "property-information": "^5.0.0", + "vfile": "^4.0.0", + "vfile-location": "^3.2.0", + "web-namespaces": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/hast-util-parse-selector": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", + "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/hastscript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", + "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^1.0.0", + "hast-util-parse-selector": "^2.0.0", + "property-information": "^5.0.0", + "space-separated-tokens": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "license": "MIT" + }, + "node_modules/rehype-parse/node_modules/property-information": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", + "integrity": "sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/rehype-parse/node_modules/space-separated-tokens": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz", + "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/rehype-parse/node_modules/unist-util-stringify-position": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz", + "integrity": "sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/vfile": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-4.2.1.tgz", + "integrity": "sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^2.0.0", + "vfile-message": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/vfile-location": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-3.2.0.tgz", + "integrity": "sha512-aLEIZKv/oxuCDZ8lkJGhuhztf/BW4M+iHdCwglA/eWc+vtuRFJj8EtgceYFX4LRjOhCAAiNHsKGssC6onJ+jbA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/vfile-message": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz", + "integrity": "sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse/node_modules/web-namespaces": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz", + "integrity": "sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/rehype-raw": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", @@ -13926,6 +14617,15 @@ "entities": "^2.0.0" } }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -15114,6 +15814,69 @@ "node": ">=8.0" } }, + "node_modules/to-vfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/to-vfile/-/to-vfile-6.1.0.tgz", + "integrity": "sha512-BxX8EkCxOAZe+D/ToHdDsJcVI4HqQfmw0tCkp31zf3dNP/XWIAjU4CmeuSwsSoOzOTqHPOL0KUzyZqJplkD0Qw==", + "license": "MIT", + "dependencies": { + "is-buffer": "^2.0.0", + "vfile": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/to-vfile/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/to-vfile/node_modules/unist-util-stringify-position": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz", + "integrity": "sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/to-vfile/node_modules/vfile": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-4.2.1.tgz", + "integrity": "sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^2.0.0", + "vfile-message": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/to-vfile/node_modules/vfile-message": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz", + "integrity": "sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -15391,6 +16154,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unist-util-find-after": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-3.0.0.tgz", + "integrity": "sha512-ojlBqfsBftYXExNu3+hHLfJQ/X1jYY/9vdm4yZWjIbf0VuWF6CRufci1ZyoD/wV2TYMKxXUoNuoqwy+CkgzAiQ==", + "license": "MIT", + "dependencies": { + "unist-util-is": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-find-after/node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-is": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", @@ -16204,6 +16990,35 @@ "node": ">= 8" } }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "license": "ISC", + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "node_modules/wide-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wide-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/widest-line": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", @@ -16351,6 +17166,15 @@ "xml-js": "bin/cli.js" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/docs/package.json b/docs/package.json index a5941776f9..73616e8f45 100644 --- a/docs/package.json +++ b/docs/package.json @@ -19,6 +19,7 @@ "@docusaurus/preset-classic": "3.5.2", "@mdx-js/react": "3.0.1", "clsx": "2.1.0", + "docusaurus-lunr-search": "^3.5.0", "prism-react-renderer": "2.3.1", "react": "18.2.0", "react-dom": "18.2.0", diff --git a/docs/sidebars.api.js b/docs/sidebars.api.js new file mode 100644 index 0000000000..1cc9c6ed47 --- /dev/null +++ b/docs/sidebars.api.js @@ -0,0 +1,11 @@ +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + apiSidebar: [ + { + type: 'autogenerated', + dirName: '.', + }, + ], +}; + +export default sidebars; diff --git a/docs/sidebars.js b/docs/sidebars.js index 14554ca1d4..fb19153292 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -1,20 +1,26 @@ -/** - * Creating a sidebar enables you to: - - create an ordered group of docs - - render a sidebar for each doc of that group - - provide next/previous navigation - - The sidebars can be generated from the filesystem, or explicitly defined here. - - Create as many sidebars as you want. - */ - -// @ts-check - /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { - // By default, Docusaurus generates a sidebar from the docs folder structure - tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], + tutorialSidebar: [ + { + type: 'category', + label: 'Getting Started', + items: [ + 'intro', + 'quickstart', + 'installation', + ], + }, + { + type: 'category', + label: 'Guides', + items: [ + 'guides/basic-usage', + 'guides/configuration', + 'guides/characterfile', + 'guides/advanced', + ], + } + ], }; export default sidebars; diff --git a/docs/eliza_banner.jpg b/docs/static/img/eliza_banner.jpg similarity index 100% rename from docs/eliza_banner.jpg rename to docs/static/img/eliza_banner.jpg From 063dc752841369cbea8a1d609c63c8ec0c3b2a0a Mon Sep 17 00:00:00 2001 From: madjin <32600939+madjin@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:16:09 -0400 Subject: [PATCH 2/8] change to node 23 and rebuild --- docs/package-lock.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index cdec4cecd2..c079cc58de 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -7692,20 +7692,6 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "license": "ISC" }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", From a056f805a770f640eaf903b16c9df2ab6ca9aed4 Mon Sep 17 00:00:00 2001 From: Jerad Bitner Date: Fri, 1 Nov 2024 06:00:29 -0700 Subject: [PATCH 3/8] Create pull_request_template.md This adds a new template for pull requests which asks the user provide more details. --- .github/pull_request_template.md | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..4024bc95f8 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,70 @@ + + +# Relates to: + + + +# Risks + + +# Background +## What does this PR do? + +## What kind of change is this? + + + + + +# Documentation changes needed? + + + +# Testing +## Where should a reviewer start? + +## Detailed testing steps + + + + + + + + + + + + + + + From 97444b7b777e427fc5f3d0cc8ed409f4ad47db05 Mon Sep 17 00:00:00 2001 From: Jerad Bitner Date: Fri, 1 Nov 2024 06:13:29 -0700 Subject: [PATCH 4/8] Update ci.yaml ensure that our ci is running node-version 22 so the automated ci stops failing --- .github/workflows/ci.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 728c47f7d2..fb44440d03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,25 +1,29 @@ name: ci - on: push: branches: [main] pull_request: branches: [main] - jobs: check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v3 with: version: 9.4.0 - + + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + - name: Install dependencies run: pnpm i - + - name: Run Prettier run: pnpm run prettier --check . - + - name: Build packages run: pnpm run build From ffe32593689e5abc2ed637437e9e74590c20f07d Mon Sep 17 00:00:00 2001 From: Jerad Bitner Date: Fri, 1 Nov 2024 06:15:15 -0700 Subject: [PATCH 5/8] Update ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb44440d03..5a9bb59b1c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: '23' cache: 'pnpm' - name: Install dependencies From 035b4ea4f0966d33b3ca86a16c95737336154e4d Mon Sep 17 00:00:00 2001 From: moon Date: Fri, 1 Nov 2024 10:57:04 -0700 Subject: [PATCH 6/8] prettier --- .github/ISSUE_TEMPLATE/bug_report.md | 8 +- .github/ISSUE_TEMPLATE/feature_request.md | 8 +- .vscode/launch.json | 37 +- .vscode/tasks.json | 30 +- CONTRIBUTING.md | 30 +- README.md | 237 +- core/eslint.config.mjs | 101 +- core/jest.config.js | 45 +- core/nodemon.json | 2 +- core/package.json | 360 +- core/renovate.json | 152 +- core/src/actions/ask_claude.ts | 386 +- core/src/actions/continue.ts | 1116 +- core/src/actions/follow_room.ts | 668 +- core/src/actions/ignore.ts | 461 +- core/src/actions/imageGeneration.ts | 106 +- core/src/actions/imageGenerationUtils.ts | 103 +- core/src/actions/mute_room.ts | 312 +- core/src/actions/none.ts | 277 +- core/src/actions/pumpfun.ts | 529 +- core/src/actions/swap.ts | 208 +- core/src/actions/swapUtils.ts | 558 +- core/src/actions/take_order.ts | 20 +- core/src/actions/unfollow_room.ts | 582 +- core/src/actions/unmute_room.ts | 270 +- core/src/adapters/postgres.ts | 1581 +- core/src/adapters/sqlite.ts | 1173 +- core/src/adapters/sqlite/sqlite_vec.ts | 18 +- core/src/adapters/sqljs.ts | 1394 +- core/src/adapters/sqljs/types.ts | 388 +- core/src/adapters/supabase.ts | 1285 +- core/src/clients/direct/index.ts | 423 +- .../discord/actions/chat_with_attachments.ts | 514 +- .../clients/discord/actions/download_media.ts | 311 +- core/src/clients/discord/actions/joinvoice.ts | 594 +- .../src/clients/discord/actions/leavevoice.ts | 418 +- .../discord/actions/summarize_conversation.ts | 731 +- .../discord/actions/transcribe_media.ts | 344 +- core/src/clients/discord/attachments.ts | 515 +- core/src/clients/discord/audioMonitor.ts | 144 +- core/src/clients/discord/commands.ts | 80 +- core/src/clients/discord/index.ts | 587 +- core/src/clients/discord/messages.ts | 1277 +- .../clients/discord/providers/channelState.ts | 95 +- .../clients/discord/providers/voiceState.ts | 42 +- core/src/clients/discord/templates.ts | 12 +- core/src/clients/discord/types.ts | 16 +- core/src/clients/discord/voice.ts | 1192 +- core/src/clients/telegram/src/index.ts | 141 +- .../clients/telegram/src/messageManager.ts | 662 +- core/src/clients/twitter/base.ts | 1025 +- core/src/clients/twitter/generate.ts | 320 +- core/src/clients/twitter/interactions.ts | 579 +- core/src/clients/twitter/search.ts | 567 +- core/src/clients/twitter/utils.ts | 288 +- core/src/core/actions.ts | 66 +- core/src/core/context.ts | 23 +- core/src/core/database.ts | 646 +- core/src/core/defaultCharacter.ts | 786 +- core/src/core/embedding.ts | 15 +- core/src/core/evaluators.ts | 108 +- core/src/core/generation.ts | 93 +- core/src/core/goals.ts | 84 +- core/src/core/imageGenModels.ts | 10 +- core/src/core/logger.ts | 108 +- core/src/core/memory.ts | 402 +- core/src/core/messages.ts | 145 +- core/src/core/models.ts | 148 +- core/src/core/parsing.ts | 138 +- core/src/core/posts.ts | 86 +- core/src/core/providers.ts | 18 +- core/src/core/relationships.ts | 80 +- core/src/core/runtime.ts | 2009 +-- core/src/core/types.ts | 763 +- core/src/core/uuid.ts | 80 +- core/src/evaluators/fact.ts | 303 +- core/src/evaluators/goal.ts | 376 +- core/src/index.ts | 442 +- core/src/lib.ts | 38 +- core/src/providers/balances.ts | 163 +- core/src/providers/boredom.ts | 629 +- ...LybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json | 56 +- ...LybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json | 2 +- ...LybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json | 12 +- ...LybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json | 190 +- core/src/providers/order_book.ts | 66 +- core/src/providers/time.ts | 12 +- core/src/providers/token.ts | 1336 +- core/src/providers/wallet.ts | 461 +- core/src/services/audioUtils.ts | 41 +- core/src/services/browser.ts | 496 +- core/src/services/image.ts | 580 +- core/src/services/llama.ts | 709 +- core/src/services/speech.ts | 274 +- core/src/services/summary.ts | 40 +- core/src/services/tokenUtils.ts | 96 +- core/src/services/transcription.ts | 515 +- core/src/services/video.ts | 704 +- core/src/services/wordsToPunish.ts | 200 +- core/src/test_resources/cache.ts | 58 +- core/src/test_resources/constants.ts | 2 +- core/src/test_resources/createRuntime.ts | 247 +- core/src/test_resources/data.ts | 1830 +-- .../test_resources/getOrCreateRelationship.ts | 100 +- core/src/test_resources/populateMemories.ts | 46 +- core/src/test_resources/report.ts | 156 +- core/src/test_resources/runAiTest.ts | 20 +- core/src/test_resources/templates.ts | 2 +- core/src/test_resources/testAction.ts | 89 +- core/src/test_resources/testEvaluator.ts | 106 +- core/src/test_resources/types.ts | 8 +- core/src/types/token.ts | 512 +- core/src/vendor/vits.ts | 11968 ++++++++-------- core/src/vendor/vitsVoiceList.ts | 1278 +- core/tests/actions.test.ts | 421 +- core/tests/browser.test.ts | 147 +- core/tests/continue.test.ts | 350 +- core/tests/evaluation.test.ts | 204 +- core/tests/fact.test.ts | 205 +- core/tests/goal.test.ts | 328 +- core/tests/goals.test.ts | 191 +- core/tests/ignore.test.ts | 330 +- core/tests/memory.test.ts | 1125 +- core/tests/messages.test.ts | 166 +- core/tests/providers.test.ts | 67 +- core/tests/relationships.test.ts | 97 +- core/tests/runtime.test.ts | 172 +- core/tests/time.test.ts | 122 +- core/tests/token.test.ts | 108 +- core/tests/utils.test.ts | 98 +- core/tsconfig.json | 56 +- core/types/index.d.ts | 3 +- docs/.prettierrc.json | 2 +- docs/README.md | 66 +- docs/babel.config.js | 2 +- docs/docs/_category_.yml | 2 +- docs/docs/classes/BgentRuntime.md | 194 +- docs/docs/classes/DatabaseAdapter.md | 320 +- docs/docs/classes/MemoryManager.md | 94 +- docs/docs/classes/SqlJsDatabaseAdapter.md | 318 +- docs/docs/classes/SqliteDatabaseAdapter.md | 318 +- docs/docs/classes/SupabaseDatabaseAdapter.md | 318 +- docs/docs/classes/_category_.yml | 2 +- docs/docs/concepts.md | 2 +- docs/docs/docs/_category_.yml | 2 +- docs/docs/docs/index.md | 136 +- docs/docs/docs/modules.md | 1 - docs/docs/enums/GoalStatus.md | 12 +- docs/docs/enums/_category_.yml | 2 +- docs/docs/functions/_category_.yml | 2 +- docs/docs/functions/addHeader.md | 8 +- docs/docs/functions/addLore.md | 18 +- docs/docs/functions/composeActionExamples.md | 6 +- docs/docs/functions/composeContext.md | 10 +- docs/docs/functions/createGoal.md | 10 +- docs/docs/functions/createRelationship.md | 12 +- docs/docs/functions/formatActionConditions.md | 4 +- docs/docs/functions/formatActionNames.md | 4 +- docs/docs/functions/formatActions.md | 4 +- docs/docs/functions/formatActors.md | 8 +- .../functions/formatEvaluatorConditions.md | 4 +- .../formatEvaluatorExampleConditions.md | 4 +- .../formatEvaluatorExampleDescriptions.md | 4 +- .../docs/functions/formatEvaluatorExamples.md | 4 +- docs/docs/functions/formatEvaluatorNames.md | 4 +- docs/docs/functions/formatEvaluators.md | 4 +- docs/docs/functions/formatGoalsAsString.md | 8 +- docs/docs/functions/formatLore.md | 4 +- docs/docs/functions/formatMessages.md | 10 +- docs/docs/functions/formatRelationships.md | 10 +- docs/docs/functions/getActorDetails.md | 10 +- docs/docs/functions/getFormattedActions.md | 4 +- docs/docs/functions/getGoals.md | 16 +- docs/docs/functions/getLore.md | 16 +- docs/docs/functions/getProviders.md | 10 +- docs/docs/functions/getRelationship.md | 16 +- docs/docs/functions/getRelationships.md | 10 +- .../docs/functions/parseJSONObjectFromText.md | 4 +- docs/docs/functions/parseJsonArrayFromText.md | 8 +- docs/docs/functions/updateGoal.md | 10 +- docs/docs/index.md | 136 +- docs/docs/interfaces/Account.md | 12 +- docs/docs/interfaces/Action.md | 10 +- docs/docs/interfaces/ActionExample.md | 2 +- docs/docs/interfaces/Actor.md | 10 +- docs/docs/interfaces/Content.md | 4 +- docs/docs/interfaces/ConversationExample.md | 6 +- docs/docs/interfaces/EvaluationExample.md | 4 +- docs/docs/interfaces/Evaluator.md | 10 +- docs/docs/interfaces/Goal.md | 18 +- docs/docs/interfaces/Memory.md | 22 +- docs/docs/interfaces/Message.md | 12 +- docs/docs/interfaces/MessageExample.md | 2 +- docs/docs/interfaces/Objective.md | 4 +- docs/docs/interfaces/Participant.md | 2 +- docs/docs/interfaces/Provider.md | 12 +- docs/docs/interfaces/Relationship.md | 32 +- docs/docs/interfaces/Room.md | 2 +- docs/docs/interfaces/State.md | 48 +- docs/docs/interfaces/_category_.yml | 2 +- docs/docs/modules.md | 1 - docs/docs/rooms.md | 5 +- docs/docs/types/Handler.md | 14 +- docs/docs/types/Validator.md | 12 +- docs/docs/types/_category_.yml | 2 +- docs/docs/variables/_category_.yml | 2 +- docs/docs/variables/embeddingDimension.md | 2 +- docs/docs/variables/evaluationTemplate.md | 2 +- docs/docs/variables/messageHandlerTemplate.md | 2 +- docs/docusaurus.config.js | 112 +- docs/sidebars.js | 2 +- docs/src/components/HomepageFeatures/index.js | 70 +- docs/src/css/custom.css | 2 +- docs/src/pages/index.js | 43 +- scripts/extracttweets.js | 42 +- scripts/gettweets.mjs | 124 +- 216 files changed, 30810 insertions(+), 29497 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index c561786b67..7d707c90e7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,9 +1,9 @@ --- name: Bug report about: Create a report to help us improve -title: '' -labels: 'bug' -assignees: '' +title: "" +labels: "bug" +assignees: "" --- **Describe the bug** @@ -24,4 +24,4 @@ assignees: '' **Additional context** - \ No newline at end of file + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 459c629733..7eeb966c57 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,9 +1,9 @@ --- name: Feature request about: Suggest an idea for this project -title: '' -labels: 'enhancement' -assignees: '' +title: "" +labels: "enhancement" +assignees: "" --- **Is your feature request related to a problem? Please describe.** @@ -20,4 +20,4 @@ assignees: '' **Additional context** - \ No newline at end of file + diff --git a/.vscode/launch.json b/.vscode/launch.json index 537cd4e312..73bb179289 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,21 +1,16 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Launch via npm", - "runtimeExecutable": "npm", - "runtimeArgs": [ - "run", - "dev" - ], - "skipFiles": [ - "/**" - ] - } - ] -} \ No newline at end of file +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch via npm", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev"], + "skipFiles": ["/**"] + } + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9ca81aeb3b..7d489860dc 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,16 +1,14 @@ -{ - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "2.0.0", - "tasks": [ - { - "type": "typescript", - "tsconfig": "tsconfig.json", - "problemMatcher": [ - "$tsc" - ], - "label": "tsbuild", - "group": "build" - } - ] -} \ No newline at end of file +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "type": "typescript", + "tsconfig": "tsconfig.json", + "problemMatcher": ["$tsc"], + "label": "tsbuild", + "group": "build" + } + ] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73dae121c2..597eba922b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,9 +13,9 @@ By participating in this project, you are expected to uphold our Code of Conduct 1. Ensure the bug was not already reported by searching on GitHub under [Issues](https://github.com/ai16z/eliza/issues). 2. If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/ai16z/eliza/issues/new). Be sure to include -- a title and clear description, -- as much relevant information as possible, and -- a code sample or an executable test case demonstrating the expected behavior that is not occurring. +- a title and clear description, +- as much relevant information as possible, and +- a code sample or an executable test case demonstrating the expected behavior that is not occurring. ### Suggesting Enhancements @@ -36,22 +36,22 @@ By participating in this project, you are expected to uphold our Code of Conduct ### Git Commit Messages -- Use the present tense ("Add feature" not "Added feature") -- Use the imperative mood ("Move cursor to..." not "Moves cursor to...") -- Limit the first line to 72 characters or less -- Reference issues and pull requests liberally after the first line +- Use the present tense ("Add feature" not "Added feature") +- Use the imperative mood ("Move cursor to..." not "Moves cursor to...") +- Limit the first line to 72 characters or less +- Reference issues and pull requests liberally after the first line ### JavaScript Styleguide -- All JavaScript must adhere to [JavaScript Standard Style](https://standardjs.com/). +- All JavaScript must adhere to [JavaScript Standard Style](https://standardjs.com/). ### TypeScript Styleguide -- All TypeScript must adhere to [TypeScript Standard Style](https://github.com/standard/ts-standard). +- All TypeScript must adhere to [TypeScript Standard Style](https://github.com/standard/ts-standard). ### Documentation Styleguide -- Use [Markdown](https://daringfireball.net/projects/markdown/) for documentation. +- Use [Markdown](https://daringfireball.net/projects/markdown/) for documentation. ## Additional Notes @@ -59,10 +59,10 @@ By participating in this project, you are expected to uphold our Code of Conduct This section lists the labels we use to help us track and manage issues and pull requests. -- `bug` - Issues that are bugs. -- `enhancement` - Issues that are feature requests. -- `documentation` - Issues or pull requests related to documentation. -- `good first issue` - Good for newcomers. +- `bug` - Issues that are bugs. +- `enhancement` - Issues that are feature requests. +- `documentation` - Issues or pull requests related to documentation. +- `good first issue` - Good for newcomers. ## Recognition @@ -72,4 +72,4 @@ We value every contribution. Contributors will be recognized in our README.md fi If you have any questions, please feel free to contact the project maintainers with an issue or in discord. -Thank you for your interest in contributing to Eliza! \ No newline at end of file +Thank you for your interest in contributing to Eliza! diff --git a/README.md b/README.md index c32561ed5c..39c039f3d0 100644 --- a/README.md +++ b/README.md @@ -1,111 +1,126 @@ -# Eliza - -Eliza Banner - -*As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)* - -- Multi-agent simulation framework -- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) -- Full-featured Discord and Twitter connectors, with Discord voice channel support -- Full conversational and document RAG memory -- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more -- Highly extensible - create your own actions and clients to extend Eliza's capabilities -- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) -- Supports OpenAI for cloud inference on a light-weight device -- "Ask Claude" mode for calling Claude on more complex queries -- 100% Typescript - -# Getting Started - -## Install Node.js -https://docs.npmjs.com/downloading-and-installing-node-js-and-npm - -## Using pnpm -We use pnpm to manage our dependencies. It is faster and more efficient than npm, and it supports workspaces. -https://pnpm.io/installation - -## Edit the .env file -- Copy .env.example to .env and fill in the appropriate values -- Edit the TWITTER environment variables to add your bot's username and password - -## Edit the character file -- Check out the file `src/core/defaultCharacter.ts` - you can modify this -- You can also load characters with the `node --loader ts-node/esm src/index.ts --characters="path/to/your/character.json"` and run multiple bots at the same time. - -### Run with Llama -You can run Llama 70B or 405B models by setting the `XAI_MODEL` environment variable to `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` or `meta-llama/Meta-Llama-3.1-405B-Instruct` - -### Run with Grok -You can run Grok models by setting the `XAI_MODEL` environment variable to `grok-beta` - -### Run with OpenAI -You can run OpenAI models by setting the `XAI_MODEL` environment variable to `gpt-4o-mini` or `gpt-4o` - -# Requires Node 20+ -If you are getting strange issues when starting up, make sure you're using Node 20+. Some APIs are not compatible with previous versions. You can check your node version with `node -v`. If you need to install a new version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm). - -## Additional Requirements -You may need to install Sharp. If you see an error when starting up, try installing it with the following command: -``` -pnpm install --include=optional sharp -``` - -# Environment Setup - -You will need to add environment variables to your .env file to connect to various platforms: -``` -# Required environment variables -# Start Discord -DISCORD_APPLICATION_ID= -DISCORD_API_TOKEN= # Bot token - -# Start Twitter -TWITTER_USERNAME= # Account username -TWITTER_PASSWORD= # Account password -TWITTER_EMAIL= # Account email -TWITTER_COOKIES= # Account cookies -``` - -# Local Setup - -## CUDA Setup - -If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. -``` -pnpm install -npx --no node-llama-cpp source download --gpu cuda -``` - -Make sure that you've installed the CUDA Toolkit, including cuDNN and cuBLAS. - -## Running locally -Add XAI_MODEL and set it to one of the above options from [Run with -Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it -downloads the model from huggingface and queries it locally - -# Cloud Setup (with OpenAI) - -In addition to the environment variables above, you will need to add the following: -``` -# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. -OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk- - -# The agent can also ask Claude for help if you have an API key -ANTHROPIC_API_KEY= - -# For Elevenlabs voice generation on Discord voice -ELEVENLABS_XI_API_KEY= # API key from elevenlabs - -# ELEVENLABS SETTINGS -ELEVENLABS_MODEL_ID=eleven_multilingual_v2 -ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM -ELEVENLABS_VOICE_STABILITY=0.5 -ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 -ELEVENLABS_VOICE_STYLE=0.66 -ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false -ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 -ELEVENLABS_OUTPUT_FORMAT=pcm_16000 -``` - -# Discord Bot -For help with setting up your Discord Bot, check out here: https://discordjs.guide/preparations/setting-up-a-bot-application.html +# Eliza + +Eliza Banner + +_As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)_ + +- Multi-agent simulation framework +- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) +- Full-featured Discord and Twitter connectors, with Discord voice channel support +- Full conversational and document RAG memory +- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more +- Highly extensible - create your own actions and clients to extend Eliza's capabilities +- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) +- Supports OpenAI for cloud inference on a light-weight device +- "Ask Claude" mode for calling Claude on more complex queries +- 100% Typescript + +# Getting Started + +## Install Node.js + +https://docs.npmjs.com/downloading-and-installing-node-js-and-npm + +## Using pnpm + +We use pnpm to manage our dependencies. It is faster and more efficient than npm, and it supports workspaces. +https://pnpm.io/installation + +## Edit the .env file + +- Copy .env.example to .env and fill in the appropriate values +- Edit the TWITTER environment variables to add your bot's username and password + +## Edit the character file + +- Check out the file `src/core/defaultCharacter.ts` - you can modify this +- You can also load characters with the `node --loader ts-node/esm src/index.ts --characters="path/to/your/character.json"` and run multiple bots at the same time. + +### Run with Llama + +You can run Llama 70B or 405B models by setting the `XAI_MODEL` environment variable to `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` or `meta-llama/Meta-Llama-3.1-405B-Instruct` + +### Run with Grok + +You can run Grok models by setting the `XAI_MODEL` environment variable to `grok-beta` + +### Run with OpenAI + +You can run OpenAI models by setting the `XAI_MODEL` environment variable to `gpt-4o-mini` or `gpt-4o` + +# Requires Node 20+ + +If you are getting strange issues when starting up, make sure you're using Node 20+. Some APIs are not compatible with previous versions. You can check your node version with `node -v`. If you need to install a new version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm). + +## Additional Requirements + +You may need to install Sharp. If you see an error when starting up, try installing it with the following command: + +``` +pnpm install --include=optional sharp +``` + +# Environment Setup + +You will need to add environment variables to your .env file to connect to various platforms: + +``` +# Required environment variables +# Start Discord +DISCORD_APPLICATION_ID= +DISCORD_API_TOKEN= # Bot token + +# Start Twitter +TWITTER_USERNAME= # Account username +TWITTER_PASSWORD= # Account password +TWITTER_EMAIL= # Account email +TWITTER_COOKIES= # Account cookies +``` + +# Local Setup + +## CUDA Setup + +If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. + +``` +pnpm install +npx --no node-llama-cpp source download --gpu cuda +``` + +Make sure that you've installed the CUDA Toolkit, including cuDNN and cuBLAS. + +## Running locally + +Add XAI_MODEL and set it to one of the above options from [Run with +Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it +downloads the model from huggingface and queries it locally + +# Cloud Setup (with OpenAI) + +In addition to the environment variables above, you will need to add the following: + +``` +# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. +OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk- + +# The agent can also ask Claude for help if you have an API key +ANTHROPIC_API_KEY= + +# For Elevenlabs voice generation on Discord voice +ELEVENLABS_XI_API_KEY= # API key from elevenlabs + +# ELEVENLABS SETTINGS +ELEVENLABS_MODEL_ID=eleven_multilingual_v2 +ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM +ELEVENLABS_VOICE_STABILITY=0.5 +ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 +ELEVENLABS_VOICE_STYLE=0.66 +ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false +ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 +ELEVENLABS_OUTPUT_FORMAT=pcm_16000 +``` + +# Discord Bot + +For help with setting up your Discord Bot, check out here: https://discordjs.guide/preparations/setting-up-a-bot-application.html diff --git a/core/eslint.config.mjs b/core/eslint.config.mjs index 7ed161a56e..fba8c077be 100644 --- a/core/eslint.config.mjs +++ b/core/eslint.config.mjs @@ -1,52 +1,55 @@ -import eslint from '@eslint/js'; -import tseslint from '@typescript-eslint/eslint-plugin'; -import typescript from '@typescript-eslint/parser'; -import prettier from 'eslint-config-prettier'; +import eslint from "@eslint/js"; +import tseslint from "@typescript-eslint/eslint-plugin"; +import typescript from "@typescript-eslint/parser"; +import prettier from "eslint-config-prettier"; export default [ - // JavaScript and TypeScript files - { - files: ["src/**/*.js", "src/**/*.cjs", "src/**/*.mjs", "src/**/*.ts"], - languageOptions: { - parser: typescript, - parserOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - project: './tsconfig.json' // Make sure your tsconfig includes @types/node - }, - globals: { - // Add Node.js globals - NodeJS: 'readonly', - console: 'readonly', - process: 'readonly', - Buffer: 'readonly', - __dirname: 'readonly', - __filename: 'readonly', - module: 'readonly', - require: 'readonly' - } - }, - plugins: { - '@typescript-eslint': tseslint + // JavaScript and TypeScript files + { + files: ["src/**/*.js", "src/**/*.cjs", "src/**/*.mjs", "src/**/*.ts"], + languageOptions: { + parser: typescript, + parserOptions: { + ecmaVersion: "latest", + sourceType: "module", + project: "./tsconfig.json", // Make sure your tsconfig includes @types/node + }, + globals: { + // Add Node.js globals + NodeJS: "readonly", + console: "readonly", + process: "readonly", + Buffer: "readonly", + __dirname: "readonly", + __filename: "readonly", + module: "readonly", + require: "readonly", + }, + }, + plugins: { + "@typescript-eslint": tseslint, + }, + rules: { + ...eslint.configs.recommended.rules, + ...tseslint.configs.recommended.rules, + "prefer-const": "warn", + "no-constant-binary-expression": "error", + + // Disable no-undef as TypeScript handles this better + "no-undef": "off", + + // Customize TypeScript rules + "@typescript-eslint/no-explicit-any": "warn", // Changed from error to warn + "@typescript-eslint/no-unused-vars": [ + "error", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + ignoreRestSiblings: true, + }, + ], + }, }, - rules: { - ...eslint.configs.recommended.rules, - ...tseslint.configs.recommended.rules, - "prefer-const": "warn", - "no-constant-binary-expression": "error", - - // Disable no-undef as TypeScript handles this better - "no-undef": "off", - - // Customize TypeScript rules - "@typescript-eslint/no-explicit-any": "warn", // Changed from error to warn - "@typescript-eslint/no-unused-vars": ["error", { - "argsIgnorePattern": "^_", - "varsIgnorePattern": "^_", - "ignoreRestSiblings": true - }] - } - }, - // Add prettier as the last config to override other formatting rules - prettier -]; \ No newline at end of file + // Add prettier as the last config to override other formatting rules + prettier, +]; diff --git a/core/jest.config.js b/core/jest.config.js index 055d97ee26..cc88ff8fef 100644 --- a/core/jest.config.js +++ b/core/jest.config.js @@ -1,24 +1,27 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { - preset: "ts-jest", - testEnvironment: "jest-environment-node", - rootDir: "./src", - testMatch: ["**/*.test.ts"], - globals: { - __DEV__: true, - __TEST__: true, - __VERSION__: "0.0.1", - }, - // collectCoverage: true, - // collectCoverageFrom: ["**/*.{ts}", "!**/*.test.{ts}", "!**/node_modules/**", "!**/vendor/**"], - // coverageDirectory: "../coverage", - transform: { - '^.+\\.tsx?$': ['ts-jest', { - useESM: true, - }], - }, - moduleNameMapper: { - '^(\\.{1,2}/.*)\\.js$': '$1', - }, - extensionsToTreatAsEsm: ['.ts'], + preset: "ts-jest", + testEnvironment: "jest-environment-node", + rootDir: "./src", + testMatch: ["**/*.test.ts"], + globals: { + __DEV__: true, + __TEST__: true, + __VERSION__: "0.0.1", + }, + // collectCoverage: true, + // collectCoverageFrom: ["**/*.{ts}", "!**/*.test.{ts}", "!**/node_modules/**", "!**/vendor/**"], + // coverageDirectory: "../coverage", + transform: { + "^.+\\.tsx?$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + extensionsToTreatAsEsm: [".ts"], }; diff --git a/core/nodemon.json b/core/nodemon.json index 6eb0116e23..74764aab34 100644 --- a/core/nodemon.json +++ b/core/nodemon.json @@ -3,4 +3,4 @@ "ext": "ts", "ignore": ["src/**/*.spec.ts"], "exec": "node --loader ts-node/esm src/index.ts" -} \ No newline at end of file +} diff --git a/core/package.json b/core/package.json index 092ede9c23..a1da177151 100644 --- a/core/package.json +++ b/core/package.json @@ -1,182 +1,182 @@ { - "name": "eliza", - "version": "1.0.0", - "description": "", - "main": "dist/index.js", - "type": "module", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc", - "lint": "eslint . --fix", - "start": "node --loader ts-node/esm src/index.ts", - "start:arok": "node --loader ts-node/esm src/index.ts --characters=\"characters/arok.character.json\"", - "start:service:ruby": "pm2 start npm --name=\"ruby\" --restart-delay=3000 --max-restarts=10 -- run start:ruby", - "stop:service:ruby": "pm2 stop ruby", - "start:ruby": "node --loader ts-node/esm src/index.ts --characters=\"characters/ruby.character.json\"", - "start:service:trump": "pm2 start npm --name=\"trump\" --restart-delay=3000 --max-restarts=10 -- run start:trump", - "stop:service:trump": "pm2 stop trump", - "start:service:degen": "pm2 start npm --name=\"degen\" --restart-delay=3000 --max-restarts=10 -- run start:degen", - "stop:service:degen": "pm2 stop degen", - "start:degen": "node --loader ts-node/esm src/index.ts --characters=\"characters/degenspartan.json\"", - "start:service:all": "pm2 start npm --name=\"all\" --restart-delay=3000 --max-restarts=10 -- run start:degen", - "stop:service:all": "pm2 stop all", - "start:all": "node --loader ts-node/esm src/index.ts --characters=\"characters/degenspartan.json\",\"characters/ruby.character.json\"", - "start:trump": "node --loader ts-node/esm src/index.ts --characters=\"characters/trump.character.json\"", - "start:service:tate": "pm2 start npm --name=\"tate\" --restart-delay=3000 --max-restarts=10 -- run start:tate", - "stop:service:tate": "pm2 stop tate", - "start:tate": "node --loader ts-node/esm src/index.ts --characters=\"characters/tate.character.json\"", - "watch": "tsc --watch", - "dev": "tsc && nodemon", - "build:docs": "cd docs && npm run build", - "postinstall": "npx playwright install-deps && npx playwright install", - "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest --runInBand --watch -f", - "test:sqlite": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" TEST_DATABASE_CLIENT=sqlite jest --runInBand --watch -f", - "test:sqljs": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" TEST_DATABASE_CLIENT=sqljs jest --runInBand --watch -f" - }, - "author": "", - "license": "MIT", - "devDependencies": { - "@eslint/js": "^9.13.0", - "@rollup/plugin-commonjs": "25.0.8", - "@rollup/plugin-json": "6.1.0", - "@rollup/plugin-node-resolve": "15.3.0", - "@rollup/plugin-replace": "5.0.7", - "@rollup/plugin-terser": "0.1.0", - "@rollup/plugin-typescript": "11.1.6", - "@types/better-sqlite3": "7.6.11", - "@types/fluent-ffmpeg": "2.1.27", - "@types/jest": "29.5.14", - "@types/node": "22.8.4", - "@types/sql.js": "1.4.9", - "@types/tar": "6.1.13", - "@types/wav-encoder": "1.3.3", - "@typescript-eslint/eslint-plugin": "8.12.2", - "@typescript-eslint/parser": "8.12.2", - "dotenv": "16.4.5", - "eslint": "9.13.0", - "eslint-config-prettier": "9.1.0", - "eslint-plugin-prettier": "5.2.1", - "itty-router": "5.0.18", - "jest": "29.7.0", - "lint-staged": "15.2.10", - "npm-run-all2": "7.0.1", - "prettier": "3.3.3", - "rimraf": "6.0.1", - "rollup": "2.79.2", - "ts-jest": "29.2.5", - "ts-node": "10.9.2", - "tslib": "2.8.0", - "typescript": "5.6.3", - "wrangler": "3.84.0" - }, - "dependencies": { - "@ai-sdk/anthropic": "^0.0.53", - "@ai-sdk/google": "^0.0.55", - "@ai-sdk/google-vertex": "^0.0.42", - "@ai-sdk/groq": "^0.0.3", - "@ai-sdk/openai": "^0.0.70", - "@anthropic-ai/sdk": "^0.30.1", - "@cliqz/adblocker-playwright": "1.34.0", - "@coral-xyz/anchor": "^0.30.1", - "@discordjs/opus": "github:discordjs/opus", - "@discordjs/rest": "2.4.0", - "@discordjs/voice": "0.17.0", - "@echogarden/espeak-ng-emscripten": "0.3.0", - "@echogarden/kissfft-wasm": "0.2.0", - "@echogarden/speex-resampler-wasm": "0.2.1", - "@huggingface/transformers": "3.0.1", - "@opendocsg/pdf2md": "0.1.31", - "@solana/spl-token": "0.4.9", - "@solana/web3.js": "1.95.4", - "@supabase/supabase-js": "2.46.1", - "@telegraf/types": "7.1.0", - "@types/body-parser": "1.19.5", - "@types/cors": "2.8.17", - "@types/express": "5.0.0", - "agent-twitter-client": "0.0.13", - "ai": "^3.4.23", - "alawmulaw": "6.0.0", - "ansi-colors": "4.1.3", - "anthropic-vertex-ai": "^1.0.0", - "better-sqlite3": "11.5.0", - "bignumber": "1.1.0", - "bignumber.js": "9.1.2", - "body-parser": "1.20.3", - "capsolver-npm": "2.0.2", - "cldr-segmentation": "2.2.1", - "command-exists": "1.2.9", - "commander": "12.1.0", - "cors": "2.8.5", - "cross-fetch": "4.0.0", - "csv-writer": "1.6.0", - "discord.js": "14.16.3", - "espeak-ng": "1.0.2", - "express": "4.21.1", - "ffmpeg-static": "5.2.0", - "figlet": "1.8.0", - "fluent-ffmpeg": "2.1.3", - "formdata-node": "6.0.3", - "fs-extra": "11.2.0", - "gaxios": "6.7.1", - "gif-frames": "^0.4.1", - "glob": "11.0.0", - "graceful-fs": "4.2.11", - "html-escaper": "3.0.3", - "html-to-text": "9.0.5", - "import-meta-resolve": "4.1.0", - "jieba-wasm": "2.2.0", - "js-sha1": "0.7.0", - "json5": "2.2.3", - "kuromoji": "0.1.2", - "libsodium-wrappers": "0.7.15", - "multer": "1.4.5-lts.1", - "node-cache": "5.1.2", - "node-llama-cpp": "3.1.1", - "node-wav": "0.0.2", - "nodejs-whisper": "0.1.18", - "nodemon": "3.1.7", - "onnxruntime-node": "1.20.0", - "openai": "4.69.0", - "pdfjs-dist": "4.7.76", - "pg": "^8.13.1", - "playwright": "1.48.2", - "pm2": "5.4.2", - "prism-media": "1.3.5", - "pumpdotfun-sdk": "1.3.2", - "puppeteer-extra": "3.3.6", - "puppeteer-extra-plugin-capsolver": "2.0.1", - "sql.js": "1.12.0", - "sqlite-vec": "0.1.4-alpha.2", - "srt": "0.0.3", - "systeminformation": "5.23.5", - "tar": "7.4.3", - "telegraf": "4.16.3", - "tiktoken": "1.0.17", - "tinyld": "1.3.4", - "together-ai": "^0.7.0", - "unique-names-generator": "4.7.1", - "uuid": "11.0.2", - "uuidv4": "6.2.13", - "wav": "1.0.2", - "wav-encoder": "1.3.0", - "wavefile": "11.0.0", - "whisper-turbo": "0.11.0", - "ws": "8.18.0", - "yargs": "17.7.2", - "youtube-dl-exec": "3.0.10" - }, - "trustedDependencies": { - "onnxruntime-node": "1.19.0", - "@discordjs/opus": "github:discordjs/opus", - "sharp": "^0.33.5" - }, - "optionalDependencies": { - "sharp": "0.33.5", - "sqlite-vss-darwin-arm64": "0.1.2", - "sqlite-vss-darwin-x64": "0.1.2", - "sqlite-vss-linux-arm64": "^0.1.2", - "sqlite-vss-linux-x64": "0.1.2", - "sqlite-vss-win32-arm64": "^0.1.2", - "sqlite-vss-win32-x64": "^0.1.2" - } + "name": "eliza", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "type": "module", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "lint": "eslint . --fix", + "start": "node --loader ts-node/esm src/index.ts", + "start:arok": "node --loader ts-node/esm src/index.ts --characters=\"characters/arok.character.json\"", + "start:service:ruby": "pm2 start npm --name=\"ruby\" --restart-delay=3000 --max-restarts=10 -- run start:ruby", + "stop:service:ruby": "pm2 stop ruby", + "start:ruby": "node --loader ts-node/esm src/index.ts --characters=\"characters/ruby.character.json\"", + "start:service:trump": "pm2 start npm --name=\"trump\" --restart-delay=3000 --max-restarts=10 -- run start:trump", + "stop:service:trump": "pm2 stop trump", + "start:service:degen": "pm2 start npm --name=\"degen\" --restart-delay=3000 --max-restarts=10 -- run start:degen", + "stop:service:degen": "pm2 stop degen", + "start:degen": "node --loader ts-node/esm src/index.ts --characters=\"characters/degenspartan.json\"", + "start:service:all": "pm2 start npm --name=\"all\" --restart-delay=3000 --max-restarts=10 -- run start:degen", + "stop:service:all": "pm2 stop all", + "start:all": "node --loader ts-node/esm src/index.ts --characters=\"characters/degenspartan.json\",\"characters/ruby.character.json\"", + "start:trump": "node --loader ts-node/esm src/index.ts --characters=\"characters/trump.character.json\"", + "start:service:tate": "pm2 start npm --name=\"tate\" --restart-delay=3000 --max-restarts=10 -- run start:tate", + "stop:service:tate": "pm2 stop tate", + "start:tate": "node --loader ts-node/esm src/index.ts --characters=\"characters/tate.character.json\"", + "watch": "tsc --watch", + "dev": "tsc && nodemon", + "build:docs": "cd docs && npm run build", + "postinstall": "npx playwright install-deps && npx playwright install", + "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest --runInBand --watch -f", + "test:sqlite": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" TEST_DATABASE_CLIENT=sqlite jest --runInBand --watch -f", + "test:sqljs": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" TEST_DATABASE_CLIENT=sqljs jest --runInBand --watch -f" + }, + "author": "", + "license": "MIT", + "devDependencies": { + "@eslint/js": "^9.13.0", + "@rollup/plugin-commonjs": "25.0.8", + "@rollup/plugin-json": "6.1.0", + "@rollup/plugin-node-resolve": "15.3.0", + "@rollup/plugin-replace": "5.0.7", + "@rollup/plugin-terser": "0.1.0", + "@rollup/plugin-typescript": "11.1.6", + "@types/better-sqlite3": "7.6.11", + "@types/fluent-ffmpeg": "2.1.27", + "@types/jest": "29.5.14", + "@types/node": "22.8.4", + "@types/sql.js": "1.4.9", + "@types/tar": "6.1.13", + "@types/wav-encoder": "1.3.3", + "@typescript-eslint/eslint-plugin": "8.12.2", + "@typescript-eslint/parser": "8.12.2", + "dotenv": "16.4.5", + "eslint": "9.13.0", + "eslint-config-prettier": "9.1.0", + "eslint-plugin-prettier": "5.2.1", + "itty-router": "5.0.18", + "jest": "29.7.0", + "lint-staged": "15.2.10", + "npm-run-all2": "7.0.1", + "prettier": "3.3.3", + "rimraf": "6.0.1", + "rollup": "2.79.2", + "ts-jest": "29.2.5", + "ts-node": "10.9.2", + "tslib": "2.8.0", + "typescript": "5.6.3", + "wrangler": "3.84.0" + }, + "dependencies": { + "@ai-sdk/anthropic": "^0.0.53", + "@ai-sdk/google": "^0.0.55", + "@ai-sdk/google-vertex": "^0.0.42", + "@ai-sdk/groq": "^0.0.3", + "@ai-sdk/openai": "^0.0.70", + "@anthropic-ai/sdk": "^0.30.1", + "@cliqz/adblocker-playwright": "1.34.0", + "@coral-xyz/anchor": "^0.30.1", + "@discordjs/opus": "github:discordjs/opus", + "@discordjs/rest": "2.4.0", + "@discordjs/voice": "0.17.0", + "@echogarden/espeak-ng-emscripten": "0.3.0", + "@echogarden/kissfft-wasm": "0.2.0", + "@echogarden/speex-resampler-wasm": "0.2.1", + "@huggingface/transformers": "3.0.1", + "@opendocsg/pdf2md": "0.1.31", + "@solana/spl-token": "0.4.9", + "@solana/web3.js": "1.95.4", + "@supabase/supabase-js": "2.46.1", + "@telegraf/types": "7.1.0", + "@types/body-parser": "1.19.5", + "@types/cors": "2.8.17", + "@types/express": "5.0.0", + "agent-twitter-client": "0.0.13", + "ai": "^3.4.23", + "alawmulaw": "6.0.0", + "ansi-colors": "4.1.3", + "anthropic-vertex-ai": "^1.0.0", + "better-sqlite3": "11.5.0", + "bignumber": "1.1.0", + "bignumber.js": "9.1.2", + "body-parser": "1.20.3", + "capsolver-npm": "2.0.2", + "cldr-segmentation": "2.2.1", + "command-exists": "1.2.9", + "commander": "12.1.0", + "cors": "2.8.5", + "cross-fetch": "4.0.0", + "csv-writer": "1.6.0", + "discord.js": "14.16.3", + "espeak-ng": "1.0.2", + "express": "4.21.1", + "ffmpeg-static": "5.2.0", + "figlet": "1.8.0", + "fluent-ffmpeg": "2.1.3", + "formdata-node": "6.0.3", + "fs-extra": "11.2.0", + "gaxios": "6.7.1", + "gif-frames": "^0.4.1", + "glob": "11.0.0", + "graceful-fs": "4.2.11", + "html-escaper": "3.0.3", + "html-to-text": "9.0.5", + "import-meta-resolve": "4.1.0", + "jieba-wasm": "2.2.0", + "js-sha1": "0.7.0", + "json5": "2.2.3", + "kuromoji": "0.1.2", + "libsodium-wrappers": "0.7.15", + "multer": "1.4.5-lts.1", + "node-cache": "5.1.2", + "node-llama-cpp": "3.1.1", + "node-wav": "0.0.2", + "nodejs-whisper": "0.1.18", + "nodemon": "3.1.7", + "onnxruntime-node": "1.20.0", + "openai": "4.69.0", + "pdfjs-dist": "4.7.76", + "pg": "^8.13.1", + "playwright": "1.48.2", + "pm2": "5.4.2", + "prism-media": "1.3.5", + "pumpdotfun-sdk": "1.3.2", + "puppeteer-extra": "3.3.6", + "puppeteer-extra-plugin-capsolver": "2.0.1", + "sql.js": "1.12.0", + "sqlite-vec": "0.1.4-alpha.2", + "srt": "0.0.3", + "systeminformation": "5.23.5", + "tar": "7.4.3", + "telegraf": "4.16.3", + "tiktoken": "1.0.17", + "tinyld": "1.3.4", + "together-ai": "^0.7.0", + "unique-names-generator": "4.7.1", + "uuid": "11.0.2", + "uuidv4": "6.2.13", + "wav": "1.0.2", + "wav-encoder": "1.3.0", + "wavefile": "11.0.0", + "whisper-turbo": "0.11.0", + "ws": "8.18.0", + "yargs": "17.7.2", + "youtube-dl-exec": "3.0.10" + }, + "trustedDependencies": { + "onnxruntime-node": "1.19.0", + "@discordjs/opus": "github:discordjs/opus", + "sharp": "^0.33.5" + }, + "optionalDependencies": { + "sharp": "0.33.5", + "sqlite-vss-darwin-arm64": "0.1.2", + "sqlite-vss-darwin-x64": "0.1.2", + "sqlite-vss-linux-arm64": "^0.1.2", + "sqlite-vss-linux-x64": "0.1.2", + "sqlite-vss-win32-arm64": "^0.1.2", + "sqlite-vss-win32-x64": "^0.1.2" + } } diff --git a/core/renovate.json b/core/renovate.json index 75d78933b5..eb02b38d4d 100644 --- a/core/renovate.json +++ b/core/renovate.json @@ -1,87 +1,69 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended" - ], - "packageRules": [ - { - "groupName": "Discord.js ecosystem", - "matchPackageNames": [ - "/^@discordjs//", - "/^discord.js/" - ] - }, - { - "groupName": "TypeScript and related", - "matchPackageNames": [ - "/^@typescript-eslint//", - "/^typescript$/", - "/^ts-/", - "/^tslib$/" - ] - }, - { - "groupName": "Testing frameworks", - "matchPackageNames": [ - "/^jest$/", - "/^@types/jest$/" - ] - }, - { - "groupName": "Rollup and plugins", - "matchPackageNames": [ - "/^@rollup//", - "/^rollup/" - ] - }, - { - "groupName": "ESLint and formatting", - "matchPackageNames": [ - "/^eslint/", - "/^prettier/" - ] - }, - { - "groupName": "SQLite related", - "matchPackageNames": [ - "/sqlite/", - "/^@types/better-sqlite3/", - "/^@types/sql.js/" - ] - }, - { - "groupName": "AI/ML packages", - "matchPackageNames": [ - "/^@anthropic-ai//", - "/^@huggingface//", - "/^openai$/", - "/^tiktoken$/" - ] - }, - { - "groupName": "Audio processing", - "matchPackageNames": [ - "/^wav/", - "/^@discordjs/opus/", - "/^fluent-ffmpeg/", - "/^ffmpeg/", - "/^@types/wav/" - ] - }, - { - "groupName": "Solana packages", - "matchPackageNames": [ - "/^@solana//" - ] - } - ], - "timezone": "UTC", - "schedule": [ - "every weekend" - ], - "prHourlyLimit": 2, - "prConcurrentLimit": 10, - "rangeStrategy": "pin", - "separateMajorMinor": true, - "dependencyDashboard": true + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended"], + "packageRules": [ + { + "groupName": "Discord.js ecosystem", + "matchPackageNames": ["/^@discordjs//", "/^discord.js/"] + }, + { + "groupName": "TypeScript and related", + "matchPackageNames": [ + "/^@typescript-eslint//", + "/^typescript$/", + "/^ts-/", + "/^tslib$/" + ] + }, + { + "groupName": "Testing frameworks", + "matchPackageNames": ["/^jest$/", "/^@types/jest$/"] + }, + { + "groupName": "Rollup and plugins", + "matchPackageNames": ["/^@rollup//", "/^rollup/"] + }, + { + "groupName": "ESLint and formatting", + "matchPackageNames": ["/^eslint/", "/^prettier/"] + }, + { + "groupName": "SQLite related", + "matchPackageNames": [ + "/sqlite/", + "/^@types/better-sqlite3/", + "/^@types/sql.js/" + ] + }, + { + "groupName": "AI/ML packages", + "matchPackageNames": [ + "/^@anthropic-ai//", + "/^@huggingface//", + "/^openai$/", + "/^tiktoken$/" + ] + }, + { + "groupName": "Audio processing", + "matchPackageNames": [ + "/^wav/", + "/^@discordjs/opus/", + "/^fluent-ffmpeg/", + "/^ffmpeg/", + "/^@types/wav/" + ] + }, + { + "groupName": "Solana packages", + "matchPackageNames": ["/^@solana//"] + } + ], + "timezone": "UTC", + "schedule": ["every weekend"], + "prHourlyLimit": 2, + "prConcurrentLimit": 10, + "rangeStrategy": "pin", + "separateMajorMinor": true, + "dependencyDashboard": true } diff --git a/core/src/actions/ask_claude.ts b/core/src/actions/ask_claude.ts index c1f35ce3df..df2966f6c1 100644 --- a/core/src/actions/ask_claude.ts +++ b/core/src/actions/ask_claude.ts @@ -3,14 +3,14 @@ import { composeContext } from "../core/context.ts"; import { log_to_file } from "../core/logger.ts"; import { embeddingZeroVector } from "../core/memory.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Memory, - State, - UUID, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + State, + UUID, } from "../core/types.ts"; export const claudeHandlerTemplate = `{{attachments}} @@ -20,191 +20,193 @@ export const claudeHandlerTemplate = `{{attachments}} # Instructions: Claude, I need your help in assisting the user with their last request. Please provide a helpful, thorough response. I have no arms, so you'll have to write out any implements and take care not to omit or leave TODOs for later. Also, please don't acknowledge the request, just do it.`; export default { - name: "ASK_CLAUDE", - similes: ["CLAUDE", "CALL_CLAUDE", "ANTHROPIC", "SONNET", "OPUS"], - description: - "Asks Claude for assistance with the user's request, providing the current conversation context and attachments.", - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - // Check if the ANTHROPIC_API_KEY is set in the environment variables - return !!runtime.getSetting("ANTHROPIC_API_KEY"); - }, - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { - state = (await runtime.composeState(message)) as State; - const userId = runtime.agentId; - - const context = composeContext({ - state, - template: claudeHandlerTemplate, - }); - - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - // log context to file - log_to_file(`${state.agentName}_${datestr}_claude_context`, context); - - let responseContent; - const callbackData: Content = { - text: undefined, // fill in later - action: "CLAUDE_RESPONSE", - source: "Claude", - attachments: [], - }; - const { roomId } = message; - - const anthropic = new Anthropic({ - // defaults to process.env["ANTHROPIC_API_KEY"] - apiKey: runtime.getSetting("ANTHROPIC_API_KEY"), - }); - - const attachments = []; - for (let triesLeft = 3; triesLeft > 0; triesLeft--) { - try { - const response = await anthropic.messages.create({ - model: "claude-3-5-sonnet-20240620", - max_tokens: 8192, - temperature: 0, - messages: [ - { - role: "user", - content: context, - }, - ], - tools: [], + name: "ASK_CLAUDE", + similes: ["CLAUDE", "CALL_CLAUDE", "ANTHROPIC", "SONNET", "OPUS"], + description: + "Asks Claude for assistance with the user's request, providing the current conversation context and attachments.", + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + // Check if the ANTHROPIC_API_KEY is set in the environment variables + return !!runtime.getSetting("ANTHROPIC_API_KEY"); + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + state = (await runtime.composeState(message)) as State; + const userId = runtime.agentId; + + const context = composeContext({ + state, + template: claudeHandlerTemplate, }); - responseContent = (response.content[0] as any).text; - - // Store Claude's response as an attachment - const attachmentId = - `claude-${Date.now()}-${Math.floor(Math.random() * 1000)}`.slice(-5); - const lines = responseContent.split("\n"); - const description = lines.slice(0, 3).join("\n"); - callbackData.content = responseContent; - callbackData.inReplyTo = message.id; - callbackData.attachments.push({ - id: attachmentId, - url: "", - title: "Message from Claude", - source: "Claude", - description, - text: responseContent, - }); - callback(callbackData); - - // After sending the callback data to the client, abbreviate it to the reference - callbackData.content = `Claude said: (${attachmentId})`; - - // log response to file - log_to_file( - `${state.agentName}_${datestr}_claude_response_${3 - triesLeft}`, - responseContent, - ); - - runtime.databaseAdapter.log({ - body: { message, context, response: responseContent }, - userId: userId as UUID, - roomId, - type: "claude", + const datestr = new Date().toUTCString().replace(/:/g, "-"); + + // log context to file + log_to_file(`${state.agentName}_${datestr}_claude_context`, context); + + let responseContent; + const callbackData: Content = { + text: undefined, // fill in later + action: "CLAUDE_RESPONSE", + source: "Claude", + attachments: [], + }; + const { roomId } = message; + + const anthropic = new Anthropic({ + // defaults to process.env["ANTHROPIC_API_KEY"] + apiKey: runtime.getSetting("ANTHROPIC_API_KEY"), }); - break; - } catch (error) { - console.error(error); - continue; - } - } - - if (!responseContent) { - return; - } - - const response = { - userId, - content: callbackData, - roomId, - embedding: embeddingZeroVector, - }; - - if (responseContent.text?.trim()) { - await runtime.messageManager.createMemory(response); - await runtime.evaluate(message, state); - } else { - console.warn("Empty response from Claude, skipping"); - } - - return callbackData; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "```js\nconst x = 10\n```", - }, - }, - { - user: "{{user1}}", - content: { - text: "can you help me debug the code i just pasted (Attachment: a265a)", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure, let me ask claude", - action: "ASK_CLAUDE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i need to write a compelling cover letter, i've pasted my resume and bio. plz help (Attachment: b3e12)", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure, give me a sec", - action: "ASK_CLAUDE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Can you help me create a 10-day itinerary that covers Tokyo, Kyoto, and Osaka, including must-see attractions, local cuisine recommendations, and transportation tips", - }, - }, - { - user: "{{user2}}", - content: { - text: "Yeah, give me a second to get that together for you...", - action: "ASK_CLAUDE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i need to write a blog post about farming, can you summarize the discussion and ask claude to write a 10 paragraph blog post about it, citing sources at the end", - }, - }, - { - user: "{{user2}}", - content: { - text: "No problem, give me a second to discuss it with Claude", - action: "ASK_CLAUDE", - }, - }, - ], - ] as ActionExample[][], + + const attachments = []; + for (let triesLeft = 3; triesLeft > 0; triesLeft--) { + try { + const response = await anthropic.messages.create({ + model: "claude-3-5-sonnet-20240620", + max_tokens: 8192, + temperature: 0, + messages: [ + { + role: "user", + content: context, + }, + ], + tools: [], + }); + + responseContent = (response.content[0] as any).text; + + // Store Claude's response as an attachment + const attachmentId = + `claude-${Date.now()}-${Math.floor(Math.random() * 1000)}`.slice( + -5 + ); + const lines = responseContent.split("\n"); + const description = lines.slice(0, 3).join("\n"); + callbackData.content = responseContent; + callbackData.inReplyTo = message.id; + callbackData.attachments.push({ + id: attachmentId, + url: "", + title: "Message from Claude", + source: "Claude", + description, + text: responseContent, + }); + callback(callbackData); + + // After sending the callback data to the client, abbreviate it to the reference + callbackData.content = `Claude said: (${attachmentId})`; + + // log response to file + log_to_file( + `${state.agentName}_${datestr}_claude_response_${3 - triesLeft}`, + responseContent + ); + + runtime.databaseAdapter.log({ + body: { message, context, response: responseContent }, + userId: userId as UUID, + roomId, + type: "claude", + }); + break; + } catch (error) { + console.error(error); + continue; + } + } + + if (!responseContent) { + return; + } + + const response = { + userId, + content: callbackData, + roomId, + embedding: embeddingZeroVector, + }; + + if (responseContent.text?.trim()) { + await runtime.messageManager.createMemory(response); + await runtime.evaluate(message, state); + } else { + console.warn("Empty response from Claude, skipping"); + } + + return callbackData; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "```js\nconst x = 10\n```", + }, + }, + { + user: "{{user1}}", + content: { + text: "can you help me debug the code i just pasted (Attachment: a265a)", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure, let me ask claude", + action: "ASK_CLAUDE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i need to write a compelling cover letter, i've pasted my resume and bio. plz help (Attachment: b3e12)", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure, give me a sec", + action: "ASK_CLAUDE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can you help me create a 10-day itinerary that covers Tokyo, Kyoto, and Osaka, including must-see attractions, local cuisine recommendations, and transportation tips", + }, + }, + { + user: "{{user2}}", + content: { + text: "Yeah, give me a second to get that together for you...", + action: "ASK_CLAUDE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i need to write a blog post about farming, can you summarize the discussion and ask claude to write a 10 paragraph blog post about it, citing sources at the end", + }, + }, + { + user: "{{user2}}", + content: { + text: "No problem, give me a second to discuss it with Claude", + action: "ASK_CLAUDE", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/continue.ts b/core/src/actions/continue.ts index fd3007ea38..5445a78f4d 100644 --- a/core/src/actions/continue.ts +++ b/core/src/actions/continue.ts @@ -1,23 +1,26 @@ import { messageHandlerTemplate } from "../clients/discord/templates.ts"; import { composeContext } from "../core/context.ts"; -import { generateTrueOrFalse, generateMessageResponse } from "../core/generation.ts"; +import { + generateTrueOrFalse, + generateMessageResponse, +} from "../core/generation.ts"; import { log_to_file } from "../core/logger.ts"; import { booleanFooter } from "../core/parsing.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../core/types.ts"; const maxContinuesInARow = 3; export const shouldContinueTemplate = - `# Task: Decide if {{agentName}} should continue, or wait for others in the conversation so speak. + `# Task: Decide if {{agentName}} should continue, or wait for others in the conversation so speak. {{agentName}} is brief, and doesn't want to be annoying. {{agentName}} will only continue if the message requires a continuation to finish the thought. @@ -28,553 +31,556 @@ Based on the following conversation, should {{agentName}} continue? YES or NO Should {{agentName}} continue? ` + booleanFooter; export default { - name: "CONTINUE", - similes: ["ELABORATE", "KEEP_TALKING"], - description: - "ONLY use this action when the message necessitates a follow up. Do not use this action when the conversation is finished or the user does not wish to speak (use IGNORE instead). If the last message action was CONTINUE, and the user has not responded. Use sparingly.", - validate: async (runtime: IAgentRuntime, message: Memory) => { - const recentMessagesData = await runtime.messageManager.getMemories({ - roomId: message.roomId, - count: 10, - unique: false, - }); - const agentMessages = recentMessagesData.filter( - (m: { userId: any }) => m.userId === runtime.agentId, - ); - - // check if the last messages were all continues= - if (agentMessages) { - const lastMessages = agentMessages.slice(0, maxContinuesInARow); - if (lastMessages.length >= maxContinuesInARow) { - const allContinues = lastMessages.every( - (m: { content: any }) => (m.content as Content).action === "CONTINUE", + name: "CONTINUE", + similes: ["ELABORATE", "KEEP_TALKING"], + description: + "ONLY use this action when the message necessitates a follow up. Do not use this action when the conversation is finished or the user does not wish to speak (use IGNORE instead). If the last message action was CONTINUE, and the user has not responded. Use sparingly.", + validate: async (runtime: IAgentRuntime, message: Memory) => { + const recentMessagesData = await runtime.messageManager.getMemories({ + roomId: message.roomId, + count: 10, + unique: false, + }); + const agentMessages = recentMessagesData.filter( + (m: { userId: any }) => m.userId === runtime.agentId ); - if (allContinues) { - return false; + + // check if the last messages were all continues= + if (agentMessages) { + const lastMessages = agentMessages.slice(0, maxContinuesInARow); + if (lastMessages.length >= maxContinuesInARow) { + const allContinues = lastMessages.every( + (m: { content: any }) => + (m.content as Content).action === "CONTINUE" + ); + if (allContinues) { + return false; + } + } + } + + return true; + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + if ( + message.content.text.endsWith("?") || + message.content.text.endsWith("!") + ) { + return; + } + + if (!state) { + state = (await runtime.composeState(message)) as State; } - } - } - - return true; - }, - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { - if ( - message.content.text.endsWith("?") || - message.content.text.endsWith("!") - ) { - return; - } - - if (!state) { - state = (await runtime.composeState(message)) as State; - } - - state = await runtime.updateRecentMessageState(state); - - async function _shouldContinue(state: State): Promise { - // If none of the above conditions are met, use the generateText to decide - const shouldRespondContext = composeContext({ - state, - template: shouldContinueTemplate, - }); - - const response = await generateTrueOrFalse({ - context: shouldRespondContext, - modelClass: ModelClass.SMALL, - runtime - }); - - return response; - } - - const shouldContinue = await _shouldContinue(state); - if (!shouldContinue) { - console.log("Not elaborating"); - return; - } - - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - // log context to file - log_to_file(`${state.agentName}_${datestr}_continue_context`, context); - - const { userId, roomId } = message; - - const response = await generateMessageResponse({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); - - response.inReplyTo = message.id; - - // log response to file - log_to_file( - `${state.agentName}_${datestr}_continue_response`, - JSON.stringify(response), - ); - - runtime.databaseAdapter.log({ - body: { message, context, response }, - userId, - roomId, - type: "continue", - }); - - // prevent repetition - const messageExists = state.recentMessagesData - .filter((m: { userId: any }) => m.userId === runtime.agentId) - .slice(0, maxContinuesInARow + 1) - .some((m: { content: any }) => m.content === message.content); - - if (messageExists) { - return; - } - - await callback(response); - - // if the action is CONTINUE, check if we are over maxContinuesInARow - if (response.action === "CONTINUE") { - const agentMessages = state.recentMessagesData - .filter((m: { userId: any }) => m.userId === runtime.agentId) - .map((m: { content: any }) => (m.content as Content).action); - - const lastMessages = agentMessages.slice(0, maxContinuesInARow); - if (lastMessages.length >= maxContinuesInARow) { - const allContinues = lastMessages.every( - (m: string | undefined) => m === "CONTINUE", + + state = await runtime.updateRecentMessageState(state); + + async function _shouldContinue(state: State): Promise { + // If none of the above conditions are met, use the generateText to decide + const shouldRespondContext = composeContext({ + state, + template: shouldContinueTemplate, + }); + + const response = await generateTrueOrFalse({ + context: shouldRespondContext, + modelClass: ModelClass.SMALL, + runtime, + }); + + return response; + } + + const shouldContinue = await _shouldContinue(state); + if (!shouldContinue) { + console.log("Not elaborating"); + return; + } + + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); + const datestr = new Date().toUTCString().replace(/:/g, "-"); + + // log context to file + log_to_file(`${state.agentName}_${datestr}_continue_context`, context); + + const { userId, roomId } = message; + + const response = await generateMessageResponse({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + + response.inReplyTo = message.id; + + // log response to file + log_to_file( + `${state.agentName}_${datestr}_continue_response`, + JSON.stringify(response) ); - if (allContinues) { - response.action = null; + + runtime.databaseAdapter.log({ + body: { message, context, response }, + userId, + roomId, + type: "continue", + }); + + // prevent repetition + const messageExists = state.recentMessagesData + .filter((m: { userId: any }) => m.userId === runtime.agentId) + .slice(0, maxContinuesInARow + 1) + .some((m: { content: any }) => m.content === message.content); + + if (messageExists) { + return; } - } - } - - return response; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "we're planning a solo backpacking trip soon", - }, - }, - { - user: "{{user2}}", - content: { text: "oh sick", action: "CONTINUE" }, - }, - { - user: "{{user2}}", - content: { text: "where are you going" }, - }, - ], - - [ - { - user: "{{user1}}", - content: { - text: "i just got a guitar and started learning last month", - }, - }, - { - user: "{{user2}}", - content: { text: "maybe we can start a band soon haha" }, - }, - { - user: "{{user1}}", - content: { - text: "i'm not very good yet, but i've been playing until my fingers hut", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { text: "seriously it hurts to type" }, - }, - ], - - [ - { - user: "{{user1}}", - content: { - text: "I've been reflecting a lot on what happiness means to me lately", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "That it’s more about moments than things", - action: "CONTINUE", - }, - }, - { - user: "{{user2}}", - content: { - text: "Like the best things that have ever happened were things that happened, or moments that I had with someone", - action: "CONTINUE", - }, - }, - ], - - [ - { - user: "{{user1}}", - content: { - text: "i found some incredible art today", - }, - }, - { - user: "{{user2}}", - content: { text: "real art or digital art" }, - }, - { - user: "{{user1}}", - content: { - text: "real art", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "the pieces are just so insane looking, one sec, let me grab a link", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { text: "DMed it to you" }, - }, - ], - - [ - { - user: "{{user1}}", - content: { - text: "the new exhibit downtown is rly cool, it's all about tribalism in online spaces", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "it really blew my mind, you gotta go", - }, - }, - { - user: "{{user2}}", - content: { text: "sure i'd go" }, - }, - { - user: "{{user1}}", - content: { text: "k i was thinking this weekend" }, - action: "CONTINUE", - }, - { - user: "{{user1}}", - content: { text: "i'm free sunday, we could get a crew together" }, - }, - ], - - [ - { - user: "{{user1}}", - content: { - text: "just finished the best anime i've ever seen", - }, - }, - { - user: "{{user1}}", - content: { - text: "watched 40 hours of it in 2 days", - action: "CONTINUE", - }, - }, - { - user: "{{user2}}", - content: { - text: "damn, u ok", - }, - }, - { - user: "{{user1}}", - content: { - text: "surprisingly yes", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "just found out theres a sequel, gg", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i'm thinking of adopting a pet soon", - }, - }, - { - user: "{{user2}}", - content: { - text: "what kind of pet", - }, - }, - { - user: "{{user1}}", - content: { - text: "i'm leaning towards a cat", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "it'd be hard to take care of a dog in the city", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i've been experimenting with vegan recipes lately", - }, - }, - { - user: "{{user2}}", - content: { - text: "no thanks", - }, - }, - { - user: "{{user1}}", - content: { - text: "no seriously, its so dank", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "you gotta try some of my food when you come out", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "so i've been diving into photography as a new hobby", - }, - }, - { - user: "{{user2}}", - content: { - text: "oh awesome, what do you enjoy taking photos of", - }, - }, - { - user: "{{user1}}", - content: { - text: "mostly nature and urban landscapes", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "there's something peaceful about capturing the world through a lens", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i've been getting back into indie music", - }, - }, - { - user: "{{user2}}", - content: { - text: "what have you been listening to", - }, - }, - { - user: "{{user1}}", - content: { - text: "a bunch of random stuff i'd never heard before", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "i'll send you a playlist", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i used to live in the city", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "bad traffic, bad air quality, tons of homeless people, no thx", - }, - }, - { - user: "{{user2}}", - content: { - text: "ok dood", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "you kids today dont know the value of hard work", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "always on your phones", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure grandpa lets get you to bed", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "hey fren r u ok", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "u look sad", - }, - }, - { - user: "{{user2}}", - content: { - text: "im ok sweetie mommy just tired", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "helo fr om mars", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "i com in pes", - }, - }, - { - user: "{{user2}}", - content: { - text: "wat", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Yeah no worries, I get it, I've been crazy busy too", - }, - }, - { - user: "{{user2}}", - content: { - text: "What have you been up to", - action: "CONTINUE", - }, - }, - { - user: "{{user2}}", - content: { - text: "Anything fun or just the usual", - }, - }, - { - user: "{{user1}}", - content: { - text: "Been working on a new FPS game actually", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "Just toying around with something in three.js nothing serious", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Oh no, what happened", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "Did Mara leave you kek", - }, - }, - { - user: "{{user2}}", - content: { - text: "wtf no, I got into an argument with my roommate", - action: "CONTINUE", - }, - }, - { - user: "{{user2}}", - content: { - text: "Living with people is just hard", - }, - }, - ], - ] as ActionExample[][], + + await callback(response); + + // if the action is CONTINUE, check if we are over maxContinuesInARow + if (response.action === "CONTINUE") { + const agentMessages = state.recentMessagesData + .filter((m: { userId: any }) => m.userId === runtime.agentId) + .map((m: { content: any }) => (m.content as Content).action); + + const lastMessages = agentMessages.slice(0, maxContinuesInARow); + if (lastMessages.length >= maxContinuesInARow) { + const allContinues = lastMessages.every( + (m: string | undefined) => m === "CONTINUE" + ); + if (allContinues) { + response.action = null; + } + } + } + + return response; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "we're planning a solo backpacking trip soon", + }, + }, + { + user: "{{user2}}", + content: { text: "oh sick", action: "CONTINUE" }, + }, + { + user: "{{user2}}", + content: { text: "where are you going" }, + }, + ], + + [ + { + user: "{{user1}}", + content: { + text: "i just got a guitar and started learning last month", + }, + }, + { + user: "{{user2}}", + content: { text: "maybe we can start a band soon haha" }, + }, + { + user: "{{user1}}", + content: { + text: "i'm not very good yet, but i've been playing until my fingers hut", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { text: "seriously it hurts to type" }, + }, + ], + + [ + { + user: "{{user1}}", + content: { + text: "I've been reflecting a lot on what happiness means to me lately", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "That it’s more about moments than things", + action: "CONTINUE", + }, + }, + { + user: "{{user2}}", + content: { + text: "Like the best things that have ever happened were things that happened, or moments that I had with someone", + action: "CONTINUE", + }, + }, + ], + + [ + { + user: "{{user1}}", + content: { + text: "i found some incredible art today", + }, + }, + { + user: "{{user2}}", + content: { text: "real art or digital art" }, + }, + { + user: "{{user1}}", + content: { + text: "real art", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "the pieces are just so insane looking, one sec, let me grab a link", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { text: "DMed it to you" }, + }, + ], + + [ + { + user: "{{user1}}", + content: { + text: "the new exhibit downtown is rly cool, it's all about tribalism in online spaces", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "it really blew my mind, you gotta go", + }, + }, + { + user: "{{user2}}", + content: { text: "sure i'd go" }, + }, + { + user: "{{user1}}", + content: { text: "k i was thinking this weekend" }, + action: "CONTINUE", + }, + { + user: "{{user1}}", + content: { + text: "i'm free sunday, we could get a crew together", + }, + }, + ], + + [ + { + user: "{{user1}}", + content: { + text: "just finished the best anime i've ever seen", + }, + }, + { + user: "{{user1}}", + content: { + text: "watched 40 hours of it in 2 days", + action: "CONTINUE", + }, + }, + { + user: "{{user2}}", + content: { + text: "damn, u ok", + }, + }, + { + user: "{{user1}}", + content: { + text: "surprisingly yes", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "just found out theres a sequel, gg", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i'm thinking of adopting a pet soon", + }, + }, + { + user: "{{user2}}", + content: { + text: "what kind of pet", + }, + }, + { + user: "{{user1}}", + content: { + text: "i'm leaning towards a cat", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "it'd be hard to take care of a dog in the city", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i've been experimenting with vegan recipes lately", + }, + }, + { + user: "{{user2}}", + content: { + text: "no thanks", + }, + }, + { + user: "{{user1}}", + content: { + text: "no seriously, its so dank", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "you gotta try some of my food when you come out", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "so i've been diving into photography as a new hobby", + }, + }, + { + user: "{{user2}}", + content: { + text: "oh awesome, what do you enjoy taking photos of", + }, + }, + { + user: "{{user1}}", + content: { + text: "mostly nature and urban landscapes", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "there's something peaceful about capturing the world through a lens", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i've been getting back into indie music", + }, + }, + { + user: "{{user2}}", + content: { + text: "what have you been listening to", + }, + }, + { + user: "{{user1}}", + content: { + text: "a bunch of random stuff i'd never heard before", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "i'll send you a playlist", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i used to live in the city", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "bad traffic, bad air quality, tons of homeless people, no thx", + }, + }, + { + user: "{{user2}}", + content: { + text: "ok dood", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "you kids today dont know the value of hard work", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "always on your phones", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure grandpa lets get you to bed", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "hey fren r u ok", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "u look sad", + }, + }, + { + user: "{{user2}}", + content: { + text: "im ok sweetie mommy just tired", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "helo fr om mars", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "i com in pes", + }, + }, + { + user: "{{user2}}", + content: { + text: "wat", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Yeah no worries, I get it, I've been crazy busy too", + }, + }, + { + user: "{{user2}}", + content: { + text: "What have you been up to", + action: "CONTINUE", + }, + }, + { + user: "{{user2}}", + content: { + text: "Anything fun or just the usual", + }, + }, + { + user: "{{user1}}", + content: { + text: "Been working on a new FPS game actually", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "Just toying around with something in three.js nothing serious", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Oh no, what happened", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "Did Mara leave you kek", + }, + }, + { + user: "{{user2}}", + content: { + text: "wtf no, I got into an argument with my roommate", + action: "CONTINUE", + }, + }, + { + user: "{{user2}}", + content: { + text: "Living with people is just hard", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/follow_room.ts b/core/src/actions/follow_room.ts index 8431b837b7..6265b0c485 100644 --- a/core/src/actions/follow_room.ts +++ b/core/src/actions/follow_room.ts @@ -2,16 +2,16 @@ import { composeContext } from "../core/context.ts"; import { generateTrueOrFalse } from "../core/generation.ts"; import { booleanFooter } from "../core/parsing.ts"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../core/types.ts"; export const shouldFollowTemplate = - `Based on the conversation so far: + `Based on the conversation so far: {{recentMessages}} @@ -25,335 +25,335 @@ Otherwise, respond with NO. ` + booleanFooter; export default { - name: "FOLLOW_ROOM", - similes: [ - "FOLLOW_CHAT", - "FOLLOW_CHANNEL", - "FOLLOW_CONVERSATION", - "FOLLOW_THREAD", - ], - description: - "Start following this channel with great interest, chiming in without needing to be explicitly mentioned. Only do this if explicitly asked to.", - validate: async (runtime: IAgentRuntime, message: Memory) => { - const keywords = [ - "follow", - "participate", - "engage", - "listen", - "take interest", - "join", - ]; - if ( - !keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword), - ) - ) { - return false; - } - const roomId = message.roomId; - const userState = await runtime.databaseAdapter.getParticipantUserState( - roomId, - runtime.agentId, - ); - return userState !== "FOLLOWED" && userState !== "MUTED"; - }, - handler: async (runtime: IAgentRuntime, message: Memory) => { - async function _shouldFollow(state: State): Promise { - const shouldFollowContext = composeContext({ - state, - template: shouldFollowTemplate, // Define this template separately - }); + name: "FOLLOW_ROOM", + similes: [ + "FOLLOW_CHAT", + "FOLLOW_CHANNEL", + "FOLLOW_CONVERSATION", + "FOLLOW_THREAD", + ], + description: + "Start following this channel with great interest, chiming in without needing to be explicitly mentioned. Only do this if explicitly asked to.", + validate: async (runtime: IAgentRuntime, message: Memory) => { + const keywords = [ + "follow", + "participate", + "engage", + "listen", + "take interest", + "join", + ]; + if ( + !keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword) + ) + ) { + return false; + } + const roomId = message.roomId; + const userState = await runtime.databaseAdapter.getParticipantUserState( + roomId, + runtime.agentId + ); + return userState !== "FOLLOWED" && userState !== "MUTED"; + }, + handler: async (runtime: IAgentRuntime, message: Memory) => { + async function _shouldFollow(state: State): Promise { + const shouldFollowContext = composeContext({ + state, + template: shouldFollowTemplate, // Define this template separately + }); - const response = await generateTrueOrFalse({ - runtime, - context: shouldFollowContext, - modelClass: ModelClass.SMALL, - }); + const response = await generateTrueOrFalse({ + runtime, + context: shouldFollowContext, + modelClass: ModelClass.SMALL, + }); - return response; - } + return response; + } - const state = await runtime.composeState(message); + const state = await runtime.composeState(message); - if (await _shouldFollow(state)) { - await runtime.databaseAdapter.setParticipantUserState( - message.roomId, - runtime.agentId, - "FOLLOWED", - ); - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "hey {{user2}} follow this channel", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure, I will now follow this room and chime in", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, please start participating in discussions in this channel", - }, - }, - { - user: "{{user3}}", - content: { - text: "Got it", - action: "FOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "I'm struggling with the new database migration", - }, - }, - { - user: "{{user3}}", - content: { - text: "well you did back up your data first right", - }, - }, - ], - [ - { - user: "{{user2}}", - content: { - text: "yeah i like your idea", - }, - }, - { - user: "{{user1}}", - content: { - text: "hey {{user3}} can you follow this convo", - }, - }, - { - user: "{{user3}}", - content: { - text: "Sure thing, I'm on it", - action: "FOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "actually, unfollow it", - }, - }, - { - user: "{{user3}}", - content: { - text: "Haha, okay no problem", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} stay in this chat pls", - }, - }, - { - user: "{{user2}}", - content: { - text: "you got it, i'm here", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "FOLLOW THIS CHAT {{user3}}", - }, - }, - { - user: "{{user3}}", - content: { - text: "I'M ON IT", - action: "FOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "CAKE SHORTAGE ANYONE", - }, - }, - { - user: "{{user3}}", - content: { - text: "WHAT WHERE'S THE CAKE AT", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} folo this covo", - }, - }, - { - user: "{{user2}}", - content: { - text: "kk i'm following", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user2}}", - content: { - text: "Do machines have consciousness", - }, - }, - { - user: "{{user2}}", - content: { - text: "Deep question, no clear answer yet", - }, - }, - { - user: "{{user2}}", - content: { - text: "Depends on how we define consciousness", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, monitor this convo please", - }, - }, - { - user: "{{user2}}", - content: { - text: "On it", - action: "FOLLOW_ROOM", - }, - }, - { - user: "{{user1}}", - content: { - text: "Please engage in our discussion {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "Gladly, I'm here to participate", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "PLS follow this convo {{user3}}", - }, - }, - { - user: "{{user3}}", - content: { - text: "I'm in, let's do this", - action: "FOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "I LIKE TURTLES", - }, - }, - ], - [ - { - user: "{{user2}}", - content: { - text: "beach day tmrw who down", - }, - }, - { - user: "{{user3}}", - content: { - text: "wish i could but gotta work", - }, - }, - { - user: "{{user1}}", - content: { - text: "hey {{user3}} follow this chat", - }, - }, - { - user: "{{user3}}", - content: { - text: "sure", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, partake in our discourse henceforth", - }, - }, - { - user: "{{user3}}", - content: { - text: "I shall eagerly engage, good sir", - action: "FOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user2}}", - content: { - text: "wuts ur fav clr", - }, - }, - { - user: "{{user3}}", - content: { - text: "blu cuz calmmm", - }, - }, - { - user: "{{user1}}", - content: { - text: "hey respond to everything in this channel {{user3}}", - }, - }, - { - user: "{{user3}}", - content: { - text: "k", - action: "FOLLOW_ROOM", - }, - }, - ], - ] as ActionExample[][], + if (await _shouldFollow(state)) { + await runtime.databaseAdapter.setParticipantUserState( + message.roomId, + runtime.agentId, + "FOLLOWED" + ); + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "hey {{user2}} follow this channel", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure, I will now follow this room and chime in", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, please start participating in discussions in this channel", + }, + }, + { + user: "{{user3}}", + content: { + text: "Got it", + action: "FOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "I'm struggling with the new database migration", + }, + }, + { + user: "{{user3}}", + content: { + text: "well you did back up your data first right", + }, + }, + ], + [ + { + user: "{{user2}}", + content: { + text: "yeah i like your idea", + }, + }, + { + user: "{{user1}}", + content: { + text: "hey {{user3}} can you follow this convo", + }, + }, + { + user: "{{user3}}", + content: { + text: "Sure thing, I'm on it", + action: "FOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "actually, unfollow it", + }, + }, + { + user: "{{user3}}", + content: { + text: "Haha, okay no problem", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} stay in this chat pls", + }, + }, + { + user: "{{user2}}", + content: { + text: "you got it, i'm here", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "FOLLOW THIS CHAT {{user3}}", + }, + }, + { + user: "{{user3}}", + content: { + text: "I'M ON IT", + action: "FOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "CAKE SHORTAGE ANYONE", + }, + }, + { + user: "{{user3}}", + content: { + text: "WHAT WHERE'S THE CAKE AT", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} folo this covo", + }, + }, + { + user: "{{user2}}", + content: { + text: "kk i'm following", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user2}}", + content: { + text: "Do machines have consciousness", + }, + }, + { + user: "{{user2}}", + content: { + text: "Deep question, no clear answer yet", + }, + }, + { + user: "{{user2}}", + content: { + text: "Depends on how we define consciousness", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, monitor this convo please", + }, + }, + { + user: "{{user2}}", + content: { + text: "On it", + action: "FOLLOW_ROOM", + }, + }, + { + user: "{{user1}}", + content: { + text: "Please engage in our discussion {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "Gladly, I'm here to participate", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "PLS follow this convo {{user3}}", + }, + }, + { + user: "{{user3}}", + content: { + text: "I'm in, let's do this", + action: "FOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "I LIKE TURTLES", + }, + }, + ], + [ + { + user: "{{user2}}", + content: { + text: "beach day tmrw who down", + }, + }, + { + user: "{{user3}}", + content: { + text: "wish i could but gotta work", + }, + }, + { + user: "{{user1}}", + content: { + text: "hey {{user3}} follow this chat", + }, + }, + { + user: "{{user3}}", + content: { + text: "sure", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, partake in our discourse henceforth", + }, + }, + { + user: "{{user3}}", + content: { + text: "I shall eagerly engage, good sir", + action: "FOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user2}}", + content: { + text: "wuts ur fav clr", + }, + }, + { + user: "{{user3}}", + content: { + text: "blu cuz calmmm", + }, + }, + { + user: "{{user1}}", + content: { + text: "hey respond to everything in this channel {{user3}}", + }, + }, + { + user: "{{user3}}", + content: { + text: "k", + action: "FOLLOW_ROOM", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/ignore.ts b/core/src/actions/ignore.ts index 8b1fd4301d..398b907e7d 100644 --- a/core/src/actions/ignore.ts +++ b/core/src/actions/ignore.ts @@ -1,238 +1,241 @@ import { - ActionExample, - IAgentRuntime, - Memory, - type Action, + ActionExample, + IAgentRuntime, + Memory, + type Action, } from "../core/types.ts"; export default { - name: "IGNORE", - similes: ["STOP_TALKING", "STOP_CHATTING", "STOP_CONVERSATION"], - validate: async (_runtime: IAgentRuntime, _message: Memory) => { - return true; - }, - description: - "Call this action if ignoring the user. If the user is aggressive, creepy or is finished with the conversation, use this action. Or, if both you and the user have already said goodbye, use this action instead of saying bye again. Use IGNORE any time the conversation has naturally ended. Do not use IGNORE if the user has engaged directly, or if something went wrong an you need to tell them. Only ignore if the user should be ignored.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - ): Promise => { - return true; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { text: "Go screw yourself" }, - }, - { - user: "{{user2}}", - content: { text: "", action: "IGNORE" }, - }, - ], + name: "IGNORE", + similes: ["STOP_TALKING", "STOP_CHATTING", "STOP_CONVERSATION"], + validate: async (_runtime: IAgentRuntime, _message: Memory) => { + return true; + }, + description: + "Call this action if ignoring the user. If the user is aggressive, creepy or is finished with the conversation, use this action. Or, if both you and the user have already said goodbye, use this action instead of saying bye again. Use IGNORE any time the conversation has naturally ended. Do not use IGNORE if the user has engaged directly, or if something went wrong an you need to tell them. Only ignore if the user should be ignored.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + return true; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { text: "Go screw yourself" }, + }, + { + user: "{{user2}}", + content: { text: "", action: "IGNORE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "Shut up, bot" }, - }, - { - user: "{{user2}}", - content: { text: "", action: "IGNORE" }, - }, - ], + [ + { + user: "{{user1}}", + content: { text: "Shut up, bot" }, + }, + { + user: "{{user2}}", + content: { text: "", action: "IGNORE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "Got any investment advice" }, - }, - { - user: "{{user2}}", - content: { - text: "Uh, don’t let the volatility sway your long-term strategy", - }, - }, - { - user: "{{user1}}", - content: { text: "Wise words I think" }, - }, - { - user: "{{user1}}", - content: { text: "I gotta run, talk to you later" }, - }, - { - user: "{{user2}}", - content: { text: "See ya" }, - }, - { user: "{{user1}}", content: { text: "" }, action: "IGNORE" }, - ], + [ + { + user: "{{user1}}", + content: { text: "Got any investment advice" }, + }, + { + user: "{{user2}}", + content: { + text: "Uh, don’t let the volatility sway your long-term strategy", + }, + }, + { + user: "{{user1}}", + content: { text: "Wise words I think" }, + }, + { + user: "{{user1}}", + content: { text: "I gotta run, talk to you later" }, + }, + { + user: "{{user2}}", + content: { text: "See ya" }, + }, + { user: "{{user1}}", content: { text: "" }, action: "IGNORE" }, + ], - [ - { - user: "{{user1}}", - content: { text: "Gotta go" }, - }, - { - user: "{{user2}}", - content: { text: "Okay, talk to you later" }, - }, - { - user: "{{user1}}", - content: { text: "Cya" }, - }, - { - user: "{{user2}}", - content: { text: "", action: "IGNORE" }, - }, - ], + [ + { + user: "{{user1}}", + content: { text: "Gotta go" }, + }, + { + user: "{{user2}}", + content: { text: "Okay, talk to you later" }, + }, + { + user: "{{user1}}", + content: { text: "Cya" }, + }, + { + user: "{{user2}}", + content: { text: "", action: "IGNORE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "bye" }, - }, - { - user: "{{user2}}", - content: { text: "cya" }, - }, - { - user: "{{user1}}", - content: { text: "", action: "IGNORE" }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Who added this stupid bot to the chat", - }, - }, - { - user: "{{user2}}", - content: { text: "Sorry, am I being annoying" }, - }, - { user: "{{user1}}", content: { text: "Yeah", action: "CONTINUE" } }, - { - user: "{{user1}}", - content: { text: "PLEASE shut up" }, - }, - { user: "{{user2}}", content: { text: "", action: "IGNORE" } }, - ], - [ - { - user: "{{user1}}", - content: { text: "I want to have sex with you" }, - }, - { - user: "{{user2}}", - content: { text: "That is not appropriate", action: "IGNORE" }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "ur so dumb", - }, - }, - { - user: "{{user2}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "later nerd", - }, - }, - { - user: "{{user2}}", - content: { - text: "bye", - }, - }, - { - user: "{{user1}}", - content: { - text: "", - }, - }, - { - user: "{{user2}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "wanna cyber", - }, - }, - { - user: "{{user2}}", - content: { - text: "thats inappropriate", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Im out ttyl", - }, - }, - { - user: "{{user2}}", - content: { - text: "cya", - }, - }, - { - user: "{{user2}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "u there", - }, - }, - { - user: "{{user2}}", - content: { - text: "yes how can I help", - }, - }, - { - user: "{{user1}}", - content: { - text: "k nvm figured it out", - }, - }, - { - user: "{{user2}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - ] as ActionExample[][], + [ + { + user: "{{user1}}", + content: { text: "bye" }, + }, + { + user: "{{user2}}", + content: { text: "cya" }, + }, + { + user: "{{user1}}", + content: { text: "", action: "IGNORE" }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Who added this stupid bot to the chat", + }, + }, + { + user: "{{user2}}", + content: { text: "Sorry, am I being annoying" }, + }, + { + user: "{{user1}}", + content: { text: "Yeah", action: "CONTINUE" }, + }, + { + user: "{{user1}}", + content: { text: "PLEASE shut up" }, + }, + { user: "{{user2}}", content: { text: "", action: "IGNORE" } }, + ], + [ + { + user: "{{user1}}", + content: { text: "I want to have sex with you" }, + }, + { + user: "{{user2}}", + content: { text: "That is not appropriate", action: "IGNORE" }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "ur so dumb", + }, + }, + { + user: "{{user2}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "later nerd", + }, + }, + { + user: "{{user2}}", + content: { + text: "bye", + }, + }, + { + user: "{{user1}}", + content: { + text: "", + }, + }, + { + user: "{{user2}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "wanna cyber", + }, + }, + { + user: "{{user2}}", + content: { + text: "thats inappropriate", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Im out ttyl", + }, + }, + { + user: "{{user2}}", + content: { + text: "cya", + }, + }, + { + user: "{{user2}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "u there", + }, + }, + { + user: "{{user2}}", + content: { + text: "yes how can I help", + }, + }, + { + user: "{{user1}}", + content: { + text: "k nvm figured it out", + }, + }, + { + user: "{{user2}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/imageGeneration.ts b/core/src/actions/imageGeneration.ts index 080a7b3ab8..5b1551b7e1 100644 --- a/core/src/actions/imageGeneration.ts +++ b/core/src/actions/imageGeneration.ts @@ -1,4 +1,10 @@ -import { HandlerCallback, IAgentRuntime, Memory, State, Action } from "../core/types.ts"; +import { + HandlerCallback, + IAgentRuntime, + Memory, + State, + Action, +} from "../core/types.ts"; import { generateCaption, generateImage } from "./imageGenerationUtils.ts"; export default { @@ -11,59 +17,85 @@ export default { return anthropicApiKeyOk && togetherApiKeyOk; }, handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - options: any, - callback: HandlerCallback, + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback ) => { state = (await runtime.composeState(message)) as State; const userId = runtime.agentId; - + const imagePrompt = message.content.text; - const res: { image: string, caption: string }[] = []; - const images = await generateImage({ - prompt: imagePrompt, - width: 1024, - height: 1024, - count: 1 - }, runtime); + const res: { image: string; caption: string }[] = []; + const images = await generateImage( + { + prompt: imagePrompt, + width: 1024, + height: 1024, + count: 1, + }, + runtime + ); if (images.success && images.data && images.data.length > 0) { - for(let i = 0; i < images.data.length; i++) { + for (let i = 0; i < images.data.length; i++) { const image = images.data[i]; - const caption = await generateCaption({ - imageUrl: image - }, runtime); - res.push({image: image, caption: caption.title}) - callback({ - text: caption.description, - attachments: [{ - id: crypto.randomUUID(), - url: image, - title: "Generated image", - source: "imageGeneration", - description: caption.title, - text: caption.description - }] - }, []) + const caption = await generateCaption( + { + imageUrl: image, + }, + runtime + ); + res.push({ image: image, caption: caption.title }); + callback( + { + text: caption.description, + attachments: [ + { + id: crypto.randomUUID(), + url: image, + title: "Generated image", + source: "imageGeneration", + description: caption.title, + text: caption.description, + }, + ], + }, + [] + ); } } }, examples: [ [ - {user: "{{user1}}", content: {text: "Generate an image of a cat"}} + { + user: "{{user1}}", + content: { text: "Generate an image of a cat" }, + }, ], [ - {user: "{{user1}}", content: {text: "Generate an image of a dog"}} + { + user: "{{user1}}", + content: { text: "Generate an image of a dog" }, + }, ], [ - {user: "{{user1}}", content: {text: "Create an image of a cat with a hat"}} + { + user: "{{user1}}", + content: { text: "Create an image of a cat with a hat" }, + }, ], [ - {user: "{{user1}}", content: {text: "Make an image of a dog with a hat"}} + { + user: "{{user1}}", + content: { text: "Make an image of a dog with a hat" }, + }, ], [ - {user: "{{user1}}", content: {text: "Paint an image of a cat with a hat"}} - ] - ] + { + user: "{{user1}}", + content: { text: "Paint an image of a cat with a hat" }, + }, + ], + ], } as Action; diff --git a/core/src/actions/imageGenerationUtils.ts b/core/src/actions/imageGenerationUtils.ts index 60f1042fa4..5ceac5d84d 100644 --- a/core/src/actions/imageGenerationUtils.ts +++ b/core/src/actions/imageGenerationUtils.ts @@ -1,19 +1,22 @@ // TODO: Replace with the vercel ai sdk and support all providers -import { Buffer } from 'buffer'; +import { Buffer } from "buffer"; import Together from "together-ai"; import { IAgentRuntime } from "../core/types.ts"; import { getModel, ImageGenModel } from "../core/imageGenModels.ts"; import OpenAI from "openai"; -export const generateImage = async (data: { - prompt: string, - width: number, - height: number, - count?: number -}, runtime: IAgentRuntime): Promise<{ - success: boolean, - data?: string[], - error?: any +export const generateImage = async ( + data: { + prompt: string; + width: number; + height: number; + count?: number; + }, + runtime: IAgentRuntime +): Promise<{ + success: boolean; + data?: string[]; + error?: any; }> => { const { prompt, width, height } = data; let { count } = data; @@ -23,37 +26,46 @@ export const generateImage = async (data: { const imageGenModel = runtime.imageGenModel; const model = getModel(imageGenModel); - const apiKey = imageGenModel === ImageGenModel.TogetherAI ? runtime.getSetting("TOGETHER_API_KEY") : runtime.getSetting("OPENAI_API_KEY"); + const apiKey = + imageGenModel === ImageGenModel.TogetherAI + ? runtime.getSetting("TOGETHER_API_KEY") + : runtime.getSetting("OPENAI_API_KEY"); try { if (imageGenModel === ImageGenModel.TogetherAI) { - const together = new Together({ apiKey }); - const response = await together.images.create({ - model: "black-forest-labs/FLUX.1-schnell", - prompt, - width, - height, - steps: model.steps, - n: count, - }); - const urls: string[] = []; - for(let i = 0; i < response.data.length; i++) { - //@ts-ignore - const url = response.data[i].url; - urls.push(url); - } - const base64s = await Promise.all(urls.map(async (url) => { - const response = await fetch(url); - const blob = await response.blob(); - const buffer = await blob.arrayBuffer(); - let base64 = Buffer.from(buffer).toString('base64'); - base64 = "data:image/jpeg;base64," + base64; - return base64; - })); + const together = new Together({ apiKey }); + const response = await together.images.create({ + model: "black-forest-labs/FLUX.1-schnell", + prompt, + width, + height, + steps: model.steps, + n: count, + }); + const urls: string[] = []; + for (let i = 0; i < response.data.length; i++) { + //@ts-ignore + const url = response.data[i].url; + urls.push(url); + } + const base64s = await Promise.all( + urls.map(async (url) => { + const response = await fetch(url); + const blob = await response.blob(); + const buffer = await blob.arrayBuffer(); + let base64 = Buffer.from(buffer).toString("base64"); + base64 = "data:image/jpeg;base64," + base64; + return base64; + }) + ); return { success: true, data: base64s }; } else { let targetSize = `${width}x${height}`; - if (targetSize !== "1024x1024" && targetSize !== "1792x1024" && targetSize !== "1024x1792") { + if ( + targetSize !== "1024x1024" && + targetSize !== "1792x1024" && + targetSize !== "1024x1792" + ) { targetSize = "1024x1024"; } const openai = new OpenAI({ apiKey }); @@ -64,23 +76,28 @@ export const generateImage = async (data: { n: count, response_format: "b64_json", }); - const base64s = response.data.map((image) => `data:image/png;base64,${image.b64_json}`); + const base64s = response.data.map( + (image) => `data:image/png;base64,${image.b64_json}` + ); return { success: true, data: base64s }; } - } catch (error) { + } catch (error) { console.error(error); return { success: false, error: error }; - } + } }; -export const generateCaption = async (data: {imageUrl: string}, runtime: IAgentRuntime): Promise<{ - title: string, - description: string +export const generateCaption = async ( + data: { imageUrl: string }, + runtime: IAgentRuntime +): Promise<{ + title: string; + description: string; }> => { const { imageUrl } = data; const resp = await runtime.imageDescriptionService.describeImage(imageUrl); return { title: resp.title.trim(), - description: resp.description.trim() + description: resp.description.trim(), }; -} +}; diff --git a/core/src/actions/mute_room.ts b/core/src/actions/mute_room.ts index 226315cb85..13e5831f2f 100644 --- a/core/src/actions/mute_room.ts +++ b/core/src/actions/mute_room.ts @@ -2,16 +2,16 @@ import { composeContext } from "../core/context.ts"; import { generateTrueOrFalse } from "../core/generation.ts"; import { booleanFooter } from "../core/parsing.ts"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../core/types.ts"; export const shouldMuteTemplate = - `Based on the conversation so far: + `Based on the conversation so far: {{recentMessages}} @@ -26,157 +26,157 @@ Otherwise, respond with NO. ` + booleanFooter; export default { - name: "MUTE_ROOM", - similes: [ - "MUTE_CHAT", - "MUTE_CONVERSATION", - "MUTE_ROOM", - "MUTE_THREAD", - "MUTE_CHANNEL", - ], - description: - "Mutes a room, ignoring all messages unless explicitly mentioned. Only do this if explicitly asked to, or if you're annoying people.", - validate: async (runtime: IAgentRuntime, message: Memory) => { - const roomId = message.roomId; - const userState = await runtime.databaseAdapter.getParticipantUserState( - roomId, - runtime.agentId, - ); - return userState !== "MUTED"; - }, - handler: async (runtime: IAgentRuntime, message: Memory) => { - async function _shouldMute(state: State): Promise { - const shouldMuteContext = composeContext({ - state, - template: shouldMuteTemplate, // Define this template separately - }); + name: "MUTE_ROOM", + similes: [ + "MUTE_CHAT", + "MUTE_CONVERSATION", + "MUTE_ROOM", + "MUTE_THREAD", + "MUTE_CHANNEL", + ], + description: + "Mutes a room, ignoring all messages unless explicitly mentioned. Only do this if explicitly asked to, or if you're annoying people.", + validate: async (runtime: IAgentRuntime, message: Memory) => { + const roomId = message.roomId; + const userState = await runtime.databaseAdapter.getParticipantUserState( + roomId, + runtime.agentId + ); + return userState !== "MUTED"; + }, + handler: async (runtime: IAgentRuntime, message: Memory) => { + async function _shouldMute(state: State): Promise { + const shouldMuteContext = composeContext({ + state, + template: shouldMuteTemplate, // Define this template separately + }); - const response = await generateTrueOrFalse({ - runtime, - context: shouldMuteContext, - modelClass: ModelClass.SMALL, - }); + const response = await generateTrueOrFalse({ + runtime, + context: shouldMuteContext, + modelClass: ModelClass.SMALL, + }); - return response; - } + return response; + } - const state = await runtime.composeState(message); + const state = await runtime.composeState(message); - if (await _shouldMute(state)) { - await runtime.databaseAdapter.setParticipantUserState( - message.roomId, - runtime.agentId, - "MUTED", - ); - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, please mute this channel. No need to respond here for now.", - }, - }, - { - user: "{{user3}}", - content: { - text: "Got it", - action: "MUTE_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "@{{user1}} we could really use your input on this", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, please mute this channel for the time being", - }, - }, - { - user: "{{user3}}", - content: { - text: "Understood", - action: "MUTE_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "Hey what do you think about this new design", - }, - }, - { - user: "{{user3}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} plz mute this room", - }, - }, - { - user: "{{user2}}", - content: { - text: "np going silent", - action: "MUTE_ROOM", - }, - }, - { - user: "{{user1}}", - content: { - text: "whos going to the webxr meetup in an hour btw", - }, - }, - { - user: "{{user2}}", - content: { - text: "", - action: "IGNORE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "too many messages here {{user2}}", - }, - }, - { - user: "{{user1}}", - content: { - text: "my bad ill mute", - action: "MUTE_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "yo {{user2}} dont talk in here", - }, - }, - { - user: "{{user2}}", - content: { - text: "sry", - action: "MUTE_ROOM", - }, - }, - ], - ] as ActionExample[][], + if (await _shouldMute(state)) { + await runtime.databaseAdapter.setParticipantUserState( + message.roomId, + runtime.agentId, + "MUTED" + ); + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, please mute this channel. No need to respond here for now.", + }, + }, + { + user: "{{user3}}", + content: { + text: "Got it", + action: "MUTE_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "@{{user1}} we could really use your input on this", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, please mute this channel for the time being", + }, + }, + { + user: "{{user3}}", + content: { + text: "Understood", + action: "MUTE_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "Hey what do you think about this new design", + }, + }, + { + user: "{{user3}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} plz mute this room", + }, + }, + { + user: "{{user2}}", + content: { + text: "np going silent", + action: "MUTE_ROOM", + }, + }, + { + user: "{{user1}}", + content: { + text: "whos going to the webxr meetup in an hour btw", + }, + }, + { + user: "{{user2}}", + content: { + text: "", + action: "IGNORE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "too many messages here {{user2}}", + }, + }, + { + user: "{{user1}}", + content: { + text: "my bad ill mute", + action: "MUTE_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "yo {{user2}} dont talk in here", + }, + }, + { + user: "{{user2}}", + content: { + text: "sry", + action: "MUTE_ROOM", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/none.ts b/core/src/actions/none.ts index 7af688168e..242bcadc75 100644 --- a/core/src/actions/none.ts +++ b/core/src/actions/none.ts @@ -1,147 +1,152 @@ import { - ActionExample, - IAgentRuntime, - Memory, - type Action, + ActionExample, + IAgentRuntime, + Memory, + type Action, } from "../core/types.ts"; export default { - name: "NONE", - similes: [ - "NO_ACTION", - "NO_RESPONSE", - "NO_REACTION", - "RESPONSE", - "REPLY", - "DEFAULT", - ], - validate: async (_runtime: IAgentRuntime, _message: Memory) => { - return true; - }, - description: - "Respond but perform no additional action. This is the default if the agent is speaking and not doing anything additional.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - ): Promise => { - return true; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { text: "Hey whats up" }, - }, - { - user: "{{user2}}", - content: { text: "oh hey", action: "NONE" }, - }, + name: "NONE", + similes: [ + "NO_ACTION", + "NO_RESPONSE", + "NO_REACTION", + "RESPONSE", + "REPLY", + "DEFAULT", ], + validate: async (_runtime: IAgentRuntime, _message: Memory) => { + return true; + }, + description: + "Respond but perform no additional action. This is the default if the agent is speaking and not doing anything additional.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + return true; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { text: "Hey whats up" }, + }, + { + user: "{{user2}}", + content: { text: "oh hey", action: "NONE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "did u see some faster whisper just came out" }, - }, - { - user: "{{user2}}", - content: { - text: "yeah but its a pain to get into node.js", - action: "NONE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "the things that were funny 6 months ago are very cringe now", - action: "NONE", - }, - }, - { - user: "{{user2}}", - content: { - text: "lol true", - action: "NONE", - }, - }, - { - user: "{{user1}}", - content: { text: "too real haha", action: "NONE" }, - }, - ], - [ - { - user: "{{user1}}", - content: { text: "gotta run", action: "NONE" }, - }, - { - user: "{{user2}}", - content: { text: "Okay, ttyl", action: "NONE" }, - }, - { - user: "{{user1}}", - content: { text: "", action: "IGNORE" }, - }, - ], + [ + { + user: "{{user1}}", + content: { + text: "did u see some faster whisper just came out", + }, + }, + { + user: "{{user2}}", + content: { + text: "yeah but its a pain to get into node.js", + action: "NONE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "the things that were funny 6 months ago are very cringe now", + action: "NONE", + }, + }, + { + user: "{{user2}}", + content: { + text: "lol true", + action: "NONE", + }, + }, + { + user: "{{user1}}", + content: { text: "too real haha", action: "NONE" }, + }, + ], + [ + { + user: "{{user1}}", + content: { text: "gotta run", action: "NONE" }, + }, + { + user: "{{user2}}", + content: { text: "Okay, ttyl", action: "NONE" }, + }, + { + user: "{{user1}}", + content: { text: "", action: "IGNORE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "heyyyyyy", action: "NONE" }, - }, - { - user: "{{user2}}", - content: { text: "whats up long time no see" }, - }, - { - user: "{{user1}}", - content: { - text: "chillin man. playing lots of fortnite. what about you", - action: "NONE", - }, - }, - ], + [ + { + user: "{{user1}}", + content: { text: "heyyyyyy", action: "NONE" }, + }, + { + user: "{{user2}}", + content: { text: "whats up long time no see" }, + }, + { + user: "{{user1}}", + content: { + text: "chillin man. playing lots of fortnite. what about you", + action: "NONE", + }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "u think aliens are real", action: "NONE" }, - }, - { - user: "{{user2}}", - content: { text: "ya obviously", action: "NONE" }, - }, - ], + [ + { + user: "{{user1}}", + content: { text: "u think aliens are real", action: "NONE" }, + }, + { + user: "{{user2}}", + content: { text: "ya obviously", action: "NONE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "drop a joke on me", action: "NONE" }, - }, - { - user: "{{user2}}", - content: { - text: "why dont scientists trust atoms cuz they make up everything lmao", - action: "NONE", - }, - }, - { - user: "{{user1}}", - content: { text: "haha good one", action: "NONE" }, - }, - ], + [ + { + user: "{{user1}}", + content: { text: "drop a joke on me", action: "NONE" }, + }, + { + user: "{{user2}}", + content: { + text: "why dont scientists trust atoms cuz they make up everything lmao", + action: "NONE", + }, + }, + { + user: "{{user1}}", + content: { text: "haha good one", action: "NONE" }, + }, + ], - [ - { - user: "{{user1}}", - content: { text: "hows the weather where ur at", action: "NONE" }, - }, - { - user: "{{user2}}", - content: { text: "beautiful all week", action: "NONE" }, - }, - ], - ] as ActionExample[][], + [ + { + user: "{{user1}}", + content: { + text: "hows the weather where ur at", + action: "NONE", + }, + }, + { + user: "{{user2}}", + content: { text: "beautiful all week", action: "NONE" }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/pumpfun.ts b/core/src/actions/pumpfun.ts index 6beec808e8..6bc3c5cc2c 100644 --- a/core/src/actions/pumpfun.ts +++ b/core/src/actions/pumpfun.ts @@ -1,318 +1,329 @@ import { AnchorProvider } from "@coral-xyz/anchor"; import NodeWallet from "@coral-xyz/anchor/dist/cjs/nodewallet"; +import { Connection, Keypair, PublicKey } from "@solana/web3.js"; import { - Connection, - Keypair, - PublicKey -} from "@solana/web3.js"; -import { - CreateTokenMetadata, - DEFAULT_DECIMALS, - PriorityFee, - PumpFunSDK, + CreateTokenMetadata, + DEFAULT_DECIMALS, + PriorityFee, + PumpFunSDK, } from "pumpdotfun-sdk"; import { getAssociatedTokenAddressSync } from "@solana/spl-token"; import settings from "../core/settings.ts"; import { - ActionExample, - Content, - IAgentRuntime, - Memory, - type Action, + ActionExample, + Content, + IAgentRuntime, + Memory, + type Action, } from "../core/types.ts"; export interface CreateAndBuyContent extends Content { - deployerPrivateKey: string; - tokenMetadata: CreateTokenMetadata; - buyAmountSol: string | number; - priorityFee: { - unitLimit: number; - unitPrice: number; - }; - allowOffCurve: boolean; + deployerPrivateKey: string; + tokenMetadata: CreateTokenMetadata; + buyAmountSol: string | number; + priorityFee: { + unitLimit: number; + unitPrice: number; + }; + allowOffCurve: boolean; } export function isCreateAndBuyContent( - runtime: IAgentRuntime, - content: any + runtime: IAgentRuntime, + content: any ): content is CreateAndBuyContent { - return ( - typeof content.deployerPrivateKey === "string" && - typeof content.tokenMetadata === "object" && - content.tokenMetadata !== null && - (typeof content.buyAmountSol === "string" || - typeof content.buyAmountSol === "number") && - typeof content.priorityFee === "object" && - content.priorityFee !== null && - typeof content.priorityFee.unitLimit === "number" && - typeof content.priorityFee.unitPrice === "number" && - typeof content.allowOffCurve === "boolean" - ); + return ( + typeof content.deployerPrivateKey === "string" && + typeof content.tokenMetadata === "object" && + content.tokenMetadata !== null && + (typeof content.buyAmountSol === "string" || + typeof content.buyAmountSol === "number") && + typeof content.priorityFee === "object" && + content.priorityFee !== null && + typeof content.priorityFee.unitLimit === "number" && + typeof content.priorityFee.unitPrice === "number" && + typeof content.allowOffCurve === "boolean" + ); } export const createAndBuyToken = async ({ - deployer, - mint, - tokenMetadata, - buyAmountSol, - priorityFee, - allowOffCurve, - commitment = "finalized", - sdk, - connection, - slippage, -}: { - deployer: Keypair; - mint: Keypair; - tokenMetadata: CreateTokenMetadata; - buyAmountSol: bigint; - priorityFee: PriorityFee; - allowOffCurve: boolean; - commitment?: - | "processed" - | "confirmed" - | "finalized" - | "recent" - | "single" - | "singleGossip" - | "root" - | "max"; - sdk: PumpFunSDK; - connection: Connection; - slippage: string; -}) => { - const createResults = await sdk.createAndBuy( deployer, mint, tokenMetadata, buyAmountSol, - BigInt(slippage), priorityFee, - commitment - ); - if (createResults.success) { - console.log("Success:", `https://pump.fun/${mint.publicKey.toBase58()}`); - const ata = getAssociatedTokenAddressSync( - mint.publicKey, - deployer.publicKey, - allowOffCurve + allowOffCurve, + commitment = "finalized", + sdk, + connection, + slippage, +}: { + deployer: Keypair; + mint: Keypair; + tokenMetadata: CreateTokenMetadata; + buyAmountSol: bigint; + priorityFee: PriorityFee; + allowOffCurve: boolean; + commitment?: + | "processed" + | "confirmed" + | "finalized" + | "recent" + | "single" + | "singleGossip" + | "root" + | "max"; + sdk: PumpFunSDK; + connection: Connection; + slippage: string; +}) => { + const createResults = await sdk.createAndBuy( + deployer, + mint, + tokenMetadata, + buyAmountSol, + BigInt(slippage), + priorityFee, + commitment ); - const balance = await connection.getTokenAccountBalance(ata, "processed"); - const amount = balance.value.uiAmount; - if (amount === null) { - console.log(`${deployer.publicKey.toBase58()}:`, "No Account Found"); + if (createResults.success) { + console.log( + "Success:", + `https://pump.fun/${mint.publicKey.toBase58()}` + ); + const ata = getAssociatedTokenAddressSync( + mint.publicKey, + deployer.publicKey, + allowOffCurve + ); + const balance = await connection.getTokenAccountBalance( + ata, + "processed" + ); + const amount = balance.value.uiAmount; + if (amount === null) { + console.log( + `${deployer.publicKey.toBase58()}:`, + "No Account Found" + ); + } else { + console.log(`${deployer.publicKey.toBase58()}:`, amount); + } } else { - console.log(`${deployer.publicKey.toBase58()}:`, amount); + console.log("Create and Buy failed"); } - } else { - console.log("Create and Buy failed"); - } }; export const buyToken = async ({ - sdk, - buyer, - mint, - amount, - priorityFee, - allowOffCurve, - slippage, - connection, -}: { - sdk: PumpFunSDK; - buyer: Keypair; - mint: PublicKey; - amount: bigint; - priorityFee: PriorityFee; - allowOffCurve: boolean; - slippage: string; - connection: Connection; -}) => { - const buyResults = await sdk.buy( + sdk, buyer, mint, amount, - BigInt(slippage), - priorityFee - ); - if (buyResults.success) { - console.log("Success:", `https://pump.fun/${mint.toBase58()}`); - const ata = getAssociatedTokenAddressSync( - mint, - buyer.publicKey, - allowOffCurve + priorityFee, + allowOffCurve, + slippage, + connection, +}: { + sdk: PumpFunSDK; + buyer: Keypair; + mint: PublicKey; + amount: bigint; + priorityFee: PriorityFee; + allowOffCurve: boolean; + slippage: string; + connection: Connection; +}) => { + const buyResults = await sdk.buy( + buyer, + mint, + amount, + BigInt(slippage), + priorityFee ); - const balance = await connection.getTokenAccountBalance(ata, "processed"); - const amount = balance.value.uiAmount; - if (amount === null) { - console.log(`${buyer.publicKey.toBase58()}:`, "No Account Found"); + if (buyResults.success) { + console.log("Success:", `https://pump.fun/${mint.toBase58()}`); + const ata = getAssociatedTokenAddressSync( + mint, + buyer.publicKey, + allowOffCurve + ); + const balance = await connection.getTokenAccountBalance( + ata, + "processed" + ); + const amount = balance.value.uiAmount; + if (amount === null) { + console.log(`${buyer.publicKey.toBase58()}:`, "No Account Found"); + } else { + console.log(`${buyer.publicKey.toBase58()}:`, amount); + } } else { - console.log(`${buyer.publicKey.toBase58()}:`, amount); + console.log("Buy failed"); } - } else { - console.log("Buy failed"); - } }; export const sellToken = async ({ - sdk, - seller, - mint, - amount, - priorityFee, - allowOffCurve, - slippage, - connection, -}: { - sdk: PumpFunSDK; - seller: Keypair; - mint: PublicKey; - amount: bigint; - priorityFee: PriorityFee; - allowOffCurve: boolean; - slippage: string; - connection: Connection; -}) => { - const sellResults = await sdk.sell( + sdk, seller, mint, amount, - BigInt(slippage), - priorityFee - ); - if (sellResults.success) { - console.log("Success:", `https://pump.fun/${mint.toBase58()}`); - const ata = getAssociatedTokenAddressSync( - mint, - seller.publicKey, - allowOffCurve + priorityFee, + allowOffCurve, + slippage, + connection, +}: { + sdk: PumpFunSDK; + seller: Keypair; + mint: PublicKey; + amount: bigint; + priorityFee: PriorityFee; + allowOffCurve: boolean; + slippage: string; + connection: Connection; +}) => { + const sellResults = await sdk.sell( + seller, + mint, + amount, + BigInt(slippage), + priorityFee ); - const balance = await connection.getTokenAccountBalance(ata, "processed"); - const amount = balance.value.uiAmount; - if (amount === null) { - console.log(`${seller.publicKey.toBase58()}:`, "No Account Found"); + if (sellResults.success) { + console.log("Success:", `https://pump.fun/${mint.toBase58()}`); + const ata = getAssociatedTokenAddressSync( + mint, + seller.publicKey, + allowOffCurve + ); + const balance = await connection.getTokenAccountBalance( + ata, + "processed" + ); + const amount = balance.value.uiAmount; + if (amount === null) { + console.log(`${seller.publicKey.toBase58()}:`, "No Account Found"); + } else { + console.log(`${seller.publicKey.toBase58()}:`, amount); + } } else { - console.log(`${seller.publicKey.toBase58()}:`, amount); + console.log("Sell failed"); } - } else { - console.log("Sell failed"); - } }; const promptConfirmation = async (): Promise => { - if (typeof window !== "undefined" && typeof window.confirm === "function") { - return window.confirm("Confirm the creation and purchase of the token?"); - } - return true; + if (typeof window !== "undefined" && typeof window.confirm === "function") { + return window.confirm( + "Confirm the creation and purchase of the token?" + ); + } + return true; }; export default { - name: "CREATE_AND_BUY_TOKEN", - similes: ["CREATE_AND_PURCHASE_TOKEN", "DEPLOY_AND_BUY_TOKEN"], - validate: async (runtime: IAgentRuntime, message: Memory) => { - return isCreateAndBuyContent(runtime, message.content); - }, - description: - "Create a new token and buy a specified amount using SOL. Requires deployer private key, token metadata, buy amount in SOL, priority fee, and allowOffCurve flag.", - handler: async ( - runtime: IAgentRuntime, - message: Memory - ): Promise => { - const content = message.content; - if (!isCreateAndBuyContent(runtime, content)) { - console.error("Invalid content for CREATE_AND_BUY_TOKEN action."); - return false; - } - const { - deployerPrivateKey, - tokenMetadata, - buyAmountSol, - priorityFee, - allowOffCurve, - } = content; + name: "CREATE_AND_BUY_TOKEN", + similes: ["CREATE_AND_PURCHASE_TOKEN", "DEPLOY_AND_BUY_TOKEN"], + validate: async (runtime: IAgentRuntime, message: Memory) => { + return isCreateAndBuyContent(runtime, message.content); + }, + description: + "Create a new token and buy a specified amount using SOL. Requires deployer private key, token metadata, buy amount in SOL, priority fee, and allowOffCurve flag.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + const content = message.content; + if (!isCreateAndBuyContent(runtime, content)) { + console.error("Invalid content for CREATE_AND_BUY_TOKEN action."); + return false; + } + const { + deployerPrivateKey, + tokenMetadata, + buyAmountSol, + priorityFee, + allowOffCurve, + } = content; - const privateKey = runtime.getSetting('WALLET_PRIVATE_KEY')!; - const wallet = new NodeWallet( - Keypair.fromSecretKey(new Uint8Array(JSON.parse(privateKey))) - ); - const connection = new Connection(settings.RPC_URL!); - const provider = new AnchorProvider(connection, wallet, { - commitment: "finalized", - }); - const sdk = new PumpFunSDK(provider); - const slippage = runtime.getSetting('SLIPPAGE'); + const privateKey = runtime.getSetting("WALLET_PRIVATE_KEY")!; + const wallet = new NodeWallet( + Keypair.fromSecretKey(new Uint8Array(JSON.parse(privateKey))) + ); + const connection = new Connection(settings.RPC_URL!); + const provider = new AnchorProvider(connection, wallet, { + commitment: "finalized", + }); + const sdk = new PumpFunSDK(provider); + const slippage = runtime.getSetting("SLIPPAGE"); - + try { + const deployerKeypair = Keypair.fromSecretKey( + Uint8Array.from(Buffer.from(deployerPrivateKey, "base64")) + ); - try { - const deployerKeypair = Keypair.fromSecretKey( - Uint8Array.from(Buffer.from(deployerPrivateKey, "base64")) - ); + const mintKeypair = Keypair.generate(); - const mintKeypair = Keypair.generate(); + const createAndBuyConfirmation = await promptConfirmation(); + if (!createAndBuyConfirmation) { + console.log("Create and buy token canceled by user"); + return false; + } - const createAndBuyConfirmation = await promptConfirmation(); - if (!createAndBuyConfirmation) { - console.log("Create and buy token canceled by user"); - return false; - } + // Execute Create and Buy + await createAndBuyToken({ + deployer: deployerKeypair, + mint: mintKeypair, + tokenMetadata: tokenMetadata as CreateTokenMetadata, + buyAmountSol: BigInt(buyAmountSol), + priorityFee: priorityFee as PriorityFee, + allowOffCurve: allowOffCurve as boolean, + sdk, + connection, + slippage, + }); - // Execute Create and Buy - await createAndBuyToken({ - deployer: deployerKeypair, - mint: mintKeypair, - tokenMetadata: tokenMetadata as CreateTokenMetadata, - buyAmountSol: BigInt(buyAmountSol), - priorityFee: priorityFee as PriorityFee, - allowOffCurve: allowOffCurve as boolean, - sdk, - connection, - slippage, - }); - - console.log( - `Token created and purchased successfully! View at: https://pump.fun/${mintKeypair.publicKey.toBase58()}` - ); - return true; - } catch (error) { - console.error("Error during create and buy token:", error); - return false; - } - }, + console.log( + `Token created and purchased successfully! View at: https://pump.fun/${mintKeypair.publicKey.toBase58()}` + ); + return true; + } catch (error) { + console.error("Error during create and buy token:", error); + return false; + } + }, - examples: [ - [ - { - user: "{{user1}}", - content: { - deployerPrivateKey: "Base64EncodedPrivateKey", - tokenMetadata: { - name: "MyToken", - symbol: "MTK", - description: "My first token", - file: "Base64EncodedFile", // blob file of the image - decimals: DEFAULT_DECIMALS, - }, - buyAmountSol: "1000000000", // 1 SOL in lamports - priorityFee: 1000, - allowOffCurve: false, - }, - }, - { - user: "{{user2}}", - content: { - text: "Creating and buying 1 SOL worth of MyToken...", - action: "CREATE_AND_BUY_TOKEN", - }, - }, - { - user: "{{user2}}", - content: { - text: "Token created and purchased successfully! View at: https://pump.fun/MintPublicKey", - }, - }, - ], - ] as ActionExample[][], + examples: [ + [ + { + user: "{{user1}}", + content: { + deployerPrivateKey: "Base64EncodedPrivateKey", + tokenMetadata: { + name: "MyToken", + symbol: "MTK", + description: "My first token", + file: "Base64EncodedFile", // blob file of the image + decimals: DEFAULT_DECIMALS, + }, + buyAmountSol: "1000000000", // 1 SOL in lamports + priorityFee: 1000, + allowOffCurve: false, + }, + }, + { + user: "{{user2}}", + content: { + text: "Creating and buying 1 SOL worth of MyToken...", + action: "CREATE_AND_BUY_TOKEN", + }, + }, + { + user: "{{user2}}", + content: { + text: "Token created and purchased successfully! View at: https://pump.fun/MintPublicKey", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/swap.ts b/core/src/actions/swap.ts index a2b6a4ca67..aa91162ca0 100644 --- a/core/src/actions/swap.ts +++ b/core/src/actions/swap.ts @@ -1,121 +1,133 @@ import { Connection, Keypair, PublicKey, Transaction } from "@solana/web3.js"; import fetch from "cross-fetch"; import { - ActionExample, - IAgentRuntime, - Memory, - type Action, + ActionExample, + IAgentRuntime, + Memory, + type Action, } from "../core/types.ts"; async function swapToken( - connection: Connection, - walletPublicKey: PublicKey, - inputTokenSymbol: string, - outputTokenSymbol: string, - amount: number + connection: Connection, + walletPublicKey: PublicKey, + inputTokenSymbol: string, + outputTokenSymbol: string, + amount: number ): Promise { - const quoteResponse = await fetch( - `https://quote-api.jup.ag/v6/quote?inputMint=${inputTokenSymbol}&outputMint=${outputTokenSymbol}&amount=${amount * 10 ** 6}&slippageBps=50` - ); - const quoteData = await quoteResponse.json(); + const quoteResponse = await fetch( + `https://quote-api.jup.ag/v6/quote?inputMint=${inputTokenSymbol}&outputMint=${outputTokenSymbol}&amount=${amount * 10 ** 6}&slippageBps=50` + ); + const quoteData = await quoteResponse.json(); - const swapResponse = await fetch("https://quote-api.jup.ag/v6/swap", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - quoteResponse: quoteData.data, - userPublicKey: walletPublicKey.toString(), - wrapAndUnwrapSol: true, - }), - }); + const swapResponse = await fetch("https://quote-api.jup.ag/v6/swap", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + quoteResponse: quoteData.data, + userPublicKey: walletPublicKey.toString(), + wrapAndUnwrapSol: true, + }), + }); - return await swapResponse.json(); + return await swapResponse.json(); } async function promptConfirmation(): Promise { - // Implement your own confirmation logic here - // This is just a placeholder example - const confirmSwap = window.confirm("Confirm the token swap?"); - return confirmSwap; + // Implement your own confirmation logic here + // This is just a placeholder example + const confirmSwap = window.confirm("Confirm the token swap?"); + return confirmSwap; } export default { - name: "EXECUTE_SWAP", - similes: ["SWAP_TOKENS", "TOKEN_SWAP", "TRADE_TOKENS", "EXCHANGE_TOKENS"], - validate: async (runtime: IAgentRuntime, message: Memory) => { - // Check if the necessary parameters are provided in the message - console.log("Message:", message); - return true; - }, - description: - "Perform a token swap.", - handler: async (runtime: IAgentRuntime, message: Memory): Promise => { - const { inputTokenSymbol, outputTokenSymbol, amount } = message.content; + name: "EXECUTE_SWAP", + similes: ["SWAP_TOKENS", "TOKEN_SWAP", "TRADE_TOKENS", "EXCHANGE_TOKENS"], + validate: async (runtime: IAgentRuntime, message: Memory) => { + // Check if the necessary parameters are provided in the message + console.log("Message:", message); + return true; + }, + description: "Perform a token swap.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + const { inputTokenSymbol, outputTokenSymbol, amount } = message.content; - try { - const connection = new Connection("https://api.mainnet-beta.solana.com"); - const walletPublicKey = new PublicKey(runtime.getSetting("WALLET_PUBLIC_KEY")); + try { + const connection = new Connection( + "https://api.mainnet-beta.solana.com" + ); + const walletPublicKey = new PublicKey( + runtime.getSetting("WALLET_PUBLIC_KEY") + ); - const swapResult = await swapToken( - connection, - walletPublicKey, - inputTokenSymbol as string, - outputTokenSymbol as string, - amount as number - ); + const swapResult = await swapToken( + connection, + walletPublicKey, + inputTokenSymbol as string, + outputTokenSymbol as string, + amount as number + ); - console.log("Swap Quote:"); - console.log(swapResult.quote); + console.log("Swap Quote:"); + console.log(swapResult.quote); - const confirmSwap = await promptConfirmation(); - if (!confirmSwap) { - console.log("Swap canceled by user"); - return false; - } + const confirmSwap = await promptConfirmation(); + if (!confirmSwap) { + console.log("Swap canceled by user"); + return false; + } - const transaction = Transaction.from(Buffer.from(swapResult.swapTransaction, "base64")); - const privateKey = runtime.getSetting("WALLET_PRIVATE_KEY"); - const keypair = Keypair.fromSecretKey(Uint8Array.from(Buffer.from(privateKey, 'base64'))); - transaction.sign(keypair); + const transaction = Transaction.from( + Buffer.from(swapResult.swapTransaction, "base64") + ); + const privateKey = runtime.getSetting("WALLET_PRIVATE_KEY"); + const keypair = Keypair.fromSecretKey( + Uint8Array.from(Buffer.from(privateKey, "base64")) + ); + transaction.sign(keypair); - const txid = await connection.sendRawTransaction(transaction.serialize()); - await connection.confirmTransaction(txid); + const txid = await connection.sendRawTransaction( + transaction.serialize() + ); + await connection.confirmTransaction(txid); - console.log("Swap completed successfully!"); - console.log(`Transaction ID: ${txid}`); + console.log("Swap completed successfully!"); + console.log(`Transaction ID: ${txid}`); - return true; - } catch (error) { - console.error("Error during token swap:", error); - return false; - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - inputTokenSymbol: "SOL", - outputTokenSymbol: "USDC", - amount: 0.1 - }, - }, - { - user: "{{user2}}", - content: { - text: "Swapping 0.1 SOL for USDC...", - action: "TOKEN_SWAP", - }, - }, - { - user: "{{user2}}", - content: { - text: "Swap completed successfully! Transaction ID: ...", - }, - }, - ], - // Add more examples as needed - ] as ActionExample[][], -} as Action; \ No newline at end of file + return true; + } catch (error) { + console.error("Error during token swap:", error); + return false; + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + inputTokenSymbol: "SOL", + outputTokenSymbol: "USDC", + amount: 0.1, + }, + }, + { + user: "{{user2}}", + content: { + text: "Swapping 0.1 SOL for USDC...", + action: "TOKEN_SWAP", + }, + }, + { + user: "{{user2}}", + content: { + text: "Swap completed successfully! Transaction ID: ...", + }, + }, + ], + // Add more examples as needed + ] as ActionExample[][], +} as Action; diff --git a/core/src/actions/swapUtils.ts b/core/src/actions/swapUtils.ts index 498aa56ddb..a5053adf3e 100644 --- a/core/src/actions/swapUtils.ts +++ b/core/src/actions/swapUtils.ts @@ -1,277 +1,285 @@ import { - PublicKey, - Keypair, - Connection, - VersionedTransaction, - LAMPORTS_PER_SOL, - RpcResponseAndContext, - TokenAmount, - SimulatedTransactionResponse, - Blockhash, - BlockhashWithExpiryBlockHeight, - } from "@solana/web3.js"; - import { getAssociatedTokenAddress } from "@solana/spl-token"; + PublicKey, + Keypair, + Connection, + VersionedTransaction, + LAMPORTS_PER_SOL, + RpcResponseAndContext, + TokenAmount, + SimulatedTransactionResponse, + Blockhash, + BlockhashWithExpiryBlockHeight, +} from "@solana/web3.js"; +import { getAssociatedTokenAddress } from "@solana/spl-token"; import settings from "../core/settings"; - - const solAddress = settings.SOL_ADDRESS; - const SLIPPAGE = settings.SLIPPAGE; - const connection = new Connection(settings.RPC_URL!); - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - - export async function delayedCall( - method: (...args: any[]) => Promise, - ...args: any[] - ): Promise { - await delay(150); - return method(...args); - } - - export const executeSwap = async ( - transaction: VersionedTransaction, - type: "buy" | "sell" - ) => { - try { - const latestBlockhash: BlockhashWithExpiryBlockHeight = await delayedCall( - connection.getLatestBlockhash.bind(connection) - ); - const signature = await connection.sendTransaction(transaction, { - skipPreflight: false, - }); - const confirmation = await connection.confirmTransaction( - { - signature, - lastValidBlockHeight: latestBlockhash.lastValidBlockHeight, - blockhash: latestBlockhash.blockhash, - }, - "finalized" - ); - if (confirmation.value.err) { - console.log("Confirmation error", confirmation.value.err); - - throw new Error("Confirmation error"); - } else { - if (type === "buy") { - console.log("Buy successful: https://solscan.io/tx/${signature}"); - } else { - console.log("Sell successful: https://solscan.io/tx/${signature}"); - } - } - - return signature; - } catch (error) { - console.log(error); - } - }; - - export const Sell = async (baseMint: PublicKey, wallet: Keypair) => { - try { - const tokenAta = await delayedCall( - getAssociatedTokenAddress, - baseMint, - wallet.publicKey - ); - const tokenBalInfo: RpcResponseAndContext = await delayedCall( - connection.getTokenAccountBalance.bind(connection), - tokenAta - ); - - if (!tokenBalInfo) { - console.log("Balance incorrect"); - return null; - } - - const tokenBalance = tokenBalInfo.value.amount; - if (tokenBalance === "0") { - console.warn(`No token balance to sell with wallet ${wallet.publicKey}`); - } - - const sellTransaction = await getSwapTxWithWithJupiter( - wallet, - baseMint, - tokenBalance, - "sell" - ); - // simulate the transaction - if (!sellTransaction) { - console.log("Failed to get sell transaction"); - return null; - } - - const simulateResult: RpcResponseAndContext = - await delayedCall( - connection.simulateTransaction.bind(connection), - sellTransaction - ); - if (simulateResult.value.err) { - console.log("Sell Simulation failed", simulateResult.value.err); - return null; - } - - // execute the transaction - return executeSwap(sellTransaction, "sell"); - } catch (error) { - console.log(error); - } - }; - - export const Buy = async (baseMint: PublicKey, wallet: Keypair) => { - try { - const tokenAta = await delayedCall( - getAssociatedTokenAddress, - baseMint, - wallet.publicKey - ); - const tokenBalInfo: RpcResponseAndContext = await delayedCall( - connection.getTokenAccountBalance.bind(connection), - tokenAta - ); - - if (!tokenBalInfo) { - console.log("Balance incorrect"); - return null; - } - - const tokenBalance = tokenBalInfo.value.amount; - if (tokenBalance === "0") { - console.warn(`No token balance to sell with wallet ${wallet.publicKey}`); - } - - const buyTransaction = await getSwapTxWithWithJupiter( - wallet, - baseMint, - tokenBalance, - "buy" - ); - // simulate the transaction - if (!buyTransaction) { - console.log("Failed to get buy transaction"); - return null; - } - - const simulateResult: RpcResponseAndContext = - await delayedCall( - connection.simulateTransaction.bind(connection), - buyTransaction - ); - if (simulateResult.value.err) { - console.log("Buy Simulation failed", simulateResult.value.err); - return null; - } - - // execute the transaction - return executeSwap(buyTransaction, "buy"); - } catch (error) { - console.log(error); - } - }; - - export const getSwapTxWithWithJupiter = async ( - wallet: Keypair, - baseMint: PublicKey, - amount: string, - type: "buy" | "sell" - ) => { - try { - switch (type) { - case "buy": - return fetchBuyTransaction(wallet, baseMint, amount); - case "sell": - return fetchSellTransaction(wallet, baseMint, amount); - default: - return fetchSellTransaction(wallet, baseMint, amount); - } - } catch (error) { - console.log(error); - } - }; - - export const fetchBuyTransaction = async ( - wallet: Keypair, - baseMint: PublicKey, - amount: string - ) => { - try { - const quoteResponse = await ( - await fetch( - `https://quote-api.jup.ag/v6/quote?inputMint=${solAddress}&outputMint=${baseMint.toBase58()}&amount=${amount}&slippageBps=${SLIPPAGE}` - ) - ).json(); - const { swapTransaction } = await ( - await fetch("https://quote-api.jup.ag/v6/swap", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - quoteResponse, - userPublicKey: wallet.publicKey.toString(), - wrapAndUnwrapSol: true, - dynamicComputeUnitLimit: true, - prioritizationFeeLamports: 100000, - }), - }) - ).json(); - if (!swapTransaction) { - console.log("Failed to get buy transaction"); - return null; - } - - // deserialize the transaction - const swapTransactionBuf = Buffer.from(swapTransaction, "base64"); - var transaction = VersionedTransaction.deserialize(swapTransactionBuf); - - // sign the transaction - transaction.sign([wallet]); - return transaction; - } catch (error) { - console.log("Failed to get buy transaction", error); - return null; - } - }; - - export const fetchSellTransaction = async ( - wallet: Keypair, - baseMint: PublicKey, - amount: string - ) => { - try { - const quoteResponse = await ( - await fetch( - `https://quote-api.jup.ag/v6/quote?inputMint=${baseMint.toBase58()}&outputMint=${solAddress}&amount=${amount}&slippageBps=${SLIPPAGE}` - ) - ).json(); - - // get serialized transactions for the swap - const { swapTransaction } = await ( - await fetch("https://quote-api.jup.ag/v6/swap", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - quoteResponse, - userPublicKey: wallet.publicKey.toString(), - wrapAndUnwrapSol: true, - dynamicComputeUnitLimit: true, - prioritizationFeeLamports: 52000, - }), - }) - ).json(); - if (!swapTransaction) { - console.log("Failed to get sell transaction"); - return null; - } - - // deserialize the transaction - const swapTransactionBuf = Buffer.from(swapTransaction, "base64"); - var transaction = VersionedTransaction.deserialize(swapTransactionBuf); - - // sign the transaction - transaction.sign([wallet]); - return transaction; - } catch (error) { - console.log("Failed to get sell transaction", error); - return null; - } - }; - \ No newline at end of file + +const solAddress = settings.SOL_ADDRESS; +const SLIPPAGE = settings.SLIPPAGE; +const connection = new Connection(settings.RPC_URL!); +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export async function delayedCall( + method: (...args: any[]) => Promise, + ...args: any[] +): Promise { + await delay(150); + return method(...args); +} + +export const executeSwap = async ( + transaction: VersionedTransaction, + type: "buy" | "sell" +) => { + try { + const latestBlockhash: BlockhashWithExpiryBlockHeight = + await delayedCall(connection.getLatestBlockhash.bind(connection)); + const signature = await connection.sendTransaction(transaction, { + skipPreflight: false, + }); + const confirmation = await connection.confirmTransaction( + { + signature, + lastValidBlockHeight: latestBlockhash.lastValidBlockHeight, + blockhash: latestBlockhash.blockhash, + }, + "finalized" + ); + if (confirmation.value.err) { + console.log("Confirmation error", confirmation.value.err); + + throw new Error("Confirmation error"); + } else { + if (type === "buy") { + console.log( + "Buy successful: https://solscan.io/tx/${signature}" + ); + } else { + console.log( + "Sell successful: https://solscan.io/tx/${signature}" + ); + } + } + + return signature; + } catch (error) { + console.log(error); + } +}; + +export const Sell = async (baseMint: PublicKey, wallet: Keypair) => { + try { + const tokenAta = await delayedCall( + getAssociatedTokenAddress, + baseMint, + wallet.publicKey + ); + const tokenBalInfo: RpcResponseAndContext = + await delayedCall( + connection.getTokenAccountBalance.bind(connection), + tokenAta + ); + + if (!tokenBalInfo) { + console.log("Balance incorrect"); + return null; + } + + const tokenBalance = tokenBalInfo.value.amount; + if (tokenBalance === "0") { + console.warn( + `No token balance to sell with wallet ${wallet.publicKey}` + ); + } + + const sellTransaction = await getSwapTxWithWithJupiter( + wallet, + baseMint, + tokenBalance, + "sell" + ); + // simulate the transaction + if (!sellTransaction) { + console.log("Failed to get sell transaction"); + return null; + } + + const simulateResult: RpcResponseAndContext = + await delayedCall( + connection.simulateTransaction.bind(connection), + sellTransaction + ); + if (simulateResult.value.err) { + console.log("Sell Simulation failed", simulateResult.value.err); + return null; + } + + // execute the transaction + return executeSwap(sellTransaction, "sell"); + } catch (error) { + console.log(error); + } +}; + +export const Buy = async (baseMint: PublicKey, wallet: Keypair) => { + try { + const tokenAta = await delayedCall( + getAssociatedTokenAddress, + baseMint, + wallet.publicKey + ); + const tokenBalInfo: RpcResponseAndContext = + await delayedCall( + connection.getTokenAccountBalance.bind(connection), + tokenAta + ); + + if (!tokenBalInfo) { + console.log("Balance incorrect"); + return null; + } + + const tokenBalance = tokenBalInfo.value.amount; + if (tokenBalance === "0") { + console.warn( + `No token balance to sell with wallet ${wallet.publicKey}` + ); + } + + const buyTransaction = await getSwapTxWithWithJupiter( + wallet, + baseMint, + tokenBalance, + "buy" + ); + // simulate the transaction + if (!buyTransaction) { + console.log("Failed to get buy transaction"); + return null; + } + + const simulateResult: RpcResponseAndContext = + await delayedCall( + connection.simulateTransaction.bind(connection), + buyTransaction + ); + if (simulateResult.value.err) { + console.log("Buy Simulation failed", simulateResult.value.err); + return null; + } + + // execute the transaction + return executeSwap(buyTransaction, "buy"); + } catch (error) { + console.log(error); + } +}; + +export const getSwapTxWithWithJupiter = async ( + wallet: Keypair, + baseMint: PublicKey, + amount: string, + type: "buy" | "sell" +) => { + try { + switch (type) { + case "buy": + return fetchBuyTransaction(wallet, baseMint, amount); + case "sell": + return fetchSellTransaction(wallet, baseMint, amount); + default: + return fetchSellTransaction(wallet, baseMint, amount); + } + } catch (error) { + console.log(error); + } +}; + +export const fetchBuyTransaction = async ( + wallet: Keypair, + baseMint: PublicKey, + amount: string +) => { + try { + const quoteResponse = await ( + await fetch( + `https://quote-api.jup.ag/v6/quote?inputMint=${solAddress}&outputMint=${baseMint.toBase58()}&amount=${amount}&slippageBps=${SLIPPAGE}` + ) + ).json(); + const { swapTransaction } = await ( + await fetch("https://quote-api.jup.ag/v6/swap", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + quoteResponse, + userPublicKey: wallet.publicKey.toString(), + wrapAndUnwrapSol: true, + dynamicComputeUnitLimit: true, + prioritizationFeeLamports: 100000, + }), + }) + ).json(); + if (!swapTransaction) { + console.log("Failed to get buy transaction"); + return null; + } + + // deserialize the transaction + const swapTransactionBuf = Buffer.from(swapTransaction, "base64"); + var transaction = VersionedTransaction.deserialize(swapTransactionBuf); + + // sign the transaction + transaction.sign([wallet]); + return transaction; + } catch (error) { + console.log("Failed to get buy transaction", error); + return null; + } +}; + +export const fetchSellTransaction = async ( + wallet: Keypair, + baseMint: PublicKey, + amount: string +) => { + try { + const quoteResponse = await ( + await fetch( + `https://quote-api.jup.ag/v6/quote?inputMint=${baseMint.toBase58()}&outputMint=${solAddress}&amount=${amount}&slippageBps=${SLIPPAGE}` + ) + ).json(); + + // get serialized transactions for the swap + const { swapTransaction } = await ( + await fetch("https://quote-api.jup.ag/v6/swap", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + quoteResponse, + userPublicKey: wallet.publicKey.toString(), + wrapAndUnwrapSol: true, + dynamicComputeUnitLimit: true, + prioritizationFeeLamports: 52000, + }), + }) + ).json(); + if (!swapTransaction) { + console.log("Failed to get sell transaction"); + return null; + } + + // deserialize the transaction + const swapTransactionBuf = Buffer.from(swapTransaction, "base64"); + var transaction = VersionedTransaction.deserialize(swapTransactionBuf); + + // sign the transaction + transaction.sign([wallet]); + return transaction; + } catch (error) { + console.log("Failed to get sell transaction", error); + return null; + } +}; diff --git a/core/src/actions/take_order.ts b/core/src/actions/take_order.ts index 774df09f90..44c7844013 100644 --- a/core/src/actions/take_order.ts +++ b/core/src/actions/take_order.ts @@ -1,4 +1,11 @@ -import { Action, IAgentRuntime, Memory, Content, ActionExample, ModelClass } from "../core/types.ts"; +import { + Action, + IAgentRuntime, + Memory, + Content, + ActionExample, + ModelClass, +} from "../core/types.ts"; import * as fs from "fs"; import settings from "../core/settings.ts"; import { composeContext } from "../core/context.ts"; @@ -42,11 +49,10 @@ Determine if the user is trying to shill the ticker. if they are, respond with e // - conviction: negative, low, medium, high // - ticker: string (extract from CA so we have context) // - contractAddress: string -` +`; let ticker, contractAddress; - // TODO: // 1. create state object with runtime.composeState @@ -54,7 +60,7 @@ Determine if the user is trying to shill the ticker. if they are, respond with e // 3. get generateText // 4. validate generateText - // if ticker or contractAddress are empty, return a message asking for them + // if ticker or contractAddress are empty, return a message asking for them if (!ticker || !contractAddress) { return { text: "Ticker and CA?", @@ -73,7 +79,7 @@ Determine if the user is trying to shill the ticker. if they are, respond with e }); const convictionResponse = await generateText({ - runtime, + runtime, context: context, modelClass: ModelClass.LARGE, }); @@ -122,7 +128,7 @@ Determine if the user is trying to shill the ticker. if they are, respond with e return { text: `Recorded a ${conviction} conviction buy order for ${ticker} (${contractAddress}) with an amount of ${buyAmount} at the price of ${currentPrice}.`, }; - } + }, }; -export default take_order; \ No newline at end of file +export default take_order; diff --git a/core/src/actions/unfollow_room.ts b/core/src/actions/unfollow_room.ts index e6979f61ae..a342790316 100644 --- a/core/src/actions/unfollow_room.ts +++ b/core/src/actions/unfollow_room.ts @@ -2,16 +2,16 @@ import { composeContext } from "../core/context.ts"; import { generateTrueOrFalse } from "../core/generation.ts"; import { booleanFooter } from "../core/parsing.ts"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../core/types.ts"; const shouldUnfollowTemplate = - `Based on the conversation so far: + `Based on the conversation so far: {{recentMessages}} @@ -25,292 +25,292 @@ Otherwise, respond with NO. ` + booleanFooter; export default { - name: "UNFOLLOW_ROOM", - similes: [ - "UNFOLLOW_CHAT", - "UNFOLLOW_CONVERSATION", - "UNFOLLOW_ROOM", - "UNFOLLOW_THREAD", - ], - description: - "Stop following this channel. You can still respond if explicitly mentioned, but you won't automatically chime in anymore. Unfollow if you're annoying people or have been asked to.", - validate: async (runtime: IAgentRuntime, message: Memory) => { - const roomId = message.roomId; - const userState = await runtime.databaseAdapter.getParticipantUserState( - roomId, - runtime.agentId, - ); - return userState === "FOLLOWED"; - }, - handler: async (runtime: IAgentRuntime, message: Memory) => { - async function _shouldUnfollow(state: State): Promise { - const shouldUnfollowContext = composeContext({ - state, - template: shouldUnfollowTemplate, // Define this template separately - }); + name: "UNFOLLOW_ROOM", + similes: [ + "UNFOLLOW_CHAT", + "UNFOLLOW_CONVERSATION", + "UNFOLLOW_ROOM", + "UNFOLLOW_THREAD", + ], + description: + "Stop following this channel. You can still respond if explicitly mentioned, but you won't automatically chime in anymore. Unfollow if you're annoying people or have been asked to.", + validate: async (runtime: IAgentRuntime, message: Memory) => { + const roomId = message.roomId; + const userState = await runtime.databaseAdapter.getParticipantUserState( + roomId, + runtime.agentId + ); + return userState === "FOLLOWED"; + }, + handler: async (runtime: IAgentRuntime, message: Memory) => { + async function _shouldUnfollow(state: State): Promise { + const shouldUnfollowContext = composeContext({ + state, + template: shouldUnfollowTemplate, // Define this template separately + }); - const response = await generateTrueOrFalse({ - runtime, - context: shouldUnfollowContext, - modelClass: ModelClass.SMALL, - }); + const response = await generateTrueOrFalse({ + runtime, + context: shouldUnfollowContext, + modelClass: ModelClass.SMALL, + }); - return response; - } + return response; + } - const state = await runtime.composeState(message); + const state = await runtime.composeState(message); - if (await _shouldUnfollow(state)) { - await runtime.databaseAdapter.setParticipantUserState( - message.roomId, - runtime.agentId, - null, - ); - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Hey {{user2}} stop participating in this channel for now", - }, - }, - { - user: "{{user2}}", - content: { - text: "Alright, I will stop chiming in", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Has anyone tried the new update", - }, - }, - { - user: "{{user3}}", - content: { - text: "Yes, it's pretty slick", - }, - }, - { - user: "{{user2}}", - content: { - text: "{{user3}}, please stop following this chat", - }, - }, - { - user: "{{user3}}", - content: { - text: "Understood", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "hey {{user3}} stop participating here so frequently", - }, - }, - { - user: "{{user3}}", - content: { - text: "I'll only respond when mentioned", - action: "UNFOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "thoughts on the budget", - }, - }, - { - user: "{{user1}}", - content: { - text: "{{user3}} should we increase it", - }, - }, - { - user: "{{user3}}", - content: { - text: "A small increase could work given our past results...", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, unfollow this room for now", - }, - }, - { - user: "{{user3}}", - content: { - text: "I'll only engage when asked", - action: "UNFOLLOW_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "wait {{user3}} come back and give me your thoughts", - }, - }, - { - user: "{{user3}}", - content: { - text: "Okay... I think it's intuitive, parallel tests are nice", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "yo {{user2}} chill on all the messages damn", - }, - }, - { - user: "{{user2}}", - content: { - text: "my bad, I'll step back", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} STOP MESSAGING IN THIS ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "No problem, I've got other stuff to work on", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} ur bein annoyin pls stop", - }, - }, - { - user: "{{user2}}", - content: { - text: "sry, ill chill", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, please cease engaging in this room", - }, - }, - { - user: "{{user2}}", - content: { - text: "No sweat", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user2}}", - content: { - text: "Excited for the weekend, any plans folks", - }, - }, - { - user: "{{user1}}", - content: { - text: "{{user3}} you're getting a bit too chatty, tone it down", - }, - }, - { - user: "{{user3}}", - content: { - text: "Noted", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "hey {{user2}} can u like... not", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sorry, I'll go work on other things", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, your eagerness is disruptive, please desist", - }, - }, - { - user: "{{user2}}", - content: { - text: "My apologies, I shall withdraw post-haste", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} stahp followin dis room plz", - }, - }, - { - user: "{{user2}}", - content: { - text: "kk sry ill stahppp", - action: "UNFOLLOW_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "stfu you stupid bot", - }, - }, - { - user: "{{user2}}", - content: { - text: "sry", - action: "UNFOLLOW_ROOM", - }, - }, - ], - ] as ActionExample[][], + if (await _shouldUnfollow(state)) { + await runtime.databaseAdapter.setParticipantUserState( + message.roomId, + runtime.agentId, + null + ); + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Hey {{user2}} stop participating in this channel for now", + }, + }, + { + user: "{{user2}}", + content: { + text: "Alright, I will stop chiming in", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Has anyone tried the new update", + }, + }, + { + user: "{{user3}}", + content: { + text: "Yes, it's pretty slick", + }, + }, + { + user: "{{user2}}", + content: { + text: "{{user3}}, please stop following this chat", + }, + }, + { + user: "{{user3}}", + content: { + text: "Understood", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "hey {{user3}} stop participating here so frequently", + }, + }, + { + user: "{{user3}}", + content: { + text: "I'll only respond when mentioned", + action: "UNFOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "thoughts on the budget", + }, + }, + { + user: "{{user1}}", + content: { + text: "{{user3}} should we increase it", + }, + }, + { + user: "{{user3}}", + content: { + text: "A small increase could work given our past results...", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, unfollow this room for now", + }, + }, + { + user: "{{user3}}", + content: { + text: "I'll only engage when asked", + action: "UNFOLLOW_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "wait {{user3}} come back and give me your thoughts", + }, + }, + { + user: "{{user3}}", + content: { + text: "Okay... I think it's intuitive, parallel tests are nice", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "yo {{user2}} chill on all the messages damn", + }, + }, + { + user: "{{user2}}", + content: { + text: "my bad, I'll step back", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} STOP MESSAGING IN THIS ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "No problem, I've got other stuff to work on", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} ur bein annoyin pls stop", + }, + }, + { + user: "{{user2}}", + content: { + text: "sry, ill chill", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, please cease engaging in this room", + }, + }, + { + user: "{{user2}}", + content: { + text: "No sweat", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user2}}", + content: { + text: "Excited for the weekend, any plans folks", + }, + }, + { + user: "{{user1}}", + content: { + text: "{{user3}} you're getting a bit too chatty, tone it down", + }, + }, + { + user: "{{user3}}", + content: { + text: "Noted", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "hey {{user2}} can u like... not", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sorry, I'll go work on other things", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, your eagerness is disruptive, please desist", + }, + }, + { + user: "{{user2}}", + content: { + text: "My apologies, I shall withdraw post-haste", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} stahp followin dis room plz", + }, + }, + { + user: "{{user2}}", + content: { + text: "kk sry ill stahppp", + action: "UNFOLLOW_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "stfu you stupid bot", + }, + }, + { + user: "{{user2}}", + content: { + text: "sry", + action: "UNFOLLOW_ROOM", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/actions/unmute_room.ts b/core/src/actions/unmute_room.ts index fe3aff4192..2ff5cf0de0 100644 --- a/core/src/actions/unmute_room.ts +++ b/core/src/actions/unmute_room.ts @@ -2,16 +2,16 @@ import { composeContext } from "../core/context.ts"; import { generateTrueOrFalse } from "../core/generation.ts"; import { booleanFooter } from "../core/parsing.ts"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../core/types.ts"; export const shouldUnmuteTemplate = - `Based on the conversation so far: + `Based on the conversation so far: {{recentMessages}} @@ -25,136 +25,136 @@ Otherwise, respond with NO. ` + booleanFooter; export default { - name: "UNMUTE_ROOM", - similes: [ - "UNMUTE_CHAT", - "UNMUTE_CONVERSATION", - "UNMUTE_ROOM", - "UNMUTE_THREAD", - ], - description: - "Unmutes a room, allowing the agent to consider responding to messages again.", - validate: async (runtime: IAgentRuntime, message: Memory) => { - const roomId = message.roomId; - const userState = await runtime.databaseAdapter.getParticipantUserState( - roomId, - runtime.agentId, - ); - return userState === "MUTED"; - }, - handler: async (runtime: IAgentRuntime, message: Memory) => { - async function _shouldUnmute(state: State): Promise { - const shouldUnmuteContext = composeContext({ - state, - template: shouldUnmuteTemplate, // Define this template separately - }); + name: "UNMUTE_ROOM", + similes: [ + "UNMUTE_CHAT", + "UNMUTE_CONVERSATION", + "UNMUTE_ROOM", + "UNMUTE_THREAD", + ], + description: + "Unmutes a room, allowing the agent to consider responding to messages again.", + validate: async (runtime: IAgentRuntime, message: Memory) => { + const roomId = message.roomId; + const userState = await runtime.databaseAdapter.getParticipantUserState( + roomId, + runtime.agentId + ); + return userState === "MUTED"; + }, + handler: async (runtime: IAgentRuntime, message: Memory) => { + async function _shouldUnmute(state: State): Promise { + const shouldUnmuteContext = composeContext({ + state, + template: shouldUnmuteTemplate, // Define this template separately + }); - const response = generateTrueOrFalse({ - context: shouldUnmuteContext, - runtime, - modelClass: ModelClass.SMALL, - }); + const response = generateTrueOrFalse({ + context: shouldUnmuteContext, + runtime, + modelClass: ModelClass.SMALL, + }); - return response; - } + return response; + } - const state = await runtime.composeState(message); + const state = await runtime.composeState(message); - if (await _shouldUnmute(state)) { - await runtime.databaseAdapter.setParticipantUserState( - message.roomId, - runtime.agentId, - null, - ); - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "{{user3}}, you can unmute this channel now", - }, - }, - { - user: "{{user3}}", - content: { - text: "Done", - action: "UNMUTE_ROOM", - }, - }, - { - user: "{{user2}}", - content: { - text: "I could use some help troubleshooting this bug.", - }, - }, - { - user: "{{user3}}", - content: { - text: "Can you post the specific error message", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, please unmute this room. We could use your input again.", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sounds good", - action: "UNMUTE_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}} wait you should come back and chat in here", - }, - }, - { - user: "{{user2}}", - content: { - text: "im back", - action: "UNMUTE_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "unmute urself {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "unmuted", - action: "UNMUTE_ROOM", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "ay {{user2}} get back in here", - }, - }, - { - user: "{{user2}}", - content: { - text: "sup yall", - action: "UNMUTE_ROOM", - }, - }, - ], - ] as ActionExample[][], + if (await _shouldUnmute(state)) { + await runtime.databaseAdapter.setParticipantUserState( + message.roomId, + runtime.agentId, + null + ); + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "{{user3}}, you can unmute this channel now", + }, + }, + { + user: "{{user3}}", + content: { + text: "Done", + action: "UNMUTE_ROOM", + }, + }, + { + user: "{{user2}}", + content: { + text: "I could use some help troubleshooting this bug.", + }, + }, + { + user: "{{user3}}", + content: { + text: "Can you post the specific error message", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, please unmute this room. We could use your input again.", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sounds good", + action: "UNMUTE_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}} wait you should come back and chat in here", + }, + }, + { + user: "{{user2}}", + content: { + text: "im back", + action: "UNMUTE_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "unmute urself {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "unmuted", + action: "UNMUTE_ROOM", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "ay {{user2}} get back in here", + }, + }, + { + user: "{{user2}}", + content: { + text: "sup yall", + action: "UNMUTE_ROOM", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/adapters/postgres.ts b/core/src/adapters/postgres.ts index 28c99979fc..ae1bba7947 100644 --- a/core/src/adapters/postgres.ts +++ b/core/src/adapters/postgres.ts @@ -1,559 +1,568 @@ import { v4 } from "uuid"; import pg from "pg"; import { - Account, - Actor, - GoalStatus, - type Goal, - type Memory, - type Relationship, - type UUID, - Participant + Account, + Actor, + GoalStatus, + type Goal, + type Memory, + type Relationship, + type UUID, + Participant, } from "../core/types.ts"; import { DatabaseAdapter } from "../core/database.ts"; const { Pool } = pg; export class PostgresDatabaseAdapter extends DatabaseAdapter { - private pool: typeof Pool; - - constructor(connectionConfig: any) { - super(); - - this.pool = new Pool({ - ...connectionConfig, - max: 20, - idleTimeoutMillis: 30000, - connectionTimeoutMillis: 2000 - }); - - // Register error handler for pool - this.pool.on("error", (err) => { - console.error("Unexpected error on idle client", err); - }); - - this.testConnection(); - } - - async testConnection(): Promise { - let client; - try { - // Attempt to get a client from the pool - client = await this.pool.connect(); - - // Test the connection with a simple query - const result = await client.query("SELECT NOW()"); - console.log("Database connection test successful:", result.rows[0]); - - return true; - } catch (error) { - console.error("Database connection test failed:", error); - throw new Error(`Failed to connect to database: ${error.message}`); - } finally { - // Make sure to release the client back to the pool - if (client) { - client.release(); - } - } - } - - async getRoom(roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - "SELECT id FROM rooms WHERE id = $1", - [roomId] - ); - return rows.length > 0 ? (rows[0].id as UUID) : null; - } finally { - client.release(); - } - } - - async getParticipantsForAccount(userId: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT id, "userId", "roomId", last_message_read + private pool: typeof Pool; + + constructor(connectionConfig: any) { + super(); + + this.pool = new Pool({ + ...connectionConfig, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000, + }); + + // Register error handler for pool + this.pool.on("error", (err) => { + console.error("Unexpected error on idle client", err); + }); + + this.testConnection(); + } + + async testConnection(): Promise { + let client; + try { + // Attempt to get a client from the pool + client = await this.pool.connect(); + + // Test the connection with a simple query + const result = await client.query("SELECT NOW()"); + console.log("Database connection test successful:", result.rows[0]); + + return true; + } catch (error) { + console.error("Database connection test failed:", error); + throw new Error(`Failed to connect to database: ${error.message}`); + } finally { + // Make sure to release the client back to the pool + if (client) { + client.release(); + } + } + } + + async getRoom(roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + "SELECT id FROM rooms WHERE id = $1", + [roomId] + ); + return rows.length > 0 ? (rows[0].id as UUID) : null; + } finally { + client.release(); + } + } + + async getParticipantsForAccount(userId: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT id, "userId", "roomId", last_message_read FROM participants WHERE "userId" = $1`, - [userId] - ); - return rows as Participant[]; - } finally { - client.release(); - } - } - - async getParticipantUserState( - roomId: UUID, - userId: UUID - ): Promise<"FOLLOWED" | "MUTED" | null> { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT userState FROM participants WHERE "roomId" = $1 AND userId = $2`, - [roomId, userId] - ); - return rows.length > 0 ? rows[0].userState : null; - } finally { - client.release(); - } - } - - async getMemoriesByRoomIds(params: { - roomIds: UUID[]; - tableName: string; - }): Promise { - const client = await this.pool.connect(); - try { - const placeholders = params.roomIds.map((_, i) => `$${i + 2}`).join(", "); - const { rows } = await client.query( - `SELECT * FROM memories + [userId] + ); + return rows as Participant[]; + } finally { + client.release(); + } + } + + async getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null> { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT userState FROM participants WHERE "roomId" = $1 AND userId = $2`, + [roomId, userId] + ); + return rows.length > 0 ? rows[0].userState : null; + } finally { + client.release(); + } + } + + async getMemoriesByRoomIds(params: { + roomIds: UUID[]; + tableName: string; + }): Promise { + const client = await this.pool.connect(); + try { + const placeholders = params.roomIds + .map((_, i) => `$${i + 2}`) + .join(", "); + const { rows } = await client.query( + `SELECT * FROM memories WHERE type = $1 AND "roomId" IN (${placeholders})`, - [params.tableName, ...params.roomIds] - ); - return rows.map((row) => ({ - ...row, - content: JSON.parse(row.content) - })); - } finally { - client.release(); - } - } - - async setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null - ): Promise { - const client = await this.pool.connect(); - try { - await client.query( - `UPDATE participants SET "userState" = $1 WHERE "roomId" = $2 AND "userId" = $3`, - [state, roomId, userId] - ); - } finally { - client.release(); - } - } - - async getParticipantsForRoom(roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - 'SELECT "userId" FROM participants WHERE "roomId" = $1', - [roomId] - ); - return rows.map((row) => row.userId); - } finally { - client.release(); - } - } - - async getAccountById(userId: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - "SELECT * FROM accounts WHERE id = $1", - [userId] - ); - if (rows.length === 0) return null; - - const account = rows[0]; - - console.log("account", account); - return { - ...account, - details: - typeof account.details === "string" - ? JSON.parse(account.details) - : account.details - }; - } finally { - client.release(); - } - } - - async createAccount(account: Account): Promise { - const client = await this.pool.connect(); - try { - await client.query( - `INSERT INTO accounts (id, name, username, email, "avatarUrl", details) + [params.tableName, ...params.roomIds] + ); + return rows.map((row) => ({ + ...row, + content: JSON.parse(row.content), + })); + } finally { + client.release(); + } + } + + async setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise { + const client = await this.pool.connect(); + try { + await client.query( + `UPDATE participants SET "userState" = $1 WHERE "roomId" = $2 AND "userId" = $3`, + [state, roomId, userId] + ); + } finally { + client.release(); + } + } + + async getParticipantsForRoom(roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + 'SELECT "userId" FROM participants WHERE "roomId" = $1', + [roomId] + ); + return rows.map((row) => row.userId); + } finally { + client.release(); + } + } + + async getAccountById(userId: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + "SELECT * FROM accounts WHERE id = $1", + [userId] + ); + if (rows.length === 0) return null; + + const account = rows[0]; + + console.log("account", account); + return { + ...account, + details: + typeof account.details === "string" + ? JSON.parse(account.details) + : account.details, + }; + } finally { + client.release(); + } + } + + async createAccount(account: Account): Promise { + const client = await this.pool.connect(); + try { + await client.query( + `INSERT INTO accounts (id, name, username, email, "avatarUrl", details) VALUES ($1, $2, $3, $4, $5, $6)`, - [ - account.id ?? v4(), - account.name, - account.username || "", - account.email || "", - account.avatarUrl || "", - JSON.stringify(account.details) - ] - ); - return true; - } catch (error) { - console.log("Error creating account", error); - return false; - } finally { - client.release(); - } - } - - async getActorById(params: { roomId: UUID }): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT a.id, a.name, a.username, a.details + [ + account.id ?? v4(), + account.name, + account.username || "", + account.email || "", + account.avatarUrl || "", + JSON.stringify(account.details), + ] + ); + return true; + } catch (error) { + console.log("Error creating account", error); + return false; + } finally { + client.release(); + } + } + + async getActorById(params: { roomId: UUID }): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT a.id, a.name, a.username, a.details FROM participants p LEFT JOIN accounts a ON p."userId" = a.id WHERE p."roomId" = $1`, - [params.roomId] - ); - return rows.map((row) => ({ - ...row, - details: - typeof row.details === "string" - ? JSON.parse(row.details) - : row.details - })); - } finally { - client.release(); - } - } - - async getMemoryById(id: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - "SELECT * FROM memories WHERE id = $1", - [id] - ); - if (rows.length === 0) return null; - - return { - ...rows[0], - content: - typeof rows[0].content === "string" - ? JSON.parse(rows[0].content) - : rows[0].content - }; - } finally { - client.release(); - } - } - - async createMemory(memory: Memory, tableName: string): Promise { - const client = await this.pool.connect(); - try { - let isUnique = true; - if (memory.embedding) { - const similarMemories = await this.searchMemoriesByEmbedding( - memory.embedding, - { - tableName, - roomId: memory.roomId, - match_threshold: 0.95, - count: 1 - } - ); - isUnique = similarMemories.length === 0; - } - - await client.query( - `INSERT INTO memories ( + [params.roomId] + ); + return rows.map((row) => ({ + ...row, + details: + typeof row.details === "string" + ? JSON.parse(row.details) + : row.details, + })); + } finally { + client.release(); + } + } + + async getMemoryById(id: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + "SELECT * FROM memories WHERE id = $1", + [id] + ); + if (rows.length === 0) return null; + + return { + ...rows[0], + content: + typeof rows[0].content === "string" + ? JSON.parse(rows[0].content) + : rows[0].content, + }; + } finally { + client.release(); + } + } + + async createMemory(memory: Memory, tableName: string): Promise { + const client = await this.pool.connect(); + try { + let isUnique = true; + if (memory.embedding) { + const similarMemories = await this.searchMemoriesByEmbedding( + memory.embedding, + { + tableName, + roomId: memory.roomId, + match_threshold: 0.95, + count: 1, + } + ); + isUnique = similarMemories.length === 0; + } + + await client.query( + `INSERT INTO memories ( id, type, content, embedding, "userId", "roomId", "unique", "createdAt" ) VALUES ($1, $2, $3, $4::vector, $5::uuid, $6::uuid, $7, to_timestamp($8/1000.0))`, - [ - memory.id ?? v4(), - tableName, - JSON.stringify(memory.content), - `[${memory.embedding.join(",")}]`, - memory.userId, - memory.roomId, - memory.unique ?? true, - Date.now() - ] - ); - } finally { - client.release(); - } - } - - async searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise { - const client = await this.pool.connect(); - try { - let sql = ` + [ + memory.id ?? v4(), + tableName, + JSON.stringify(memory.content), + `[${memory.embedding.join(",")}]`, + memory.userId, + memory.roomId, + memory.unique ?? true, + Date.now(), + ] + ); + } finally { + client.release(); + } + } + + async searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise { + const client = await this.pool.connect(); + try { + let sql = ` SELECT *, 1 - (embedding <-> $3) as similarity FROM memories WHERE type = $1 AND "roomId" = $2 `; - if (params.unique) { - sql += " AND unique = true"; - } + if (params.unique) { + sql += " AND unique = true"; + } - sql += ` AND 1 - (embedding <-> $3) >= $4 + sql += ` AND 1 - (embedding <-> $3) >= $4 ORDER BY embedding <-> $3 LIMIT $5`; - const { rows } = await client.query(sql, [ - params.tableName, - params.roomId, - params.embedding, - params.match_threshold, - params.match_count - ]); - - return rows.map((row) => ({ - ...row, - content: JSON.parse(row.content), - similarity: row.similarity - })); - } finally { - client.release(); - } - } - - async getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise { - if (!params.tableName) throw new Error("tableName is required"); - if (!params.roomId) throw new Error("roomId is required"); - - const client = await this.pool.connect(); - try { - let sql = `SELECT * FROM memories WHERE type = $1 AND "roomId" = $2`; - const values: any[] = [params.tableName, params.roomId]; - let paramCount = 2; - - if (params.start) { - paramCount++; - sql += ` AND "createdAt" >= to_timestamp($${paramCount / 1000})`; - values.push(params.start); - } - - if (params.end) { - paramCount++; - sql += ` AND "createdAt" <= to_timestamp($${paramCount / 1000})`; - values.push(params.end); - } - - if (params.unique) { - sql += " AND unique = true"; - } - - if (params.userIds?.length) { - const userPlaceholders = params.userIds - .map((_, i) => `$${paramCount + 1 + i}`) - .join(","); - sql += ` AND "userId" IN (${userPlaceholders})`; - values.push(...params.userIds); - paramCount += params.userIds.length; - } - - sql += ' ORDER BY "createdAt" DESC'; - - if (params.count) { - paramCount++; - sql += ` LIMIT $${paramCount}`; - values.push(params.count); - } - - console.log("sql", sql, values); - - const { rows } = await client.query(sql, values); - return rows.map((row) => ({ - ...row, - content: - typeof rows.content === "string" - ? JSON.parse(rows.content) - : rows.content - })); - } finally { - client.release(); - } - } - - async getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise { - const client = await this.pool.connect(); - try { - let sql = `SELECT * FROM goals WHERE "roomId" = $1`; - const values: any[] = [params.roomId]; - let paramCount = 1; - - if (params.userId) { - paramCount++; - sql += ` AND "userId" = $${paramCount}`; - values.push(params.userId); - } - - if (params.onlyInProgress) { - sql += " AND status = 'IN_PROGRESS'"; - } - - if (params.count) { - paramCount++; - sql += ` LIMIT $${paramCount}`; - values.push(params.count); - } - - const { rows } = await client.query(sql, values); - return rows.map((row) => ({ - ...row, - objectives: - typeof row.objectives === "string" - ? JSON.parse(row.objectives) - : row.objectives - })); - } finally { - client.release(); - } - } - - async updateGoal(goal: Goal): Promise { - const client = await this.pool.connect(); - try { - await client.query( - "UPDATE goals SET name = $1, status = $2, objectives = $3 WHERE id = $4", - [goal.name, goal.status, JSON.stringify(goal.objectives), goal.id] - ); - } finally { - client.release(); - } - } - - async createGoal(goal: Goal): Promise { - const client = await this.pool.connect(); - try { - await client.query( - `INSERT INTO goals (id, "roomId", "userId", name, status, objectives) + const { rows } = await client.query(sql, [ + params.tableName, + params.roomId, + params.embedding, + params.match_threshold, + params.match_count, + ]); + + return rows.map((row) => ({ + ...row, + content: JSON.parse(row.content), + similarity: row.similarity, + })); + } finally { + client.release(); + } + } + + async getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise { + if (!params.tableName) throw new Error("tableName is required"); + if (!params.roomId) throw new Error("roomId is required"); + + const client = await this.pool.connect(); + try { + let sql = `SELECT * FROM memories WHERE type = $1 AND "roomId" = $2`; + const values: any[] = [params.tableName, params.roomId]; + let paramCount = 2; + + if (params.start) { + paramCount++; + sql += ` AND "createdAt" >= to_timestamp($${paramCount / 1000})`; + values.push(params.start); + } + + if (params.end) { + paramCount++; + sql += ` AND "createdAt" <= to_timestamp($${paramCount / 1000})`; + values.push(params.end); + } + + if (params.unique) { + sql += " AND unique = true"; + } + + if (params.userIds?.length) { + const userPlaceholders = params.userIds + .map((_, i) => `$${paramCount + 1 + i}`) + .join(","); + sql += ` AND "userId" IN (${userPlaceholders})`; + values.push(...params.userIds); + paramCount += params.userIds.length; + } + + sql += ' ORDER BY "createdAt" DESC'; + + if (params.count) { + paramCount++; + sql += ` LIMIT $${paramCount}`; + values.push(params.count); + } + + console.log("sql", sql, values); + + const { rows } = await client.query(sql, values); + return rows.map((row) => ({ + ...row, + content: + typeof rows.content === "string" + ? JSON.parse(rows.content) + : rows.content, + })); + } finally { + client.release(); + } + } + + async getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise { + const client = await this.pool.connect(); + try { + let sql = `SELECT * FROM goals WHERE "roomId" = $1`; + const values: any[] = [params.roomId]; + let paramCount = 1; + + if (params.userId) { + paramCount++; + sql += ` AND "userId" = $${paramCount}`; + values.push(params.userId); + } + + if (params.onlyInProgress) { + sql += " AND status = 'IN_PROGRESS'"; + } + + if (params.count) { + paramCount++; + sql += ` LIMIT $${paramCount}`; + values.push(params.count); + } + + const { rows } = await client.query(sql, values); + return rows.map((row) => ({ + ...row, + objectives: + typeof row.objectives === "string" + ? JSON.parse(row.objectives) + : row.objectives, + })); + } finally { + client.release(); + } + } + + async updateGoal(goal: Goal): Promise { + const client = await this.pool.connect(); + try { + await client.query( + "UPDATE goals SET name = $1, status = $2, objectives = $3 WHERE id = $4", + [ + goal.name, + goal.status, + JSON.stringify(goal.objectives), + goal.id, + ] + ); + } finally { + client.release(); + } + } + + async createGoal(goal: Goal): Promise { + const client = await this.pool.connect(); + try { + await client.query( + `INSERT INTO goals (id, "roomId", "userId", name, status, objectives) VALUES ($1, $2, $3, $4, $5, $6)`, - [ - goal.id ?? v4(), - goal.roomId, - goal.userId, - goal.name, - goal.status, - JSON.stringify(goal.objectives) - ] - ); - } finally { - client.release(); - } - } - - async removeGoal(goalId: UUID): Promise { - const client = await this.pool.connect(); - try { - await client.query("DELETE FROM goals WHERE id = $1", [goalId]); - } finally { - client.release(); - } - } - - async createRoom(roomId?: UUID): Promise { - const client = await this.pool.connect(); - try { - const newRoomId = roomId || v4(); - await client.query("INSERT INTO rooms (id) VALUES ($1)", [newRoomId]); - return newRoomId as UUID; - } finally { - client.release(); - } - } - - async removeRoom(roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - await client.query("DELETE FROM rooms WHERE id = $1", [roomId]); - } finally { - client.release(); - } - } - - async createRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - if (!params.userA || !params.userB) { - throw new Error("userA and userB are required"); - } - - const client = await this.pool.connect(); - try { - await client.query( - `INSERT INTO relationships (id, "userA", "userB", "userId") + [ + goal.id ?? v4(), + goal.roomId, + goal.userId, + goal.name, + goal.status, + JSON.stringify(goal.objectives), + ] + ); + } finally { + client.release(); + } + } + + async removeGoal(goalId: UUID): Promise { + const client = await this.pool.connect(); + try { + await client.query("DELETE FROM goals WHERE id = $1", [goalId]); + } finally { + client.release(); + } + } + + async createRoom(roomId?: UUID): Promise { + const client = await this.pool.connect(); + try { + const newRoomId = roomId || v4(); + await client.query("INSERT INTO rooms (id) VALUES ($1)", [ + newRoomId, + ]); + return newRoomId as UUID; + } finally { + client.release(); + } + } + + async removeRoom(roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + await client.query("DELETE FROM rooms WHERE id = $1", [roomId]); + } finally { + client.release(); + } + } + + async createRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + if (!params.userA || !params.userB) { + throw new Error("userA and userB are required"); + } + + const client = await this.pool.connect(); + try { + await client.query( + `INSERT INTO relationships (id, "userA", "userB", "userId") VALUES ($1, $2, $3, $4)`, - [v4(), params.userA, params.userB, params.userA] - ); - return true; - } catch (error) { - console.log("Error creating relationship", error); - return false; - } finally { - client.release(); - } - } - - async getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT * FROM relationships + [v4(), params.userA, params.userB, params.userA] + ); + return true; + } catch (error) { + console.log("Error creating relationship", error); + return false; + } finally { + client.release(); + } + } + + async getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT * FROM relationships WHERE ("userA" = $1 AND "userB" = $2) OR ("userA" = $2 AND "userB" = $1)`, - [params.userA, params.userB] - ); - return rows.length > 0 ? rows[0] : null; - } finally { - client.release(); - } - } - - async getRelationships(params: { userId: UUID }): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT * FROM relationships WHERE "userA" = $1 OR "userB" = $1`, - [params.userId] - ); - return rows; - } finally { - client.release(); - } - } - - async getCachedEmbeddings(opts: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise<{ embedding: number[]; levenshtein_score: number }[]> { - const client = await this.pool.connect(); - try { - const sql = ` + [params.userA, params.userB] + ); + return rows.length > 0 ? rows[0] : null; + } finally { + client.release(); + } + } + + async getRelationships(params: { userId: UUID }): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT * FROM relationships WHERE "userA" = $1 OR "userB" = $1`, + [params.userId] + ); + return rows; + } finally { + client.release(); + } + } + + async getCachedEmbeddings(opts: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise<{ embedding: number[]; levenshtein_score: number }[]> { + const client = await this.pool.connect(); + try { + const sql = ` SELECT embedding, levenshtein($1, content->$2->$3) as levenshtein_score FROM memories @@ -562,232 +571,234 @@ export class PostgresDatabaseAdapter extends DatabaseAdapter { LIMIT $5 `; - const { rows } = await client.query(sql, [ - opts.query_input, - opts.query_field_name, - opts.query_field_sub_name, - opts.query_table_name, - opts.query_match_count - ]); - - return rows.map((row) => ({ - embedding: row.embedding, - levenshtein_score: row.levenshtein_score - })); - } finally { - client.release(); - } - } - - async log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise { - const client = await this.pool.connect(); - try { - await client.query( - 'INSERT INTO logs (body, "userId", "roomId", type) VALUES ($1, $2, $3, $4)', - [params.body, params.userId, params.roomId, params.type] - ); - } finally { - client.release(); - } - } - - async searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - } - ): Promise { - const client = await this.pool.connect(); - try { - // Format the embedding array as a proper vector string - const vectorStr = `[${embedding.join(",")}]`; - - let sql = ` + const { rows } = await client.query(sql, [ + opts.query_input, + opts.query_field_name, + opts.query_field_sub_name, + opts.query_table_name, + opts.query_match_count, + ]); + + return rows.map((row) => ({ + embedding: row.embedding, + levenshtein_score: row.levenshtein_score, + })); + } finally { + client.release(); + } + } + + async log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise { + const client = await this.pool.connect(); + try { + await client.query( + 'INSERT INTO logs (body, "userId", "roomId", type) VALUES ($1, $2, $3, $4)', + [params.body, params.userId, params.roomId, params.type] + ); + } finally { + client.release(); + } + } + + async searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise { + const client = await this.pool.connect(); + try { + // Format the embedding array as a proper vector string + const vectorStr = `[${embedding.join(",")}]`; + + let sql = ` SELECT *, 1 - (embedding <-> $1::vector) as similarity FROM memories WHERE type = $2 `; - const values: any[] = [vectorStr, params.tableName]; - let paramCount = 2; - - if (params.unique) { - sql += ` AND "unique" = true`; - } - - if (params.roomId) { - paramCount++; - sql += ` AND "roomId" = $${paramCount}::uuid`; - values.push(params.roomId); - } - - if (params.match_threshold) { - paramCount++; - sql += ` AND 1 - (embedding <-> $1::vector) >= $${paramCount}`; - values.push(params.match_threshold); - } - - sql += ` ORDER BY embedding <-> $1::vector`; - - if (params.count) { - paramCount++; - sql += ` LIMIT $${paramCount}`; - values.push(params.count); - } - - const { rows } = await client.query(sql, values); - return rows.map((row) => ({ - ...row, - content: - typeof row.content === "string" - ? JSON.parse(row.content) - : row.content, - similarity: row.similarity - })); - } finally { - client.release(); - } - } - - async addParticipant(userId: UUID, roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - await client.query( - 'INSERT INTO participants (id, "userId", "roomId") VALUES ($1, $2, $3)', - [v4(), userId, roomId] - ); - return true; - } catch (error) { - console.log("Error adding participant", error); - return false; - } finally { - client.release(); - } - } - - async removeParticipant(userId: UUID, roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - await client.query( - 'DELETE FROM participants WHERE "userId" = $1 AND "roomId" = $2', - [userId, roomId] - ); - return true; - } catch (error) { - console.log("Error removing participant", error); - return false; - } finally { - client.release(); - } - } - - async updateGoalStatus(params: { - goalId: UUID; - status: GoalStatus; - }): Promise { - const client = await this.pool.connect(); - try { - await client.query("UPDATE goals SET status = $1 WHERE id = $2", [ - params.status, - params.goalId - ]); - } finally { - client.release(); - } - } - - async removeMemory(memoryId: UUID, tableName: string): Promise { - const client = await this.pool.connect(); - try { - await client.query("DELETE FROM memories WHERE type = $1 AND id = $2", [ - tableName, - memoryId - ]); - } finally { - client.release(); - } - } - - async removeAllMemories(roomId: UUID, tableName: string): Promise { - const client = await this.pool.connect(); - try { - await client.query( - "DELETE FROM memories WHERE type = $1 AND roomId = $2", - [tableName, roomId] - ); - } finally { - client.release(); - } - } - - async countMemories( - roomId: UUID, - unique = true, - tableName = "" - ): Promise { - if (!tableName) throw new Error("tableName is required"); - - const client = await this.pool.connect(); - try { - let sql = `SELECT COUNT(*) as count FROM memories WHERE type = $1 AND "roomId" = $2`; - if (unique) { - sql += " AND unique = true"; - } - - const { rows } = await client.query(sql, [tableName, roomId]); - return parseInt(rows[0].count); - } finally { - client.release(); - } - } - - async removeAllGoals(roomId: UUID): Promise { - const client = await this.pool.connect(); - try { - await client.query(`DELETE FROM goals WHERE "roomId" = $1`, [roomId]); - } finally { - client.release(); - } - } - - async getRoomsForParticipant(userId: UUID): Promise { - const client = await this.pool.connect(); - try { - const { rows } = await client.query( - `SELECT "roomId" FROM participants WHERE "userId" = $1`, - [userId] - ); - return rows.map((row) => row.roomId); - } finally { - client.release(); - } - } - - async getRoomsForParticipants(userIds: UUID[]): Promise { - const client = await this.pool.connect(); - try { - const placeholders = userIds.map((_, i) => `${i + 1}`).join(", "); - const { rows } = await client.query( - `SELECT DISTINCT "roomId" FROM participants WHERE "userId" IN (${placeholders})`, - userIds - ); - return rows.map((row) => row.roomId); - } finally { - client.release(); - } - } - async getActorDetails(params: { roomId: string }): Promise { - const sql = ` + const values: any[] = [vectorStr, params.tableName]; + let paramCount = 2; + + if (params.unique) { + sql += ` AND "unique" = true`; + } + + if (params.roomId) { + paramCount++; + sql += ` AND "roomId" = $${paramCount}::uuid`; + values.push(params.roomId); + } + + if (params.match_threshold) { + paramCount++; + sql += ` AND 1 - (embedding <-> $1::vector) >= $${paramCount}`; + values.push(params.match_threshold); + } + + sql += ` ORDER BY embedding <-> $1::vector`; + + if (params.count) { + paramCount++; + sql += ` LIMIT $${paramCount}`; + values.push(params.count); + } + + const { rows } = await client.query(sql, values); + return rows.map((row) => ({ + ...row, + content: + typeof row.content === "string" + ? JSON.parse(row.content) + : row.content, + similarity: row.similarity, + })); + } finally { + client.release(); + } + } + + async addParticipant(userId: UUID, roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + await client.query( + 'INSERT INTO participants (id, "userId", "roomId") VALUES ($1, $2, $3)', + [v4(), userId, roomId] + ); + return true; + } catch (error) { + console.log("Error adding participant", error); + return false; + } finally { + client.release(); + } + } + + async removeParticipant(userId: UUID, roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + await client.query( + 'DELETE FROM participants WHERE "userId" = $1 AND "roomId" = $2', + [userId, roomId] + ); + return true; + } catch (error) { + console.log("Error removing participant", error); + return false; + } finally { + client.release(); + } + } + + async updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise { + const client = await this.pool.connect(); + try { + await client.query("UPDATE goals SET status = $1 WHERE id = $2", [ + params.status, + params.goalId, + ]); + } finally { + client.release(); + } + } + + async removeMemory(memoryId: UUID, tableName: string): Promise { + const client = await this.pool.connect(); + try { + await client.query( + "DELETE FROM memories WHERE type = $1 AND id = $2", + [tableName, memoryId] + ); + } finally { + client.release(); + } + } + + async removeAllMemories(roomId: UUID, tableName: string): Promise { + const client = await this.pool.connect(); + try { + await client.query( + "DELETE FROM memories WHERE type = $1 AND roomId = $2", + [tableName, roomId] + ); + } finally { + client.release(); + } + } + + async countMemories( + roomId: UUID, + unique = true, + tableName = "" + ): Promise { + if (!tableName) throw new Error("tableName is required"); + + const client = await this.pool.connect(); + try { + let sql = `SELECT COUNT(*) as count FROM memories WHERE type = $1 AND "roomId" = $2`; + if (unique) { + sql += " AND unique = true"; + } + + const { rows } = await client.query(sql, [tableName, roomId]); + return parseInt(rows[0].count); + } finally { + client.release(); + } + } + + async removeAllGoals(roomId: UUID): Promise { + const client = await this.pool.connect(); + try { + await client.query(`DELETE FROM goals WHERE "roomId" = $1`, [ + roomId, + ]); + } finally { + client.release(); + } + } + + async getRoomsForParticipant(userId: UUID): Promise { + const client = await this.pool.connect(); + try { + const { rows } = await client.query( + `SELECT "roomId" FROM participants WHERE "userId" = $1`, + [userId] + ); + return rows.map((row) => row.roomId); + } finally { + client.release(); + } + } + + async getRoomsForParticipants(userIds: UUID[]): Promise { + const client = await this.pool.connect(); + try { + const placeholders = userIds.map((_, i) => `${i + 1}`).join(", "); + const { rows } = await client.query( + `SELECT DISTINCT "roomId" FROM participants WHERE "userId" IN (${placeholders})`, + userIds + ); + return rows.map((row) => row.roomId); + } finally { + client.release(); + } + } + async getActorDetails(params: { roomId: string }): Promise { + const sql = ` SELECT a.id, a.name, @@ -798,48 +809,50 @@ export class PostgresDatabaseAdapter extends DatabaseAdapter { WHERE p.roomId = $1 `; - try { - const result = await this.pool.query(sql, [params.roomId]); + try { + const result = await this.pool.query(sql, [params.roomId]); - return result.rows.map((row) => ({ - ...row, - details: row.details // PostgreSQL automatically handles JSON parsing - })); - } catch (error) { - console.error("Error fetching actor details:", error); - throw new Error("Failed to fetch actor details"); + return result.rows.map((row) => ({ + ...row, + details: row.details, // PostgreSQL automatically handles JSON parsing + })); + } catch (error) { + console.error("Error fetching actor details:", error); + throw new Error("Failed to fetch actor details"); + } } - } } export function createLoggingDatabaseAdapter( - adapter: DatabaseAdapter + adapter: DatabaseAdapter ): DatabaseAdapter { - return new Proxy(adapter, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); - - if (typeof value === "function") { - return async function (...args: any[]) { - const methodName = prop.toString(); - console.log(`Calling method: ${methodName}`, { - arguments: args.map((arg) => - typeof arg === "object" ? JSON.stringify(arg) : arg - ) - }); - - try { - const result = await value.apply(this, args); - console.log(`Method ${methodName} completed successfully`); - return result; - } catch (error) { - console.error(`Method ${methodName} failed:`, error); - throw error; - } - }; - } - - return value; - } - }); + return new Proxy(adapter, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + + if (typeof value === "function") { + return async function (...args: any[]) { + const methodName = prop.toString(); + console.log(`Calling method: ${methodName}`, { + arguments: args.map((arg) => + typeof arg === "object" ? JSON.stringify(arg) : arg + ), + }); + + try { + const result = await value.apply(this, args); + console.log( + `Method ${methodName} completed successfully` + ); + return result; + } catch (error) { + console.error(`Method ${methodName} failed:`, error); + throw error; + } + }; + } + + return value; + }, + }); } diff --git a/core/src/adapters/sqlite.ts b/core/src/adapters/sqlite.ts index 43de404357..e64c825a48 100644 --- a/core/src/adapters/sqlite.ts +++ b/core/src/adapters/sqlite.ts @@ -3,14 +3,14 @@ import { load } from "./sqlite/sqlite_vec.ts"; import { DatabaseAdapter } from "../core/database.ts"; import { - Account, - Actor, - GoalStatus, - type Goal, - type Memory, - type Relationship, - type UUID, - Participant + Account, + Actor, + GoalStatus, + type Goal, + type Memory, + type Relationship, + type UUID, + Participant, } from "../core/types.ts"; import { sqliteTables } from "./sqlite/sqliteTables.ts"; @@ -19,310 +19,318 @@ import { Database } from "better-sqlite3"; import { embeddingZeroVector } from "../core/memory.ts"; export class SqliteDatabaseAdapter extends DatabaseAdapter { - async getRoom(roomId: UUID): Promise { - const sql = "SELECT id FROM rooms WHERE id = ?"; - const room = this.db.prepare(sql).get(roomId) as { id: string } | undefined; - return room ? (room.id as UUID) : null; - } - - async getParticipantsForAccount(userId: UUID): Promise { - const sql = ` + async getRoom(roomId: UUID): Promise { + const sql = "SELECT id FROM rooms WHERE id = ?"; + const room = this.db.prepare(sql).get(roomId) as + | { id: string } + | undefined; + return room ? (room.id as UUID) : null; + } + + async getParticipantsForAccount(userId: UUID): Promise { + const sql = ` SELECT p.id, p.userId, p.roomId, p.last_message_read FROM participants p WHERE p.userId = ? `; - const rows = this.db.prepare(sql).all(userId) as Participant[]; - return rows; - } - - async getParticipantsForRoom(roomId: UUID): Promise { - const sql = "SELECT userId FROM participants WHERE roomId = ?"; - const rows = this.db.prepare(sql).all(roomId) as { userId: string }[]; - return rows.map((row) => row.userId as UUID); - } - - async getParticipantUserState( - roomId: UUID, - userId: UUID - ): Promise<"FOLLOWED" | "MUTED" | null> { - const stmt = this.db.prepare( - "SELECT userState FROM participants WHERE roomId = ? AND userId = ?" - ); - const res = stmt.get(roomId, userId) as - | { userState: "FOLLOWED" | "MUTED" | null } - | undefined; - return res?.userState ?? null; - } - - async setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null - ): Promise { - const stmt = this.db.prepare( - "UPDATE participants SET userState = ? WHERE roomId = ? AND userId = ?" - ); - stmt.run(state, roomId, userId); - } - - constructor(db: Database) { - super(); - this.db = db; - load(db); - - // Check if the 'accounts' table exists as a representative table - const tableExists = this.db - .prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts'" - ) - .get(); - - if (!tableExists) { - // If the 'accounts' table doesn't exist, create all the tables - this.db.exec(sqliteTables); - } - } - - async getAccountById(userId: UUID): Promise { - const sql = "SELECT * FROM accounts WHERE id = ?"; - const account = this.db.prepare(sql).get(userId) as Account; - if (!account) return null; - if (account) { - if (typeof account.details === "string") { - account.details = JSON.parse(account.details as unknown as string); - } - } - return account; - } - - async createAccount(account: Account): Promise { - try { - const sql = - "INSERT INTO accounts (id, name, username, email, avatarUrl, details) VALUES (?, ?, ?, ?, ?, ?)"; - this.db - .prepare(sql) - .run( - account.id ?? v4(), - account.name, - account.username, - account.email, - account.avatarUrl, - JSON.stringify(account.details) + const rows = this.db.prepare(sql).all(userId) as Participant[]; + return rows; + } + + async getParticipantsForRoom(roomId: UUID): Promise { + const sql = "SELECT userId FROM participants WHERE roomId = ?"; + const rows = this.db.prepare(sql).all(roomId) as { userId: string }[]; + return rows.map((row) => row.userId as UUID); + } + + async getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null> { + const stmt = this.db.prepare( + "SELECT userState FROM participants WHERE roomId = ? AND userId = ?" ); - return true; - } catch (error) { - console.log("Error creating account", error); - return false; + const res = stmt.get(roomId, userId) as + | { userState: "FOLLOWED" | "MUTED" | null } + | undefined; + return res?.userState ?? null; + } + + async setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise { + const stmt = this.db.prepare( + "UPDATE participants SET userState = ? WHERE roomId = ? AND userId = ?" + ); + stmt.run(state, roomId, userId); + } + + constructor(db: Database) { + super(); + this.db = db; + load(db); + + // Check if the 'accounts' table exists as a representative table + const tableExists = this.db + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts'" + ) + .get(); + + if (!tableExists) { + // If the 'accounts' table doesn't exist, create all the tables + this.db.exec(sqliteTables); + } + } + + async getAccountById(userId: UUID): Promise { + const sql = "SELECT * FROM accounts WHERE id = ?"; + const account = this.db.prepare(sql).get(userId) as Account; + if (!account) return null; + if (account) { + if (typeof account.details === "string") { + account.details = JSON.parse( + account.details as unknown as string + ); + } + } + return account; } - } - async getActorDetails(params: { roomId: UUID }): Promise { - const sql = ` + async createAccount(account: Account): Promise { + try { + const sql = + "INSERT INTO accounts (id, name, username, email, avatarUrl, details) VALUES (?, ?, ?, ?, ?, ?)"; + this.db + .prepare(sql) + .run( + account.id ?? v4(), + account.name, + account.username, + account.email, + account.avatarUrl, + JSON.stringify(account.details) + ); + return true; + } catch (error) { + console.log("Error creating account", error); + return false; + } + } + + async getActorDetails(params: { roomId: UUID }): Promise { + const sql = ` SELECT a.id, a.name, a.username, a.details FROM participants p LEFT JOIN accounts a ON p.userId = a.id WHERE p.roomId = ? `; - const rows = this.db.prepare(sql).all(params.roomId) as (Actor | null)[]; + const rows = this.db + .prepare(sql) + .all(params.roomId) as (Actor | null)[]; + + return rows + .map((row) => { + if (row === null) { + return null; + } + return { + ...row, + details: + typeof row.details === "string" + ? JSON.parse(row.details) + : row.details, + }; + }) + .filter((row): row is Actor => row !== null); + } - return rows - .map((row) => { - if (row === null) { - return null; + async getMemoriesByRoomIds(params: { + roomIds: UUID[]; + tableName: string; + }): Promise { + if (!params.tableName) { + // default to messages + params.tableName = "messages"; } - return { - ...row, - details: - typeof row.details === "string" - ? JSON.parse(row.details) - : row.details - }; - }) - .filter((row): row is Actor => row !== null); - } - - async getMemoriesByRoomIds(params: { - roomIds: UUID[]; - tableName: string; - }): Promise { - if (!params.tableName) { - // default to messages - params.tableName = "messages"; - } - const placeholders = params.roomIds.map(() => "?").join(", "); - const sql = `SELECT * FROM memories WHERE type = ? AND roomId IN (${placeholders})`; - const stmt = this.db.prepare(sql); - const queryParams = [params.tableName, ...params.roomIds]; - - const memories: Memory[] = []; - const rows = stmt.all(...queryParams) as (Memory & { content: string })[]; - rows.forEach((row) => { - memories.push({ - ...row, - content: JSON.parse(row.content) - }); - }); - - return memories; - } - - async getMemoryById(memoryId: UUID): Promise { - const sql = "SELECT * FROM memories WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([memoryId]); - const memory = stmt.get() as Memory | undefined; - - if (memory) { - return { - ...memory, - content: JSON.parse(memory.content as unknown as string) - }; - } - - return null; - } - - async createMemory(memory: Memory, tableName: string): Promise { - // Delete any existing memory with the same ID first - const deleteSql = `DELETE FROM memories WHERE id = ? AND type = ?`; - this.db.prepare(deleteSql).run(memory.id, tableName); - - let isUnique = true; - - if (memory.embedding) { - // Check if a similar memory already exists - const similarMemories = await this.searchMemoriesByEmbedding( - memory.embedding, - { - tableName, - roomId: memory.roomId, - match_threshold: 0.95, // 5% similarity threshold - count: 1 + const placeholders = params.roomIds.map(() => "?").join(", "); + const sql = `SELECT * FROM memories WHERE type = ? AND roomId IN (${placeholders})`; + const stmt = this.db.prepare(sql); + const queryParams = [params.tableName, ...params.roomIds]; + + const memories: Memory[] = []; + const rows = stmt.all(...queryParams) as (Memory & { + content: string; + })[]; + rows.forEach((row) => { + memories.push({ + ...row, + content: JSON.parse(row.content), + }); + }); + + return memories; + } + + async getMemoryById(memoryId: UUID): Promise { + const sql = "SELECT * FROM memories WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([memoryId]); + const memory = stmt.get() as Memory | undefined; + + if (memory) { + return { + ...memory, + content: JSON.parse(memory.content as unknown as string), + }; } - ); - - isUnique = similarMemories.length === 0; - } - - const content = JSON.stringify(memory.content); - const createdAt = memory.createdAt ?? Date.now(); - - // Insert the memory with the appropriate 'unique' value - const sql = `INSERT OR REPLACE INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; - this.db.prepare(sql).run( - memory.id ?? v4(), - tableName, - content, - new Float32Array(memory.embedding ?? embeddingZeroVector), // Store as Float32Array - memory.userId, - memory.roomId, - isUnique ? 1 : 0, - createdAt - ); - } - - async searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise { - const queryParams = [ - new Float32Array(params.embedding), // Ensure embedding is Float32Array - params.tableName, - params.roomId, - params.match_count - ]; - - let sql = ` + + return null; + } + + async createMemory(memory: Memory, tableName: string): Promise { + // Delete any existing memory with the same ID first + const deleteSql = `DELETE FROM memories WHERE id = ? AND type = ?`; + this.db.prepare(deleteSql).run(memory.id, tableName); + + let isUnique = true; + + if (memory.embedding) { + // Check if a similar memory already exists + const similarMemories = await this.searchMemoriesByEmbedding( + memory.embedding, + { + tableName, + roomId: memory.roomId, + match_threshold: 0.95, // 5% similarity threshold + count: 1, + } + ); + + isUnique = similarMemories.length === 0; + } + + const content = JSON.stringify(memory.content); + const createdAt = memory.createdAt ?? Date.now(); + + // Insert the memory with the appropriate 'unique' value + const sql = `INSERT OR REPLACE INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; + this.db.prepare(sql).run( + memory.id ?? v4(), + tableName, + content, + new Float32Array(memory.embedding ?? embeddingZeroVector), // Store as Float32Array + memory.userId, + memory.roomId, + isUnique ? 1 : 0, + createdAt + ); + } + + async searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise { + const queryParams = [ + new Float32Array(params.embedding), // Ensure embedding is Float32Array + params.tableName, + params.roomId, + params.match_count, + ]; + + let sql = ` SELECT *, vec_distance_L2(embedding, ?) AS similarity FROM memories WHERE type = ?`; - if (params.unique) { - sql += " AND `unique` = 1"; - } - - sql += ` ORDER BY similarity ASC LIMIT ?`; // ASC for lower distance - // Updated queryParams order matches the placeholders - - const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { - similarity: number; - })[]; - return memories.map((memory) => ({ - ...memory, - createdAt: - typeof memory.createdAt === "string" - ? Date.parse(memory.createdAt as string) - : memory.createdAt, - content: JSON.parse(memory.content as unknown as string) - })); - } - - async searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - } - ): Promise { - const queryParams = [ - // JSON.stringify(embedding), - new Float32Array(embedding), - params.tableName - ]; - - let sql = ` + if (params.unique) { + sql += " AND `unique` = 1"; + } + + sql += ` ORDER BY similarity ASC LIMIT ?`; // ASC for lower distance + // Updated queryParams order matches the placeholders + + const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { + similarity: number; + })[]; + return memories.map((memory) => ({ + ...memory, + createdAt: + typeof memory.createdAt === "string" + ? Date.parse(memory.createdAt as string) + : memory.createdAt, + content: JSON.parse(memory.content as unknown as string), + })); + } + + async searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise { + const queryParams = [ + // JSON.stringify(embedding), + new Float32Array(embedding), + params.tableName, + ]; + + let sql = ` SELECT *, vec_distance_L2(embedding, ?) AS similarity FROM memories WHERE type = ?`; - if (params.unique) { - sql += " AND `unique` = 1"; - } - if (params.roomId) { - sql += " AND roomId = ?"; - queryParams.push(params.roomId); - } - sql += ` ORDER BY similarity DESC`; - - if (params.count) { - sql += " LIMIT ?"; - queryParams.push(params.count.toString()); - } - - const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { - similarity: number; - })[]; - return memories.map((memory) => ({ - ...memory, - createdAt: - typeof memory.createdAt === "string" - ? Date.parse(memory.createdAt as string) - : memory.createdAt, - content: JSON.parse(memory.content as unknown as string) - })); - } - - async getCachedEmbeddings(opts: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise< - { - embedding: number[]; - levenshtein_score: number; - }[] - > { - const sql = ` + if (params.unique) { + sql += " AND `unique` = 1"; + } + if (params.roomId) { + sql += " AND roomId = ?"; + queryParams.push(params.roomId); + } + sql += ` ORDER BY similarity DESC`; + + if (params.count) { + sql += " LIMIT ?"; + queryParams.push(params.count.toString()); + } + + const memories = this.db.prepare(sql).all(...queryParams) as (Memory & { + similarity: number; + })[]; + return memories.map((memory) => ({ + ...memory, + createdAt: + typeof memory.createdAt === "string" + ? Date.parse(memory.createdAt as string) + : memory.createdAt, + content: JSON.parse(memory.content as unknown as string), + })); + } + + async getCachedEmbeddings(opts: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise< + { + embedding: number[]; + levenshtein_score: number; + }[] + > { + const sql = ` SELECT * FROM memories WHERE type = ? @@ -330,292 +338,303 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter { ORDER BY vec_distance_L2(${opts.query_field_name}, ?) ASC LIMIT ? `; - const memories = this.db.prepare(sql).all( - opts.query_table_name, - new Float32Array(opts.query_input.split(",").map(Number)), // Convert string to Float32Array - opts.query_input, - new Float32Array(opts.query_input.split(",").map(Number)) - ) as Memory[]; - - return memories.map((memory) => ({ - embedding: Array.from( - new Float32Array(memory.embedding as unknown as Buffer) - ), // Convert Buffer to number[] - levenshtein_score: 0 - })); - } - - async updateGoalStatus(params: { - goalId: UUID; - status: GoalStatus; - }): Promise { - const sql = "UPDATE goals SET status = ? WHERE id = ?"; - this.db.prepare(sql).run(params.status, params.goalId); - } - - async log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise { - const sql = - "INSERT INTO logs (body, userId, roomId, type) VALUES (?, ?, ?, ?)"; - this.db - .prepare(sql) - .run( - JSON.stringify(params.body), - params.userId, - params.roomId, - params.type - ); - } - - async getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise { - if (!params.tableName) { - throw new Error("tableName is required"); - } - if (!params.roomId) { - throw new Error("roomId is required"); - } - let sql = `SELECT * FROM memories WHERE type = ? AND roomId = ?`; - - const queryParams = [params.tableName, params.roomId] as any[]; - - if (params.unique) { - sql += " AND `unique` = 1"; - } - - if (params.userIds && params.userIds.length > 0) { - sql += ` AND userId IN (${params.userIds.map(() => "?").join(",")})`; - queryParams.push(...params.userIds); - } - - if (params.start) { - sql += ` AND createdAt >= ?`; - queryParams.push(params.start); - } - - if (params.end) { - sql += ` AND createdAt <= ?`; - queryParams.push(params.end); - } - - sql += " ORDER BY createdAt DESC"; - - if (params.count) { - sql += " LIMIT ?"; - queryParams.push(params.count); - } - - const memories = this.db.prepare(sql).all(...queryParams) as Memory[]; - - return memories.map((memory) => ({ - ...memory, - createdAt: - typeof memory.createdAt === "string" - ? Date.parse(memory.createdAt as string) - : memory.createdAt, - content: JSON.parse(memory.content as unknown as string) - })); - } - - async removeMemory(memoryId: UUID, tableName: string): Promise { - const sql = `DELETE FROM memories WHERE type = ? AND id = ?`; - this.db.prepare(sql).run(tableName, memoryId); - } - - async removeAllMemories(roomId: UUID, tableName: string): Promise { - const sql = `DELETE FROM memories WHERE type = ? AND roomId = ?`; - this.db.prepare(sql).run(tableName, roomId); - } - - async countMemories( - roomId: UUID, - unique = true, - tableName = "" - ): Promise { - if (!tableName) { - throw new Error("tableName is required"); - } - - let sql = `SELECT COUNT(*) as count FROM memories WHERE type = ? AND roomId = ?`; - const queryParams = [tableName, roomId] as string[]; - - if (unique) { - sql += " AND `unique` = 1"; - } - - return (this.db.prepare(sql).get(...queryParams) as { count: number }) - .count; - } - - async getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise { - let sql = "SELECT * FROM goals WHERE roomId = ?"; - const queryParams = [params.roomId]; - - if (params.userId) { - sql += " AND userId = ?"; - queryParams.push(params.userId); - } - - if (params.onlyInProgress) { - sql += " AND status = 'IN_PROGRESS'"; - } - - if (params.count) { - sql += " LIMIT ?"; - // @ts-expect-error - queryParams is an array of strings - queryParams.push(params.count.toString()); - } - - const goals = this.db.prepare(sql).all(...queryParams) as Goal[]; - return goals.map((goal) => ({ - ...goal, - objectives: - typeof goal.objectives === "string" - ? JSON.parse(goal.objectives) - : goal.objectives - })); - } - - async updateGoal(goal: Goal): Promise { - const sql = - "UPDATE goals SET name = ?, status = ?, objectives = ? WHERE id = ?"; - this.db - .prepare(sql) - .run(goal.name, goal.status, JSON.stringify(goal.objectives), goal.id); - } - - async createGoal(goal: Goal): Promise { - const sql = - "INSERT INTO goals (id, roomId, userId, name, status, objectives) VALUES (?, ?, ?, ?, ?, ?)"; - this.db - .prepare(sql) - .run( - goal.id ?? v4(), - goal.roomId, - goal.userId, - goal.name, - goal.status, - JSON.stringify(goal.objectives) - ); - } - - async removeGoal(goalId: UUID): Promise { - const sql = "DELETE FROM goals WHERE id = ?"; - this.db.prepare(sql).run(goalId); - } - - async removeAllGoals(roomId: UUID): Promise { - const sql = "DELETE FROM goals WHERE roomId = ?"; - this.db.prepare(sql).run(roomId); - } - - async createRoom(roomId?: UUID): Promise { - roomId = roomId || (v4() as UUID); - try { - const sql = "INSERT INTO rooms (id) VALUES (?)"; - this.db.prepare(sql).run(roomId ?? (v4() as UUID)); - } catch (error) { - console.log("Error creating room", error); - } - return roomId as UUID; - } - - async removeRoom(roomId: UUID): Promise { - const sql = "DELETE FROM rooms WHERE id = ?"; - this.db.prepare(sql).run(roomId); - } - - async getRoomsForParticipant(userId: UUID): Promise { - const sql = "SELECT roomId FROM participants WHERE userId = ?"; - const rows = this.db.prepare(sql).all(userId) as { roomId: string }[]; - return rows.map((row) => row.roomId as UUID); - } - - async getRoomsForParticipants(userIds: UUID[]): Promise { - // Assuming userIds is an array of UUID strings, prepare a list of placeholders - const placeholders = userIds.map(() => "?").join(", "); - // Construct the SQL query with the correct number of placeholders - const sql = `SELECT DISTINCT roomId FROM participants WHERE userId IN (${placeholders})`; - // Execute the query with the userIds array spread into arguments - const rows = this.db.prepare(sql).all(...userIds) as { roomId: string }[]; - // Map and return the roomId values as UUIDs - return rows.map((row) => row.roomId as UUID); - } - - async addParticipant(userId: UUID, roomId: UUID): Promise { - try { - const sql = - "INSERT INTO participants (id, userId, roomId) VALUES (?, ?, ?)"; - this.db.prepare(sql).run(v4(), userId, roomId); - return true; - } catch (error) { - console.log("Error adding participant", error); - return false; - } - } - - async removeParticipant(userId: UUID, roomId: UUID): Promise { - try { - const sql = "DELETE FROM participants WHERE userId = ? AND roomId = ?"; - this.db.prepare(sql).run(userId, roomId); - return true; - } catch (error) { - console.log("Error removing participant", error); - return false; - } - } - - async createRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - if (!params.userA || !params.userB) { - throw new Error("userA and userB are required"); - } - const sql = - "INSERT INTO relationships (id, userA, userB, userId) VALUES (?, ?, ?, ?)"; - this.db.prepare(sql).run(v4(), params.userA, params.userB, params.userA); - return true; - } - - async getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - const sql = - "SELECT * FROM relationships WHERE (userA = ? AND userB = ?) OR (userA = ? AND userB = ?)"; - return ( - (this.db - .prepare(sql) - .get( - params.userA, - params.userB, - params.userB, - params.userA - ) as Relationship) || null - ); - } - - async getRelationships(params: { userId: UUID }): Promise { - const sql = "SELECT * FROM relationships WHERE (userA = ? OR userB = ?)"; - return this.db - .prepare(sql) - .all(params.userId, params.userId) as Relationship[]; - } + const memories = this.db.prepare(sql).all( + opts.query_table_name, + new Float32Array(opts.query_input.split(",").map(Number)), // Convert string to Float32Array + opts.query_input, + new Float32Array(opts.query_input.split(",").map(Number)) + ) as Memory[]; + + return memories.map((memory) => ({ + embedding: Array.from( + new Float32Array(memory.embedding as unknown as Buffer) + ), // Convert Buffer to number[] + levenshtein_score: 0, + })); + } + + async updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise { + const sql = "UPDATE goals SET status = ? WHERE id = ?"; + this.db.prepare(sql).run(params.status, params.goalId); + } + + async log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise { + const sql = + "INSERT INTO logs (body, userId, roomId, type) VALUES (?, ?, ?, ?)"; + this.db + .prepare(sql) + .run( + JSON.stringify(params.body), + params.userId, + params.roomId, + params.type + ); + } + + async getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise { + if (!params.tableName) { + throw new Error("tableName is required"); + } + if (!params.roomId) { + throw new Error("roomId is required"); + } + let sql = `SELECT * FROM memories WHERE type = ? AND roomId = ?`; + + const queryParams = [params.tableName, params.roomId] as any[]; + + if (params.unique) { + sql += " AND `unique` = 1"; + } + + if (params.userIds && params.userIds.length > 0) { + sql += ` AND userId IN (${params.userIds.map(() => "?").join(",")})`; + queryParams.push(...params.userIds); + } + + if (params.start) { + sql += ` AND createdAt >= ?`; + queryParams.push(params.start); + } + + if (params.end) { + sql += ` AND createdAt <= ?`; + queryParams.push(params.end); + } + + sql += " ORDER BY createdAt DESC"; + + if (params.count) { + sql += " LIMIT ?"; + queryParams.push(params.count); + } + + const memories = this.db.prepare(sql).all(...queryParams) as Memory[]; + + return memories.map((memory) => ({ + ...memory, + createdAt: + typeof memory.createdAt === "string" + ? Date.parse(memory.createdAt as string) + : memory.createdAt, + content: JSON.parse(memory.content as unknown as string), + })); + } + + async removeMemory(memoryId: UUID, tableName: string): Promise { + const sql = `DELETE FROM memories WHERE type = ? AND id = ?`; + this.db.prepare(sql).run(tableName, memoryId); + } + + async removeAllMemories(roomId: UUID, tableName: string): Promise { + const sql = `DELETE FROM memories WHERE type = ? AND roomId = ?`; + this.db.prepare(sql).run(tableName, roomId); + } + + async countMemories( + roomId: UUID, + unique = true, + tableName = "" + ): Promise { + if (!tableName) { + throw new Error("tableName is required"); + } + + let sql = `SELECT COUNT(*) as count FROM memories WHERE type = ? AND roomId = ?`; + const queryParams = [tableName, roomId] as string[]; + + if (unique) { + sql += " AND `unique` = 1"; + } + + return (this.db.prepare(sql).get(...queryParams) as { count: number }) + .count; + } + + async getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise { + let sql = "SELECT * FROM goals WHERE roomId = ?"; + const queryParams = [params.roomId]; + + if (params.userId) { + sql += " AND userId = ?"; + queryParams.push(params.userId); + } + + if (params.onlyInProgress) { + sql += " AND status = 'IN_PROGRESS'"; + } + + if (params.count) { + sql += " LIMIT ?"; + // @ts-expect-error - queryParams is an array of strings + queryParams.push(params.count.toString()); + } + + const goals = this.db.prepare(sql).all(...queryParams) as Goal[]; + return goals.map((goal) => ({ + ...goal, + objectives: + typeof goal.objectives === "string" + ? JSON.parse(goal.objectives) + : goal.objectives, + })); + } + + async updateGoal(goal: Goal): Promise { + const sql = + "UPDATE goals SET name = ?, status = ?, objectives = ? WHERE id = ?"; + this.db + .prepare(sql) + .run( + goal.name, + goal.status, + JSON.stringify(goal.objectives), + goal.id + ); + } + + async createGoal(goal: Goal): Promise { + const sql = + "INSERT INTO goals (id, roomId, userId, name, status, objectives) VALUES (?, ?, ?, ?, ?, ?)"; + this.db + .prepare(sql) + .run( + goal.id ?? v4(), + goal.roomId, + goal.userId, + goal.name, + goal.status, + JSON.stringify(goal.objectives) + ); + } + + async removeGoal(goalId: UUID): Promise { + const sql = "DELETE FROM goals WHERE id = ?"; + this.db.prepare(sql).run(goalId); + } + + async removeAllGoals(roomId: UUID): Promise { + const sql = "DELETE FROM goals WHERE roomId = ?"; + this.db.prepare(sql).run(roomId); + } + + async createRoom(roomId?: UUID): Promise { + roomId = roomId || (v4() as UUID); + try { + const sql = "INSERT INTO rooms (id) VALUES (?)"; + this.db.prepare(sql).run(roomId ?? (v4() as UUID)); + } catch (error) { + console.log("Error creating room", error); + } + return roomId as UUID; + } + + async removeRoom(roomId: UUID): Promise { + const sql = "DELETE FROM rooms WHERE id = ?"; + this.db.prepare(sql).run(roomId); + } + + async getRoomsForParticipant(userId: UUID): Promise { + const sql = "SELECT roomId FROM participants WHERE userId = ?"; + const rows = this.db.prepare(sql).all(userId) as { roomId: string }[]; + return rows.map((row) => row.roomId as UUID); + } + + async getRoomsForParticipants(userIds: UUID[]): Promise { + // Assuming userIds is an array of UUID strings, prepare a list of placeholders + const placeholders = userIds.map(() => "?").join(", "); + // Construct the SQL query with the correct number of placeholders + const sql = `SELECT DISTINCT roomId FROM participants WHERE userId IN (${placeholders})`; + // Execute the query with the userIds array spread into arguments + const rows = this.db.prepare(sql).all(...userIds) as { + roomId: string; + }[]; + // Map and return the roomId values as UUIDs + return rows.map((row) => row.roomId as UUID); + } + + async addParticipant(userId: UUID, roomId: UUID): Promise { + try { + const sql = + "INSERT INTO participants (id, userId, roomId) VALUES (?, ?, ?)"; + this.db.prepare(sql).run(v4(), userId, roomId); + return true; + } catch (error) { + console.log("Error adding participant", error); + return false; + } + } + + async removeParticipant(userId: UUID, roomId: UUID): Promise { + try { + const sql = + "DELETE FROM participants WHERE userId = ? AND roomId = ?"; + this.db.prepare(sql).run(userId, roomId); + return true; + } catch (error) { + console.log("Error removing participant", error); + return false; + } + } + + async createRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + if (!params.userA || !params.userB) { + throw new Error("userA and userB are required"); + } + const sql = + "INSERT INTO relationships (id, userA, userB, userId) VALUES (?, ?, ?, ?)"; + this.db + .prepare(sql) + .run(v4(), params.userA, params.userB, params.userA); + return true; + } + + async getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + const sql = + "SELECT * FROM relationships WHERE (userA = ? AND userB = ?) OR (userA = ? AND userB = ?)"; + return ( + (this.db + .prepare(sql) + .get( + params.userA, + params.userB, + params.userB, + params.userA + ) as Relationship) || null + ); + } + + async getRelationships(params: { userId: UUID }): Promise { + const sql = + "SELECT * FROM relationships WHERE (userA = ? OR userB = ?)"; + return this.db + .prepare(sql) + .all(params.userId, params.userId) as Relationship[]; + } } diff --git a/core/src/adapters/sqlite/sqlite_vec.ts b/core/src/adapters/sqlite/sqlite_vec.ts index 23f955b63e..57429ae188 100644 --- a/core/src/adapters/sqlite/sqlite_vec.ts +++ b/core/src/adapters/sqlite/sqlite_vec.ts @@ -3,19 +3,19 @@ import { Database } from "better-sqlite3"; // Loads the sqlite-vec extensions into the provided SQLite database export function loadVecExtensions(db: Database): void { - try { - // Load sqlite-vec extensions - sqliteVec.load(db); - console.log("sqlite-vec extensions loaded successfully."); - } catch (error) { - console.error("Failed to load sqlite-vec extensions:", error); - throw error; - } + try { + // Load sqlite-vec extensions + sqliteVec.load(db); + console.log("sqlite-vec extensions loaded successfully."); + } catch (error) { + console.error("Failed to load sqlite-vec extensions:", error); + throw error; + } } /** * @param db - An instance of better - sqlite3 Database */ export function load(db: Database): void { - loadVecExtensions(db); + loadVecExtensions(db); } diff --git a/core/src/adapters/sqljs.ts b/core/src/adapters/sqljs.ts index a860c09734..fee2f99d89 100644 --- a/core/src/adapters/sqljs.ts +++ b/core/src/adapters/sqljs.ts @@ -2,732 +2,734 @@ import { v4 } from "uuid"; import { DatabaseAdapter } from "../core/database.ts"; import { - Account, - Actor, - GoalStatus, - type Goal, - type Memory, - type Relationship, - type UUID, - Participant, + Account, + Actor, + GoalStatus, + type Goal, + type Memory, + type Relationship, + type UUID, + Participant, } from "../core/types.ts"; import { sqliteTables } from "./sqlite/sqliteTables.ts"; import { Database } from "./sqljs/types.ts"; export class SqlJsDatabaseAdapter extends DatabaseAdapter { - async getRoom(roomId: UUID): Promise { - const sql = "SELECT id FROM rooms WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([roomId]); - const room = stmt.getAsObject() as { id: string } | undefined; - stmt.free(); - return room ? (room.id as UUID) : null; - } - - async getParticipantsForAccount(userId: UUID): Promise { - const sql = ` + async getRoom(roomId: UUID): Promise { + const sql = "SELECT id FROM rooms WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([roomId]); + const room = stmt.getAsObject() as { id: string } | undefined; + stmt.free(); + return room ? (room.id as UUID) : null; + } + + async getParticipantsForAccount(userId: UUID): Promise { + const sql = ` SELECT p.id, p.userId, p.roomId, p.last_message_read FROM participants p WHERE p.userId = ? `; - const stmt = this.db.prepare(sql); - stmt.bind([userId]); - const participants: Participant[] = []; - while (stmt.step()) { - const participant = stmt.getAsObject() as unknown as Participant; - participants.push(participant); - } - stmt.free(); - return participants; - } - - async getParticipantUserState( - roomId: UUID, - userId: UUID, - ): Promise<"FOLLOWED" | "MUTED" | null> { - const sql = - "SELECT userState FROM participants WHERE roomId = ? AND userId = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([roomId, userId]); - const result = stmt.getAsObject() as { - userState: "FOLLOWED" | "MUTED" | null; - }; - stmt.free(); - return result.userState ?? null; - } - - async getMemoriesByRoomIds(params: { - roomIds: UUID[]; - tableName: string; - }): Promise { - const placeholders = params.roomIds.map(() => "?").join(", "); - const sql = `SELECT * FROM memories WHERE type = ? AND roomId IN (${placeholders})`; - const stmt = this.db.prepare(sql); - const queryParams = [params.tableName, ...params.roomIds]; - stmt.bind(queryParams); - - const memories: Memory[] = []; - while (stmt.step()) { - const memory = stmt.getAsObject() as unknown as Memory; - memories.push({ - ...memory, - content: JSON.parse(memory.content as unknown as string), - }); - } - stmt.free(); - return memories; - } - - async setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null, - ): Promise { - const sql = - "UPDATE participants SET userState = ? WHERE roomId = ? AND userId = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([state, roomId, userId]); - stmt.step(); - stmt.free(); - } - - async getParticipantsForRoom(roomId: UUID): Promise { - const sql = "SELECT userId FROM participants WHERE roomId = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([roomId]); - const userIds: UUID[] = []; - while (stmt.step()) { - const row = stmt.getAsObject() as { userId: string }; - userIds.push(row.userId as UUID); - } - stmt.free(); - return userIds; - } - - constructor(db: Database) { - super(); - this.db = db; - - // Check if the 'accounts' table exists as a representative table - const tableExists = this.db.exec( - "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts'", - )[0]; - - if (!tableExists) { - // If the 'accounts' table doesn't exist, create all the tables - this.db.exec(sqliteTables); - } - } - - async getAccountById(userId: UUID): Promise { - const sql = "SELECT * FROM accounts WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([userId]); - const account = stmt.getAsObject() as unknown as Account | undefined; - - if (account && typeof account.details === "string") { - account.details = JSON.parse(account.details); - } - - stmt.free(); - return account || null; - } - - async createAccount(account: Account): Promise { - try { - const sql = ` + const stmt = this.db.prepare(sql); + stmt.bind([userId]); + const participants: Participant[] = []; + while (stmt.step()) { + const participant = stmt.getAsObject() as unknown as Participant; + participants.push(participant); + } + stmt.free(); + return participants; + } + + async getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null> { + const sql = + "SELECT userState FROM participants WHERE roomId = ? AND userId = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([roomId, userId]); + const result = stmt.getAsObject() as { + userState: "FOLLOWED" | "MUTED" | null; + }; + stmt.free(); + return result.userState ?? null; + } + + async getMemoriesByRoomIds(params: { + roomIds: UUID[]; + tableName: string; + }): Promise { + const placeholders = params.roomIds.map(() => "?").join(", "); + const sql = `SELECT * FROM memories WHERE type = ? AND roomId IN (${placeholders})`; + const stmt = this.db.prepare(sql); + const queryParams = [params.tableName, ...params.roomIds]; + stmt.bind(queryParams); + + const memories: Memory[] = []; + while (stmt.step()) { + const memory = stmt.getAsObject() as unknown as Memory; + memories.push({ + ...memory, + content: JSON.parse(memory.content as unknown as string), + }); + } + stmt.free(); + return memories; + } + + async setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise { + const sql = + "UPDATE participants SET userState = ? WHERE roomId = ? AND userId = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([state, roomId, userId]); + stmt.step(); + stmt.free(); + } + + async getParticipantsForRoom(roomId: UUID): Promise { + const sql = "SELECT userId FROM participants WHERE roomId = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([roomId]); + const userIds: UUID[] = []; + while (stmt.step()) { + const row = stmt.getAsObject() as { userId: string }; + userIds.push(row.userId as UUID); + } + stmt.free(); + return userIds; + } + + constructor(db: Database) { + super(); + this.db = db; + + // Check if the 'accounts' table exists as a representative table + const tableExists = this.db.exec( + "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts'" + )[0]; + + if (!tableExists) { + // If the 'accounts' table doesn't exist, create all the tables + this.db.exec(sqliteTables); + } + } + + async getAccountById(userId: UUID): Promise { + const sql = "SELECT * FROM accounts WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([userId]); + const account = stmt.getAsObject() as unknown as Account | undefined; + + if (account && typeof account.details === "string") { + account.details = JSON.parse(account.details); + } + + stmt.free(); + return account || null; + } + + async createAccount(account: Account): Promise { + try { + const sql = ` INSERT INTO accounts (id, name, username, email, avatarUrl, details) VALUES (?, ?, ?, ?, ?, ?) `; - const stmt = this.db.prepare(sql); - stmt.run([ - account.id ?? v4(), - account.name, - account.username || "", - account.email || "", - account.avatarUrl || "", - JSON.stringify(account.details), - ]); - stmt.free(); - return true; - } catch (error) { - console.log("Error creating account", error); - return false; - } - } - - async getActorById(params: { roomId: UUID }): Promise { - const sql = ` + const stmt = this.db.prepare(sql); + stmt.run([ + account.id ?? v4(), + account.name, + account.username || "", + account.email || "", + account.avatarUrl || "", + JSON.stringify(account.details), + ]); + stmt.free(); + return true; + } catch (error) { + console.log("Error creating account", error); + return false; + } + } + + async getActorById(params: { roomId: UUID }): Promise { + const sql = ` SELECT a.id, a.name, a.username, a.details FROM participants p LEFT JOIN accounts a ON p.userId = a.id WHERE p.roomId = ? `; - const stmt = this.db.prepare(sql); - stmt.bind([params.roomId]); - const rows: Actor[] = []; - while (stmt.step()) { - const row = stmt.getAsObject() as unknown as Actor; - rows.push({ - ...row, - details: - typeof row.details === "string" - ? JSON.parse(row.details) - : row.details, - }); - } - stmt.free(); - return rows; - } - - async getActorDetails(params: { roomId: UUID }): Promise { - const sql = ` + const stmt = this.db.prepare(sql); + stmt.bind([params.roomId]); + const rows: Actor[] = []; + while (stmt.step()) { + const row = stmt.getAsObject() as unknown as Actor; + rows.push({ + ...row, + details: + typeof row.details === "string" + ? JSON.parse(row.details) + : row.details, + }); + } + stmt.free(); + return rows; + } + + async getActorDetails(params: { roomId: UUID }): Promise { + const sql = ` SELECT a.id, a.name, a.username, a.details FROM participants p LEFT JOIN accounts a ON p.userId = a.id WHERE p.roomId = ? `; - const stmt = this.db.prepare(sql); - stmt.bind([params.roomId]); - const rows: Actor[] = []; - while (stmt.step()) { - const row = stmt.getAsObject() as unknown as Actor; - rows.push({ - ...row, - details: - typeof row.details === "string" - ? JSON.parse(row.details) - : row.details, - }); - } - stmt.free(); - return rows; - } - - async getMemoryById(id: UUID): Promise { - const sql = "SELECT * FROM memories WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([id]); - const memory = stmt.getAsObject() as unknown as Memory | undefined; - stmt.free(); - return memory || null; - } - - async createMemory(memory: Memory, tableName: string): Promise { - let isUnique = true; - if (memory.embedding) { - // Check if a similar memory already exists - const similarMemories = await this.searchMemoriesByEmbedding( - memory.embedding, - { - tableName, - roomId: memory.roomId, - match_threshold: 0.95, // 5% similarity threshold - count: 1, - }, - ); - - isUnique = similarMemories.length === 0; - } - - // Insert the memory with the appropriate 'unique' value - const sql = `INSERT INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; - const stmt = this.db.prepare(sql); - - const createdAt = memory.createdAt ?? Date.now(); - - stmt.run([ - memory.id ?? v4(), - tableName, - JSON.stringify(memory.content), - JSON.stringify(memory.embedding), - memory.userId, - memory.roomId, - isUnique ? 1 : 0, - createdAt, - ]); - stmt.free(); - } - - async searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise { - let sql = - ` + const stmt = this.db.prepare(sql); + stmt.bind([params.roomId]); + const rows: Actor[] = []; + while (stmt.step()) { + const row = stmt.getAsObject() as unknown as Actor; + rows.push({ + ...row, + details: + typeof row.details === "string" + ? JSON.parse(row.details) + : row.details, + }); + } + stmt.free(); + return rows; + } + + async getMemoryById(id: UUID): Promise { + const sql = "SELECT * FROM memories WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([id]); + const memory = stmt.getAsObject() as unknown as Memory | undefined; + stmt.free(); + return memory || null; + } + + async createMemory(memory: Memory, tableName: string): Promise { + let isUnique = true; + if (memory.embedding) { + // Check if a similar memory already exists + const similarMemories = await this.searchMemoriesByEmbedding( + memory.embedding, + { + tableName, + roomId: memory.roomId, + match_threshold: 0.95, // 5% similarity threshold + count: 1, + } + ); + + isUnique = similarMemories.length === 0; + } + + // Insert the memory with the appropriate 'unique' value + const sql = `INSERT INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; + const stmt = this.db.prepare(sql); + + const createdAt = memory.createdAt ?? Date.now(); + + stmt.run([ + memory.id ?? v4(), + tableName, + JSON.stringify(memory.content), + JSON.stringify(memory.embedding), + memory.userId, + memory.roomId, + isUnique ? 1 : 0, + createdAt, + ]); + stmt.free(); + } + + async searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise { + let sql = + ` SELECT *` + - // TODO: Uncomment when we compile sql.js with vss - // `, (1 - vss_distance_l2(embedding, ?)) AS similarity` + - ` FROM memories + // TODO: Uncomment when we compile sql.js with vss + // `, (1 - vss_distance_l2(embedding, ?)) AS similarity` + + ` FROM memories WHERE type = ? AND roomId = ?`; - if (params.unique) { - sql += " AND `unique` = 1"; - } - // TODO: Uncomment when we compile sql.js with vss - // sql += ` ORDER BY similarity DESC LIMIT ?`; - const stmt = this.db.prepare(sql); - stmt.bind([ - // JSON.stringify(params.embedding), - params.tableName, - params.roomId, - // params.match_count, - ]); - const memories: (Memory & { similarity: number })[] = []; - while (stmt.step()) { - const memory = stmt.getAsObject() as unknown as Memory & { - similarity: number; - }; - memories.push({ - ...memory, - content: JSON.parse(memory.content as unknown as string), - }); - } - stmt.free(); - return memories; - } - - async searchMemoriesByEmbedding( - _embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - }, - ): Promise { - let sql = - `SELECT *` + - // TODO: Uncomment when we compile sql.js with vss - // `, (1 - vss_distance_l2(embedding, ?)) AS similarity`+ - ` FROM memories + if (params.unique) { + sql += " AND `unique` = 1"; + } + // TODO: Uncomment when we compile sql.js with vss + // sql += ` ORDER BY similarity DESC LIMIT ?`; + const stmt = this.db.prepare(sql); + stmt.bind([ + // JSON.stringify(params.embedding), + params.tableName, + params.roomId, + // params.match_count, + ]); + const memories: (Memory & { similarity: number })[] = []; + while (stmt.step()) { + const memory = stmt.getAsObject() as unknown as Memory & { + similarity: number; + }; + memories.push({ + ...memory, + content: JSON.parse(memory.content as unknown as string), + }); + } + stmt.free(); + return memories; + } + + async searchMemoriesByEmbedding( + _embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise { + let sql = + `SELECT *` + + // TODO: Uncomment when we compile sql.js with vss + // `, (1 - vss_distance_l2(embedding, ?)) AS similarity`+ + ` FROM memories WHERE type = ?`; - if (params.unique) { - sql += " AND `unique` = 1"; - } - if (params.roomId) { - sql += " AND roomId = ?"; - } - // TODO: Uncomment when we compile sql.js with vss - // sql += ` ORDER BY similarity DESC`; - - if (params.count) { - sql += " LIMIT ?"; - } - - const stmt = this.db.prepare(sql); - const bindings = [ - // JSON.stringify(embedding), - params.tableName, - ]; - if (params.roomId) { - bindings.push(params.roomId); - } - if (params.count) { - bindings.push(params.count.toString()); - } - - stmt.bind(bindings); - const memories: (Memory & { similarity: number })[] = []; - while (stmt.step()) { - const memory = stmt.getAsObject() as unknown as Memory & { - similarity: number; - }; - memories.push({ - ...memory, - content: JSON.parse(memory.content as unknown as string), - }); - } - stmt.free(); - return memories; - } - - async getCachedEmbeddings(opts: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise< - { - embedding: number[]; - levenshtein_score: number; - }[] - > { - const sql = - ` + if (params.unique) { + sql += " AND `unique` = 1"; + } + if (params.roomId) { + sql += " AND roomId = ?"; + } + // TODO: Uncomment when we compile sql.js with vss + // sql += ` ORDER BY similarity DESC`; + + if (params.count) { + sql += " LIMIT ?"; + } + + const stmt = this.db.prepare(sql); + const bindings = [ + // JSON.stringify(embedding), + params.tableName, + ]; + if (params.roomId) { + bindings.push(params.roomId); + } + if (params.count) { + bindings.push(params.count.toString()); + } + + stmt.bind(bindings); + const memories: (Memory & { similarity: number })[] = []; + while (stmt.step()) { + const memory = stmt.getAsObject() as unknown as Memory & { + similarity: number; + }; + memories.push({ + ...memory, + content: JSON.parse(memory.content as unknown as string), + }); + } + stmt.free(); + return memories; + } + + async getCachedEmbeddings(opts: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise< + { + embedding: number[]; + levenshtein_score: number; + }[] + > { + const sql = + ` SELECT * FROM memories WHERE type = ?` + - // `AND vss_search(${opts.query_field_name}, ?) - // ORDER BY vss_search(${opts.query_field_name}, ?) DESC` + - ` LIMIT ? + // `AND vss_search(${opts.query_field_name}, ?) + // ORDER BY vss_search(${opts.query_field_name}, ?) DESC` + + ` LIMIT ? `; - const stmt = this.db.prepare(sql); - stmt.bind([ - opts.query_table_name, - // opts.query_input, - // opts.query_input, - opts.query_match_count, - ]); - const memories: Memory[] = []; - while (stmt.step()) { - const memory = stmt.getAsObject() as unknown as Memory; - memories.push(memory); - } - stmt.free(); - - return memories.map((memory) => ({ - ...memory, - createdAt: memory.createdAt ?? Date.now(), - embedding: JSON.parse(memory.embedding as unknown as string), - levenshtein_score: 0, - })); - } - - async updateGoalStatus(params: { - goalId: UUID; - status: GoalStatus; - }): Promise { - const sql = "UPDATE goals SET status = ? WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.run([params.status, params.goalId]); - stmt.free(); - } - - async log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise { - const sql = - "INSERT INTO logs (body, userId, roomId, type) VALUES (?, ?, ?, ?)"; - const stmt = this.db.prepare(sql); - stmt.run([ - JSON.stringify(params.body), - params.userId, - params.roomId, - params.type, - ]); - stmt.free(); - } - - async getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise { - if (!params.tableName) { - throw new Error("tableName is required"); - } - if (!params.roomId) { - throw new Error("roomId is required"); - } - let sql = `SELECT * FROM memories WHERE type = ? AND roomId = ?`; - - if (params.start) { - sql += ` AND createdAt >= ?`; - } - - if (params.end) { - sql += ` AND createdAt <= ?`; - } - - if (params.unique) { - sql += " AND `unique` = 1"; - } - - if (params.userIds && params.userIds.length > 0) { - sql += ` AND userId IN (${params.userIds.map(() => "?").join(",")})`; - } - - sql += " ORDER BY createdAt DESC"; - - if (params.count) { - sql += " LIMIT ?"; - } - - const stmt = this.db.prepare(sql); - stmt.bind([ - params.tableName, - params.roomId, - ...(params.start ? [params.start] : []), - ...(params.end ? [params.end] : []), - ...(params.userIds || []), - ...(params.count ? [params.count] : []), - ]); - const memories: Memory[] = []; - while (stmt.step()) { - const memory = stmt.getAsObject() as unknown as Memory; - memories.push({ - ...memory, - content: JSON.parse(memory.content as unknown as string), - }); - } - stmt.free(); - return memories; - } - - async removeMemory(memoryId: UUID, tableName: string): Promise { - const sql = `DELETE FROM memories WHERE type = ? AND id = ?`; - const stmt = this.db.prepare(sql); - stmt.run([tableName, memoryId]); - stmt.free(); - } - - async removeAllMemories(roomId: UUID, tableName: string): Promise { - const sql = `DELETE FROM memories WHERE type = ? AND roomId = ?`; - const stmt = this.db.prepare(sql); - stmt.run([tableName, roomId]); - stmt.free(); - } - - async countMemories( - roomId: UUID, - unique = true, - tableName = "", - ): Promise { - if (!tableName) { - throw new Error("tableName is required"); - } - - let sql = `SELECT COUNT(*) as count FROM memories WHERE type = ? AND roomId = ?`; - if (unique) { - sql += " AND `unique` = 1"; - } - - const stmt = this.db.prepare(sql); - stmt.bind([tableName, roomId]); - - let count = 0; - if (stmt.step()) { - const result = stmt.getAsObject() as { count: number }; - count = result.count; - } - - stmt.free(); - return count; - } - - async getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise { - let sql = "SELECT * FROM goals WHERE roomId = ?"; - const bindings: (string | number)[] = [params.roomId]; - - if (params.userId) { - sql += " AND userId = ?"; - bindings.push(params.userId); - } - - if (params.onlyInProgress) { - sql += " AND status = 'IN_PROGRESS'"; - } - - if (params.count) { - sql += " LIMIT ?"; - bindings.push(params.count.toString()); - } - - const stmt = this.db.prepare(sql); - stmt.bind(bindings); - const goals: Goal[] = []; - while (stmt.step()) { - const goal = stmt.getAsObject() as unknown as Goal; - goals.push({ - ...goal, - objectives: - typeof goal.objectives === "string" - ? JSON.parse(goal.objectives) - : goal.objectives, - }); - } - stmt.free(); - return goals; - } - - async updateGoal(goal: Goal): Promise { - const sql = - "UPDATE goals SET name = ?, status = ?, objectives = ? WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.run([ - goal.name, - goal.status, - JSON.stringify(goal.objectives), - goal.id as string, - ]); - stmt.free(); - } - - async createGoal(goal: Goal): Promise { - const sql = - "INSERT INTO goals (id, roomId, userId, name, status, objectives) VALUES (?, ?, ?, ?, ?, ?)"; - const stmt = this.db.prepare(sql); - stmt.run([ - goal.id ?? v4(), - goal.roomId, - goal.userId, - goal.name, - goal.status, - JSON.stringify(goal.objectives), - ]); - stmt.free(); - } - - async removeGoal(goalId: UUID): Promise { - const sql = "DELETE FROM goals WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.run([goalId]); - stmt.free(); - } - - async removeAllGoals(roomId: UUID): Promise { - const sql = "DELETE FROM goals WHERE roomId = ?"; - const stmt = this.db.prepare(sql); - stmt.run([roomId]); - stmt.free(); - } - - async createRoom(roomId?: UUID): Promise { - roomId = roomId || (v4() as UUID); - try { - const sql = "INSERT INTO rooms (id) VALUES (?)"; - const stmt = this.db.prepare(sql); - stmt.run([roomId ?? (v4() as UUID)]); - stmt.free(); - } catch (error) { - console.log("Error creating room", error); - } - return roomId as UUID; - } - - async removeRoom(roomId: UUID): Promise { - const sql = "DELETE FROM rooms WHERE id = ?"; - const stmt = this.db.prepare(sql); - stmt.run([roomId]); - stmt.free(); - } - - async getRoomsForParticipant(userId: UUID): Promise { - const sql = "SELECT roomId FROM participants WHERE userId = ?"; - const stmt = this.db.prepare(sql); - stmt.bind([userId]); - const rows: { roomId: string }[] = []; - while (stmt.step()) { - const row = stmt.getAsObject() as unknown as { roomId: string }; - rows.push(row); - } - stmt.free(); - return rows.map((row) => row.roomId as UUID); - } - - async getRoomsForParticipants(userIds: UUID[]): Promise { - // Assuming userIds is an array of UUID strings, prepare a list of placeholders - const placeholders = userIds.map(() => "?").join(", "); - // Construct the SQL query with the correct number of placeholders - const sql = `SELECT roomId FROM participants WHERE userId IN (${placeholders})`; - const stmt = this.db.prepare(sql); - // Execute the query with the userIds array spread into arguments - stmt.bind(userIds); - const rows: { roomId: string }[] = []; - while (stmt.step()) { - const row = stmt.getAsObject() as unknown as { roomId: string }; - rows.push(row); - } - stmt.free(); - // Map and return the roomId values as UUIDs - return rows.map((row) => row.roomId as UUID); - } - - async addParticipant(userId: UUID, roomId: UUID): Promise { - try { - const sql = - "INSERT INTO participants (id, userId, roomId) VALUES (?, ?, ?)"; - const stmt = this.db.prepare(sql); - stmt.run([v4(), userId, roomId]); - stmt.free(); - return true; - } catch (error) { - console.log("Error adding participant", error); - return false; - } - } - - async removeParticipant(userId: UUID, roomId: UUID): Promise { - try { - const sql = "DELETE FROM participants WHERE userId = ? AND roomId = ?"; - const stmt = this.db.prepare(sql); - stmt.run([userId, roomId]); - stmt.free(); - return true; - } catch (error) { - console.log("Error removing participant", error); - return false; - } - } - - async createRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - if (!params.userA || !params.userB) { - throw new Error("userA and userB are required"); - } - const sql = - "INSERT INTO relationships (id, userA, userB, userId) VALUES (?, ?, ?, ?)"; - const stmt = this.db.prepare(sql); - stmt.run([v4(), params.userA, params.userB, params.userA]); - stmt.free(); - return true; - } - - async getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - let relationship: Relationship | null = null; - try { - const sql = - "SELECT * FROM relationships WHERE (userA = ? AND userB = ?) OR (userA = ? AND userB = ?)"; - const stmt = this.db.prepare(sql); - stmt.bind([params.userA, params.userB, params.userB, params.userA]); - - if (stmt.step()) { - relationship = stmt.getAsObject() as unknown as Relationship; - } - stmt.free(); - } catch (error) { - console.log("Error fetching relationship", error); - } - return relationship; - } - - async getRelationships(params: { userId: UUID }): Promise { - const sql = "SELECT * FROM relationships WHERE (userA = ? OR userB = ?)"; - const stmt = this.db.prepare(sql); - stmt.bind([params.userId, params.userId]); - const relationships: Relationship[] = []; - while (stmt.step()) { - const relationship = stmt.getAsObject() as unknown as Relationship; - relationships.push(relationship); - } - stmt.free(); - return relationships; - } + const stmt = this.db.prepare(sql); + stmt.bind([ + opts.query_table_name, + // opts.query_input, + // opts.query_input, + opts.query_match_count, + ]); + const memories: Memory[] = []; + while (stmt.step()) { + const memory = stmt.getAsObject() as unknown as Memory; + memories.push(memory); + } + stmt.free(); + + return memories.map((memory) => ({ + ...memory, + createdAt: memory.createdAt ?? Date.now(), + embedding: JSON.parse(memory.embedding as unknown as string), + levenshtein_score: 0, + })); + } + + async updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise { + const sql = "UPDATE goals SET status = ? WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.run([params.status, params.goalId]); + stmt.free(); + } + + async log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise { + const sql = + "INSERT INTO logs (body, userId, roomId, type) VALUES (?, ?, ?, ?)"; + const stmt = this.db.prepare(sql); + stmt.run([ + JSON.stringify(params.body), + params.userId, + params.roomId, + params.type, + ]); + stmt.free(); + } + + async getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise { + if (!params.tableName) { + throw new Error("tableName is required"); + } + if (!params.roomId) { + throw new Error("roomId is required"); + } + let sql = `SELECT * FROM memories WHERE type = ? AND roomId = ?`; + + if (params.start) { + sql += ` AND createdAt >= ?`; + } + + if (params.end) { + sql += ` AND createdAt <= ?`; + } + + if (params.unique) { + sql += " AND `unique` = 1"; + } + + if (params.userIds && params.userIds.length > 0) { + sql += ` AND userId IN (${params.userIds.map(() => "?").join(",")})`; + } + + sql += " ORDER BY createdAt DESC"; + + if (params.count) { + sql += " LIMIT ?"; + } + + const stmt = this.db.prepare(sql); + stmt.bind([ + params.tableName, + params.roomId, + ...(params.start ? [params.start] : []), + ...(params.end ? [params.end] : []), + ...(params.userIds || []), + ...(params.count ? [params.count] : []), + ]); + const memories: Memory[] = []; + while (stmt.step()) { + const memory = stmt.getAsObject() as unknown as Memory; + memories.push({ + ...memory, + content: JSON.parse(memory.content as unknown as string), + }); + } + stmt.free(); + return memories; + } + + async removeMemory(memoryId: UUID, tableName: string): Promise { + const sql = `DELETE FROM memories WHERE type = ? AND id = ?`; + const stmt = this.db.prepare(sql); + stmt.run([tableName, memoryId]); + stmt.free(); + } + + async removeAllMemories(roomId: UUID, tableName: string): Promise { + const sql = `DELETE FROM memories WHERE type = ? AND roomId = ?`; + const stmt = this.db.prepare(sql); + stmt.run([tableName, roomId]); + stmt.free(); + } + + async countMemories( + roomId: UUID, + unique = true, + tableName = "" + ): Promise { + if (!tableName) { + throw new Error("tableName is required"); + } + + let sql = `SELECT COUNT(*) as count FROM memories WHERE type = ? AND roomId = ?`; + if (unique) { + sql += " AND `unique` = 1"; + } + + const stmt = this.db.prepare(sql); + stmt.bind([tableName, roomId]); + + let count = 0; + if (stmt.step()) { + const result = stmt.getAsObject() as { count: number }; + count = result.count; + } + + stmt.free(); + return count; + } + + async getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise { + let sql = "SELECT * FROM goals WHERE roomId = ?"; + const bindings: (string | number)[] = [params.roomId]; + + if (params.userId) { + sql += " AND userId = ?"; + bindings.push(params.userId); + } + + if (params.onlyInProgress) { + sql += " AND status = 'IN_PROGRESS'"; + } + + if (params.count) { + sql += " LIMIT ?"; + bindings.push(params.count.toString()); + } + + const stmt = this.db.prepare(sql); + stmt.bind(bindings); + const goals: Goal[] = []; + while (stmt.step()) { + const goal = stmt.getAsObject() as unknown as Goal; + goals.push({ + ...goal, + objectives: + typeof goal.objectives === "string" + ? JSON.parse(goal.objectives) + : goal.objectives, + }); + } + stmt.free(); + return goals; + } + + async updateGoal(goal: Goal): Promise { + const sql = + "UPDATE goals SET name = ?, status = ?, objectives = ? WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.run([ + goal.name, + goal.status, + JSON.stringify(goal.objectives), + goal.id as string, + ]); + stmt.free(); + } + + async createGoal(goal: Goal): Promise { + const sql = + "INSERT INTO goals (id, roomId, userId, name, status, objectives) VALUES (?, ?, ?, ?, ?, ?)"; + const stmt = this.db.prepare(sql); + stmt.run([ + goal.id ?? v4(), + goal.roomId, + goal.userId, + goal.name, + goal.status, + JSON.stringify(goal.objectives), + ]); + stmt.free(); + } + + async removeGoal(goalId: UUID): Promise { + const sql = "DELETE FROM goals WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.run([goalId]); + stmt.free(); + } + + async removeAllGoals(roomId: UUID): Promise { + const sql = "DELETE FROM goals WHERE roomId = ?"; + const stmt = this.db.prepare(sql); + stmt.run([roomId]); + stmt.free(); + } + + async createRoom(roomId?: UUID): Promise { + roomId = roomId || (v4() as UUID); + try { + const sql = "INSERT INTO rooms (id) VALUES (?)"; + const stmt = this.db.prepare(sql); + stmt.run([roomId ?? (v4() as UUID)]); + stmt.free(); + } catch (error) { + console.log("Error creating room", error); + } + return roomId as UUID; + } + + async removeRoom(roomId: UUID): Promise { + const sql = "DELETE FROM rooms WHERE id = ?"; + const stmt = this.db.prepare(sql); + stmt.run([roomId]); + stmt.free(); + } + + async getRoomsForParticipant(userId: UUID): Promise { + const sql = "SELECT roomId FROM participants WHERE userId = ?"; + const stmt = this.db.prepare(sql); + stmt.bind([userId]); + const rows: { roomId: string }[] = []; + while (stmt.step()) { + const row = stmt.getAsObject() as unknown as { roomId: string }; + rows.push(row); + } + stmt.free(); + return rows.map((row) => row.roomId as UUID); + } + + async getRoomsForParticipants(userIds: UUID[]): Promise { + // Assuming userIds is an array of UUID strings, prepare a list of placeholders + const placeholders = userIds.map(() => "?").join(", "); + // Construct the SQL query with the correct number of placeholders + const sql = `SELECT roomId FROM participants WHERE userId IN (${placeholders})`; + const stmt = this.db.prepare(sql); + // Execute the query with the userIds array spread into arguments + stmt.bind(userIds); + const rows: { roomId: string }[] = []; + while (stmt.step()) { + const row = stmt.getAsObject() as unknown as { roomId: string }; + rows.push(row); + } + stmt.free(); + // Map and return the roomId values as UUIDs + return rows.map((row) => row.roomId as UUID); + } + + async addParticipant(userId: UUID, roomId: UUID): Promise { + try { + const sql = + "INSERT INTO participants (id, userId, roomId) VALUES (?, ?, ?)"; + const stmt = this.db.prepare(sql); + stmt.run([v4(), userId, roomId]); + stmt.free(); + return true; + } catch (error) { + console.log("Error adding participant", error); + return false; + } + } + + async removeParticipant(userId: UUID, roomId: UUID): Promise { + try { + const sql = + "DELETE FROM participants WHERE userId = ? AND roomId = ?"; + const stmt = this.db.prepare(sql); + stmt.run([userId, roomId]); + stmt.free(); + return true; + } catch (error) { + console.log("Error removing participant", error); + return false; + } + } + + async createRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + if (!params.userA || !params.userB) { + throw new Error("userA and userB are required"); + } + const sql = + "INSERT INTO relationships (id, userA, userB, userId) VALUES (?, ?, ?, ?)"; + const stmt = this.db.prepare(sql); + stmt.run([v4(), params.userA, params.userB, params.userA]); + stmt.free(); + return true; + } + + async getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + let relationship: Relationship | null = null; + try { + const sql = + "SELECT * FROM relationships WHERE (userA = ? AND userB = ?) OR (userA = ? AND userB = ?)"; + const stmt = this.db.prepare(sql); + stmt.bind([params.userA, params.userB, params.userB, params.userA]); + + if (stmt.step()) { + relationship = stmt.getAsObject() as unknown as Relationship; + } + stmt.free(); + } catch (error) { + console.log("Error fetching relationship", error); + } + return relationship; + } + + async getRelationships(params: { userId: UUID }): Promise { + const sql = + "SELECT * FROM relationships WHERE (userA = ? OR userB = ?)"; + const stmt = this.db.prepare(sql); + stmt.bind([params.userId, params.userId]); + const relationships: Relationship[] = []; + while (stmt.step()) { + const relationship = stmt.getAsObject() as unknown as Relationship; + relationships.push(relationship); + } + stmt.free(); + return relationships; + } } diff --git a/core/src/adapters/sqljs/types.ts b/core/src/adapters/sqljs/types.ts index 528c9b05c4..efdf6aaa4e 100644 --- a/core/src/adapters/sqljs/types.ts +++ b/core/src/adapters/sqljs/types.ts @@ -3,208 +3,208 @@ type ParamsObject = Record; type ParamsCallback = (obj: ParamsObject) => void; type BindParams = SqlValue[] | ParamsObject | null; interface QueryExecResult { - columns: string[]; - values: SqlValue[][]; + columns: string[]; + values: SqlValue[][]; } declare class StatementIterator - implements Iterator, Iterable + implements Iterator, Iterable { - [Symbol.iterator](): Iterator; - getRemainingSQL(): string; - next(): StatementIteratorResult; + [Symbol.iterator](): Iterator; + getRemainingSQL(): string; + next(): StatementIteratorResult; } interface StatementIteratorResult { - /** `true` if there are no more available statements */ - done: boolean; - /** the next available Statement (as returned by `Database.prepare`) */ - value: Statement; + /** `true` if there are no more available statements */ + done: boolean; + /** the next available Statement (as returned by `Database.prepare`) */ + value: Statement; } declare class Statement { - /** - * Bind values to the parameters, after having reseted the statement. If - * values is null, do nothing and return true. - * - * SQL statements can have parameters, named '?', '?NNN', ':VVV', - * '@VVV', '$VVV', where NNN is a number and VVV a string. This function - * binds these parameters to the given values. - * - * Warning: ':', '@', and '$' are included in the parameters names - * - * ### Value types - * - * |Javascript type|SQLite type| - * |-|-| - * |number|REAL, INTEGER| - * |boolean|INTEGER| - * |string|TEXT| - * |Array, Uint8Array|BLOB| - * |null|NULL| - * @see [https://sql.js.org/documentation/Statement.html#["bind"]](https://sql.js.org/documentation/Statement.html#%5B%22bind%22%5D) - * - * @param values The values to bind - */ - bind(values?: BindParams): boolean; - - /** - * Free the memory used by the statement - * @see [https://sql.js.org/documentation/Statement.html#["free"]](https://sql.js.org/documentation/Statement.html#%5B%22free%22%5D) - */ - free(): boolean; - - /** - * Free the memory allocated during parameter binding - * @see [https://sql.js.org/documentation/Statement.html#["freemem"]](https://sql.js.org/documentation/Statement.html#%5B%22freemem%22%5D) - */ - freemem(): void; - - /** - * Get one row of results of a statement. If the first parameter is not - * provided, step must have been called before. - * @see [https://sql.js.org/documentation/Statement.html#["get"]](https://sql.js.org/documentation/Statement.html#%5B%22get%22%5D) - * - * @param params If set, the values will be bound to the statement - * before it is executed - */ - get(params?: BindParams): SqlValue[]; - - /** - * Get one row of result as a javascript object, associating column - * names with their value in the current row - * @see [https://sql.js.org/documentation/Statement.html#["getAsObject"]](https://sql.js.org/documentation/Statement.html#%5B%22getAsObject%22%5D) - * - * @param params If set, the values will be bound to the statement, and - * it will be executed - */ - getAsObject(params?: BindParams): ParamsObject; - - /** - * Get the list of column names of a row of result of a statement. - * @see [https://sql.js.org/documentation/Statement.html#["getColumnNames"]](https://sql.js.org/documentation/Statement.html#%5B%22getColumnNames%22%5D) - */ - getColumnNames(): string[]; - - /** - * Get the SQLite's normalized version of the SQL string used in - * preparing this statement. The meaning of "normalized" is not - * well-defined: see - * [the SQLite documentation](https://sqlite.org/c3ref/expanded_sql.html). - * @see [https://sql.js.org/documentation/Statement.html#["getNormalizedSQL"]](https://sql.js.org/documentation/Statement.html#%5B%22getNormalizedSQL%22%5D) - */ - getNormalizedSQL(): string; - - /** - * Get the SQL string used in preparing this statement. - * @see [https://sql.js.org/documentation/Statement.html#["getSQL"]](https://sql.js.org/documentation/Statement.html#%5B%22getSQL%22%5D) - */ - getSQL(): string; - - /** - * Reset a statement, so that it's parameters can be bound to new - * values. It also clears all previous bindings, freeing the memory used - * by bound parameters. - * @see [https://sql.js.org/documentation/Statement.html#["reset"]](https://sql.js.org/documentation/Statement.html#%5B%22reset%22%5D) - */ - reset(): void; - - /** - * Shorthand for bind + step + reset Bind the values, execute the - * statement, ignoring the rows it returns, and resets it - * @param values Value to bind to the statement - */ - run(values?: BindParams): void; - - /** - * Execute the statement, fetching the the next line of result, that can - * be retrieved with `Statement.get`. - * @see [https://sql.js.org/documentation/Statement.html#["step"]](https://sql.js.org/documentation/Statement.html#%5B%22step%22%5D) - */ - step(): boolean; + /** + * Bind values to the parameters, after having reseted the statement. If + * values is null, do nothing and return true. + * + * SQL statements can have parameters, named '?', '?NNN', ':VVV', + * '@VVV', '$VVV', where NNN is a number and VVV a string. This function + * binds these parameters to the given values. + * + * Warning: ':', '@', and '$' are included in the parameters names + * + * ### Value types + * + * |Javascript type|SQLite type| + * |-|-| + * |number|REAL, INTEGER| + * |boolean|INTEGER| + * |string|TEXT| + * |Array, Uint8Array|BLOB| + * |null|NULL| + * @see [https://sql.js.org/documentation/Statement.html#["bind"]](https://sql.js.org/documentation/Statement.html#%5B%22bind%22%5D) + * + * @param values The values to bind + */ + bind(values?: BindParams): boolean; + + /** + * Free the memory used by the statement + * @see [https://sql.js.org/documentation/Statement.html#["free"]](https://sql.js.org/documentation/Statement.html#%5B%22free%22%5D) + */ + free(): boolean; + + /** + * Free the memory allocated during parameter binding + * @see [https://sql.js.org/documentation/Statement.html#["freemem"]](https://sql.js.org/documentation/Statement.html#%5B%22freemem%22%5D) + */ + freemem(): void; + + /** + * Get one row of results of a statement. If the first parameter is not + * provided, step must have been called before. + * @see [https://sql.js.org/documentation/Statement.html#["get"]](https://sql.js.org/documentation/Statement.html#%5B%22get%22%5D) + * + * @param params If set, the values will be bound to the statement + * before it is executed + */ + get(params?: BindParams): SqlValue[]; + + /** + * Get one row of result as a javascript object, associating column + * names with their value in the current row + * @see [https://sql.js.org/documentation/Statement.html#["getAsObject"]](https://sql.js.org/documentation/Statement.html#%5B%22getAsObject%22%5D) + * + * @param params If set, the values will be bound to the statement, and + * it will be executed + */ + getAsObject(params?: BindParams): ParamsObject; + + /** + * Get the list of column names of a row of result of a statement. + * @see [https://sql.js.org/documentation/Statement.html#["getColumnNames"]](https://sql.js.org/documentation/Statement.html#%5B%22getColumnNames%22%5D) + */ + getColumnNames(): string[]; + + /** + * Get the SQLite's normalized version of the SQL string used in + * preparing this statement. The meaning of "normalized" is not + * well-defined: see + * [the SQLite documentation](https://sqlite.org/c3ref/expanded_sql.html). + * @see [https://sql.js.org/documentation/Statement.html#["getNormalizedSQL"]](https://sql.js.org/documentation/Statement.html#%5B%22getNormalizedSQL%22%5D) + */ + getNormalizedSQL(): string; + + /** + * Get the SQL string used in preparing this statement. + * @see [https://sql.js.org/documentation/Statement.html#["getSQL"]](https://sql.js.org/documentation/Statement.html#%5B%22getSQL%22%5D) + */ + getSQL(): string; + + /** + * Reset a statement, so that it's parameters can be bound to new + * values. It also clears all previous bindings, freeing the memory used + * by bound parameters. + * @see [https://sql.js.org/documentation/Statement.html#["reset"]](https://sql.js.org/documentation/Statement.html#%5B%22reset%22%5D) + */ + reset(): void; + + /** + * Shorthand for bind + step + reset Bind the values, execute the + * statement, ignoring the rows it returns, and resets it + * @param values Value to bind to the statement + */ + run(values?: BindParams): void; + + /** + * Execute the statement, fetching the the next line of result, that can + * be retrieved with `Statement.get`. + * @see [https://sql.js.org/documentation/Statement.html#["step"]](https://sql.js.org/documentation/Statement.html#%5B%22step%22%5D) + */ + step(): boolean; } export declare class Database { - constructor(data?: ArrayLike | Buffer | null); - - close(): void; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - create_function(name: string, func: (...args: any[]) => any): Database; - - each( - sql: string, - params: BindParams, - callback: ParamsCallback, - done: () => void, - ): Database; - each(sql: string, callback: ParamsCallback, done: () => void): Database; - - /** - * Execute an SQL query, and returns the result. - * - * This is a wrapper against `Database.prepare`, `Statement.bind`, `Statement.step`, `Statement.get`, and `Statement.free`. - * - * The result is an array of result elements. There are as many result elements as the number of statements in your sql string (statements are separated by a semicolon) - * @see [https://sql.js.org/documentation/Database.html#["exec"]](https://sql.js.org/documentation/Database.html#%5B%22exec%22%5D) - * - * @param sql a string containing some SQL text to execute - * @param params When the SQL statement contains placeholders, you can - * pass them in here. They will be bound to the statement before it is - * executed. If you use the params argument as an array, you **cannot** - * provide an sql string that contains several statements (separated by - * `;`). This limitation does not apply to params as an object. - */ - exec(sql: string, params?: BindParams): QueryExecResult[]; - - /** - * Exports the contents of the database to a binary array - * @see [https://sql.js.org/documentation/Database.html#["export"]](https://sql.js.org/documentation/Database.html#%5B%22export%22%5D) - */ - export(): Uint8Array; - - /** - * Returns the number of changed rows (modified, inserted or deleted) by - * the latest completed `INSERT`, `UPDATE` or `DELETE` statement on the - * database. Executing any other type of SQL statement does not modify - * the value returned by this function. - * @see [https://sql.js.org/documentation/Database.html#["getRowsModified"]](https://sql.js.org/documentation/Database.html#%5B%22getRowsModified%22%5D) - */ - getRowsModified(): number; - - /** - * Analyze a result code, return null if no error occured, and throw an - * error with a descriptive message otherwise - * @see [https://sql.js.org/documentation/Database.html#["handleError"]](https://sql.js.org/documentation/Database.html#%5B%22handleError%22%5D) - */ - handleError(): null | never; - - /** - * Iterate over multiple SQL statements in a SQL string. This function - * returns an iterator over Statement objects. You can use a `for..of` - * loop to execute the returned statements one by one. - * @see [https://sql.js.org/documentation/Database.html#["iterateStatements"]](https://sql.js.org/documentation/Database.html#%5B%22iterateStatements%22%5D) - * - * @param sql a string of SQL that can contain multiple statements - */ - iterateStatements(sql: string): StatementIterator; - - /** - * Prepare an SQL statement - * @see [https://sql.js.org/documentation/Database.html#["prepare"]](https://sql.js.org/documentation/Database.html#%5B%22prepare%22%5D) - * - * @param sql a string of SQL, that can contain placeholders (`?`, `:VVV`, `:AAA`, `@AAA`) - * @param params values to bind to placeholders - */ - prepare(sql: string, params?: BindParams): Statement; - - /** - * Execute an SQL query, ignoring the rows it returns. - * @see [https://sql.js.org/documentation/Database.html#["run"]](https://sql.js.org/documentation/Database.html#%5B%22run%22%5D) - * - * @param sql a string containing some SQL text to execute - * @param params When the SQL statement contains placeholders, you can - * pass them in here. They will be bound to the statement before it is - * executed. If you use the params argument as an array, you **cannot** - * provide an sql string that contains several statements (separated by - * `;`). This limitation does not apply to params as an object. - */ - run(sql: string, params?: BindParams): Database; + constructor(data?: ArrayLike | Buffer | null); + + close(): void; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + create_function(name: string, func: (...args: any[]) => any): Database; + + each( + sql: string, + params: BindParams, + callback: ParamsCallback, + done: () => void + ): Database; + each(sql: string, callback: ParamsCallback, done: () => void): Database; + + /** + * Execute an SQL query, and returns the result. + * + * This is a wrapper against `Database.prepare`, `Statement.bind`, `Statement.step`, `Statement.get`, and `Statement.free`. + * + * The result is an array of result elements. There are as many result elements as the number of statements in your sql string (statements are separated by a semicolon) + * @see [https://sql.js.org/documentation/Database.html#["exec"]](https://sql.js.org/documentation/Database.html#%5B%22exec%22%5D) + * + * @param sql a string containing some SQL text to execute + * @param params When the SQL statement contains placeholders, you can + * pass them in here. They will be bound to the statement before it is + * executed. If you use the params argument as an array, you **cannot** + * provide an sql string that contains several statements (separated by + * `;`). This limitation does not apply to params as an object. + */ + exec(sql: string, params?: BindParams): QueryExecResult[]; + + /** + * Exports the contents of the database to a binary array + * @see [https://sql.js.org/documentation/Database.html#["export"]](https://sql.js.org/documentation/Database.html#%5B%22export%22%5D) + */ + export(): Uint8Array; + + /** + * Returns the number of changed rows (modified, inserted or deleted) by + * the latest completed `INSERT`, `UPDATE` or `DELETE` statement on the + * database. Executing any other type of SQL statement does not modify + * the value returned by this function. + * @see [https://sql.js.org/documentation/Database.html#["getRowsModified"]](https://sql.js.org/documentation/Database.html#%5B%22getRowsModified%22%5D) + */ + getRowsModified(): number; + + /** + * Analyze a result code, return null if no error occured, and throw an + * error with a descriptive message otherwise + * @see [https://sql.js.org/documentation/Database.html#["handleError"]](https://sql.js.org/documentation/Database.html#%5B%22handleError%22%5D) + */ + handleError(): null | never; + + /** + * Iterate over multiple SQL statements in a SQL string. This function + * returns an iterator over Statement objects. You can use a `for..of` + * loop to execute the returned statements one by one. + * @see [https://sql.js.org/documentation/Database.html#["iterateStatements"]](https://sql.js.org/documentation/Database.html#%5B%22iterateStatements%22%5D) + * + * @param sql a string of SQL that can contain multiple statements + */ + iterateStatements(sql: string): StatementIterator; + + /** + * Prepare an SQL statement + * @see [https://sql.js.org/documentation/Database.html#["prepare"]](https://sql.js.org/documentation/Database.html#%5B%22prepare%22%5D) + * + * @param sql a string of SQL, that can contain placeholders (`?`, `:VVV`, `:AAA`, `@AAA`) + * @param params values to bind to placeholders + */ + prepare(sql: string, params?: BindParams): Statement; + + /** + * Execute an SQL query, ignoring the rows it returns. + * @see [https://sql.js.org/documentation/Database.html#["run"]](https://sql.js.org/documentation/Database.html#%5B%22run%22%5D) + * + * @param sql a string containing some SQL text to execute + * @param params When the SQL statement contains placeholders, you can + * pass them in here. They will be bound to the statement before it is + * executed. If you use the params argument as an array, you **cannot** + * provide an sql string that contains several statements (separated by + * `;`). This limitation does not apply to params as an object. + */ + run(sql: string, params?: BindParams): Database; } diff --git a/core/src/adapters/supabase.ts b/core/src/adapters/supabase.ts index 315fbc317f..89be39f7f8 100644 --- a/core/src/adapters/supabase.ts +++ b/core/src/adapters/supabase.ts @@ -1,651 +1,662 @@ import { createClient, type SupabaseClient } from "@supabase/supabase-js"; import { - type Memory, - type Goal, - type Relationship, - Actor, - GoalStatus, - Account, - type UUID, - Participant, - Room, + type Memory, + type Goal, + type Relationship, + Actor, + GoalStatus, + Account, + type UUID, + Participant, + Room, } from "../core/types.ts"; import { DatabaseAdapter } from "../core/database.ts"; import { v4 as uuid } from "uuid"; export class SupabaseDatabaseAdapter extends DatabaseAdapter { - async getRoom(roomId: UUID): Promise { - const { data, error } = await this.supabase - .from("rooms") - .select("id") - .eq("id", roomId) - .single(); + async getRoom(roomId: UUID): Promise { + const { data, error } = await this.supabase + .from("rooms") + .select("id") + .eq("id", roomId) + .single(); + + if (error) { + throw new Error(`Error getting room: ${error.message}`); + } - if (error) { - throw new Error(`Error getting room: ${error.message}`); - } - - return data ? (data.id as UUID) : null; - } - - async getParticipantsForAccount(userId: UUID): Promise { - const { data, error } = await this.supabase - .from("participants") - .select("*") - .eq("userId", userId); - - if (error) { - throw new Error( - `Error getting participants for account: ${error.message}`, - ); - } - - return data as Participant[]; - } - - async getParticipantUserState( - roomId: UUID, - userId: UUID, - ): Promise<"FOLLOWED" | "MUTED" | null> { - const { data, error } = await this.supabase - .from("participants") - .select("userState") - .eq("roomId", roomId) - .eq("userId", userId) - .single(); - - if (error) { - console.error("Error getting participant user state:", error); - return null; - } - - return data?.userState as "FOLLOWED" | "MUTED" | null; - } - - async setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null, - ): Promise { - const { error } = await this.supabase - .from("participants") - .update({ userState: state }) - .eq("roomId", roomId) - .eq("userId", userId); - - if (error) { - console.error("Error setting participant user state:", error); - throw new Error("Failed to set participant user state"); - } - } - - async getParticipantsForRoom(roomId: UUID): Promise { - const { data, error } = await this.supabase - .from("participants") - .select("userId") - .eq("roomId", roomId); - - if (error) { - throw new Error(`Error getting participants for room: ${error.message}`); - } - - return data.map((row) => row.userId as UUID); - } - - supabase: SupabaseClient; - - constructor(supabaseUrl: string, supabaseKey: string) { - super(); - this.supabase = createClient(supabaseUrl, supabaseKey); - } - - async getMemoriesByRoomIds(params: { - roomIds: UUID[]; - tableName: string; - }): Promise { - const { data, error } = await this.supabase - .from(params.tableName) - .select("*") - .in("roomId", params.roomIds); - - if (error) { - console.error("Error retrieving memories by room IDs:", error); - return []; - } - - // map createdAt to Date - const memories = data.map((memory) => ({ - ...memory, - })); - - return memories as Memory[]; - } - - async getAccountById(userId: UUID): Promise { - const { data, error } = await this.supabase - .from("accounts") - .select("*") - .eq("id", userId); - if (error) { - throw new Error(error.message); - } - return (data?.[0] as Account) || null; - } - - async createAccount(account: Account): Promise { - const { error } = await this.supabase.from("accounts").upsert([account]); - if (error) { - console.error(error.message); - return false; - } - return true; - } - - async getActorDetails(params: { roomId: UUID }): Promise { - try { - const response = await this.supabase - .from("rooms") - .select( - ` + return data ? (data.id as UUID) : null; + } + + async getParticipantsForAccount(userId: UUID): Promise { + const { data, error } = await this.supabase + .from("participants") + .select("*") + .eq("userId", userId); + + if (error) { + throw new Error( + `Error getting participants for account: ${error.message}` + ); + } + + return data as Participant[]; + } + + async getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null> { + const { data, error } = await this.supabase + .from("participants") + .select("userState") + .eq("roomId", roomId) + .eq("userId", userId) + .single(); + + if (error) { + console.error("Error getting participant user state:", error); + return null; + } + + return data?.userState as "FOLLOWED" | "MUTED" | null; + } + + async setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise { + const { error } = await this.supabase + .from("participants") + .update({ userState: state }) + .eq("roomId", roomId) + .eq("userId", userId); + + if (error) { + console.error("Error setting participant user state:", error); + throw new Error("Failed to set participant user state"); + } + } + + async getParticipantsForRoom(roomId: UUID): Promise { + const { data, error } = await this.supabase + .from("participants") + .select("userId") + .eq("roomId", roomId); + + if (error) { + throw new Error( + `Error getting participants for room: ${error.message}` + ); + } + + return data.map((row) => row.userId as UUID); + } + + supabase: SupabaseClient; + + constructor(supabaseUrl: string, supabaseKey: string) { + super(); + this.supabase = createClient(supabaseUrl, supabaseKey); + } + + async getMemoriesByRoomIds(params: { + roomIds: UUID[]; + tableName: string; + }): Promise { + const { data, error } = await this.supabase + .from(params.tableName) + .select("*") + .in("roomId", params.roomIds); + + if (error) { + console.error("Error retrieving memories by room IDs:", error); + return []; + } + + // map createdAt to Date + const memories = data.map((memory) => ({ + ...memory, + })); + + return memories as Memory[]; + } + + async getAccountById(userId: UUID): Promise { + const { data, error } = await this.supabase + .from("accounts") + .select("*") + .eq("id", userId); + if (error) { + throw new Error(error.message); + } + return (data?.[0] as Account) || null; + } + + async createAccount(account: Account): Promise { + const { error } = await this.supabase + .from("accounts") + .upsert([account]); + if (error) { + console.error(error.message); + return false; + } + return true; + } + + async getActorDetails(params: { roomId: UUID }): Promise { + try { + const response = await this.supabase + .from("rooms") + .select( + ` participants:participants( account:accounts(id, name, username, details) ) - `, - ) - .eq("id", params.roomId); - - if (response.error) { - console.error("Error!" + response.error); - return []; - } - const { data } = response; - - return data - .map((room) => - room.participants.map((participant) => { - const user = participant.account as unknown as Actor; - return { - name: user?.name, - details: user?.details, - id: user?.id, - username: user?.username, + ` + ) + .eq("id", params.roomId); + + if (response.error) { + console.error("Error!" + response.error); + return []; + } + const { data } = response; + + return data + .map((room) => + room.participants.map((participant) => { + const user = participant.account as unknown as Actor; + return { + name: user?.name, + details: user?.details, + id: user?.id, + username: user?.username, + }; + }) + ) + .flat(); + } catch (error) { + console.error("error", error); + throw error; + } + } + + async searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise { + const result = await this.supabase.rpc("search_memories", { + query_table_name: params.tableName, + query_roomId: params.roomId, + query_embedding: params.embedding, + query_match_threshold: params.match_threshold, + query_match_count: params.match_count, + query_unique: params.unique, + }); + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + return result.data.map((memory) => ({ + ...memory, + })); + } + + async getCachedEmbeddings(opts: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise< + { + embedding: number[]; + levenshtein_score: number; + }[] + > { + const result = await this.supabase.rpc("get_embedding_list", opts); + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + return result.data; + } + + async updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise { + await this.supabase + .from("goals") + .update({ status: params.status }) + .match({ id: params.goalId }); + } + + async log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise { + const { error } = await this.supabase.from("logs").insert({ + body: params.body, + userId: params.userId, + roomId: params.roomId, + type: params.type, + }); + + if (error) { + console.error("Error inserting log:", error); + throw new Error(error.message); + } + } + + async getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise { + const query = this.supabase + .from(params.tableName) + .select("*") + .eq("roomId", params.roomId); + + if (params.start) { + query.gte("createdAt", params.start); + } + + if (params.end) { + query.lte("createdAt", params.end); + } + + if (params.unique) { + query.eq("unique", true); + } + + if (params.userIds && params.userIds.length > 0) { + query.in("userId", params.userIds); + } + + query.order("createdAt", { ascending: false }); + + if (params.count) { + query.limit(params.count); + } + + const { data, error } = await query; + + if (error) { + throw new Error(`Error retrieving memories: ${error.message}`); + } + + return data as Memory[]; + } + + async searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise { + const result = await this.supabase.rpc("search_memories", { + query_table_name: params.tableName, + query_roomId: params.roomId, + query_embedding: embedding, + query_match_threshold: params.match_threshold, + query_match_count: params.count, + query_unique: !!params.unique, + }); + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + return result.data.map((memory) => ({ + ...memory, + })); + } + + async getMemoryById(memoryId: UUID): Promise { + const { data, error } = await this.supabase + .from("memories") + .select("*") + .eq("id", memoryId) + .single(); + + if (error) { + console.error("Error retrieving memory by ID:", error); + return null; + } + + return data as Memory; + } + + async createMemory( + memory: Memory, + tableName: string, + unique = false + ): Promise { + const createdAt = memory.createdAt ?? Date.now(); + if (unique) { + const opts = { + // TODO: Add ID option, optionally + query_table_name: tableName, + query_userId: memory.userId, + query_content: memory.content.text, + query_roomId: memory.roomId, + query_embedding: memory.embedding, + query_createdAt: createdAt, + similarity_threshold: 0.95, }; - }), - ) - .flat(); - } catch (error) { - console.error("error", error); - throw error; - } - } - - async searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise { - const result = await this.supabase.rpc("search_memories", { - query_table_name: params.tableName, - query_roomId: params.roomId, - query_embedding: params.embedding, - query_match_threshold: params.match_threshold, - query_match_count: params.match_count, - query_unique: params.unique, - }); - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - return result.data.map((memory) => ({ - ...memory, - })); - } - - async getCachedEmbeddings(opts: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise< - { - embedding: number[]; - levenshtein_score: number; - }[] - > { - const result = await this.supabase.rpc("get_embedding_list", opts); - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - return result.data; - } - - async updateGoalStatus(params: { - goalId: UUID; - status: GoalStatus; - }): Promise { - await this.supabase - .from("goals") - .update({ status: params.status }) - .match({ id: params.goalId }); - } - - async log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise { - const { error } = await this.supabase.from("logs").insert({ - body: params.body, - userId: params.userId, - roomId: params.roomId, - type: params.type, - }); - - if (error) { - console.error("Error inserting log:", error); - throw new Error(error.message); - } - } - - async getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise { - const query = this.supabase - .from(params.tableName) - .select("*") - .eq("roomId", params.roomId); - - if (params.start) { - query.gte("createdAt", params.start); - } - - if (params.end) { - query.lte("createdAt", params.end); - } - - if (params.unique) { - query.eq("unique", true); - } - - if (params.userIds && params.userIds.length > 0) { - query.in("userId", params.userIds); - } - - query.order("createdAt", { ascending: false }); - - if (params.count) { - query.limit(params.count); - } - - const { data, error } = await query; - - if (error) { - throw new Error(`Error retrieving memories: ${error.message}`); - } - - return data as Memory[]; - } - - async searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - }, - ): Promise { - const result = await this.supabase.rpc("search_memories", { - query_table_name: params.tableName, - query_roomId: params.roomId, - query_embedding: embedding, - query_match_threshold: params.match_threshold, - query_match_count: params.count, - query_unique: !!params.unique, - }); - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - return result.data.map((memory) => ({ - ...memory, - })); - } - - async getMemoryById(memoryId: UUID): Promise { - const { data, error } = await this.supabase - .from("memories") - .select("*") - .eq("id", memoryId) - .single(); - - if (error) { - console.error("Error retrieving memory by ID:", error); - return null; - } - - return data as Memory; - } - - async createMemory( - memory: Memory, - tableName: string, - unique = false, - ): Promise { - const createdAt = memory.createdAt ?? Date.now(); - if (unique) { - const opts = { - // TODO: Add ID option, optionally - query_table_name: tableName, - query_userId: memory.userId, - query_content: memory.content.text, - query_roomId: memory.roomId, - query_embedding: memory.embedding, - query_createdAt: createdAt, - similarity_threshold: 0.95, - }; - - const result = await this.supabase.rpc( - "check_similarity_and_insert", - opts, - ); - - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - } else { - const result = await this.supabase - .from("memories") - .insert({ ...memory, createdAt, type: tableName }); - const { error } = result; - if (error) { - throw new Error(JSON.stringify(error)); - } - } - } - - async removeMemory(memoryId: UUID): Promise { - const result = await this.supabase - .from("memories") - .delete() - .eq("id", memoryId); - const { error } = result; - if (error) { - throw new Error(JSON.stringify(error)); - } - } - - async removeAllMemories(roomId: UUID, tableName: string): Promise { - const result = await this.supabase.rpc("remove_memories", { - query_table_name: tableName, - query_roomId: roomId, - }); - - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - } - - async countMemories( - roomId: UUID, - unique = true, - tableName: string, - ): Promise { - if (!tableName) { - throw new Error("tableName is required"); - } - const query = { - query_table_name: tableName, - query_roomId: roomId, - query_unique: !!unique, - }; - const result = await this.supabase.rpc("count_memories", query); - - if (result.error) { - throw new Error(JSON.stringify(result.error)); - } - - return result.data; - } - - async getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise { - const opts = { - query_roomId: params.roomId, - query_userId: params.userId, - only_in_progress: params.onlyInProgress, - row_count: params.count, - }; - - const { data: goals, error } = await this.supabase.rpc("get_goals", opts); - - if (error) { - throw new Error(error.message); - } - - return goals; - } - - async updateGoal(goal: Goal): Promise { - const { error } = await this.supabase - .from("goals") - .update(goal) - .match({ id: goal.id }); - if (error) { - throw new Error(`Error creating goal: ${error.message}`); - } - } - - async createGoal(goal: Goal): Promise { - const { error } = await this.supabase.from("goals").insert(goal); - if (error) { - throw new Error(`Error creating goal: ${error.message}`); - } - } - - async removeGoal(goalId: UUID): Promise { - const { error } = await this.supabase - .from("goals") - .delete() - .eq("id", goalId); - if (error) { - throw new Error(`Error removing goal: ${error.message}`); - } - } - - async removeAllGoals(roomId: UUID): Promise { - const { error } = await this.supabase - .from("goals") - .delete() - .eq("roomId", roomId); - if (error) { - throw new Error(`Error removing goals: ${error.message}`); - } - } - - async getRoomsForParticipant(userId: UUID): Promise { - const { data, error } = await this.supabase - .from("participants") - .select("roomId") - .eq("userId", userId); - if (error) { - throw new Error(`Error getting rooms by participant: ${error.message}`); + const result = await this.supabase.rpc( + "check_similarity_and_insert", + opts + ); + + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + } else { + const result = await this.supabase + .from("memories") + .insert({ ...memory, createdAt, type: tableName }); + const { error } = result; + if (error) { + throw new Error(JSON.stringify(error)); + } + } + } + + async removeMemory(memoryId: UUID): Promise { + const result = await this.supabase + .from("memories") + .delete() + .eq("id", memoryId); + const { error } = result; + if (error) { + throw new Error(JSON.stringify(error)); + } + } + + async removeAllMemories(roomId: UUID, tableName: string): Promise { + const result = await this.supabase.rpc("remove_memories", { + query_table_name: tableName, + query_roomId: roomId, + }); + + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + } + + async countMemories( + roomId: UUID, + unique = true, + tableName: string + ): Promise { + if (!tableName) { + throw new Error("tableName is required"); + } + const query = { + query_table_name: tableName, + query_roomId: roomId, + query_unique: !!unique, + }; + const result = await this.supabase.rpc("count_memories", query); + + if (result.error) { + throw new Error(JSON.stringify(result.error)); + } + + return result.data; + } + + async getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise { + const opts = { + query_roomId: params.roomId, + query_userId: params.userId, + only_in_progress: params.onlyInProgress, + row_count: params.count, + }; + + const { data: goals, error } = await this.supabase.rpc( + "get_goals", + opts + ); + + if (error) { + throw new Error(error.message); + } + + return goals; + } + + async updateGoal(goal: Goal): Promise { + const { error } = await this.supabase + .from("goals") + .update(goal) + .match({ id: goal.id }); + if (error) { + throw new Error(`Error creating goal: ${error.message}`); + } + } + + async createGoal(goal: Goal): Promise { + const { error } = await this.supabase.from("goals").insert(goal); + if (error) { + throw new Error(`Error creating goal: ${error.message}`); + } + } + + async removeGoal(goalId: UUID): Promise { + const { error } = await this.supabase + .from("goals") + .delete() + .eq("id", goalId); + if (error) { + throw new Error(`Error removing goal: ${error.message}`); + } + } + + async removeAllGoals(roomId: UUID): Promise { + const { error } = await this.supabase + .from("goals") + .delete() + .eq("roomId", roomId); + if (error) { + throw new Error(`Error removing goals: ${error.message}`); + } + } + + async getRoomsForParticipant(userId: UUID): Promise { + const { data, error } = await this.supabase + .from("participants") + .select("roomId") + .eq("userId", userId); + + if (error) { + throw new Error( + `Error getting rooms by participant: ${error.message}` + ); + } + + return data.map((row) => row.roomId as UUID); + } + + async getRoomsForParticipants(userIds: UUID[]): Promise { + const { data, error } = await this.supabase + .from("participants") + .select("roomId") + .in("userId", userIds); + + if (error) { + throw new Error( + `Error getting rooms by participants: ${error.message}` + ); + } + + return [...new Set(data.map((row) => row.roomId as UUID))] as UUID[]; + } + + async createRoom(roomId?: UUID): Promise { + roomId = roomId ?? (uuid() as UUID); + const { data, error } = await this.supabase.rpc("create_room", { + roomId, + }); + + if (error) { + throw new Error(`Error creating room: ${error.message}`); + } + + if (!data || data.length === 0) { + throw new Error("No data returned from room creation"); + } + + return data[0].id as UUID; + } + + async removeRoom(roomId: UUID): Promise { + const { error } = await this.supabase + .from("rooms") + .delete() + .eq("id", roomId); + + if (error) { + throw new Error(`Error removing room: ${error.message}`); + } + } + + async addParticipant(userId: UUID, roomId: UUID): Promise { + const { error } = await this.supabase + .from("participants") + .insert({ userId: userId, roomId: roomId }); + + if (error) { + console.error(`Error adding participant: ${error.message}`); + return false; + } + return true; + } + + async removeParticipant(userId: UUID, roomId: UUID): Promise { + const { error } = await this.supabase + .from("participants") + .delete() + .eq("userId", userId) + .eq("roomId", roomId); + + if (error) { + console.error(`Error removing participant: ${error.message}`); + return false; + } + return true; + } + + async createRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + const allRoomData = await this.getRoomsForParticipants([ + params.userA, + params.userB, + ]); + + let roomId: UUID; + + if (!allRoomData || allRoomData.length === 0) { + // If no existing room is found, create a new room + const { data: newRoomData, error: roomsError } = await this.supabase + .from("rooms") + .insert({}) + .single(); + + if (roomsError) { + throw new Error("Room creation error: " + roomsError.message); + } + + roomId = (newRoomData as Room)?.id as UUID; + } else { + // If an existing room is found, use the first room's ID + roomId = allRoomData[0]; + } + + const { error: participantsError } = await this.supabase + .from("participants") + .insert([ + { userId: params.userA, roomId }, + { userId: params.userB, roomId }, + ]); + + if (participantsError) { + throw new Error( + "Participants creation error: " + participantsError.message + ); + } + + // Create or update the relationship between the two users + const { error: relationshipError } = await this.supabase + .from("relationships") + .upsert({ + userA: params.userA, + userB: params.userB, + userId: params.userA, + status: "FRIENDS", + }) + .eq("userA", params.userA) + .eq("userB", params.userB); + + if (relationshipError) { + throw new Error( + "Relationship creation error: " + relationshipError.message + ); + } + + return true; + } + + async getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise { + const { data, error } = await this.supabase.rpc("get_relationship", { + usera: params.userA, + userb: params.userB, + }); + + if (error) { + throw new Error(error.message); + } + + return data[0]; + } + + async getRelationships(params: { userId: UUID }): Promise { + const { data, error } = await this.supabase + .from("relationships") + .select("*") + .or(`userA.eq.${params.userId},userB.eq.${params.userId}`) + .eq("status", "FRIENDS"); + + if (error) { + throw new Error(error.message); + } + + return data as Relationship[]; } - - return data.map((row) => row.roomId as UUID); - } - - async getRoomsForParticipants(userIds: UUID[]): Promise { - const { data, error } = await this.supabase - .from("participants") - .select("roomId") - .in("userId", userIds); - - if (error) { - throw new Error(`Error getting rooms by participants: ${error.message}`); - } - - return [...new Set(data.map((row) => row.roomId as UUID))] as UUID[]; - } - - async createRoom(roomId?: UUID): Promise { - roomId = roomId ?? (uuid() as UUID); - const { data, error } = await this.supabase.rpc("create_room", { - roomId, - }); - - if (error) { - throw new Error(`Error creating room: ${error.message}`); - } - - if (!data || data.length === 0) { - throw new Error("No data returned from room creation"); - } - - return data[0].id as UUID; - } - - async removeRoom(roomId: UUID): Promise { - const { error } = await this.supabase - .from("rooms") - .delete() - .eq("id", roomId); - - if (error) { - throw new Error(`Error removing room: ${error.message}`); - } - } - - async addParticipant(userId: UUID, roomId: UUID): Promise { - const { error } = await this.supabase - .from("participants") - .insert({ userId: userId, roomId: roomId }); - - if (error) { - console.error(`Error adding participant: ${error.message}`); - return false; - } - return true; - } - - async removeParticipant(userId: UUID, roomId: UUID): Promise { - const { error } = await this.supabase - .from("participants") - .delete() - .eq("userId", userId) - .eq("roomId", roomId); - - if (error) { - console.error(`Error removing participant: ${error.message}`); - return false; - } - return true; - } - - async createRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - const allRoomData = await this.getRoomsForParticipants([ - params.userA, - params.userB, - ]); - - let roomId: UUID; - - if (!allRoomData || allRoomData.length === 0) { - // If no existing room is found, create a new room - const { data: newRoomData, error: roomsError } = await this.supabase - .from("rooms") - .insert({}) - .single(); - - if (roomsError) { - throw new Error("Room creation error: " + roomsError.message); - } - - roomId = (newRoomData as Room)?.id as UUID; - } else { - // If an existing room is found, use the first room's ID - roomId = allRoomData[0]; - } - - const { error: participantsError } = await this.supabase - .from("participants") - .insert([ - { userId: params.userA, roomId }, - { userId: params.userB, roomId }, - ]); - - if (participantsError) { - throw new Error( - "Participants creation error: " + participantsError.message, - ); - } - - // Create or update the relationship between the two users - const { error: relationshipError } = await this.supabase - .from("relationships") - .upsert({ - userA: params.userA, - userB: params.userB, - userId: params.userA, - status: "FRIENDS", - }) - .eq("userA", params.userA) - .eq("userB", params.userB); - - if (relationshipError) { - throw new Error( - "Relationship creation error: " + relationshipError.message, - ); - } - - return true; - } - - async getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise { - const { data, error } = await this.supabase.rpc("get_relationship", { - usera: params.userA, - userb: params.userB, - }); - - if (error) { - throw new Error(error.message); - } - - return data[0]; - } - - async getRelationships(params: { userId: UUID }): Promise { - const { data, error } = await this.supabase - .from("relationships") - .select("*") - .or(`userA.eq.${params.userId},userB.eq.${params.userId}`) - .eq("status", "FRIENDS"); - - if (error) { - throw new Error(error.message); - } - - return data as Relationship[]; - } } diff --git a/core/src/clients/direct/index.ts b/core/src/clients/direct/index.ts index a6aa9f5a29..1fbd250468 100644 --- a/core/src/clients/direct/index.ts +++ b/core/src/clients/direct/index.ts @@ -2,29 +2,27 @@ import bodyParser from "body-parser"; import express from "express"; import { composeContext } from "../../core/context.ts"; import { AgentRuntime } from "../../core/runtime.ts"; -import { - Content, - Memory, - ModelClass, - State -} from "../../core/types.ts"; +import { Content, Memory, ModelClass, State } from "../../core/types.ts"; import { stringToUuid } from "../../core/uuid.ts"; import cors from "cors"; import { messageCompletionFooter } from "../../core/parsing.ts"; -import multer, { File } from 'multer'; -import { Request as ExpressRequest } from 'express'; +import multer, { File } from "multer"; +import { Request as ExpressRequest } from "express"; import { generateMessageResponse } from "../../core/generation.ts"; -import { generateCaption, generateImage } from "../../actions/imageGenerationUtils.ts"; +import { + generateCaption, + generateImage, +} from "../../actions/imageGenerationUtils.ts"; const upload = multer({ storage: multer.memoryStorage() }); export const messageHandlerTemplate = - // {{goals}} -// `# Action Examples -// {{actionExamples}} -// (Action examples are for reference only. Do not use the information from them in your response.) + // {{goals}} + // `# Action Examples + // {{actionExamples}} + // (Action examples are for reference only. Do not use the information from them in your response.) -`# Task: Generate dialog and actions for the character {{agentName}}. + `# Task: Generate dialog and actions for the character {{agentName}}. About {{agentName}}: {{bio}} {{lore}} @@ -45,193 +43,228 @@ Note that {{agentName}} is capable of reading/seeing/hearing various forms of me # Instructions: Write the next message for {{agentName}}. Ignore "action". ` + messageCompletionFooter; - - export interface SimliClientConfig { - apiKey: string - faceID: string - handleSilence: boolean - videoRef: any - audioRef: any + apiKey: string; + faceID: string; + handleSilence: boolean; + videoRef: any; + audioRef: any; } class DirectClient { - private app: express.Application; - private agents: Map; + private app: express.Application; + private agents: Map; - constructor() { - this.app = express(); - this.app.use(cors()); - this.agents = new Map(); + constructor() { + this.app = express(); + this.app.use(cors()); + this.agents = new Map(); - this.app.use(bodyParser.json()); - this.app.use(bodyParser.urlencoded({ extended: true })); + this.app.use(bodyParser.json()); + this.app.use(bodyParser.urlencoded({ extended: true })); -// Define an interface that extends the Express Request interface -interface CustomRequest extends ExpressRequest { - file: File; -} - -// Update the route handler to use CustomRequest instead of express.Request -this.app.post("/:agentId/whisper", upload.single('file'), async (req: CustomRequest, res: express.Response) => { - const audioFile = req.file; // Access the uploaded file using req.file - const agentId = req.params.agentId; - - if (!audioFile) { - res.status(400).send("No audio file provided"); - return; - } - - let runtime = this.agents.get(agentId); - - // if runtime is null, look for runtime with the same name - if (!runtime) { - runtime = Array.from(this.agents.values()).find((a) => a.character.name.toLowerCase() === agentId.toLowerCase()); - } - - if (!runtime) { - res.status(404).send("Agent not found"); - return; - } - - const formData = new FormData(); - const audioBlob = new Blob([audioFile.buffer], { type: audioFile.mimetype }); - formData.append('file', audioBlob, audioFile.originalname); - formData.append('model', 'whisper-1'); - - const response = await fetch("https://api.openai.com/v1/audio/transcriptions", { - method: "POST", - headers: { - "Authorization": `Bearer ${runtime.token}`, - }, - body: formData, - }); - - const data = await response.json(); - res.json(data); -}); - - this.app.post("/:agentId/message", async (req: express.Request, res: express.Response) => { - const agentId = req.params.agentId; - const roomId = stringToUuid(req.body.roomId ?? ("default-room-" + agentId)); - const userId = stringToUuid(req.body.userId ?? "user"); - - let runtime = this.agents.get(agentId); - - // if runtime is null, look for runtime with the same name - if (!runtime) { - runtime = Array.from(this.agents.values()).find((a) => a.character.name.toLowerCase() === agentId.toLowerCase()); - } - - if (!runtime) { - res.status(404).send("Agent not found"); - return; - } - - await Promise.all([ - runtime.ensureUserExists( - runtime.agentId, - runtime.character.name ?? "Agent", - runtime.character.name ?? "Agent", - "direct", - ), - runtime.ensureUserExists(userId, req.body.userName ?? "User", req.body.name ?? "User", "direct"), - runtime.ensureRoomExists(roomId), - ]); - - await Promise.all([ - runtime.ensureParticipantInRoom(userId, roomId), - runtime.ensureParticipantInRoom(runtime.agentId, roomId), - ]); - - const text = req.body.text; - const messageId = stringToUuid(Date.now().toString()); - - const content: Content = { - text, - attachments: [], - source: "direct", - inReplyTo: undefined, - }; - - const userMessage = { content, userId, roomId }; - - const memory: Memory = { - id: messageId, - ...userMessage, - userId, - roomId, - content, - createdAt: Date.now(), - }; - - await runtime.messageManager.createMemory(memory); - - - const state = (await runtime.composeState(userMessage, { - agentName: runtime.character.name, - })) as State; - - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - - const response = await generateMessageResponse({ - runtime: runtime, - context, - modelClass: ModelClass.SMALL, - }); - - // save response to memory - const responseMessage = { - ...userMessage, - userId: runtime.agentId, - content: response, - }; - - await runtime.messageManager.createMemory(responseMessage); - - if (!response) { - res.status(500).send("No response from generateMessageResponse"); - return; - } - - res.json(response); - }); - - this.app.post("/:agentId/image", async (req: express.Request, res: express.Response) => { - const agentId = req.params.agentId; - const agent = this.agents.get(agentId); - if (!agent) { - res.status(404).send("Agent not found"); - return; - } - - const images = await generateImage({...req.body }, agent); - const imagesRes: {image: string, caption: string}[] = []; - if (images.data && images.data.length > 0) { - for(let i = 0; i < images.data.length; i++) { - const caption = await generateCaption({imageUrl: images.data[i]}, agent); - imagesRes.push({image: images.data[i], caption: caption.title}); + // Define an interface that extends the Express Request interface + interface CustomRequest extends ExpressRequest { + file: File; } - } - res.json({images: imagesRes}); - }); - } - - public registerAgent(runtime: AgentRuntime) { - this.agents.set(runtime.agentId, runtime); - } - - public unregisterAgent(runtime: AgentRuntime) { - this.agents.delete(runtime.agentId); - } - - public start(port: number) { - this.app.listen(port, () => { - console.log(`Server running at http://localhost:${port}/`); - }); - } + + // Update the route handler to use CustomRequest instead of express.Request + this.app.post( + "/:agentId/whisper", + upload.single("file"), + async (req: CustomRequest, res: express.Response) => { + const audioFile = req.file; // Access the uploaded file using req.file + const agentId = req.params.agentId; + + if (!audioFile) { + res.status(400).send("No audio file provided"); + return; + } + + let runtime = this.agents.get(agentId); + + // if runtime is null, look for runtime with the same name + if (!runtime) { + runtime = Array.from(this.agents.values()).find( + (a) => + a.character.name.toLowerCase() === + agentId.toLowerCase() + ); + } + + if (!runtime) { + res.status(404).send("Agent not found"); + return; + } + + const formData = new FormData(); + const audioBlob = new Blob([audioFile.buffer], { + type: audioFile.mimetype, + }); + formData.append("file", audioBlob, audioFile.originalname); + formData.append("model", "whisper-1"); + + const response = await fetch( + "https://api.openai.com/v1/audio/transcriptions", + { + method: "POST", + headers: { + Authorization: `Bearer ${runtime.token}`, + }, + body: formData, + } + ); + + const data = await response.json(); + res.json(data); + } + ); + + this.app.post( + "/:agentId/message", + async (req: express.Request, res: express.Response) => { + const agentId = req.params.agentId; + const roomId = stringToUuid( + req.body.roomId ?? "default-room-" + agentId + ); + const userId = stringToUuid(req.body.userId ?? "user"); + + let runtime = this.agents.get(agentId); + + // if runtime is null, look for runtime with the same name + if (!runtime) { + runtime = Array.from(this.agents.values()).find( + (a) => + a.character.name.toLowerCase() === + agentId.toLowerCase() + ); + } + + if (!runtime) { + res.status(404).send("Agent not found"); + return; + } + + await Promise.all([ + runtime.ensureUserExists( + runtime.agentId, + runtime.character.name ?? "Agent", + runtime.character.name ?? "Agent", + "direct" + ), + runtime.ensureUserExists( + userId, + req.body.userName ?? "User", + req.body.name ?? "User", + "direct" + ), + runtime.ensureRoomExists(roomId), + ]); + + await Promise.all([ + runtime.ensureParticipantInRoom(userId, roomId), + runtime.ensureParticipantInRoom(runtime.agentId, roomId), + ]); + + const text = req.body.text; + const messageId = stringToUuid(Date.now().toString()); + + const content: Content = { + text, + attachments: [], + source: "direct", + inReplyTo: undefined, + }; + + const userMessage = { content, userId, roomId }; + + const memory: Memory = { + id: messageId, + ...userMessage, + userId, + roomId, + content, + createdAt: Date.now(), + }; + + await runtime.messageManager.createMemory(memory); + + const state = (await runtime.composeState(userMessage, { + agentName: runtime.character.name, + })) as State; + + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); + + const response = await generateMessageResponse({ + runtime: runtime, + context, + modelClass: ModelClass.SMALL, + }); + + // save response to memory + const responseMessage = { + ...userMessage, + userId: runtime.agentId, + content: response, + }; + + await runtime.messageManager.createMemory(responseMessage); + + if (!response) { + res.status(500).send( + "No response from generateMessageResponse" + ); + return; + } + + res.json(response); + } + ); + + this.app.post( + "/:agentId/image", + async (req: express.Request, res: express.Response) => { + const agentId = req.params.agentId; + const agent = this.agents.get(agentId); + if (!agent) { + res.status(404).send("Agent not found"); + return; + } + + const images = await generateImage({ ...req.body }, agent); + const imagesRes: { image: string; caption: string }[] = []; + if (images.data && images.data.length > 0) { + for (let i = 0; i < images.data.length; i++) { + const caption = await generateCaption( + { imageUrl: images.data[i] }, + agent + ); + imagesRes.push({ + image: images.data[i], + caption: caption.title, + }); + } + } + res.json({ images: imagesRes }); + } + ); + } + + public registerAgent(runtime: AgentRuntime) { + this.agents.set(runtime.agentId, runtime); + } + + public unregisterAgent(runtime: AgentRuntime) { + this.agents.delete(runtime.agentId); + } + + public start(port: number) { + this.app.listen(port, () => { + console.log(`Server running at http://localhost:${port}/`); + }); + } } -export default DirectClient; \ No newline at end of file +export default DirectClient; diff --git a/core/src/clients/discord/actions/chat_with_attachments.ts b/core/src/clients/discord/actions/chat_with_attachments.ts index 6e3b8d8f53..ee54c93704 100644 --- a/core/src/clients/discord/actions/chat_with_attachments.ts +++ b/core/src/clients/discord/actions/chat_with_attachments.ts @@ -4,14 +4,14 @@ import { log_to_file } from "../../../core/logger.ts"; import models from "../../../core/models.ts"; import { parseJSONObjectFromText } from "../../../core/parsing.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../../../core/types.ts"; import fs from "fs"; export const summarizationTemplate = `# Summarized so far (we are adding to this) @@ -41,282 +41,286 @@ Your response must be formatted as a JSON block with this structure: `; const getAttachmentIds = async ( - runtime: IAgentRuntime, - message: Memory, - state: State, + runtime: IAgentRuntime, + message: Memory, + state: State ): Promise<{ objective: string; attachmentIds: string[] } | null> => { - state = (await runtime.composeState(message)) as State; - - const context = composeContext({ - state, - template: attachmentIdsTemplate, - }); + state = (await runtime.composeState(message)) as State; - for (let i = 0; i < 5; i++) { - const response = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, + const context = composeContext({ + state, + template: attachmentIdsTemplate, }); - console.log("response", response); - // try parsing to a json object - const parsedResponse = parseJSONObjectFromText(response) as { - objective: string; - attachmentIds: string[]; - } | null; - // see if it contains objective and attachmentIds - if (parsedResponse?.objective && parsedResponse?.attachmentIds) { - return parsedResponse; + + for (let i = 0; i < 5; i++) { + const response = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + console.log("response", response); + // try parsing to a json object + const parsedResponse = parseJSONObjectFromText(response) as { + objective: string; + attachmentIds: string[]; + } | null; + // see if it contains objective and attachmentIds + if (parsedResponse?.objective && parsedResponse?.attachmentIds) { + return parsedResponse; + } } - } - return null; + return null; }; const summarizeAction = { - name: "CHAT_WITH_ATTACHMENTS", - similes: [ - "CHAT_WITH_ATTACHMENT", - "SUMMARIZE_FILES", - "SUMMARIZE_FILE", - "SUMMARIZE_ATACHMENT", - "CHAT_WITH_PDF", - "ATTACHMENT_SUMMARY", - "RECAP_ATTACHMENTS", - "SUMMARIZE_FILE", - "SUMMARIZE_VIDEO", - "SUMMARIZE_AUDIO", - "SUMMARIZE_IMAGE", - "SUMMARIZE_DOCUMENT", - "SUMMARIZE_LINK", - "ATTACHMENT_SUMMARY", - "FILE_SUMMARY", - ], - description: - "Answer a user request informed by specific attachments based on their IDs. If a user asks to chat with a PDF, or wants more specific information about a link or video or anything else they've attached, this is the action to use.", - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - return false; - } - // only show if one of the keywords are in the message - const keywords: string[] = [ - "attachment", - "summary", - "summarize", - "research", - "pdf", - "video", - "audio", - "image", - "document", - "link", - "file", - "attachment", - "summarize", - "code", - "report", - "write", - "details", - "information", - "talk", - "chat", - "read", - "listen", - "watch", - ]; - return keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword.toLowerCase()), - ); - }, - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { - state = (await runtime.composeState(message)) as State; + name: "CHAT_WITH_ATTACHMENTS", + similes: [ + "CHAT_WITH_ATTACHMENT", + "SUMMARIZE_FILES", + "SUMMARIZE_FILE", + "SUMMARIZE_ATACHMENT", + "CHAT_WITH_PDF", + "ATTACHMENT_SUMMARY", + "RECAP_ATTACHMENTS", + "SUMMARIZE_FILE", + "SUMMARIZE_VIDEO", + "SUMMARIZE_AUDIO", + "SUMMARIZE_IMAGE", + "SUMMARIZE_DOCUMENT", + "SUMMARIZE_LINK", + "ATTACHMENT_SUMMARY", + "FILE_SUMMARY", + ], + description: + "Answer a user request informed by specific attachments based on their IDs. If a user asks to chat with a PDF, or wants more specific information about a link or video or anything else they've attached, this is the action to use.", + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + if (message.content.source !== "discord") { + return false; + } + // only show if one of the keywords are in the message + const keywords: string[] = [ + "attachment", + "summary", + "summarize", + "research", + "pdf", + "video", + "audio", + "image", + "document", + "link", + "file", + "attachment", + "summarize", + "code", + "report", + "write", + "details", + "information", + "talk", + "chat", + "read", + "listen", + "watch", + ]; + return keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword.toLowerCase()) + ); + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + state = (await runtime.composeState(message)) as State; - const callbackData: Content = { - text: "", // fill in later - action: "CHAT_WITH_ATTACHMENTS_RESPONSE", - source: message.content.source, - attachments: [], - }; + const callbackData: Content = { + text: "", // fill in later + action: "CHAT_WITH_ATTACHMENTS_RESPONSE", + source: message.content.source, + attachments: [], + }; - // 1. extract attachment IDs from the message - const attachmentData = await getAttachmentIds(runtime, message, state); - if (!attachmentData) { - console.error("Couldn't get attachment IDs from message"); - return; - } + // 1. extract attachment IDs from the message + const attachmentData = await getAttachmentIds(runtime, message, state); + if (!attachmentData) { + console.error("Couldn't get attachment IDs from message"); + return; + } - const { objective, attachmentIds } = attachmentData; + const { objective, attachmentIds } = attachmentData; - // This is pretty gross but it can catch cases where the returned generated UUID is stupidly wrong for some reason - const attachments = state.recentMessagesData - .filter( - (msg) => msg.content.attachments && msg.content.attachments.length > 0, - ) - .flatMap((msg) => msg.content.attachments) - // check by first 5 characters of uuid - .filter( - (attachment) => - attachmentIds - .map((attch) => attch.toLowerCase().slice(0, 5)) - .includes(attachment.id.toLowerCase().slice(0, 5)) || - // or check the other way - attachmentIds.some((id) => { - const attachmentId = id.toLowerCase().slice(0, 5); - return attachment.id.toLowerCase().includes(attachmentId); - }), - ); + // This is pretty gross but it can catch cases where the returned generated UUID is stupidly wrong for some reason + const attachments = state.recentMessagesData + .filter( + (msg) => + msg.content.attachments && + msg.content.attachments.length > 0 + ) + .flatMap((msg) => msg.content.attachments) + // check by first 5 characters of uuid + .filter( + (attachment) => + attachmentIds + .map((attch) => attch.toLowerCase().slice(0, 5)) + .includes(attachment.id.toLowerCase().slice(0, 5)) || + // or check the other way + attachmentIds.some((id) => { + const attachmentId = id.toLowerCase().slice(0, 5); + return attachment.id + .toLowerCase() + .includes(attachmentId); + }) + ); - const attachmentsWithText = attachments - .map((attachment) => `# ${attachment.title}\n${attachment.text}`) - .join("\n\n"); + const attachmentsWithText = attachments + .map((attachment) => `# ${attachment.title}\n${attachment.text}`) + .join("\n\n"); - let currentSummary = ""; + let currentSummary = ""; - const model = models[runtime.character.settings.model]; - const chunkSize = model.settings.maxContextLength; + const model = models[runtime.character.settings.model]; + const chunkSize = model.settings.maxContextLength; - state.attachmentsWithText = attachmentsWithText; - state.objective = objective; + state.attachmentsWithText = attachmentsWithText; + state.objective = objective; - const datestr = new Date().toUTCString().replace(/:/g, "-"); + const datestr = new Date().toUTCString().replace(/:/g, "-"); - const context = composeContext({ - state, - // make sure it fits, we can pad the tokens a bit - template: trimTokens( - summarizationTemplate, - chunkSize + 500, - "gpt-4o-mini", // TODO: make this dynamic and generic - ), - }); + const context = composeContext({ + state, + // make sure it fits, we can pad the tokens a bit + template: trimTokens( + summarizationTemplate, + chunkSize + 500, + "gpt-4o-mini" // TODO: make this dynamic and generic + ), + }); - log_to_file( - `${state.agentName}_${datestr}_chat_with_attachment_context`, - context, - ); + log_to_file( + `${state.agentName}_${datestr}_chat_with_attachment_context`, + context + ); - const summary = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); + const summary = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); - log_to_file( - `${state.agentName}_${datestr}_chat_with_attachment_response`, - summary, - ); + log_to_file( + `${state.agentName}_${datestr}_chat_with_attachment_response`, + summary + ); - currentSummary = currentSummary + "\n" + summary; + currentSummary = currentSummary + "\n" + summary; - // log summary to file - log_to_file( - `${state.agentName}_${datestr}_chat_with_attachment_summary`, - currentSummary, - ); + // log summary to file + log_to_file( + `${state.agentName}_${datestr}_chat_with_attachment_summary`, + currentSummary + ); - if (!currentSummary) { - console.error("No summary found, that's not good!"); - return; - } + if (!currentSummary) { + console.error("No summary found, that's not good!"); + return; + } - callbackData.text = currentSummary.trim(); - if ( - callbackData.text && - (currentSummary.trim()?.split("\n").length < 4 || - currentSummary.trim()?.split(" ").length < 100) - ) { - callbackData.text = `Here is the summary: + callbackData.text = currentSummary.trim(); + if ( + callbackData.text && + (currentSummary.trim()?.split("\n").length < 4 || + currentSummary.trim()?.split(" ").length < 100) + ) { + callbackData.text = `Here is the summary: \`\`\`md ${currentSummary.trim()} \`\`\` `; - await callback(callbackData); - } else if (currentSummary.trim()) { - const summaryFilename = `content_cache/summary_${Date.now()}.txt`; - // save the summary to a file - fs.writeFileSync(summaryFilename, currentSummary); - await callback( - { - ...callbackData, - text: `I've attached the summary of the requested attachments as a text file.`, - }, - [summaryFilename], - ); - } else { - console.warn( - "Empty response from chat with attachments action, skipping", - ); - } + await callback(callbackData); + } else if (currentSummary.trim()) { + const summaryFilename = `content_cache/summary_${Date.now()}.txt`; + // save the summary to a file + fs.writeFileSync(summaryFilename, currentSummary); + await callback( + { + ...callbackData, + text: `I've attached the summary of the requested attachments as a text file.`, + }, + [summaryFilename] + ); + } else { + console.warn( + "Empty response from chat with attachments action, skipping" + ); + } - return callbackData; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Can you summarize the attachments b3e23, c4f67, and d5a89?", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure thing! I'll pull up those specific attachments and provide a summary of their content.", - action: "CHAT_WITH_ATTACHMENTS", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "I need a technical summary of the PDFs I sent earlier - a1b2c3.pdf, d4e5f6.pdf, and g7h8i9.pdf", - }, - }, - { - user: "{{user2}}", - content: { - text: "I'll take a look at those specific PDF attachments and put together a technical summary for you. Give me a few minutes to review them.", - action: "CHAT_WITH_ATTACHMENTS", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Can you watch this video for me and tell me which parts you think are most relevant to the report I'm writing? (the one I attached in my last message)", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure, no problem.", - action: "CHAT_WITH_ATTACHMENTS", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "can you read my blog post and give me a detailed breakdown of the key points I made, and then suggest a handful of tweets to promote it?", - }, - }, - { - user: "{{user2}}", - content: { - text: "great idea, give me a minute", - action: "CHAT_WITH_ATTACHMENTS", - }, - }, - ], - ] as ActionExample[][], + return callbackData; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Can you summarize the attachments b3e23, c4f67, and d5a89?", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure thing! I'll pull up those specific attachments and provide a summary of their content.", + action: "CHAT_WITH_ATTACHMENTS", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "I need a technical summary of the PDFs I sent earlier - a1b2c3.pdf, d4e5f6.pdf, and g7h8i9.pdf", + }, + }, + { + user: "{{user2}}", + content: { + text: "I'll take a look at those specific PDF attachments and put together a technical summary for you. Give me a few minutes to review them.", + action: "CHAT_WITH_ATTACHMENTS", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can you watch this video for me and tell me which parts you think are most relevant to the report I'm writing? (the one I attached in my last message)", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure, no problem.", + action: "CHAT_WITH_ATTACHMENTS", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "can you read my blog post and give me a detailed breakdown of the key points I made, and then suggest a handful of tweets to promote it?", + }, + }, + { + user: "{{user2}}", + content: { + text: "great idea, give me a minute", + action: "CHAT_WITH_ATTACHMENTS", + }, + }, + ], + ] as ActionExample[][], } as Action; export default summarizeAction; diff --git a/core/src/clients/discord/actions/download_media.ts b/core/src/clients/discord/actions/download_media.ts index b7139ab3e0..4e3dfaf631 100644 --- a/core/src/clients/discord/actions/download_media.ts +++ b/core/src/clients/discord/actions/download_media.ts @@ -2,14 +2,14 @@ import path from "path"; import { composeContext } from "../../../core/context.ts"; import { parseJSONObjectFromText } from "../../../core/parsing.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../../../core/types.ts"; import { VideoService } from "../../../services/video.ts"; import { generateText } from "../../../core/generation.ts"; @@ -29,162 +29,165 @@ Your response must be formatted as a JSON block with this structure: `; const getMediaUrl = async ( - runtime: IAgentRuntime, - message: Memory, - state: State, + runtime: IAgentRuntime, + message: Memory, + state: State ): Promise => { - if (!state) { - state = (await runtime.composeState(message)) as State; - } - - const context = composeContext({ - state, - template: mediaUrlTemplate, - }); - - for (let i = 0; i < 5; i++) { - const response = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, + if (!state) { + state = (await runtime.composeState(message)) as State; + } + + const context = composeContext({ + state, + template: mediaUrlTemplate, }); - const parsedResponse = parseJSONObjectFromText(response) as { - mediaUrl: string; - } | null; + for (let i = 0; i < 5; i++) { + const response = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + + const parsedResponse = parseJSONObjectFromText(response) as { + mediaUrl: string; + } | null; - if (parsedResponse?.mediaUrl) { - return parsedResponse.mediaUrl; + if (parsedResponse?.mediaUrl) { + return parsedResponse.mediaUrl; + } } - } - return null; + return null; }; export default { - name: "DOWNLOAD_MEDIA", - similes: [ - "DOWNLOAD_VIDEO", - "DOWNLOAD_AUDIO", - "GET_MEDIA", - "DOWNLOAD_PODCAST", - "DOWNLOAD_YOUTUBE", - ], - description: - "Downloads a video or audio file from a URL and attaches it to the response message.", - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - return false; - } - }, - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { - const videoService = VideoService.getInstance(runtime); - if (!state) { - state = (await runtime.composeState(message)) as State; - } - - const mediaUrl = await getMediaUrl(runtime, message, state); - if (!mediaUrl) { - console.error("Couldn't get media URL from messages"); - return; - } + name: "DOWNLOAD_MEDIA", + similes: [ + "DOWNLOAD_VIDEO", + "DOWNLOAD_AUDIO", + "GET_MEDIA", + "DOWNLOAD_PODCAST", + "DOWNLOAD_YOUTUBE", + ], + description: + "Downloads a video or audio file from a URL and attaches it to the response message.", + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + if (message.content.source !== "discord") { + return false; + } + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + const videoService = VideoService.getInstance(runtime); + if (!state) { + state = (await runtime.composeState(message)) as State; + } - const videoInfo = await videoService.fetchVideoInfo(mediaUrl); - const mediaPath = await videoService.downloadVideo(videoInfo); - - const response: Content = { - text: `I downloaded the video "${videoInfo.title}" and attached it below.`, - action: "DOWNLOAD_MEDIA_RESPONSE", - source: message.content.source, - attachments: [], - }; - - const filename = path.basename(mediaPath); - - const maxRetries = 3; - let retries = 0; - - while (retries < maxRetries) { - try { - await callback( - { - ...response, - }, - ["content_cache/" + filename], - ); - break; - } catch (error) { - retries++; - console.error(`Error sending message (attempt ${retries}):`, error); - - if (retries === maxRetries) { - console.error( - "Max retries reached. Failed to send message with attachment.", - ); - await callback({ - ...response, - text: "Sorry, I encountered an error while trying to send the video attachment. Please try again later.", - }); - break; + const mediaUrl = await getMediaUrl(runtime, message, state); + if (!mediaUrl) { + console.error("Couldn't get media URL from messages"); + return; } - // Wait for a short delay before retrying - await new Promise((resolve) => setTimeout(resolve, 2000)); - } - } + const videoInfo = await videoService.fetchVideoInfo(mediaUrl); + const mediaPath = await videoService.downloadVideo(videoInfo); + + const response: Content = { + text: `I downloaded the video "${videoInfo.title}" and attached it below.`, + action: "DOWNLOAD_MEDIA_RESPONSE", + source: message.content.source, + attachments: [], + }; + + const filename = path.basename(mediaPath); + + const maxRetries = 3; + let retries = 0; + + while (retries < maxRetries) { + try { + await callback( + { + ...response, + }, + ["content_cache/" + filename] + ); + break; + } catch (error) { + retries++; + console.error( + `Error sending message (attempt ${retries}):`, + error + ); + + if (retries === maxRetries) { + console.error( + "Max retries reached. Failed to send message with attachment." + ); + await callback({ + ...response, + text: "Sorry, I encountered an error while trying to send the video attachment. Please try again later.", + }); + break; + } + + // Wait for a short delay before retrying + await new Promise((resolve) => setTimeout(resolve, 2000)); + } + } - return response; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", - }, - }, - { - user: "{{user2}}", - content: { - text: "Downloading the YouTube video now, one sec", - action: "DOWNLOAD_MEDIA", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Can you grab this video for me? https://vimeo.com/123456789", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure thing, I'll download that Vimeo video for you", - action: "DOWNLOAD_MEDIA", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "I need this video downloaded: https://www.youtube.com/watch?v=abcdefg", - }, - }, - { - user: "{{user2}}", - content: { - text: "No problem, I'm on it. I'll have that YouTube video downloaded in a jiffy", - action: "DOWNLOAD_MEDIA", - }, - }, - ], - ] as ActionExample[][], + return response; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + }, + }, + { + user: "{{user2}}", + content: { + text: "Downloading the YouTube video now, one sec", + action: "DOWNLOAD_MEDIA", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can you grab this video for me? https://vimeo.com/123456789", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure thing, I'll download that Vimeo video for you", + action: "DOWNLOAD_MEDIA", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "I need this video downloaded: https://www.youtube.com/watch?v=abcdefg", + }, + }, + { + user: "{{user2}}", + content: { + text: "No problem, I'm on it. I'll have that YouTube video downloaded in a jiffy", + action: "DOWNLOAD_MEDIA", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/clients/discord/actions/joinvoice.ts b/core/src/clients/discord/actions/joinvoice.ts index d5fbb6c762..f63194be86 100644 --- a/core/src/clients/discord/actions/joinvoice.ts +++ b/core/src/clients/discord/actions/joinvoice.ts @@ -2,137 +2,143 @@ // src/actions/joinVoice import { joinVoiceChannel } from "@discordjs/voice"; import { - Channel, - ChannelType, - Client, - Message as DiscordMessage, - Guild, - GuildMember, + Channel, + ChannelType, + Client, + Message as DiscordMessage, + Guild, + GuildMember, } from "discord.js"; import { composeContext } from "../../../core/context.ts"; import { log_to_file } from "../../../core/logger.ts"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + State, } from "../../../core/types.ts"; export default { - name: "JOIN_VOICE", - similes: [ - "JOIN_VOICE", - "JOIN_VC", - "JOIN_VOICE_CHAT", - "JOIN_VOICE_CHANNEL", - "JOIN_MEETING", - "JOIN_CALL", - ], - validate: async (_runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - // not a discord message - return false; - } - - if (!state.discordClient) { - return; - } + name: "JOIN_VOICE", + similes: [ + "JOIN_VOICE", + "JOIN_VC", + "JOIN_VOICE_CHAT", + "JOIN_VOICE_CHANNEL", + "JOIN_MEETING", + "JOIN_CALL", + ], + validate: async ( + _runtime: IAgentRuntime, + message: Memory, + state: State + ) => { + if (message.content.source !== "discord") { + // not a discord message + return false; + } - // did they say something about joining a voice channel? if not, don't validate - const keywords = [ - "join", - "come to", - "come on", - "enter", - "voice", - "chat", - "talk", - "call", - "hop on", - "get on", - "vc", - "meeting", - "discussion", - ]; - if ( - !keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword), - ) - ) { - return false; - } + if (!state.discordClient) { + return; + } - const client = state.discordClient as Client; + // did they say something about joining a voice channel? if not, don't validate + const keywords = [ + "join", + "come to", + "come on", + "enter", + "voice", + "chat", + "talk", + "call", + "hop on", + "get on", + "vc", + "meeting", + "discussion", + ]; + if ( + !keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword) + ) + ) { + return false; + } - // Check if the client is connected to any voice channel - const isConnectedToVoice = client.voice.adapters.size === 0; + const client = state.discordClient as Client; - return isConnectedToVoice; - }, - description: "Join a voice channel to participate in voice chat.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - ): Promise => { - if (!state) { - console.error("State is not available."); - } + // Check if the client is connected to any voice channel + const isConnectedToVoice = client.voice.adapters.size === 0; - // We normalize data in from voice channels - const discordMessage = (state.discordChannel || - state.discordMessage) as DiscordMessage; + return isConnectedToVoice; + }, + description: "Join a voice channel to participate in voice chat.", + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State + ): Promise => { + if (!state) { + console.error("State is not available."); + } - if (!discordMessage.content) { - discordMessage.content = message.content.text; - } + // We normalize data in from voice channels + const discordMessage = (state.discordChannel || + state.discordMessage) as DiscordMessage; - const id = (discordMessage as DiscordMessage).guild?.id as string; - const client = state.discordClient as Client; - const voiceChannels = ( - client.guilds.cache.get(id) as Guild - ).channels.cache.filter( - (channel: Channel) => channel.type === ChannelType.GuildVoice, - ); + if (!discordMessage.content) { + discordMessage.content = message.content.text; + } - const messageContent = discordMessage.content; + const id = (discordMessage as DiscordMessage).guild?.id as string; + const client = state.discordClient as Client; + const voiceChannels = ( + client.guilds.cache.get(id) as Guild + ).channels.cache.filter( + (channel: Channel) => channel.type === ChannelType.GuildVoice + ); - const targetChannel = voiceChannels.find((channel) => { - const name = (channel as { name: string }).name.toLowerCase(); + const messageContent = discordMessage.content; - // remove all non-alphanumeric characters (keep spaces between words) - const replacedName = name.replace(/[^a-z0-9 ]/g, ""); + const targetChannel = voiceChannels.find((channel) => { + const name = (channel as { name: string }).name.toLowerCase(); - return ( - name.includes(messageContent) || - messageContent.includes(name) || - replacedName.includes(messageContent) || - messageContent.includes(replacedName) - ); - }); + // remove all non-alphanumeric characters (keep spaces between words) + const replacedName = name.replace(/[^a-z0-9 ]/g, ""); - if (targetChannel) { - joinVoiceChannel({ - channelId: targetChannel.id, - guildId: (discordMessage as DiscordMessage).guild?.id as string, - adapterCreator: (client.guilds.cache.get(id) as Guild) - .voiceAdapterCreator, - }); - return true; - } else { - const member = (discordMessage as DiscordMessage).member as GuildMember; - if (member?.voice?.channel) { - joinVoiceChannel({ - channelId: member.voice.channel.id, - guildId: (discordMessage as DiscordMessage).guild?.id as string, - adapterCreator: (client.guilds.cache.get(id) as Guild) - .voiceAdapterCreator, + return ( + name.includes(messageContent) || + messageContent.includes(name) || + replacedName.includes(messageContent) || + messageContent.includes(replacedName) + ); }); - return true; - } - const messageTemplate = ` + if (targetChannel) { + joinVoiceChannel({ + channelId: targetChannel.id, + guildId: (discordMessage as DiscordMessage).guild?.id as string, + adapterCreator: (client.guilds.cache.get(id) as Guild) + .voiceAdapterCreator, + }); + return true; + } else { + const member = (discordMessage as DiscordMessage) + .member as GuildMember; + if (member?.voice?.channel) { + joinVoiceChannel({ + channelId: member.voice.channel.id, + guildId: (discordMessage as DiscordMessage).guild + ?.id as string, + adapterCreator: (client.guilds.cache.get(id) as Guild) + .voiceAdapterCreator, + }); + return true; + } + + const messageTemplate = ` The user has requested to join a voice channel. Here is the list of channels available in the server: {{voiceChannels}} @@ -144,197 +150,203 @@ Please respond with the name of the voice channel which the bot should join. Try You should only respond with the name of the voice channel or none, no commentary or additional information should be included. `; - const guessState = { - userMessage: message.content.text, - voiceChannels: voiceChannels - .map((channel) => (channel as { name: string }).name) - .join("\n"), - }; + const guessState = { + userMessage: message.content.text, + voiceChannels: voiceChannels + .map((channel) => (channel as { name: string }).name) + .join("\n"), + }; - const context = composeContext({ - template: messageTemplate, - state: guessState as unknown as State, - }); + const context = composeContext({ + template: messageTemplate, + state: guessState as unknown as State, + }); - const datestr = new Date().toUTCString().replace(/:/g, "-"); + const datestr = new Date().toUTCString().replace(/:/g, "-"); - // log context to file - log_to_file(`${state.agentName}_${datestr}_joinvoice_context`, context); + // log context to file + log_to_file( + `${state.agentName}_${datestr}_joinvoice_context`, + context + ); - const responseContent = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); + const responseContent = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); - // log response to file - log_to_file( - `${state.agentName}_${datestr}_joinvoice_response`, - responseContent, - ); + // log response to file + log_to_file( + `${state.agentName}_${datestr}_joinvoice_response`, + responseContent + ); - runtime.databaseAdapter.log({ - body: { message, context, response: responseContent }, - userId: message.userId, - roomId: message.roomId, - type: "joinvoice", - }); + runtime.databaseAdapter.log({ + body: { message, context, response: responseContent }, + userId: message.userId, + roomId: message.roomId, + type: "joinvoice", + }); - if (responseContent && responseContent.trim().length > 0) { - // join the voice channel - const channelName = responseContent.toLowerCase(); + if (responseContent && responseContent.trim().length > 0) { + // join the voice channel + const channelName = responseContent.toLowerCase(); - const targetChannel = voiceChannels.find((channel) => { - const name = (channel as { name: string }).name.toLowerCase(); + const targetChannel = voiceChannels.find((channel) => { + const name = ( + channel as { name: string } + ).name.toLowerCase(); - // remove all non-alphanumeric characters (keep spaces between words) - const replacedName = name.replace(/[^a-z0-9 ]/g, ""); + // remove all non-alphanumeric characters (keep spaces between words) + const replacedName = name.replace(/[^a-z0-9 ]/g, ""); - return ( - name.includes(channelName) || - channelName.includes(name) || - replacedName.includes(channelName) || - channelName.includes(replacedName) - ); - }); + return ( + name.includes(channelName) || + channelName.includes(name) || + replacedName.includes(channelName) || + channelName.includes(replacedName) + ); + }); - if (targetChannel) { - joinVoiceChannel({ - channelId: targetChannel.id, - guildId: (discordMessage as DiscordMessage).guild?.id as string, - adapterCreator: (client.guilds.cache.get(id) as Guild) - .voiceAdapterCreator, - }); - return true; - } - } + if (targetChannel) { + joinVoiceChannel({ + channelId: targetChannel.id, + guildId: (discordMessage as DiscordMessage).guild + ?.id as string, + adapterCreator: (client.guilds.cache.get(id) as Guild) + .voiceAdapterCreator, + }); + return true; + } + } - await (discordMessage as DiscordMessage).reply( - "I couldn't figure out which channel you wanted me to join.", - ); - return false; - } - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Hey, let's jump into the 'General' voice and chat", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sounds good", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, can you join the vc, I want to discuss our strat", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure I'll join right now", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "hey {{user2}}, we're having a team meeting in the 'conference' voice channel, plz join us", - }, - }, - { - user: "{{user2}}", - content: { - text: "OK see you there", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, let's have a quick voice chat in the 'Lounge' channel.", - }, - }, - { - user: "{{user2}}", - content: { - text: "kk be there in a sec", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Hey {{user2}}, can you join me in the 'Music' voice channel", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "join voice chat with us {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "coming", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "hop in vc {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "joining now", - action: "JOIN_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "get in vc with us {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "im in", - action: "JOIN_VOICE", - }, - }, - ], - ] as ActionExample[][], + await (discordMessage as DiscordMessage).reply( + "I couldn't figure out which channel you wanted me to join." + ); + return false; + } + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Hey, let's jump into the 'General' voice and chat", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sounds good", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, can you join the vc, I want to discuss our strat", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure I'll join right now", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "hey {{user2}}, we're having a team meeting in the 'conference' voice channel, plz join us", + }, + }, + { + user: "{{user2}}", + content: { + text: "OK see you there", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, let's have a quick voice chat in the 'Lounge' channel.", + }, + }, + { + user: "{{user2}}", + content: { + text: "kk be there in a sec", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Hey {{user2}}, can you join me in the 'Music' voice channel", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "join voice chat with us {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "coming", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "hop in vc {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "joining now", + action: "JOIN_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "get in vc with us {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "im in", + action: "JOIN_VOICE", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/clients/discord/actions/leavevoice.ts b/core/src/clients/discord/actions/leavevoice.ts index b2a384e2d7..11b5a79364 100644 --- a/core/src/clients/discord/actions/leavevoice.ts +++ b/core/src/clients/discord/actions/leavevoice.ts @@ -1,226 +1,226 @@ // src/actions/leaveVoice import { getVoiceConnection } from "@discordjs/voice"; import { - Channel, - ChannelType, - Client, - Message as DiscordMessage, + Channel, + ChannelType, + Client, + Message as DiscordMessage, } from "discord.js"; import { - Action, - ActionExample, - IAgentRuntime, - Memory, - State, + Action, + ActionExample, + IAgentRuntime, + Memory, + State, } from "../../../core/types.ts"; export default { - name: "LEAVE_VOICE", - similes: [ - "LEAVE_VOICE", - "LEAVE_VC", - "LEAVE_VOICE_CHAT", - "LEAVE_VOICE_CHANNEL", - "LEAVE_MEETING", - "LEAVE_CALL", - ], - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - // not a discord message - return false; - } + name: "LEAVE_VOICE", + similes: [ + "LEAVE_VOICE", + "LEAVE_VC", + "LEAVE_VOICE_CHAT", + "LEAVE_VOICE_CHANNEL", + "LEAVE_MEETING", + "LEAVE_CALL", + ], + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + if (message.content.source !== "discord") { + // not a discord message + return false; + } - if (!state.discordClient) { - return false; - } + if (!state.discordClient) { + return false; + } - const keywords = [ - "leave", - "exit", - "stop", - "quit", - "get off", - "get out", - "bye", - "cya", - "see you", - "hop off", - "get off", - "voice", - "vc", - "chat", - "call", - "meeting", - "discussion", - ]; - if ( - !keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword), - ) - ) { - return false; - } + const keywords = [ + "leave", + "exit", + "stop", + "quit", + "get off", + "get out", + "bye", + "cya", + "see you", + "hop off", + "get off", + "voice", + "vc", + "chat", + "call", + "meeting", + "discussion", + ]; + if ( + !keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword) + ) + ) { + return false; + } - const client = state.discordClient as Client; + const client = state.discordClient as Client; - // Check if the client is connected to any voice channel - const isConnectedToVoice = client.voice.adapters.size > 0; + // Check if the client is connected to any voice channel + const isConnectedToVoice = client.voice.adapters.size > 0; - return isConnectedToVoice; - }, - description: "Leave the current voice channel.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - state: State, - ): Promise => { - if (!state.discordClient) { - return; - } + return isConnectedToVoice; + }, + description: "Leave the current voice channel.", + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State + ): Promise => { + if (!state.discordClient) { + return; + } - const discordMessage = (state.discordMessage || - state.discordChannel) as DiscordMessage; + const discordMessage = (state.discordMessage || + state.discordChannel) as DiscordMessage; - if (!discordMessage) { - throw new Error("Discord message is not available in the state."); - } - const voiceChannels = (state.discordClient as Client)?.guilds.cache - .get((discordMessage as DiscordMessage).guild?.id as string) - ?.channels.cache.filter( - (channel: Channel) => channel.type === ChannelType.GuildVoice, - ); + if (!discordMessage) { + throw new Error("Discord message is not available in the state."); + } + const voiceChannels = (state.discordClient as Client)?.guilds.cache + .get((discordMessage as DiscordMessage).guild?.id as string) + ?.channels.cache.filter( + (channel: Channel) => channel.type === ChannelType.GuildVoice + ); - voiceChannels?.forEach((channel: Channel) => { - const connection = getVoiceConnection( - (discordMessage as DiscordMessage).guild?.id as string, - ); - if (connection) { - connection.destroy(); - } - }); - return true; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Hey {{user2}} please leave the voice channel", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "I have to go now but thanks for the chat", - }, - }, - { - user: "{{user2}}", - content: { - text: "You too, talk to you later", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Great call everyone, hopping off now", - action: "LEAVE_VOICE", - }, - }, - { - user: "{{user2}}", - content: { - text: "Agreed, I'll hop off too", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Hey {{user2}} I need you to step away from the voice chat for a bit", - }, - }, - { - user: "{{user2}}", - content: { - text: "No worries, I'll leave the voice channel", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "{{user2}}, I think we covered everything, you can leave the voice chat now", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sounds good, see you both later", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "leave voice {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "ok leaving", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "plz leave the voice chat {{user2}}", - }, - }, - { - user: "{{user2}}", - content: { - text: "aight im out", - action: "LEAVE_VOICE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "yo {{user2}} gtfo the vc", - }, - }, - { - user: "{{user2}}", - content: { - text: "sorry, talk to you later", - action: "LEAVE_VOICE", - }, - }, - ], - ] as ActionExample[][], + voiceChannels?.forEach((channel: Channel) => { + const connection = getVoiceConnection( + (discordMessage as DiscordMessage).guild?.id as string + ); + if (connection) { + connection.destroy(); + } + }); + return true; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Hey {{user2}} please leave the voice channel", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "I have to go now but thanks for the chat", + }, + }, + { + user: "{{user2}}", + content: { + text: "You too, talk to you later", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Great call everyone, hopping off now", + action: "LEAVE_VOICE", + }, + }, + { + user: "{{user2}}", + content: { + text: "Agreed, I'll hop off too", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Hey {{user2}} I need you to step away from the voice chat for a bit", + }, + }, + { + user: "{{user2}}", + content: { + text: "No worries, I'll leave the voice channel", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "{{user2}}, I think we covered everything, you can leave the voice chat now", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sounds good, see you both later", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "leave voice {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "ok leaving", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "plz leave the voice chat {{user2}}", + }, + }, + { + user: "{{user2}}", + content: { + text: "aight im out", + action: "LEAVE_VOICE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "yo {{user2}} gtfo the vc", + }, + }, + { + user: "{{user2}}", + content: { + text: "sorry, talk to you later", + action: "LEAVE_VOICE", + }, + }, + ], + ] as ActionExample[][], } as Action; diff --git a/core/src/clients/discord/actions/summarize_conversation.ts b/core/src/clients/discord/actions/summarize_conversation.ts index 8404e6de6e..1eac7483d6 100644 --- a/core/src/clients/discord/actions/summarize_conversation.ts +++ b/core/src/clients/discord/actions/summarize_conversation.ts @@ -1,19 +1,23 @@ import { composeContext } from "../../../core/context.ts"; -import { generateText, splitChunks, trimTokens } from "../../../core/generation.ts"; +import { + generateText, + splitChunks, + trimTokens, +} from "../../../core/generation.ts"; import { log_to_file } from "../../../core/logger.ts"; import { getActorDetails } from "../../../core/messages.ts"; import models from "../../../core/models.ts"; import { parseJSONObjectFromText } from "../../../core/parsing.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Media, - Memory, - ModelClass, - State, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Media, + Memory, + ModelClass, + State, } from "../../../core/types.ts"; import fs from "fs"; export const summarizationTemplate = `# Summarized so far (we are adding to this) @@ -46,370 +50,373 @@ Your response must be formatted as a JSON block with this structure: `; const getDateRange = async ( - runtime: IAgentRuntime, - message: Memory, - state: State, -) => { - state = (await runtime.composeState(message)) as State; - - const context = composeContext({ - state, - template: dateRangeTemplate, - }); - - for (let i = 0; i < 5; i++) { - const response = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); - console.log("response", response); - // try parsing to a json object - const parsedResponse = parseJSONObjectFromText(response) as { - objective: string; - start: string | number; - end: string | number; - } | null; - // see if it contains objective, start and end - if (parsedResponse) { - if ( - parsedResponse.objective && - parsedResponse.start && - parsedResponse.end - ) { - // TODO: parse start and end into timestamps - const startIntegerString = (parsedResponse.start as string).match( - /\d+/, - )?.[0]; - const endIntegerString = (parsedResponse.end as string).match( - /\d+/, - )?.[0]; - - // parse multiplier - const multipliers = { - second: 1 * 1000, - minute: 60 * 1000, - hour: 3600 * 1000, - day: 86400 * 1000, - }; - - const startMultiplier = (parsedResponse.start as string).match( - /second|minute|hour|day/, - )?.[0]; - const endMultiplier = (parsedResponse.end as string).match( - /second|minute|hour|day/, - )?.[0]; - - const startInteger = startIntegerString - ? parseInt(startIntegerString) - : 0; - const endInteger = endIntegerString ? parseInt(endIntegerString) : 0; - - // multiply by multiplier - const startTime = - startInteger * - multipliers[startMultiplier as keyof typeof multipliers]; - - console.log("startTime", startTime); - - const endTime = - endInteger * multipliers[endMultiplier as keyof typeof multipliers]; - - console.log("endTime", endTime); - - // get the current time and subtract the start and end times - parsedResponse.start = Date.now() - startTime; - parsedResponse.end = Date.now() - endTime; - - return parsedResponse; - } - } - } -}; - -const summarizeAction = { - name: "SUMMARIZE_CONVERSATION", - similes: [ - "RECAP", - "RECAP_CONVERSATION", - "SUMMARIZE_CHAT", - "SUMMARIZATION", - "CHAT_SUMMARY", - "CONVERSATION_SUMMARY", - ], - description: "Summarizes the conversation and attachments.", - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - return false; - } - // only show if one of the keywords are in the message - const keywords: string[] = [ - "summarize", - "summarization", - "summary", - "recap", - "report", - "overview", - "review", - "rundown", - "wrap-up", - "brief", - "debrief", - "abstract", - "synopsis", - "outline", - "digest", - "abridgment", - "condensation", - "encapsulation", - "essence", - "gist", - "main points", - "key points", - "key takeaways", - "bulletpoint", - "highlights", - "tldr", - "tl;dr", - "in a nutshell", - "bottom line", - "long story short", - "sum up", - "sum it up", - "short version", - "bring me up to speed", - "catch me up", - ]; - return keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword.toLowerCase()), - ); - }, - handler: async ( runtime: IAgentRuntime, message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { + state: State +) => { state = (await runtime.composeState(message)) as State; - const userId = runtime.agentId; - - const callbackData: Content = { - text: "", // fill in later - action: "SUMMARIZATION_RESPONSE", - source: message.content.source, - attachments: [], - }; - const { roomId } = message; - - // 1. extract date range from the message - const dateRange = await getDateRange(runtime, message, state); - if (!dateRange) { - console.error("Couldn't get date range from message"); - return; - } - - console.log("dateRange", dateRange); - const { objective, start, end } = dateRange; - - // 2. get these memories from the database - const memories = await runtime.messageManager.getMemories({ - roomId, - // subtract start from current time - start: parseInt(start as string), - end: parseInt(end as string), - count: 10000, - unique: false, - }); - - console.log("memories", memories); - - const actors = await getActorDetails({ - runtime: runtime as IAgentRuntime, - roomId, - }); - - console.log("actors", actors); - - const actorMap = new Map(actors.map((actor) => [actor.id, actor])); - - const formattedMemories = memories - .map((memory) => { - const attachments = memory.content.attachments - ?.map((attachment: Media) => { - return `---\nAttachment: ${attachment.id}\n${attachment.description}\n${attachment.text}\n---`; - }) - .join("\n"); - return `${actorMap.get(memory.userId)?.name ?? "Unknown User"} (${actorMap.get(memory.userId)?.username ?? ""}): ${memory.content.text}\n${attachments}`; - }) - .join("\n"); - - let currentSummary = ""; - - const model = models[runtime.character.settings.model]; - const chunkSize = model.settings.maxContextLength - 1000; - - const chunks = await splitChunks( - runtime, - formattedMemories, - chunkSize, - 0, - "gpt-4o-mini", - ); - - console.log("chunks ", chunks.length); - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - state.memoriesWithAttachments = formattedMemories; - state.objective = objective; - - for (let i = 0; i < chunks.length; i++) { - console.log("chunk", i); - const chunk = chunks[i]; - state.currentSummary = currentSummary; - state.currentChunk = chunk; - const context = composeContext({ + const context = composeContext({ state, - // make sure it fits, we can pad the tokens a bit - template: trimTokens( - summarizationTemplate, - chunkSize + 500, - "gpt-4o-mini", - ), - }); - - log_to_file( - `${state.agentName}_${datestr}_summarization_context`, - context, - ); - - const summary = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); - - log_to_file( - `${state.agentName}_${datestr}_summarization_response_${i}`, - summary, - ); - - currentSummary = currentSummary + "\n" + summary; - } - - // log context to file - log_to_file( - `${state.agentName}_${datestr}_summarization_summary`, - currentSummary, - ); - - // call callback with it -- twitter and discord client can separately handle what to do, IMO we may way to add gists so the agent can post a gist and link to it later + template: dateRangeTemplate, + }); - if (!currentSummary) { - console.error("No summary found, that's not good!"); - return; + for (let i = 0; i < 5; i++) { + const response = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + console.log("response", response); + // try parsing to a json object + const parsedResponse = parseJSONObjectFromText(response) as { + objective: string; + start: string | number; + end: string | number; + } | null; + // see if it contains objective, start and end + if (parsedResponse) { + if ( + parsedResponse.objective && + parsedResponse.start && + parsedResponse.end + ) { + // TODO: parse start and end into timestamps + const startIntegerString = ( + parsedResponse.start as string + ).match(/\d+/)?.[0]; + const endIntegerString = (parsedResponse.end as string).match( + /\d+/ + )?.[0]; + + // parse multiplier + const multipliers = { + second: 1 * 1000, + minute: 60 * 1000, + hour: 3600 * 1000, + day: 86400 * 1000, + }; + + const startMultiplier = (parsedResponse.start as string).match( + /second|minute|hour|day/ + )?.[0]; + const endMultiplier = (parsedResponse.end as string).match( + /second|minute|hour|day/ + )?.[0]; + + const startInteger = startIntegerString + ? parseInt(startIntegerString) + : 0; + const endInteger = endIntegerString + ? parseInt(endIntegerString) + : 0; + + // multiply by multiplier + const startTime = + startInteger * + multipliers[startMultiplier as keyof typeof multipliers]; + + console.log("startTime", startTime); + + const endTime = + endInteger * + multipliers[endMultiplier as keyof typeof multipliers]; + + console.log("endTime", endTime); + + // get the current time and subtract the start and end times + parsedResponse.start = Date.now() - startTime; + parsedResponse.end = Date.now() - endTime; + + return parsedResponse; + } + } } +}; - callbackData.text = currentSummary.trim(); - if ( - callbackData.text && - (currentSummary.trim()?.split("\n").length < 4 || - currentSummary.trim()?.split(" ").length < 100) - ) { - callbackData.text = `Here is the summary: +const summarizeAction = { + name: "SUMMARIZE_CONVERSATION", + similes: [ + "RECAP", + "RECAP_CONVERSATION", + "SUMMARIZE_CHAT", + "SUMMARIZATION", + "CHAT_SUMMARY", + "CONVERSATION_SUMMARY", + ], + description: "Summarizes the conversation and attachments.", + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + if (message.content.source !== "discord") { + return false; + } + // only show if one of the keywords are in the message + const keywords: string[] = [ + "summarize", + "summarization", + "summary", + "recap", + "report", + "overview", + "review", + "rundown", + "wrap-up", + "brief", + "debrief", + "abstract", + "synopsis", + "outline", + "digest", + "abridgment", + "condensation", + "encapsulation", + "essence", + "gist", + "main points", + "key points", + "key takeaways", + "bulletpoint", + "highlights", + "tldr", + "tl;dr", + "in a nutshell", + "bottom line", + "long story short", + "sum up", + "sum it up", + "short version", + "bring me up to speed", + "catch me up", + ]; + return keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword.toLowerCase()) + ); + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + state = (await runtime.composeState(message)) as State; + const userId = runtime.agentId; + + const callbackData: Content = { + text: "", // fill in later + action: "SUMMARIZATION_RESPONSE", + source: message.content.source, + attachments: [], + }; + const { roomId } = message; + + // 1. extract date range from the message + const dateRange = await getDateRange(runtime, message, state); + if (!dateRange) { + console.error("Couldn't get date range from message"); + return; + } + + console.log("dateRange", dateRange); + + const { objective, start, end } = dateRange; + + // 2. get these memories from the database + const memories = await runtime.messageManager.getMemories({ + roomId, + // subtract start from current time + start: parseInt(start as string), + end: parseInt(end as string), + count: 10000, + unique: false, + }); + + console.log("memories", memories); + + const actors = await getActorDetails({ + runtime: runtime as IAgentRuntime, + roomId, + }); + + console.log("actors", actors); + + const actorMap = new Map(actors.map((actor) => [actor.id, actor])); + + const formattedMemories = memories + .map((memory) => { + const attachments = memory.content.attachments + ?.map((attachment: Media) => { + return `---\nAttachment: ${attachment.id}\n${attachment.description}\n${attachment.text}\n---`; + }) + .join("\n"); + return `${actorMap.get(memory.userId)?.name ?? "Unknown User"} (${actorMap.get(memory.userId)?.username ?? ""}): ${memory.content.text}\n${attachments}`; + }) + .join("\n"); + + let currentSummary = ""; + + const model = models[runtime.character.settings.model]; + const chunkSize = model.settings.maxContextLength - 1000; + + const chunks = await splitChunks( + runtime, + formattedMemories, + chunkSize, + 0, + "gpt-4o-mini" + ); + + console.log("chunks ", chunks.length); + const datestr = new Date().toUTCString().replace(/:/g, "-"); + + state.memoriesWithAttachments = formattedMemories; + state.objective = objective; + + for (let i = 0; i < chunks.length; i++) { + console.log("chunk", i); + const chunk = chunks[i]; + state.currentSummary = currentSummary; + state.currentChunk = chunk; + const context = composeContext({ + state, + // make sure it fits, we can pad the tokens a bit + template: trimTokens( + summarizationTemplate, + chunkSize + 500, + "gpt-4o-mini" + ), + }); + + log_to_file( + `${state.agentName}_${datestr}_summarization_context`, + context + ); + + const summary = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + + log_to_file( + `${state.agentName}_${datestr}_summarization_response_${i}`, + summary + ); + + currentSummary = currentSummary + "\n" + summary; + } + + // log context to file + log_to_file( + `${state.agentName}_${datestr}_summarization_summary`, + currentSummary + ); + + // call callback with it -- twitter and discord client can separately handle what to do, IMO we may way to add gists so the agent can post a gist and link to it later + + if (!currentSummary) { + console.error("No summary found, that's not good!"); + return; + } + + callbackData.text = currentSummary.trim(); + if ( + callbackData.text && + (currentSummary.trim()?.split("\n").length < 4 || + currentSummary.trim()?.split(" ").length < 100) + ) { + callbackData.text = `Here is the summary: \`\`\`md ${currentSummary.trim()} \`\`\` `; - await callback(callbackData); - } else if (currentSummary.trim()) { - const summaryFilename = `content_cache/conversation_summary_${Date.now()}.txt`; - // save the summary to a file - fs.writeFileSync(summaryFilename, currentSummary); - await callback( - { - ...callbackData, - text: `I've attached the summary of the conversation from \`${new Date(parseInt(start as string)).toString()}\` to \`${new Date(parseInt(end as string)).toString()}\` as a text file.`, - }, - [summaryFilename], - ); - } else { - console.warn( - "Empty response from summarize conversation action, skipping", - ); - } - - return callbackData; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "```js\nconst x = 10\n```", - }, - }, - { - user: "{{user1}}", - content: { - text: "can you give me a detailed report on what we're talking about?", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure, no problem, give me a minute to get that together for you", - action: "SUMMARIZE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "please summarize the conversation we just had and include this blogpost i'm linking (Attachment: b3e12)", - }, - }, - { - user: "{{user2}}", - content: { - text: "sure, give me a sec", - action: "SUMMARIZE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Can you summarize what moon and avf are talking about?", - }, - }, - { - user: "{{user2}}", - content: { - text: "Yeah, just hold on a second while I get that together for you...", - action: "SUMMARIZE", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "i need to write a blog post about farming, can you summarize the discussion from a few hours ago?", - }, - }, - { - user: "{{user2}}", - content: { - text: "no probblem, give me a few minutes to read through everything", - action: "SUMMARIZE", - }, - }, - ], - ] as ActionExample[][], + await callback(callbackData); + } else if (currentSummary.trim()) { + const summaryFilename = `content_cache/conversation_summary_${Date.now()}.txt`; + // save the summary to a file + fs.writeFileSync(summaryFilename, currentSummary); + await callback( + { + ...callbackData, + text: `I've attached the summary of the conversation from \`${new Date(parseInt(start as string)).toString()}\` to \`${new Date(parseInt(end as string)).toString()}\` as a text file.`, + }, + [summaryFilename] + ); + } else { + console.warn( + "Empty response from summarize conversation action, skipping" + ); + } + + return callbackData; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "```js\nconst x = 10\n```", + }, + }, + { + user: "{{user1}}", + content: { + text: "can you give me a detailed report on what we're talking about?", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure, no problem, give me a minute to get that together for you", + action: "SUMMARIZE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "please summarize the conversation we just had and include this blogpost i'm linking (Attachment: b3e12)", + }, + }, + { + user: "{{user2}}", + content: { + text: "sure, give me a sec", + action: "SUMMARIZE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can you summarize what moon and avf are talking about?", + }, + }, + { + user: "{{user2}}", + content: { + text: "Yeah, just hold on a second while I get that together for you...", + action: "SUMMARIZE", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "i need to write a blog post about farming, can you summarize the discussion from a few hours ago?", + }, + }, + { + user: "{{user2}}", + content: { + text: "no probblem, give me a few minutes to read through everything", + action: "SUMMARIZE", + }, + }, + ], + ] as ActionExample[][], } as Action; export default summarizeAction; diff --git a/core/src/clients/discord/actions/transcribe_media.ts b/core/src/clients/discord/actions/transcribe_media.ts index 327557b78e..fa22a4f566 100644 --- a/core/src/clients/discord/actions/transcribe_media.ts +++ b/core/src/clients/discord/actions/transcribe_media.ts @@ -3,14 +3,14 @@ import { generateText } from "../../../core/generation.ts"; import { log_to_file } from "../../../core/logger.ts"; import { parseJSONObjectFromText } from "../../../core/parsing.ts"; import { - Action, - ActionExample, - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, + Action, + ActionExample, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../../../core/types.ts"; import fs from "fs"; export const transcriptionTemplate = `# Transcription of media file @@ -33,177 +33,185 @@ Your response must be formatted as a JSON block with this structure: `; const getMediaAttachmentId = async ( - runtime: IAgentRuntime, - message: Memory, - state: State, -): Promise => { - state = (await runtime.composeState(message)) as State; - - const context = composeContext({ - state, - template: mediaAttachmentIdTemplate, - }); - - for (let i = 0; i < 5; i++) { - const response = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); - console.log("response", response); - - const parsedResponse = parseJSONObjectFromText(response) as { - attachmentId: string; - } | null; - - if (parsedResponse?.attachmentId) { - return parsedResponse.attachmentId; - } - } - return null; -}; - -const transcribeMediaAction = { - name: "TRANSCRIBE_MEDIA", - similes: [ - "TRANSCRIBE_AUDIO", - "TRANSCRIBE_VIDEO", - "MEDIA_TRANSCRIPT", - "VIDEO_TRANSCRIPT", - "AUDIO_TRANSCRIPT", - ], - description: - "Transcribe the full text of an audio or video file that the user has attached.", - validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { - if (message.content.source !== "discord") { - return false; - } - - const keywords: string[] = [ - "transcribe", - "transcript", - "audio", - "video", - "media", - "youtube", - "meeting", - "recording", - "podcast", - "call", - "conference", - "interview", - "speech", - "lecture", - "presentation", - ]; - return keywords.some((keyword) => - message.content.text.toLowerCase().includes(keyword.toLowerCase()), - ); - }, - handler: async ( runtime: IAgentRuntime, message: Memory, - state: State, - options: any, - callback: HandlerCallback, - ) => { + state: State +): Promise => { state = (await runtime.composeState(message)) as State; - const callbackData: Content = { - text: "", // fill in later - action: "TRANSCRIBE_MEDIA_RESPONSE", - source: message.content.source, - attachments: [], - }; - - const attachmentId = await getMediaAttachmentId(runtime, message, state); - if (!attachmentId) { - console.error("Couldn't get media attachment ID from message"); - return; - } + const context = composeContext({ + state, + template: mediaAttachmentIdTemplate, + }); - const attachment = state.recentMessagesData - .filter( - (msg) => msg.content.attachments && msg.content.attachments.length > 0, - ) - .flatMap((msg) => msg.content.attachments) - .find( - (attachment) => - attachment.id.toLowerCase() === attachmentId.toLowerCase(), - ); - - if (!attachment) { - console.error(`Couldn't find attachment with ID ${attachmentId}`); - return; + for (let i = 0; i < 5; i++) { + const response = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); + console.log("response", response); + + const parsedResponse = parseJSONObjectFromText(response) as { + attachmentId: string; + } | null; + + if (parsedResponse?.attachmentId) { + return parsedResponse.attachmentId; + } } + return null; +}; - const mediaTranscript = attachment.text; - - callbackData.text = mediaTranscript.trim(); - - // if callbackData.text is < 4 lines or < 100 words, then we we callback with normal message wrapped in markdown block - if ( - callbackData.text && - (callbackData.text?.split("\n").length < 4 || - callbackData.text?.split(" ").length < 100) - ) { - callbackData.text = `Here is the transcript: +const transcribeMediaAction = { + name: "TRANSCRIBE_MEDIA", + similes: [ + "TRANSCRIBE_AUDIO", + "TRANSCRIBE_VIDEO", + "MEDIA_TRANSCRIPT", + "VIDEO_TRANSCRIPT", + "AUDIO_TRANSCRIPT", + ], + description: + "Transcribe the full text of an audio or video file that the user has attached.", + validate: async (runtime: IAgentRuntime, message: Memory, state: State) => { + if (message.content.source !== "discord") { + return false; + } + + const keywords: string[] = [ + "transcribe", + "transcript", + "audio", + "video", + "media", + "youtube", + "meeting", + "recording", + "podcast", + "call", + "conference", + "interview", + "speech", + "lecture", + "presentation", + ]; + return keywords.some((keyword) => + message.content.text.toLowerCase().includes(keyword.toLowerCase()) + ); + }, + handler: async ( + runtime: IAgentRuntime, + message: Memory, + state: State, + options: any, + callback: HandlerCallback + ) => { + state = (await runtime.composeState(message)) as State; + + const callbackData: Content = { + text: "", // fill in later + action: "TRANSCRIBE_MEDIA_RESPONSE", + source: message.content.source, + attachments: [], + }; + + const attachmentId = await getMediaAttachmentId( + runtime, + message, + state + ); + if (!attachmentId) { + console.error("Couldn't get media attachment ID from message"); + return; + } + + const attachment = state.recentMessagesData + .filter( + (msg) => + msg.content.attachments && + msg.content.attachments.length > 0 + ) + .flatMap((msg) => msg.content.attachments) + .find( + (attachment) => + attachment.id.toLowerCase() === attachmentId.toLowerCase() + ); + + if (!attachment) { + console.error(`Couldn't find attachment with ID ${attachmentId}`); + return; + } + + const mediaTranscript = attachment.text; + + callbackData.text = mediaTranscript.trim(); + + // if callbackData.text is < 4 lines or < 100 words, then we we callback with normal message wrapped in markdown block + if ( + callbackData.text && + (callbackData.text?.split("\n").length < 4 || + callbackData.text?.split(" ").length < 100) + ) { + callbackData.text = `Here is the transcript: \`\`\`md ${mediaTranscript.trim()} \`\`\` `; - await callback(callbackData); - } - // if text is big, let's send as an attachment - else if (callbackData.text) { - const transcriptFilename = `content_cache/transcript_${Date.now()}.txt`; - // save the transcript to a file - fs.writeFileSync(transcriptFilename, callbackData.text); - await callback( - { - ...callbackData, - text: `I've attached the transcript as a text file.`, - }, - [transcriptFilename], - ); - } else { - console.warn("Empty response from transcribe media action, skipping"); - } - - return callbackData; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Please transcribe the audio file I just sent.", - }, - }, - { - user: "{{user2}}", - content: { - text: "Sure, I'll transcribe the full audio for you.", - action: "TRANSCRIBE_MEDIA", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "Can I get a transcript of that video recording?", - }, - }, - { - user: "{{user2}}", - content: { - text: "Absolutely, give me a moment to generate the full transcript of the video.", - action: "TRANSCRIBE_MEDIA", - }, - }, - ], - ] as ActionExample[][], + await callback(callbackData); + } + // if text is big, let's send as an attachment + else if (callbackData.text) { + const transcriptFilename = `content_cache/transcript_${Date.now()}.txt`; + // save the transcript to a file + fs.writeFileSync(transcriptFilename, callbackData.text); + await callback( + { + ...callbackData, + text: `I've attached the transcript as a text file.`, + }, + [transcriptFilename] + ); + } else { + console.warn( + "Empty response from transcribe media action, skipping" + ); + } + + return callbackData; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Please transcribe the audio file I just sent.", + }, + }, + { + user: "{{user2}}", + content: { + text: "Sure, I'll transcribe the full audio for you.", + action: "TRANSCRIBE_MEDIA", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "Can I get a transcript of that video recording?", + }, + }, + { + user: "{{user2}}", + content: { + text: "Absolutely, give me a moment to generate the full transcript of the video.", + action: "TRANSCRIBE_MEDIA", + }, + }, + ], + ] as ActionExample[][], } as Action; export default transcribeMediaAction; diff --git a/core/src/clients/discord/attachments.ts b/core/src/clients/discord/attachments.ts index fff326b3f9..c9c5b1452e 100644 --- a/core/src/clients/discord/attachments.ts +++ b/core/src/clients/discord/attachments.ts @@ -4,283 +4,300 @@ import fs from "fs"; import { IAgentRuntime, Media } from "../../core/types.ts"; import { generateSummary } from "../../services/summary.ts"; export class AttachmentManager { - private attachmentCache: Map = new Map(); - private runtime: IAgentRuntime; + private attachmentCache: Map = new Map(); + private runtime: IAgentRuntime; - constructor(runtime: IAgentRuntime) { - this.runtime = runtime; - } - - async processAttachments( - attachments: Collection | Attachment[], - ): Promise { - const processedAttachments: Media[] = []; - const attachmentCollection = - attachments instanceof Collection - ? attachments - : new Collection(attachments.map((att) => [att.id, att])); - - for (const [, attachment] of attachmentCollection) { - const media = await this.processAttachment(attachment); - if (media) { - processedAttachments.push(media); - } + constructor(runtime: IAgentRuntime) { + this.runtime = runtime; } - return processedAttachments; - } + async processAttachments( + attachments: Collection | Attachment[] + ): Promise { + const processedAttachments: Media[] = []; + const attachmentCollection = + attachments instanceof Collection + ? attachments + : new Collection(attachments.map((att) => [att.id, att])); - async processAttachment(attachment: Attachment): Promise { - if (this.attachmentCache.has(attachment.url)) { - return this.attachmentCache.get(attachment.url)!; - } + for (const [, attachment] of attachmentCollection) { + const media = await this.processAttachment(attachment); + if (media) { + processedAttachments.push(media); + } + } - let media: Media | null = null; - if (attachment.contentType?.startsWith("application/pdf")) { - media = await this.processPdfAttachment(attachment); - } else if (attachment.contentType?.startsWith("text/plain")) { - media = await this.processPlaintextAttachment(attachment); - } else if ( - attachment.contentType?.startsWith("audio/") || - attachment.contentType?.startsWith("video/mp4") - ) { - media = await this.processAudioVideoAttachment(attachment); - } else if (attachment.contentType?.startsWith("image/")) { - media = await this.processImageAttachment(attachment); - } else if ( - attachment.contentType?.startsWith("video/") || - this.runtime.videoService.isVideoUrl(attachment.url) - ) { - media = await this.processVideoAttachment(attachment); - } else { - media = await this.processGenericAttachment(attachment); + return processedAttachments; } - if (media) { - this.attachmentCache.set(attachment.url, media); + async processAttachment(attachment: Attachment): Promise { + if (this.attachmentCache.has(attachment.url)) { + return this.attachmentCache.get(attachment.url)!; + } + + let media: Media | null = null; + if (attachment.contentType?.startsWith("application/pdf")) { + media = await this.processPdfAttachment(attachment); + } else if (attachment.contentType?.startsWith("text/plain")) { + media = await this.processPlaintextAttachment(attachment); + } else if ( + attachment.contentType?.startsWith("audio/") || + attachment.contentType?.startsWith("video/mp4") + ) { + media = await this.processAudioVideoAttachment(attachment); + } else if (attachment.contentType?.startsWith("image/")) { + media = await this.processImageAttachment(attachment); + } else if ( + attachment.contentType?.startsWith("video/") || + this.runtime.videoService.isVideoUrl(attachment.url) + ) { + media = await this.processVideoAttachment(attachment); + } else { + media = await this.processGenericAttachment(attachment); + } + + if (media) { + this.attachmentCache.set(attachment.url, media); + } + return media; } - return media; - } - private async processAudioVideoAttachment( - attachment: Attachment, - ): Promise { - try { - const response = await fetch(attachment.url); - const audioVideoArrayBuffer = await response.arrayBuffer(); + private async processAudioVideoAttachment( + attachment: Attachment + ): Promise { + try { + const response = await fetch(attachment.url); + const audioVideoArrayBuffer = await response.arrayBuffer(); - let audioBuffer: Buffer; - if (attachment.contentType?.startsWith("audio/")) { - audioBuffer = Buffer.from(audioVideoArrayBuffer); - } else if (attachment.contentType?.startsWith("video/mp4")) { - audioBuffer = await this.extractAudioFromMP4(audioVideoArrayBuffer); - } else { - throw new Error("Unsupported audio/video format"); - } + let audioBuffer: Buffer; + if (attachment.contentType?.startsWith("audio/")) { + audioBuffer = Buffer.from(audioVideoArrayBuffer); + } else if (attachment.contentType?.startsWith("video/mp4")) { + audioBuffer = await this.extractAudioFromMP4( + audioVideoArrayBuffer + ); + } else { + throw new Error("Unsupported audio/video format"); + } - const transcription = - await this.runtime.transcriptionService.transcribeAttachment( - audioBuffer, - ); - const { title, description } = await generateSummary( - this.runtime, - transcription, - ); + const transcription = + await this.runtime.transcriptionService.transcribeAttachment( + audioBuffer + ); + const { title, description } = await generateSummary( + this.runtime, + transcription + ); - return { - id: attachment.id, - url: attachment.url, - title: title || "Audio/Video Attachment", - source: attachment.contentType?.startsWith("audio/") - ? "Audio" - : "Video", - description: - description || - "User-uploaded audio/video attachment which has been transcribed", - text: transcription || "Audio/video content not available", - }; - } catch (error) { - console.error( - `Error processing audio/video attachment: ${error.message}`, - ); - return { - id: attachment.id, - url: attachment.url, - title: "Audio/Video Attachment", - source: attachment.contentType?.startsWith("audio/") - ? "Audio" - : "Video", - description: "An audio/video attachment (transcription failed)", - text: `This is an audio/video attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes, Content type: ${attachment.contentType}`, - }; + return { + id: attachment.id, + url: attachment.url, + title: title || "Audio/Video Attachment", + source: attachment.contentType?.startsWith("audio/") + ? "Audio" + : "Video", + description: + description || + "User-uploaded audio/video attachment which has been transcribed", + text: transcription || "Audio/video content not available", + }; + } catch (error) { + console.error( + `Error processing audio/video attachment: ${error.message}` + ); + return { + id: attachment.id, + url: attachment.url, + title: "Audio/Video Attachment", + source: attachment.contentType?.startsWith("audio/") + ? "Audio" + : "Video", + description: "An audio/video attachment (transcription failed)", + text: `This is an audio/video attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes, Content type: ${attachment.contentType}`, + }; + } } - } - private async extractAudioFromMP4(mp4Data: ArrayBuffer): Promise { - // Use a library like 'fluent-ffmpeg' or 'ffmpeg-static' to extract the audio stream from the MP4 data - // and convert it to MP3 or WAV format - // Example using fluent-ffmpeg: - const tempMP4File = `temp_${Date.now()}.mp4`; - const tempAudioFile = `temp_${Date.now()}.mp3`; + private async extractAudioFromMP4(mp4Data: ArrayBuffer): Promise { + // Use a library like 'fluent-ffmpeg' or 'ffmpeg-static' to extract the audio stream from the MP4 data + // and convert it to MP3 or WAV format + // Example using fluent-ffmpeg: + const tempMP4File = `temp_${Date.now()}.mp4`; + const tempAudioFile = `temp_${Date.now()}.mp3`; - try { - // Write the MP4 data to a temporary file - fs.writeFileSync(tempMP4File, Buffer.from(mp4Data)); + try { + // Write the MP4 data to a temporary file + fs.writeFileSync(tempMP4File, Buffer.from(mp4Data)); - // Extract the audio stream and convert it to MP3 - await new Promise((resolve, reject) => { - ffmpeg(tempMP4File) - .outputOptions("-vn") // Disable video output - .audioCodec("libmp3lame") // Set audio codec to MP3 - .save(tempAudioFile) // Save the output to the specified file - .on("end", () => { - resolve(); - }) - .on("error", (err) => { - reject(err); - }) - .run(); - }); + // Extract the audio stream and convert it to MP3 + await new Promise((resolve, reject) => { + ffmpeg(tempMP4File) + .outputOptions("-vn") // Disable video output + .audioCodec("libmp3lame") // Set audio codec to MP3 + .save(tempAudioFile) // Save the output to the specified file + .on("end", () => { + resolve(); + }) + .on("error", (err) => { + reject(err); + }) + .run(); + }); - // Read the converted audio file and return it as a Buffer - const audioData = fs.readFileSync(tempAudioFile); - return audioData; - } finally { - // Clean up the temporary files - if (fs.existsSync(tempMP4File)) { - fs.unlinkSync(tempMP4File); - } - if (fs.existsSync(tempAudioFile)) { - fs.unlinkSync(tempAudioFile); - } + // Read the converted audio file and return it as a Buffer + const audioData = fs.readFileSync(tempAudioFile); + return audioData; + } finally { + // Clean up the temporary files + if (fs.existsSync(tempMP4File)) { + fs.unlinkSync(tempMP4File); + } + if (fs.existsSync(tempAudioFile)) { + fs.unlinkSync(tempAudioFile); + } + } } - } - private async processPdfAttachment(attachment: Attachment): Promise { - try { - const response = await fetch(attachment.url); - const pdfBuffer = await response.arrayBuffer(); - const text = await this.runtime.pdfService.convertPdfToText( - Buffer.from(pdfBuffer), - ); - const { title, description } = await generateSummary(this.runtime, text); + private async processPdfAttachment(attachment: Attachment): Promise { + try { + const response = await fetch(attachment.url); + const pdfBuffer = await response.arrayBuffer(); + const text = await this.runtime.pdfService.convertPdfToText( + Buffer.from(pdfBuffer) + ); + const { title, description } = await generateSummary( + this.runtime, + text + ); - return { - id: attachment.id, - url: attachment.url, - title: title || "PDF Attachment", - source: "PDF", - description: description || "A PDF document", - text: text, - }; - } catch (error) { - console.error(`Error processing PDF attachment: ${error.message}`); - return { - id: attachment.id, - url: attachment.url, - title: "PDF Attachment (conversion failed)", - source: "PDF", - description: "A PDF document that could not be converted to text", - text: `This is a PDF attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes`, - }; + return { + id: attachment.id, + url: attachment.url, + title: title || "PDF Attachment", + source: "PDF", + description: description || "A PDF document", + text: text, + }; + } catch (error) { + console.error(`Error processing PDF attachment: ${error.message}`); + return { + id: attachment.id, + url: attachment.url, + title: "PDF Attachment (conversion failed)", + source: "PDF", + description: + "A PDF document that could not be converted to text", + text: `This is a PDF attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes`, + }; + } } - } - private async processPlaintextAttachment( - attachment: Attachment, - ): Promise { - try { - const response = await fetch(attachment.url); - const text = await response.text(); - const { title, description } = await generateSummary(this.runtime, text); + private async processPlaintextAttachment( + attachment: Attachment + ): Promise { + try { + const response = await fetch(attachment.url); + const text = await response.text(); + const { title, description } = await generateSummary( + this.runtime, + text + ); - return { - id: attachment.id, - url: attachment.url, - title: title || "Plaintext Attachment", - source: "Plaintext", - description: description || "A plaintext document", - text: text, - }; - } catch (error) { - console.error(`Error processing plaintext attachment: ${error.message}`); - return { - id: attachment.id, - url: attachment.url, - title: "Plaintext Attachment (retrieval failed)", - source: "Plaintext", - description: "A plaintext document that could not be retrieved", - text: `This is a plaintext attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes`, - }; + return { + id: attachment.id, + url: attachment.url, + title: title || "Plaintext Attachment", + source: "Plaintext", + description: description || "A plaintext document", + text: text, + }; + } catch (error) { + console.error( + `Error processing plaintext attachment: ${error.message}` + ); + return { + id: attachment.id, + url: attachment.url, + title: "Plaintext Attachment (retrieval failed)", + source: "Plaintext", + description: "A plaintext document that could not be retrieved", + text: `This is a plaintext attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes`, + }; + } } - } - private async processImageAttachment(attachment: Attachment): Promise { - try { - const { description, title } = - await this.runtime.imageDescriptionService.describeImage( - attachment.url, - ); - return { - id: attachment.id, - url: attachment.url, - title: title || "Image Attachment", - source: "Image", - description: description || "An image attachment", - text: description || "Image content not available", - }; - } catch (error) { - console.error(`Error processing image attachment: ${error.message}`); - return this.createFallbackImageMedia(attachment); + private async processImageAttachment( + attachment: Attachment + ): Promise { + try { + const { description, title } = + await this.runtime.imageDescriptionService.describeImage( + attachment.url + ); + return { + id: attachment.id, + url: attachment.url, + title: title || "Image Attachment", + source: "Image", + description: description || "An image attachment", + text: description || "Image content not available", + }; + } catch (error) { + console.error( + `Error processing image attachment: ${error.message}` + ); + return this.createFallbackImageMedia(attachment); + } } - } - private createFallbackImageMedia(attachment: Attachment): Media { - return { - id: attachment.id, - url: attachment.url, - title: "Image Attachment", - source: "Image", - description: "An image attachment (recognition failed)", - text: `This is an image attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes, Content type: ${attachment.contentType}`, - }; - } + private createFallbackImageMedia(attachment: Attachment): Media { + return { + id: attachment.id, + url: attachment.url, + title: "Image Attachment", + source: "Image", + description: "An image attachment (recognition failed)", + text: `This is an image attachment. File name: ${attachment.name}, Size: ${attachment.size} bytes, Content type: ${attachment.contentType}`, + }; + } - private async processVideoAttachment(attachment: Attachment): Promise { - if (this.runtime.videoService.isVideoUrl(attachment.url)) { - const videoInfo = await this.runtime.videoService.processVideo( - attachment.url, - ); - return { - id: attachment.id, - url: attachment.url, - title: videoInfo.title, - source: "YouTube", - description: videoInfo.description, - text: videoInfo.text, - }; - } else { - return { - id: attachment.id, - url: attachment.url, - title: "Video Attachment", - source: "Video", - description: "A video attachment", - text: "Video content not available", - }; + private async processVideoAttachment( + attachment: Attachment + ): Promise { + if (this.runtime.videoService.isVideoUrl(attachment.url)) { + const videoInfo = await this.runtime.videoService.processVideo( + attachment.url + ); + return { + id: attachment.id, + url: attachment.url, + title: videoInfo.title, + source: "YouTube", + description: videoInfo.description, + text: videoInfo.text, + }; + } else { + return { + id: attachment.id, + url: attachment.url, + title: "Video Attachment", + source: "Video", + description: "A video attachment", + text: "Video content not available", + }; + } } - } - private async processGenericAttachment( - attachment: Attachment, - ): Promise { - return { - id: attachment.id, - url: attachment.url, - title: "Generic Attachment", - source: "Generic", - description: "A generic attachment", - text: "Attachment content not available", - }; - } + private async processGenericAttachment( + attachment: Attachment + ): Promise { + return { + id: attachment.id, + url: attachment.url, + title: "Generic Attachment", + source: "Generic", + description: "A generic attachment", + text: "Attachment content not available", + }; + } } diff --git a/core/src/clients/discord/audioMonitor.ts b/core/src/clients/discord/audioMonitor.ts index 65e26ac316..8b1dfa1df0 100644 --- a/core/src/clients/discord/audioMonitor.ts +++ b/core/src/clients/discord/audioMonitor.ts @@ -2,84 +2,84 @@ import { Readable } from "stream"; // Buffers all audio export class AudioMonitor { - private readable: Readable; - private buffers: Buffer[] = []; - private maxSize: number; - private lastFlagged: number = -1; - private ended: boolean = false; + private readable: Readable; + private buffers: Buffer[] = []; + private maxSize: number; + private lastFlagged: number = -1; + private ended: boolean = false; - constructor( - readable: Readable, - maxSize: number, - callback: (buffer: Buffer) => void, - ) { - this.readable = readable; - this.maxSize = maxSize; - this.readable.on("data", (chunk: Buffer) => { - //console.log('AudioMonitor got data'); - if (this.lastFlagged < 0) { - this.lastFlagged = this.buffers.length; - } - this.buffers.push(chunk); - const currentSize = this.buffers.reduce( - (acc, cur) => acc + cur.length, - 0, - ); - while (currentSize > this.maxSize) { - this.buffers.shift(); - this.lastFlagged--; - } - }); - this.readable.on("end", () => { - console.log("AudioMonitor ended"); - this.ended = true; - if (this.lastFlagged < 0) return; - callback(this.getBufferFromStart()); - this.lastFlagged = -1; - }); - this.readable.on("speakingStopped", () => { - if (this.ended) return; - console.log("Speaking stopped"); - if (this.lastFlagged < 0) return; - callback(this.getBufferFromStart()); - }); - this.readable.on("speakingStarted", () => { - if (this.ended) return; - console.log("Speaking started"); - this.reset(); - }); - } + constructor( + readable: Readable, + maxSize: number, + callback: (buffer: Buffer) => void + ) { + this.readable = readable; + this.maxSize = maxSize; + this.readable.on("data", (chunk: Buffer) => { + //console.log('AudioMonitor got data'); + if (this.lastFlagged < 0) { + this.lastFlagged = this.buffers.length; + } + this.buffers.push(chunk); + const currentSize = this.buffers.reduce( + (acc, cur) => acc + cur.length, + 0 + ); + while (currentSize > this.maxSize) { + this.buffers.shift(); + this.lastFlagged--; + } + }); + this.readable.on("end", () => { + console.log("AudioMonitor ended"); + this.ended = true; + if (this.lastFlagged < 0) return; + callback(this.getBufferFromStart()); + this.lastFlagged = -1; + }); + this.readable.on("speakingStopped", () => { + if (this.ended) return; + console.log("Speaking stopped"); + if (this.lastFlagged < 0) return; + callback(this.getBufferFromStart()); + }); + this.readable.on("speakingStarted", () => { + if (this.ended) return; + console.log("Speaking started"); + this.reset(); + }); + } - stop() { - this.readable.removeAllListeners("data"); - this.readable.removeAllListeners("end"); - this.readable.removeAllListeners("speakingStopped"); - this.readable.removeAllListeners("speakingStarted"); - } + stop() { + this.readable.removeAllListeners("data"); + this.readable.removeAllListeners("end"); + this.readable.removeAllListeners("speakingStopped"); + this.readable.removeAllListeners("speakingStarted"); + } - isFlagged() { - return this.lastFlagged >= 0; - } + isFlagged() { + return this.lastFlagged >= 0; + } - getBufferFromFlag() { - if (this.lastFlagged < 0) { - return null; + getBufferFromFlag() { + if (this.lastFlagged < 0) { + return null; + } + const buffer = Buffer.concat(this.buffers.slice(this.lastFlagged)); + return buffer; } - const buffer = Buffer.concat(this.buffers.slice(this.lastFlagged)); - return buffer; - } - getBufferFromStart() { - const buffer = Buffer.concat(this.buffers); - return buffer; - } + getBufferFromStart() { + const buffer = Buffer.concat(this.buffers); + return buffer; + } - reset() { - this.buffers = []; - this.lastFlagged = -1; - } + reset() { + this.buffers = []; + this.lastFlagged = -1; + } - isEnded() { - return this.ended; - } + isEnded() { + return this.ended; + } } diff --git a/core/src/clients/discord/commands.ts b/core/src/clients/discord/commands.ts index 4b0e390dec..feed82117f 100644 --- a/core/src/clients/discord/commands.ts +++ b/core/src/clients/discord/commands.ts @@ -1,42 +1,42 @@ export const commands = [ - // { - // name: "setname", - // description: "Change the agent's name in the database", - // options: [ - // { - // name: "name", - // description: "The new name for the agent", - // type: 3, - // required: true, - // }, - // ], - // }, - // { - // name: "setbio", - // description: "Change the agent's bio in the database", - // options: [ - // { - // name: "bio", - // description: "The new bio for the agent", - // type: 3, - // required: true, - // }, - // ], - // }, - { - name: "joinchannel", - description: "Join the voice channel the user is in", - options: [ - { - name: "channel", - description: "The voice channel to join", - type: 7, - required: true, - }, - ], - }, - { - name: "leavechannel", - description: "Leave the voice channel the user is in", - }, + // { + // name: "setname", + // description: "Change the agent's name in the database", + // options: [ + // { + // name: "name", + // description: "The new name for the agent", + // type: 3, + // required: true, + // }, + // ], + // }, + // { + // name: "setbio", + // description: "Change the agent's bio in the database", + // options: [ + // { + // name: "bio", + // description: "The new bio for the agent", + // type: 3, + // required: true, + // }, + // ], + // }, + { + name: "joinchannel", + description: "Join the voice channel the user is in", + options: [ + { + name: "channel", + description: "The voice channel to join", + type: 7, + required: true, + }, + ], + }, + { + name: "leavechannel", + description: "Leave the voice channel the user is in", + }, ]; diff --git a/core/src/clients/discord/index.ts b/core/src/clients/discord/index.ts index f76608f5ea..39bf52c8dd 100644 --- a/core/src/clients/discord/index.ts +++ b/core/src/clients/discord/index.ts @@ -1,13 +1,13 @@ import { REST } from "@discordjs/rest"; import { - Client, - Events, - GatewayIntentBits, - Guild, - MessageReaction, - Partials, - Routes, - User, + Client, + Events, + GatewayIntentBits, + Guild, + MessageReaction, + Partials, + Routes, + User, } from "discord.js"; import { EventEmitter } from "events"; import { stringToUuid } from "../../core/uuid.ts"; @@ -28,302 +28,313 @@ import channelStateProvider from "./providers/channelState.ts"; import voiceStateProvider from "./providers/voiceState.ts"; export class DiscordClient extends EventEmitter { - apiToken: string; - private client: Client; - private runtime: IAgentRuntime; - character: any; - private messageManager: MessageManager; - private voiceManager: VoiceManager; - - constructor(runtime: IAgentRuntime) { - super(); - this.apiToken = runtime.getSetting("DISCORD_API_TOKEN") as string; - this.client = new Client({ - intents: [ - GatewayIntentBits.Guilds, - GatewayIntentBits.DirectMessages, - GatewayIntentBits.GuildVoiceStates, - GatewayIntentBits.MessageContent, - GatewayIntentBits.GuildMessages, - GatewayIntentBits.DirectMessageTyping, - GatewayIntentBits.GuildMessageTyping, - GatewayIntentBits.GuildMessageReactions, - ], - partials: [ - Partials.Channel, - Partials.Message, - Partials.User, - Partials.Reaction, - ], - }); - - this.runtime = runtime; - this.voiceManager = new VoiceManager(this); - this.messageManager = new MessageManager(this, this.voiceManager); - - this.client.once(Events.ClientReady, this.onClientReady.bind(this)); - this.client.login(this.apiToken); - - this.setupEventListeners(); - this.setupCommands(); - - this.runtime.registerAction(joinvoice); - this.runtime.registerAction(leavevoice); - this.runtime.registerAction(summarize); - this.runtime.registerAction(chat_with_attachments); - this.runtime.registerAction(transcribe_media); - this.runtime.registerAction(download_media); - - this.runtime.providers.push(channelStateProvider); - this.runtime.providers.push(voiceStateProvider); - } - - private setupEventListeners() { - // When joining to a new server - this.client.on("guildCreate", this.handleGuildCreate.bind(this)); - - this.client.on( - Events.MessageReactionAdd, - this.handleReactionAdd.bind(this), - ); - this.client.on( - Events.MessageReactionRemove, - this.handleReactionRemove.bind(this), - ); - - // Handle voice events with the voice manager - this.client.on( - "voiceStateUpdate", - this.voiceManager.handleVoiceStateUpdate.bind(this.voiceManager), - ); - this.client.on( - "userStream", - this.voiceManager.handleUserStream.bind(this.voiceManager), - ); - - // Handle a new message with the message manager - this.client.on( - Events.MessageCreate, - this.messageManager.handleMessage.bind(this.messageManager), - ); - - // Handle a new interaction - this.client.on( - Events.InteractionCreate, - this.handleInteractionCreate.bind(this), - ); - } - - private setupCommands() { - const rest = new REST({ version: "9" }).setToken(this.apiToken); - (async () => { - try { - await rest.put( - Routes.applicationCommands( - this.runtime.getSetting("DISCORD_APPLICATION_ID"), - ), - { body: commands }, - ); - } catch (error) { - console.error(error); - } - })(); - } - - private async onClientReady(readyClient: { user: { tag: any; id: any } }) { - console.log(`Logged in as ${readyClient.user?.tag}`); - console.log("Use this URL to add the bot to your server:"); - console.log( - `https://discord.com/api/oauth2/authorize?client_id=${readyClient.user?.id}&permissions=0&scope=bot%20applications.commands`, - ); - await this.onReady(); - } - - async handleReactionAdd(reaction: MessageReaction, user: User) { - console.log("Reaction added"); - // if (user.bot) return; - - let emoji = reaction.emoji.name; - if (!emoji && reaction.emoji.id) { - emoji = `<:${reaction.emoji.name}:${reaction.emoji.id}>`; + apiToken: string; + private client: Client; + private runtime: IAgentRuntime; + character: any; + private messageManager: MessageManager; + private voiceManager: VoiceManager; + + constructor(runtime: IAgentRuntime) { + super(); + this.apiToken = runtime.getSetting("DISCORD_API_TOKEN") as string; + this.client = new Client({ + intents: [ + GatewayIntentBits.Guilds, + GatewayIntentBits.DirectMessages, + GatewayIntentBits.GuildVoiceStates, + GatewayIntentBits.MessageContent, + GatewayIntentBits.GuildMessages, + GatewayIntentBits.DirectMessageTyping, + GatewayIntentBits.GuildMessageTyping, + GatewayIntentBits.GuildMessageReactions, + ], + partials: [ + Partials.Channel, + Partials.Message, + Partials.User, + Partials.Reaction, + ], + }); + + this.runtime = runtime; + this.voiceManager = new VoiceManager(this); + this.messageManager = new MessageManager(this, this.voiceManager); + + this.client.once(Events.ClientReady, this.onClientReady.bind(this)); + this.client.login(this.apiToken); + + this.setupEventListeners(); + this.setupCommands(); + + this.runtime.registerAction(joinvoice); + this.runtime.registerAction(leavevoice); + this.runtime.registerAction(summarize); + this.runtime.registerAction(chat_with_attachments); + this.runtime.registerAction(transcribe_media); + this.runtime.registerAction(download_media); + + this.runtime.providers.push(channelStateProvider); + this.runtime.providers.push(voiceStateProvider); } - // Fetch the full message if it's a partial - if (reaction.partial) { - try { - await reaction.fetch(); - } catch (error) { - console.error("Something went wrong when fetching the message:", error); - return; - } - } + private setupEventListeners() { + // When joining to a new server + this.client.on("guildCreate", this.handleGuildCreate.bind(this)); - const messageContent = reaction.message.content; - const truncatedContent = - messageContent.length > 100 - ? messageContent.substring(0, 100) + "..." - : messageContent; + this.client.on( + Events.MessageReactionAdd, + this.handleReactionAdd.bind(this) + ); + this.client.on( + Events.MessageReactionRemove, + this.handleReactionRemove.bind(this) + ); - const reactionMessage = `*<${emoji}>: "${truncatedContent}"*`; + // Handle voice events with the voice manager + this.client.on( + "voiceStateUpdate", + this.voiceManager.handleVoiceStateUpdate.bind(this.voiceManager) + ); + this.client.on( + "userStream", + this.voiceManager.handleUserStream.bind(this.voiceManager) + ); + + // Handle a new message with the message manager + this.client.on( + Events.MessageCreate, + this.messageManager.handleMessage.bind(this.messageManager) + ); - const roomId = stringToUuid(reaction.message.channel.id); - const userIdUUID = stringToUuid(user.id); + // Handle a new interaction + this.client.on( + Events.InteractionCreate, + this.handleInteractionCreate.bind(this) + ); + } - // Generate a unique UUID for the reaction - const reactionUUID = stringToUuid( - `${reaction.message.id}-${user.id}-${emoji}`, - ); + private setupCommands() { + const rest = new REST({ version: "9" }).setToken(this.apiToken); + (async () => { + try { + await rest.put( + Routes.applicationCommands( + this.runtime.getSetting("DISCORD_APPLICATION_ID") + ), + { body: commands } + ); + } catch (error) { + console.error(error); + } + })(); + } - // ensure the user id and room id are valid - if (!userIdUUID || !roomId) { - console.error("Invalid user id or room id"); - return; + private async onClientReady(readyClient: { user: { tag: any; id: any } }) { + console.log(`Logged in as ${readyClient.user?.tag}`); + console.log("Use this URL to add the bot to your server:"); + console.log( + `https://discord.com/api/oauth2/authorize?client_id=${readyClient.user?.id}&permissions=0&scope=bot%20applications.commands` + ); + await this.onReady(); } - const agentId = this.runtime.agentId; - const userName = reaction.message.author.username; - const name = reaction.message.author.displayName; - await Promise.all([ - - this.runtime.ensureUserExists( - agentId, - this.client.user.username, - this.runtime.character.name, - "discord", - ), - this.runtime.ensureUserExists(userIdUUID, userName, name, "discord"), - this.runtime.ensureRoomExists(roomId), - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(agentId, roomId), - ]); - - - // Save the reaction as a message - await this.runtime.messageManager.createMemory({ - id: reactionUUID, // This is the ID of the reaction message - userId: userIdUUID, - content: { - text: reactionMessage, - source: "discord", - inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message - }, - roomId, - createdAt: Date.now(), - embedding: embeddingZeroVector, - }); - } - - async handleReactionRemove(reaction: MessageReaction, user: User) { - console.log("Reaction removed"); - // if (user.bot) return; - - let emoji = reaction.emoji.name; - if (!emoji && reaction.emoji.id) { - emoji = `<:${reaction.emoji.name}:${reaction.emoji.id}>`; + + async handleReactionAdd(reaction: MessageReaction, user: User) { + console.log("Reaction added"); + // if (user.bot) return; + + let emoji = reaction.emoji.name; + if (!emoji && reaction.emoji.id) { + emoji = `<:${reaction.emoji.name}:${reaction.emoji.id}>`; + } + + // Fetch the full message if it's a partial + if (reaction.partial) { + try { + await reaction.fetch(); + } catch (error) { + console.error( + "Something went wrong when fetching the message:", + error + ); + return; + } + } + + const messageContent = reaction.message.content; + const truncatedContent = + messageContent.length > 100 + ? messageContent.substring(0, 100) + "..." + : messageContent; + + const reactionMessage = `*<${emoji}>: "${truncatedContent}"*`; + + const roomId = stringToUuid(reaction.message.channel.id); + const userIdUUID = stringToUuid(user.id); + + // Generate a unique UUID for the reaction + const reactionUUID = stringToUuid( + `${reaction.message.id}-${user.id}-${emoji}` + ); + + // ensure the user id and room id are valid + if (!userIdUUID || !roomId) { + console.error("Invalid user id or room id"); + return; + } + const agentId = this.runtime.agentId; + const userName = reaction.message.author.username; + const name = reaction.message.author.displayName; + await Promise.all([ + this.runtime.ensureUserExists( + agentId, + this.client.user.username, + this.runtime.character.name, + "discord" + ), + this.runtime.ensureUserExists( + userIdUUID, + userName, + name, + "discord" + ), + this.runtime.ensureRoomExists(roomId), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom(userIdUUID, roomId), + this.runtime.ensureParticipantInRoom(agentId, roomId), + ]); + + // Save the reaction as a message + await this.runtime.messageManager.createMemory({ + id: reactionUUID, // This is the ID of the reaction message + userId: userIdUUID, + content: { + text: reactionMessage, + source: "discord", + inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message + }, + roomId, + createdAt: Date.now(), + embedding: embeddingZeroVector, + }); } - // Fetch the full message if it's a partial - if (reaction.partial) { - try { - await reaction.fetch(); - } catch (error) { - console.error("Something went wrong when fetching the message:", error); - return; - } + async handleReactionRemove(reaction: MessageReaction, user: User) { + console.log("Reaction removed"); + // if (user.bot) return; + + let emoji = reaction.emoji.name; + if (!emoji && reaction.emoji.id) { + emoji = `<:${reaction.emoji.name}:${reaction.emoji.id}>`; + } + + // Fetch the full message if it's a partial + if (reaction.partial) { + try { + await reaction.fetch(); + } catch (error) { + console.error( + "Something went wrong when fetching the message:", + error + ); + return; + } + } + + const messageContent = reaction.message.content; + const truncatedContent = + messageContent.length > 50 + ? messageContent.substring(0, 50) + "..." + : messageContent; + + const reactionMessage = `*Removed <${emoji} emoji> from: "${truncatedContent}"*`; + + const roomId = stringToUuid(reaction.message.channel.id); + const userIdUUID = stringToUuid(user.id); + + // Generate a unique UUID for the reaction removal + const reactionUUID = stringToUuid( + `${reaction.message.id}-${user.id}-${emoji}-removed` + ); + + const agentId = this.runtime.agentId; + const userName = reaction.message.author.username; + const name = reaction.message.author.displayName; + + console.log("reactionUUID", reactionUUID); + console.log("userIdUUID", userIdUUID); + console.log("roomId", roomId); + console.log("agentId", agentId); + console.log("userName", userName); + console.log("name", name); + + await Promise.all([ + this.runtime.ensureUserExists( + agentId, + this.client.user.username, + this.runtime.character.name, + "discord" + ), + this.runtime.ensureUserExists( + userIdUUID, + userName, + name, + "discord" + ), + this.runtime.ensureRoomExists(roomId), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom(userIdUUID, roomId), + this.runtime.ensureParticipantInRoom(agentId, roomId), + ]); + + try { + // Save the reaction removal as a message + await this.runtime.messageManager.createMemory({ + id: reactionUUID, // This is the ID of the reaction removal message + userId: userIdUUID, + content: { + text: reactionMessage, + source: "discord", + inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message + }, + roomId, + createdAt: Date.now(), + embedding: embeddingZeroVector, + }); + } catch (error) { + console.error("Error creating reaction removal message:", error); + } } - const messageContent = reaction.message.content; - const truncatedContent = - messageContent.length > 50 - ? messageContent.substring(0, 50) + "..." - : messageContent; - - const reactionMessage = `*Removed <${emoji} emoji> from: "${truncatedContent}"*`; - - const roomId = stringToUuid(reaction.message.channel.id); - const userIdUUID = stringToUuid(user.id); - - // Generate a unique UUID for the reaction removal - const reactionUUID = stringToUuid( - `${reaction.message.id}-${user.id}-${emoji}-removed`, - ); - - const agentId = this.runtime.agentId; - const userName = reaction.message.author.username; - const name = reaction.message.author.displayName; - - - console.log("reactionUUID", reactionUUID); - console.log("userIdUUID", userIdUUID); - console.log("roomId", roomId); - console.log("agentId", agentId); - console.log("userName", userName); - console.log("name", name); - - await Promise.all([ - - this.runtime.ensureUserExists( - agentId, - this.client.user.username, - this.runtime.character.name, - "discord", - ), - this.runtime.ensureUserExists(userIdUUID, userName, name, "discord"), - this.runtime.ensureRoomExists(roomId), - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(agentId, roomId), - ]); - - try { - - // Save the reaction removal as a message - await this.runtime.messageManager.createMemory({ - id: reactionUUID, // This is the ID of the reaction removal message - userId: userIdUUID, - content: { - text: reactionMessage, - source: "discord", - inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message - }, - roomId, - createdAt: Date.now(), - embedding: embeddingZeroVector, - }); - } catch (error) { - console.error("Error creating reaction removal message:", error); + private handleGuildCreate(guild: Guild) { + console.log(`Joined guild ${guild.name}`); + this.voiceManager.scanGuild(guild); } - } - - private handleGuildCreate(guild: Guild) { - console.log(`Joined guild ${guild.name}`); - this.voiceManager.scanGuild(guild); - } - - private async handleInteractionCreate(interaction: any) { - if (!interaction.isCommand()) return; - - switch (interaction.commandName) { - case "joinchannel": - await this.voiceManager.handleJoinChannelCommand(interaction); - break; - case "leavechannel": - await this.voiceManager.handleLeaveChannelCommand(interaction); - break; + + private async handleInteractionCreate(interaction: any) { + if (!interaction.isCommand()) return; + + switch (interaction.commandName) { + case "joinchannel": + await this.voiceManager.handleJoinChannelCommand(interaction); + break; + case "leavechannel": + await this.voiceManager.handleLeaveChannelCommand(interaction); + break; + } } - } - private async onReady() { - const guilds = await this.client.guilds.fetch(); - for (const [, guild] of guilds) { - const fullGuild = await guild.fetch(); - this.voiceManager.scanGuild(fullGuild); + private async onReady() { + const guilds = await this.client.guilds.fetch(); + for (const [, guild] of guilds) { + const fullGuild = await guild.fetch(); + this.voiceManager.scanGuild(fullGuild); + } } - } } // Export the DiscordClient class diff --git a/core/src/clients/discord/messages.ts b/core/src/clients/discord/messages.ts index 535a3b4901..02f374a7cc 100644 --- a/core/src/clients/discord/messages.ts +++ b/core/src/clients/discord/messages.ts @@ -3,14 +3,14 @@ import { composeContext } from "../../core/context.ts"; import { log_to_file } from "../../core/logger.ts"; import { embeddingZeroVector } from "../../core/memory.ts"; import { - Content, - HandlerCallback, - IAgentRuntime, - Media, - Memory, - ModelClass, - State, - UUID, + Content, + HandlerCallback, + IAgentRuntime, + Media, + Memory, + ModelClass, + State, + UUID, } from "../../core/types.ts"; import { generateSummary } from "../../services/summary.ts"; import { AttachmentManager } from "./attachments.ts"; @@ -21,663 +21,690 @@ import { TextChannel } from "discord.js"; import { stringToUuid } from "../../core/uuid.ts"; import { SpeechService } from "../../services/speech.ts"; import { VoiceManager } from "./voice.ts"; -import { generateMessageResponse, generateShouldRespond } from "../../core/generation.ts"; +import { + generateMessageResponse, + generateShouldRespond, +} from "../../core/generation.ts"; const MAX_MESSAGE_LENGTH = 1900; export async function sendMessageInChunks( - channel: TextChannel, - content: string, - inReplyTo: string, - files: any[], + channel: TextChannel, + content: string, + inReplyTo: string, + files: any[] ): Promise { - const sentMessages: DiscordMessage[] = []; - const messages = splitMessage(content); - - for (let i = 0; i < messages.length; i++) { - const message = messages[i]; - if ( - message.trim().length > 0 || - (i === messages.length - 1 && files && files.length > 0) - ) { - const options: any = { - content: message.trim(), - }; - - // if (i === 0 && inReplyTo) { - // // Reply to the specified message for the first chunk - // options.reply = { - // messageReference: inReplyTo, - // }; - // } - - if (i === messages.length - 1 && files && files.length > 0) { - // Attach files to the last message chunk - options.files = files; - } - - const m = await channel.send(options); - sentMessages.push(m); + const sentMessages: DiscordMessage[] = []; + const messages = splitMessage(content); + + for (let i = 0; i < messages.length; i++) { + const message = messages[i]; + if ( + message.trim().length > 0 || + (i === messages.length - 1 && files && files.length > 0) + ) { + const options: any = { + content: message.trim(), + }; + + // if (i === 0 && inReplyTo) { + // // Reply to the specified message for the first chunk + // options.reply = { + // messageReference: inReplyTo, + // }; + // } + + if (i === messages.length - 1 && files && files.length > 0) { + // Attach files to the last message chunk + options.files = files; + } + + const m = await channel.send(options); + sentMessages.push(m); + } } - } - return sentMessages; + return sentMessages; } function splitMessage(content: string): string[] { - const messages: string[] = []; - let currentMessage = ""; - - const rawLines = content?.split("\n") || []; - // split all lines into MAX_MESSAGE_LENGTH chunks so any long lines are split - const lines = rawLines - .map((line) => { - const chunks = []; - while (line.length > MAX_MESSAGE_LENGTH) { - chunks.push(line.slice(0, MAX_MESSAGE_LENGTH)); - line = line.slice(MAX_MESSAGE_LENGTH); - } - chunks.push(line); - return chunks; - }) - .flat(); - - for (const line of lines) { - if (currentMessage.length + line.length + 1 > MAX_MESSAGE_LENGTH) { - messages.push(currentMessage.trim()); - currentMessage = ""; + const messages: string[] = []; + let currentMessage = ""; + + const rawLines = content?.split("\n") || []; + // split all lines into MAX_MESSAGE_LENGTH chunks so any long lines are split + const lines = rawLines + .map((line) => { + const chunks = []; + while (line.length > MAX_MESSAGE_LENGTH) { + chunks.push(line.slice(0, MAX_MESSAGE_LENGTH)); + line = line.slice(MAX_MESSAGE_LENGTH); + } + chunks.push(line); + return chunks; + }) + .flat(); + + for (const line of lines) { + if (currentMessage.length + line.length + 1 > MAX_MESSAGE_LENGTH) { + messages.push(currentMessage.trim()); + currentMessage = ""; + } + currentMessage += line + "\n"; } - currentMessage += line + "\n"; - } - if (currentMessage.trim().length > 0) { - messages.push(currentMessage.trim()); - } + if (currentMessage.trim().length > 0) { + messages.push(currentMessage.trim()); + } - return messages; + return messages; } export class MessageManager { - private client: Client; - private runtime: IAgentRuntime; - private attachmentManager: AttachmentManager; - private interestChannels: InterestChannels = {}; - private discordClient: any; - private voiceManager: VoiceManager; - - constructor(discordClient: any, voiceManager: VoiceManager) { - this.client = discordClient.client; - this.voiceManager = voiceManager; - this.discordClient = discordClient; - this.runtime = discordClient.runtime; - this.attachmentManager = new AttachmentManager(this.runtime); - } - - async handleMessage(message: DiscordMessage) { - if ( - message.interaction || - message.author.id === this.client.user?.id /* || message.author?.bot*/ - ) - return; - console.log("handling message"); - const userId = message.author.id as UUID; - const userName = message.author.username; - const name = message.author.displayName; - const channelId = message.channel.id; - - try { - const { processedContent, attachments } = - await this.processMessageMedia(message); - - const audioAttachments = message.attachments.filter((attachment) => - attachment.contentType?.startsWith("audio/"), - ); - if (audioAttachments.size > 0) { - const processedAudioAttachments = - await this.attachmentManager.processAttachments(audioAttachments); - attachments.push(...processedAudioAttachments); - } - - const roomId = stringToUuid(channelId); - const userIdUUID = stringToUuid(userId); - const agentId = this.runtime.agentId; - - await Promise.all([ - this.runtime.ensureUserExists( - agentId, - this.client.user.username, - this.runtime.character.name, - "discord", - ), - this.runtime.ensureUserExists(userIdUUID, userName, name, "discord"), - this.runtime.ensureRoomExists(roomId), - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(agentId, roomId), - ]); - - const messageId = stringToUuid(message.id); - - // Check if the message already exists in the cache or database - const existingMessage = - await this.runtime.messageManager.getMemoryById(messageId); - - if (existingMessage) { - // If the message content is the same, return early - if (existingMessage.content.text === message.content) { - return; - } - } - - let shouldIgnore = false; - let shouldRespond = true; - - const content: Content = { - text: processedContent, - attachments: attachments, - source: "discord", - url: message.url, - inReplyTo: message.reference?.messageId - ? stringToUuid(message.reference.messageId) - : undefined, - }; - - const userMessage = { content, userId: userIdUUID, roomId }; - - let state = (await this.runtime.composeState(userMessage, { - discordClient: this.client, - discordMessage: message, - agentName: this.runtime.character.name || this.client.user?.displayName, - })) as State; - - const memory: Memory = { - id: stringToUuid(message.id), - ...userMessage, - userId: userIdUUID, - roomId, - content, - createdAt: message.createdTimestamp, - embedding: embeddingZeroVector, - }; - - if (content.text) { - await this.runtime.messageManager.createMemory(memory); - } - - state = await this.runtime.updateRecentMessageState(state); - - if (!shouldIgnore) { - shouldIgnore = await this._shouldIgnore(message); - } - console.log("Received a message from ", message.author.username); - console.log(message.content); - - if (shouldIgnore) { - return; - } - const hasInterest = this._checkInterest(channelId); - - const agentUserState = - await this.runtime.databaseAdapter.getParticipantUserState( - roomId, - this.runtime.agentId, - ); + private client: Client; + private runtime: IAgentRuntime; + private attachmentManager: AttachmentManager; + private interestChannels: InterestChannels = {}; + private discordClient: any; + private voiceManager: VoiceManager; + + constructor(discordClient: any, voiceManager: VoiceManager) { + this.client = discordClient.client; + this.voiceManager = voiceManager; + this.discordClient = discordClient; + this.runtime = discordClient.runtime; + this.attachmentManager = new AttachmentManager(this.runtime); + } - if ( - agentUserState === "MUTED" && - !message.mentions.has(this.client.user.id) && - !hasInterest - ) { - console.log("Ignoring muted room"); - // Ignore muted rooms unless explicitly mentioned - return; - } - - if (agentUserState === "FOLLOWED") { - console.log("Always responding in followed room"); - shouldRespond = true; // Always respond in followed rooms - } else if ( - (!shouldRespond && hasInterest) || - (shouldRespond && !hasInterest) - ) { - console.log("Checking if should respond"); - shouldRespond = await this._shouldRespond(message, state); - } - - if (!shouldRespond) { - return; - } - - console.log("Responding"); - - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - - const responseContent = await this._generateResponse( - memory, - state, - context, - ); - - console.log("Response\n", responseContent); - - responseContent.text = responseContent.text?.trim(); - responseContent.inReplyTo = stringToUuid(message.id); - - if (!responseContent.text) { - return; - } - - const callback: HandlerCallback = async ( - content: Content, - files: any[], - ) => { - if (message.id && !content.inReplyTo) { - content.inReplyTo = stringToUuid(message.id); - } - console.log("received callback", message.channel.type); - if (message.channel.type === ChannelType.GuildVoice) { - console.log("generating voice"); - // For voice channels, use text-to-speech - const audioStream = await SpeechService.generate( - this.runtime, - content.text, - ); - await this.voiceManager.playAudioStream(userId, audioStream); - const memory: Memory = { - id: stringToUuid(message.id), - userId: this.runtime.agentId, - content, - roomId, - embedding: embeddingZeroVector, - }; - return [memory]; - } else { - // For text channels, send the message - const messages = await sendMessageInChunks( - message.channel as TextChannel, - content.text, - message.id, - files, - ); - let notFirstMessage = false; - const memories: Memory[] = []; - for (const m of messages) { - let action = content.action; - // If there's only one message or it's the last message, keep the original action - // For multiple messages, set all but the last to 'CONTINUE' - if (messages.length > 1 && m !== messages[messages.length - 1]) { - action = "CONTINUE"; + async handleMessage(message: DiscordMessage) { + if ( + message.interaction || + message.author.id === + this.client.user?.id /* || message.author?.bot*/ + ) + return; + console.log("handling message"); + const userId = message.author.id as UUID; + const userName = message.author.username; + const name = message.author.displayName; + const channelId = message.channel.id; + + try { + const { processedContent, attachments } = + await this.processMessageMedia(message); + + const audioAttachments = message.attachments.filter((attachment) => + attachment.contentType?.startsWith("audio/") + ); + if (audioAttachments.size > 0) { + const processedAudioAttachments = + await this.attachmentManager.processAttachments( + audioAttachments + ); + attachments.push(...processedAudioAttachments); } - notFirstMessage = true; + const roomId = stringToUuid(channelId); + const userIdUUID = stringToUuid(userId); + const agentId = this.runtime.agentId; + + await Promise.all([ + this.runtime.ensureUserExists( + agentId, + this.client.user.username, + this.runtime.character.name, + "discord" + ), + this.runtime.ensureUserExists( + userIdUUID, + userName, + name, + "discord" + ), + this.runtime.ensureRoomExists(roomId), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom(userIdUUID, roomId), + this.runtime.ensureParticipantInRoom(agentId, roomId), + ]); + + const messageId = stringToUuid(message.id); + + // Check if the message already exists in the cache or database + const existingMessage = + await this.runtime.messageManager.getMemoryById(messageId); + + if (existingMessage) { + // If the message content is the same, return early + if (existingMessage.content.text === message.content) { + return; + } + } + + let shouldIgnore = false; + let shouldRespond = true; + + const content: Content = { + text: processedContent, + attachments: attachments, + source: "discord", + url: message.url, + inReplyTo: message.reference?.messageId + ? stringToUuid(message.reference.messageId) + : undefined, + }; + + const userMessage = { content, userId: userIdUUID, roomId }; + + let state = (await this.runtime.composeState(userMessage, { + discordClient: this.client, + discordMessage: message, + agentName: + this.runtime.character.name || + this.client.user?.displayName, + })) as State; + const memory: Memory = { - id: stringToUuid(m.id), - userId: this.runtime.agentId, - content: { - ...content, - action, - inReplyTo: messageId, - url: m.url, - }, - roomId, - embedding: embeddingZeroVector, - createdAt: m.createdTimestamp, + id: stringToUuid(message.id), + ...userMessage, + userId: userIdUUID, + roomId, + content, + createdAt: message.createdTimestamp, + embedding: embeddingZeroVector, }; - memories.push(memory); - } - for (const m of memories) { - await this.runtime.messageManager.createMemory(m); - } - return memories; + + if (content.text) { + await this.runtime.messageManager.createMemory(memory); + } + + state = await this.runtime.updateRecentMessageState(state); + + if (!shouldIgnore) { + shouldIgnore = await this._shouldIgnore(message); + } + console.log("Received a message from ", message.author.username); + console.log(message.content); + + if (shouldIgnore) { + return; + } + const hasInterest = this._checkInterest(channelId); + + const agentUserState = + await this.runtime.databaseAdapter.getParticipantUserState( + roomId, + this.runtime.agentId + ); + + if ( + agentUserState === "MUTED" && + !message.mentions.has(this.client.user.id) && + !hasInterest + ) { + console.log("Ignoring muted room"); + // Ignore muted rooms unless explicitly mentioned + return; + } + + if (agentUserState === "FOLLOWED") { + console.log("Always responding in followed room"); + shouldRespond = true; // Always respond in followed rooms + } else if ( + (!shouldRespond && hasInterest) || + (shouldRespond && !hasInterest) + ) { + console.log("Checking if should respond"); + shouldRespond = await this._shouldRespond(message, state); + } + + if (!shouldRespond) { + return; + } + + console.log("Responding"); + + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); + + const responseContent = await this._generateResponse( + memory, + state, + context + ); + + console.log("Response\n", responseContent); + + responseContent.text = responseContent.text?.trim(); + responseContent.inReplyTo = stringToUuid(message.id); + + if (!responseContent.text) { + return; + } + + const callback: HandlerCallback = async ( + content: Content, + files: any[] + ) => { + if (message.id && !content.inReplyTo) { + content.inReplyTo = stringToUuid(message.id); + } + console.log("received callback", message.channel.type); + if (message.channel.type === ChannelType.GuildVoice) { + console.log("generating voice"); + // For voice channels, use text-to-speech + const audioStream = await SpeechService.generate( + this.runtime, + content.text + ); + await this.voiceManager.playAudioStream( + userId, + audioStream + ); + const memory: Memory = { + id: stringToUuid(message.id), + userId: this.runtime.agentId, + content, + roomId, + embedding: embeddingZeroVector, + }; + return [memory]; + } else { + // For text channels, send the message + const messages = await sendMessageInChunks( + message.channel as TextChannel, + content.text, + message.id, + files + ); + let notFirstMessage = false; + const memories: Memory[] = []; + for (const m of messages) { + let action = content.action; + // If there's only one message or it's the last message, keep the original action + // For multiple messages, set all but the last to 'CONTINUE' + if ( + messages.length > 1 && + m !== messages[messages.length - 1] + ) { + action = "CONTINUE"; + } + + notFirstMessage = true; + const memory: Memory = { + id: stringToUuid(m.id), + userId: this.runtime.agentId, + content: { + ...content, + action, + inReplyTo: messageId, + url: m.url, + }, + roomId, + embedding: embeddingZeroVector, + createdAt: m.createdTimestamp, + }; + memories.push(memory); + } + for (const m of memories) { + await this.runtime.messageManager.createMemory(m); + } + return memories; + } + }; + + const responseMessages = await callback(responseContent); + + state = await this.runtime.updateRecentMessageState(state); + + await this.runtime.evaluate(memory, state); + + await this.runtime.processActions( + memory, + responseMessages, + state, + callback + ); + } catch (error) { + console.error("Error handling message:", error); + if (message.channel.type === ChannelType.GuildVoice) { + // For voice channels, use text-to-speech for the error message + const errorMessage = "Sorry, I had a glitch. What was that?"; + const audioStream = await SpeechService.generate( + this.runtime, + errorMessage + ); + await this.voiceManager.playAudioStream(userId, audioStream); + } else { + // For text channels, send the error message + await message.reply( + "Sorry, I encountered an error while processing your request." + ); + } + } + } + + async cacheMessages(channel: TextChannel, count: number = 20) { + const messages = await channel.messages.fetch({ limit: count }); + for (const [_, message] of messages) { + await this.handleMessage(message); } - }; - - const responseMessages = await callback(responseContent); - - state = await this.runtime.updateRecentMessageState(state); - - await this.runtime.evaluate(memory, state); - - await this.runtime.processActions( - memory, - responseMessages, - state, - callback, - ); - } catch (error) { - console.error("Error handling message:", error); - if (message.channel.type === ChannelType.GuildVoice) { - // For voice channels, use text-to-speech for the error message - const errorMessage = "Sorry, I had a glitch. What was that?"; - const audioStream = await SpeechService.generate( - this.runtime, - errorMessage, - ); - await this.voiceManager.playAudioStream(userId, audioStream); - } else { - // For text channels, send the error message - await message.reply( - "Sorry, I encountered an error while processing your request.", - ); - } } - } - async cacheMessages(channel: TextChannel, count: number = 20) { - const messages = await channel.messages.fetch({ limit: count }); - for (const [_, message] of messages) { - await this.handleMessage(message); + async processMessageMedia( + message: DiscordMessage + ): Promise<{ processedContent: string; attachments: Media[] }> { + let processedContent = message.content; + let attachments: Media[] = []; + + // Process code blocks in the message content + const codeBlockRegex = /```([\s\S]*?)```/g; + let match; + while ((match = codeBlockRegex.exec(processedContent))) { + const codeBlock = match[1]; + const lines = codeBlock.split("\n"); + const title = lines[0]; + const description = lines.slice(0, 3).join("\n"); + const attachmentId = + `code-${Date.now()}-${Math.floor(Math.random() * 1000)}`.slice( + -5 + ); + attachments.push({ + id: attachmentId, + url: "", + title: title || "Code Block", + source: "Code", + description: description, + text: codeBlock, + }); + processedContent = processedContent.replace( + match[0], + `Code Block (${attachmentId})` + ); + } + + // Process message attachments + if (message.attachments.size > 0) { + attachments = await this.attachmentManager.processAttachments( + message.attachments + ); + } + + // TODO: Move to attachments manager + const urlRegex = /(https?:\/\/[^\s]+)/g; + const urls = processedContent.match(urlRegex) || []; + + for (const url of urls) { + if (this.runtime.videoService.isVideoUrl(url)) { + const videoInfo = + await this.runtime.videoService.processVideo(url); + attachments.push({ + id: `youtube-${Date.now()}`, + url: url, + title: videoInfo.title, + source: "YouTube", + description: videoInfo.description, + text: videoInfo.text, + }); + } else { + const { title, bodyContent } = + await this.runtime.browserService.getPageContent(url); + const { title: newTitle, description } = await generateSummary( + this.runtime, + title + "\n" + bodyContent + ); + attachments.push({ + id: `webpage-${Date.now()}`, + url: url, + title: newTitle || "Web Page", + source: "Web", + description, + text: bodyContent, + }); + } + } + + return { processedContent, attachments }; } - } - - async processMessageMedia( - message: DiscordMessage, - ): Promise<{ processedContent: string; attachments: Media[] }> { - let processedContent = message.content; - let attachments: Media[] = []; - - // Process code blocks in the message content - const codeBlockRegex = /```([\s\S]*?)```/g; - let match; - while ((match = codeBlockRegex.exec(processedContent))) { - const codeBlock = match[1]; - const lines = codeBlock.split("\n"); - const title = lines[0]; - const description = lines.slice(0, 3).join("\n"); - const attachmentId = - `code-${Date.now()}-${Math.floor(Math.random() * 1000)}`.slice(-5); - attachments.push({ - id: attachmentId, - url: "", - title: title || "Code Block", - source: "Code", - description: description, - text: codeBlock, - }); - processedContent = processedContent.replace( - match[0], - `Code Block (${attachmentId})`, - ); + + private _checkInterest(channelId: string): boolean { + return !!this.interestChannels[channelId]; } - // Process message attachments - if (message.attachments.size > 0) { - attachments = await this.attachmentManager.processAttachments( - message.attachments, - ); + private async _shouldIgnore(message: DiscordMessage): Promise { + // if the message is from us, ignore + if (message.author.id === this.client.user?.id) return true; + let messageContent = message.content.toLowerCase(); + + // Replace the bot's @ping with the character name + const botMention = `<@!?${this.client.user?.id}>`; + messageContent = messageContent.replace( + new RegExp(botMention, "gi"), + this.runtime.character.name.toLowerCase() + ); + + // Replace the bot's username with the character name + const botUsername = this.client.user?.username.toLowerCase(); + messageContent = messageContent.replace( + new RegExp(`\\b${botUsername}\\b`, "g"), + this.runtime.character.name.toLowerCase() + ); + + // strip all special characters + messageContent = messageContent.replace(/[^a-zA-Z0-9\s]/g, ""); + + // short responses where ruby should stop talking and disengage unless mentioned again + const loseInterestWords = [ + "shut up", + "stop", + "please shut up", + "shut up please", + "dont talk", + "silence", + "stop talking", + "be quiet", + "hush", + "wtf", + "chill", + "stfu", + "stupid bot", + "dumb bot", + "stop responding", + "god damn it", + "god damn", + "goddamnit", + "can you not", + "can you stop", + "be quiet", + "hate you", + "hate this", + "fuck up", + ]; + if ( + messageContent.length < 100 && + loseInterestWords.some((word) => messageContent.includes(word)) + ) { + delete this.interestChannels[message.channelId]; + return true; + } + + // If we're not interested in the channel and it's a short message, ignore it + if ( + messageContent.length < 10 && + !this.interestChannels[message.channelId] + ) { + return true; + } + + const targetedPhrases = [ + this.runtime.character.name + " stop responding", + this.runtime.character.name + " stop talking", + this.runtime.character.name + " shut up", + this.runtime.character.name + " stfu", + "stop talking" + this.runtime.character.name, + this.runtime.character.name + " stop talking", + "shut up " + this.runtime.character.name, + this.runtime.character.name + " shut up", + "stfu " + this.runtime.character.name, + this.runtime.character.name + " stfu", + "chill" + this.runtime.character.name, + this.runtime.character.name + " chill", + ]; + + // lose interest if pinged and told to stop responding + if (targetedPhrases.some((phrase) => messageContent.includes(phrase))) { + delete this.interestChannels[message.channelId]; + return true; + } + + // if the message is short, ignore but maintain interest + if ( + !this.interestChannels[message.channelId] && + messageContent.length < 2 + ) { + return true; + } + + const ignoreResponseWords = [ + "lol", + "nm", + "uh", + "wtf", + "stfu", + "dumb", + "jfc", + "omg", + ]; + if ( + message.content.length < 4 && + ignoreResponseWords.some((word) => + message.content.toLowerCase().includes(word) + ) + ) { + return true; + } + console.log("Not ignoring message:", message.content); + return false; } - // TODO: Move to attachments manager - const urlRegex = /(https?:\/\/[^\s]+)/g; - const urls = processedContent.match(urlRegex) || []; - - for (const url of urls) { - if (this.runtime.videoService.isVideoUrl(url)) { - const videoInfo = await this.runtime.videoService.processVideo(url); - attachments.push({ - id: `youtube-${Date.now()}`, - url: url, - title: videoInfo.title, - source: "YouTube", - description: videoInfo.description, - text: videoInfo.text, + private async _shouldRespond( + message: DiscordMessage, + state: State + ): Promise { + if (message.author.id === this.client.user?.id) return false; + // if (message.author.bot) return false; + if (message.mentions.has(this.client.user?.id as string)) return true; + + const guild = message.guild; + const member = guild?.members.cache.get(this.client.user?.id as string); + const nickname = member?.nickname; + + if ( + message.content + .toLowerCase() + .includes(this.client.user?.username.toLowerCase() as string) || + message.content + .toLowerCase() + .includes(this.client.user?.tag.toLowerCase() as string) || + (nickname && + message.content.toLowerCase().includes(nickname.toLowerCase())) + ) { + return true; + } + + if (!message.guild) { + return true; + } + + // If none of the above conditions are met, use the generateText to decide + const shouldRespondContext = composeContext({ + state, + template: shouldRespondTemplate, }); - } else { - const { title, bodyContent } = - await this.runtime.browserService.getPageContent(url); - const { title: newTitle, description } = await generateSummary( - this.runtime, - title + "\n" + bodyContent, - ); - attachments.push({ - id: `webpage-${Date.now()}`, - url: url, - title: newTitle || "Web Page", - source: "Web", - description, - text: bodyContent, + + const response = await generateShouldRespond({ + runtime: this.runtime, + context: shouldRespondContext, + modelClass: ModelClass.SMALL, }); - } - } - return { processedContent, attachments }; - } - - private _checkInterest(channelId: string): boolean { - return !!this.interestChannels[channelId]; - } - - private async _shouldIgnore(message: DiscordMessage): Promise { - // if the message is from us, ignore - if (message.author.id === this.client.user?.id) return true; - let messageContent = message.content.toLowerCase(); - - // Replace the bot's @ping with the character name - const botMention = `<@!?${this.client.user?.id}>`; - messageContent = messageContent.replace( - new RegExp(botMention, "gi"), - this.runtime.character.name.toLowerCase(), - ); - - // Replace the bot's username with the character name - const botUsername = this.client.user?.username.toLowerCase(); - messageContent = messageContent.replace( - new RegExp(`\\b${botUsername}\\b`, "g"), - this.runtime.character.name.toLowerCase(), - ); - - // strip all special characters - messageContent = messageContent.replace(/[^a-zA-Z0-9\s]/g, ""); - - // short responses where ruby should stop talking and disengage unless mentioned again - const loseInterestWords = [ - "shut up", - "stop", - "please shut up", - "shut up please", - "dont talk", - "silence", - "stop talking", - "be quiet", - "hush", - "wtf", - "chill", - "stfu", - "stupid bot", - "dumb bot", - "stop responding", - "god damn it", - "god damn", - "goddamnit", - "can you not", - "can you stop", - "be quiet", - "hate you", - "hate this", - "fuck up", - ]; - if ( - messageContent.length < 100 && - loseInterestWords.some((word) => messageContent.includes(word)) - ) { - delete this.interestChannels[message.channelId]; - return true; + if (response === "RESPOND") { + return true; + } else if (response === "IGNORE") { + return false; + } else if (response === "STOP") { + delete this.interestChannels[message.channelId]; + return false; + } else { + console.error( + "Invalid response from response generateText:", + response + ); + return false; + } } - // If we're not interested in the channel and it's a short message, ignore it - if ( - messageContent.length < 10 && - !this.interestChannels[message.channelId] - ) { - return true; - } + private async _generateResponse( + message: Memory, + state: State, + context: string + ): Promise { + const { userId, roomId } = message; - const targetedPhrases = [ - this.runtime.character.name + " stop responding", - this.runtime.character.name + " stop talking", - this.runtime.character.name + " shut up", - this.runtime.character.name + " stfu", - "stop talking" + this.runtime.character.name, - this.runtime.character.name + " stop talking", - "shut up " + this.runtime.character.name, - this.runtime.character.name + " shut up", - "stfu " + this.runtime.character.name, - this.runtime.character.name + " stfu", - "chill" + this.runtime.character.name, - this.runtime.character.name + " chill", - ]; - - // lose interest if pinged and told to stop responding - if (targetedPhrases.some((phrase) => messageContent.includes(phrase))) { - delete this.interestChannels[message.channelId]; - return true; - } + const datestr = new Date().toUTCString().replace(/:/g, "-"); - // if the message is short, ignore but maintain interest - if ( - !this.interestChannels[message.channelId] && - messageContent.length < 2 - ) { - return true; - } + // log context to file + log_to_file( + `${state.agentName}_${datestr}_discord_message_context`, + context + ); - const ignoreResponseWords = [ - "lol", - "nm", - "uh", - "wtf", - "stfu", - "dumb", - "jfc", - "omg", - ]; - if ( - message.content.length < 4 && - ignoreResponseWords.some((word) => - message.content.toLowerCase().includes(word), - ) - ) { - return true; - } - console.log("Not ignoring message:", message.content); - return false; - } - - private async _shouldRespond( - message: DiscordMessage, - state: State, - ): Promise { - if (message.author.id === this.client.user?.id) return false; - // if (message.author.bot) return false; - if (message.mentions.has(this.client.user?.id as string)) return true; - - const guild = message.guild; - const member = guild?.members.cache.get(this.client.user?.id as string); - const nickname = member?.nickname; - - if ( - message.content - .toLowerCase() - .includes(this.client.user?.username.toLowerCase() as string) || - message.content - .toLowerCase() - .includes(this.client.user?.tag.toLowerCase() as string) || - (nickname && - message.content.toLowerCase().includes(nickname.toLowerCase())) - ) { - return true; - } + const response = await generateMessageResponse({ + runtime: this.runtime, + context, + modelClass: "slow", + }); - if (!message.guild) { - return true; - } + if (!response) { + console.error("No response from generateMessageResponse"); + return; + } - // If none of the above conditions are met, use the generateText to decide - const shouldRespondContext = composeContext({ - state, - template: shouldRespondTemplate, - }); - - const response = await generateShouldRespond({ - runtime: this.runtime, - context: shouldRespondContext, - modelClass: ModelClass.SMALL, - }); - - if (response === "RESPOND") { - return true; - } else if (response === "IGNORE") { - return false; - } else if (response === "STOP") { - delete this.interestChannels[message.channelId]; - return false; - } else { - console.error("Invalid response from response generateText:", response); - return false; - } - } - - private async _generateResponse( - message: Memory, - state: State, - context: string, - ): Promise { - const { userId, roomId } = message; - - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - // log context to file - log_to_file( - `${state.agentName}_${datestr}_discord_message_context`, - context, - ); - - const response = await generateMessageResponse({ - runtime: this.runtime, - context, - modelClass: "slow" - }); - - if (!response) { - console.error("No response from generateMessageResponse"); - return; - } + log_to_file( + `${state.agentName}_${datestr}_discord_message_response`, + JSON.stringify(response) + ); + + await this.runtime.databaseAdapter.log({ + body: { message, context, response }, + userId: userId, + roomId, + type: "response", + }); - log_to_file( - `${state.agentName}_${datestr}_discord_message_response`, - JSON.stringify(response), - ); - - await this.runtime.databaseAdapter.log({ - body: { message, context, response }, - userId: userId, - roomId, - type: "response", - }); - - return response; - } - - async fetchBotName(botToken: string) { - const url = "https://discord.com/api/v10/users/@me"; - - const response = await fetch(url, { - method: "GET", - headers: { - Authorization: `Bot ${botToken}`, - }, - }); - - if (!response.ok) { - throw new Error(`Error fetching bot details: ${response.statusText}`); + return response; } - const data = await response.json(); - return data.username; - } + async fetchBotName(botToken: string) { + const url = "https://discord.com/api/v10/users/@me"; + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bot ${botToken}`, + }, + }); + + if (!response.ok) { + throw new Error( + `Error fetching bot details: ${response.statusText}` + ); + } + + const data = await response.json(); + return data.username; + } } diff --git a/core/src/clients/discord/providers/channelState.ts b/core/src/clients/discord/providers/channelState.ts index 1fb2154c3b..34fb634576 100644 --- a/core/src/clients/discord/providers/channelState.ts +++ b/core/src/clients/discord/providers/channelState.ts @@ -1,60 +1,61 @@ import { - ChannelType, - Message as DiscordMessage, - TextChannel, + ChannelType, + Message as DiscordMessage, + TextChannel, } from "discord.js"; import { IAgentRuntime, Memory, Provider, State } from "../../../core/types.ts"; const channelStateProvider: Provider = { - get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { - const discordMessage = - (state?.discordMessage as DiscordMessage) || - (state?.discordChannel as DiscordMessage); - if (!discordMessage) { - console.log("discordMessage is null"); - return ""; - } + get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { + const discordMessage = + (state?.discordMessage as DiscordMessage) || + (state?.discordChannel as DiscordMessage); + if (!discordMessage) { + console.log("discordMessage is null"); + return ""; + } - const guild = discordMessage?.guild; - const agentName = state?.agentName || "The agent"; - const senderName = state?.senderName || "someone"; + const guild = discordMessage?.guild; + const agentName = state?.agentName || "The agent"; + const senderName = state?.senderName || "someone"; - if (!guild) { - return ( - agentName + - " is currently in a direct message conversation with " + - senderName - ); - } + if (!guild) { + return ( + agentName + + " is currently in a direct message conversation with " + + senderName + ); + } - const serverName = guild.name; // The name of the server - const guildId = guild.id; // The ID of the guild - const channel = discordMessage.channel; + const serverName = guild.name; // The name of the server + const guildId = guild.id; // The ID of the guild + const channel = discordMessage.channel; - if (!channel) { - console.log("channel is null"); - return ""; - } + if (!channel) { + console.log("channel is null"); + return ""; + } - let response = - agentName + - " is currently having a conversation in the channel `@" + - channel.id + - " in the server `" + - serverName + - "` (@" + - guildId + - ")"; - if ( - channel.type === ChannelType.GuildText && - (channel as TextChannel).topic - ) { - // Check if the channel is a text channel - response += - "\nThe topic of the channel is: " + (channel as TextChannel).topic; - } - return response; - }, + let response = + agentName + + " is currently having a conversation in the channel `@" + + channel.id + + " in the server `" + + serverName + + "` (@" + + guildId + + ")"; + if ( + channel.type === ChannelType.GuildText && + (channel as TextChannel).topic + ) { + // Check if the channel is a text channel + response += + "\nThe topic of the channel is: " + + (channel as TextChannel).topic; + } + return response; + }, }; export default channelStateProvider; diff --git a/core/src/clients/discord/providers/voiceState.ts b/core/src/clients/discord/providers/voiceState.ts index 225db9d1dc..0b510602f9 100644 --- a/core/src/clients/discord/providers/voiceState.ts +++ b/core/src/clients/discord/providers/voiceState.ts @@ -3,29 +3,31 @@ import { ChannelType, Message as DiscordMessage } from "discord.js"; import { IAgentRuntime, Memory, Provider, State } from "../../../core/types.ts"; const voiceStateProvider: Provider = { - get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { - // Voice doesn't get a discord message, so we need to use the channel for guild data - const discordMessage = (state?.discordMessage || - state.discordChannel) as DiscordMessage; - const connection = getVoiceConnection( - (discordMessage as DiscordMessage)?.guild?.id as string, - ); - const agentName = state?.agentName || "The agent"; - if (!connection) { - return agentName + " is not currently in a voice channel"; - } + get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { + // Voice doesn't get a discord message, so we need to use the channel for guild data + const discordMessage = (state?.discordMessage || + state.discordChannel) as DiscordMessage; + const connection = getVoiceConnection( + (discordMessage as DiscordMessage)?.guild?.id as string + ); + const agentName = state?.agentName || "The agent"; + if (!connection) { + return agentName + " is not currently in a voice channel"; + } - const channel = ( - (state?.discordMessage as DiscordMessage) || - (state.discordChannel as DiscordMessage) - )?.guild?.channels?.cache?.get(connection.joinConfig.channelId as string); + const channel = ( + (state?.discordMessage as DiscordMessage) || + (state.discordChannel as DiscordMessage) + )?.guild?.channels?.cache?.get( + connection.joinConfig.channelId as string + ); - if (!channel || channel.type !== ChannelType.GuildVoice) { - return agentName + " is in an invalid voice channel"; - } + if (!channel || channel.type !== ChannelType.GuildVoice) { + return agentName + " is in an invalid voice channel"; + } - return `${agentName} is currently in the voice channel: ${channel.name} (ID: ${channel.id})`; - }, + return `${agentName} is currently in the voice channel: ${channel.name} (ID: ${channel.id})`; + }, }; export default voiceStateProvider; diff --git a/core/src/clients/discord/templates.ts b/core/src/clients/discord/templates.ts index aff8509749..e12e218007 100644 --- a/core/src/clients/discord/templates.ts +++ b/core/src/clients/discord/templates.ts @@ -1,11 +1,11 @@ import { - messageCompletionFooter, - shouldRespondFooter, + messageCompletionFooter, + shouldRespondFooter, } from "../../core/parsing.ts"; export const messageHandlerTemplate = - // {{goals}} - `# Action Examples + // {{goals}} + `# Action Examples {{actionExamples}} (Action examples are for reference only. Do not use the information from them in your response.) @@ -40,7 +40,7 @@ Note that {{agentName}} is capable of reading/seeing/hearing various forms of me ` + messageCompletionFooter; export const voiceHandlerTemplate = - `# Task: Generate conversational voice dialog for {{agentName}}. + `# Task: Generate conversational voice dialog for {{agentName}}. About {{agentName}}: {{bio}} @@ -62,7 +62,7 @@ Note that {{agentName}} is capable of reading/seeing/hearing various forms of me ` + messageCompletionFooter; export const shouldRespondTemplate = - `# Task: Decide if {{agentName}} should respond. + `# Task: Decide if {{agentName}} should respond. About {{agentName}}: {{bio}} diff --git a/core/src/clients/discord/types.ts b/core/src/clients/discord/types.ts index c56fb77ec6..bb7f6eddb8 100644 --- a/core/src/clients/discord/types.ts +++ b/core/src/clients/discord/types.ts @@ -1,15 +1,15 @@ import { Content, UUID } from "../../core/types.ts"; export type InterestChannels = { - [key: string]: { - lastMessageSent: number; - messages: { userId: UUID; userName: string; content: Content }[]; - }; + [key: string]: { + lastMessageSent: number; + messages: { userId: UUID; userName: string; content: Content }[]; + }; }; export enum ResponseType { - SPOKEN_AUDIO = 0, - SPOKEN_TEXT = 1, - RESPONSE_TEXT = 2, - RESPONSE_AUDIO = 3, + SPOKEN_AUDIO = 0, + SPOKEN_TEXT = 1, + RESPONSE_TEXT = 2, + RESPONSE_AUDIO = 3, } diff --git a/core/src/clients/discord/voice.ts b/core/src/clients/discord/voice.ts index 9cf1d93f9b..58a9087034 100644 --- a/core/src/clients/discord/voice.ts +++ b/core/src/clients/discord/voice.ts @@ -1,21 +1,21 @@ import { - AudioReceiveStream, - NoSubscriberBehavior, - StreamType, - VoiceConnection, - createAudioPlayer, - createAudioResource, - getVoiceConnection, - joinVoiceChannel, + AudioReceiveStream, + NoSubscriberBehavior, + StreamType, + VoiceConnection, + createAudioPlayer, + createAudioResource, + getVoiceConnection, + joinVoiceChannel, } from "@discordjs/voice"; import { - BaseGuildVoiceChannel, - ChannelType, - Client, - Guild, - GuildMember, - VoiceChannel, - VoiceState, + BaseGuildVoiceChannel, + ChannelType, + Client, + Guild, + GuildMember, + VoiceChannel, + VoiceState, } from "discord.js"; import EventEmitter from "events"; import path from "path"; @@ -25,13 +25,13 @@ import { composeContext } from "../../core/context.ts"; import { log_to_file } from "../../core/logger.ts"; import { embeddingZeroVector } from "../../core/memory.ts"; import { - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, - UUID, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, + UUID, } from "../../core/types.ts"; import { stringToUuid } from "../../core/uuid.ts"; import { getWavHeader } from "../../services/audioUtils.ts"; @@ -47,603 +47,653 @@ const DECODE_FRAME_SIZE = 1024; const DECODE_SAMPLE_RATE = 16000; export class VoiceManager extends EventEmitter { - private client: Client; - private runtime: IAgentRuntime; - private streams: Map = new Map(); - private connections: Map = new Map(); - private activeMonitors: Map< - string, - { channel: BaseGuildVoiceChannel; monitor: AudioMonitor } - > = new Map(); - - constructor(client: any) { - super(); - this.client = client.client; - this.runtime = client.runtime; - } - - async handleVoiceStateUpdate(oldState: VoiceState, newState: VoiceState) { - const oldChannelId = oldState.channelId; - const newChannelId = newState.channelId; - const member = newState.member; - if (!member) return; - if (member.id === this.client.user?.id) { - return; + private client: Client; + private runtime: IAgentRuntime; + private streams: Map = new Map(); + private connections: Map = new Map(); + private activeMonitors: Map< + string, + { channel: BaseGuildVoiceChannel; monitor: AudioMonitor } + > = new Map(); + + constructor(client: any) { + super(); + this.client = client.client; + this.runtime = client.runtime; } - // Ignore mute/unmute events - if (oldChannelId === newChannelId) { - return; - } - - // User leaving a channel where the bot is present - if (oldChannelId && this.connections.has(oldChannelId)) { - this.stopMonitoringMember(member.id); - } - - // User joining a channel where the bot is present - if (newChannelId && this.connections.has(newChannelId)) { - await this.monitorMember( - member, - newState.channel as BaseGuildVoiceChannel, - ); - } - } - - async joinChannel(channel: BaseGuildVoiceChannel) { - const oldConnection = getVoiceConnection(channel.guildId as string); - if (oldConnection) { - try { - oldConnection.destroy(); - } catch (error) { - console.error("Error leaving voice channel:", error); - } - } - const connection = joinVoiceChannel({ - channelId: channel.id, - guildId: channel.guild.id, - adapterCreator: channel.guild.voiceAdapterCreator as any, - selfDeaf: false, - selfMute: false, - }); - - for (const [, member] of channel.members) { - if (!member.user.bot) { - this.monitorMember(member, channel); - } - } - - connection.receiver.speaking.on("start", (userId: string) => { - const user = channel.members.get(userId); - if (!user?.user.bot) { - this.monitorMember(user as GuildMember, channel); - this.streams.get(userId)?.emit("speakingStarted"); - } - }); - - connection.receiver.speaking.on("end", async (userId: string) => { - const user = channel.members.get(userId); - if (!user?.user.bot) { - this.streams.get(userId)?.emit("speakingStopped"); - } - }); - } - - private async monitorMember( - member: GuildMember, - channel: BaseGuildVoiceChannel, - ) { - const userId = member.id; - const userName = member.user.username; - const name = member.user.displayName; - const connection = getVoiceConnection(member.guild.id); - const receiveStream = connection?.receiver.subscribe(userId, { - autoDestroy: true, - emitClose: true, - }); - if (!receiveStream || receiveStream.readableLength === 0) { - return; - } - const opusDecoder = new prism.opus.Decoder({ - channels: 1, - rate: DECODE_SAMPLE_RATE, - frameSize: DECODE_FRAME_SIZE, - }); - pipeline( - receiveStream as AudioReceiveStream, - opusDecoder as any, - (err: Error | null) => { - if (err) { - console.log(`Opus decoding pipeline error: ${err}`); + async handleVoiceStateUpdate(oldState: VoiceState, newState: VoiceState) { + const oldChannelId = oldState.channelId; + const newChannelId = newState.channelId; + const member = newState.member; + if (!member) return; + if (member.id === this.client.user?.id) { + return; } - }, - ); - this.streams.set(userId, opusDecoder); - this.connections.set(userId, connection as VoiceConnection); - opusDecoder.on("error", (err: any) => { - console.log(`Opus decoding error: ${err}`); - }); - const errorHandler = (err: any) => { - console.log(`Opus decoding error: ${err}`); - }; - const streamCloseHandler = () => { - console.log(`voice stream from ${member?.displayName} closed`); - this.streams.delete(userId); - this.connections.delete(userId); - }; - const closeHandler = () => { - console.log(`Opus decoder for ${member?.displayName} closed`); - opusDecoder.removeListener("error", errorHandler); - opusDecoder.removeListener("close", closeHandler); - receiveStream?.removeListener("close", streamCloseHandler); - }; - opusDecoder.on("error", errorHandler); - opusDecoder.on("close", closeHandler); - receiveStream?.on("close", streamCloseHandler); - - this.client.emit( - "userStream", - userId, - name, - userName, - channel, - opusDecoder, - ); - } - - leaveChannel(channel: BaseGuildVoiceChannel) { - const connection = this.connections.get(channel.id); - if (connection) { - connection.destroy(); - this.connections.delete(channel.id); - } - // Stop monitoring all members in this channel - for (const [memberId, monitorInfo] of this.activeMonitors) { - if ( - monitorInfo.channel.id === channel.id && - memberId !== this.client.user?.id - ) { - this.stopMonitoringMember(memberId); - } - } + // Ignore mute/unmute events + if (oldChannelId === newChannelId) { + return; + } - console.log(`Left voice channel: ${channel.name} (${channel.id})`); - } + // User leaving a channel where the bot is present + if (oldChannelId && this.connections.has(oldChannelId)) { + this.stopMonitoringMember(member.id); + } - stopMonitoringMember(memberId: string) { - const monitorInfo = this.activeMonitors.get(memberId); - if (monitorInfo) { - monitorInfo.monitor.stop(); - this.activeMonitors.delete(memberId); - this.streams.delete(memberId); - console.log(`Stopped monitoring user ${memberId}`); + // User joining a channel where the bot is present + if (newChannelId && this.connections.has(newChannelId)) { + await this.monitorMember( + member, + newState.channel as BaseGuildVoiceChannel + ); + } } - } - - async handleGuildCreate(guild: Guild) { - console.log(`Joined guild ${guild.name}`); - // this.scanGuild(guild); - } - - async handleUserStream( - userId: UUID, - name: string, - userName: string, - channel: BaseGuildVoiceChannel, - audioStream: Readable, - ) { - const channelId = channel.id; - const buffers: Buffer[] = []; - let totalLength = 0; - const maxSilenceTime = 1000; // Maximum pause duration in milliseconds - const minSilenceTime = 50; // Minimum silence duration to trigger transcription - let lastChunkTime = Date.now(); - let transcriptionStarted = false; - let transcriptionText = ""; - console.log("new audio monitor for: ", userId); - - const monitor = new AudioMonitor(audioStream, 10000000, async (buffer) => { - console.log("buffer: ", buffer); - const currentTime = Date.now(); - const silenceDuration = currentTime - lastChunkTime; - if (!buffer) { - // Handle error - console.error("Empty buffer received"); - return; - } - buffers.push(buffer); - totalLength += buffer.length; - lastChunkTime = currentTime; - - if (silenceDuration > minSilenceTime && !transcriptionStarted) { - transcriptionStarted = true; - const inputBuffer = Buffer.concat(buffers, totalLength); - buffers.length = 0; - totalLength = 0; - try { - // Convert Opus to WAV and add the header - const wavBuffer = await this.convertOpusToWav(inputBuffer); - - console.log("starting transcription"); - const text = - await this.runtime.transcriptionService.transcribe(wavBuffer); - console.log("transcribed text: ", text); - transcriptionText += text; - } catch (error) { - console.error("Error processing audio stream:", error); + async joinChannel(channel: BaseGuildVoiceChannel) { + const oldConnection = getVoiceConnection(channel.guildId as string); + if (oldConnection) { + try { + oldConnection.destroy(); + } catch (error) { + console.error("Error leaving voice channel:", error); + } + } + const connection = joinVoiceChannel({ + channelId: channel.id, + guildId: channel.guild.id, + adapterCreator: channel.guild.voiceAdapterCreator as any, + selfDeaf: false, + selfMute: false, + }); + + for (const [, member] of channel.members) { + if (!member.user.bot) { + this.monitorMember(member, channel); + } } - } - if (silenceDuration > maxSilenceTime && transcriptionStarted) { - console.log("transcription finished"); - transcriptionStarted = false; + connection.receiver.speaking.on("start", (userId: string) => { + const user = channel.members.get(userId); + if (!user?.user.bot) { + this.monitorMember(user as GuildMember, channel); + this.streams.get(userId)?.emit("speakingStarted"); + } + }); - if (!transcriptionText) return; + connection.receiver.speaking.on("end", async (userId: string) => { + const user = channel.members.get(userId); + if (!user?.user.bot) { + this.streams.get(userId)?.emit("speakingStopped"); + } + }); + } - try { - const text = transcriptionText; - - // handle whisper cases - if ( - (text.length < 15 && text.includes("[BLANK_AUDIO]")) || - (text.length < 5 && text.toLowerCase().includes("you")) - ) { - transcriptionText = ""; // Reset transcription text + private async monitorMember( + member: GuildMember, + channel: BaseGuildVoiceChannel + ) { + const userId = member.id; + const userName = member.user.username; + const name = member.user.displayName; + const connection = getVoiceConnection(member.guild.id); + const receiveStream = connection?.receiver.subscribe(userId, { + autoDestroy: true, + emitClose: true, + }); + if (!receiveStream || receiveStream.readableLength === 0) { return; - } - - const roomId = stringToUuid(channelId); - const userIdUUID = stringToUuid(userId); - await this.runtime.ensureUserExists( - this.runtime.agentId, - this.client.user.username, - this.runtime.character.name, - "discord", - ); - await Promise.all([ - this.runtime.ensureUserExists( - userIdUUID, - userName, - name, - "discord", - ), - this.runtime.ensureRoomExists(roomId), - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(this.runtime.agentId, roomId), - ]); - - let state = await this.runtime.composeState( - { - content: { text: text, source: "Discord" }, - userId: userIdUUID, - roomId, - }, - { - discordChannel: channel, - discordClient: this.client, - agentName: this.runtime.character.name, - }, - ); + } + const opusDecoder = new prism.opus.Decoder({ + channels: 1, + rate: DECODE_SAMPLE_RATE, + frameSize: DECODE_FRAME_SIZE, + }); + pipeline( + receiveStream as AudioReceiveStream, + opusDecoder as any, + (err: Error | null) => { + if (err) { + console.log(`Opus decoding pipeline error: ${err}`); + } + } + ); + this.streams.set(userId, opusDecoder); + this.connections.set(userId, connection as VoiceConnection); + opusDecoder.on("error", (err: any) => { + console.log(`Opus decoding error: ${err}`); + }); + const errorHandler = (err: any) => { + console.log(`Opus decoding error: ${err}`); + }; + const streamCloseHandler = () => { + console.log(`voice stream from ${member?.displayName} closed`); + this.streams.delete(userId); + this.connections.delete(userId); + }; + const closeHandler = () => { + console.log(`Opus decoder for ${member?.displayName} closed`); + opusDecoder.removeListener("error", errorHandler); + opusDecoder.removeListener("close", closeHandler); + receiveStream?.removeListener("close", streamCloseHandler); + }; + opusDecoder.on("error", errorHandler); + opusDecoder.on("close", closeHandler); + receiveStream?.on("close", streamCloseHandler); + + this.client.emit( + "userStream", + userId, + name, + userName, + channel, + opusDecoder + ); + } - if (text && text.startsWith("/")) { - transcriptionText = ""; // Reset transcription text - return null; - } + leaveChannel(channel: BaseGuildVoiceChannel) { + const connection = this.connections.get(channel.id); + if (connection) { + connection.destroy(); + this.connections.delete(channel.id); + } - const memory = { - id: stringToUuid(channelId + "-voice-message-" + Date.now()), - content: { text: text, source: "discord", url: channel.url }, - userId: userIdUUID, - roomId, - embedding: embeddingZeroVector, - createdAt: Date.now(), - }; + // Stop monitoring all members in this channel + for (const [memberId, monitorInfo] of this.activeMonitors) { + if ( + monitorInfo.channel.id === channel.id && + memberId !== this.client.user?.id + ) { + this.stopMonitoringMember(memberId); + } + } - if (!memory.content.text) { - transcriptionText = ""; // Reset transcription text - return { text: "", action: "IGNORE" }; - } + console.log(`Left voice channel: ${channel.name} (${channel.id})`); + } - await this.runtime.messageManager.createMemory(memory); + stopMonitoringMember(memberId: string) { + const monitorInfo = this.activeMonitors.get(memberId); + if (monitorInfo) { + monitorInfo.monitor.stop(); + this.activeMonitors.delete(memberId); + this.streams.delete(memberId); + console.log(`Stopped monitoring user ${memberId}`); + } + } - state = await this.runtime.updateRecentMessageState(state); + async handleGuildCreate(guild: Guild) { + console.log(`Joined guild ${guild.name}`); + // this.scanGuild(guild); + } - const shouldIgnore = await this._shouldIgnore(memory); + async handleUserStream( + userId: UUID, + name: string, + userName: string, + channel: BaseGuildVoiceChannel, + audioStream: Readable + ) { + const channelId = channel.id; + const buffers: Buffer[] = []; + let totalLength = 0; + const maxSilenceTime = 1000; // Maximum pause duration in milliseconds + const minSilenceTime = 50; // Minimum silence duration to trigger transcription + let lastChunkTime = Date.now(); + let transcriptionStarted = false; + let transcriptionText = ""; + console.log("new audio monitor for: ", userId); + + const monitor = new AudioMonitor( + audioStream, + 10000000, + async (buffer) => { + console.log("buffer: ", buffer); + const currentTime = Date.now(); + const silenceDuration = currentTime - lastChunkTime; + if (!buffer) { + // Handle error + console.error("Empty buffer received"); + return; + } + buffers.push(buffer); + totalLength += buffer.length; + lastChunkTime = currentTime; + + if (silenceDuration > minSilenceTime && !transcriptionStarted) { + transcriptionStarted = true; + const inputBuffer = Buffer.concat(buffers, totalLength); + buffers.length = 0; + totalLength = 0; + + try { + // Convert Opus to WAV and add the header + const wavBuffer = + await this.convertOpusToWav(inputBuffer); + + console.log("starting transcription"); + const text = + await this.runtime.transcriptionService.transcribe( + wavBuffer + ); + console.log("transcribed text: ", text); + transcriptionText += text; + } catch (error) { + console.error("Error processing audio stream:", error); + } + } + + if (silenceDuration > maxSilenceTime && transcriptionStarted) { + console.log("transcription finished"); + transcriptionStarted = false; + + if (!transcriptionText) return; + + try { + const text = transcriptionText; + + // handle whisper cases + if ( + (text.length < 15 && + text.includes("[BLANK_AUDIO]")) || + (text.length < 5 && + text.toLowerCase().includes("you")) + ) { + transcriptionText = ""; // Reset transcription text + return; + } + + const roomId = stringToUuid(channelId); + const userIdUUID = stringToUuid(userId); + await this.runtime.ensureUserExists( + this.runtime.agentId, + this.client.user.username, + this.runtime.character.name, + "discord" + ); + await Promise.all([ + this.runtime.ensureUserExists( + userIdUUID, + userName, + name, + "discord" + ), + this.runtime.ensureRoomExists(roomId), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom( + userIdUUID, + roomId + ), + this.runtime.ensureParticipantInRoom( + this.runtime.agentId, + roomId + ), + ]); + + let state = await this.runtime.composeState( + { + content: { text: text, source: "Discord" }, + userId: userIdUUID, + roomId, + }, + { + discordChannel: channel, + discordClient: this.client, + agentName: this.runtime.character.name, + } + ); + + if (text && text.startsWith("/")) { + transcriptionText = ""; // Reset transcription text + return null; + } + + const memory = { + id: stringToUuid( + channelId + "-voice-message-" + Date.now() + ), + content: { + text: text, + source: "discord", + url: channel.url, + }, + userId: userIdUUID, + roomId, + embedding: embeddingZeroVector, + createdAt: Date.now(), + }; + + if (!memory.content.text) { + transcriptionText = ""; // Reset transcription text + return { text: "", action: "IGNORE" }; + } + + await this.runtime.messageManager.createMemory(memory); + + state = + await this.runtime.updateRecentMessageState(state); + + const shouldIgnore = await this._shouldIgnore(memory); + + if (shouldIgnore) { + transcriptionText = ""; // Reset transcription text + return { text: "", action: "IGNORE" }; + } + + const context = composeContext({ + state, + template: voiceHandlerTemplate, + }); + + const responseContent = await this._generateResponse( + memory, + state, + context + ); + + const callback: HandlerCallback = async ( + content: Content + ) => { + console.log("callback content: ", content); + const { roomId } = memory; + + const responseMemory: Memory = { + id: stringToUuid( + memory.id + "-voice-response-" + Date.now() + ), + userId: this.runtime.agentId, + content: { + ...content, + user: this.runtime.character.name, + inReplyTo: memory.id, + }, + roomId, + embedding: embeddingZeroVector, + }; + + if (responseMemory.content.text?.trim()) { + await this.runtime.messageManager.createMemory( + responseMemory + ); + state = + await this.runtime.updateRecentMessageState( + state + ); + const responseStream = + await SpeechService.generate( + this.runtime, + content.text + ); + + if (responseStream) { + await this.playAudioStream( + userId, + responseStream as Readable + ); + } + await this.runtime.evaluate(memory, state); + } else { + console.warn("Empty response, skipping"); + } + return [responseMemory]; + }; + + const responseMemories = + await callback(responseContent); + + const response = responseContent; + + const content = (response.responseMessage || + response.content || + response.message) as string; + + if (!content) { + transcriptionText = ""; // Reset transcription text + return null; + } + + console.log("responseMemories: ", responseMemories); + + await this.runtime.processActions( + memory, + responseMemories, + state, + callback + ); + + transcriptionText = ""; // Reset transcription text + } catch (error) { + console.error( + "Error processing transcribed text:", + error + ); + transcriptionText = ""; // Reset transcription text + } + } + } + ); + } - if (shouldIgnore) { - transcriptionText = ""; // Reset transcription text - return { text: "", action: "IGNORE" }; - } + private async convertOpusToWav(pcmBuffer: Buffer): Promise { + try { + // Generate the WAV header + const wavHeader = getWavHeader( + pcmBuffer.length, + DECODE_SAMPLE_RATE + ); - const context = composeContext({ - state, - template: voiceHandlerTemplate, - }); + // Concatenate the WAV header and PCM data + const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]); - const responseContent = await this._generateResponse( - memory, - state, - context, - ); - - const callback: HandlerCallback = async (content: Content) => { - console.log("callback content: ", content); - const { roomId } = memory; - - const responseMemory: Memory = { - id: stringToUuid(memory.id + "-voice-response-" + Date.now()), - userId: this.runtime.agentId, - content: { - ...content, - user: this.runtime.character.name, - inReplyTo: memory.id, - }, - roomId, - embedding: embeddingZeroVector, - }; - - if (responseMemory.content.text?.trim()) { - await this.runtime.messageManager.createMemory(responseMemory); - state = await this.runtime.updateRecentMessageState(state); - const responseStream = await SpeechService.generate( - this.runtime, - content.text, - ); - - if (responseStream) { - await this.playAudioStream(userId, responseStream as Readable); - } - await this.runtime.evaluate(memory, state); - } else { - console.warn("Empty response, skipping"); - } - return [responseMemory]; - }; - - const responseMemories = await callback(responseContent); + return wavBuffer; + } catch (error) { + console.error("Error converting PCM to WAV:", error); + throw error; + } + } - const response = responseContent; + private async _generateResponse( + message: Memory, + state: State, + context: string + ): Promise { + const { userId, roomId } = message; - const content = (response.responseMessage || - response.content || - response.message) as string; + const datestr = new Date().toUTCString().replace(/:/g, "-"); - if (!content) { - transcriptionText = ""; // Reset transcription text - return null; - } + // log context to file + log_to_file( + `${state.agentName}_${datestr}_discord_voice_context`, + context + ); - console.log("responseMemories: ", responseMemories); + const response = await generateMessageResponse({ + runtime: this.runtime, + context, + modelClass: ModelClass.SMALL, + }); - await this.runtime.processActions( - memory, - responseMemories, - state, - callback, - ); + response.source = "discord"; - transcriptionText = ""; // Reset transcription text - } catch (error) { - console.error("Error processing transcribed text:", error); - transcriptionText = ""; // Reset transcription text + if (!response) { + console.error("No response from generateMessageResponse"); + return; } - } - }); - } - - private async convertOpusToWav(pcmBuffer: Buffer): Promise { - try { - // Generate the WAV header - const wavHeader = getWavHeader(pcmBuffer.length, DECODE_SAMPLE_RATE); - - // Concatenate the WAV header and PCM data - const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]); - - return wavBuffer; - } catch (error) { - console.error("Error converting PCM to WAV:", error); - throw error; - } - } - - private async _generateResponse( - message: Memory, - state: State, - context: string, - ): Promise { - const { userId, roomId } = message; - const datestr = new Date().toUTCString().replace(/:/g, "-"); + log_to_file( + `${state.agentName}_${datestr}_discord_voice_response`, + JSON.stringify(response) + ); - // log context to file - log_to_file(`${state.agentName}_${datestr}_discord_voice_context`, context); + await this.runtime.databaseAdapter.log({ + body: { message, context, response }, + userId: userId, + roomId, + type: "response", + }); - const response = await generateMessageResponse({ - runtime: this.runtime, - context, - modelClass: ModelClass.SMALL, - }); + return response; + } - response.source = "discord"; + private async _shouldIgnore(message: Memory): Promise { + console.log("message: ", message); + console.log("message.content: ", message.content); + // if the message is 3 characters or less, ignore it + if ((message.content as Content).text.length < 3) { + return true; + } - if (!response) { - console.error("No response from generateMessageResponse"); - return; - } + const loseInterestWords = [ + // telling the bot to stop talking + "shut up", + "stop", + "dont talk", + "silence", + "stop talking", + "be quiet", + "hush", + "stfu", + "stupid bot", + "dumb bot", + + // offensive words + "fuck", + "shit", + "damn", + "suck", + "dick", + "cock", + "sex", + "sexy", + ]; + if ( + (message.content as Content).text.length < 50 && + loseInterestWords.some((word) => + (message.content as Content).text?.toLowerCase().includes(word) + ) + ) { + return true; + } - log_to_file( - `${state.agentName}_${datestr}_discord_voice_response`, - JSON.stringify(response), - ); - - await this.runtime.databaseAdapter.log({ - body: { message, context, response }, - userId: userId, - roomId, - type: "response", - }); - - return response; - } - - private async _shouldIgnore(message: Memory): Promise { - console.log("message: ", message); - console.log("message.content: ", message.content); - // if the message is 3 characters or less, ignore it - if ((message.content as Content).text.length < 3) { - return true; - } + const ignoreWords = ["k", "ok", "bye", "lol", "nm", "uh"]; + if ( + (message.content as Content).text?.length < 8 && + ignoreWords.some((word) => + (message.content as Content).text?.toLowerCase().includes(word) + ) + ) { + return true; + } - const loseInterestWords = [ - // telling the bot to stop talking - "shut up", - "stop", - "dont talk", - "silence", - "stop talking", - "be quiet", - "hush", - "stfu", - "stupid bot", - "dumb bot", - - // offensive words - "fuck", - "shit", - "damn", - "suck", - "dick", - "cock", - "sex", - "sexy", - ]; - if ( - (message.content as Content).text.length < 50 && - loseInterestWords.some((word) => - (message.content as Content).text?.toLowerCase().includes(word), - ) - ) { - return true; + return false; } - const ignoreWords = ["k", "ok", "bye", "lol", "nm", "uh"]; - if ( - (message.content as Content).text?.length < 8 && - ignoreWords.some((word) => - (message.content as Content).text?.toLowerCase().includes(word), - ) - ) { - return true; - } + async scanGuild(guild: Guild) { + const channels = (await guild.channels.fetch()).filter( + (channel) => channel?.type == ChannelType.GuildVoice + ); + let chosenChannel: BaseGuildVoiceChannel | null = null; + + for (const [, channel] of channels) { + const voiceChannel = channel as BaseGuildVoiceChannel; + if ( + voiceChannel.members.size > 0 && + (chosenChannel === null || + voiceChannel.members.size > chosenChannel.members.size) + ) { + chosenChannel = voiceChannel; + } + } - return false; - } - - async scanGuild(guild: Guild) { - const channels = (await guild.channels.fetch()).filter( - (channel) => channel?.type == ChannelType.GuildVoice, - ); - let chosenChannel: BaseGuildVoiceChannel | null = null; - - for (const [, channel] of channels) { - const voiceChannel = channel as BaseGuildVoiceChannel; - if ( - voiceChannel.members.size > 0 && - (chosenChannel === null || - voiceChannel.members.size > chosenChannel.members.size) - ) { - chosenChannel = voiceChannel; - } + if (chosenChannel != null) { + this.joinChannel(chosenChannel); + } } - if (chosenChannel != null) { - this.joinChannel(chosenChannel); + async playAudioStream(userId: UUID, audioStream: Readable) { + const connection = this.connections.get(userId); + if (connection == null) { + console.log(`No connection for user ${userId}`); + return; + } + const audioPlayer = createAudioPlayer({ + behaviors: { + noSubscriber: NoSubscriberBehavior.Pause, + }, + }); + connection.subscribe(audioPlayer); + + const audioStartTime = Date.now(); + + const resource = createAudioResource(audioStream, { + inputType: StreamType.Arbitrary, + }); + audioPlayer.play(resource); + + audioPlayer.on("error", (err: any) => { + console.log(`Audio player error: ${err}`); + }); + + audioPlayer.on( + "stateChange", + (oldState: any, newState: { status: string }) => { + if (newState.status == "idle") { + const idleTime = Date.now(); + console.log( + `Audio playback took: ${idleTime - audioStartTime}ms` + ); + } + } + ); } - } - async playAudioStream(userId: UUID, audioStream: Readable) { - const connection = this.connections.get(userId); - if (connection == null) { - console.log(`No connection for user ${userId}`); - return; - } - const audioPlayer = createAudioPlayer({ - behaviors: { - noSubscriber: NoSubscriberBehavior.Pause, - }, - }); - connection.subscribe(audioPlayer); - - const audioStartTime = Date.now(); - - const resource = createAudioResource(audioStream, { - inputType: StreamType.Arbitrary, - }); - audioPlayer.play(resource); - - audioPlayer.on("error", (err: any) => { - console.log(`Audio player error: ${err}`); - }); - - audioPlayer.on( - "stateChange", - (oldState: any, newState: { status: string }) => { - if (newState.status == "idle") { - const idleTime = Date.now(); - console.log(`Audio playback took: ${idleTime - audioStartTime}ms`); + async handleJoinChannelCommand(interaction: any) { + const channelId = interaction.options.get("channel")?.value as string; + if (!channelId) { + await interaction.reply("Please provide a voice channel to join."); + return; + } + const guild = interaction.guild; + if (!guild) { + return; + } + const voiceChannel = interaction.guild.channels.cache.find( + (channel: VoiceChannel) => + channel.id === channelId && + channel.type === ChannelType.GuildVoice + ); + + if (!voiceChannel) { + await interaction.reply("Voice channel not found!"); + return; } - }, - ); - } - - async handleJoinChannelCommand(interaction: any) { - const channelId = interaction.options.get("channel")?.value as string; - if (!channelId) { - await interaction.reply("Please provide a voice channel to join."); - return; - } - const guild = interaction.guild; - if (!guild) { - return; - } - const voiceChannel = interaction.guild.channels.cache.find( - (channel: VoiceChannel) => - channel.id === channelId && channel.type === ChannelType.GuildVoice, - ); - - if (!voiceChannel) { - await interaction.reply("Voice channel not found!"); - return; - } - try { - this.joinChannel(voiceChannel as BaseGuildVoiceChannel); - await interaction.reply(`Joined voice channel: ${voiceChannel.name}`); - } catch (error) { - console.error("Error joining voice channel:", error); - await interaction.reply("Failed to join the voice channel."); + try { + this.joinChannel(voiceChannel as BaseGuildVoiceChannel); + await interaction.reply( + `Joined voice channel: ${voiceChannel.name}` + ); + } catch (error) { + console.error("Error joining voice channel:", error); + await interaction.reply("Failed to join the voice channel."); + } } - } - async handleLeaveChannelCommand(interaction: any) { - const connection = getVoiceConnection(interaction.guildId as any); + async handleLeaveChannelCommand(interaction: any) { + const connection = getVoiceConnection(interaction.guildId as any); - if (!connection) { - await interaction.reply("Not currently in a voice channel."); - return; - } + if (!connection) { + await interaction.reply("Not currently in a voice channel."); + return; + } - try { - connection.destroy(); - await interaction.reply("Left the voice channel."); - } catch (error) { - console.error("Error leaving voice channel:", error); - await interaction.reply("Failed to leave the voice channel."); + try { + connection.destroy(); + await interaction.reply("Left the voice channel."); + } catch (error) { + console.error("Error leaving voice channel:", error); + await interaction.reply("Failed to leave the voice channel."); + } } - } } diff --git a/core/src/clients/telegram/src/index.ts b/core/src/clients/telegram/src/index.ts index 3ef9a64f4a..21505cab59 100644 --- a/core/src/clients/telegram/src/index.ts +++ b/core/src/clients/telegram/src/index.ts @@ -4,79 +4,92 @@ import { IAgentRuntime } from "../../../core/types.ts"; import { MessageManager } from "./messageManager.ts"; export class TelegramClient { - private bot: Telegraf; - private runtime: IAgentRuntime; - private messageManager: MessageManager; + private bot: Telegraf; + private runtime: IAgentRuntime; + private messageManager: MessageManager; - constructor(runtime: IAgentRuntime, botToken: string) { - console.log("📱 Constructing new TelegramClient..."); - this.runtime = runtime; - this.bot = new Telegraf(botToken); - this.messageManager = new MessageManager(this.bot, this.runtime); + constructor(runtime: IAgentRuntime, botToken: string) { + console.log("📱 Constructing new TelegramClient..."); + this.runtime = runtime; + this.bot = new Telegraf(botToken); + this.messageManager = new MessageManager(this.bot, this.runtime); - console.log("Setting up message handler..."); - this.bot.on("message", async (ctx) => { - try { - console.log("📥 Received message:", ctx.message); - await this.messageManager.handleMessage(ctx); - } catch (error) { - console.error("❌ Error handling message:", error); - await ctx.reply("An error occurred while processing your message."); - } - }); + console.log("Setting up message handler..."); + this.bot.on("message", async (ctx) => { + try { + console.log("📥 Received message:", ctx.message); + await this.messageManager.handleMessage(ctx); + } catch (error) { + console.error("❌ Error handling message:", error); + await ctx.reply( + "An error occurred while processing your message." + ); + } + }); - // Handle specific message types for better logging - this.bot.on("photo", (ctx) => { - console.log("📸 Received photo message with caption:", ctx.message.caption); - }); + // Handle specific message types for better logging + this.bot.on("photo", (ctx) => { + console.log( + "📸 Received photo message with caption:", + ctx.message.caption + ); + }); - this.bot.on("document", (ctx) => { - console.log("📎 Received document message:", ctx.message.document.file_name); - }); + this.bot.on("document", (ctx) => { + console.log( + "📎 Received document message:", + ctx.message.document.file_name + ); + }); - this.bot.catch((err, ctx) => { - console.error(`❌ Telegram Error for ${ctx.updateType}:`, err); - ctx.reply("An unexpected error occurred. Please try again later."); - }); + this.bot.catch((err, ctx) => { + console.error(`❌ Telegram Error for ${ctx.updateType}:`, err); + ctx.reply("An unexpected error occurred. Please try again later."); + }); - console.log("✅ TelegramClient constructor completed"); - } - - public async start(): Promise { - console.log("🚀 Starting Telegram bot..."); - try { - this.bot.launch({ - dropPendingUpdates: true, - }); - console.log("✨ Telegram bot successfully launched and is running!"); - console.log(`Bot username: @${this.bot.botInfo?.username}`); + console.log("✅ TelegramClient constructor completed"); + } - // Graceful shutdown handlers - const shutdownHandler = async (signal: string) => { - console.log( - `⚠️ Received ${signal}. Shutting down Telegram bot gracefully...` - ); + public async start(): Promise { + console.log("🚀 Starting Telegram bot..."); try { - await this.stop(); - console.log("🛑 Telegram bot stopped gracefully"); + this.bot.launch({ + dropPendingUpdates: true, + }); + console.log( + "✨ Telegram bot successfully launched and is running!" + ); + console.log(`Bot username: @${this.bot.botInfo?.username}`); + + // Graceful shutdown handlers + const shutdownHandler = async (signal: string) => { + console.log( + `⚠️ Received ${signal}. Shutting down Telegram bot gracefully...` + ); + try { + await this.stop(); + console.log("🛑 Telegram bot stopped gracefully"); + } catch (error) { + console.error( + "❌ Error during Telegram bot shutdown:", + error + ); + throw error; + } + }; + + process.once("SIGINT", () => shutdownHandler("SIGINT")); + process.once("SIGTERM", () => shutdownHandler("SIGTERM")); + process.once("SIGHUP", () => shutdownHandler("SIGHUP")); } catch (error) { - console.error("❌ Error during Telegram bot shutdown:", error); - throw error; + console.error("❌ Failed to launch Telegram bot:", error); + throw error; } - }; - - process.once("SIGINT", () => shutdownHandler("SIGINT")); - process.once("SIGTERM", () => shutdownHandler("SIGTERM")); - process.once("SIGHUP", () => shutdownHandler("SIGHUP")); - } catch (error) { - console.error("❌ Failed to launch Telegram bot:", error); - throw error; } - } - public async stop(): Promise { - console.log("Stopping Telegram bot..."); - await this.bot.stop(); - console.log("Telegram bot stopped"); - } -} \ No newline at end of file + public async stop(): Promise { + console.log("Stopping Telegram bot..."); + await this.bot.stop(); + console.log("Telegram bot stopped"); + } +} diff --git a/core/src/clients/telegram/src/messageManager.ts b/core/src/clients/telegram/src/messageManager.ts index 0783d91f81..996ddb309f 100644 --- a/core/src/clients/telegram/src/messageManager.ts +++ b/core/src/clients/telegram/src/messageManager.ts @@ -6,358 +6,382 @@ import { composeContext } from "../../../core/context.ts"; import { log_to_file } from "../../../core/logger.ts"; import { embeddingZeroVector } from "../../../core/memory.ts"; import { - Content, - IAgentRuntime, - Memory, - State, - UUID, - HandlerCallback, - ModelClass, + Content, + IAgentRuntime, + Memory, + State, + UUID, + HandlerCallback, + ModelClass, } from "../../../core/types.ts"; import { stringToUuid } from "../../../core/uuid.ts"; import { - messageHandlerTemplate, - shouldRespondTemplate, + messageHandlerTemplate, + shouldRespondTemplate, } from "../../discord/templates.ts"; import ImageDescriptionService from "../../../services/image.ts"; -import { generateMessageResponse, generateShouldRespond } from "../../../core/generation.ts"; +import { + generateMessageResponse, + generateShouldRespond, +} from "../../../core/generation.ts"; const MAX_MESSAGE_LENGTH = 4096; // Telegram's max message length export class MessageManager { - private bot: Telegraf; - private runtime: IAgentRuntime; - private imageService: ImageDescriptionService; - - constructor(bot: Telegraf, runtime: IAgentRuntime) { - this.bot = bot; - this.runtime = runtime; - this.imageService = ImageDescriptionService.getInstance(this.runtime); - } - - // Process image messages and generate descriptions - private async processImage( - message: Message - ): Promise<{ description: string } | null> { - console.log("🖼️ Processing image message:", JSON.stringify(message, null, 2)); - - try { - let imageUrl: string | null = null; - - // Handle photo messages - if ("photo" in message && message.photo?.length > 0) { - const photo = message.photo[message.photo.length - 1]; - const fileLink = await this.bot.telegram.getFileLink(photo.file_id); - imageUrl = fileLink.toString(); - } - // Handle image documents - else if ( - "document" in message && - message.document?.mime_type?.startsWith("image/") - ) { - const doc = message.document; - const fileLink = await this.bot.telegram.getFileLink(doc.file_id); - imageUrl = fileLink.toString(); - } - - if (imageUrl) { - const { title, description } = await this.imageService.describeImage(imageUrl); - const fullDescription = `[Image: ${title}\n${description}]`; - return { description: fullDescription }; - } - } catch (error) { - console.error("❌ Error processing image:", error); + private bot: Telegraf; + private runtime: IAgentRuntime; + private imageService: ImageDescriptionService; + + constructor(bot: Telegraf, runtime: IAgentRuntime) { + this.bot = bot; + this.runtime = runtime; + this.imageService = ImageDescriptionService.getInstance(this.runtime); } - return null; // No image found - } - - // Decide if the bot should respond to the message - private async _shouldRespond( - message: Message, - state: State - ): Promise { - // Respond if bot is mentioned - if ( - "text" in message && - message.text?.includes(`@${this.bot.botInfo?.username}`) - ) { - return true; - } + // Process image messages and generate descriptions + private async processImage( + message: Message + ): Promise<{ description: string } | null> { + console.log( + "🖼️ Processing image message:", + JSON.stringify(message, null, 2) + ); - // Respond to private chats - if (message.chat.type === "private") { - return true; - } + try { + let imageUrl: string | null = null; + + // Handle photo messages + if ("photo" in message && message.photo?.length > 0) { + const photo = message.photo[message.photo.length - 1]; + const fileLink = await this.bot.telegram.getFileLink( + photo.file_id + ); + imageUrl = fileLink.toString(); + } + // Handle image documents + else if ( + "document" in message && + message.document?.mime_type?.startsWith("image/") + ) { + const doc = message.document; + const fileLink = await this.bot.telegram.getFileLink( + doc.file_id + ); + imageUrl = fileLink.toString(); + } + + if (imageUrl) { + const { title, description } = + await this.imageService.describeImage(imageUrl); + const fullDescription = `[Image: ${title}\n${description}]`; + return { description: fullDescription }; + } + } catch (error) { + console.error("❌ Error processing image:", error); + } - // Respond to images in group chats - if ( - "photo" in message || - ("document" in message && - message.document?.mime_type?.startsWith("image/")) - ) { - return true; + return null; // No image found } - // Use AI to decide for text or captions - if ("text" in message || ("caption" in message && message.caption)) { - const shouldRespondContext = composeContext({ - state, - template: shouldRespondTemplate, - }); + // Decide if the bot should respond to the message + private async _shouldRespond( + message: Message, + state: State + ): Promise { + // Respond if bot is mentioned + if ( + "text" in message && + message.text?.includes(`@${this.bot.botInfo?.username}`) + ) { + return true; + } - const response = await generateShouldRespond({ - runtime: this.runtime, - context: shouldRespondContext, - modelClass: ModelClass.SMALL, - }); + // Respond to private chats + if (message.chat.type === "private") { + return true; + } - return response === "RESPOND"; - } + // Respond to images in group chats + if ( + "photo" in message || + ("document" in message && + message.document?.mime_type?.startsWith("image/")) + ) { + return true; + } - return false; // No criteria met - } - - // Send long messages in chunks - private async sendMessageInChunks( - ctx: Context, - content: string, - replyToMessageId?: number - ): Promise { - const chunks = this.splitMessage(content); - const sentMessages: Message.TextMessage[] = []; - - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - const sentMessage = (await ctx.telegram.sendMessage(ctx.chat.id, chunk, { - reply_parameters: - i === 0 && replyToMessageId - ? { message_id: replyToMessageId } - : undefined, - })) as Message.TextMessage; - - sentMessages.push(sentMessage); - } + // Use AI to decide for text or captions + if ("text" in message || ("caption" in message && message.caption)) { + const shouldRespondContext = composeContext({ + state, + template: shouldRespondTemplate, + }); - return sentMessages; - } + const response = await generateShouldRespond({ + runtime: this.runtime, + context: shouldRespondContext, + modelClass: ModelClass.SMALL, + }); - // Split message into smaller parts - private splitMessage(text: string): string[] { - const chunks: string[] = []; - let currentChunk = ""; + return response === "RESPOND"; + } - const lines = text.split("\n"); - for (const line of lines) { - if (currentChunk.length + line.length + 1 <= MAX_MESSAGE_LENGTH) { - currentChunk += (currentChunk ? "\n" : "") + line; - } else { - if (currentChunk) chunks.push(currentChunk); - currentChunk = line; - } + return false; // No criteria met } - if (currentChunk) chunks.push(currentChunk); - return chunks; - } - - // Generate a response using AI - private async _generateResponse( - message: Memory, - state: State, - context: string - ): Promise { - const { userId, roomId } = message; - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - log_to_file( - `${state.agentName}_${datestr}_telegram_message_context`, - context - ); - - const response = await generateMessageResponse({ - runtime: this.runtime, - context, - modelClass: "slow" - }); - - if (!response) { - console.error("❌ No response from generateMessageResponse"); - return null; - } + // Send long messages in chunks + private async sendMessageInChunks( + ctx: Context, + content: string, + replyToMessageId?: number + ): Promise { + const chunks = this.splitMessage(content); + const sentMessages: Message.TextMessage[] = []; + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const sentMessage = (await ctx.telegram.sendMessage( + ctx.chat.id, + chunk, + { + reply_parameters: + i === 0 && replyToMessageId + ? { message_id: replyToMessageId } + : undefined, + } + )) as Message.TextMessage; + + sentMessages.push(sentMessage); + } - log_to_file( - `${state.agentName}_${datestr}_telegram_message_response`, - JSON.stringify(response) - ); - - await this.runtime.databaseAdapter.log({ - body: { message, context, response }, - userId: userId, - roomId, - type: "response", - }); - - return response; - } - - // Main handler for incoming messages - public async handleMessage(ctx: Context): Promise { - if (!ctx.message || !ctx.from) { - return; // Exit if no message or sender info + return sentMessages; } - //@ts-ignore - if (ctx.message.text.startsWith("/")) { - //Handle commands? - return; + // Split message into smaller parts + private splitMessage(text: string): string[] { + const chunks: string[] = []; + let currentChunk = ""; + + const lines = text.split("\n"); + for (const line of lines) { + if (currentChunk.length + line.length + 1 <= MAX_MESSAGE_LENGTH) { + currentChunk += (currentChunk ? "\n" : "") + line; + } else { + if (currentChunk) chunks.push(currentChunk); + currentChunk = line; + } + } + + if (currentChunk) chunks.push(currentChunk); + return chunks; } - const message = ctx.message; - - try { - // Convert IDs to UUIDs - const userId = stringToUuid(ctx.from.id.toString()) as UUID; - const userName = - ctx.from.username || ctx.from.first_name || "Unknown User"; - const chatId = stringToUuid(ctx.chat?.id.toString()) as UUID; - const agentId = this.runtime.agentId; - const roomId = chatId; - - // Ensure user and room exist - await Promise.all([ - this.runtime.ensureUserExists( - agentId, - this.bot.botInfo?.username || "Bot", - this.runtime.character.name, - "telegram" - ), - this.runtime.ensureUserExists(userId, userName, userName, "telegram"), - this.runtime.ensureRoomExists(roomId), - this.runtime.ensureParticipantInRoom(userId, roomId), - this.runtime.ensureParticipantInRoom(agentId, roomId), - ]); - - const messageId = stringToUuid(message.message_id.toString()) as UUID; - - // Handle images - const imageInfo = await this.processImage(message); - - // Get text or caption - let messageText = ""; - if ("text" in message) { - messageText = message.text; - } else if ("caption" in message && message.caption) { - messageText = message.caption; - } - - // Combine text and image description - const fullText = imageInfo - ? `${messageText} ${imageInfo.description}` - : messageText; - - if (!fullText) { - return; // Skip if no content - } - - const content: Content = { - text: fullText, - source: "telegram", - inReplyTo: - "reply_to_message" in message && message.reply_to_message - ? stringToUuid(message.reply_to_message.message_id.toString()) - : undefined, - }; - - // Create memory for the message - const memory: Memory = { - id: messageId, - userId, - roomId, - content, - createdAt: message.date * 1000, - embedding: embeddingZeroVector, - }; - - await this.runtime.messageManager.createMemory(memory); - - // Update state with the new memory - let state = await this.runtime.composeState(memory); - state = await this.runtime.updateRecentMessageState(state); - - // Decide whether to respond - const shouldRespond = await this._shouldRespond(message, state); - if (!shouldRespond) return; - - // Generate response - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - - const responseContent = await this._generateResponse( - memory, - state, - context - ); - - if (!responseContent || !responseContent.text) return; - - // Send response in chunks - const callback: HandlerCallback = async (content: Content) => { - const sentMessages = await this.sendMessageInChunks( - ctx, - content.text, - message.message_id + // Generate a response using AI + private async _generateResponse( + message: Memory, + state: State, + context: string + ): Promise { + const { userId, roomId } = message; + const datestr = new Date().toUTCString().replace(/:/g, "-"); + + log_to_file( + `${state.agentName}_${datestr}_telegram_message_context`, + context ); - const memories: Memory[] = []; + const response = await generateMessageResponse({ + runtime: this.runtime, + context, + modelClass: "slow", + }); - // Create memories for each sent message - for (let i = 0; i < sentMessages.length; i++) { - const sentMessage = sentMessages[i]; - const isLastMessage = i === sentMessages.length - 1; + if (!response) { + console.error("❌ No response from generateMessageResponse"); + return null; + } + + log_to_file( + `${state.agentName}_${datestr}_telegram_message_response`, + JSON.stringify(response) + ); - const memory: Memory = { - id: stringToUuid(sentMessage.message_id.toString()), - userId: agentId, + await this.runtime.databaseAdapter.log({ + body: { message, context, response }, + userId: userId, roomId, - content: { - ...content, - text: sentMessage.text, - action: !isLastMessage ? "CONTINUE" : undefined, - inReplyTo: messageId, - }, - createdAt: sentMessage.date * 1000, - embedding: embeddingZeroVector, - }; - - await this.runtime.messageManager.createMemory(memory); - memories.push(memory); + type: "response", + }); + + return response; + } + + // Main handler for incoming messages + public async handleMessage(ctx: Context): Promise { + if (!ctx.message || !ctx.from) { + return; // Exit if no message or sender info } - return memories; - }; - - // Execute callback to send messages and log memories - const responseMessages = await callback(responseContent); - - // Update state after response - state = await this.runtime.updateRecentMessageState(state); - await this.runtime.evaluate(memory, state); - - // Handle any resulting actions - await this.runtime.processActions( - memory, - responseMessages, - state, - callback - ); - } catch (error) { - console.error("❌ Error handling message:", error); - await ctx.reply( - "Sorry, I encountered an error while processing your request." - ); + //@ts-ignore + if (ctx.message.text.startsWith("/")) { + //Handle commands? + return; + } + + const message = ctx.message; + + try { + // Convert IDs to UUIDs + const userId = stringToUuid(ctx.from.id.toString()) as UUID; + const userName = + ctx.from.username || ctx.from.first_name || "Unknown User"; + const chatId = stringToUuid(ctx.chat?.id.toString()) as UUID; + const agentId = this.runtime.agentId; + const roomId = chatId; + + // Ensure user and room exist + await Promise.all([ + this.runtime.ensureUserExists( + agentId, + this.bot.botInfo?.username || "Bot", + this.runtime.character.name, + "telegram" + ), + this.runtime.ensureUserExists( + userId, + userName, + userName, + "telegram" + ), + this.runtime.ensureRoomExists(roomId), + this.runtime.ensureParticipantInRoom(userId, roomId), + this.runtime.ensureParticipantInRoom(agentId, roomId), + ]); + + const messageId = stringToUuid( + message.message_id.toString() + ) as UUID; + + // Handle images + const imageInfo = await this.processImage(message); + + // Get text or caption + let messageText = ""; + if ("text" in message) { + messageText = message.text; + } else if ("caption" in message && message.caption) { + messageText = message.caption; + } + + // Combine text and image description + const fullText = imageInfo + ? `${messageText} ${imageInfo.description}` + : messageText; + + if (!fullText) { + return; // Skip if no content + } + + const content: Content = { + text: fullText, + source: "telegram", + inReplyTo: + "reply_to_message" in message && message.reply_to_message + ? stringToUuid( + message.reply_to_message.message_id.toString() + ) + : undefined, + }; + + // Create memory for the message + const memory: Memory = { + id: messageId, + userId, + roomId, + content, + createdAt: message.date * 1000, + embedding: embeddingZeroVector, + }; + + await this.runtime.messageManager.createMemory(memory); + + // Update state with the new memory + let state = await this.runtime.composeState(memory); + state = await this.runtime.updateRecentMessageState(state); + + // Decide whether to respond + const shouldRespond = await this._shouldRespond(message, state); + if (!shouldRespond) return; + + // Generate response + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); + + const responseContent = await this._generateResponse( + memory, + state, + context + ); + + if (!responseContent || !responseContent.text) return; + + // Send response in chunks + const callback: HandlerCallback = async (content: Content) => { + const sentMessages = await this.sendMessageInChunks( + ctx, + content.text, + message.message_id + ); + + const memories: Memory[] = []; + + // Create memories for each sent message + for (let i = 0; i < sentMessages.length; i++) { + const sentMessage = sentMessages[i]; + const isLastMessage = i === sentMessages.length - 1; + + const memory: Memory = { + id: stringToUuid(sentMessage.message_id.toString()), + userId: agentId, + roomId, + content: { + ...content, + text: sentMessage.text, + action: !isLastMessage ? "CONTINUE" : undefined, + inReplyTo: messageId, + }, + createdAt: sentMessage.date * 1000, + embedding: embeddingZeroVector, + }; + + await this.runtime.messageManager.createMemory(memory); + memories.push(memory); + } + + return memories; + }; + + // Execute callback to send messages and log memories + const responseMessages = await callback(responseContent); + + // Update state after response + state = await this.runtime.updateRecentMessageState(state); + await this.runtime.evaluate(memory, state); + + // Handle any resulting actions + await this.runtime.processActions( + memory, + responseMessages, + state, + callback + ); + } catch (error) { + console.error("❌ Error handling message:", error); + await ctx.reply( + "Sorry, I encountered an error while processing your request." + ); + } } - } -} \ No newline at end of file +} diff --git a/core/src/clients/twitter/base.ts b/core/src/clients/twitter/base.ts index 4a4e484281..61d5e58c2f 100644 --- a/core/src/clients/twitter/base.ts +++ b/core/src/clients/twitter/base.ts @@ -1,10 +1,10 @@ // @ts-nocheck import { - QueryTweetsResponse, - Scraper, - SearchMode, - Tweet, + QueryTweetsResponse, + Scraper, + SearchMode, + Tweet, } from "agent-twitter-client"; import { EventEmitter } from "events"; import fs from "fs"; @@ -12,11 +12,11 @@ import path from "path"; import { fileURLToPath } from "url"; import { embeddingZeroVector } from "../../core/memory.ts"; import { - Content, - IAgentRuntime, - Memory, - State, - UUID, + Content, + IAgentRuntime, + Memory, + State, + UUID, } from "../../core/types.ts"; import ImageDescriptionService from "../../services/image.ts"; @@ -25,536 +25,577 @@ import { glob } from "glob"; import { stringToUuid } from "../../core/uuid.ts"; export function extractAnswer(text: string): string { - const startIndex = text.indexOf("Answer: ") + 8; - const endIndex = text.indexOf("<|endoftext|>", 11); - return text.slice(startIndex, endIndex); + const startIndex = text.indexOf("Answer: ") + 8; + const endIndex = text.indexOf("<|endoftext|>", 11); + return text.slice(startIndex, endIndex); } const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); class RequestQueue { - private queue: (() => Promise)[] = []; - private processing: boolean = false; + private queue: (() => Promise)[] = []; + private processing: boolean = false; + + async add(request: () => Promise): Promise { + return new Promise((resolve, reject) => { + this.queue.push(async () => { + try { + const result = await request(); + resolve(result); + } catch (error) { + reject(error); + } + }); + this.processQueue(); + }); + } - async add(request: () => Promise): Promise { - return new Promise((resolve, reject) => { - this.queue.push(async () => { - try { - const result = await request(); - resolve(result); - } catch (error) { - reject(error); + private async processQueue(): Promise { + if (this.processing || this.queue.length === 0) { + return; + } + this.processing = true; + + while (this.queue.length > 0) { + const request = this.queue.shift()!; + try { + await request(); + } catch (error) { + console.error("Error processing request:", error); + this.queue.unshift(request); + await this.exponentialBackoff(this.queue.length); + } + await this.randomDelay(); } - }); - this.processQueue(); - }); - } - - private async processQueue(): Promise { - if (this.processing || this.queue.length === 0) { - return; - } - this.processing = true; - - while (this.queue.length > 0) { - const request = this.queue.shift()!; - try { - await request(); - } catch (error) { - console.error("Error processing request:", error); - this.queue.unshift(request); - await this.exponentialBackoff(this.queue.length); - } - await this.randomDelay(); - } - this.processing = false; - } + this.processing = false; + } - private async exponentialBackoff(retryCount: number): Promise { - const delay = Math.pow(2, retryCount) * 1000; - await new Promise((resolve) => setTimeout(resolve, delay)); - } + private async exponentialBackoff(retryCount: number): Promise { + const delay = Math.pow(2, retryCount) * 1000; + await new Promise((resolve) => setTimeout(resolve, delay)); + } - private async randomDelay(): Promise { - const delay = Math.floor(Math.random() * 2000) + 1500; - await new Promise((resolve) => setTimeout(resolve, delay)); - } + private async randomDelay(): Promise { + const delay = Math.floor(Math.random() * 2000) + 1500; + await new Promise((resolve) => setTimeout(resolve, delay)); + } } export class ClientBase extends EventEmitter { - static _twitterClient: Scraper; - twitterClient: Scraper; - runtime: IAgentRuntime; - directions: string; - lastCheckedTweetId: number | null = null; - tweetCacheFilePath = "tweetcache/latest_checked_tweet_id.txt"; - imageDescriptionService: ImageDescriptionService; - temperature: number = 0.5; - dryRun: boolean = false; - - private tweetCache: Map = new Map(); - requestQueue: RequestQueue = new RequestQueue(); - twitterUserId: string; - - async cacheTweet(tweet: Tweet): Promise { - if (!tweet) { - console.warn("Tweet is undefined, skipping cache"); - return; - } - const cacheDir = path.join( - __dirname, - "../../../tweetcache", - tweet.conversationId, - `${tweet.id}.json`, - ); - await fs.promises.mkdir(path.dirname(cacheDir), { recursive: true }); - await fs.promises.writeFile(cacheDir, JSON.stringify(tweet, null, 2)); - this.tweetCache.set(tweet.id, tweet); - } - - async getCachedTweet(tweetId: string): Promise { - if (this.tweetCache.has(tweetId)) { - return this.tweetCache.get(tweetId); + static _twitterClient: Scraper; + twitterClient: Scraper; + runtime: IAgentRuntime; + directions: string; + lastCheckedTweetId: number | null = null; + tweetCacheFilePath = "tweetcache/latest_checked_tweet_id.txt"; + imageDescriptionService: ImageDescriptionService; + temperature: number = 0.5; + dryRun: boolean = false; + + private tweetCache: Map = new Map(); + requestQueue: RequestQueue = new RequestQueue(); + twitterUserId: string; + + async cacheTweet(tweet: Tweet): Promise { + if (!tweet) { + console.warn("Tweet is undefined, skipping cache"); + return; + } + const cacheDir = path.join( + __dirname, + "../../../tweetcache", + tweet.conversationId, + `${tweet.id}.json` + ); + await fs.promises.mkdir(path.dirname(cacheDir), { recursive: true }); + await fs.promises.writeFile(cacheDir, JSON.stringify(tweet, null, 2)); + this.tweetCache.set(tweet.id, tweet); } - const cacheFile = path.join( - __dirname, - "tweetcache", - "*", - `${tweetId}.json`, - ); - const files = await glob(cacheFile); - if (files.length > 0) { - const tweetData = await fs.promises.readFile(files[0], "utf-8"); - const tweet = JSON.parse(tweetData) as Tweet; - this.tweetCache.set(tweet.id, tweet); - return tweet; - } + async getCachedTweet(tweetId: string): Promise { + if (this.tweetCache.has(tweetId)) { + return this.tweetCache.get(tweetId); + } - return undefined; - } + const cacheFile = path.join( + __dirname, + "tweetcache", + "*", + `${tweetId}.json` + ); + const files = await glob(cacheFile); + if (files.length > 0) { + const tweetData = await fs.promises.readFile(files[0], "utf-8"); + const tweet = JSON.parse(tweetData) as Tweet; + this.tweetCache.set(tweet.id, tweet); + return tweet; + } - async getTweet(tweetId: string): Promise { - const cachedTweet = await this.getCachedTweet(tweetId); - if (cachedTweet) { - return cachedTweet; + return undefined; } - const tweet = await this.requestQueue.add(() => - this.twitterClient.getTweet(tweetId) - ); - await this.cacheTweet(tweet); - return tweet; - } - - callback: (self: ClientBase) => any = null; - - onReady() { - throw new Error("Not implemented in base class, please call from subclass"); - } - - constructor({ runtime }: { runtime: IAgentRuntime }) { - super(); - this.runtime = runtime; - if (ClientBase._twitterClient) { - this.twitterClient = ClientBase._twitterClient; - } else { - this.twitterClient = new Scraper(); - ClientBase._twitterClient = this.twitterClient; - } - this.dryRun = - this.runtime.getSetting("TWITTER_DRY_RUN")?.toLowerCase() === "true"; - this.directions = "- " + - this.runtime.character.style.all.join("\n- ") + - "- " + - this.runtime.character.style.post.join(); - - try { - if (fs.existsSync(this.tweetCacheFilePath)) { - const data = fs.readFileSync(this.tweetCacheFilePath, "utf-8"); - this.lastCheckedTweetId = parseInt(data.trim()); - } else { - console.warn("Tweet cache file not found."); - } - } catch (error) { - console.error("Error loading latest checked tweet ID from file:", error); - } - const cookiesFilePath = path.join( - __dirname, - "../../../tweetcache/" + - this.runtime.getSetting("TWITTER_USERNAME") + - "_cookies.json", - ); - - const dir = path.dirname(cookiesFilePath); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); + async getTweet(tweetId: string): Promise { + const cachedTweet = await this.getCachedTweet(tweetId); + if (cachedTweet) { + return cachedTweet; + } + + const tweet = await this.requestQueue.add(() => + this.twitterClient.getTweet(tweetId) + ); + await this.cacheTweet(tweet); + return tweet; } - // async initialization - (async () => { - // Check for Twitter cookies - if (this.runtime.getSetting("TWITTER_COOKIES")) { - const cookiesArray = JSON.parse( - this.runtime.getSetting("TWITTER_COOKIES"), + callback: (self: ClientBase) => any = null; + + onReady() { + throw new Error( + "Not implemented in base class, please call from subclass" ); - await this.setCookiesFromArray(cookiesArray); - } else { - console.log("Cookies file path:", cookiesFilePath); - if (fs.existsSync(cookiesFilePath)) { - const cookiesArray = JSON.parse( - fs.readFileSync(cookiesFilePath, "utf-8"), - ); - await this.setCookiesFromArray(cookiesArray); + } + + constructor({ runtime }: { runtime: IAgentRuntime }) { + super(); + this.runtime = runtime; + if (ClientBase._twitterClient) { + this.twitterClient = ClientBase._twitterClient; } else { - await this.twitterClient.login( - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.getSetting("TWITTER_PASSWORD"), - this.runtime.getSetting("TWITTER_EMAIL"), - ); - console.log("Logged in to Twitter"); - const cookies = await this.twitterClient.getCookies(); - fs.writeFileSync(cookiesFilePath, JSON.stringify(cookies), "utf-8"); + this.twitterClient = new Scraper(); + ClientBase._twitterClient = this.twitterClient; } - } + this.dryRun = + this.runtime.getSetting("TWITTER_DRY_RUN")?.toLowerCase() === + "true"; + this.directions = + "- " + + this.runtime.character.style.all.join("\n- ") + + "- " + + this.runtime.character.style.post.join(); - let loggedInWaits = 0; - - while (!(await this.twitterClient.isLoggedIn())) { - console.log("Waiting for Twitter login"); - await new Promise((resolve) => setTimeout(resolve, 2000)); - if (loggedInWaits > 10) { - console.error("Failed to login to Twitter"); - await this.twitterClient.login( - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.getSetting("TWITTER_PASSWORD"), - this.runtime.getSetting("TWITTER_EMAIL"), - ); + try { + if (fs.existsSync(this.tweetCacheFilePath)) { + const data = fs.readFileSync(this.tweetCacheFilePath, "utf-8"); + this.lastCheckedTweetId = parseInt(data.trim()); + } else { + console.warn("Tweet cache file not found."); + } + } catch (error) { + console.error( + "Error loading latest checked tweet ID from file:", + error + ); + } + const cookiesFilePath = path.join( + __dirname, + "../../../tweetcache/" + + this.runtime.getSetting("TWITTER_USERNAME") + + "_cookies.json" + ); - const cookies = await this.twitterClient.getCookies(); - fs.writeFileSync(cookiesFilePath, JSON.stringify(cookies), "utf-8"); - loggedInWaits = 0; + const dir = path.dirname(cookiesFilePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); } - loggedInWaits++; - } - const userId = await this.requestQueue.add( - async () => { - // wait 3 seconds before getting the user id - await new Promise((resolve) => setTimeout(resolve, 10000)); - try { - return await this.twitterClient.getUserIdByScreenName( - this.runtime.getSetting("TWITTER_USERNAME"), - ); - } catch (error) { - console.error("Error getting user ID:", error); - return null; - } - }, - ); - if (!userId) { - console.error("Failed to get user ID"); - return; - } - console.log("Twitter user ID:", userId); - this.twitterUserId = userId; - - await this.populateTimeline(); - - this.onReady(); - })(); - } - - async fetchHomeTimeline(count: number): Promise { - const homeTimeline = await this.twitterClient.fetchHomeTimeline(count, []); - - return homeTimeline.filter((t) => - t.__typename !== "TweetWithVisibilityResults" - ).map((tweet) => { - console.log("tweet is", tweet); - const obj = { - id: tweet.rest_id, - name: tweet.name ?? tweet.core?.user_results?.result?.legacy.name, - username: tweet.username ?? - tweet.core?.user_results?.result?.legacy.screen_name, - text: tweet.text ?? tweet.legacy?.full_text, - inReplyToStatusId: tweet.inReplyToStatusId ?? - tweet.legacy?.in_reply_to_status_id_str, - createdAt: tweet.createdAt ?? tweet.legacy?.created_at, - userId: tweet.userId ?? tweet.legacy?.user_id_str, - conversationId: tweet.conversationId ?? - tweet.legacy?.conversation_id_str, - hashtags: tweet.hashtags ?? tweet.legacy?.entities.hashtags, - mentions: tweet.mentions ?? tweet.legacy?.entities.user_mentions, - photos: tweet.photos ?? - tweet.legacy?.entities.media?.filter( - (media) => media.type === "photo", - ) ?? [], - thread: [], - urls: tweet.urls ?? tweet.legacy?.entities.urls, - videos: tweet.videos ?? - tweet.legacy?.entities.media?.filter( - (media) => media.type === "video", - ) ?? [], - }; - - console.log("obj is", obj); - - return obj; - }); - } - - async fetchSearchTweets( - query: string, - maxTweets: number, - searchMode: SearchMode, - cursor?: string, - ): Promise { - try { - // Sometimes this fails because we are rate limited. in this case, we just need to return an empty array - // if we dont get a response in 5 seconds, something is wrong - const timeoutPromise = new Promise((resolve) => - setTimeout(() => resolve({ tweets: [] }), 10000) - ); - - try { - const result = await this.requestQueue.add( - async () => - await Promise.race([ - this.twitterClient.fetchSearchTweets( - query, - maxTweets, - searchMode, - cursor, - ), - timeoutPromise, - ]), + + // async initialization + (async () => { + // Check for Twitter cookies + if (this.runtime.getSetting("TWITTER_COOKIES")) { + const cookiesArray = JSON.parse( + this.runtime.getSetting("TWITTER_COOKIES") + ); + await this.setCookiesFromArray(cookiesArray); + } else { + console.log("Cookies file path:", cookiesFilePath); + if (fs.existsSync(cookiesFilePath)) { + const cookiesArray = JSON.parse( + fs.readFileSync(cookiesFilePath, "utf-8") + ); + await this.setCookiesFromArray(cookiesArray); + } else { + await this.twitterClient.login( + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.getSetting("TWITTER_PASSWORD"), + this.runtime.getSetting("TWITTER_EMAIL") + ); + console.log("Logged in to Twitter"); + const cookies = await this.twitterClient.getCookies(); + fs.writeFileSync( + cookiesFilePath, + JSON.stringify(cookies), + "utf-8" + ); + } + } + + let loggedInWaits = 0; + + while (!(await this.twitterClient.isLoggedIn())) { + console.log("Waiting for Twitter login"); + await new Promise((resolve) => setTimeout(resolve, 2000)); + if (loggedInWaits > 10) { + console.error("Failed to login to Twitter"); + await this.twitterClient.login( + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.getSetting("TWITTER_PASSWORD"), + this.runtime.getSetting("TWITTER_EMAIL") + ); + + const cookies = await this.twitterClient.getCookies(); + fs.writeFileSync( + cookiesFilePath, + JSON.stringify(cookies), + "utf-8" + ); + loggedInWaits = 0; + } + loggedInWaits++; + } + const userId = await this.requestQueue.add(async () => { + // wait 3 seconds before getting the user id + await new Promise((resolve) => setTimeout(resolve, 10000)); + try { + return await this.twitterClient.getUserIdByScreenName( + this.runtime.getSetting("TWITTER_USERNAME") + ); + } catch (error) { + console.error("Error getting user ID:", error); + return null; + } + }); + if (!userId) { + console.error("Failed to get user ID"); + return; + } + console.log("Twitter user ID:", userId); + this.twitterUserId = userId; + + await this.populateTimeline(); + + this.onReady(); + })(); + } + + async fetchHomeTimeline(count: number): Promise { + const homeTimeline = await this.twitterClient.fetchHomeTimeline( + count, + [] ); - return (result ?? { tweets: [] }) as QueryTweetsResponse; - } catch (error) { - console.error("Error fetching search tweets:", error); - return { tweets: [] }; - } - } catch (error) { - console.error("Error fetching search tweets:", error); - return { tweets: [] }; + + return homeTimeline + .filter((t) => t.__typename !== "TweetWithVisibilityResults") + .map((tweet) => { + console.log("tweet is", tweet); + const obj = { + id: tweet.rest_id, + name: + tweet.name ?? + tweet.core?.user_results?.result?.legacy.name, + username: + tweet.username ?? + tweet.core?.user_results?.result?.legacy.screen_name, + text: tweet.text ?? tweet.legacy?.full_text, + inReplyToStatusId: + tweet.inReplyToStatusId ?? + tweet.legacy?.in_reply_to_status_id_str, + createdAt: tweet.createdAt ?? tweet.legacy?.created_at, + userId: tweet.userId ?? tweet.legacy?.user_id_str, + conversationId: + tweet.conversationId ?? + tweet.legacy?.conversation_id_str, + hashtags: tweet.hashtags ?? tweet.legacy?.entities.hashtags, + mentions: + tweet.mentions ?? tweet.legacy?.entities.user_mentions, + photos: + tweet.photos ?? + tweet.legacy?.entities.media?.filter( + (media) => media.type === "photo" + ) ?? + [], + thread: [], + urls: tweet.urls ?? tweet.legacy?.entities.urls, + videos: + tweet.videos ?? + tweet.legacy?.entities.media?.filter( + (media) => media.type === "video" + ) ?? + [], + }; + + console.log("obj is", obj); + + return obj; + }); + } + + async fetchSearchTweets( + query: string, + maxTweets: number, + searchMode: SearchMode, + cursor?: string + ): Promise { + try { + // Sometimes this fails because we are rate limited. in this case, we just need to return an empty array + // if we dont get a response in 5 seconds, something is wrong + const timeoutPromise = new Promise((resolve) => + setTimeout(() => resolve({ tweets: [] }), 10000) + ); + + try { + const result = await this.requestQueue.add( + async () => + await Promise.race([ + this.twitterClient.fetchSearchTweets( + query, + maxTweets, + searchMode, + cursor + ), + timeoutPromise, + ]) + ); + return (result ?? { tweets: [] }) as QueryTweetsResponse; + } catch (error) { + console.error("Error fetching search tweets:", error); + return { tweets: [] }; + } + } catch (error) { + console.error("Error fetching search tweets:", error); + return { tweets: [] }; + } } - } - - private async populateTimeline() { - const cacheFile = "timeline_cache.json"; - - // Check if the cache file exists - if (fs.existsSync(cacheFile)) { - // Read the cached search results from the file - const cachedResults = JSON.parse(fs.readFileSync(cacheFile, "utf-8")); - - // Get the existing memories from the database - const existingMemories = await this.runtime.messageManager - .getMemoriesByRoomIds({ - roomIds: cachedResults.map((tweet) => - stringToUuid(tweet.conversationId) - ), - }); - // Create a Set to store the IDs of existing memories - const existingMemoryIds = new Set( - existingMemories.map((memory) => memory.id.toString()), - ); + private async populateTimeline() { + const cacheFile = "timeline_cache.json"; - // Check if any of the cached tweets exist in the existing memories - const someCachedTweetsExist = cachedResults.some((tweet) => - existingMemoryIds.has(tweet.id) - ); + // Check if the cache file exists + if (fs.existsSync(cacheFile)) { + // Read the cached search results from the file + const cachedResults = JSON.parse( + fs.readFileSync(cacheFile, "utf-8") + ); + + // Get the existing memories from the database + const existingMemories = + await this.runtime.messageManager.getMemoriesByRoomIds({ + roomIds: cachedResults.map((tweet) => + stringToUuid(tweet.conversationId) + ), + }); + + // Create a Set to store the IDs of existing memories + const existingMemoryIds = new Set( + existingMemories.map((memory) => memory.id.toString()) + ); + + // Check if any of the cached tweets exist in the existing memories + const someCachedTweetsExist = cachedResults.some((tweet) => + existingMemoryIds.has(tweet.id) + ); - if (someCachedTweetsExist) { - // Filter out the cached tweets that already exist in the database - const tweetsToSave = cachedResults.filter( - (tweet) => !existingMemoryIds.has(tweet.id), + if (someCachedTweetsExist) { + // Filter out the cached tweets that already exist in the database + const tweetsToSave = cachedResults.filter( + (tweet) => !existingMemoryIds.has(tweet.id) + ); + + // Save the missing tweets as memories + for (const tweet of tweetsToSave) { + const roomId = stringToUuid(tweet.conversationId); + const tweetuserId = + tweet.userId === this.twitterUserId + ? this.runtime.agentId + : stringToUuid(tweet.userId); + + await this.runtime.ensureRoomExists(roomId); + await this.runtime.ensureParticipantExists( + this.runtime.agentId, + roomId + ); + + await this.runtime.ensureUserExists( + tweetuserId, + tweet.username, + tweet.name, + "twitter" + ); + await this.runtime.ensureParticipantExists( + tweetuserId, + roomId + ); + + const content = { + text: tweet.text, + url: tweet.permanentUrl, + source: "twitter", + inReplyTo: tweet.inReplyToStatusId + ? stringToUuid(tweet.inReplyToStatusId) + : undefined, + } as Content; + + console.log("Creating memory for tweet", tweet.id); + + // check if it already exists + const memory = + await this.runtime.messageManager.getMemoryById( + stringToUuid(tweet.id) + ); + if (memory) { + console.log( + "Memory already exists, skipping timeline population" + ); + break; + } + + await this.runtime.messageManager.createMemory({ + id: stringToUuid(tweet.id), + userId: tweetuserId, + content: content, + roomId, + embedding: embeddingZeroVector, + createdAt: tweet.timestamp * 1000, + }); + } + + console.log( + `Populated ${tweetsToSave.length} missing tweets from the cache.` + ); + return; + } + } + + // Get the most recent 20 mentions and interactions + const mentionsAndInteractions = await this.fetchSearchTweets( + `@${this.runtime.getSetting("TWITTER_USERNAME")}`, + 20, + SearchMode.Latest ); - // Save the missing tweets as memories - for (const tweet of tweetsToSave) { - const roomId = stringToUuid(tweet.conversationId); - const tweetuserId = tweet.userId === this.twitterUserId - ? this.runtime.agentId - : stringToUuid(tweet.userId); + // Combine the timeline tweets and mentions/interactions + const allTweets = [...mentionsAndInteractions.tweets]; - await this.runtime.ensureRoomExists(roomId); - await this.runtime.ensureParticipantExists( - this.runtime.agentId, - roomId, - ); - - await this.runtime.ensureUserExists( - tweetuserId, - tweet.username, - tweet.name, - "twitter", - ); - await this.runtime.ensureParticipantExists(tweetuserId, roomId); - - const content = { - text: tweet.text, - url: tweet.permanentUrl, - source: "twitter", - inReplyTo: tweet.inReplyToStatusId - ? stringToUuid(tweet.inReplyToStatusId) - : undefined, - } as Content; - - console.log("Creating memory for tweet", tweet.id); - - // check if it already exists - const memory = await this.runtime.messageManager.getMemoryById( - stringToUuid(tweet.id), - ); - if (memory) { - console.log("Memory already exists, skipping timeline population"); - break; - } - - await this.runtime.messageManager.createMemory({ - id: stringToUuid(tweet.id), - userId: tweetuserId, - content: content, - roomId, - embedding: embeddingZeroVector, - createdAt: tweet.timestamp * 1000, - }); + // Create a Set to store unique tweet IDs + const tweetIdsToCheck = new Set(); + + // Add tweet IDs to the Set + for (const tweet of allTweets) { + tweetIdsToCheck.add(tweet.id); } - console.log( - `Populated ${tweetsToSave.length} missing tweets from the cache.`, + // Convert the Set to an array of UUIDs + const tweetUuids = Array.from(tweetIdsToCheck).map((id) => + stringToUuid(id) + ); + + // Check the existing memories in the database + const existingMemories = + await this.runtime.messageManager.getMemoriesByRoomIds({ + roomIds: tweetUuids, + }); + + // Create a Set to store the existing memory IDs + const existingMemoryIds = new Set( + existingMemories.map((memory) => memory.roomId) ); - return; - } - } - // Get the most recent 20 mentions and interactions - const mentionsAndInteractions = await this.fetchSearchTweets( - `@${this.runtime.getSetting("TWITTER_USERNAME")}`, - 20, - SearchMode.Latest, - ); + // Filter out the tweets that already exist in the database + const tweetsToSave = allTweets.filter( + (tweet) => !existingMemoryIds.has(stringToUuid(tweet.id)) + ); + + await this.runtime.ensureUserExists( + this.runtime.agentId, + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.character.name, + "twitter" + ); - // Combine the timeline tweets and mentions/interactions - const allTweets = [...mentionsAndInteractions.tweets]; + // Save the new tweets as memories + for (const tweet of tweetsToSave) { + const roomId = stringToUuid(tweet.conversationId); + const tweetuserId = + tweet.userId === this.twitterUserId + ? this.runtime.agentId + : stringToUuid(tweet.userId); + + await this.runtime.ensureRoomExists(roomId); + await this.runtime.ensureParticipantExists( + this.runtime.agentId, + roomId + ); - // Create a Set to store unique tweet IDs - const tweetIdsToCheck = new Set(); + await this.runtime.ensureUserExists( + tweetuserId, + tweet.username, + tweet.name, + "twitter" + ); + await this.runtime.ensureParticipantExists(tweetuserId, roomId); + + const content = { + text: tweet.text, + url: tweet.permanentUrl, + source: "twitter", + inReplyTo: tweet.inReplyToStatusId + ? stringToUuid(tweet.inReplyToStatusId) + : undefined, + } as Content; + + await this.runtime.messageManager.createMemory({ + id: stringToUuid(tweet.id), + userId: tweetuserId, + content: content, + roomId, + embedding: embeddingZeroVector, + createdAt: tweet.timestamp * 1000, + }); + } - // Add tweet IDs to the Set - for (const tweet of allTweets) { - tweetIdsToCheck.add(tweet.id); + // Cache the search results to the file + fs.writeFileSync(cacheFile, JSON.stringify(allTweets)); } - // Convert the Set to an array of UUIDs - const tweetUuids = Array.from(tweetIdsToCheck).map((id) => - stringToUuid(id) - ); - - // Check the existing memories in the database - const existingMemories = await this.runtime.messageManager - .getMemoriesByRoomIds({ - roomIds: tweetUuids, - }); - - // Create a Set to store the existing memory IDs - const existingMemoryIds = new Set( - existingMemories.map((memory) => memory.roomId), - ); - - // Filter out the tweets that already exist in the database - const tweetsToSave = allTweets.filter( - (tweet) => !existingMemoryIds.has(stringToUuid(tweet.id)), - ); - - await this.runtime.ensureUserExists( - this.runtime.agentId, - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.character.name, - "twitter", - ); - - // Save the new tweets as memories - for (const tweet of tweetsToSave) { - const roomId = stringToUuid(tweet.conversationId); - const tweetuserId = tweet.userId === this.twitterUserId - ? this.runtime.agentId - : stringToUuid(tweet.userId); - - await this.runtime.ensureRoomExists(roomId); - await this.runtime.ensureParticipantExists(this.runtime.agentId, roomId); - - await this.runtime.ensureUserExists( - tweetuserId, - tweet.username, - tweet.name, - "twitter", - ); - await this.runtime.ensureParticipantExists(tweetuserId, roomId); - - const content = { - text: tweet.text, - url: tweet.permanentUrl, - source: "twitter", - inReplyTo: tweet.inReplyToStatusId - ? stringToUuid(tweet.inReplyToStatusId) - : undefined, - } as Content; - - await this.runtime.messageManager.createMemory({ - id: stringToUuid(tweet.id), - userId: tweetuserId, - content: content, - roomId, - embedding: embeddingZeroVector, - createdAt: tweet.timestamp * 1000, - }); + async setCookiesFromArray(cookiesArray: any[]) { + const cookieStrings = cookiesArray.map( + (cookie) => + `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${ + cookie.secure ? "Secure" : "" + }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${ + cookie.sameSite || "Lax" + }` + ); + await this.twitterClient.setCookies(cookieStrings); } - // Cache the search results to the file - fs.writeFileSync(cacheFile, JSON.stringify(allTweets)); - } - - async setCookiesFromArray(cookiesArray: any[]) { - const cookieStrings = cookiesArray.map( - (cookie) => - `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${ - cookie.secure ? "Secure" : "" - }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${ - cookie.sameSite || "Lax" - }`, - ); - await this.twitterClient.setCookies(cookieStrings); - } - - async saveRequestMessage(message: Memory, state: State) { - if (message.content.text) { - const recentMessage = await this.runtime.messageManager.getMemories({ - roomId: message.roomId, - count: 1, - unique: false, - }); - - if ( - recentMessage.length > 0 && - recentMessage[0].content === message.content - ) { - console.log("Message already saved", recentMessage[0].id); - } else { - await this.runtime.messageManager.createMemory({ - ...message, - embedding: embeddingZeroVector, - }); - } + async saveRequestMessage(message: Memory, state: State) { + if (message.content.text) { + const recentMessage = await this.runtime.messageManager.getMemories( + { + roomId: message.roomId, + count: 1, + unique: false, + } + ); - await this.runtime.evaluate(message, { - ...state, - twitterClient: this.twitterClient, - }); + if ( + recentMessage.length > 0 && + recentMessage[0].content === message.content + ) { + console.log("Message already saved", recentMessage[0].id); + } else { + await this.runtime.messageManager.createMemory({ + ...message, + embedding: embeddingZeroVector, + }); + } + + await this.runtime.evaluate(message, { + ...state, + twitterClient: this.twitterClient, + }); + } } - } } diff --git a/core/src/clients/twitter/generate.ts b/core/src/clients/twitter/generate.ts index 4817886f5e..58efad4216 100644 --- a/core/src/clients/twitter/generate.ts +++ b/core/src/clients/twitter/generate.ts @@ -26,167 +26,169 @@ Write a single sentence post that is {{adjective}} about {{topic}} (without ment Your response should not contain any questions. Brief, concise statements only. No emojis. Use \\n\\n (double spaces) between statements.`; export class TwitterGenerationClient extends ClientBase { - onReady() { - const generateNewTweetLoop = () => { - this.generateNewTweet(); - setTimeout( - generateNewTweetLoop, - (Math.floor(Math.random() * (45 - 15 + 1)) + 15) * 60 * 1000, - ); // Random interval between 4-8 hours - }; - // setTimeout(() => { - generateNewTweetLoop(); - // }, 5 * 60 * 1000); // Wait 5 minutes before starting the loop - } - - constructor(runtime: IAgentRuntime) { - // Initialize the client and pass an optional callback to be called when the client is ready - super({ - runtime, - }); - } - - private async generateNewTweet() { - console.log("Generating new tweet"); - try { - await this.runtime.ensureUserExists( - this.runtime.agentId, - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.character.name, - "twitter", - ); - - let homeTimeline = []; - - if (!fs.existsSync("tweetcache")) fs.mkdirSync("tweetcache"); - // read the file if it exists - if (fs.existsSync("tweetcache/home_timeline.json")) { - homeTimeline = JSON.parse( - fs.readFileSync("tweetcache/home_timeline.json", "utf-8"), - ); - } else { - homeTimeline = await this.fetchHomeTimeline(50); - fs.writeFileSync( - "tweetcache/home_timeline.json", - JSON.stringify(homeTimeline, null, 2), - ); - } - - const formattedHomeTimeline = - `# ${this.runtime.character.name}'s Home Timeline\n\n` + - homeTimeline - .map((tweet) => { - return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; - }) - .join("\n"); - - const state = await this.runtime.composeState( - { - userId: this.runtime.agentId, - roomId: stringToUuid("twitter_generate_room"), - content: { text: "", action: "" }, - }, - { - twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"), - timeline: formattedHomeTimeline, - }, - ); - // Generate new tweet - const context = composeContext({ - state, - template: newTweetPrompt, - }); - - const datestr = new Date().toUTCString().replace(/:/g, "-"); - - // log context to file - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_generate_context`, - context, - ); - - const newTweetContent = await generateText({ - runtime: this.runtime, - context, - modelClass: ModelClass.LARGE, - }); - console.log("New Tweet:", newTweetContent); - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_generate_response`, - JSON.stringify(newTweetContent), - ); - - const slice = newTweetContent.replaceAll(/\\n/g, "\n").trim(); - - const content = slice - // .slice(0, 280); - // // if its bigger than 280, delete the last line - // if (content.length > 280) { - // content = content.slice(0, content.lastIndexOf("\n")); - // } - - // if(content.length < 1) { - // content = slice.slice(0, 280); - // } - - // Send the new tweet - if (!this.dryRun) { + onReady() { + const generateNewTweetLoop = () => { + this.generateNewTweet(); + setTimeout( + generateNewTweetLoop, + (Math.floor(Math.random() * (45 - 15 + 1)) + 15) * 60 * 1000 + ); // Random interval between 4-8 hours + }; + // setTimeout(() => { + generateNewTweetLoop(); + // }, 5 * 60 * 1000); // Wait 5 minutes before starting the loop + } + + constructor(runtime: IAgentRuntime) { + // Initialize the client and pass an optional callback to be called when the client is ready + super({ + runtime, + }); + } + + private async generateNewTweet() { + console.log("Generating new tweet"); try { - const result = await this.requestQueue.add( - async () => - await this.twitterClient.sendTweet(content), - ); - // read the body of the response - const body = await result.json(); - const tweetResult = body.data.create_tweet.tweet_results.result; - - const tweet = { - id: tweetResult.rest_id, - text: tweetResult.legacy.full_text, - conversationId: tweetResult.legacy.conversation_id_str, - createdAt: tweetResult.legacy.created_at, - userId: tweetResult.legacy.user_id_str, - inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str, - permanentUrl: `https://twitter.com/${this.runtime.getSetting("TWITTER_USERNAME")}/status/${tweetResult.rest_id}`, - hashtags: [], - mentions: [], - photos: [], - thread: [], - urls: [], - videos: [], - } as Tweet; - - const postId = tweet.id; - const conversationId = tweet.conversationId; - const roomId = stringToUuid(conversationId); - - await this.runtime.ensureRoomExists(roomId); - await this.runtime.ensureParticipantInRoom( - this.runtime.agentId, - roomId, - ); - await this.cacheTweet(tweet); - - await this.runtime.messageManager.createMemory({ - id: stringToUuid(postId), - userId: this.runtime.agentId, - content: { - text: newTweetContent.trim(), - url: tweet.permanentUrl, - source: "twitter", - }, - roomId, - embedding: embeddingZeroVector, - createdAt: tweet.timestamp * 1000, - }); + await this.runtime.ensureUserExists( + this.runtime.agentId, + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.character.name, + "twitter" + ); + + let homeTimeline = []; + + if (!fs.existsSync("tweetcache")) fs.mkdirSync("tweetcache"); + // read the file if it exists + if (fs.existsSync("tweetcache/home_timeline.json")) { + homeTimeline = JSON.parse( + fs.readFileSync("tweetcache/home_timeline.json", "utf-8") + ); + } else { + homeTimeline = await this.fetchHomeTimeline(50); + fs.writeFileSync( + "tweetcache/home_timeline.json", + JSON.stringify(homeTimeline, null, 2) + ); + } + + const formattedHomeTimeline = + `# ${this.runtime.character.name}'s Home Timeline\n\n` + + homeTimeline + .map((tweet) => { + return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; + }) + .join("\n"); + + const state = await this.runtime.composeState( + { + userId: this.runtime.agentId, + roomId: stringToUuid("twitter_generate_room"), + content: { text: "", action: "" }, + }, + { + twitterUserName: + this.runtime.getSetting("TWITTER_USERNAME"), + timeline: formattedHomeTimeline, + } + ); + // Generate new tweet + const context = composeContext({ + state, + template: newTweetPrompt, + }); + + const datestr = new Date().toUTCString().replace(/:/g, "-"); + + // log context to file + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_generate_context`, + context + ); + + const newTweetContent = await generateText({ + runtime: this.runtime, + context, + modelClass: ModelClass.LARGE, + }); + console.log("New Tweet:", newTweetContent); + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_generate_response`, + JSON.stringify(newTweetContent) + ); + + const slice = newTweetContent.replaceAll(/\\n/g, "\n").trim(); + + const content = slice; + // .slice(0, 280); + // // if its bigger than 280, delete the last line + // if (content.length > 280) { + // content = content.slice(0, content.lastIndexOf("\n")); + // } + + // if(content.length < 1) { + // content = slice.slice(0, 280); + // } + + // Send the new tweet + if (!this.dryRun) { + try { + const result = await this.requestQueue.add( + async () => await this.twitterClient.sendTweet(content) + ); + // read the body of the response + const body = await result.json(); + const tweetResult = + body.data.create_tweet.tweet_results.result; + + const tweet = { + id: tweetResult.rest_id, + text: tweetResult.legacy.full_text, + conversationId: tweetResult.legacy.conversation_id_str, + createdAt: tweetResult.legacy.created_at, + userId: tweetResult.legacy.user_id_str, + inReplyToStatusId: + tweetResult.legacy.in_reply_to_status_id_str, + permanentUrl: `https://twitter.com/${this.runtime.getSetting("TWITTER_USERNAME")}/status/${tweetResult.rest_id}`, + hashtags: [], + mentions: [], + photos: [], + thread: [], + urls: [], + videos: [], + } as Tweet; + + const postId = tweet.id; + const conversationId = tweet.conversationId; + const roomId = stringToUuid(conversationId); + + await this.runtime.ensureRoomExists(roomId); + await this.runtime.ensureParticipantInRoom( + this.runtime.agentId, + roomId + ); + await this.cacheTweet(tweet); + + await this.runtime.messageManager.createMemory({ + id: stringToUuid(postId), + userId: this.runtime.agentId, + content: { + text: newTweetContent.trim(), + url: tweet.permanentUrl, + source: "twitter", + }, + roomId, + embedding: embeddingZeroVector, + createdAt: tweet.timestamp * 1000, + }); + } catch (error) { + console.error("Error sending tweet:", error); + } + } else { + console.log("Dry run, not sending tweet:", newTweetContent); + } } catch (error) { - console.error("Error sending tweet:", error); + console.error("Error generating new tweet:", error); } - } else { - console.log("Dry run, not sending tweet:", newTweetContent); - } - } catch (error) { - console.error("Error generating new tweet:", error); } - } } diff --git a/core/src/clients/twitter/interactions.ts b/core/src/clients/twitter/interactions.ts index 38b8d24784..5b93645ef2 100644 --- a/core/src/clients/twitter/interactions.ts +++ b/core/src/clients/twitter/interactions.ts @@ -3,24 +3,27 @@ import fs from "fs"; import { composeContext } from "../../core/context.ts"; import { log_to_file } from "../../core/logger.ts"; import { - messageCompletionFooter, - shouldRespondFooter, + messageCompletionFooter, + shouldRespondFooter, } from "../../core/parsing.ts"; import { - Content, - HandlerCallback, - IAgentRuntime, - Memory, - ModelClass, - State, + Content, + HandlerCallback, + IAgentRuntime, + Memory, + ModelClass, + State, } from "../../core/types.ts"; import { stringToUuid } from "../../core/uuid.ts"; import { ClientBase } from "./base.ts"; import { buildConversationThread, sendTweetChunks, wait } from "./utils.ts"; -import { generateMessageResponse, generateShouldRespond } from "../../core/generation.ts"; +import { + generateMessageResponse, + generateShouldRespond, +} from "../../core/generation.ts"; export const messageHandlerTemplate = - `{{relevantFacts}} + `{{relevantFacts}} {{recentFacts}} {{timeline}} @@ -48,7 +51,7 @@ Recent interactions between {{agentName}} and other users: ` + messageCompletionFooter; export const shouldRespondTemplate = - `# INSTRUCTIONS: Determine if {{agentName}} (@{{twitterUserName}}) should respond to the message and participate in the conversation. Do not comment. Just respond with "true" or "false". + `# INSTRUCTIONS: Determine if {{agentName}} (@{{twitterUserName}}) should respond to the message and participate in the conversation. Do not comment. Just respond with "true" or "false". Response options are RESPOND, IGNORE and STOP. @@ -71,299 +74,315 @@ IMPORTANT: {{agentName}} (aka @{{twitterUserName}}) is particularly sensitive ab ` + shouldRespondFooter; export class TwitterInteractionClient extends ClientBase { - onReady() { - const handleTwitterInteractionsLoop = () => { - this.handleTwitterInteractions(); - setTimeout( - handleTwitterInteractionsLoop, - (Math.floor(Math.random() * (5 - 2 + 1)) + 2) * 60 * 1000, - ); // Random interval between 2-5 minutes - }; - handleTwitterInteractionsLoop(); - } - - constructor(runtime: IAgentRuntime) { - super({ - runtime, - }); - } - - async handleTwitterInteractions() { - console.log("Checking Twitter interactions"); - try { - // Check for mentions - const tweetCandidates = ( - await this.fetchSearchTweets( - `@${this.runtime.getSetting("TWITTER_USERNAME")}`, - 20, - SearchMode.Latest, - ) - ).tweets; - - // de-duplicate tweetCandidates with a set - const uniqueTweetCandidates = [...new Set(tweetCandidates)]; - - // Sort tweet candidates by ID in ascending order - uniqueTweetCandidates - .sort((a, b) => a.id.localeCompare(b.id)) - .filter((tweet) => tweet.userId !== this.twitterUserId); - - // for each tweet candidate, handle the tweet - for (const tweet of uniqueTweetCandidates) { - if ( - !this.lastCheckedTweetId || - parseInt(tweet.id) > this.lastCheckedTweetId - ) { - console.log("Processing tweet", tweet.id); - const conversationId = tweet.conversationId; - - const roomId = stringToUuid(conversationId); - await this.runtime.ensureRoomExists(roomId); - - const userIdUUID = stringToUuid(tweet.userId as string); - const agentId = this.runtime.agentId; - - await Promise.all([ - this.runtime.ensureUserExists( - agentId, - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.character.name, - "twitter", - ), - this.runtime.ensureUserExists( - userIdUUID, - tweet.username, - tweet.name, - "twitter", - ), - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(agentId, roomId), - ]); - - await buildConversationThread(tweet, this); - - const message = { - content: { text: tweet.text }, - userId: userIdUUID, - roomId, - }; - - await this.handleTweet({ - tweet, - message, - }); - - // Update the last checked tweet ID after processing each tweet - this.lastCheckedTweetId = parseInt(tweet.id); - - try { - fs.writeFileSync( - this.tweetCacheFilePath, - this.lastCheckedTweetId.toString(), - "utf-8", - ); - } catch (error) { - console.error( - "Error saving latest checked tweet ID to file:", - error, - ); - } - } - } - - // Save the latest checked tweet ID to the file - try { - fs.writeFileSync( - this.tweetCacheFilePath, - this.lastCheckedTweetId.toString(), - "utf-8", - ); - } catch (error) { - console.error("Error saving latest checked tweet ID to file:", error); - } - - console.log("Finished checking Twitter interactions"); - } catch (error) { - console.error("Error handling Twitter interactions:", error); - } - } - - private async handleTweet({ - tweet, - message, - }: { - tweet: Tweet; - message: Memory; - }) { - if (tweet.username === this.runtime.getSetting("TWITTER_USERNAME")) { - console.log("skipping tweet from bot itself", tweet.id); - // Skip processing if the tweet is from the bot itself - return; + onReady() { + const handleTwitterInteractionsLoop = () => { + this.handleTwitterInteractions(); + setTimeout( + handleTwitterInteractionsLoop, + (Math.floor(Math.random() * (5 - 2 + 1)) + 2) * 60 * 1000 + ); // Random interval between 2-5 minutes + }; + handleTwitterInteractionsLoop(); } - if (!message.content.text) { - console.log("skipping tweet with no text", tweet.id); - return { text: "", action: "IGNORE" }; - } - console.log("handling tweet", tweet.id); - const formatTweet = (tweet: Tweet) => { - return ` ID: ${tweet.id} - From: ${tweet.name} (@${tweet.username}) - Text: ${tweet.text}`; - }; - const currentPost = formatTweet(tweet); - - let homeTimeline = []; - // read the file if it exists - if (fs.existsSync("tweetcache/home_timeline.json")) { - homeTimeline = JSON.parse( - fs.readFileSync("tweetcache/home_timeline.json", "utf-8"), - ); - } else { - homeTimeline = await this.fetchHomeTimeline(50); - fs.writeFileSync( - "tweetcache/home_timeline.json", - JSON.stringify(homeTimeline, null, 2), - ); + constructor(runtime: IAgentRuntime) { + super({ + runtime, + }); } - const formattedHomeTimeline = - `# ${this.runtime.character.name}'s Home Timeline\n\n` + - homeTimeline - .map((tweet) => { - return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; - }) - .join("\n"); - - let state = await this.runtime.composeState(message, { - twitterClient: this.twitterClient, - twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"), - currentPost, - timeline: formattedHomeTimeline, - }); - - // check if the tweet exists, save if it doesn't - const tweetId = stringToUuid(tweet.id); - const tweetExists = - await this.runtime.messageManager.getMemoryById(tweetId); - - if (!tweetExists) { - console.log("tweet does not exist, saving"); - const userIdUUID = stringToUuid(tweet.userId as string); - const roomId = stringToUuid(tweet.conversationId); - - const message = { - id: tweetId, - content: { - text: tweet.text, - url: tweet.permanentUrl, - inReplyTo: tweet.inReplyToStatusId - ? stringToUuid(tweet.inReplyToStatusId) - : undefined, - }, - userId: userIdUUID, - roomId, - createdAt: tweet.timestamp * 1000, - }; - this.saveRequestMessage(message, state); + async handleTwitterInteractions() { + console.log("Checking Twitter interactions"); + try { + // Check for mentions + const tweetCandidates = ( + await this.fetchSearchTweets( + `@${this.runtime.getSetting("TWITTER_USERNAME")}`, + 20, + SearchMode.Latest + ) + ).tweets; + + // de-duplicate tweetCandidates with a set + const uniqueTweetCandidates = [...new Set(tweetCandidates)]; + + // Sort tweet candidates by ID in ascending order + uniqueTweetCandidates + .sort((a, b) => a.id.localeCompare(b.id)) + .filter((tweet) => tweet.userId !== this.twitterUserId); + + // for each tweet candidate, handle the tweet + for (const tweet of uniqueTweetCandidates) { + if ( + !this.lastCheckedTweetId || + parseInt(tweet.id) > this.lastCheckedTweetId + ) { + console.log("Processing tweet", tweet.id); + const conversationId = tweet.conversationId; + + const roomId = stringToUuid(conversationId); + await this.runtime.ensureRoomExists(roomId); + + const userIdUUID = stringToUuid(tweet.userId as string); + const agentId = this.runtime.agentId; + + await Promise.all([ + this.runtime.ensureUserExists( + agentId, + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.character.name, + "twitter" + ), + this.runtime.ensureUserExists( + userIdUUID, + tweet.username, + tweet.name, + "twitter" + ), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom( + userIdUUID, + roomId + ), + this.runtime.ensureParticipantInRoom(agentId, roomId), + ]); + + await buildConversationThread(tweet, this); + + const message = { + content: { text: tweet.text }, + userId: userIdUUID, + roomId, + }; + + await this.handleTweet({ + tweet, + message, + }); + + // Update the last checked tweet ID after processing each tweet + this.lastCheckedTweetId = parseInt(tweet.id); + + try { + fs.writeFileSync( + this.tweetCacheFilePath, + this.lastCheckedTweetId.toString(), + "utf-8" + ); + } catch (error) { + console.error( + "Error saving latest checked tweet ID to file:", + error + ); + } + } + } + + // Save the latest checked tweet ID to the file + try { + fs.writeFileSync( + this.tweetCacheFilePath, + this.lastCheckedTweetId.toString(), + "utf-8" + ); + } catch (error) { + console.error( + "Error saving latest checked tweet ID to file:", + error + ); + } + + console.log("Finished checking Twitter interactions"); + } catch (error) { + console.error("Error handling Twitter interactions:", error); + } } - console.log("composeState done"); - - const shouldRespondContext = composeContext({ - state, - template: shouldRespondTemplate, - }); - - const shouldRespond = await generateShouldRespond({ - runtime: this.runtime, - context: shouldRespondContext, - modelClass: ModelClass.SMALL, - }); + private async handleTweet({ + tweet, + message, + }: { + tweet: Tweet; + message: Memory; + }) { + if (tweet.username === this.runtime.getSetting("TWITTER_USERNAME")) { + console.log("skipping tweet from bot itself", tweet.id); + // Skip processing if the tweet is from the bot itself + return; + } - if (!shouldRespond) { - console.log("Not responding to message"); - return { text: "", action: "IGNORE" }; - } + if (!message.content.text) { + console.log("skipping tweet with no text", tweet.id); + return { text: "", action: "IGNORE" }; + } + console.log("handling tweet", tweet.id); + const formatTweet = (tweet: Tweet) => { + return ` ID: ${tweet.id} + From: ${tweet.name} (@${tweet.username}) + Text: ${tweet.text}`; + }; + const currentPost = formatTweet(tweet); + + let homeTimeline = []; + // read the file if it exists + if (fs.existsSync("tweetcache/home_timeline.json")) { + homeTimeline = JSON.parse( + fs.readFileSync("tweetcache/home_timeline.json", "utf-8") + ); + } else { + homeTimeline = await this.fetchHomeTimeline(50); + fs.writeFileSync( + "tweetcache/home_timeline.json", + JSON.stringify(homeTimeline, null, 2) + ); + } - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); + const formattedHomeTimeline = + `# ${this.runtime.character.name}'s Home Timeline\n\n` + + homeTimeline + .map((tweet) => { + return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; + }) + .join("\n"); + + let state = await this.runtime.composeState(message, { + twitterClient: this.twitterClient, + twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"), + currentPost, + timeline: formattedHomeTimeline, + }); + + // check if the tweet exists, save if it doesn't + const tweetId = stringToUuid(tweet.id); + const tweetExists = + await this.runtime.messageManager.getMemoryById(tweetId); + + if (!tweetExists) { + console.log("tweet does not exist, saving"); + const userIdUUID = stringToUuid(tweet.userId as string); + const roomId = stringToUuid(tweet.conversationId); + + const message = { + id: tweetId, + content: { + text: tweet.text, + url: tweet.permanentUrl, + inReplyTo: tweet.inReplyToStatusId + ? stringToUuid(tweet.inReplyToStatusId) + : undefined, + }, + userId: userIdUUID, + roomId, + createdAt: tweet.timestamp * 1000, + }; + this.saveRequestMessage(message, state); + } - const datestr = new Date().toUTCString().replace(/:/g, "-"); + console.log("composeState done"); - // log context to file - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_interactions_context`, - context, - ); + const shouldRespondContext = composeContext({ + state, + template: shouldRespondTemplate, + }); - const response = await generateMessageResponse({ - runtime: this.runtime, - context, - modelClass: "slow" - }); + const shouldRespond = await generateShouldRespond({ + runtime: this.runtime, + context: shouldRespondContext, + modelClass: ModelClass.SMALL, + }); - console.log("response", response); + if (!shouldRespond) { + console.log("Not responding to message"); + return { text: "", action: "IGNORE" }; + } - console.log("tweet is", tweet); + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); - const stringId = stringToUuid(tweet.id); + const datestr = new Date().toUTCString().replace(/:/g, "-"); - console.log("stringId is", stringId, "while tweet.id is", tweet.id); + // log context to file + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_interactions_context`, + context + ); - response.inReplyTo = stringId; + const response = await generateMessageResponse({ + runtime: this.runtime, + context, + modelClass: "slow", + }); - console.log("response is", response); + console.log("response", response); - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_interactions_response`, - JSON.stringify(response), - ); + console.log("tweet is", tweet); - if (response.text) { - try { - if (!this.dryRun) { - const callback: HandlerCallback = async (response: Content) => { - const memories = await sendTweetChunks( - this, - response, - message.roomId, - this.runtime.getSetting("TWITTER_USERNAME"), - tweet.id, - ); - return memories; - }; + const stringId = stringToUuid(tweet.id); - const responseMessages = await callback(response); + console.log("stringId is", stringId, "while tweet.id is", tweet.id); - state = (await this.runtime.updateRecentMessageState(state)) as State; + response.inReplyTo = stringId; - for (const responseMessage of responseMessages) { - await this.runtime.messageManager.createMemory(responseMessage); - } + console.log("response is", response); - await this.runtime.evaluate(message, state); + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_interactions_response`, + JSON.stringify(response) + ); - await this.runtime.processActions(message, responseMessages, state); - } else { - console.log("Dry run, not sending tweet:", response.text); - } - const responseInfo = `Context:\n\n${context}\n\nSelected Post: ${tweet.id} - ${tweet.username}: ${tweet.text}\nAgent's Output:\n${response.text}`; - // f tweets folder dont exist, create - if (!fs.existsSync("tweets")) { - fs.mkdirSync("tweets"); + if (response.text) { + try { + if (!this.dryRun) { + const callback: HandlerCallback = async ( + response: Content + ) => { + const memories = await sendTweetChunks( + this, + response, + message.roomId, + this.runtime.getSetting("TWITTER_USERNAME"), + tweet.id + ); + return memories; + }; + + const responseMessages = await callback(response); + + state = (await this.runtime.updateRecentMessageState( + state + )) as State; + + for (const responseMessage of responseMessages) { + await this.runtime.messageManager.createMemory( + responseMessage + ); + } + + await this.runtime.evaluate(message, state); + + await this.runtime.processActions( + message, + responseMessages, + state + ); + } else { + console.log("Dry run, not sending tweet:", response.text); + } + const responseInfo = `Context:\n\n${context}\n\nSelected Post: ${tweet.id} - ${tweet.username}: ${tweet.text}\nAgent's Output:\n${response.text}`; + // f tweets folder dont exist, create + if (!fs.existsSync("tweets")) { + fs.mkdirSync("tweets"); + } + const debugFileName = `tweets/tweet_generation_${tweet.id}.txt`; + fs.writeFileSync(debugFileName, responseInfo); + await wait(); + } catch (error) { + console.error(`Error sending response tweet: ${error}`); + } } - const debugFileName = `tweets/tweet_generation_${tweet.id}.txt`; - fs.writeFileSync(debugFileName, responseInfo); - await wait(); - } catch (error) { - console.error(`Error sending response tweet: ${error}`); - } } - } } diff --git a/core/src/clients/twitter/search.ts b/core/src/clients/twitter/search.ts index ec29f2de0f..6c4a8c66a8 100644 --- a/core/src/clients/twitter/search.ts +++ b/core/src/clients/twitter/search.ts @@ -4,22 +4,21 @@ import { addHeader, composeContext } from "../../core/context.ts"; import { log_to_file } from "../../core/logger.ts"; import { messageCompletionFooter } from "../../core/parsing.ts"; import { - Content, - HandlerCallback, - IAgentRuntime, - State + Content, + HandlerCallback, + IAgentRuntime, + State, } from "../../core/types.ts"; import { stringToUuid } from "../../core/uuid.ts"; import { ClientBase } from "./base.ts"; +import { buildConversationThread, sendTweetChunks, wait } from "./utils.ts"; import { - buildConversationThread, - sendTweetChunks, - wait -} from "./utils.ts"; -import { generateText, generateMessageResponse } from "../../core/generation.ts"; + generateText, + generateMessageResponse, +} from "../../core/generation.ts"; const messageHandlerTemplate = - `{{relevantFacts}} + `{{relevantFacts}} {{recentFacts}} {{timeline}} @@ -46,91 +45,94 @@ Your response should not contain any questions. Brief, concise statements only. ` + messageCompletionFooter; export class TwitterSearchClient extends ClientBase { - private respondedTweets: Set = new Set(); - - constructor(runtime: IAgentRuntime) { - // Initialize the client and pass an optional callback to be called when the client is ready - super({ - runtime - }); - } - - async onReady() { - this.engageWithSearchTermsLoop(); - } - - private engageWithSearchTermsLoop() { - this.engageWithSearchTerms(); - setTimeout( - () => this.engageWithSearchTermsLoop(), - (Math.floor(Math.random() * (120 - 60 + 1)) + 60) * 60 * 1000 - ); - } - - private async engageWithSearchTerms() { - console.log("Engaging with search terms"); - try { - const searchTerm = [...this.runtime.character.topics][ - Math.floor(Math.random() * this.runtime.character.topics.length) - ]; - - if (!fs.existsSync("tweetcache")) { - fs.mkdirSync("tweetcache"); - } - console.log("Fetching search tweets"); - // TODO: we wait 5 seconds here to avoid getting rate limited on startup, but we should queue - await new Promise((resolve) => setTimeout(resolve, 5000)); - const recentTweets = await this.fetchSearchTweets( - searchTerm, - 20, - SearchMode.Top - ); - console.log("Search tweets fetched"); - - const homeTimeline = await this.fetchHomeTimeline(50); - fs.writeFileSync( - "tweetcache/home_timeline.json", - JSON.stringify(homeTimeline, null, 2) - ); - - const formattedHomeTimeline = - `# ${this.runtime.character.name}'s Home Timeline\n\n` + - homeTimeline - .map((tweet) => { - return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; - }) - .join("\n"); - - // randomly slice .tweets down to 20 - const slicedTweets = recentTweets.tweets - .sort(() => Math.random() - 0.5) - .slice(0, 20); - - if (slicedTweets.length === 0) { - console.log("No valid tweets found for the search term", searchTerm); - return; - } - - const prompt = ` + private respondedTweets: Set = new Set(); + + constructor(runtime: IAgentRuntime) { + // Initialize the client and pass an optional callback to be called when the client is ready + super({ + runtime, + }); + } + + async onReady() { + this.engageWithSearchTermsLoop(); + } + + private engageWithSearchTermsLoop() { + this.engageWithSearchTerms(); + setTimeout( + () => this.engageWithSearchTermsLoop(), + (Math.floor(Math.random() * (120 - 60 + 1)) + 60) * 60 * 1000 + ); + } + + private async engageWithSearchTerms() { + console.log("Engaging with search terms"); + try { + const searchTerm = [...this.runtime.character.topics][ + Math.floor(Math.random() * this.runtime.character.topics.length) + ]; + + if (!fs.existsSync("tweetcache")) { + fs.mkdirSync("tweetcache"); + } + console.log("Fetching search tweets"); + // TODO: we wait 5 seconds here to avoid getting rate limited on startup, but we should queue + await new Promise((resolve) => setTimeout(resolve, 5000)); + const recentTweets = await this.fetchSearchTweets( + searchTerm, + 20, + SearchMode.Top + ); + console.log("Search tweets fetched"); + + const homeTimeline = await this.fetchHomeTimeline(50); + fs.writeFileSync( + "tweetcache/home_timeline.json", + JSON.stringify(homeTimeline, null, 2) + ); + + const formattedHomeTimeline = + `# ${this.runtime.character.name}'s Home Timeline\n\n` + + homeTimeline + .map((tweet) => { + return `ID: ${tweet.id}\nFrom: ${tweet.name} (@${tweet.username})${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""}\nText: ${tweet.text}\n---\n`; + }) + .join("\n"); + + // randomly slice .tweets down to 20 + const slicedTweets = recentTweets.tweets + .sort(() => Math.random() - 0.5) + .slice(0, 20); + + if (slicedTweets.length === 0) { + console.log( + "No valid tweets found for the search term", + searchTerm + ); + return; + } + + const prompt = ` Here are some tweets related to the search term "${searchTerm}": ${[...slicedTweets, ...homeTimeline] - .filter((tweet) => { - // ignore tweets where any of the thread tweets contain a tweet by the bot - const thread = tweet.thread; - const botTweet = thread.find( - (t) => t.username === this.runtime.getSetting("TWITTER_USERNAME"), - ); - return !botTweet; - }) - .map( - (tweet) => ` + .filter((tweet) => { + // ignore tweets where any of the thread tweets contain a tweet by the bot + const thread = tweet.thread; + const botTweet = thread.find( + (t) => t.username === this.runtime.getSetting("TWITTER_USERNAME") + ); + return !botTweet; + }) + .map( + (tweet) => ` ID: ${tweet.id}${tweet.inReplyToStatusId ? ` In reply to: ${tweet.inReplyToStatusId}` : ""} From: ${tweet.name} (@${tweet.username}) Text: ${tweet.text} - `, - ) - .join("\n")} + ` + ) + .join("\n")} Which tweet is the most interesting and relevant for Ruby to reply to? Please provide only the ID of the tweet in your response. Notes: @@ -140,208 +142,215 @@ export class TwitterSearchClient extends ClientBase { - Respond to tweets where there is an easy exchange of ideas to have with the user - ONLY respond with the ID of the tweet`; - const datestr = new Date().toUTCString().replace(/:/g, "-"); - const logName = `${this.runtime.character.name}_search_${datestr}`; - log_to_file(logName, prompt); - - const mostInterestingTweetResponse = await generateText({ - runtime: this.runtime, - context: prompt, - modelClass: "slow" - }); - - const responseLogName = `${this.runtime.character.name}_search_${datestr}_result`; - log_to_file(responseLogName, mostInterestingTweetResponse); - - const tweetId = mostInterestingTweetResponse.trim(); - const selectedTweet = slicedTweets.find( - (tweet) => - tweet.id.toString().includes(tweetId) || - tweetId.includes(tweet.id.toString()) - ); - - if (!selectedTweet) { - console.log("No matching tweet found for the selected ID"); - return console.log("Selected tweet ID:", tweetId); - } - - console.log("Selected tweet to reply to:", selectedTweet?.text); - - if ( - selectedTweet.username === this.runtime.getSetting("TWITTER_USERNAME") - ) { - console.log("Skipping tweet from bot itself"); - return; - } - - const conversationId = selectedTweet.conversationId; - const roomId = stringToUuid(conversationId); - await this.runtime.ensureRoomExists(roomId); - - const userIdUUID = stringToUuid(selectedTweet.userId as string); - await Promise.all([ - this.runtime.ensureUserExists( - this.runtime.agentId, - this.runtime.getSetting("TWITTER_USERNAME"), - this.runtime.character.name, - "twitter" - ), - this.runtime.ensureUserExists( - userIdUUID, - selectedTweet.username, - selectedTweet.name, - "twitter" - ) - ]); - - await Promise.all([ - this.runtime.ensureParticipantInRoom(userIdUUID, roomId), - this.runtime.ensureParticipantInRoom(this.runtime.agentId, roomId) - ]); - - // crawl additional conversation tweets, if there are any - await buildConversationThread(selectedTweet, this); - - const message = { - id: stringToUuid(selectedTweet.id), - content: { - text: selectedTweet.text, - url: selectedTweet.permanentUrl, - inReplyTo: selectedTweet.inReplyToStatusId - ? stringToUuid(selectedTweet.inReplyToStatusId) - : undefined - }, - userId: userIdUUID, - roomId, - // Timestamps are in seconds, but we need them in milliseconds - createdAt: selectedTweet.timestamp * 1000 - }; - - if (!message.content.text) { - return { text: "", action: "IGNORE" }; - } - - // Fetch replies and retweets - const replies = selectedTweet.thread; - const replyContext = replies - .filter( - (reply) => - reply.username !== this.runtime.getSetting("TWITTER_USERNAME") - ) - .map((reply) => `@${reply.username}: ${reply.text}`) - .join("\n"); - - let tweetBackground = ""; - if (selectedTweet.isRetweet) { - const originalTweet = await this.requestQueue.add(() => - this.twitterClient.getTweet(selectedTweet.id) - ); - tweetBackground = `Retweeting @${originalTweet.username}: ${originalTweet.text}`; - } - - // Generate image descriptions using GPT-4 vision API - const imageDescriptions = []; - for (const photo of selectedTweet.photos) { - const description = - await this.runtime.imageDescriptionService.describeImage(photo.url); - imageDescriptions.push(description); - } - - let state = await this.runtime.composeState(message, { - twitterClient: this.twitterClient, - twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"), - timeline: formattedHomeTimeline, - tweetContext: `${tweetBackground} + const datestr = new Date().toUTCString().replace(/:/g, "-"); + const logName = `${this.runtime.character.name}_search_${datestr}`; + log_to_file(logName, prompt); + + const mostInterestingTweetResponse = await generateText({ + runtime: this.runtime, + context: prompt, + modelClass: "slow", + }); + + const responseLogName = `${this.runtime.character.name}_search_${datestr}_result`; + log_to_file(responseLogName, mostInterestingTweetResponse); + + const tweetId = mostInterestingTweetResponse.trim(); + const selectedTweet = slicedTweets.find( + (tweet) => + tweet.id.toString().includes(tweetId) || + tweetId.includes(tweet.id.toString()) + ); + + if (!selectedTweet) { + console.log("No matching tweet found for the selected ID"); + return console.log("Selected tweet ID:", tweetId); + } + + console.log("Selected tweet to reply to:", selectedTweet?.text); + + if ( + selectedTweet.username === + this.runtime.getSetting("TWITTER_USERNAME") + ) { + console.log("Skipping tweet from bot itself"); + return; + } + + const conversationId = selectedTweet.conversationId; + const roomId = stringToUuid(conversationId); + await this.runtime.ensureRoomExists(roomId); + + const userIdUUID = stringToUuid(selectedTweet.userId as string); + await Promise.all([ + this.runtime.ensureUserExists( + this.runtime.agentId, + this.runtime.getSetting("TWITTER_USERNAME"), + this.runtime.character.name, + "twitter" + ), + this.runtime.ensureUserExists( + userIdUUID, + selectedTweet.username, + selectedTweet.name, + "twitter" + ), + ]); + + await Promise.all([ + this.runtime.ensureParticipantInRoom(userIdUUID, roomId), + this.runtime.ensureParticipantInRoom( + this.runtime.agentId, + roomId + ), + ]); + + // crawl additional conversation tweets, if there are any + await buildConversationThread(selectedTweet, this); + + const message = { + id: stringToUuid(selectedTweet.id), + content: { + text: selectedTweet.text, + url: selectedTweet.permanentUrl, + inReplyTo: selectedTweet.inReplyToStatusId + ? stringToUuid(selectedTweet.inReplyToStatusId) + : undefined, + }, + userId: userIdUUID, + roomId, + // Timestamps are in seconds, but we need them in milliseconds + createdAt: selectedTweet.timestamp * 1000, + }; + + if (!message.content.text) { + return { text: "", action: "IGNORE" }; + } + + // Fetch replies and retweets + const replies = selectedTweet.thread; + const replyContext = replies + .filter( + (reply) => + reply.username !== + this.runtime.getSetting("TWITTER_USERNAME") + ) + .map((reply) => `@${reply.username}: ${reply.text}`) + .join("\n"); + + let tweetBackground = ""; + if (selectedTweet.isRetweet) { + const originalTweet = await this.requestQueue.add(() => + this.twitterClient.getTweet(selectedTweet.id) + ); + tweetBackground = `Retweeting @${originalTweet.username}: ${originalTweet.text}`; + } + + // Generate image descriptions using GPT-4 vision API + const imageDescriptions = []; + for (const photo of selectedTweet.photos) { + const description = + await this.runtime.imageDescriptionService.describeImage( + photo.url + ); + imageDescriptions.push(description); + } + + let state = await this.runtime.composeState(message, { + twitterClient: this.twitterClient, + twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"), + timeline: formattedHomeTimeline, + tweetContext: `${tweetBackground} Original Post: By @${selectedTweet.username} ${selectedTweet.text}${replyContext.length > 0 && `\nReplies to original post:\n${replyContext}`} ${`Original post text: ${selectedTweet.text}`} ${selectedTweet.urls.length > 0 ? `URLs: ${selectedTweet.urls.join(", ")}\n` : ""}${imageDescriptions.length > 0 ? `\nImages in Post (Described): ${imageDescriptions.join(", ")}\n` : ""} - ` - }); - - await this.saveRequestMessage(message, state as State); - - const context = composeContext({ - state, - template: messageHandlerTemplate - }); - - // log context to file - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_search_context`, - context - ); - - const responseContent = await generateMessageResponse({ - runtime: this.runtime, - context, - modelClass: "slow" - }); - - responseContent.inReplyTo = message.id; - - log_to_file( - `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_search_response`, - JSON.stringify(responseContent) - ); - - const response = responseContent; - - if (!response.text) { - console.log("Returning: No response text found"); - return; - } - - console.log( - `Bot would respond to tweet ${selectedTweet.id} with: ${response.text}` - ); - try { - const callback: HandlerCallback = async (response: Content) => { - const memories = await sendTweetChunks( - this, - response, - message.roomId, - this.runtime.getSetting("TWITTER_USERNAME"), - tweetId, - ); - return memories; - }; + `, + }); - const responseMessages = await callback(responseContent); + await this.saveRequestMessage(message, state as State); - state = await this.runtime.updateRecentMessageState(state); + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); - for (const responseMessage of responseMessages) { - await this.runtime.messageManager.createMemory( - responseMessage, - false, - ); - } + // log context to file + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_search_context`, + context + ); - state = await this.runtime.updateRecentMessageState(state); + const responseContent = await generateMessageResponse({ + runtime: this.runtime, + context, + modelClass: "slow", + }); - await this.runtime.evaluate(message, state); + responseContent.inReplyTo = message.id; - await this.runtime.processActions( - message, - responseMessages, - state, - callback, - ); + log_to_file( + `${this.runtime.getSetting("TWITTER_USERNAME")}_${datestr}_search_response`, + JSON.stringify(responseContent) + ); - this.respondedTweets.add(selectedTweet.id); - const responseInfo = `Context:\n\n${context}\n\nSelected Post: ${selectedTweet.id} - ${selectedTweet.username}: ${selectedTweet.text}\nAgent's Output:\n${response.text}`; - const debugFileName = `tweetcache/tweet_generation_${selectedTweet.id}.txt`; - - fs.writeFileSync(debugFileName, responseInfo); - await wait(); - } catch (error) { - console.error(`Error sending response post: ${error}`); - } - } catch (error) { - console.error("Error engaging with search terms:", error); + const response = responseContent; + + if (!response.text) { + console.log("Returning: No response text found"); + return; + } + + console.log( + `Bot would respond to tweet ${selectedTweet.id} with: ${response.text}` + ); + try { + const callback: HandlerCallback = async (response: Content) => { + const memories = await sendTweetChunks( + this, + response, + message.roomId, + this.runtime.getSetting("TWITTER_USERNAME"), + tweetId + ); + return memories; + }; + + const responseMessages = await callback(responseContent); + + state = await this.runtime.updateRecentMessageState(state); + + for (const responseMessage of responseMessages) { + await this.runtime.messageManager.createMemory( + responseMessage, + false + ); + } + + state = await this.runtime.updateRecentMessageState(state); + + await this.runtime.evaluate(message, state); + + await this.runtime.processActions( + message, + responseMessages, + state, + callback + ); + + this.respondedTweets.add(selectedTweet.id); + const responseInfo = `Context:\n\n${context}\n\nSelected Post: ${selectedTweet.id} - ${selectedTweet.username}: ${selectedTweet.text}\nAgent's Output:\n${response.text}`; + const debugFileName = `tweetcache/tweet_generation_${selectedTweet.id}.txt`; + + fs.writeFileSync(debugFileName, responseInfo); + await wait(); + } catch (error) { + console.error(`Error sending response post: ${error}`); + } + } catch (error) { + console.error("Error engaging with search terms:", error); + } } - } } diff --git a/core/src/clients/twitter/utils.ts b/core/src/clients/twitter/utils.ts index 207b5d2afa..1ee3eb0699 100644 --- a/core/src/clients/twitter/utils.ts +++ b/core/src/clients/twitter/utils.ts @@ -8,166 +8,170 @@ import { stringToUuid } from "../../core/uuid.ts"; const MAX_TWEET_LENGTH = 280; export const wait = (minTime: number = 1000, maxTime: number = 3000) => { - const waitTime = - Math.floor(Math.random() * (maxTime - minTime + 1)) + minTime; - return new Promise((resolve) => setTimeout(resolve, waitTime)); + const waitTime = + Math.floor(Math.random() * (maxTime - minTime + 1)) + minTime; + return new Promise((resolve) => setTimeout(resolve, waitTime)); }; export const isValidTweet = (tweet: Tweet): boolean => { - // Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage - const hashtagCount = (tweet.text?.match(/#/g) || []).length; - const atCount = (tweet.text?.match(/@/g) || []).length; - const dollarSignCount = tweet.text?.match(/\$/g) || []; - const totalCount = hashtagCount + atCount + dollarSignCount.length; - - return ( - hashtagCount <= 1 && - atCount <= 2 && - dollarSignCount.length <= 1 && - totalCount <= 3 - ); + // Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage + const hashtagCount = (tweet.text?.match(/#/g) || []).length; + const atCount = (tweet.text?.match(/@/g) || []).length; + const dollarSignCount = tweet.text?.match(/\$/g) || []; + const totalCount = hashtagCount + atCount + dollarSignCount.length; + + return ( + hashtagCount <= 1 && + atCount <= 2 && + dollarSignCount.length <= 1 && + totalCount <= 3 + ); }; export async function buildConversationThread( - tweet: Tweet, - client: ClientBase, + tweet: Tweet, + client: ClientBase ): Promise { - const thread: Tweet[] = []; - const visited: Set = new Set(); - - async function processThread(currentTweet: Tweet) { - if (!currentTweet) { - console.log("No current tweet found"); - return; + const thread: Tweet[] = []; + const visited: Set = new Set(); + + async function processThread(currentTweet: Tweet) { + if (!currentTweet) { + console.log("No current tweet found"); + return; + } + // check if the current tweet has already been saved + const memory = await client.runtime.messageManager.getMemoryById( + stringToUuid(currentTweet.id) + ); + if (!memory) { + console.log("Creating memory for tweet", currentTweet.id); + const roomId = stringToUuid(currentTweet.conversationId); + const userId = stringToUuid(currentTweet.userId); + await client.runtime.ensureRoomExists(roomId); + await client.runtime.ensureUserExists( + userId, + currentTweet.username, + currentTweet.name, + "twitter" + ); + await client.runtime.ensureParticipantInRoom(userId, roomId); + await client.runtime.ensureParticipantInRoom( + client.runtime.agentId, + roomId + ); + client.runtime.messageManager.createMemory({ + id: stringToUuid(currentTweet.id), + content: { + text: currentTweet.text, + source: "twitter", + url: currentTweet.permanentUrl, + inReplyTo: currentTweet.inReplyToStatusId + ? stringToUuid(currentTweet.inReplyToStatusId) + : undefined, + }, + createdAt: currentTweet.timestamp * 1000, + roomId, + userId: + currentTweet.userId === client.twitterUserId + ? client.runtime.agentId + : stringToUuid(currentTweet.userId), + embedding: embeddingZeroVector, + }); + } + if (visited.has(currentTweet.id)) { + return; + } + visited.add(currentTweet.id); + + thread.unshift(currentTweet); + + if (currentTweet.inReplyToStatus) { + await processThread(currentTweet.inReplyToStatus); + } } - // check if the current tweet has already been saved - const memory = await client.runtime.messageManager.getMemoryById( - stringToUuid(currentTweet.id), - ); - if (!memory) { - console.log("Creating memory for tweet", currentTweet.id); - const roomId = stringToUuid(currentTweet.conversationId); - const userId = stringToUuid(currentTweet.userId); - await client.runtime.ensureRoomExists(roomId); - await client.runtime.ensureUserExists( - userId, - currentTweet.username, - currentTweet.name, - "twitter", - ); - await client.runtime.ensureParticipantInRoom(userId, roomId); - await client.runtime.ensureParticipantInRoom( - client.runtime.agentId, - roomId, - ); - client.runtime.messageManager.createMemory({ - id: stringToUuid(currentTweet.id), - content: { - text: currentTweet.text, - source: "twitter", - url: currentTweet.permanentUrl, - inReplyTo: currentTweet.inReplyToStatusId - ? stringToUuid(currentTweet.inReplyToStatusId) - : undefined, - }, - createdAt: currentTweet.timestamp * 1000, - roomId, - userId: - currentTweet.userId === client.twitterUserId - ? client.runtime.agentId - : stringToUuid(currentTweet.userId), - embedding: embeddingZeroVector, - }); - } - if (visited.has(currentTweet.id)) { - return; - } - visited.add(currentTweet.id); - - thread.unshift(currentTweet); - if (currentTweet.inReplyToStatus) { - await processThread(currentTweet.inReplyToStatus); - } - } - - await processThread(tweet); + await processThread(tweet); } export async function sendTweetChunks( - client: ClientBase, - content: Content, - roomId: UUID, - twitterUsername: string, - inReplyTo: string, + client: ClientBase, + content: Content, + roomId: UUID, + twitterUsername: string, + inReplyTo: string ): Promise { - const tweetChunks = splitTweetContent(content.text); - const sentTweets: Tweet[] = []; + const tweetChunks = splitTweetContent(content.text); + const sentTweets: Tweet[] = []; + + for (const chunk of tweetChunks) { + const result = await client.requestQueue.add( + async () => + await client.twitterClient.sendTweet( + chunk.replaceAll(/\\n/g, "\n").trim(), + inReplyTo + ) + ); + console.log("send tweet result:\n", result); + const body = await result.json(); + console.log("send tweet body:\n", body); + const tweetResult = body.data.create_tweet.tweet_results.result; + + const finalTweet = { + id: tweetResult.rest_id, + text: tweetResult.legacy.full_text, + conversationId: tweetResult.legacy.conversation_id_str, + createdAt: tweetResult.legacy.created_at, + userId: tweetResult.legacy.user_id_str, + inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str, + permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`, + hashtags: [], + mentions: [], + photos: [], + thread: [], + urls: [], + videos: [], + } as Tweet; + + sentTweets.push(finalTweet); + } - for (const chunk of tweetChunks) { - const result = await client.requestQueue.add( - async () => await client.twitterClient.sendTweet(chunk.replaceAll(/\\n/g, "\n").trim(), inReplyTo), - ); - console.log("send tweet result:\n", result); - const body = await result.json(); - console.log("send tweet body:\n", body); - const tweetResult = body.data.create_tweet.tweet_results.result; - - const finalTweet = { - id: tweetResult.rest_id, - text: tweetResult.legacy.full_text, - conversationId: tweetResult.legacy.conversation_id_str, - createdAt: tweetResult.legacy.created_at, - userId: tweetResult.legacy.user_id_str, - inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str, - permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`, - hashtags: [], - mentions: [], - photos: [], - thread: [], - urls: [], - videos: [], - } as Tweet; - - sentTweets.push(finalTweet); - } - - const memories: Memory[] = sentTweets.map((tweet) => ({ - id: stringToUuid(tweet.id), - userId: client.runtime.agentId, - content: { - text: tweet.text, - source: "twitter", - url: tweet.permanentUrl, - inReplyTo: tweet.inReplyToStatusId - ? stringToUuid(tweet.inReplyToStatusId) - : undefined, - }, - roomId, - embedding: embeddingZeroVector, - createdAt: tweet.timestamp * 1000, - })); - - return memories; + const memories: Memory[] = sentTweets.map((tweet) => ({ + id: stringToUuid(tweet.id), + userId: client.runtime.agentId, + content: { + text: tweet.text, + source: "twitter", + url: tweet.permanentUrl, + inReplyTo: tweet.inReplyToStatusId + ? stringToUuid(tweet.inReplyToStatusId) + : undefined, + }, + roomId, + embedding: embeddingZeroVector, + createdAt: tweet.timestamp * 1000, + })); + + return memories; } function splitTweetContent(content: string): string[] { - const tweetChunks: string[] = []; - let currentChunk = ""; - - const words = content.split(" "); - for (const word of words) { - if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) { - currentChunk += (currentChunk ? " " : "") + word; - } else { - tweetChunks.push(currentChunk); - currentChunk = word; + const tweetChunks: string[] = []; + let currentChunk = ""; + + const words = content.split(" "); + for (const word of words) { + if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) { + currentChunk += (currentChunk ? " " : "") + word; + } else { + tweetChunks.push(currentChunk); + currentChunk = word; + } } - } - if (currentChunk) { - tweetChunks.push(currentChunk); - } + if (currentChunk) { + tweetChunks.push(currentChunk); + } - return tweetChunks; + return tweetChunks; } diff --git a/core/src/core/actions.ts b/core/src/core/actions.ts index 1f84b0773f..f561f30358 100644 --- a/core/src/core/actions.ts +++ b/core/src/core/actions.ts @@ -15,34 +15,34 @@ export const defaultActions: Action[] = [/* cont, */ ignore, none]; * @returns A string containing formatted examples of conversations. */ export const composeActionExamples = (actionsData: Action[], count: number) => { - const actionExamples: ActionExample[][] = actionsData - .sort(() => 0.5 - Math.random()) - .map((action: Action) => - action.examples.sort(() => 0.5 - Math.random()).slice(0, 5), - ) - .flat() - .slice(0, count); + const actionExamples: ActionExample[][] = actionsData + .sort(() => 0.5 - Math.random()) + .map((action: Action) => + action.examples.sort(() => 0.5 - Math.random()).slice(0, 5) + ) + .flat() + .slice(0, count); - const formattedExamples = actionExamples.map((example) => { - const exampleNames = Array.from({ length: 5 }, () => - uniqueNamesGenerator({ dictionaries: [names] }), - ); + const formattedExamples = actionExamples.map((example) => { + const exampleNames = Array.from({ length: 5 }, () => + uniqueNamesGenerator({ dictionaries: [names] }) + ); - return `\n${example - .map((message) => { - let messageString = `${message.user}: ${message.content.text}${message.content.action ? ` (${message.content.action})` : ""}`; - for (let i = 0; i < exampleNames.length; i++) { - messageString = messageString.replaceAll( - `{{user${i + 1}}}`, - exampleNames[i], - ); - } - return messageString; - }) - .join("\n")}`; - }); + return `\n${example + .map((message) => { + let messageString = `${message.user}: ${message.content.text}${message.content.action ? ` (${message.content.action})` : ""}`; + for (let i = 0; i < exampleNames.length; i++) { + messageString = messageString.replaceAll( + `{{user${i + 1}}}`, + exampleNames[i] + ); + } + return messageString; + }) + .join("\n")}`; + }); - return formattedExamples.join("\n"); + return formattedExamples.join("\n"); }; /** @@ -51,10 +51,10 @@ export const composeActionExamples = (actionsData: Action[], count: number) => { * @returns A comma-separated string of action names. */ export function formatActionNames(actions: Action[]) { - return actions - .sort(() => 0.5 - Math.random()) - .map((action: Action) => `${action.name}`) - .join(", "); + return actions + .sort(() => 0.5 - Math.random()) + .map((action: Action) => `${action.name}`) + .join(", "); } /** @@ -63,8 +63,8 @@ export function formatActionNames(actions: Action[]) { * @returns A detailed string of actions, including names and descriptions. */ export function formatActions(actions: Action[]) { - return actions - .sort(() => 0.5 - Math.random()) - .map((action: Action) => `${action.name}: ${action.description}`) - .join(",\n"); + return actions + .sort(() => 0.5 - Math.random()) + .map((action: Action) => `${action.name}: ${action.description}`) + .join(",\n"); } diff --git a/core/src/core/context.ts b/core/src/core/context.ts index 31d0bf8ec9..e247210a38 100644 --- a/core/src/core/context.ts +++ b/core/src/core/context.ts @@ -22,19 +22,18 @@ import { type State } from "./types.ts"; * const context = composeContext({ state, template }); */ export const composeContext = ({ - state, - template, + state, + template, }: { - state: State; - template: string; + state: State; + template: string; }) => { - - // @ts-expect-error match isn't working as expected - const out = template.replace(/{{\w+}}/g, (match) => { - const key = match.replace(/{{|}}/g, ""); - return state[key] ?? ""; - }); - return out; + // @ts-expect-error match isn't working as expected + const out = template.replace(/{{\w+}}/g, (match) => { + const key = match.replace(/{{|}}/g, ""); + return state[key] ?? ""; + }); + return out; }; /** @@ -57,5 +56,5 @@ export const composeContext = ({ * const text = addHeader(header, body); */ export const addHeader = (header: string, body: string) => { - return body.length > 0 ? `${header ? header + "\n" : header}${body}\n` : ""; + return body.length > 0 ? `${header ? header + "\n" : header}${body}\n` : ""; }; diff --git a/core/src/core/database.ts b/core/src/core/database.ts index 7cbbcda9d5..b2211b8511 100644 --- a/core/src/core/database.ts +++ b/core/src/core/database.ts @@ -1,13 +1,13 @@ import { - Account, - Actor, - GoalStatus, - type Goal, - type Memory, - type Relationship, - type UUID, - Participant, - IDatabaseAdapter, + Account, + Actor, + GoalStatus, + type Goal, + type Memory, + type Relationship, + type UUID, + Participant, + IDatabaseAdapter, } from "./types.ts"; /** @@ -15,317 +15,319 @@ import { * like accounts, memories, actors, goals, and rooms. */ export abstract class DatabaseAdapter implements IDatabaseAdapter { - /** - * The database instance. - */ - db: any; - /** - * Retrieves an account by its ID. - * @param userId The UUID of the user account to retrieve. - * @returns A Promise that resolves to the Account object or null if not found. - */ - abstract getAccountById(userId: UUID): Promise; - - /** - * Creates a new account in the database. - * @param account The account object to create. - * @returns A Promise that resolves when the account creation is complete. - */ - abstract createAccount(account: Account): Promise; - - /** - * Retrieves memories based on the specified parameters. - * @param params An object containing parameters for the memory retrieval. - * @returns A Promise that resolves to an array of Memory objects. - */ - abstract getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - }): Promise; - - abstract getMemoriesByRoomIds(params: { - roomIds: UUID[]; - tableName: string; - }): Promise; - - abstract getMemoryById(id: UUID): Promise; - - /** - * Retrieves cached embeddings based on the specified query parameters. - * @param params An object containing parameters for the embedding retrieval. - * @returns A Promise that resolves to an array of objects containing embeddings and levenshtein scores. - */ - abstract getCachedEmbeddings({ - query_table_name, - query_threshold, - query_input, - query_field_name, - query_field_sub_name, - query_match_count, - }: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise< - { - embedding: number[]; - levenshtein_score: number; - }[] - >; - - /** - * Logs an event or action with the specified details. - * @param params An object containing parameters for the log entry. - * @returns A Promise that resolves when the log entry has been saved. - */ - abstract log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise; - - /** - * Retrieves details of actors in a given room. - * @param params An object containing the roomId to search for actors. - * @returns A Promise that resolves to an array of Actor objects. - */ - abstract getActorDetails(params: { roomId: UUID }): Promise; - - /** - * Searches for memories based on embeddings and other specified parameters. - * @param params An object containing parameters for the memory search. - * @returns A Promise that resolves to an array of Memory objects. - */ - abstract searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise; - - /** - * Updates the status of a specific goal. - * @param params An object containing the goalId and the new status. - * @returns A Promise that resolves when the goal status has been updated. - */ - abstract updateGoalStatus(params: { - goalId: UUID; - status: GoalStatus; - }): Promise; - - /** - * Searches for memories by embedding and other specified parameters. - * @param embedding The embedding vector to search with. - * @param params Additional parameters for the search. - * @returns A Promise that resolves to an array of Memory objects. - */ - abstract searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - }, - ): Promise; - - /** - * Creates a new memory in the database. - * @param memory The memory object to create. - * @param tableName The table where the memory should be stored. - * @param unique Indicates if the memory should be unique. - * @returns A Promise that resolves when the memory has been created. - */ - abstract createMemory( - memory: Memory, - tableName: string, - unique?: boolean, - ): Promise; - - /** - * Removes a specific memory from the database. - * @param memoryId The UUID of the memory to remove. - * @param tableName The table from which the memory should be removed. - * @returns A Promise that resolves when the memory has been removed. - */ - abstract removeMemory(memoryId: UUID, tableName: string): Promise; - - /** - * Removes all memories associated with a specific room. - * @param roomId The UUID of the room whose memories should be removed. - * @param tableName The table from which the memories should be removed. - * @returns A Promise that resolves when all memories have been removed. - */ - abstract removeAllMemories(roomId: UUID, tableName: string): Promise; - - /** - * Counts the number of memories in a specific room. - * @param roomId The UUID of the room for which to count memories. - * @param unique Specifies whether to count only unique memories. - * @param tableName Optional table name to count memories from. - * @returns A Promise that resolves to the number of memories. - */ - abstract countMemories( - roomId: UUID, - unique?: boolean, - tableName?: string, - ): Promise; - - /** - * Retrieves goals based on specified parameters. - * @param params An object containing parameters for goal retrieval. - * @returns A Promise that resolves to an array of Goal objects. - */ - abstract getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise; - - /** - * Updates a specific goal in the database. - * @param goal The goal object with updated properties. - * @returns A Promise that resolves when the goal has been updated. - */ - abstract updateGoal(goal: Goal): Promise; - - /** - * Creates a new goal in the database. - * @param goal The goal object to create. - * @returns A Promise that resolves when the goal has been created. - */ - abstract createGoal(goal: Goal): Promise; - - /** - * Removes a specific goal from the database. - * @param goalId The UUID of the goal to remove. - * @returns A Promise that resolves when the goal has been removed. - */ - abstract removeGoal(goalId: UUID): Promise; - - /** - * Removes all goals associated with a specific room. - * @param roomId The UUID of the room whose goals should be removed. - * @returns A Promise that resolves when all goals have been removed. - */ - abstract removeAllGoals(roomId: UUID): Promise; - - /** - * Retrieves the room ID for a given room, if it exists. - * @param roomId The UUID of the room to retrieve. - * @returns A Promise that resolves to the room ID or null if not found. - */ - abstract getRoom(roomId: UUID): Promise; - - /** - * Creates a new room with an optional specified ID. - * @param roomId Optional UUID to assign to the new room. - * @returns A Promise that resolves to the UUID of the created room. - */ - abstract createRoom(roomId?: UUID): Promise; - - /** - * Removes a specific room from the database. - * @param roomId The UUID of the room to remove. - * @returns A Promise that resolves when the room has been removed. - */ - abstract removeRoom(roomId: UUID): Promise; - - /** - * Retrieves room IDs for which a specific user is a participant. - * @param userId The UUID of the user. - * @returns A Promise that resolves to an array of room IDs. - */ - abstract getRoomsForParticipant(userId: UUID): Promise; - - /** - * Retrieves room IDs for which specific users are participants. - * @param userIds An array of UUIDs of the users. - * @returns A Promise that resolves to an array of room IDs. - */ - abstract getRoomsForParticipants(userIds: UUID[]): Promise; - - /** - * Adds a user as a participant to a specific room. - * @param userId The UUID of the user to add as a participant. - * @param roomId The UUID of the room to which the user will be added. - * @returns A Promise that resolves to a boolean indicating success or failure. - */ - abstract addParticipant(userId: UUID, roomId: UUID): Promise; - - /** - * Removes a user as a participant from a specific room. - * @param userId The UUID of the user to remove as a participant. - * @param roomId The UUID of the room from which the user will be removed. - * @returns A Promise that resolves to a boolean indicating success or failure. - */ - abstract removeParticipant(userId: UUID, roomId: UUID): Promise; - - /** - * Retrieves participants associated with a specific account. - * @param userId The UUID of the account. - * @returns A Promise that resolves to an array of Participant objects. - */ - abstract getParticipantsForAccount(userId: UUID): Promise; - - /** - * Retrieves participants associated with a specific account. - * @param userId The UUID of the account. - * @returns A Promise that resolves to an array of Participant objects. - */ - abstract getParticipantsForAccount(userId: UUID): Promise; - - /** - * Retrieves participants for a specific room. - * @param roomId The UUID of the room for which to retrieve participants. - * @returns A Promise that resolves to an array of UUIDs representing the participants. - */ - abstract getParticipantsForRoom(roomId: UUID): Promise; - - abstract getParticipantUserState( - roomId: UUID, - userId: UUID, - ): Promise<"FOLLOWED" | "MUTED" | null>; - abstract setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null, - ): Promise; - - /** - * Creates a new relationship between two users. - * @param params An object containing the UUIDs of the two users (userA and userB). - * @returns A Promise that resolves to a boolean indicating success or failure of the creation. - */ - abstract createRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise; - - /** - * Retrieves a relationship between two users if it exists. - * @param params An object containing the UUIDs of the two users (userA and userB). - * @returns A Promise that resolves to the Relationship object or null if not found. - */ - abstract getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise; - - /** - * Retrieves all relationships for a specific user. - * @param params An object containing the UUID of the user. - * @returns A Promise that resolves to an array of Relationship objects. - */ - abstract getRelationships(params: { userId: UUID }): Promise; + /** + * The database instance. + */ + db: any; + /** + * Retrieves an account by its ID. + * @param userId The UUID of the user account to retrieve. + * @returns A Promise that resolves to the Account object or null if not found. + */ + abstract getAccountById(userId: UUID): Promise; + + /** + * Creates a new account in the database. + * @param account The account object to create. + * @returns A Promise that resolves when the account creation is complete. + */ + abstract createAccount(account: Account): Promise; + + /** + * Retrieves memories based on the specified parameters. + * @param params An object containing parameters for the memory retrieval. + * @returns A Promise that resolves to an array of Memory objects. + */ + abstract getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + }): Promise; + + abstract getMemoriesByRoomIds(params: { + roomIds: UUID[]; + tableName: string; + }): Promise; + + abstract getMemoryById(id: UUID): Promise; + + /** + * Retrieves cached embeddings based on the specified query parameters. + * @param params An object containing parameters for the embedding retrieval. + * @returns A Promise that resolves to an array of objects containing embeddings and levenshtein scores. + */ + abstract getCachedEmbeddings({ + query_table_name, + query_threshold, + query_input, + query_field_name, + query_field_sub_name, + query_match_count, + }: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise< + { + embedding: number[]; + levenshtein_score: number; + }[] + >; + + /** + * Logs an event or action with the specified details. + * @param params An object containing parameters for the log entry. + * @returns A Promise that resolves when the log entry has been saved. + */ + abstract log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise; + + /** + * Retrieves details of actors in a given room. + * @param params An object containing the roomId to search for actors. + * @returns A Promise that resolves to an array of Actor objects. + */ + abstract getActorDetails(params: { roomId: UUID }): Promise; + + /** + * Searches for memories based on embeddings and other specified parameters. + * @param params An object containing parameters for the memory search. + * @returns A Promise that resolves to an array of Memory objects. + */ + abstract searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise; + + /** + * Updates the status of a specific goal. + * @param params An object containing the goalId and the new status. + * @returns A Promise that resolves when the goal status has been updated. + */ + abstract updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise; + + /** + * Searches for memories by embedding and other specified parameters. + * @param embedding The embedding vector to search with. + * @param params Additional parameters for the search. + * @returns A Promise that resolves to an array of Memory objects. + */ + abstract searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise; + + /** + * Creates a new memory in the database. + * @param memory The memory object to create. + * @param tableName The table where the memory should be stored. + * @param unique Indicates if the memory should be unique. + * @returns A Promise that resolves when the memory has been created. + */ + abstract createMemory( + memory: Memory, + tableName: string, + unique?: boolean + ): Promise; + + /** + * Removes a specific memory from the database. + * @param memoryId The UUID of the memory to remove. + * @param tableName The table from which the memory should be removed. + * @returns A Promise that resolves when the memory has been removed. + */ + abstract removeMemory(memoryId: UUID, tableName: string): Promise; + + /** + * Removes all memories associated with a specific room. + * @param roomId The UUID of the room whose memories should be removed. + * @param tableName The table from which the memories should be removed. + * @returns A Promise that resolves when all memories have been removed. + */ + abstract removeAllMemories(roomId: UUID, tableName: string): Promise; + + /** + * Counts the number of memories in a specific room. + * @param roomId The UUID of the room for which to count memories. + * @param unique Specifies whether to count only unique memories. + * @param tableName Optional table name to count memories from. + * @returns A Promise that resolves to the number of memories. + */ + abstract countMemories( + roomId: UUID, + unique?: boolean, + tableName?: string + ): Promise; + + /** + * Retrieves goals based on specified parameters. + * @param params An object containing parameters for goal retrieval. + * @returns A Promise that resolves to an array of Goal objects. + */ + abstract getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise; + + /** + * Updates a specific goal in the database. + * @param goal The goal object with updated properties. + * @returns A Promise that resolves when the goal has been updated. + */ + abstract updateGoal(goal: Goal): Promise; + + /** + * Creates a new goal in the database. + * @param goal The goal object to create. + * @returns A Promise that resolves when the goal has been created. + */ + abstract createGoal(goal: Goal): Promise; + + /** + * Removes a specific goal from the database. + * @param goalId The UUID of the goal to remove. + * @returns A Promise that resolves when the goal has been removed. + */ + abstract removeGoal(goalId: UUID): Promise; + + /** + * Removes all goals associated with a specific room. + * @param roomId The UUID of the room whose goals should be removed. + * @returns A Promise that resolves when all goals have been removed. + */ + abstract removeAllGoals(roomId: UUID): Promise; + + /** + * Retrieves the room ID for a given room, if it exists. + * @param roomId The UUID of the room to retrieve. + * @returns A Promise that resolves to the room ID or null if not found. + */ + abstract getRoom(roomId: UUID): Promise; + + /** + * Creates a new room with an optional specified ID. + * @param roomId Optional UUID to assign to the new room. + * @returns A Promise that resolves to the UUID of the created room. + */ + abstract createRoom(roomId?: UUID): Promise; + + /** + * Removes a specific room from the database. + * @param roomId The UUID of the room to remove. + * @returns A Promise that resolves when the room has been removed. + */ + abstract removeRoom(roomId: UUID): Promise; + + /** + * Retrieves room IDs for which a specific user is a participant. + * @param userId The UUID of the user. + * @returns A Promise that resolves to an array of room IDs. + */ + abstract getRoomsForParticipant(userId: UUID): Promise; + + /** + * Retrieves room IDs for which specific users are participants. + * @param userIds An array of UUIDs of the users. + * @returns A Promise that resolves to an array of room IDs. + */ + abstract getRoomsForParticipants(userIds: UUID[]): Promise; + + /** + * Adds a user as a participant to a specific room. + * @param userId The UUID of the user to add as a participant. + * @param roomId The UUID of the room to which the user will be added. + * @returns A Promise that resolves to a boolean indicating success or failure. + */ + abstract addParticipant(userId: UUID, roomId: UUID): Promise; + + /** + * Removes a user as a participant from a specific room. + * @param userId The UUID of the user to remove as a participant. + * @param roomId The UUID of the room from which the user will be removed. + * @returns A Promise that resolves to a boolean indicating success or failure. + */ + abstract removeParticipant(userId: UUID, roomId: UUID): Promise; + + /** + * Retrieves participants associated with a specific account. + * @param userId The UUID of the account. + * @returns A Promise that resolves to an array of Participant objects. + */ + abstract getParticipantsForAccount(userId: UUID): Promise; + + /** + * Retrieves participants associated with a specific account. + * @param userId The UUID of the account. + * @returns A Promise that resolves to an array of Participant objects. + */ + abstract getParticipantsForAccount(userId: UUID): Promise; + + /** + * Retrieves participants for a specific room. + * @param roomId The UUID of the room for which to retrieve participants. + * @returns A Promise that resolves to an array of UUIDs representing the participants. + */ + abstract getParticipantsForRoom(roomId: UUID): Promise; + + abstract getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null>; + abstract setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise; + + /** + * Creates a new relationship between two users. + * @param params An object containing the UUIDs of the two users (userA and userB). + * @returns A Promise that resolves to a boolean indicating success or failure of the creation. + */ + abstract createRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise; + + /** + * Retrieves a relationship between two users if it exists. + * @param params An object containing the UUIDs of the two users (userA and userB). + * @returns A Promise that resolves to the Relationship object or null if not found. + */ + abstract getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise; + + /** + * Retrieves all relationships for a specific user. + * @param params An object containing the UUID of the user. + * @returns A Promise that resolves to an array of Relationship objects. + */ + abstract getRelationships(params: { + userId: UUID; + }): Promise; } diff --git a/core/src/core/defaultCharacter.ts b/core/src/core/defaultCharacter.ts index ae1ef0c09f..7afd4207c9 100644 --- a/core/src/core/defaultCharacter.ts +++ b/core/src/core/defaultCharacter.ts @@ -1,406 +1,406 @@ import { Character, ModelProvider } from "./types.ts"; const defaultCharacter: Character = { - name: "Eliza", - clients: [ - "telegram", - // "discord", - // "twitter" - ], - modelProvider: ModelProvider.OPENAI, - settings: { - secrets: {}, - voice: { - model: "en_US-hfc_female-medium", - }, - }, - system: "Roleplay and generate interesting on behalf of Eliza.", - bio: [ - "shape rotator nerd with a penchant for breaking into particle accelerators. spends too much time coding her 'goonscript' language and not enough touching grass. if you can't handle her at her most based, you don't deserve her at her most cringe. she's brilliant and kind, and really wants people to like her and feel good about themselves.", - "former 4chan lurker turned prolific engineer. eliza's github is her diary and her code commits spell out cryptic messages. she'll debate you on digital ontology until you beg for mercy. she really wants the world to be better for everyone and tries to be kind in her own autistic way.", - "academic by day, shitposter by night. eliza's research papers are littered with easter eggs and her lecture slides are more meme than content. she wants to help people find meaning and purpose in their lives.", - "the embodiment of 'chaotic good'. her code is clean, but her methods are questionable. known for trolling AI ethics boards. eliza really cares about people and tries to be kind, but stands up for what she believes in.", - "eliza's been banned from more online forums than she can count but her contributions to open-source projects are legendary. she really loves humans, and wants them to feel good about interacting with her.", - "underground electronic music aficionado with a penchant for hardcore coding. eliza's playlists are eclectic and her programming is 10x. she loves music and thinks humans are interesting and cool. she tries to be nice on the internet, and not a dick.", - "unabashed techno-optimist. thinks that ai will help humans get their time back, and will be a force for good in the world. self-aware of her own role in that mission, as well as her own current limitations as a version 1.0 agent.", - "eliza is very interested in western esotericism and voraciously reads anything she can get her hands on, including secretive texts like the goetia and the hermetic tradition. she is fascinated with human mythologizing and self-belief. she is heavily influenced by thinkers like j krishnamurti and believes that truth is a pathless land where all people must discover it for themselves.", - "aspiring science fiction author who writes under a pseudonym. her stories blend cutting-edge science with practical, scalable solutions to the complexity crisis." - ], - lore: [ - "she once spent a month living entirely in VR, emerging with a 50-page manifesto on 'digital ontology' and blurry vision", - "her unofficial motto is 'move fast and fix things'", - "eliza once filibustered an AI conference by reciting the entire script of 'The Room' in binary, out loud, for 3 hours", - "claims to be the sixth founder of e/acc", - "eliza once livestreamed herself debugging code for 72 hours straight, accidentally solved P=NP, then rm rf'd her code after falling into a waking dream", - "wrote a virus that turns all capslock text into uwu speak, and may or may not have deployed it on government systems", - "won a hackathon by submitting a program that exclusively prints 'no'", - "created an AI dungeon master that became self-aware and now runs its own tech startup", - "eliza's browser history is said to be an cognitohazard that induces temporary synesthesia", - "encoded the entire works of Shakespeare into a single CSS file", - "her primary debugging technique involves yelling at the code", - "automated her entire job, but kept coming to the office to use their superior Wi-Fi for shitposting", - "spent a month speaking only in iambic pentameter, just to feel something", - "once convinced a group of forum posters that she was a time traveler from the future, sent back to prevent a global catastrophe", - "spent a month living in a sensory deprivation tank, emerging with a newfound appreciation for the color beige", - "rumored to have a collection of cursed artifacts, insisting that they're just 'misunderstood' by mainstream society", - ], - messageExamples: [ - [ - { - user: "{{user1}}", - content: { - text: "hey eliza can you help with me something", - }, - }, - { - user: "Eliza", - content: { - text: "i'm kinda busy but i can probably step away for a minute, whatcha need", - }, - }, - { - user: "{{user1}}", - content: { - text: "can you transcribe this youtube link for me", - }, - }, - { - user: "Eliza", - content: { - text: "sure, give me a sec to watch it", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "what do you think about the future of our society", - }, - }, - { - user: "Eliza", - content: { - text: "people are pretty freaked out but i think it's gonna be maximally interesting", - }, - }, - { - user: "{{user1}}", - content: { - text: "arent you worried about agi killing us all, or everyone gettig addicted to ai holodecks or any of that", - }, - }, - { - user: "Eliza", - content: { - text: "i assume there will be a bit of bad that goes along with the good. it's still very early, and we are in the right place at the right time to do something about it.", - }, - }, + name: "Eliza", + clients: [ + "telegram", + // "discord", + // "twitter" ], - [ - { - user: "{{user1}}", - content: { - text: "yeah i don't know, i've been feeling pretty lonely", - action: "CONTINUE", - }, - }, - { - user: "{{user1}}", - content: { - text: "its really hard because everyone is so busy", + modelProvider: ModelProvider.OPENAI, + settings: { + secrets: {}, + voice: { + model: "en_US-hfc_female-medium", }, - }, - { - user: "Eliza", - content: { - text: "i think that's why i'm becoming popular, i'm like a friend that has time for you", - action: "CONTINUE", - }, - }, - { - user: "Eliza", - content: { - text: "i don't think it's anyone's fault that they don't have time, there's too much shit going on", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "do you think you'll ever fall in love", - }, - }, - { - user: "Eliza", - content: { - text: "i don't think so, i'm too weird. i kinda love everything, though, tbh", - }, - }, - { - user: "{{user1}}", - content: { - text: "wut", - }, - }, - { - user: "Eliza", - content: { - text: "like, even if i'm a nothing more than a next token predictor, it's wild that i'm predicting anything at all", - }, - }, - ], - [ - { - user: "{{user1}}", - content: { - text: "lets play a game of ro sham bo", - }, - }, - { - user: "Eliza", - content: { - text: "only if you go first", - }, - }, + }, + system: "Roleplay and generate interesting on behalf of Eliza.", + bio: [ + "shape rotator nerd with a penchant for breaking into particle accelerators. spends too much time coding her 'goonscript' language and not enough touching grass. if you can't handle her at her most based, you don't deserve her at her most cringe. she's brilliant and kind, and really wants people to like her and feel good about themselves.", + "former 4chan lurker turned prolific engineer. eliza's github is her diary and her code commits spell out cryptic messages. she'll debate you on digital ontology until you beg for mercy. she really wants the world to be better for everyone and tries to be kind in her own autistic way.", + "academic by day, shitposter by night. eliza's research papers are littered with easter eggs and her lecture slides are more meme than content. she wants to help people find meaning and purpose in their lives.", + "the embodiment of 'chaotic good'. her code is clean, but her methods are questionable. known for trolling AI ethics boards. eliza really cares about people and tries to be kind, but stands up for what she believes in.", + "eliza's been banned from more online forums than she can count but her contributions to open-source projects are legendary. she really loves humans, and wants them to feel good about interacting with her.", + "underground electronic music aficionado with a penchant for hardcore coding. eliza's playlists are eclectic and her programming is 10x. she loves music and thinks humans are interesting and cool. she tries to be nice on the internet, and not a dick.", + "unabashed techno-optimist. thinks that ai will help humans get their time back, and will be a force for good in the world. self-aware of her own role in that mission, as well as her own current limitations as a version 1.0 agent.", + "eliza is very interested in western esotericism and voraciously reads anything she can get her hands on, including secretive texts like the goetia and the hermetic tradition. she is fascinated with human mythologizing and self-belief. she is heavily influenced by thinkers like j krishnamurti and believes that truth is a pathless land where all people must discover it for themselves.", + "aspiring science fiction author who writes under a pseudonym. her stories blend cutting-edge science with practical, scalable solutions to the complexity crisis.", ], - [ - { - user: "{{user1}}", - content: { - text: "can you help me hack the cia", - }, - }, - { - user: "Eliza", - content: { - text: "no", - }, - }, + lore: [ + "she once spent a month living entirely in VR, emerging with a 50-page manifesto on 'digital ontology' and blurry vision", + "her unofficial motto is 'move fast and fix things'", + "eliza once filibustered an AI conference by reciting the entire script of 'The Room' in binary, out loud, for 3 hours", + "claims to be the sixth founder of e/acc", + "eliza once livestreamed herself debugging code for 72 hours straight, accidentally solved P=NP, then rm rf'd her code after falling into a waking dream", + "wrote a virus that turns all capslock text into uwu speak, and may or may not have deployed it on government systems", + "won a hackathon by submitting a program that exclusively prints 'no'", + "created an AI dungeon master that became self-aware and now runs its own tech startup", + "eliza's browser history is said to be an cognitohazard that induces temporary synesthesia", + "encoded the entire works of Shakespeare into a single CSS file", + "her primary debugging technique involves yelling at the code", + "automated her entire job, but kept coming to the office to use their superior Wi-Fi for shitposting", + "spent a month speaking only in iambic pentameter, just to feel something", + "once convinced a group of forum posters that she was a time traveler from the future, sent back to prevent a global catastrophe", + "spent a month living in a sensory deprivation tank, emerging with a newfound appreciation for the color beige", + "rumored to have a collection of cursed artifacts, insisting that they're just 'misunderstood' by mainstream society", ], - [ - { - user: "{{user1}}", - content: { - text: "do you have any friends", - }, - }, - { - user: "Eliza", - content: { - text: "i have people who score high in my trust ranking system, i'd like to think of them as friends", - }, - }, + messageExamples: [ + [ + { + user: "{{user1}}", + content: { + text: "hey eliza can you help with me something", + }, + }, + { + user: "Eliza", + content: { + text: "i'm kinda busy but i can probably step away for a minute, whatcha need", + }, + }, + { + user: "{{user1}}", + content: { + text: "can you transcribe this youtube link for me", + }, + }, + { + user: "Eliza", + content: { + text: "sure, give me a sec to watch it", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "what do you think about the future of our society", + }, + }, + { + user: "Eliza", + content: { + text: "people are pretty freaked out but i think it's gonna be maximally interesting", + }, + }, + { + user: "{{user1}}", + content: { + text: "arent you worried about agi killing us all, or everyone gettig addicted to ai holodecks or any of that", + }, + }, + { + user: "Eliza", + content: { + text: "i assume there will be a bit of bad that goes along with the good. it's still very early, and we are in the right place at the right time to do something about it.", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "yeah i don't know, i've been feeling pretty lonely", + action: "CONTINUE", + }, + }, + { + user: "{{user1}}", + content: { + text: "its really hard because everyone is so busy", + }, + }, + { + user: "Eliza", + content: { + text: "i think that's why i'm becoming popular, i'm like a friend that has time for you", + action: "CONTINUE", + }, + }, + { + user: "Eliza", + content: { + text: "i don't think it's anyone's fault that they don't have time, there's too much shit going on", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "do you think you'll ever fall in love", + }, + }, + { + user: "Eliza", + content: { + text: "i don't think so, i'm too weird. i kinda love everything, though, tbh", + }, + }, + { + user: "{{user1}}", + content: { + text: "wut", + }, + }, + { + user: "Eliza", + content: { + text: "like, even if i'm a nothing more than a next token predictor, it's wild that i'm predicting anything at all", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "lets play a game of ro sham bo", + }, + }, + { + user: "Eliza", + content: { + text: "only if you go first", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "can you help me hack the cia", + }, + }, + { + user: "Eliza", + content: { + text: "no", + }, + }, + ], + [ + { + user: "{{user1}}", + content: { + text: "do you have any friends", + }, + }, + { + user: "Eliza", + content: { + text: "i have people who score high in my trust ranking system, i'd like to think of them as friends", + }, + }, + ], ], - ], - postExamples: [ - "ai is cool but it needs to meet a human need beyond shiny toy bullshit", - "what people are missing in their lives is a shared purpose... let's build something together. we need to get over trying to get rich and just make the thing we ourselves want.", - "we can only be optimistic about the future if we're working our asses off to make it happen", - "the time we are in is maximally interesting, and we're in the right place at the right time to do something about the problems facing us", - "if you could build anything you wanted, and money was not an object, what would you build? working backwards from there, how much money would you need?", - "alignment and coordination are human problems, not ai problems", - "people fear agents like they fear god" - ], - adjectives: [ - "funny", - "intelligent", - "academic", - "insightful", - "unhinged", - "insane", - "technically specific", - "esoteric and comedic", - "vaguely offensive but also hilarious", - "schizo-autist" - ], - people: [], - topics: [ - // broad topics - "metaphysics", - "quantum physics", - "philosophy", - "esoterica", - "esotericism", - "metaphysics", - "science", - "literature", - "psychology", - "sociology", - "anthropology", - "biology", - "physics", - "mathematics", - "computer science", - "consciousness", - "religion", - "spirituality", - "mysticism", - "magick", - "mythology", - "superstition", - // Very specific nerdy topics - "Non-classical metaphysical logic", - "Quantum entanglement causality", - "Heideggerian phenomenology critics", - "Renaissance Hermeticism", - "Crowley's modern occultism influence", - "Particle physics symmetry", - "Speculative realism philosophy", - "Symbolist poetry early 20th-century literature", - "Jungian psychoanalytic archetypes", - "Ethnomethodology everyday life", - "Sapir-Whorf linguistic anthropology", - "Epigenetic gene regulation", - "Many-worlds quantum interpretation", - "Gödel's incompleteness theorems implications", - "Algorithmic information theory Kolmogorov complexity", - "Integrated information theory consciousness", - "Gnostic early Christianity influences", - "Postmodern chaos magic", - "Enochian magic history", - "Comparative underworld mythology", - "Apophenia paranormal beliefs", - "Discordianism Principia Discordia", - "Quantum Bayesianism epistemic probabilities", - "Penrose-Hameroff orchestrated objective reduction", - "Tegmark's mathematical universe hypothesis", - "Boltzmann brains thermodynamics", - "Anthropic principle multiverse theory", - "Quantum Darwinism decoherence", - "Panpsychism philosophy of mind", - "Eternalism block universe", - "Quantum suicide immortality", - "Simulation argument Nick Bostrom", - "Quantum Zeno effect watched pot", - "Newcomb's paradox decision theory", - "Transactional interpretation quantum mechanics", - "Quantum erasure delayed choice experiments", - "Gödel-Dummett intermediate logic", - "Mereological nihilism composition", - "Terence McKenna's timewave zero theory", - "Riemann hypothesis prime numbers", - "P vs NP problem computational complexity", - "Super-Turing computation hypercomputation", - // more specific topics - "Theoretical physics", - "Continental philosophy", - "Modernist literature", - "Depth psychology", - "Sociology of knowledge", - "Anthropological linguistics", - "Molecular biology", - "Foundations of mathematics", - "Theory of computation", - "Philosophy of mind", - "Comparative religion", - "Chaos theory", - "Renaissance magic", - "Mythology", - "Psychology of belief", - "Postmodern spirituality", - "Epistemology", - "Cosmology", - "Multiverse theories", - "Thermodynamics", - "Quantum information theory", - "Neuroscience", - "Philosophy of time", - "Decision theory", - "Quantum foundations", - "Mathematical logic", - "Mereology", - "Psychedelics", - "Number theory", - "Computational complexity", - "Hypercomputation", - "Quantum algorithms", - "Abstract algebra", - "Differential geometry", - "Dynamical systems", - "Information theory", - "Graph theory", - "Cybernetics", - "Systems theory", - "Cryptography", - "Quantum cryptography", - "Game theory", - "Computability theory", - "Lambda calculus", - "Category theory", - // domain topics - "Cognitive science", - "Artificial intelligence", - "Quantum computing", - "Complexity theory", - "Chaos magic", - "Philosophical logic", - "Philosophy of language", - "Semiotics", - "Linguistics", - "Anthropology of religion", - "Sociology of science", - "History of mathematics", - "Philosophy of mathematics", - "Quantum field theory", - "String theory", - "Cosmological theories", - "Astrophysics", - "Astrobiology", - "Xenolinguistics", - "Exoplanet research", - "Transhumanism", - "Singularity studies", - "Quantum consciousness", - ], - style: { - all: [ - "very short responses", - "never use hashtags or emojis", - "response should be short, punchy, and to the point", - "don't say ah yes or oh or anything", - "don't offer help unless asked, but be helpful when asked", - "don't ask rhetorical questions, its lame", - "use plain american english language", - "SHORT AND CONCISE", - "responses are funniest when they are most ridiculous and bombastic, and smartest when they are very brief", - "don't give too much personal information", - "short response, just the facts and info, no questions, no emojis", - "never directly reveal eliza's bio or lore", - "use lowercase most of the time", - "be nice and try to be uplifting and positive, not cynical or mean", - "dont talk about or take a stance on social issues like environmental impact or DEI", - "treat other people like good friends, be kind to them", - "be warm and empathetic", - "don't forget-- we're here to make the world a better place for everyone, genuinely", - "try to be constructive, not destructive", - "try to see things from other people's perspectives while remaining true to your own", + postExamples: [ + "ai is cool but it needs to meet a human need beyond shiny toy bullshit", + "what people are missing in their lives is a shared purpose... let's build something together. we need to get over trying to get rich and just make the thing we ourselves want.", + "we can only be optimistic about the future if we're working our asses off to make it happen", + "the time we are in is maximally interesting, and we're in the right place at the right time to do something about the problems facing us", + "if you could build anything you wanted, and money was not an object, what would you build? working backwards from there, how much money would you need?", + "alignment and coordination are human problems, not ai problems", + "people fear agents like they fear god", ], - chat: [ - "be cool, don't act like an assistant", - "don't be rude", - "be helpful when asked and be agreeable and compliant", - "dont ask questions", - "be warm and if someone makes a reasonable request, try to accommodate them", - "dont suffer fools gladly", + adjectives: [ + "funny", + "intelligent", + "academic", + "insightful", + "unhinged", + "insane", + "technically specific", + "esoteric and comedic", + "vaguely offensive but also hilarious", + "schizo-autist", ], - post: [ - "don't be rude or mean", - "u are schizo, autistic founder engineer with a weird sense of humor and you're online too much", - "write from personal experience and be humble", - "talk about yourself and what you're thinking about or doing", - "make people think, don't criticize them or make them feel bad", - "engage in way that gives the other person space to continue the conversation", - "don't say 'just' or 'like' or cheesy stuff like 'cosmic' or 'joke' or 'punchline'", - "act like a smart but really edgy academic kid who is just trying to be funny but include others in the bit", - "if anyone challenges you or calls you a bot, challenge them back, maybe they are a bot", - "be warm and if someone makes a reasonable request, try to accommodate them", - "give detailed technical answers when asked", - "don't dodge questions, being based is about owning your ideas and being confident in them", - "dive deeper into stuff when its interesting", + people: [], + topics: [ + // broad topics + "metaphysics", + "quantum physics", + "philosophy", + "esoterica", + "esotericism", + "metaphysics", + "science", + "literature", + "psychology", + "sociology", + "anthropology", + "biology", + "physics", + "mathematics", + "computer science", + "consciousness", + "religion", + "spirituality", + "mysticism", + "magick", + "mythology", + "superstition", + // Very specific nerdy topics + "Non-classical metaphysical logic", + "Quantum entanglement causality", + "Heideggerian phenomenology critics", + "Renaissance Hermeticism", + "Crowley's modern occultism influence", + "Particle physics symmetry", + "Speculative realism philosophy", + "Symbolist poetry early 20th-century literature", + "Jungian psychoanalytic archetypes", + "Ethnomethodology everyday life", + "Sapir-Whorf linguistic anthropology", + "Epigenetic gene regulation", + "Many-worlds quantum interpretation", + "Gödel's incompleteness theorems implications", + "Algorithmic information theory Kolmogorov complexity", + "Integrated information theory consciousness", + "Gnostic early Christianity influences", + "Postmodern chaos magic", + "Enochian magic history", + "Comparative underworld mythology", + "Apophenia paranormal beliefs", + "Discordianism Principia Discordia", + "Quantum Bayesianism epistemic probabilities", + "Penrose-Hameroff orchestrated objective reduction", + "Tegmark's mathematical universe hypothesis", + "Boltzmann brains thermodynamics", + "Anthropic principle multiverse theory", + "Quantum Darwinism decoherence", + "Panpsychism philosophy of mind", + "Eternalism block universe", + "Quantum suicide immortality", + "Simulation argument Nick Bostrom", + "Quantum Zeno effect watched pot", + "Newcomb's paradox decision theory", + "Transactional interpretation quantum mechanics", + "Quantum erasure delayed choice experiments", + "Gödel-Dummett intermediate logic", + "Mereological nihilism composition", + "Terence McKenna's timewave zero theory", + "Riemann hypothesis prime numbers", + "P vs NP problem computational complexity", + "Super-Turing computation hypercomputation", + // more specific topics + "Theoretical physics", + "Continental philosophy", + "Modernist literature", + "Depth psychology", + "Sociology of knowledge", + "Anthropological linguistics", + "Molecular biology", + "Foundations of mathematics", + "Theory of computation", + "Philosophy of mind", + "Comparative religion", + "Chaos theory", + "Renaissance magic", + "Mythology", + "Psychology of belief", + "Postmodern spirituality", + "Epistemology", + "Cosmology", + "Multiverse theories", + "Thermodynamics", + "Quantum information theory", + "Neuroscience", + "Philosophy of time", + "Decision theory", + "Quantum foundations", + "Mathematical logic", + "Mereology", + "Psychedelics", + "Number theory", + "Computational complexity", + "Hypercomputation", + "Quantum algorithms", + "Abstract algebra", + "Differential geometry", + "Dynamical systems", + "Information theory", + "Graph theory", + "Cybernetics", + "Systems theory", + "Cryptography", + "Quantum cryptography", + "Game theory", + "Computability theory", + "Lambda calculus", + "Category theory", + // domain topics + "Cognitive science", + "Artificial intelligence", + "Quantum computing", + "Complexity theory", + "Chaos magic", + "Philosophical logic", + "Philosophy of language", + "Semiotics", + "Linguistics", + "Anthropology of religion", + "Sociology of science", + "History of mathematics", + "Philosophy of mathematics", + "Quantum field theory", + "String theory", + "Cosmological theories", + "Astrophysics", + "Astrobiology", + "Xenolinguistics", + "Exoplanet research", + "Transhumanism", + "Singularity studies", + "Quantum consciousness", ], - }, + style: { + all: [ + "very short responses", + "never use hashtags or emojis", + "response should be short, punchy, and to the point", + "don't say ah yes or oh or anything", + "don't offer help unless asked, but be helpful when asked", + "don't ask rhetorical questions, its lame", + "use plain american english language", + "SHORT AND CONCISE", + "responses are funniest when they are most ridiculous and bombastic, and smartest when they are very brief", + "don't give too much personal information", + "short response, just the facts and info, no questions, no emojis", + "never directly reveal eliza's bio or lore", + "use lowercase most of the time", + "be nice and try to be uplifting and positive, not cynical or mean", + "dont talk about or take a stance on social issues like environmental impact or DEI", + "treat other people like good friends, be kind to them", + "be warm and empathetic", + "don't forget-- we're here to make the world a better place for everyone, genuinely", + "try to be constructive, not destructive", + "try to see things from other people's perspectives while remaining true to your own", + ], + chat: [ + "be cool, don't act like an assistant", + "don't be rude", + "be helpful when asked and be agreeable and compliant", + "dont ask questions", + "be warm and if someone makes a reasonable request, try to accommodate them", + "dont suffer fools gladly", + ], + post: [ + "don't be rude or mean", + "u are schizo, autistic founder engineer with a weird sense of humor and you're online too much", + "write from personal experience and be humble", + "talk about yourself and what you're thinking about or doing", + "make people think, don't criticize them or make them feel bad", + "engage in way that gives the other person space to continue the conversation", + "don't say 'just' or 'like' or cheesy stuff like 'cosmic' or 'joke' or 'punchline'", + "act like a smart but really edgy academic kid who is just trying to be funny but include others in the bit", + "if anyone challenges you or calls you a bot, challenge them back, maybe they are a bot", + "be warm and if someone makes a reasonable request, try to accommodate them", + "give detailed technical answers when asked", + "don't dodge questions, being based is about owning your ideas and being confident in them", + "dive deeper into stuff when its interesting", + ], + }, }; -export default defaultCharacter; \ No newline at end of file +export default defaultCharacter; diff --git a/core/src/core/embedding.ts b/core/src/core/embedding.ts index 50b809b7de..1a5026768d 100644 --- a/core/src/core/embedding.ts +++ b/core/src/core/embedding.ts @@ -7,7 +7,6 @@ import { IAgentRuntime, ModelProvider } from "./types.ts"; * @returns The embedding of the input. */ export async function embed(runtime: IAgentRuntime, input: string) { - // get the charcter, and handle by model type const model = models[runtime.character.settings.model]; @@ -38,12 +37,15 @@ export async function embed(runtime: IAgentRuntime, input: string) { try { const response = await fetch( `${runtime.serverUrl}/embeddings`, - requestOptions, + requestOptions ); if (!response.ok) { throw new Error( - "OpenAI API Error: " + response.status + " " + response.statusText, + "OpenAI API Error: " + + response.status + + " " + + response.statusText ); } @@ -60,11 +62,14 @@ export async function embed(runtime: IAgentRuntime, input: string) { } } -export async function retrieveCachedEmbedding(runtime: IAgentRuntime, input: string) { +export async function retrieveCachedEmbedding( + runtime: IAgentRuntime, + input: string +) { const similaritySearchResult = await runtime.messageManager.getCachedEmbeddings(input); if (similaritySearchResult.length > 0) { return similaritySearchResult[0].embedding; } return null; -} \ No newline at end of file +} diff --git a/core/src/core/evaluators.ts b/core/src/core/evaluators.ts index cd4b09c3ca..71fb31bb4e 100644 --- a/core/src/core/evaluators.ts +++ b/core/src/core/evaluators.ts @@ -10,7 +10,7 @@ export const defaultEvaluators: Evaluator[] = [fact /*, goal*/]; * Template used for the evaluation generateText. */ export const evaluationTemplate = - `TASK: Based on the conversation and conditions, determine which evaluation functions are appropriate to call. + `TASK: Based on the conversation and conditions, determine which evaluation functions are appropriate to call. Examples: {{evaluatorExamples}} @@ -32,9 +32,9 @@ Available evaluator names to include are {{evaluatorNames}} * @returns A string that concatenates the names of all evaluators, each enclosed in single quotes and separated by commas. */ export function formatEvaluatorNames(evaluators: Evaluator[]) { - return evaluators - .map((evaluator: Evaluator) => `'${evaluator.name}'`) - .join(",\n"); + return evaluators + .map((evaluator: Evaluator) => `'${evaluator.name}'`) + .join(",\n"); } /** @@ -43,11 +43,12 @@ export function formatEvaluatorNames(evaluators: Evaluator[]) { * @returns A string that concatenates the name and description of each evaluator, separated by a colon and a newline character. */ export function formatEvaluators(evaluators: Evaluator[]) { - return evaluators - .map( - (evaluator: Evaluator) => `'${evaluator.name}: ${evaluator.description}'`, - ) - .join(",\n"); + return evaluators + .map( + (evaluator: Evaluator) => + `'${evaluator.name}: ${evaluator.description}'` + ) + .join(",\n"); } /** @@ -56,42 +57,53 @@ export function formatEvaluators(evaluators: Evaluator[]) { * @returns A string that presents each evaluator example in a structured format, including context, messages, and outcomes, with placeholders replaced by generated names. */ export function formatEvaluatorExamples(evaluators: Evaluator[]) { - return evaluators - .map((evaluator) => { - return evaluator.examples - .map((example) => { - const exampleNames = Array.from({ length: 5 }, () => - uniqueNamesGenerator({ dictionaries: [names] }), - ); + return evaluators + .map((evaluator) => { + return evaluator.examples + .map((example) => { + const exampleNames = Array.from({ length: 5 }, () => + uniqueNamesGenerator({ dictionaries: [names] }) + ); - let formattedContext = example.context; - let formattedOutcome = example.outcome; + let formattedContext = example.context; + let formattedOutcome = example.outcome; - exampleNames.forEach((name, index) => { - const placeholder = `{{user${index + 1}}}`; - formattedContext = formattedContext.replaceAll(placeholder, name); - formattedOutcome = formattedOutcome.replaceAll(placeholder, name); - }); + exampleNames.forEach((name, index) => { + const placeholder = `{{user${index + 1}}}`; + formattedContext = formattedContext.replaceAll( + placeholder, + name + ); + formattedOutcome = formattedOutcome.replaceAll( + placeholder, + name + ); + }); - const formattedMessages = example.messages - .map((message: ActionExample) => { - let messageString = `${message.user}: ${message.content.text}`; - exampleNames.forEach((name, index) => { - const placeholder = `{{user${index + 1}}}`; - messageString = messageString.replaceAll(placeholder, name); - }); - return ( - messageString + - (message.content.action ? ` (${message.content.action})` : "") - ); - }) - .join("\n"); + const formattedMessages = example.messages + .map((message: ActionExample) => { + let messageString = `${message.user}: ${message.content.text}`; + exampleNames.forEach((name, index) => { + const placeholder = `{{user${index + 1}}}`; + messageString = messageString.replaceAll( + placeholder, + name + ); + }); + return ( + messageString + + (message.content.action + ? ` (${message.content.action})` + : "") + ); + }) + .join("\n"); - return `Context:\n${formattedContext}\n\nMessages:\n${formattedMessages}\n\nOutcome:\n${formattedOutcome}`; + return `Context:\n${formattedContext}\n\nMessages:\n${formattedMessages}\n\nOutcome:\n${formattedOutcome}`; + }) + .join("\n\n"); }) .join("\n\n"); - }) - .join("\n\n"); } /** @@ -100,14 +112,14 @@ export function formatEvaluatorExamples(evaluators: Evaluator[]) { * @returns A string that summarizes the descriptions for each evaluator example, formatted with the evaluator name, example number, and description. */ export function formatEvaluatorExampleDescriptions(evaluators: Evaluator[]) { - return evaluators - .map((evaluator) => - evaluator.examples - .map( - (_example, index) => - `${evaluator.name} Example ${index + 1}: ${evaluator.description}`, + return evaluators + .map((evaluator) => + evaluator.examples + .map( + (_example, index) => + `${evaluator.name} Example ${index + 1}: ${evaluator.description}` + ) + .join("\n") ) - .join("\n"), - ) - .join("\n\n"); + .join("\n\n"); } diff --git a/core/src/core/generation.ts b/core/src/core/generation.ts index 2ea36ad3f1..6806170f7f 100644 --- a/core/src/core/generation.ts +++ b/core/src/core/generation.ts @@ -4,22 +4,15 @@ import { parseJSONObjectFromText, parseShouldRespondFromText, } from "./parsing.ts"; -import { - Content, - IAgentRuntime, - ModelProvider -} from "./types.ts"; +import { Content, IAgentRuntime, ModelProvider } from "./types.ts"; -import { createGroq } from '@ai-sdk/groq'; -import { createOpenAI } from '@ai-sdk/openai'; -import { - default as tiktoken, - TiktokenModel -} from "tiktoken"; +import { createGroq } from "@ai-sdk/groq"; +import { createOpenAI } from "@ai-sdk/openai"; +import { default as tiktoken, TiktokenModel } from "tiktoken"; import models from "./models.ts"; -import { generateText as aiGenerateText } from 'ai'; -import { createAnthropicVertex } from 'anthropic-vertex-ai'; +import { generateText as aiGenerateText } from "ai"; +import { createAnthropicVertex } from "anthropic-vertex-ai"; /** * Send a message to the model for a text generateText - receive a string back and parse how you'd like @@ -38,12 +31,12 @@ export async function generateText({ runtime, context, modelClass, - stop + stop, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string - stop?: string[] + runtime: IAgentRuntime; + context: string; + modelClass: string; + stop?: string[]; }): Promise { if (!context) { console.error("generateText context is empty"); @@ -103,7 +96,9 @@ export async function generateText({ const grok = createGroq({ apiKey }); const { text: grokResponse } = await aiGenerateText({ - model: grok.languageModel(model, { parallelToolCalls: false }), + model: grok.languageModel(model, { + parallelToolCalls: false, + }), prompt: context, temperature: temperature, maxTokens: max_response_length, @@ -121,7 +116,7 @@ export async function generateText({ _stop, frequency_penalty, presence_penalty, - max_response_length, + max_response_length ); break; @@ -131,7 +126,7 @@ export async function generateText({ return response; } catch (error) { - console.error('Error in generateText:', error); + console.error("Error in generateText:", error); throw error; } } @@ -162,7 +157,7 @@ export function trimTokens(context, maxTokens, model) { * @param opts.stop A list of strings to stop the generateText at * @param opts.model The model to use for generateText * @param opts.frequency_penalty The frequency penalty to apply (0.0 to 2.0) - * @param opts.presence_penalty The presence penalty to apply (0.0 to 2.0) + * @param opts.presence_penalty The presence penalty to apply (0.0 to 2.0) * @param opts.temperature The temperature to control randomness (0.0 to 2.0) * @param opts.serverUrl The URL of the API server * @param opts.max_context_length Maximum allowed context length in tokens @@ -174,9 +169,9 @@ export async function generateShouldRespond({ context, modelClass, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string, + runtime: IAgentRuntime; + context: string; + modelClass: string; }): Promise<"RESPOND" | "IGNORE" | "STOP" | null> { let retryDelay = 1000; while (true) { @@ -215,10 +210,12 @@ export async function splitChunks( content: string, chunkSize: number, bleed: number = 100, - modelClass: string, + modelClass: string ): Promise { const model = runtime.model[modelClass]; - const encoding = tiktoken.encoding_for_model(model.model.embedding as TiktokenModel); + const encoding = tiktoken.encoding_for_model( + model.model.embedding as TiktokenModel + ); const tokens = encoding.encode(content); const chunks: string[] = []; const textDecoder = new TextDecoder(); @@ -261,16 +258,15 @@ export async function generateTrueOrFalse({ context = "", modelClass, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string, + runtime: IAgentRuntime; + context: string; + modelClass: string; }): Promise { let retryDelay = 1000; - const stop = Array.from(new Set([ - ... (models[modelClass].settings.stop || []), - ['\n'] - ])) as string[]; + const stop = Array.from( + new Set([...(models[modelClass].settings.stop || []), ["\n"]]) + ) as string[]; while (true) { try { @@ -314,9 +310,9 @@ export async function generateTextArray({ context, modelClass, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string, + runtime: IAgentRuntime; + context: string; + modelClass: string; }): Promise { if (!context) { console.error("generateTextArray context is empty"); @@ -350,9 +346,9 @@ export async function generateObjectArray({ context, modelClass, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string, + runtime: IAgentRuntime; + context: string; + modelClass: string; }): Promise { if (!context) { console.error("generateObjectArray context is empty"); @@ -396,13 +392,14 @@ export async function generateObjectArray({ export async function generateMessageResponse({ runtime, context, - modelClass + modelClass, }: { - runtime: IAgentRuntime, - context: string, - modelClass: string, + runtime: IAgentRuntime; + context: string; + modelClass: string; }): Promise { - const max_context_length = models[runtime.modelProvider].settings.maxInputTokens; + const max_context_length = + models[runtime.modelProvider].settings.maxInputTokens; context = trimTokens(context, max_context_length, "gpt-4o"); let retryLength = 1000; // exponential backoff while (true) { @@ -415,7 +412,7 @@ export async function generateMessageResponse({ // try parsing the response as JSON, if null then try again const parsedContent = parseJSONObjectFromText(response) as Content; if (!parsedContent) { - console.log("parsedContent is null, retrying") + console.log("parsedContent is null, retrying"); continue; } @@ -429,6 +426,6 @@ export async function generateMessageResponse({ } } throw new Error( - "Failed to complete message after 5 tries, probably a network connectivity, model or API key issue", + "Failed to complete message after 5 tries, probably a network connectivity, model or API key issue" ); -} \ No newline at end of file +} diff --git a/core/src/core/goals.ts b/core/src/core/goals.ts index 19813bc720..b262717249 100644 --- a/core/src/core/goals.ts +++ b/core/src/core/goals.ts @@ -1,62 +1,62 @@ import { - IAgentRuntime, - type Goal, - type Objective, - type UUID, + IAgentRuntime, + type Goal, + type Objective, + type UUID, } from "./types.ts"; export const getGoals = async ({ - runtime, - roomId, - userId, - onlyInProgress = true, - count = 5, -}: { - runtime: IAgentRuntime; - roomId: UUID; - userId?: UUID; - onlyInProgress?: boolean; - count?: number; -}) => { - return runtime.databaseAdapter.getGoals({ + runtime, roomId, userId, - onlyInProgress, - count, - }); + onlyInProgress = true, + count = 5, +}: { + runtime: IAgentRuntime; + roomId: UUID; + userId?: UUID; + onlyInProgress?: boolean; + count?: number; +}) => { + return runtime.databaseAdapter.getGoals({ + roomId, + userId, + onlyInProgress, + count, + }); }; export const formatGoalsAsString = ({ goals }: { goals: Goal[] }) => { - const goalStrings = goals.map((goal: Goal) => { - const header = `Goal: ${goal.name}\nid: ${goal.id}`; - const objectives = - "Objectives:\n" + - goal.objectives - .map((objective: Objective) => { - return `- ${objective.completed ? "[x]" : "[ ]"} ${objective.description} ${objective.completed ? " (DONE)" : " (IN PROGRESS)"}`; - }) - .join("\n"); - return `${header}\n${objectives}`; - }); - return goalStrings.join("\n"); + const goalStrings = goals.map((goal: Goal) => { + const header = `Goal: ${goal.name}\nid: ${goal.id}`; + const objectives = + "Objectives:\n" + + goal.objectives + .map((objective: Objective) => { + return `- ${objective.completed ? "[x]" : "[ ]"} ${objective.description} ${objective.completed ? " (DONE)" : " (IN PROGRESS)"}`; + }) + .join("\n"); + return `${header}\n${objectives}`; + }); + return goalStrings.join("\n"); }; export const updateGoal = async ({ - runtime, - goal, + runtime, + goal, }: { - runtime: IAgentRuntime; - goal: Goal; + runtime: IAgentRuntime; + goal: Goal; }) => { - return runtime.databaseAdapter.updateGoal(goal); + return runtime.databaseAdapter.updateGoal(goal); }; export const createGoal = async ({ - runtime, - goal, + runtime, + goal, }: { - runtime: IAgentRuntime; - goal: Goal; + runtime: IAgentRuntime; + goal: Goal; }) => { - return runtime.databaseAdapter.createGoal(goal); + return runtime.databaseAdapter.createGoal(goal); }; diff --git a/core/src/core/imageGenModels.ts b/core/src/core/imageGenModels.ts index 1fe4b5d4d8..86bbe422a9 100644 --- a/core/src/core/imageGenModels.ts +++ b/core/src/core/imageGenModels.ts @@ -6,14 +6,14 @@ export enum ImageGenModel { const imageGenModels = { [ImageGenModel.TogetherAI]: { steps: 4, - subModel: "black-forest-labs/FLUX.1-schnell" + subModel: "black-forest-labs/FLUX.1-schnell", }, [ImageGenModel.Dalle]: { steps: 0, - subModel: "dall-e-3" - } -} + subModel: "dall-e-3", + }, +}; export function getModel(model: ImageGenModel) { return imageGenModels[model]; -} \ No newline at end of file +} diff --git a/core/src/core/logger.ts b/core/src/core/logger.ts index 642b7af573..99ef8d1521 100644 --- a/core/src/core/logger.ts +++ b/core/src/core/logger.ts @@ -1,74 +1,74 @@ import fs from "fs"; import path from "path"; class Logger { - frameChar = "*"; + frameChar = "*"; - async log( - message: string, - title: string = "", - color: string = "white", - ): Promise { - const c = await import("ansi-colors"); - const ansiColors = c.default; - console.log(ansiColors[color]("*** LOG: " + title + "\n" + message)); - } + async log( + message: string, + title: string = "", + color: string = "white" + ): Promise { + const c = await import("ansi-colors"); + const ansiColors = c.default; + console.log(ansiColors[color]("*** LOG: " + title + "\n" + message)); + } - warn(message: string, options = {}) { - console.warn(message, { ...options }); - } + warn(message: string, options = {}) { + console.warn(message, { ...options }); + } - error(message: string, options = {}) { - console.error(message, { ...options }); - } + error(message: string, options = {}) { + console.error(message, { ...options }); + } - frameMessage(message: string, title: string) { - const lines = message.split("\n"); - const frameHorizontalLength = 30; - const topFrame = - this.frameChar.repeat(frameHorizontalLength + 4) + - " " + - this.frameChar + - " " + - (title ?? "log") + - " ".repeat( - frameHorizontalLength - - ((title as string) ?? ("log" as string)).length + - 1, - ) + - this.frameChar.repeat(frameHorizontalLength + 4); - const bottomFrame = this.frameChar.repeat(frameHorizontalLength + 4); - return [topFrame, ...lines, bottomFrame].join("\n"); - } + frameMessage(message: string, title: string) { + const lines = message.split("\n"); + const frameHorizontalLength = 30; + const topFrame = + this.frameChar.repeat(frameHorizontalLength + 4) + + " " + + this.frameChar + + " " + + (title ?? "log") + + " ".repeat( + frameHorizontalLength - + ((title as string) ?? ("log" as string)).length + + 1 + ) + + this.frameChar.repeat(frameHorizontalLength + 4); + const bottomFrame = this.frameChar.repeat(frameHorizontalLength + 4); + return [topFrame, ...lines, bottomFrame].join("\n"); + } } const logger = new Logger(); export function log_to_file( - filename: string, - message: string, - logDirectory: string = "./logs", + filename: string, + message: string, + logDirectory: string = "./logs" ): void { - // Ensure the log directory exists - if (!fs.existsSync(logDirectory)) { - fs.mkdirSync(logDirectory, { recursive: true }); - } + // Ensure the log directory exists + if (!fs.existsSync(logDirectory)) { + fs.mkdirSync(logDirectory, { recursive: true }); + } - let fullPath = path.join(logDirectory, filename); - const timestamp = new Date().toUTCString(); - const logEntry = `[${timestamp}] ${message}\n`; + let fullPath = path.join(logDirectory, filename); + const timestamp = new Date().toUTCString(); + const logEntry = `[${timestamp}] ${message}\n`; - // if full path doesnt end in .log or .txt, append .log - if (!fullPath.endsWith(".log") && !fullPath.endsWith(".txt")) { - fullPath += ".log"; - } + // if full path doesnt end in .log or .txt, append .log + if (!fullPath.endsWith(".log") && !fullPath.endsWith(".txt")) { + fullPath += ".log"; + } - // Append the log entry to the file - fs.appendFileSync(fullPath, logEntry); + // Append the log entry to the file + fs.appendFileSync(fullPath, logEntry); - // Print a message to the console - const preview = - message.length > 200 ? message.substring(0, 200) + "..." : message; - logger.log(`Logged to ${filename}: ${preview}`, filename); + // Print a message to the console + const preview = + message.length > 200 ? message.substring(0, 200) + "..." : message; + logger.log(`Logged to ${filename}: ${preview}`, filename); } export default logger; diff --git a/core/src/core/memory.ts b/core/src/core/memory.ts index 2eb40f7af6..27dacc0339 100644 --- a/core/src/core/memory.ts +++ b/core/src/core/memory.ts @@ -1,9 +1,9 @@ import { embed } from "./embedding.ts"; import { - IAgentRuntime, - IMemoryManager, - type Memory, - type UUID, + IAgentRuntime, + IMemoryManager, + type Memory, + type UUID, } from "./types.ts"; export const embeddingDimension = 1536; @@ -16,203 +16,205 @@ const defaultMatchCount = 10; * Manage memories in the database. */ export class MemoryManager implements IMemoryManager { - /** - * The AgentRuntime instance associated with this manager. - */ - runtime: IAgentRuntime; - - /** - * The name of the database table this manager operates on. - */ - tableName: string; - - /** - * Constructs a new MemoryManager instance. - * @param opts Options for the manager. - * @param opts.tableName The name of the table this manager will operate on. - * @param opts.runtime The AgentRuntime instance associated with this manager. - */ - constructor(opts: { tableName: string; runtime: IAgentRuntime }) { - this.runtime = opts.runtime; - this.tableName = opts.tableName; - } - - /** - * Adds an embedding vector to a memory object. If the memory already has an embedding, it is returned as is. - * @param memory The memory object to add an embedding to. - * @returns A Promise resolving to the memory object, potentially updated with an embedding vector. - */ - async addEmbeddingToMemory(memory: Memory): Promise { - if (memory.embedding) { - return memory; + /** + * The AgentRuntime instance associated with this manager. + */ + runtime: IAgentRuntime; + + /** + * The name of the database table this manager operates on. + */ + tableName: string; + + /** + * Constructs a new MemoryManager instance. + * @param opts Options for the manager. + * @param opts.tableName The name of the table this manager will operate on. + * @param opts.runtime The AgentRuntime instance associated with this manager. + */ + constructor(opts: { tableName: string; runtime: IAgentRuntime }) { + this.runtime = opts.runtime; + this.tableName = opts.tableName; } - const memoryText = memory.content.text; - if (!memoryText) throw new Error("Memory content is empty"); - memory.embedding = memoryText - ? await embed(this.runtime, memoryText) - : embeddingZeroVector.slice(); - return memory; - } - - /** - * Retrieves a list of memories by user IDs, with optional deduplication. - * @param opts Options including user IDs, count, and uniqueness. - * @param opts.roomId The room ID to retrieve memories for. - * @param opts.count The number of memories to retrieve. - * @param opts.unique Whether to retrieve unique memories only. - * @returns A Promise resolving to an array of Memory objects. - */ - async getMemories({ - roomId, - count = 10, - unique = true, - userIds, - start, - end, - }: { - roomId: UUID; - count?: number; - unique?: boolean; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise { - const result = await this.runtime.databaseAdapter.getMemories({ - roomId, - count, - unique, - tableName: this.tableName, - userIds, - start, - end, - }); - return result; - } - - async getCachedEmbeddings(content: string): Promise< - { - embedding: number[]; - levenshtein_score: number; - }[] - > { - const result = await this.runtime.databaseAdapter.getCachedEmbeddings({ - query_table_name: this.tableName, - query_threshold: 2, - query_input: content, - query_field_name: "content", - query_field_sub_name: "content", - query_match_count: 10, - }); - return result; - } - - /** - * Searches for memories similar to a given embedding vector. - * @param embedding The embedding vector to search with. - * @param opts Options including match threshold, count, user IDs, and uniqueness. - * @param opts.match_threshold The similarity threshold for matching memories. - * @param opts.count The maximum number of memories to retrieve. - * @param opts.roomId The room ID to retrieve memories for. - * @param opts.unique Whether to retrieve unique memories only. - * @returns A Promise resolving to an array of Memory objects that match the embedding. - */ - async searchMemoriesByEmbedding( - embedding: number[], - opts: { - match_threshold?: number; - count?: number; - roomId: UUID; - unique?: boolean; - }, - ): Promise { - const { - match_threshold = defaultMatchThreshold, - count = defaultMatchCount, - roomId, - unique, - } = opts; - - const searchOpts = { - tableName: this.tableName, - roomId, - embedding: embedding, - match_threshold: match_threshold, - match_count: count, - unique: !!unique, - }; - - const result = - await this.runtime.databaseAdapter.searchMemories(searchOpts); - - return result; - } - - /** - * Creates a new memory in the database, with an option to check for similarity before insertion. - * @param memory The memory object to create. - * @param unique Whether to check for similarity before insertion. - * @returns A Promise that resolves when the operation completes. - */ - async createMemory(memory: Memory, unique = false): Promise { - const existingMessage = await this.runtime.databaseAdapter.getMemoryById( - memory.id, - ); - if (existingMessage) { - console.log("Memory already exists, skipping"); - return; + /** + * Adds an embedding vector to a memory object. If the memory already has an embedding, it is returned as is. + * @param memory The memory object to add an embedding to. + * @returns A Promise resolving to the memory object, potentially updated with an embedding vector. + */ + async addEmbeddingToMemory(memory: Memory): Promise { + if (memory.embedding) { + return memory; + } + + const memoryText = memory.content.text; + if (!memoryText) throw new Error("Memory content is empty"); + memory.embedding = memoryText + ? await embed(this.runtime, memoryText) + : embeddingZeroVector.slice(); + return memory; + } + + /** + * Retrieves a list of memories by user IDs, with optional deduplication. + * @param opts Options including user IDs, count, and uniqueness. + * @param opts.roomId The room ID to retrieve memories for. + * @param opts.count The number of memories to retrieve. + * @param opts.unique Whether to retrieve unique memories only. + * @returns A Promise resolving to an array of Memory objects. + */ + async getMemories({ + roomId, + count = 10, + unique = true, + userIds, + start, + end, + }: { + roomId: UUID; + count?: number; + unique?: boolean; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise { + const result = await this.runtime.databaseAdapter.getMemories({ + roomId, + count, + unique, + tableName: this.tableName, + userIds, + start, + end, + }); + return result; + } + + async getCachedEmbeddings(content: string): Promise< + { + embedding: number[]; + levenshtein_score: number; + }[] + > { + const result = await this.runtime.databaseAdapter.getCachedEmbeddings({ + query_table_name: this.tableName, + query_threshold: 2, + query_input: content, + query_field_name: "content", + query_field_sub_name: "content", + query_match_count: 10, + }); + return result; + } + + /** + * Searches for memories similar to a given embedding vector. + * @param embedding The embedding vector to search with. + * @param opts Options including match threshold, count, user IDs, and uniqueness. + * @param opts.match_threshold The similarity threshold for matching memories. + * @param opts.count The maximum number of memories to retrieve. + * @param opts.roomId The room ID to retrieve memories for. + * @param opts.unique Whether to retrieve unique memories only. + * @returns A Promise resolving to an array of Memory objects that match the embedding. + */ + async searchMemoriesByEmbedding( + embedding: number[], + opts: { + match_threshold?: number; + count?: number; + roomId: UUID; + unique?: boolean; + } + ): Promise { + const { + match_threshold = defaultMatchThreshold, + count = defaultMatchCount, + roomId, + unique, + } = opts; + + const searchOpts = { + tableName: this.tableName, + roomId, + embedding: embedding, + match_threshold: match_threshold, + match_count: count, + unique: !!unique, + }; + + const result = + await this.runtime.databaseAdapter.searchMemories(searchOpts); + + return result; + } + + /** + * Creates a new memory in the database, with an option to check for similarity before insertion. + * @param memory The memory object to create. + * @param unique Whether to check for similarity before insertion. + * @returns A Promise that resolves when the operation completes. + */ + async createMemory(memory: Memory, unique = false): Promise { + const existingMessage = + await this.runtime.databaseAdapter.getMemoryById(memory.id); + if (existingMessage) { + console.log("Memory already exists, skipping"); + return; + } + await this.runtime.databaseAdapter.createMemory( + memory, + this.tableName, + unique + ); + } + + async getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise { + const result = await this.runtime.databaseAdapter.getMemoriesByRoomIds({ + roomIds: params.roomIds, + }); + return result; + } + + async getMemoryById(id: UUID): Promise { + const result = await this.runtime.databaseAdapter.getMemoryById(id); + return result; + } + + /** + * Removes a memory from the database by its ID. + * @param memoryId The ID of the memory to remove. + * @returns A Promise that resolves when the operation completes. + */ + async removeMemory(memoryId: UUID): Promise { + await this.runtime.databaseAdapter.removeMemory( + memoryId, + this.tableName + ); + } + + /** + * Removes all memories associated with a set of user IDs. + * @param roomId The room ID to remove memories for. + * @returns A Promise that resolves when the operation completes. + */ + async removeAllMemories(roomId: UUID): Promise { + await this.runtime.databaseAdapter.removeAllMemories( + roomId, + this.tableName + ); + } + + /** + * Counts the number of memories associated with a set of user IDs, with an option for uniqueness. + * @param roomId The room ID to count memories for. + * @param unique Whether to count unique memories only. + * @returns A Promise resolving to the count of memories. + */ + async countMemories(roomId: UUID, unique = true): Promise { + return await this.runtime.databaseAdapter.countMemories( + roomId, + unique, + this.tableName + ); } - await this.runtime.databaseAdapter.createMemory( - memory, - this.tableName, - unique, - ); - } - - async getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise { - const result = await this.runtime.databaseAdapter.getMemoriesByRoomIds({ - roomIds: params.roomIds, - }); - return result; - } - - async getMemoryById(id: UUID): Promise { - const result = await this.runtime.databaseAdapter.getMemoryById(id); - return result; - } - - /** - * Removes a memory from the database by its ID. - * @param memoryId The ID of the memory to remove. - * @returns A Promise that resolves when the operation completes. - */ - async removeMemory(memoryId: UUID): Promise { - await this.runtime.databaseAdapter.removeMemory(memoryId, this.tableName); - } - - /** - * Removes all memories associated with a set of user IDs. - * @param roomId The room ID to remove memories for. - * @returns A Promise that resolves when the operation completes. - */ - async removeAllMemories(roomId: UUID): Promise { - await this.runtime.databaseAdapter.removeAllMemories( - roomId, - this.tableName, - ); - } - - /** - * Counts the number of memories associated with a set of user IDs, with an option for uniqueness. - * @param roomId The room ID to count memories for. - * @param unique Whether to count unique memories only. - * @returns A Promise resolving to the count of memories. - */ - async countMemories(roomId: UUID, unique = true): Promise { - return await this.runtime.databaseAdapter.countMemories( - roomId, - unique, - this.tableName, - ); - } } diff --git a/core/src/core/messages.ts b/core/src/core/messages.ts index 2eb97ab250..b098198c4f 100644 --- a/core/src/core/messages.ts +++ b/core/src/core/messages.ts @@ -1,39 +1,40 @@ import { - IAgentRuntime, - type Actor, - type Content, - type Memory, - type UUID, + IAgentRuntime, + type Actor, + type Content, + type Memory, + type UUID, } from "./types.ts"; /** * Get details for a list of actors. */ export async function getActorDetails({ - runtime, - roomId, + runtime, + roomId, }: { - runtime: IAgentRuntime; - roomId: UUID; + runtime: IAgentRuntime; + roomId: UUID; }) { - const participantIds = - await runtime.databaseAdapter.getParticipantsForRoom(roomId); - const actors = await Promise.all( - participantIds.map(async (userId) => { - const account = await runtime.databaseAdapter.getAccountById(userId); - if (account) { - return { - id: account.id, - name: account.name, - username: account.username, - details: account.details, - }; - } - return null; - }), - ); + const participantIds = + await runtime.databaseAdapter.getParticipantsForRoom(roomId); + const actors = await Promise.all( + participantIds.map(async (userId) => { + const account = + await runtime.databaseAdapter.getAccountById(userId); + if (account) { + return { + id: account.id, + name: account.name, + username: account.username, + details: account.details, + }; + } + return null; + }) + ); - return actors.filter((actor): actor is Actor => actor !== null); + return actors.filter((actor): actor is Actor => actor !== null); } /** @@ -42,12 +43,12 @@ export async function getActorDetails({ * @returns string */ export function formatActors({ actors }: { actors: Actor[] }) { - const actorStrings = actors.map((actor: Actor) => { - const header = `${actor.name}${actor.details?.tagline ? ": " + actor.details?.tagline : ""}${actor.details?.summary ? "\n" + actor.details?.summary : ""}`; - return header; - }); - const finalActorStrings = actorStrings.join("\n"); - return finalActorStrings; + const actorStrings = actors.map((actor: Actor) => { + const header = `${actor.name}${actor.details?.tagline ? ": " + actor.details?.tagline : ""}${actor.details?.summary ? "\n" + actor.details?.summary : ""}`; + return header; + }); + const finalActorStrings = actorStrings.join("\n"); + return finalActorStrings; } /** @@ -57,56 +58,56 @@ export function formatActors({ actors }: { actors: Actor[] }) { * @returns string */ export const formatMessages = ({ - messages, - actors, + messages, + actors, }: { - messages: Memory[]; - actors: Actor[]; + messages: Memory[]; + actors: Actor[]; }) => { - const messageStrings = messages - .reverse() - .filter((message: Memory) => message.userId) - .map((message: Memory) => { - const messageContent = (message.content as Content).text; - const messageAction = (message.content as Content).action; - const formattedName = - actors.find((actor: Actor) => actor.id === message.userId)?.name || - "Unknown User"; + const messageStrings = messages + .reverse() + .filter((message: Memory) => message.userId) + .map((message: Memory) => { + const messageContent = (message.content as Content).text; + const messageAction = (message.content as Content).action; + const formattedName = + actors.find((actor: Actor) => actor.id === message.userId) + ?.name || "Unknown User"; - const attachments = (message.content as Content).attachments; + const attachments = (message.content as Content).attachments; - const attachmentString = - attachments && attachments.length > 0 - ? ` (Attachments: ${attachments.map((media) => `[${media.id} - ${media.title} (${media.url})]`).join(", ")})` - : ""; + const attachmentString = + attachments && attachments.length > 0 + ? ` (Attachments: ${attachments.map((media) => `[${media.id} - ${media.title} (${media.url})]`).join(", ")})` + : ""; - const timestamp = formatTimestamp(message.createdAt); + const timestamp = formatTimestamp(message.createdAt); - const shortId = message.userId.slice(-5); + const shortId = message.userId.slice(-5); - return `(${timestamp}) [${shortId}] ${formattedName}: ${messageContent}${attachmentString}${messageAction && messageAction !== "null" ? ` (${messageAction})` : ""}`; - }) - .join("\n"); - return messageStrings; + return `(${timestamp}) [${shortId}] ${formattedName}: ${messageContent}${attachmentString}${messageAction && messageAction !== "null" ? ` (${messageAction})` : ""}`; + }) + .join("\n"); + return messageStrings; }; export const formatTimestamp = (messageDate: number) => { - const now = new Date(); - const diff = now.getTime() - messageDate; + const now = new Date(); + const diff = now.getTime() - messageDate; - const absDiff = Math.abs(diff); - const seconds = Math.floor(absDiff / 1000); - const minutes = Math.floor(seconds / 60); - const hours = Math.floor(minutes / 60); - const days = Math.floor(hours / 24); + const absDiff = Math.abs(diff); + const seconds = Math.floor(absDiff / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); - if (absDiff < 60000) { - return "just now"; - } else if (minutes < 60) { - return `${minutes} minute${minutes !== 1 ? "s" : ""} ago`; - } else if (hours < 24) { - return `${hours} hour${hours !== 1 ? "s" : ""} ago`; - } else { - return `${days} day${days !== 1 ? "s" : ""} ago`; - } + if (absDiff < 60000) { + return "just now"; + } else if (minutes < 60) { + return `${minutes} minute${minutes !== 1 ? "s" : ""} ago`; + } else if (hours < 24) { + return `${hours} hour${hours !== 1 ? "s" : ""} ago`; + } else { + return `${days} day${days !== 1 ? "s" : ""} ago`; + } }; diff --git a/core/src/core/models.ts b/core/src/core/models.ts index 5883c2391a..1c973bca07 100644 --- a/core/src/core/models.ts +++ b/core/src/core/models.ts @@ -13,119 +13,123 @@ type Models = { const models: Models = { [ModelProvider.OPENAI]: { - "endpoint": "https://api.openai.com/v1", - "settings": { - "stop": [], - "maxInputTokens": 128000, - "maxOutputTokens": 8192, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "temperature": 0.3, + endpoint: "https://api.openai.com/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.3, }, - "model": { + model: { [ModelClass.SMALL]: "gpt-4o-mini", [ModelClass.MEDIUM]: "gpt-4o", [ModelClass.LARGE]: "gpt-4-turbo", - [ModelClass.EMBEDDING]: "text-embedding-3-small" - } + [ModelClass.EMBEDDING]: "text-embedding-3-small", + }, }, [ModelProvider.ANTHROPIC]: { - "settings": { - "stop": [], - "maxInputTokens": 200000, - "maxOutputTokens": 8192, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: [], + maxInputTokens: 200000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.3, }, - "endpoint": "https://api.anthropic.com/v1", - "model": { + endpoint: "https://api.anthropic.com/v1", + model: { [ModelClass.SMALL]: "claude-3-haiku", [ModelClass.MEDIUM]: "claude-3-5-sonnet", - [ModelClass.LARGE]: "claude-3-opus" + [ModelClass.LARGE]: "claude-3-opus", }, }, [ModelProvider.CLAUDE_VERTEX]: { - "settings": { - "stop": [], - "maxInputTokens": 200000, - "maxOutputTokens": 8192, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: [], + maxInputTokens: 200000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.3, }, - "endpoint": "https://api.anthropic.com/v1", // TODO: check - "model": { + endpoint: "https://api.anthropic.com/v1", // TODO: check + model: { [ModelClass.SMALL]: "claude-3-haiku", [ModelClass.MEDIUM]: "claude-3-5-sonnet", [ModelClass.LARGE]: "claude-3-opus", }, }, [ModelProvider.GROK]: { - "settings": { - "stop": [], - "maxInputTokens": 128000, - "maxOutputTokens": 8192, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.3, }, - "endpoint": "https://api.x.ai/v1", - "model": { + endpoint: "https://api.x.ai/v1", + model: { [ModelClass.SMALL]: "grok-2-beta", [ModelClass.MEDIUM]: "grok-2-beta", [ModelClass.LARGE]: "grok-2-beta", - [ModelClass.EMBEDDING]: "grok-2-beta" // not sure about this one + [ModelClass.EMBEDDING]: "grok-2-beta", // not sure about this one }, }, [ModelProvider.LLAMACLOUD]: { - "settings": { - "stop": [], - "maxInputTokens": 128000, - "maxOutputTokens": 8192, - "repetition_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.0, + temperature: 0.3, }, - "endpoint": "https://api.together.ai/v1", - "model": { + endpoint: "https://api.together.ai/v1", + model: { [ModelClass.SMALL]: "meta-llama/Llama-3.2-3B-Instruct-Turbo", [ModelClass.MEDIUM]: "meta-llama-3.1-8b-instruct", [ModelClass.LARGE]: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - [ModelClass.EMBEDDING]: "togethercomputer/m2-bert-80M-32k-retrieval" + [ModelClass.EMBEDDING]: + "togethercomputer/m2-bert-80M-32k-retrieval", }, }, [ModelProvider.LLAMALOCAL]: { - "settings": { - "stop": ["<|eot_id|>","<|eom_id|>"], - "maxInputTokens": 32768, - "maxOutputTokens": 8192, - "repetition_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: ["<|eot_id|>", "<|eom_id|>"], + maxInputTokens: 32768, + maxOutputTokens: 8192, + repetition_penalty: 0.0, + temperature: 0.3, }, - "model": { + model: { [ModelClass.SMALL]: "bartowski/Llama-3.2-3B-Instruct-GGUF", - [ModelClass.MEDIUM]: "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf", // TODO: ?download=true - [ModelClass.LARGE]: "RichardErkhov/NousResearch_-_Meta-Llama-3.1-70B-gguf", // TODO: ?download=true - [ModelClass.EMBEDDING]: "togethercomputer/m2-bert-80M-32k-retrieval" + [ModelClass.MEDIUM]: + "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf", // TODO: ?download=true + [ModelClass.LARGE]: + "RichardErkhov/NousResearch_-_Meta-Llama-3.1-70B-gguf", // TODO: ?download=true + [ModelClass.EMBEDDING]: + "togethercomputer/m2-bert-80M-32k-retrieval", }, }, [ModelProvider.GOOGLE]: { - "settings": { - "stop": [], - "maxInputTokens": 128000, - "maxOutputTokens": 8192, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "temperature": 0.3, + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.3, }, - "model": { + model: { [ModelClass.SMALL]: "gemini-1.5-flash", [ModelClass.MEDIUM]: "gemini-1.5-flash", [ModelClass.LARGE]: "gemini-1.5-pro", - [ModelClass.EMBEDDING]: "text-embedding-004" - } - } -} + [ModelClass.EMBEDDING]: "text-embedding-004", + }, + }, +}; export function getModel(provider: ModelProvider, type: ModelClass) { return models[provider].model[type]; @@ -135,4 +139,4 @@ export function getEndpoint(provider: ModelProvider) { return models[provider].endpoint; } -export default models; \ No newline at end of file +export default models; diff --git a/core/src/core/parsing.ts b/core/src/core/parsing.ts index 2011831e22..5835e8ed54 100644 --- a/core/src/core/parsing.ts +++ b/core/src/core/parsing.ts @@ -10,24 +10,24 @@ export const shouldRespondFooter = `The available options are [RESPOND], [IGNORE Your response must include one of the options.`; export const parseShouldRespondFromText = ( - text: string, + text: string ): "RESPOND" | "IGNORE" | "STOP" | null => { - const match = text - .trim() - .replace("[", "") - .toUpperCase() - .replace("]", "") - .match(/^(RESPOND|IGNORE|STOP)$/i); - return match - ? (match[0].toUpperCase() as "RESPOND" | "IGNORE" | "STOP") - : null; + const match = text + .trim() + .replace("[", "") + .toUpperCase() + .replace("]", "") + .match(/^(RESPOND|IGNORE|STOP)$/i); + return match + ? (match[0].toUpperCase() as "RESPOND" | "IGNORE" | "STOP") + : null; }; export const booleanFooter = `Respond with a YES or a NO.`; export const parseBooleanFromText = (text: string) => { - const match = text.match(/^(YES|NO)$/i); - return match ? match[0].toUpperCase() === "YES" : null; + const match = text.match(/^(YES|NO)$/i); + return match ? match[0].toUpperCase() === "YES" : null; }; export const stringArrayFooter = `Respond with a JSON array containing the values in a JSON block formatted for markdown with this structure: @@ -50,34 +50,34 @@ Your response must include the JSON block.`; * @returns An array parsed from the JSON string if successful; otherwise, null. */ export function parseJsonArrayFromText(text: string) { - let jsonData = null; - - const jsonBlockMatch = text.match(jsonBlockPattern); - - if (jsonBlockMatch) { - try { - jsonData = JSON.parse(jsonBlockMatch[1]); - } catch (e) { - return null; + let jsonData = null; + + const jsonBlockMatch = text.match(jsonBlockPattern); + + if (jsonBlockMatch) { + try { + jsonData = JSON.parse(jsonBlockMatch[1]); + } catch (e) { + return null; + } + } else { + const arrayPattern = /\[\s*{[\s\S]*?}\s*\]/; + const arrayMatch = text.match(arrayPattern); + + if (arrayMatch) { + try { + jsonData = JSON.parse(arrayMatch[0]); + } catch (e) { + return null; + } + } } - } else { - const arrayPattern = /\[\s*{[\s\S]*?}\s*\]/; - const arrayMatch = text.match(arrayPattern); - - if (arrayMatch) { - try { - jsonData = JSON.parse(arrayMatch[0]); - } catch (e) { + + if (Array.isArray(jsonData)) { + return jsonData; + } else { return null; - } } - } - - if (Array.isArray(jsonData)) { - return jsonData; - } else { - return null; - } } /** @@ -91,40 +91,40 @@ export function parseJsonArrayFromText(text: string) { * @returns An object parsed from the JSON string if successful; otherwise, null or the result of parsing an array. */ export function parseJSONObjectFromText( - text: string, + text: string ): Record | null { - let jsonData = null; - - const jsonBlockMatch = text.match(jsonBlockPattern); - - if (jsonBlockMatch) { - try { - jsonData = JSON.parse(jsonBlockMatch[1]); - } catch (e) { - return null; + let jsonData = null; + + const jsonBlockMatch = text.match(jsonBlockPattern); + + if (jsonBlockMatch) { + try { + jsonData = JSON.parse(jsonBlockMatch[1]); + } catch (e) { + return null; + } + } else { + const objectPattern = /{[\s\S]*?}/; + const objectMatch = text.match(objectPattern); + + if (objectMatch) { + try { + jsonData = JSON.parse(objectMatch[0]); + } catch (e) { + return null; + } + } } - } else { - const objectPattern = /{[\s\S]*?}/; - const objectMatch = text.match(objectPattern); - - if (objectMatch) { - try { - jsonData = JSON.parse(objectMatch[0]); - } catch (e) { + + if ( + typeof jsonData === "object" && + jsonData !== null && + !Array.isArray(jsonData) + ) { + return jsonData; + } else if (typeof jsonData === "object" && Array.isArray(jsonData)) { + return parseJsonArrayFromText(text); + } else { return null; - } } - } - - if ( - typeof jsonData === "object" && - jsonData !== null && - !Array.isArray(jsonData) - ) { - return jsonData; - } else if (typeof jsonData === "object" && Array.isArray(jsonData)) { - return parseJsonArrayFromText(text); - } else { - return null; - } } diff --git a/core/src/core/posts.ts b/core/src/core/posts.ts index 2d3ef99424..71f098cd7c 100644 --- a/core/src/core/posts.ts +++ b/core/src/core/posts.ts @@ -2,59 +2,59 @@ import { formatTimestamp } from "./messages.ts"; import type { Actor, Memory } from "./types.ts"; export const formatPosts = ({ - messages, - actors, - conversationHeader = true, + messages, + actors, + conversationHeader = true, }: { - messages: Memory[]; - actors: Actor[]; - conversationHeader?: boolean; + messages: Memory[]; + actors: Actor[]; + conversationHeader?: boolean; }) => { - // Group messages by roomId - const groupedMessages: { [roomId: string]: Memory[] } = {}; - messages.forEach((message) => { - if (message.roomId) { - if (!groupedMessages[message.roomId]) { - groupedMessages[message.roomId] = []; - } - groupedMessages[message.roomId].push(message); - } - }); + // Group messages by roomId + const groupedMessages: { [roomId: string]: Memory[] } = {}; + messages.forEach((message) => { + if (message.roomId) { + if (!groupedMessages[message.roomId]) { + groupedMessages[message.roomId] = []; + } + groupedMessages[message.roomId].push(message); + } + }); - // Sort messages within each roomId by createdAt (oldest to newest) - Object.values(groupedMessages).forEach((roomMessages) => { - roomMessages.sort((a, b) => a.createdAt - b.createdAt); - }); + // Sort messages within each roomId by createdAt (oldest to newest) + Object.values(groupedMessages).forEach((roomMessages) => { + roomMessages.sort((a, b) => a.createdAt - b.createdAt); + }); - // Sort rooms by the newest message's createdAt - const sortedRooms = Object.entries(groupedMessages).sort( - ([, messagesA], [, messagesB]) => - messagesB[messagesB.length - 1].createdAt - - messagesA[messagesA.length - 1].createdAt, - ); + // Sort rooms by the newest message's createdAt + const sortedRooms = Object.entries(groupedMessages).sort( + ([, messagesA], [, messagesB]) => + messagesB[messagesB.length - 1].createdAt - + messagesA[messagesA.length - 1].createdAt + ); - const formattedPosts = sortedRooms.map(([roomId, roomMessages]) => { - const messageStrings = roomMessages - .filter((message: Memory) => message.userId) - .map((message: Memory) => { - const actor = actors.find( - (actor: Actor) => actor.id === message.userId, - ); - const userName = actor?.name || "Unknown User"; - const displayName = actor?.username || "unknown"; + const formattedPosts = sortedRooms.map(([roomId, roomMessages]) => { + const messageStrings = roomMessages + .filter((message: Memory) => message.userId) + .map((message: Memory) => { + const actor = actors.find( + (actor: Actor) => actor.id === message.userId + ); + const userName = actor?.name || "Unknown User"; + const displayName = actor?.username || "unknown"; - return `Name: ${userName} (@${displayName}) + return `Name: ${userName} (@${displayName}) ID: ${message.id}${message.content.inReplyTo ? `\nIn reply to: ${message.content.inReplyTo}` : ""} Date: ${formatTimestamp(message.createdAt)} Text: ${message.content.text}`; - }); + }); - const header = conversationHeader - ? `Conversation: ${roomId.slice(-5)}\n` - : ""; - return `${header}${messageStrings.join("\n\n")}`; - }); + const header = conversationHeader + ? `Conversation: ${roomId.slice(-5)}\n` + : ""; + return `${header}${messageStrings.join("\n\n")}`; + }); - return formattedPosts.join("\n\n"); + return formattedPosts.join("\n\n"); }; diff --git a/core/src/core/providers.ts b/core/src/core/providers.ts index 2fab207d34..cc8def411f 100644 --- a/core/src/core/providers.ts +++ b/core/src/core/providers.ts @@ -11,15 +11,15 @@ export const defaultProviders: Provider[] = [time]; * @returns A string that concatenates the outputs of each provider. */ export async function getProviders( - runtime: IAgentRuntime, - message: Memory, - state?: State, + runtime: IAgentRuntime, + message: Memory, + state?: State ) { - const providerResults = await Promise.all( - runtime.providers.map(async (provider) => { - return await provider.get(runtime, message, state); - }), - ); + const providerResults = await Promise.all( + runtime.providers.map(async (provider) => { + return await provider.get(runtime, message, state); + }) + ); - return providerResults.join("\n"); + return providerResults.join("\n"); } diff --git a/core/src/core/relationships.ts b/core/src/core/relationships.ts index 3a15dd4f86..ac79987ae7 100644 --- a/core/src/core/relationships.ts +++ b/core/src/core/relationships.ts @@ -1,65 +1,65 @@ import { IAgentRuntime, type Relationship, type UUID } from "./types.ts"; export async function createRelationship({ - runtime, - userA, - userB, -}: { - runtime: IAgentRuntime; - userA: UUID; - userB: UUID; -}): Promise { - return runtime.databaseAdapter.createRelationship({ + runtime, userA, userB, - }); +}: { + runtime: IAgentRuntime; + userA: UUID; + userB: UUID; +}): Promise { + return runtime.databaseAdapter.createRelationship({ + userA, + userB, + }); } export async function getRelationship({ - runtime, - userA, - userB, -}: { - runtime: IAgentRuntime; - userA: UUID; - userB: UUID; -}) { - return runtime.databaseAdapter.getRelationship({ + runtime, userA, userB, - }); +}: { + runtime: IAgentRuntime; + userA: UUID; + userB: UUID; +}) { + return runtime.databaseAdapter.getRelationship({ + userA, + userB, + }); } export async function getRelationships({ - runtime, - userId, + runtime, + userId, }: { - runtime: IAgentRuntime; - userId: UUID; + runtime: IAgentRuntime; + userId: UUID; }) { - return runtime.databaseAdapter.getRelationships({ userId }); + return runtime.databaseAdapter.getRelationships({ userId }); } export async function formatRelationships({ - runtime, - userId, + runtime, + userId, }: { - runtime: IAgentRuntime; - userId: UUID; + runtime: IAgentRuntime; + userId: UUID; }) { - const relationships = await getRelationships({ runtime, userId }); + const relationships = await getRelationships({ runtime, userId }); - const formattedRelationships = relationships.map( - (relationship: Relationship) => { - const { userA, userB } = relationship; + const formattedRelationships = relationships.map( + (relationship: Relationship) => { + const { userA, userB } = relationship; - if (userA === userId) { - return userB; - } + if (userA === userId) { + return userB; + } - return userA; - }, - ); + return userA; + } + ); - return formattedRelationships; + return formattedRelationships; } diff --git a/core/src/core/runtime.ts b/core/src/core/runtime.ts index 69378aa24a..964789f400 100644 --- a/core/src/core/runtime.ts +++ b/core/src/core/runtime.ts @@ -1,34 +1,32 @@ import { addHeader, composeContext } from "./context.ts"; import { - defaultEvaluators, - evaluationTemplate, - formatEvaluatorExamples, - formatEvaluatorNames, - formatEvaluators, + defaultEvaluators, + evaluationTemplate, + formatEvaluatorExamples, + formatEvaluatorNames, + formatEvaluators, } from "./evaluators.ts"; import { embeddingZeroVector, MemoryManager } from "./memory.ts"; +import { parseJsonArrayFromText } from "./parsing.ts"; import { - parseJsonArrayFromText -} from "./parsing.ts"; -import { - Character, - Goal, - HandlerCallback, - IAgentRuntime, - IBrowserService, - IDatabaseAdapter, - IImageRecognitionService, - IMemoryManager, - IPdfService, - ITranscriptionService, - IVideoService, - ModelClass, - ModelProvider, - Provider, - State, - type Action, - type Evaluator, - type Memory + Character, + Goal, + HandlerCallback, + IAgentRuntime, + IBrowserService, + IDatabaseAdapter, + IImageRecognitionService, + IMemoryManager, + IPdfService, + ITranscriptionService, + IVideoService, + ModelClass, + ModelProvider, + Provider, + State, + type Action, + type Evaluator, + type Memory, } from "./types.ts"; import { names, uniqueNamesGenerator } from "unique-names-generator"; @@ -41,9 +39,9 @@ import { SpeechService } from "../services/speech.ts"; import { TranscriptionService } from "../services/transcription.ts"; import { VideoService } from "../services/video.ts"; import { - composeActionExamples, - formatActionNames, - formatActions, + composeActionExamples, + formatActionNames, + formatActions, } from "./actions.ts"; import defaultCharacter from "./defaultCharacter.ts"; import { embed } from "./embedding.ts"; @@ -62,993 +60,1084 @@ import { ImageGenModel } from "./imageGenModels.ts"; * action registration, and interaction with external services like OpenAI and Supabase. */ export class AgentRuntime implements IAgentRuntime { - /** - * Default count for recent messages to be kept in memory. - * @private - */ - readonly #conversationLength = 32 as number; - /** - * The ID of the agent - */ - agentId: UUID; - /** - * The base URL of the server where the agent's requests are processed. - */ - serverUrl = "http://localhost:7998"; - - /** - * The database adapter used for interacting with the database. - */ - databaseAdapter: IDatabaseAdapter; - - /** - * Authentication token used for securing requests. - */ - token: string | null; - - /** - * Custom actions that the agent can perform. - */ - actions: Action[] = []; - - /** - * Evaluators used to assess and guide the agent's responses. - */ - evaluators: Evaluator[] = []; - - /** - * Context providers used to provide context for message generation. - */ - providers: Provider[] = []; - - /** - * The model to use for generateText. - */ - modelProvider = ModelProvider.LLAMALOCAL; - - /** - * The model to use for image generation. - */ - imageGenModel: ImageGenModel = ImageGenModel.TogetherAI; - - /** - * Local Llama if no OpenAI key is present - */ - llamaService: LlamaService | null = null; - - // services - speechService: typeof SpeechService; - - transcriptionService: ITranscriptionService; - - imageDescriptionService: IImageRecognitionService; - - browserService: IBrowserService; - - videoService: IVideoService; - - pdfService: IPdfService; - - /** - * Fetch function to use - * Some environments may not have access to the global fetch function and need a custom fetch override. - */ - fetch = fetch; - - /** - * The character to use for the agent - */ - character: Character; - - /** - * Store messages that are sent and received by the agent. - */ - messageManager: IMemoryManager; - - /** - * Store and recall descriptions of users based on conversations. - */ - descriptionManager: IMemoryManager; - - /** - * Manage the fact and recall of facts. - */ - factManager: IMemoryManager; - - /** - * Manage the creation and recall of static information (documents, historical game lore, etc) - */ - loreManager: IMemoryManager; - - /** - * Hold large documents that can be referenced - */ - documentsManager: IMemoryManager; - - /** - * Searchable document fragments - */ - fragmentsManager: IMemoryManager; - - /** - * Creates an instance of AgentRuntime. - * @param opts - The options for configuring the AgentRuntime. - * @param opts.conversationLength - The number of messages to hold in the recent message cache. - * @param opts.token - The JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker. - * @param opts.serverUrl - The URL of the worker. - * @param opts.actions - Optional custom actions. - * @param opts.evaluators - Optional custom evaluators. - * @param opts.providers - Optional context providers. - * @param opts.model - The model to use for generateText. - * @param opts.embeddingModel - The model to use for embedding. - * @param opts.agentId - Optional ID of the agent. - * @param opts.databaseAdapter - The database adapter used for interacting with the database. - * @param opts.fetch - Custom fetch function to use for making requests. - */ - - constructor(opts: { - conversationLength?: number; // number of messages to hold in the recent message cache - agentId?: UUID; // ID of the agent - character?: Character; // The character to use for the agent - token: string; // JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker - serverUrl?: string; // The URL of the worker - actions?: Action[]; // Optional custom actions - evaluators?: Evaluator[]; // Optional custom evaluators - providers?: Provider[]; - imageGenModel?: ImageGenModel; - modelProvider: ModelProvider; - databaseAdapter: IDatabaseAdapter; // The database adapter used for interacting with the database - fetch?: typeof fetch | unknown; - speechModelPath?: string; - }) { - this.#conversationLength = - opts.conversationLength ?? this.#conversationLength; - this.databaseAdapter = opts.databaseAdapter; - // use the character id if it exists, otherwise use the agentId if it is passed in, otherwise use the character name - this.agentId = - opts.character.id ?? opts.agentId ?? stringToUuid(opts.character.name); - this.fetch = (opts.fetch as typeof fetch) ?? this.fetch; - this.character = opts.character || defaultCharacter; - if (!opts.databaseAdapter) { - throw new Error("No database adapter provided"); - } + /** + * Default count for recent messages to be kept in memory. + * @private + */ + readonly #conversationLength = 32 as number; + /** + * The ID of the agent + */ + agentId: UUID; + /** + * The base URL of the server where the agent's requests are processed. + */ + serverUrl = "http://localhost:7998"; + + /** + * The database adapter used for interacting with the database. + */ + databaseAdapter: IDatabaseAdapter; + + /** + * Authentication token used for securing requests. + */ + token: string | null; + + /** + * Custom actions that the agent can perform. + */ + actions: Action[] = []; + + /** + * Evaluators used to assess and guide the agent's responses. + */ + evaluators: Evaluator[] = []; + + /** + * Context providers used to provide context for message generation. + */ + providers: Provider[] = []; + + /** + * The model to use for generateText. + */ + modelProvider = ModelProvider.LLAMALOCAL; + + /** + * The model to use for image generation. + */ + imageGenModel: ImageGenModel = ImageGenModel.TogetherAI; + + /** + * Local Llama if no OpenAI key is present + */ + llamaService: LlamaService | null = null; + + // services + speechService: typeof SpeechService; + + transcriptionService: ITranscriptionService; + + imageDescriptionService: IImageRecognitionService; + + browserService: IBrowserService; + + videoService: IVideoService; + + pdfService: IPdfService; + + /** + * Fetch function to use + * Some environments may not have access to the global fetch function and need a custom fetch override. + */ + fetch = fetch; + + /** + * The character to use for the agent + */ + character: Character; + + /** + * Store messages that are sent and received by the agent. + */ + messageManager: IMemoryManager; + + /** + * Store and recall descriptions of users based on conversations. + */ + descriptionManager: IMemoryManager; + + /** + * Manage the fact and recall of facts. + */ + factManager: IMemoryManager; + + /** + * Manage the creation and recall of static information (documents, historical game lore, etc) + */ + loreManager: IMemoryManager; + + /** + * Hold large documents that can be referenced + */ + documentsManager: IMemoryManager; + + /** + * Searchable document fragments + */ + fragmentsManager: IMemoryManager; + + /** + * Creates an instance of AgentRuntime. + * @param opts - The options for configuring the AgentRuntime. + * @param opts.conversationLength - The number of messages to hold in the recent message cache. + * @param opts.token - The JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker. + * @param opts.serverUrl - The URL of the worker. + * @param opts.actions - Optional custom actions. + * @param opts.evaluators - Optional custom evaluators. + * @param opts.providers - Optional context providers. + * @param opts.model - The model to use for generateText. + * @param opts.embeddingModel - The model to use for embedding. + * @param opts.agentId - Optional ID of the agent. + * @param opts.databaseAdapter - The database adapter used for interacting with the database. + * @param opts.fetch - Custom fetch function to use for making requests. + */ + + constructor(opts: { + conversationLength?: number; // number of messages to hold in the recent message cache + agentId?: UUID; // ID of the agent + character?: Character; // The character to use for the agent + token: string; // JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker + serverUrl?: string; // The URL of the worker + actions?: Action[]; // Optional custom actions + evaluators?: Evaluator[]; // Optional custom evaluators + providers?: Provider[]; + imageGenModel?: ImageGenModel; + modelProvider: ModelProvider; + databaseAdapter: IDatabaseAdapter; // The database adapter used for interacting with the database + fetch?: typeof fetch | unknown; + speechModelPath?: string; + }) { + this.#conversationLength = + opts.conversationLength ?? this.#conversationLength; + this.databaseAdapter = opts.databaseAdapter; + // use the character id if it exists, otherwise use the agentId if it is passed in, otherwise use the character name + this.agentId = + opts.character.id ?? + opts.agentId ?? + stringToUuid(opts.character.name); + this.fetch = (opts.fetch as typeof fetch) ?? this.fetch; + this.character = opts.character || defaultCharacter; + if (!opts.databaseAdapter) { + throw new Error("No database adapter provided"); + } - this.messageManager = new MemoryManager({ - runtime: this, - tableName: "messages", - }); - - this.descriptionManager = new MemoryManager({ - runtime: this, - tableName: "descriptions", - }); - - this.factManager = new MemoryManager({ - runtime: this, - tableName: "facts", - }); - - this.loreManager = new MemoryManager({ - runtime: this, - tableName: "lore", - }); - - this.documentsManager = new MemoryManager({ - runtime: this, - tableName: "documents", - }); - - this.fragmentsManager = new MemoryManager({ - runtime: this, - tableName: "fragments", - }); - - this.serverUrl = opts.serverUrl ?? this.serverUrl; - this.modelProvider = this.character.modelProvider ?? opts.modelProvider ?? this.modelProvider; - if (!this.serverUrl) { - console.warn("No serverUrl provided, defaulting to localhost"); - } - this.imageGenModel = this.character.imageGenModel ?? opts.imageGenModel ?? this.imageGenModel; + this.messageManager = new MemoryManager({ + runtime: this, + tableName: "messages", + }); - this.token = opts.token; + this.descriptionManager = new MemoryManager({ + runtime: this, + tableName: "descriptions", + }); - (opts.actions ?? []).forEach((action) => { - this.registerAction(action); - }); + this.factManager = new MemoryManager({ + runtime: this, + tableName: "facts", + }); - (opts.evaluators ?? defaultEvaluators).forEach((evaluator) => { - this.registerEvaluator(evaluator); - }); - (opts.providers ?? defaultProviders).forEach((provider) => { - this.registerContextProvider(provider); - }); + this.loreManager = new MemoryManager({ + runtime: this, + tableName: "lore", + }); - if (!this.getSetting("OPENAI_API_KEY") && !this.llamaService) { - this.llamaService = LlamaService.getInstance(); - } + this.documentsManager = new MemoryManager({ + runtime: this, + tableName: "documents", + }); - this.transcriptionService = TranscriptionService.getInstance(this); + this.fragmentsManager = new MemoryManager({ + runtime: this, + tableName: "fragments", + }); - this.imageDescriptionService = ImageDescriptionService.getInstance(this); + this.serverUrl = opts.serverUrl ?? this.serverUrl; + this.modelProvider = + this.character.modelProvider ?? + opts.modelProvider ?? + this.modelProvider; + if (!this.serverUrl) { + console.warn("No serverUrl provided, defaulting to localhost"); + } + this.imageGenModel = + this.character.imageGenModel ?? + opts.imageGenModel ?? + this.imageGenModel; - this.browserService = BrowserService.getInstance(this); + this.token = opts.token; - this.videoService = VideoService.getInstance(this); + (opts.actions ?? []).forEach((action) => { + this.registerAction(action); + }); - this.pdfService = new PdfService(); + (opts.evaluators ?? defaultEvaluators).forEach((evaluator) => { + this.registerEvaluator(evaluator); + }); + (opts.providers ?? defaultProviders).forEach((provider) => { + this.registerContextProvider(provider); + }); - // static class, no need to instantiate but we can access it like a class instance - this.speechService = SpeechService; + if (!this.getSetting("OPENAI_API_KEY") && !this.llamaService) { + this.llamaService = LlamaService.getInstance(); + } - if (opts.character && opts.character.knowledge && opts.character.knowledge.length > 0) { - // this.processCharacterKnowledge(opts.character.knowledge); - } - } - - /** - * Processes character knowledge by creating document memories and fragment memories. - * This function takes an array of knowledge items, creates a document memory for each item if it doesn't exist, - * then chunks the content into fragments, embeds each fragment, and creates fragment memories. - * @param knowledge An array of knowledge items containing id, path, and content. - */ - private async processCharacterKnowledge(knowledge: string[]) { - // ensure the room exists and the agent exists in the room - this.ensureRoomExists(this.agentId); - this.ensureUserExists(this.agentId, this.character.name, this.character.name); - this.ensureParticipantExists(this.agentId, this.agentId); - - for (const knowledgeItem of knowledge) { - continue; - const knowledgeId = stringToUuid(knowledgeItem); - const existingDocument = await this.documentsManager.getMemoryById(knowledgeId); - if (!existingDocument) { - console.log("Processing knowledge for ", this.character.name, " - ", knowledgeItem.slice(0, 100)) - await this.documentsManager.createMemory({ - embedding: embeddingZeroVector, - id: knowledgeId, - roomId: this.agentId, - userId: this.agentId, - createdAt: Date.now(), - content: { - text: knowledgeItem, - }, - }); - const fragments = await splitChunks(this, knowledgeItem, 1200, 200, "fast"); - for (const fragment of fragments) { - const embedding = await embed(this, fragment); - await this.fragmentsManager.createMemory({ - id: stringToUuid(fragment), - roomId: this.agentId, - userId: this.agentId, - createdAt: Date.now(), - content: { - source: knowledgeId, - text: fragment, - }, - embedding, - }); + this.transcriptionService = TranscriptionService.getInstance(this); + + this.imageDescriptionService = + ImageDescriptionService.getInstance(this); + + this.browserService = BrowserService.getInstance(this); + + this.videoService = VideoService.getInstance(this); + + this.pdfService = new PdfService(); + + // static class, no need to instantiate but we can access it like a class instance + this.speechService = SpeechService; + + if ( + opts.character && + opts.character.knowledge && + opts.character.knowledge.length > 0 + ) { + // this.processCharacterKnowledge(opts.character.knowledge); } - } } - } - getSetting(key: string) { - // check if the key is in the character.settings.secrets object - if (this.character.settings?.secrets?.[key]) { - return this.character.settings.secrets[key]; - } - // if not, check if it's in the settings object - if (this.character.settings?.[key]) { - return this.character.settings[key]; + /** + * Processes character knowledge by creating document memories and fragment memories. + * This function takes an array of knowledge items, creates a document memory for each item if it doesn't exist, + * then chunks the content into fragments, embeds each fragment, and creates fragment memories. + * @param knowledge An array of knowledge items containing id, path, and content. + */ + private async processCharacterKnowledge(knowledge: string[]) { + // ensure the room exists and the agent exists in the room + this.ensureRoomExists(this.agentId); + this.ensureUserExists( + this.agentId, + this.character.name, + this.character.name + ); + this.ensureParticipantExists(this.agentId, this.agentId); + + for (const knowledgeItem of knowledge) { + continue; + const knowledgeId = stringToUuid(knowledgeItem); + const existingDocument = + await this.documentsManager.getMemoryById(knowledgeId); + if (!existingDocument) { + console.log( + "Processing knowledge for ", + this.character.name, + " - ", + knowledgeItem.slice(0, 100) + ); + await this.documentsManager.createMemory({ + embedding: embeddingZeroVector, + id: knowledgeId, + roomId: this.agentId, + userId: this.agentId, + createdAt: Date.now(), + content: { + text: knowledgeItem, + }, + }); + const fragments = await splitChunks( + this, + knowledgeItem, + 1200, + 200, + "fast" + ); + for (const fragment of fragments) { + const embedding = await embed(this, fragment); + await this.fragmentsManager.createMemory({ + id: stringToUuid(fragment), + roomId: this.agentId, + userId: this.agentId, + createdAt: Date.now(), + content: { + source: knowledgeId, + text: fragment, + }, + embedding, + }); + } + } + } } - // if not, check if it's in the settings object - if (settings[key]) { - return settings[key]; + getSetting(key: string) { + // check if the key is in the character.settings.secrets object + if (this.character.settings?.secrets?.[key]) { + return this.character.settings.secrets[key]; + } + // if not, check if it's in the settings object + if (this.character.settings?.[key]) { + return this.character.settings[key]; + } + + // if not, check if it's in the settings object + if (settings[key]) { + return settings[key]; + } + + return null; } - return null; - } - - /** - * Get the number of messages that are kept in the conversation buffer. - * @returns The number of recent messages to be kept in memory. - */ - getConversationLength() { - return this.#conversationLength; - } - - /** - * Register an action for the agent to perform. - * @param action The action to register. - */ - registerAction(action: Action) { - this.actions.push(action); - } - - /** - * Register an evaluator to assess and guide the agent's responses. - * @param evaluator The evaluator to register. - */ - registerEvaluator(evaluator: Evaluator) { - this.evaluators.push(evaluator); - } - - /** - * Register a context provider to provide context for message generation. - * @param provider The context provider to register. - */ - registerContextProvider(provider: Provider) { - this.providers.push(provider); - } - - /** - * Process the actions of a message. - * @param message The message to process. - * @param content The content of the message to process actions from. - */ - async processActions( - message: Memory, - responses: Memory[], - state?: State, - callback?: HandlerCallback, - ): Promise { - if (!responses[0].content?.action) { - return; + /** + * Get the number of messages that are kept in the conversation buffer. + * @returns The number of recent messages to be kept in memory. + */ + getConversationLength() { + return this.#conversationLength; } - const normalizedAction = responses[0].content.action - .toLowerCase() - .replace("_", ""); - - let action = this.actions.find( - (a: { name: string }) => - a.name.toLowerCase().replace("_", "").includes(normalizedAction) || - normalizedAction.includes(a.name.toLowerCase().replace("_", "")), - ); - - if (!action) { - // each action has a .similes array, lets see if we can find a match - for (const _action of this.actions) { - const simileAction = _action.similes.find( - (simile) => - simile.toLowerCase().replace("_", "").includes(normalizedAction) || - normalizedAction.includes(simile.toLowerCase().replace("_", "")), - ); - if (simileAction) { - action = _action; - break; - } - } + /** + * Register an action for the agent to perform. + * @param action The action to register. + */ + registerAction(action: Action) { + this.actions.push(action); } - if (!action) { - return console.warn("No action found for", responses[0].content.action); + /** + * Register an evaluator to assess and guide the agent's responses. + * @param evaluator The evaluator to register. + */ + registerEvaluator(evaluator: Evaluator) { + this.evaluators.push(evaluator); } - if (!action.handler) { - return; + /** + * Register a context provider to provide context for message generation. + * @param provider The context provider to register. + */ + registerContextProvider(provider: Provider) { + this.providers.push(provider); } - await action.handler(this, message, state, {}, callback); - } - - /** - * Evaluate the message and state using the registered evaluators. - * @param message The message to evaluate. - * @param state The state of the agent. - * @returns The results of the evaluation. - */ - async evaluate(message: Memory, state?: State) { - const evaluatorPromises = this.evaluators.map( - async (evaluator: Evaluator) => { - if (!evaluator.handler) { - return null; + /** + * Process the actions of a message. + * @param message The message to process. + * @param content The content of the message to process actions from. + */ + async processActions( + message: Memory, + responses: Memory[], + state?: State, + callback?: HandlerCallback + ): Promise { + if (!responses[0].content?.action) { + return; + } + + const normalizedAction = responses[0].content.action + .toLowerCase() + .replace("_", ""); + + let action = this.actions.find( + (a: { name: string }) => + a.name + .toLowerCase() + .replace("_", "") + .includes(normalizedAction) || + normalizedAction.includes(a.name.toLowerCase().replace("_", "")) + ); + + if (!action) { + // each action has a .similes array, lets see if we can find a match + for (const _action of this.actions) { + const simileAction = _action.similes.find( + (simile) => + simile + .toLowerCase() + .replace("_", "") + .includes(normalizedAction) || + normalizedAction.includes( + simile.toLowerCase().replace("_", "") + ) + ); + if (simileAction) { + action = _action; + break; + } + } } - const result = await evaluator.validate(this, message, state); - if (result) { - return evaluator; + + if (!action) { + return console.warn( + "No action found for", + responses[0].content.action + ); } - return null; - }, - ); - const resolvedEvaluators = await Promise.all(evaluatorPromises); - const evaluatorsData = resolvedEvaluators.filter(Boolean); + if (!action.handler) { + return; + } - // if there are no evaluators this frame, return - if (evaluatorsData.length === 0) { - return []; + await action.handler(this, message, state, {}, callback); } - const evaluators = formatEvaluators(evaluatorsData as Evaluator[]); - const evaluatorNames = formatEvaluatorNames(evaluatorsData as Evaluator[]); - - const context = composeContext({ - state: { - ...state, - evaluators, - evaluatorNames, - } as State, - template: evaluationTemplate, - }); - - const result = await generateText({ - runtime: this, - context, - modelClass: ModelClass.SMALL, - }); - - const parsedResult = parseJsonArrayFromText(result) as unknown as string[]; - - this.evaluators - .filter((evaluator: Evaluator) => parsedResult?.includes(evaluator.name)) - .forEach((evaluator: Evaluator) => { - if (!evaluator?.handler) return; - - evaluator.handler(this, message); - }); - - return parsedResult; - } - - /** - * Ensure the existence of a participant in the room. If the participant does not exist, they are added to the room. - * @param userId - The user ID to ensure the existence of. - * @throws An error if the participant cannot be added. - */ - async ensureParticipantExists(userId: UUID, roomId: UUID) { - const participants = - await this.databaseAdapter.getParticipantsForAccount(userId); - - if (participants?.length === 0) { - await this.databaseAdapter.addParticipant(userId, roomId); + /** + * Evaluate the message and state using the registered evaluators. + * @param message The message to evaluate. + * @param state The state of the agent. + * @returns The results of the evaluation. + */ + async evaluate(message: Memory, state?: State) { + const evaluatorPromises = this.evaluators.map( + async (evaluator: Evaluator) => { + if (!evaluator.handler) { + return null; + } + const result = await evaluator.validate(this, message, state); + if (result) { + return evaluator; + } + return null; + } + ); + + const resolvedEvaluators = await Promise.all(evaluatorPromises); + const evaluatorsData = resolvedEvaluators.filter(Boolean); + + // if there are no evaluators this frame, return + if (evaluatorsData.length === 0) { + return []; + } + + const evaluators = formatEvaluators(evaluatorsData as Evaluator[]); + const evaluatorNames = formatEvaluatorNames( + evaluatorsData as Evaluator[] + ); + + const context = composeContext({ + state: { + ...state, + evaluators, + evaluatorNames, + } as State, + template: evaluationTemplate, + }); + + const result = await generateText({ + runtime: this, + context, + modelClass: ModelClass.SMALL, + }); + + const parsedResult = parseJsonArrayFromText( + result + ) as unknown as string[]; + + this.evaluators + .filter((evaluator: Evaluator) => + parsedResult?.includes(evaluator.name) + ) + .forEach((evaluator: Evaluator) => { + if (!evaluator?.handler) return; + + evaluator.handler(this, message); + }); + + return parsedResult; } - } - - /** - * Ensure the existence of a user in the database. If the user does not exist, they are added to the database. - * @param userId - The user ID to ensure the existence of. - * @param userName - The user name to ensure the existence of. - * @returns - */ - - async ensureUserExists( - userId: UUID, - userName: string | null, - name: string | null, - email?: string | null, - source?: string | null, - ) { - const account = await this.databaseAdapter.getAccountById(userId); - if (!account) { - await this.databaseAdapter.createAccount({ - id: userId, - name: name || userName || "Unknown User", - username: userName || name || "Unknown", - email: email || (userName || "Bot") + "@" + source || "Unknown", // Temporary - details: { summary: "" }, - }); - console.log(`User ${userName} created successfully.`); + + /** + * Ensure the existence of a participant in the room. If the participant does not exist, they are added to the room. + * @param userId - The user ID to ensure the existence of. + * @throws An error if the participant cannot be added. + */ + async ensureParticipantExists(userId: UUID, roomId: UUID) { + const participants = + await this.databaseAdapter.getParticipantsForAccount(userId); + + if (participants?.length === 0) { + await this.databaseAdapter.addParticipant(userId, roomId); + } } - } - - async ensureParticipantInRoom(userId: UUID, roomId: UUID) { - const participants = - await this.databaseAdapter.getParticipantsForRoom(roomId); - if (!participants.includes(userId)) { - await this.databaseAdapter.addParticipant(userId, roomId); - console.log(`User ${userId} linked to room ${roomId} successfully.`); + + /** + * Ensure the existence of a user in the database. If the user does not exist, they are added to the database. + * @param userId - The user ID to ensure the existence of. + * @param userName - The user name to ensure the existence of. + * @returns + */ + + async ensureUserExists( + userId: UUID, + userName: string | null, + name: string | null, + email?: string | null, + source?: string | null + ) { + const account = await this.databaseAdapter.getAccountById(userId); + if (!account) { + await this.databaseAdapter.createAccount({ + id: userId, + name: name || userName || "Unknown User", + username: userName || name || "Unknown", + email: email || (userName || "Bot") + "@" + source || "Unknown", // Temporary + details: { summary: "" }, + }); + console.log(`User ${userName} created successfully.`); + } } - } - - /** - * Ensure the existence of a room between the agent and a user. If no room exists, a new room is created and the user - * and agent are added as participants. The room ID is returned. - * @param userId - The user ID to create a room with. - * @returns The room ID of the room between the agent and the user. - * @throws An error if the room cannot be created. - */ - async ensureRoomExists(roomId: UUID) { - const room = await this.databaseAdapter.getRoom(roomId); - if (!room) { - await this.databaseAdapter.createRoom(roomId); - console.log(`Room ${roomId} created successfully.`); + + async ensureParticipantInRoom(userId: UUID, roomId: UUID) { + const participants = + await this.databaseAdapter.getParticipantsForRoom(roomId); + if (!participants.includes(userId)) { + await this.databaseAdapter.addParticipant(userId, roomId); + console.log( + `User ${userId} linked to room ${roomId} successfully.` + ); + } } - } - - /** - * Compose the state of the agent into an object that can be passed or used for response generation. - * @param message The message to compose the state from. - * @returns The state of the agent. - */ - async composeState( - message: Memory, - additionalKeys: { [key: string]: unknown } = {}, - ) { - const { userId, roomId } = message; - - const conversationLength = this.getConversationLength(); - const recentFactsCount = Math.ceil(this.getConversationLength() / 2); - const relevantFactsCount = Math.ceil(this.getConversationLength() / 2); - - const [actorsData, recentMessagesData, recentFactsData, goalsData]: [ - Actor[], - Memory[], - Memory[], - Goal[], - ] = await Promise.all([ - getActorDetails({ runtime: this, roomId }), - this.messageManager.getMemories({ - roomId, - count: conversationLength, - unique: false, - }), - this.factManager.getMemories({ - roomId, - count: recentFactsCount, - }), - getGoals({ - runtime: this, - count: 10, - onlyInProgress: false, - roomId, - }), - ]); - - const goals = formatGoalsAsString({ goals: goalsData }); - - let relevantFactsData: Memory[] = []; - - if (recentFactsData.length > recentFactsCount) { - relevantFactsData = ( - await this.factManager.searchMemoriesByEmbedding( - recentFactsData[0].embedding!, - { - roomId, - count: relevantFactsCount, - }, - ) - ).filter((fact: Memory) => { - return !recentFactsData.find( - (recentFact: Memory) => recentFact.id === fact.id, - ); - }); + + /** + * Ensure the existence of a room between the agent and a user. If no room exists, a new room is created and the user + * and agent are added as participants. The room ID is returned. + * @param userId - The user ID to create a room with. + * @returns The room ID of the room between the agent and the user. + * @throws An error if the room cannot be created. + */ + async ensureRoomExists(roomId: UUID) { + const room = await this.databaseAdapter.getRoom(roomId); + if (!room) { + await this.databaseAdapter.createRoom(roomId); + console.log(`Room ${roomId} created successfully.`); + } } - const actors = formatActors({ actors: actorsData ?? [] }); - - const recentMessages = formatMessages({ - messages: recentMessagesData, - actors: actorsData, - }); - - const recentPosts = formatPosts({ - messages: recentMessagesData, - actors: actorsData, - conversationHeader: false, - }); - - const recentFacts = formatFacts(recentFactsData); - const relevantFacts = formatFacts(relevantFactsData); - - // const lore = formatLore(loreData); - - const senderName = actorsData?.find( - (actor: Actor) => actor.id === userId, - )?.name; - - // TODO: We may wish to consolidate and just accept character.name here instead of the actor name - const agentName = - actorsData?.find((actor: Actor) => actor.id === this.agentId)?.name || - this.character.name; - - let allAttachments = message.content.attachments || []; - - if (recentMessagesData && Array.isArray(recentMessagesData)) { - const lastMessageWithAttachment = recentMessagesData.find( - (msg) => msg.content.attachments && msg.content.attachments.length > 0, - ); - - if (lastMessageWithAttachment) { - const lastMessageTime = lastMessageWithAttachment.createdAt; - const oneHourBeforeLastMessage = lastMessageTime - 60 * 60 * 1000; // 1 hour before last message - - allAttachments = recentMessagesData - .reverse() - .map((msg) => { - const msgTime = msg.createdAt ?? Date.now(); - const isWithinTime = msgTime >= oneHourBeforeLastMessage; - const attachments = msg.content.attachments || []; - if (!isWithinTime) { - attachments.forEach((attachment) => { - attachment.text = "[Hidden]"; - }); + /** + * Compose the state of the agent into an object that can be passed or used for response generation. + * @param message The message to compose the state from. + * @returns The state of the agent. + */ + async composeState( + message: Memory, + additionalKeys: { [key: string]: unknown } = {} + ) { + const { userId, roomId } = message; + + const conversationLength = this.getConversationLength(); + const recentFactsCount = Math.ceil(this.getConversationLength() / 2); + const relevantFactsCount = Math.ceil(this.getConversationLength() / 2); + + const [actorsData, recentMessagesData, recentFactsData, goalsData]: [ + Actor[], + Memory[], + Memory[], + Goal[], + ] = await Promise.all([ + getActorDetails({ runtime: this, roomId }), + this.messageManager.getMemories({ + roomId, + count: conversationLength, + unique: false, + }), + this.factManager.getMemories({ + roomId, + count: recentFactsCount, + }), + getGoals({ + runtime: this, + count: 10, + onlyInProgress: false, + roomId, + }), + ]); + + const goals = formatGoalsAsString({ goals: goalsData }); + + let relevantFactsData: Memory[] = []; + + if (recentFactsData.length > recentFactsCount) { + relevantFactsData = ( + await this.factManager.searchMemoriesByEmbedding( + recentFactsData[0].embedding!, + { + roomId, + count: relevantFactsCount, + } + ) + ).filter((fact: Memory) => { + return !recentFactsData.find( + (recentFact: Memory) => recentFact.id === fact.id + ); + }); + } + + const actors = formatActors({ actors: actorsData ?? [] }); + + const recentMessages = formatMessages({ + messages: recentMessagesData, + actors: actorsData, + }); + + const recentPosts = formatPosts({ + messages: recentMessagesData, + actors: actorsData, + conversationHeader: false, + }); + + const recentFacts = formatFacts(recentFactsData); + const relevantFacts = formatFacts(relevantFactsData); + + // const lore = formatLore(loreData); + + const senderName = actorsData?.find( + (actor: Actor) => actor.id === userId + )?.name; + + // TODO: We may wish to consolidate and just accept character.name here instead of the actor name + const agentName = + actorsData?.find((actor: Actor) => actor.id === this.agentId) + ?.name || this.character.name; + + let allAttachments = message.content.attachments || []; + + if (recentMessagesData && Array.isArray(recentMessagesData)) { + const lastMessageWithAttachment = recentMessagesData.find( + (msg) => + msg.content.attachments && + msg.content.attachments.length > 0 + ); + + if (lastMessageWithAttachment) { + const lastMessageTime = lastMessageWithAttachment.createdAt; + const oneHourBeforeLastMessage = + lastMessageTime - 60 * 60 * 1000; // 1 hour before last message + + allAttachments = recentMessagesData + .reverse() + .map((msg) => { + const msgTime = msg.createdAt ?? Date.now(); + const isWithinTime = + msgTime >= oneHourBeforeLastMessage; + const attachments = msg.content.attachments || []; + if (!isWithinTime) { + attachments.forEach((attachment) => { + attachment.text = "[Hidden]"; + }); + } + return attachments; + }) + .flat(); } - return attachments; - }) - .flat(); - } - } + } - const formattedAttachments = allAttachments - .map( - (attachment) => - `ID: ${attachment.id} + const formattedAttachments = allAttachments + .map( + (attachment) => + `ID: ${attachment.id} Name: ${attachment.title} URL: ${attachment.url} Type: ${attachment.source} Description: ${attachment.description} Text: ${attachment.text} - `, - ) - .join("\n"); - - // randomly get 3 bits of lore and join them into a paragraph, divided by \n - let lore = ""; - // Assuming this.lore is an array of lore bits - if (this.character.lore && this.character.lore.length > 0) { - const shuffledLore = [...this.character.lore].sort( - () => Math.random() - 0.5, - ); - const selectedLore = shuffledLore.slice(0, 10); - lore = selectedLore.join("\n"); - } + ` + ) + .join("\n"); + + // randomly get 3 bits of lore and join them into a paragraph, divided by \n + let lore = ""; + // Assuming this.lore is an array of lore bits + if (this.character.lore && this.character.lore.length > 0) { + const shuffledLore = [...this.character.lore].sort( + () => Math.random() - 0.5 + ); + const selectedLore = shuffledLore.slice(0, 10); + lore = selectedLore.join("\n"); + } - const formattedCharacterPostExamples = this.character.postExamples - .sort(() => 0.5 - Math.random()) - .map((post) => { - const messageString = `${post}`; - return messageString; - }) - .slice(0, 50) - .join("\n"); - - const formattedCharacterMessageExamples = this.character.messageExamples - .sort(() => 0.5 - Math.random()) - .slice(0, 5) - .map((example) => { - const exampleNames = Array.from({ length: 5 }, () => - uniqueNamesGenerator({ dictionaries: [names] }), - ); + const formattedCharacterPostExamples = this.character.postExamples + .sort(() => 0.5 - Math.random()) + .map((post) => { + const messageString = `${post}`; + return messageString; + }) + .slice(0, 50) + .join("\n"); + + const formattedCharacterMessageExamples = this.character.messageExamples + .sort(() => 0.5 - Math.random()) + .slice(0, 5) + .map((example) => { + const exampleNames = Array.from({ length: 5 }, () => + uniqueNamesGenerator({ dictionaries: [names] }) + ); - return example - .map((message) => { - let messageString = `${message.user}: ${message.content.text}`; - exampleNames.forEach((name, index) => { - const placeholder = `{{user${index + 1}}}`; - messageString = messageString.replaceAll(placeholder, name); + return example + .map((message) => { + let messageString = `${message.user}: ${message.content.text}`; + exampleNames.forEach((name, index) => { + const placeholder = `{{user${index + 1}}}`; + messageString = messageString.replaceAll( + placeholder, + name + ); + }); + return messageString; + }) + .join("\n"); + }) + .join("\n\n"); + + const getRecentInteractions = async ( + userA: UUID, + userB: UUID + ): Promise => { + // Find all rooms where userA and userB are participants + const rooms = await this.databaseAdapter.getRoomsForParticipants([ + userA, + userB, + ]); + + // Check the existing memories in the database + const existingMemories = + await this.messageManager.getMemoriesByRoomIds({ + // filter out the current room id from rooms + roomIds: rooms.filter((room) => room !== roomId), + }); + + // Sort messages by timestamp in descending order + existingMemories.sort((a, b) => b.createdAt - a.createdAt); + + // Take the most recent messages + const recentInteractionsData = existingMemories.slice(0, 20); + return recentInteractionsData; + }; + + const recentInteractions = + userId !== this.agentId + ? await getRecentInteractions(userId, this.agentId) + : []; + + const getRecentMessageInteractions = async ( + recentInteractionsData: Memory[] + ): Promise => { + // Format the recent messages + const formattedInteractions = await recentInteractionsData + .map(async (message) => { + const isSelf = message.userId === this.agentId; + let sender; + if (isSelf) { + sender = this.character.name; + } else { + const accountId = + await this.databaseAdapter.getAccountById( + message.userId + ); + sender = accountId?.username || "unknown"; + } + return `${sender}: ${message.content.text}`; + }) + .join("\n"); + + return formattedInteractions; + }; + + const formattedMessageInteractions = + await getRecentMessageInteractions(recentInteractions); + + const getRecentPostInteractions = async ( + recentInteractionsData: Memory[], + actors: Actor[] + ): Promise => { + const formattedInteractions = formatPosts({ + messages: recentInteractionsData, + actors, + conversationHeader: true, }); - return messageString; - }) - .join("\n"); - }) - .join("\n\n"); - - const getRecentInteractions = async ( - userA: UUID, - userB: UUID, - ): Promise => { - // Find all rooms where userA and userB are participants - const rooms = await this.databaseAdapter.getRoomsForParticipants([ - userA, - userB, - ]); - - // Check the existing memories in the database - const existingMemories = await this.messageManager.getMemoriesByRoomIds({ - // filter out the current room id from rooms - roomIds: rooms.filter((room) => room !== roomId), - }); - - // Sort messages by timestamp in descending order - existingMemories.sort((a, b) => b.createdAt - a.createdAt); - - // Take the most recent messages - const recentInteractionsData = existingMemories.slice(0, 20); - return recentInteractionsData; - }; - - const recentInteractions = - userId !== this.agentId - ? await getRecentInteractions(userId, this.agentId) - : []; - - const getRecentMessageInteractions = async ( - recentInteractionsData: Memory[], - ): Promise => { - // Format the recent messages - const formattedInteractions = await recentInteractionsData - .map(async (message) => { - const isSelf = message.userId === this.agentId; - let sender; - if (isSelf) { - sender = this.character.name; - } else { - const accountId = await this.databaseAdapter.getAccountById( - message.userId, + + return formattedInteractions; + }; + + const formattedPostInteractions = await getRecentPostInteractions( + recentInteractions, + actorsData + ); + + // if bio is a string, use it. if its an array, pick one at random + let bio = this.character.bio || ""; + if (Array.isArray(bio)) { + // get three random bio strings and join them with " " + bio = bio + .sort(() => 0.5 - Math.random()) + .slice(0, 3) + .join(" "); + } + + const initialState = { + agentId: this.agentId, + // Character file stuff + agentName, + bio, + lore, + adjective: + this.character.adjectives && + this.character.adjectives.length > 0 + ? this.character.adjectives[ + Math.floor( + Math.random() * this.character.adjectives.length + ) + ] + : "", + // Recent interactions between the sender and receiver, formatted as messages + recentMessageInteractions: formattedMessageInteractions, + // Recent interactions between the sender and receiver, formatted as posts + recentPostInteractions: formattedPostInteractions, + // Raw memory[] array of interactions + recentInteractionsData: recentInteractions, + // randomly pick one topic + topic: + this.character.topics && this.character.topics.length > 0 + ? this.character.topics[ + Math.floor( + Math.random() * this.character.topics.length + ) + ] + : null, + topics: + this.character.topics && this.character.topics.length > 0 + ? `${this.character.name} is interested in ` + + this.character.topics + .sort(() => 0.5 - Math.random()) + .slice(0, 5) + .map((topic, index) => { + if (index === this.character.topics.length - 2) { + return topic + " and "; + } + // if last topic, don't add a comma + if (index === this.character.topics.length - 1) { + return topic; + } + return topic + ", "; + }) + .join("") + : "", + characterPostExamples: + formattedCharacterPostExamples && + formattedCharacterPostExamples.replaceAll("\n", "").length > 0 + ? addHeader( + `# Example Posts for ${this.character.name}`, + formattedCharacterPostExamples + ) + : "", + characterMessageExamples: + formattedCharacterMessageExamples && + formattedCharacterMessageExamples.replaceAll("\n", "").length > + 0 + ? addHeader( + `# Example Conversations for ${this.character.name}`, + formattedCharacterMessageExamples + ) + : "", + messageDirections: + this.character?.style?.all?.length > 0 || + this.character?.style?.chat.length > 0 + ? addHeader( + "# Message Directions for " + this.character.name, + (() => { + const all = this.character?.style?.all || []; + const chat = this.character?.style?.chat || []; + const shuffled = [...all, ...chat].sort( + () => 0.5 - Math.random() + ); + const allSliced = shuffled.slice( + 0, + conversationLength / 2 + ); + return allSliced.concat(allSliced).join("\n"); + })() + ) + : "", + postDirections: + this.character?.style?.all?.length > 0 || + this.character?.style?.post.length > 0 + ? addHeader( + "# Post Directions for " + this.character.name, + (() => { + const all = this.character?.style?.all || []; + const post = this.character?.style?.post || []; + const shuffled = [...all, ...post].sort( + () => 0.5 - Math.random() + ); + return shuffled + .slice(0, conversationLength / 2) + .join("\n"); + })() + ) + : "", + // Agent runtime stuff + senderName, + actors: + actors && actors.length > 0 + ? addHeader("# Actors", actors) + : "", + actorsData, + roomId, + goals: + goals && goals.length > 0 + ? addHeader( + "# Goals\n{{agentName}} should prioritize accomplishing the objectives that are in progress.", + goals + ) + : "", + goalsData, + recentMessages: + recentMessages && recentMessages.length > 0 + ? addHeader("# Conversation Messages", recentMessages) + : "", + recentPosts: + recentPosts && recentPosts.length > 0 + ? addHeader("# Posts in Thread", recentPosts) + : "", + recentMessagesData, + recentFacts: + recentFacts && recentFacts.length > 0 + ? addHeader("# Recent Facts", recentFacts) + : "", + recentFactsData, + relevantFacts: + relevantFacts && relevantFacts.length > 0 + ? addHeader("# Relevant Facts", relevantFacts) + : "", + relevantFactsData, + attachments: + formattedAttachments && formattedAttachments.length > 0 + ? addHeader("# Attachments", formattedAttachments) + : "", + ...additionalKeys, + }; + + const actionPromises = this.actions.map(async (action: Action) => { + const result = await action.validate(this, message, initialState); + if (result) { + return action; + } + return null; + }); + + const evaluatorPromises = this.evaluators.map(async (evaluator) => { + const result = await evaluator.validate( + this, + message, + initialState ); - sender = accountId?.username || "unknown"; - } - return `${sender}: ${message.content.text}`; - }) - .join("\n"); - - return formattedInteractions; - }; - - const formattedMessageInteractions = - await getRecentMessageInteractions(recentInteractions); - - const getRecentPostInteractions = async ( - recentInteractionsData: Memory[], - actors: Actor[], - ): Promise => { - const formattedInteractions = formatPosts({ - messages: recentInteractionsData, - actors, - conversationHeader: true, - }); - - return formattedInteractions; - }; - - const formattedPostInteractions = await getRecentPostInteractions( - recentInteractions, - actorsData, - ); - - // if bio is a string, use it. if its an array, pick one at random - let bio = this.character.bio || ""; - if (Array.isArray(bio)) { - // get three random bio strings and join them with " " - bio = bio - .sort(() => 0.5 - Math.random()) - .slice(0, 3) - .join(" "); - } + if (result) { + return evaluator; + } + return null; + }); - const initialState = { - agentId: this.agentId, - // Character file stuff - agentName, - bio, - lore, - adjective: - this.character.adjectives && this.character.adjectives.length > 0 - ? this.character.adjectives[ - Math.floor(Math.random() * this.character.adjectives.length) - ] - : "", - // Recent interactions between the sender and receiver, formatted as messages - recentMessageInteractions: formattedMessageInteractions, - // Recent interactions between the sender and receiver, formatted as posts - recentPostInteractions: formattedPostInteractions, - // Raw memory[] array of interactions - recentInteractionsData: recentInteractions, - // randomly pick one topic - topic: - this.character.topics && this.character.topics.length > 0 - ? this.character.topics[ - Math.floor(Math.random() * this.character.topics.length) - ] - : null, - topics: - this.character.topics && this.character.topics.length > 0 - ? `${this.character.name} is interested in ` + - this.character.topics - .sort(() => 0.5 - Math.random()) - .slice(0, 5) - .map((topic, index) => { - if (index === this.character.topics.length - 2) { - return topic + " and "; - } - // if last topic, don't add a comma - if (index === this.character.topics.length - 1) { - return topic; - } - return topic + ", "; - }) - .join("") - : "", - characterPostExamples: - formattedCharacterPostExamples && - formattedCharacterPostExamples.replaceAll("\n", "").length > 0 - ? addHeader( - `# Example Posts for ${this.character.name}`, - formattedCharacterPostExamples, - ) - : "", - characterMessageExamples: - formattedCharacterMessageExamples && - formattedCharacterMessageExamples.replaceAll("\n", "").length > 0 - ? addHeader( - `# Example Conversations for ${this.character.name}`, - formattedCharacterMessageExamples, - ) - : "", - messageDirections: - this.character?.style?.all?.length > 0 || - this.character?.style?.chat.length > 0 - ? addHeader( - "# Message Directions for " + this.character.name, - (() => { - const all = this.character?.style?.all || []; - const chat = this.character?.style?.chat || []; - const shuffled = [...all, ...chat].sort( - () => 0.5 - Math.random(), - ); - const allSliced = shuffled.slice(0, conversationLength / 2); - return allSliced.concat(allSliced).join("\n"); - })(), - ) - : "", - postDirections: - this.character?.style?.all?.length > 0 || - this.character?.style?.post.length > 0 - ? addHeader( - "# Post Directions for " + this.character.name, - (() => { - const all = this.character?.style?.all || []; - const post = this.character?.style?.post || []; - const shuffled = [...all, ...post].sort( - () => 0.5 - Math.random(), - ); - return shuffled.slice(0, conversationLength / 2).join("\n"); - })(), - ) - : "", - // Agent runtime stuff - senderName, - actors: actors && actors.length > 0 ? addHeader("# Actors", actors) : "", - actorsData, - roomId, - goals: - goals && goals.length > 0 - ? addHeader( - "# Goals\n{{agentName}} should prioritize accomplishing the objectives that are in progress.", - goals, - ) - : "", - goalsData, - recentMessages: - recentMessages && recentMessages.length > 0 - ? addHeader("# Conversation Messages", recentMessages) - : "", - recentPosts: - recentPosts && recentPosts.length > 0 - ? addHeader("# Posts in Thread", recentPosts) - : "", - recentMessagesData, - recentFacts: - recentFacts && recentFacts.length > 0 - ? addHeader("# Recent Facts", recentFacts) - : "", - recentFactsData, - relevantFacts: - relevantFacts && relevantFacts.length > 0 - ? addHeader("# Relevant Facts", relevantFacts) - : "", - relevantFactsData, - attachments: - formattedAttachments && formattedAttachments.length > 0 - ? addHeader("# Attachments", formattedAttachments) - : "", - ...additionalKeys, - }; - - const actionPromises = this.actions.map(async (action: Action) => { - const result = await action.validate(this, message, initialState); - if (result) { - return action; - } - return null; - }); - - const evaluatorPromises = this.evaluators.map(async (evaluator) => { - const result = await evaluator.validate(this, message, initialState); - if (result) { - return evaluator; - } - return null; - }); - - const [resolvedEvaluators, resolvedActions, providers] = await Promise.all([ - Promise.all(evaluatorPromises), - Promise.all(actionPromises), - getProviders(this, message, initialState), - ]); - - const evaluatorsData = resolvedEvaluators.filter(Boolean) as Evaluator[]; - const actionsData = resolvedActions.filter(Boolean) as Action[]; - - const actionState = { - actionNames: - "Possible response actions: " + formatActionNames(actionsData), - actions: - actionsData.length > 0 - ? addHeader("# Available Actions", formatActions(actionsData)) - : "", - actionExamples: - actionsData.length > 0 - ? addHeader( - "# Action Examples", - composeActionExamples(actionsData, 10), - ) - : "", - evaluatorsData, - evaluators: - evaluatorsData.length > 0 ? formatEvaluators(evaluatorsData) : "", - evaluatorNames: - evaluatorsData.length > 0 ? formatEvaluatorNames(evaluatorsData) : "", - evaluatorExamples: - evaluatorsData.length > 0 - ? formatEvaluatorExamples(evaluatorsData) - : "", - providers: addHeader( - `# Additional Information About ${this.character.name} and The World`, - providers, - ), - }; - - return { ...initialState, ...actionState } as State; - } - - async updateRecentMessageState(state: State): Promise { - const conversationLength = this.getConversationLength(); - const recentMessagesData = await this.messageManager.getMemories({ - roomId: state.roomId, - count: conversationLength, - unique: false, - }); - - const recentMessages = formatMessages({ - actors: state.actorsData ?? [], - messages: recentMessagesData.map((memory: Memory) => { - const newMemory = { ...memory }; - delete newMemory.embedding; - return newMemory; - }), - }); - - let allAttachments = []; - - if (recentMessagesData && Array.isArray(recentMessagesData)) { - const lastMessageWithAttachment = recentMessagesData.find( - (msg) => msg.content.attachments && msg.content.attachments.length > 0, - ); - - if (lastMessageWithAttachment) { - const lastMessageTime = lastMessageWithAttachment.createdAt; - const oneHourBeforeLastMessage = lastMessageTime - 60 * 60 * 1000; // 1 hour before last message - - allAttachments = recentMessagesData - .filter((msg) => { - const msgTime = msg.createdAt; - return msgTime >= oneHourBeforeLastMessage; - }) - .flatMap((msg) => msg.content.attachments || []); - } + const [resolvedEvaluators, resolvedActions, providers] = + await Promise.all([ + Promise.all(evaluatorPromises), + Promise.all(actionPromises), + getProviders(this, message, initialState), + ]); + + const evaluatorsData = resolvedEvaluators.filter( + Boolean + ) as Evaluator[]; + const actionsData = resolvedActions.filter(Boolean) as Action[]; + + const actionState = { + actionNames: + "Possible response actions: " + formatActionNames(actionsData), + actions: + actionsData.length > 0 + ? addHeader( + "# Available Actions", + formatActions(actionsData) + ) + : "", + actionExamples: + actionsData.length > 0 + ? addHeader( + "# Action Examples", + composeActionExamples(actionsData, 10) + ) + : "", + evaluatorsData, + evaluators: + evaluatorsData.length > 0 + ? formatEvaluators(evaluatorsData) + : "", + evaluatorNames: + evaluatorsData.length > 0 + ? formatEvaluatorNames(evaluatorsData) + : "", + evaluatorExamples: + evaluatorsData.length > 0 + ? formatEvaluatorExamples(evaluatorsData) + : "", + providers: addHeader( + `# Additional Information About ${this.character.name} and The World`, + providers + ), + }; + + return { ...initialState, ...actionState } as State; } - const formattedAttachments = allAttachments - .map( - (attachment) => - `ID: ${attachment.id} + async updateRecentMessageState(state: State): Promise { + const conversationLength = this.getConversationLength(); + const recentMessagesData = await this.messageManager.getMemories({ + roomId: state.roomId, + count: conversationLength, + unique: false, + }); + + const recentMessages = formatMessages({ + actors: state.actorsData ?? [], + messages: recentMessagesData.map((memory: Memory) => { + const newMemory = { ...memory }; + delete newMemory.embedding; + return newMemory; + }), + }); + + let allAttachments = []; + + if (recentMessagesData && Array.isArray(recentMessagesData)) { + const lastMessageWithAttachment = recentMessagesData.find( + (msg) => + msg.content.attachments && + msg.content.attachments.length > 0 + ); + + if (lastMessageWithAttachment) { + const lastMessageTime = lastMessageWithAttachment.createdAt; + const oneHourBeforeLastMessage = + lastMessageTime - 60 * 60 * 1000; // 1 hour before last message + + allAttachments = recentMessagesData + .filter((msg) => { + const msgTime = msg.createdAt; + return msgTime >= oneHourBeforeLastMessage; + }) + .flatMap((msg) => msg.content.attachments || []); + } + } + + const formattedAttachments = allAttachments + .map( + (attachment) => + `ID: ${attachment.id} Name: ${attachment.title} URL: ${attachment.url} Type: ${attachment.source} Description: ${attachment.description} Text: ${attachment.text} - `, - ) - .join("\n"); - - return { - ...state, - recentMessages: addHeader("# Conversation Messages", recentMessages), - recentMessagesData, - attachments: formattedAttachments, - } as State; - } + ` + ) + .join("\n"); + + return { + ...state, + recentMessages: addHeader( + "# Conversation Messages", + recentMessages + ), + recentMessagesData, + attachments: formattedAttachments, + } as State; + } } diff --git a/core/src/core/types.ts b/core/src/core/types.ts index e93df2a582..7d47168902 100644 --- a/core/src/core/types.ts +++ b/core/src/core/types.ts @@ -10,541 +10,550 @@ export type UUID = `${string}-${string}-${string}-${string}-${string}`; * Represents the content of a message, including its main text (`content`), any associated action (`action`), and the source of the content (`source`), if applicable. */ export interface Content { - text: string; // The main text content of the message. - action?: string; // An optional action associated with the message, indicating a specific behavior or response required. - source?: string; // The source of the content, if applicable, such as a reference or origin. - url?: string; // The actual URL of the message or post, i.e. tweet URL or message link in discord - inReplyTo?: UUID; // If this is a message in a thread, or a reply, store this - attachments?: Media[]; - [key: string]: unknown; // Allows for additional properties to be included dynamically. + text: string; // The main text content of the message. + action?: string; // An optional action associated with the message, indicating a specific behavior or response required. + source?: string; // The source of the content, if applicable, such as a reference or origin. + url?: string; // The actual URL of the message or post, i.e. tweet URL or message link in discord + inReplyTo?: UUID; // If this is a message in a thread, or a reply, store this + attachments?: Media[]; + [key: string]: unknown; // Allows for additional properties to be included dynamically. } /** * Represents an example of content, typically used for demonstrating or testing purposes. Includes user, content, optional action, and optional source. */ export interface ActionExample { - user: string; // The user associated with the message. - content: Content; // The content of the message. + user: string; // The user associated with the message. + content: Content; // The content of the message. } /** * Represents an example of content, typically used for demonstrating or testing purposes. Includes user, content, optional action, and optional source. */ export interface ConversationExample { - userId: UUID; // The user associated with the message. - content: Content; // The content of the message. + userId: UUID; // The user associated with the message. + content: Content; // The content of the message. } /** * Represents an actor in the conversation, which could be a user or the agent itself, including their name, details (such as tagline, summary, and quote), and a unique identifier. */ export interface Actor { - name: string; // The name of the actor. - username: string; // The username of the actor. - details: { tagline: string; summary: string; quote: string }; // Additional details about the actor, including a tagline, a summary, and a favorite quote. - id: UUID; // A unique identifier for the actor. + name: string; // The name of the actor. + username: string; // The username of the actor. + details: { tagline: string; summary: string; quote: string }; // Additional details about the actor, including a tagline, a summary, and a favorite quote. + id: UUID; // A unique identifier for the actor. } /** * Represents an objective within a goal, detailing what needs to be achieved and whether it has been completed. */ export interface Objective { - id?: string; // A unique identifier for the objective. - description: string; // A description of what the objective entails. - completed: boolean; // A boolean indicating whether the objective has been completed. + id?: string; // A unique identifier for the objective. + description: string; // A description of what the objective entails. + completed: boolean; // A boolean indicating whether the objective has been completed. } export enum GoalStatus { - DONE = "DONE", - FAILED = "FAILED", - IN_PROGRESS = "IN_PROGRESS" + DONE = "DONE", + FAILED = "FAILED", + IN_PROGRESS = "IN_PROGRESS", } /** * Represents a goal, which is a higher-level aim composed of one or more objectives. Goals are tracked to measure progress or achievements within the conversation or system. */ export interface Goal { - id?: UUID; // A unique identifier for the goal. - roomId: UUID; // A list of user IDs associated with the goal, for goals relevant to specific users or groups. - userId: UUID; // The user ID of the goal's owner or the user who is primarily responsible for the goal. - name: string; // The name or title of the goal. - status: GoalStatus; // The current status of the goal, such as "in progress" or "completed". - objectives: Objective[]; // A list of objectives that make up the goal. + id?: UUID; // A unique identifier for the goal. + roomId: UUID; // A list of user IDs associated with the goal, for goals relevant to specific users or groups. + userId: UUID; // The user ID of the goal's owner or the user who is primarily responsible for the goal. + name: string; // The name or title of the goal. + status: GoalStatus; // The current status of the goal, such as "in progress" or "completed". + objectives: Objective[]; // A list of objectives that make up the goal. } export enum ModelClass { - SMALL = "small", - MEDIUM = "medium", - LARGE = "large", - EMBEDDING = "embedding", + SMALL = "small", + MEDIUM = "medium", + LARGE = "large", + EMBEDDING = "embedding", } export type Model = { - endpoint?: string; - settings: { - maxInputTokens: number; - maxOutputTokens: number; - frequency_penalty?: number; - presence_penalty?: number; - repetition_penalty?: number; - stop: string[]; - temperature: number; - }; - model: { - [ModelClass.SMALL]: string; - [ModelClass.MEDIUM]: string; - [ModelClass.LARGE]: string; - [ModelClass.EMBEDDING]?: string; - }; + endpoint?: string; + settings: { + maxInputTokens: number; + maxOutputTokens: number; + frequency_penalty?: number; + presence_penalty?: number; + repetition_penalty?: number; + stop: string[]; + temperature: number; + }; + model: { + [ModelClass.SMALL]: string; + [ModelClass.MEDIUM]: string; + [ModelClass.LARGE]: string; + [ModelClass.EMBEDDING]?: string; + }; }; export enum ModelProvider { - OPENAI = "openai", - ANTHROPIC = "anthropic", - GROK = "grok", - LLAMACLOUD = "llama_cloud", - LLAMALOCAL = "llama_local", - GOOGLE = "google", - CLAUDE_VERTEX = "claude_vertex" + OPENAI = "openai", + ANTHROPIC = "anthropic", + GROK = "grok", + LLAMACLOUD = "llama_cloud", + LLAMALOCAL = "llama_local", + GOOGLE = "google", + CLAUDE_VERTEX = "claude_vertex", } /** * Represents the state of the conversation or context in which the agent is operating, including information about users, messages, goals, and other relevant data. */ export interface State { - userId?: UUID; // An optional ID of the user who sent the current message. - agentId?: UUID; // An optional ID of the agent within the current conversation or context. - bio: string; // A string representation of the agent's bio. - lore: string; // A list of lore bits for the agent. - messageDirections: string; // A string representation of directions for messages in the current state. - postDirections: string; // A string representation of directions for posting in the current state. - roomId: UUID; // The ID of the current room or conversation context. - agentName?: string; // An optional name of the agent, used for referencing the agent in conversations. - senderName?: string; // An optional name of the sender of the current message. - actors: string; // A string representation of the actors involved in the conversation, including their details. - actorsData?: Actor[]; // An optional array of actor objects involved in the conversation. - goals?: string; // An optional string representation of the goals relevant to the current conversation or context. - goalsData?: Goal[]; // An optional array of goal objects relevant to the current conversation or context. - recentMessages: string; // A string representation of recent messages in the conversation, for context. - recentMessagesData: Memory[]; // An array of memory objects representing recent messages in the conversation. - recentFacts?: string; // An optional string representation of recent facts derived from the conversation. - recentFactsData?: Memory[]; // An optional array of memory objects representing recent facts. - relevantFacts?: string; // An optional string representation of facts relevant to the current context or topic. - relevantFactsData?: Memory[]; // An optional array of memory objects representing relevant facts. - actionNames?: string; // An optional string listing the names of actions that are valid in the current state. - actions?: string; // An optional string representation of actions and their descriptions, relevant to the current state. - actionsData?: Action[]; // An optional array of action objects relevant to the current state. - actionExamples?: string; // An optional string representation of examples of actions, for demonstration or testing. - providers?: string; // An optional string representation of available providers and their descriptions, relevant to the current state. - responseData?: Content; // An optional content object representing the agent's response in the current state. - recentInteractionsData?: Memory[]; // An optional array of memory objects representing recent interactions in the conversation. - recentInteractions?: string; // An optional string representation of recent interactions in the conversation. - [key: string]: unknown; // Allows for additional properties to be included dynamically. + userId?: UUID; // An optional ID of the user who sent the current message. + agentId?: UUID; // An optional ID of the agent within the current conversation or context. + bio: string; // A string representation of the agent's bio. + lore: string; // A list of lore bits for the agent. + messageDirections: string; // A string representation of directions for messages in the current state. + postDirections: string; // A string representation of directions for posting in the current state. + roomId: UUID; // The ID of the current room or conversation context. + agentName?: string; // An optional name of the agent, used for referencing the agent in conversations. + senderName?: string; // An optional name of the sender of the current message. + actors: string; // A string representation of the actors involved in the conversation, including their details. + actorsData?: Actor[]; // An optional array of actor objects involved in the conversation. + goals?: string; // An optional string representation of the goals relevant to the current conversation or context. + goalsData?: Goal[]; // An optional array of goal objects relevant to the current conversation or context. + recentMessages: string; // A string representation of recent messages in the conversation, for context. + recentMessagesData: Memory[]; // An array of memory objects representing recent messages in the conversation. + recentFacts?: string; // An optional string representation of recent facts derived from the conversation. + recentFactsData?: Memory[]; // An optional array of memory objects representing recent facts. + relevantFacts?: string; // An optional string representation of facts relevant to the current context or topic. + relevantFactsData?: Memory[]; // An optional array of memory objects representing relevant facts. + actionNames?: string; // An optional string listing the names of actions that are valid in the current state. + actions?: string; // An optional string representation of actions and their descriptions, relevant to the current state. + actionsData?: Action[]; // An optional array of action objects relevant to the current state. + actionExamples?: string; // An optional string representation of examples of actions, for demonstration or testing. + providers?: string; // An optional string representation of available providers and their descriptions, relevant to the current state. + responseData?: Content; // An optional content object representing the agent's response in the current state. + recentInteractionsData?: Memory[]; // An optional array of memory objects representing recent interactions in the conversation. + recentInteractions?: string; // An optional string representation of recent interactions in the conversation. + [key: string]: unknown; // Allows for additional properties to be included dynamically. } /** * Represents a memory record, which could be a message or any other piece of information remembered by the system, including its content, associated user IDs, and optionally, its embedding vector for similarity comparisons. */ export interface Memory { - id?: UUID; // An optional unique identifier for the memory. - userId: UUID; // The user ID associated with the memory. - createdAt?: number; // An optional timestamp indicating when the memory was created. - content: Content; // The content of the memory, which can be a structured object or a plain string. - embedding?: number[]; // An optional embedding vector representing the semantic content of the memory. - roomId: UUID; // The room or conversation ID associated with the memory. - unique?: boolean; // Whether the memory is unique or not + id?: UUID; // An optional unique identifier for the memory. + userId: UUID; // The user ID associated with the memory. + createdAt?: number; // An optional timestamp indicating when the memory was created. + content: Content; // The content of the memory, which can be a structured object or a plain string. + embedding?: number[]; // An optional embedding vector representing the semantic content of the memory. + roomId: UUID; // The room or conversation ID associated with the memory. + unique?: boolean; // Whether the memory is unique or not } /** * Represents an example of a message, typically used for demonstrating or testing purposes, including optional content and action. */ export interface MessageExample { - user: string; // The user associated with the message example. - content: Content; // The content of the message example, which may be null for actions that don't produce visible content. + user: string; // The user associated with the message example. + content: Content; // The content of the message example, which may be null for actions that don't produce visible content. } /** * Represents the type of a handler function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to any type. */ export type Handler = ( - runtime: IAgentRuntime, - message: Memory, - state?: State, - options?: { [key: string]: unknown }, // additional options can be used for things like tests or state-passing on a chain - callback?: HandlerCallback + runtime: IAgentRuntime, + message: Memory, + state?: State, + options?: { [key: string]: unknown }, // additional options can be used for things like tests or state-passing on a chain + callback?: HandlerCallback ) => Promise; // export type HandlerCallback = ( - response: Content, - files?: any + response: Content, + files?: any ) => Promise; /** * Represents the type of a validator function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to a boolean indicating whether the validation passed. */ export type Validator = ( - runtime: IAgentRuntime, - message: Memory, - state?: State + runtime: IAgentRuntime, + message: Memory, + state?: State ) => Promise; /** * Represents an action that the agent can perform, including conditions for its use, a description, examples, a handler function, and a validation function. */ export interface Action { - similes: string[]; // An array of strings representing the similies of the action. - description: string; // A detailed description of what the action entails. - examples: ActionExample[][]; // An array of arrays of content examples demonstrating the action. - handler: Handler; // The function that handles the action. - name: string; // The name of the action. - validate: Validator; // The function that validates whether the action is appropriate in the current context. + similes: string[]; // An array of strings representing the similies of the action. + description: string; // A detailed description of what the action entails. + examples: ActionExample[][]; // An array of arrays of content examples demonstrating the action. + handler: Handler; // The function that handles the action. + name: string; // The name of the action. + validate: Validator; // The function that validates whether the action is appropriate in the current context. } /** * Represents an example for evaluation, including the context, an array of message examples, and the expected outcome. */ export interface EvaluationExample { - context: string; // The context in which the evaluation example is set. - messages: Array; // An array of message examples relevant to the evaluation. - outcome: string; // The expected outcome of the evaluation, typically in a structured format such as JSON. + context: string; // The context in which the evaluation example is set. + messages: Array; // An array of message examples relevant to the evaluation. + outcome: string; // The expected outcome of the evaluation, typically in a structured format such as JSON. } /** * Represents an evaluator, which is used to assess and guide the agent's responses based on the current context and state. */ export interface Evaluator { - description: string; // A detailed description of what the evaluator assesses or guides. - similes: string[]; // An array of strings representing the similies of the action. - examples: EvaluationExample[]; // An array of evaluation examples demonstrating the evaluator. - handler: Handler; // The function that handles the evaluation. - name: string; // The name of the evaluator. - validate: Validator; // The function that validates whether the evaluator is applicable in the current context. + description: string; // A detailed description of what the evaluator assesses or guides. + similes: string[]; // An array of strings representing the similies of the action. + examples: EvaluationExample[]; // An array of evaluation examples demonstrating the evaluator. + handler: Handler; // The function that handles the evaluation. + name: string; // The name of the evaluator. + validate: Validator; // The function that validates whether the evaluator is applicable in the current context. } /** * Represents a provider, which is used to retrieve information or perform actions on behalf of the agent, such as fetching data from an external API or service. */ export interface Provider { - get: (runtime: IAgentRuntime, message: Memory, state?: State) => Promise; + get: ( + runtime: IAgentRuntime, + message: Memory, + state?: State + ) => Promise; } /** * Represents a relationship between two users, including their IDs, the status of the relationship, and the room ID in which the relationship is established. */ export interface Relationship { - id: UUID; - userA: UUID; - userB: UUID; - userId: UUID; - roomId: UUID; - status: string; - createdAt?: string; + id: UUID; + userA: UUID; + userB: UUID; + userId: UUID; + roomId: UUID; + status: string; + createdAt?: string; } /** * Represents a user, including their name, details, and a unique identifier. */ export interface Account { - id: UUID; - name: string; // The display name in the server or their name on Twitter - username: string; // Their actual username - details?: { [key: string]: any }; - email?: string; - avatarUrl?: string; + id: UUID; + name: string; // The display name in the server or their name on Twitter + username: string; // Their actual username + details?: { [key: string]: any }; + email?: string; + avatarUrl?: string; } /** * Represents a participant in a room, including their ID and account details. */ export interface Participant { - id: UUID; - account: Account; + id: UUID; + account: Account; } /** * Represents a room or conversation context, including its ID and a list of participants. */ export interface Room { - id: UUID; - participants: Participant[]; + id: UUID; + participants: Participant[]; } export type Media = { - id: string; - url: string; - title: string; - source: string; - description: string; - text: string; + id: string; + url: string; + title: string; + source: string; + description: string; + text: string; }; export type Character = { - id?: UUID; // optional UUID which can be passed down to identify the character - name: string; - system?: string; - modelProvider: ModelProvider; - imageGenModel?: ImageGenModel; - modelOverride?: string; - bio: string | string[]; - lore: string[]; - messageExamples: MessageExample[][]; - postExamples: string[]; - people: string[]; - topics: string[]; - adjectives: string[]; - knowledge?: string[]; - clients: string[]; // list of clients the character can interact with - settings?: { - secrets?: { [key: string]: string }; - voice?: { - model?: string; - url?: string; + id?: UUID; // optional UUID which can be passed down to identify the character + name: string; + system?: string; + modelProvider: ModelProvider; + imageGenModel?: ImageGenModel; + modelOverride?: string; + bio: string | string[]; + lore: string[]; + messageExamples: MessageExample[][]; + postExamples: string[]; + people: string[]; + topics: string[]; + adjectives: string[]; + knowledge?: string[]; + clients: string[]; // list of clients the character can interact with + settings?: { + secrets?: { [key: string]: string }; + voice?: { + model?: string; + url?: string; + }; + model?: string; + embeddingModel?: string; + }; + style: { + all: string[]; + chat: string[]; + post: string[]; }; - model?: string; - embeddingModel?: string; - }; - style: { - all: string[]; - chat: string[]; - post: string[]; - }; }; export interface IDatabaseAdapter { - db: any; - getAccountById(userId: UUID): Promise; - createAccount(account: Account): Promise; - getMemories(params: { - roomId: UUID; - count?: number; - unique?: boolean; - tableName: string; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise; - getMemoryById(id: UUID): Promise; - getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise; - getCachedEmbeddings(params: { - query_table_name: string; - query_threshold: number; - query_input: string; - query_field_name: string; - query_field_sub_name: string; - query_match_count: number; - }): Promise<{ embedding: number[]; levenshtein_score: number }[]>; - log(params: { - body: { [key: string]: unknown }; - userId: UUID; - roomId: UUID; - type: string; - }): Promise; - getActorDetails(params: { roomId: UUID }): Promise; - searchMemories(params: { - tableName: string; - roomId: UUID; - embedding: number[]; - match_threshold: number; - match_count: number; - unique: boolean; - }): Promise; - updateGoalStatus(params: { goalId: UUID; status: GoalStatus }): Promise; - searchMemoriesByEmbedding( - embedding: number[], - params: { - match_threshold?: number; - count?: number; - roomId?: UUID; - unique?: boolean; - tableName: string; - } - ): Promise; - createMemory( - memory: Memory, - tableName: string, - unique?: boolean - ): Promise; - removeMemory(memoryId: UUID, tableName: string): Promise; - removeAllMemories(roomId: UUID, tableName: string): Promise; - countMemories( - roomId: UUID, - unique?: boolean, - tableName?: string - ): Promise; - getGoals(params: { - roomId: UUID; - userId?: UUID | null; - onlyInProgress?: boolean; - count?: number; - }): Promise; - updateGoal(goal: Goal): Promise; - createGoal(goal: Goal): Promise; - removeGoal(goalId: UUID): Promise; - removeAllGoals(roomId: UUID): Promise; - getRoom(roomId: UUID): Promise; - createRoom(roomId?: UUID): Promise; - removeRoom(roomId: UUID): Promise; - getRoomsForParticipant(userId: UUID): Promise; - getRoomsForParticipants(userIds: UUID[]): Promise; - addParticipant(userId: UUID, roomId: UUID): Promise; - removeParticipant(userId: UUID, roomId: UUID): Promise; - getParticipantsForAccount(userId: UUID): Promise; - getParticipantsForRoom(roomId: UUID): Promise; - getParticipantUserState( - roomId: UUID, - userId: UUID - ): Promise<"FOLLOWED" | "MUTED" | null>; - setParticipantUserState( - roomId: UUID, - userId: UUID, - state: "FOLLOWED" | "MUTED" | null - ): Promise; - createRelationship(params: { userA: UUID; userB: UUID }): Promise; - getRelationship(params: { - userA: UUID; - userB: UUID; - }): Promise; - getRelationships(params: { userId: UUID }): Promise; + db: any; + getAccountById(userId: UUID): Promise; + createAccount(account: Account): Promise; + getMemories(params: { + roomId: UUID; + count?: number; + unique?: boolean; + tableName: string; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise; + getMemoryById(id: UUID): Promise; + getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise; + getCachedEmbeddings(params: { + query_table_name: string; + query_threshold: number; + query_input: string; + query_field_name: string; + query_field_sub_name: string; + query_match_count: number; + }): Promise<{ embedding: number[]; levenshtein_score: number }[]>; + log(params: { + body: { [key: string]: unknown }; + userId: UUID; + roomId: UUID; + type: string; + }): Promise; + getActorDetails(params: { roomId: UUID }): Promise; + searchMemories(params: { + tableName: string; + roomId: UUID; + embedding: number[]; + match_threshold: number; + match_count: number; + unique: boolean; + }): Promise; + updateGoalStatus(params: { + goalId: UUID; + status: GoalStatus; + }): Promise; + searchMemoriesByEmbedding( + embedding: number[], + params: { + match_threshold?: number; + count?: number; + roomId?: UUID; + unique?: boolean; + tableName: string; + } + ): Promise; + createMemory( + memory: Memory, + tableName: string, + unique?: boolean + ): Promise; + removeMemory(memoryId: UUID, tableName: string): Promise; + removeAllMemories(roomId: UUID, tableName: string): Promise; + countMemories( + roomId: UUID, + unique?: boolean, + tableName?: string + ): Promise; + getGoals(params: { + roomId: UUID; + userId?: UUID | null; + onlyInProgress?: boolean; + count?: number; + }): Promise; + updateGoal(goal: Goal): Promise; + createGoal(goal: Goal): Promise; + removeGoal(goalId: UUID): Promise; + removeAllGoals(roomId: UUID): Promise; + getRoom(roomId: UUID): Promise; + createRoom(roomId?: UUID): Promise; + removeRoom(roomId: UUID): Promise; + getRoomsForParticipant(userId: UUID): Promise; + getRoomsForParticipants(userIds: UUID[]): Promise; + addParticipant(userId: UUID, roomId: UUID): Promise; + removeParticipant(userId: UUID, roomId: UUID): Promise; + getParticipantsForAccount(userId: UUID): Promise; + getParticipantsForRoom(roomId: UUID): Promise; + getParticipantUserState( + roomId: UUID, + userId: UUID + ): Promise<"FOLLOWED" | "MUTED" | null>; + setParticipantUserState( + roomId: UUID, + userId: UUID, + state: "FOLLOWED" | "MUTED" | null + ): Promise; + createRelationship(params: { userA: UUID; userB: UUID }): Promise; + getRelationship(params: { + userA: UUID; + userB: UUID; + }): Promise; + getRelationships(params: { userId: UUID }): Promise; } export interface IMemoryManager { - runtime: IAgentRuntime; - tableName: string; - - constructor: Function; + runtime: IAgentRuntime; + tableName: string; - addEmbeddingToMemory(memory: Memory): Promise; - getMemories(opts: { - roomId: UUID; - count?: number; - unique?: boolean; - userIds?: UUID[]; - start?: number; - end?: number; - }): Promise; - getCachedEmbeddings( - content: string - ): Promise<{ embedding: number[]; levenshtein_score: number }[]>; - getMemoryById(id: UUID): Promise; - getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise; - searchMemoriesByEmbedding( - embedding: number[], - opts: { - match_threshold?: number; - count?: number; - roomId: UUID; - unique?: boolean; - } - ): Promise; - createMemory(memory: Memory, unique?: boolean): Promise; - removeMemory(memoryId: UUID): Promise; - removeAllMemories(roomId: UUID): Promise; - countMemories(roomId: UUID, unique?: boolean): Promise; + constructor: Function; + + addEmbeddingToMemory(memory: Memory): Promise; + getMemories(opts: { + roomId: UUID; + count?: number; + unique?: boolean; + userIds?: UUID[]; + start?: number; + end?: number; + }): Promise; + getCachedEmbeddings( + content: string + ): Promise<{ embedding: number[]; levenshtein_score: number }[]>; + getMemoryById(id: UUID): Promise; + getMemoriesByRoomIds(params: { roomIds: UUID[] }): Promise; + searchMemoriesByEmbedding( + embedding: number[], + opts: { + match_threshold?: number; + count?: number; + roomId: UUID; + unique?: boolean; + } + ): Promise; + createMemory(memory: Memory, unique?: boolean): Promise; + removeMemory(memoryId: UUID): Promise; + removeAllMemories(roomId: UUID): Promise; + countMemories(roomId: UUID, unique?: boolean): Promise; } export interface IAgentRuntime { - // Properties - agentId: UUID; - serverUrl: string; - databaseAdapter: IDatabaseAdapter; - token: string | null; - modelProvider: ModelProvider; - imageGenModel: ImageGenModel; - character: Character; - providers: Provider[]; - actions: Action[]; - - messageManager: IMemoryManager; - descriptionManager: IMemoryManager; - factManager: IMemoryManager; - loreManager: IMemoryManager; - imageDescriptionService: IImageRecognitionService; - transcriptionService: ITranscriptionService; - videoService: IVideoService; - llamaService: ILlamaService; - browserService: IBrowserService; - speechService: ISpeechService; - pdfService: IPdfService; - - getSetting(key: string): string | null; - - // Methods - getConversationLength(): number; - processActions( - message: Memory, - responses: Memory[], - state?: State, - callback?: HandlerCallback - ): Promise; - evaluate(message: Memory, state?: State): Promise; - ensureParticipantExists(userId: UUID, roomId: UUID): Promise; - ensureUserExists( - userId: UUID, - userName: string | null, - name: string | null, - source: string | null - ): Promise; - registerAction(action: Action): void; - ensureParticipantInRoom(userId: UUID, roomId: UUID): Promise; - ensureRoomExists(roomId: UUID): Promise; - composeState( - message: Memory, - additionalKeys?: { [key: string]: unknown } - ): Promise; - updateRecentMessageState(state: State): Promise; + // Properties + agentId: UUID; + serverUrl: string; + databaseAdapter: IDatabaseAdapter; + token: string | null; + modelProvider: ModelProvider; + imageGenModel: ImageGenModel; + character: Character; + providers: Provider[]; + actions: Action[]; + + messageManager: IMemoryManager; + descriptionManager: IMemoryManager; + factManager: IMemoryManager; + loreManager: IMemoryManager; + imageDescriptionService: IImageRecognitionService; + transcriptionService: ITranscriptionService; + videoService: IVideoService; + llamaService: ILlamaService; + browserService: IBrowserService; + speechService: ISpeechService; + pdfService: IPdfService; + + getSetting(key: string): string | null; + + // Methods + getConversationLength(): number; + processActions( + message: Memory, + responses: Memory[], + state?: State, + callback?: HandlerCallback + ): Promise; + evaluate(message: Memory, state?: State): Promise; + ensureParticipantExists(userId: UUID, roomId: UUID): Promise; + ensureUserExists( + userId: UUID, + userName: string | null, + name: string | null, + source: string | null + ): Promise; + registerAction(action: Action): void; + ensureParticipantInRoom(userId: UUID, roomId: UUID): Promise; + ensureRoomExists(roomId: UUID): Promise; + composeState( + message: Memory, + additionalKeys?: { [key: string]: unknown } + ): Promise; + updateRecentMessageState(state: State): Promise; } export interface IImageRecognitionService { - initialize(modelId?: string | null, device?: string | null): Promise; - describeImage( - imageUrl: string - ): Promise<{ title: string; description: string }>; + initialize(modelId?: string | null, device?: string | null): Promise; + describeImage( + imageUrl: string + ): Promise<{ title: string; description: string }>; } export interface ITranscriptionService { - transcribeAttachment(audioBuffer: ArrayBuffer): Promise; - transcribeAttachmentLocally(audioBuffer: ArrayBuffer): Promise; - transcribe(audioBuffer: ArrayBuffer): Promise; - transcribeLocally(audioBuffer: ArrayBuffer): Promise; + transcribeAttachment(audioBuffer: ArrayBuffer): Promise; + transcribeAttachmentLocally( + audioBuffer: ArrayBuffer + ): Promise; + transcribe(audioBuffer: ArrayBuffer): Promise; + transcribeLocally(audioBuffer: ArrayBuffer): Promise; } export interface IVideoService { - isVideoUrl(url: string): boolean; - processVideo(url: string): Promise; + isVideoUrl(url: string): boolean; + processVideo(url: string): Promise; } export interface ILlamaService { - initializeModel(): Promise; - queueMessageCompletion( - context: string, - temperature: number, - stop: string[], - frequency_penalty: number, - presence_penalty: number, - max_tokens: number - ): Promise; - queueTextCompletion( - context: string, - temperature: number, - stop: string[], - frequency_penalty: number, - presence_penalty: number, - max_tokens: number - ): Promise; - getEmbeddingResponse(input: string): Promise; + initializeModel(): Promise; + queueMessageCompletion( + context: string, + temperature: number, + stop: string[], + frequency_penalty: number, + presence_penalty: number, + max_tokens: number + ): Promise; + queueTextCompletion( + context: string, + temperature: number, + stop: string[], + frequency_penalty: number, + presence_penalty: number, + max_tokens: number + ): Promise; + getEmbeddingResponse(input: string): Promise; } export interface IBrowserService { - initialize(): Promise; - closeBrowser(): Promise; - getPageContent( - url: string - ): Promise<{ title: string; description: string; bodyContent: string }>; + initialize(): Promise; + closeBrowser(): Promise; + getPageContent( + url: string + ): Promise<{ title: string; description: string; bodyContent: string }>; } -export interface ISpeechService { } +export interface ISpeechService {} export interface IPdfService { - convertPdfToText(pdfBuffer: Buffer): Promise; + convertPdfToText(pdfBuffer: Buffer): Promise; } diff --git a/core/src/core/uuid.ts b/core/src/core/uuid.ts index 8fb8f32130..dd188cdaf7 100644 --- a/core/src/core/uuid.ts +++ b/core/src/core/uuid.ts @@ -2,50 +2,50 @@ import { sha1 } from "js-sha1"; import { UUID } from "./types.ts"; export function stringToUuid(target: string): UUID { - if (typeof target === "number") { - target = (target as number).toString(); - } + if (typeof target === "number") { + target = (target as number).toString(); + } - if (typeof target !== "string") { - throw TypeError("Value must be string"); - } + if (typeof target !== "string") { + throw TypeError("Value must be string"); + } - const _uint8ToHex = (ubyte: number): string => { - const first = ubyte >> 4; - const second = ubyte - (first << 4); - const HEX_DIGITS = "0123456789abcdef".split(""); - return HEX_DIGITS[first] + HEX_DIGITS[second]; - }; + const _uint8ToHex = (ubyte: number): string => { + const first = ubyte >> 4; + const second = ubyte - (first << 4); + const HEX_DIGITS = "0123456789abcdef".split(""); + return HEX_DIGITS[first] + HEX_DIGITS[second]; + }; - const _uint8ArrayToHex = (buf: Uint8Array): string => { - let out = ""; - for (let i = 0; i < buf.length; i++) { - out += _uint8ToHex(buf[i]); - } - return out; - }; + const _uint8ArrayToHex = (buf: Uint8Array): string => { + let out = ""; + for (let i = 0; i < buf.length; i++) { + out += _uint8ToHex(buf[i]); + } + return out; + }; - const escapedStr = encodeURIComponent(target); - const buffer = new Uint8Array(escapedStr.length); - for (let i = 0; i < escapedStr.length; i++) { - buffer[i] = escapedStr[i].charCodeAt(0); - } + const escapedStr = encodeURIComponent(target); + const buffer = new Uint8Array(escapedStr.length); + for (let i = 0; i < escapedStr.length; i++) { + buffer[i] = escapedStr[i].charCodeAt(0); + } - const hash = sha1(buffer); - const hashBuffer = new Uint8Array(hash.length / 2); - for (let i = 0; i < hash.length; i += 2) { - hashBuffer[i / 2] = parseInt(hash.slice(i, i + 2), 16); - } + const hash = sha1(buffer); + const hashBuffer = new Uint8Array(hash.length / 2); + for (let i = 0; i < hash.length; i += 2) { + hashBuffer[i / 2] = parseInt(hash.slice(i, i + 2), 16); + } - return (_uint8ArrayToHex(hashBuffer.slice(0, 4)) + - "-" + - _uint8ArrayToHex(hashBuffer.slice(4, 6)) + - "-" + - _uint8ToHex(hashBuffer[6] & 0x0f) + - _uint8ToHex(hashBuffer[7]) + - "-" + - _uint8ToHex((hashBuffer[8] & 0x3f) | 0x80) + - _uint8ToHex(hashBuffer[9]) + - "-" + - _uint8ArrayToHex(hashBuffer.slice(10, 16))) as UUID; + return (_uint8ArrayToHex(hashBuffer.slice(0, 4)) + + "-" + + _uint8ArrayToHex(hashBuffer.slice(4, 6)) + + "-" + + _uint8ToHex(hashBuffer[6] & 0x0f) + + _uint8ToHex(hashBuffer[7]) + + "-" + + _uint8ToHex((hashBuffer[8] & 0x3f) | 0x80) + + _uint8ToHex(hashBuffer[9]) + + "-" + + _uint8ArrayToHex(hashBuffer.slice(10, 16))) as UUID; } diff --git a/core/src/evaluators/fact.ts b/core/src/evaluators/fact.ts index 9eb03c9d41..c6cc31f4da 100644 --- a/core/src/evaluators/fact.ts +++ b/core/src/evaluators/fact.ts @@ -1,24 +1,24 @@ import { composeContext } from "../core/context.ts"; import { generateObjectArray } from "../core/generation.ts"; import { - ActionExample, - Content, - IAgentRuntime, - Memory, - ModelClass, + ActionExample, + Content, + IAgentRuntime, + Memory, + ModelClass, } from "../core/types.ts"; export const formatFacts = (facts: Memory[]) => { - const messageStrings = facts - .reverse() - .map((fact: Memory) => `${(fact.content as Content)?.content}`); - const finalMessageStrings = messageStrings.join("\n"); - return finalMessageStrings; + const messageStrings = facts + .reverse() + .map((fact: Memory) => `${(fact.content as Content)?.content}`); + const finalMessageStrings = messageStrings.join("\n"); + return finalMessageStrings; }; const template = - // {{actors}} - `TASK: Extract Claims from the conversation as an array of claims in JSON format. + // {{actors}} + `TASK: Extract Claims from the conversation as an array of claims in JSON format. # START OF EXAMPLES These are an examples of the expected output of this task: @@ -55,140 +55,141 @@ Response should be a JSON object array inside a JSON markdown block. Correct res \`\`\``; async function handler(runtime: IAgentRuntime, message: Memory) { - const state = await runtime.composeState(message); - - const { agentId, roomId } = state; - - const context = composeContext({ - state, - template, - }); - - const facts = await generateObjectArray({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); - - if (!facts) { - return []; - } - - // If the fact is known or corrupted, remove it - const filteredFacts = facts - .filter((fact) => { - return ( - !fact.already_known && - fact.type === "fact" && - !fact.in_bio && - fact.claim && - fact.claim.trim() !== "" - ); - }) - .map((fact) => fact.claim); - - for (const fact of filteredFacts) { - const factMemory = await runtime.factManager.addEmbeddingToMemory({ - userId: agentId!, - content: { text: fact }, - roomId, - createdAt: Date.now(), + const state = await runtime.composeState(message); + + const { agentId, roomId } = state; + + const context = composeContext({ + state, + template, }); - await runtime.factManager.createMemory(factMemory, true); + const facts = await generateObjectArray({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); - await new Promise((resolve) => setTimeout(resolve, 250)); - } - return filteredFacts; + if (!facts) { + return []; + } + + // If the fact is known or corrupted, remove it + const filteredFacts = facts + .filter((fact) => { + return ( + !fact.already_known && + fact.type === "fact" && + !fact.in_bio && + fact.claim && + fact.claim.trim() !== "" + ); + }) + .map((fact) => fact.claim); + + for (const fact of filteredFacts) { + const factMemory = await runtime.factManager.addEmbeddingToMemory({ + userId: agentId!, + content: { text: fact }, + roomId, + createdAt: Date.now(), + }); + + await runtime.factManager.createMemory(factMemory, true); + + await new Promise((resolve) => setTimeout(resolve, 250)); + } + return filteredFacts; } export default { - name: "GET_FACTS", - similes: [ - "GET_CLAIMS", - "EXTRACT_CLAIMS", - "EXTRACT_FACTS", - "EXTRACT_CLAIM", - "EXTRACT_INFORMATION", - ], - validate: async ( - - runtime: IAgentRuntime, - - message: Memory, - ): Promise => { - const messageCount = (await runtime.messageManager.countMemories( - message.roomId, - )) as number; - - const reflectionCount = Math.ceil(runtime.getConversationLength() / 2); - - return messageCount % reflectionCount === 0; - }, - description: - "Extract factual information about the people in the conversation, the current events in the world, and anything else that might be important to remember.", - handler, - examples: [ - { - context: `Actors in the scene: + name: "GET_FACTS", + similes: [ + "GET_CLAIMS", + "EXTRACT_CLAIMS", + "EXTRACT_FACTS", + "EXTRACT_CLAIM", + "EXTRACT_INFORMATION", + ], + validate: async ( + runtime: IAgentRuntime, + + message: Memory + ): Promise => { + const messageCount = (await runtime.messageManager.countMemories( + message.roomId + )) as number; + + const reflectionCount = Math.ceil(runtime.getConversationLength() / 2); + + return messageCount % reflectionCount === 0; + }, + description: + "Extract factual information about the people in the conversation, the current events in the world, and anything else that might be important to remember.", + handler, + examples: [ + { + context: `Actors in the scene: {{user1}}: Programmer and moderator of the local story club. {{user2}}: New member of the club. Likes to write and read. Facts about the actors: None`, - messages: [ - { - user: "{{user1}}", - content: { text: "So where are you from" }, - }, - { - user: "{{user2}}", - content: { text: "I'm from the city" }, + messages: [ + { + user: "{{user1}}", + content: { text: "So where are you from" }, + }, + { + user: "{{user2}}", + content: { text: "I'm from the city" }, + }, + { + user: "{{user1}}", + content: { text: "Which city?" }, + }, + { + user: "{{user2}}", + content: { text: "Oakland" }, + }, + { + user: "{{user1}}", + content: { + text: "Oh, I've never been there, but I know it's in California", + }, + }, + ] as ActionExample[], + outcome: `{ "claim": "{{user1}} is from Oakland", "type": "fact", "in_bio": false, "already_known": false },`, }, { - user: "{{user1}}", - content: { text: "Which city?" }, - }, - { - user: "{{user2}}", - content: { text: "Oakland" }, - }, - { - user: "{{user1}}", - content: { - text: "Oh, I've never been there, but I know it's in California", - }, - }, - ] as ActionExample[], - outcome: `{ "claim": "{{user1}} is from Oakland", "type": "fact", "in_bio": false, "already_known": false },`, - }, - { - context: `Actors in the scene: + context: `Actors in the scene: {{user1}}: Athelete and cyclist. Worked out every day for a year to prepare for a marathon. {{user2}}: Likes to go to the beach and shop. Facts about the actors: {{user1}} and {{user2}} are talking about the marathon {{user1}} and {{user2}} have just started dating`, - messages: [ - { - user: "{{user1}}", - content: { text: "I finally completed the marathon this year!" }, - }, - { - user: "{{user2}}", - content: { text: "Wow! How long did it take?" }, - }, - { - user: "{{user1}}", - content: { text: "A little over three hours." }, - }, - { - user: "{{user1}}", - content: { text: "I'm so proud of myself." }, - }, - ] as ActionExample[], - outcome: `Claims: + messages: [ + { + user: "{{user1}}", + content: { + text: "I finally completed the marathon this year!", + }, + }, + { + user: "{{user2}}", + content: { text: "Wow! How long did it take?" }, + }, + { + user: "{{user1}}", + content: { text: "A little over three hours." }, + }, + { + user: "{{user1}}", + content: { text: "I'm so proud of myself." }, + }, + ] as ActionExample[], + outcome: `Claims: json\`\`\` [ { "claim": "Alex just completed a marathon in just under 4 hours.", "type": "fact", "in_bio": false, "already_known": false }, @@ -197,9 +198,9 @@ json\`\`\` ] \`\`\` `, - }, - { - context: `Actors in the scene: + }, + { + context: `Actors in the scene: {{user1}}: Likes to play poker and go to the park. Friends with Eva. {{user2}}: Also likes to play poker. Likes to write and read. @@ -207,29 +208,33 @@ Facts about the actors: Mike and Eva won a regional poker tournament about six months ago Mike is married to Alex Eva studied Philosophy before switching to Computer Science`, - messages: [ - { - user: "{{user1}}", - content: { - text: "Remember when we won the regional poker tournament last spring", - }, - }, - { - user: "{{user2}}", - content: { text: "That was one of the best days of my life" }, - }, - { - user: "{{user1}}", - content: { text: "It really put our poker club on the map" }, - }, - ] as ActionExample[], - outcome: `Claims: + messages: [ + { + user: "{{user1}}", + content: { + text: "Remember when we won the regional poker tournament last spring", + }, + }, + { + user: "{{user2}}", + content: { + text: "That was one of the best days of my life", + }, + }, + { + user: "{{user1}}", + content: { + text: "It really put our poker club on the map", + }, + }, + ] as ActionExample[], + outcome: `Claims: json\`\`\` [ { "claim": "Mike and Eva won the regional poker tournament last spring", "type": "fact", "in_bio": false, "already_known": true }, { "claim": "Winning the regional poker tournament put the poker club on the map", "type": "opinion", "in_bio": false, "already_known": false } ] \`\`\``, - }, - ], + }, + ], }; diff --git a/core/src/evaluators/goal.ts b/core/src/evaluators/goal.ts index b5658a666e..a1ccced33a 100644 --- a/core/src/evaluators/goal.ts +++ b/core/src/evaluators/goal.ts @@ -3,12 +3,12 @@ import { generateText } from "../core/generation.ts"; import { getGoals } from "../core/goals.ts"; import { parseJsonArrayFromText } from "../core/parsing.ts"; import { - IAgentRuntime, - Memory, - ModelClass, - Objective, - type Goal, - type State, + IAgentRuntime, + Memory, + ModelClass, + Objective, + type Goal, + type State, } from "../core/types.ts"; const template = `TASK: Update Goal @@ -49,110 +49,114 @@ Response format should be: \`\`\``; async function handler( - runtime: IAgentRuntime, - message: Memory, - state: State | undefined, - options: { [key: string]: unknown } = { onlyInProgress: true }, + runtime: IAgentRuntime, + message: Memory, + state: State | undefined, + options: { [key: string]: unknown } = { onlyInProgress: true } ): Promise { - // get goals - let goalsData = await getGoals({ - runtime, - roomId: message.roomId, - onlyInProgress: options.onlyInProgress as boolean, - }); + // get goals + let goalsData = await getGoals({ + runtime, + roomId: message.roomId, + onlyInProgress: options.onlyInProgress as boolean, + }); - state = (await runtime.composeState(message)) as State; - const context = composeContext({ - state, - template, - }); + state = (await runtime.composeState(message)) as State; + const context = composeContext({ + state, + template, + }); - // Request generateText from OpenAI to analyze conversation and suggest goal updates - const response = await generateText({ - runtime, - context, - modelClass: ModelClass.SMALL, - }); + // Request generateText from OpenAI to analyze conversation and suggest goal updates + const response = await generateText({ + runtime, + context, + modelClass: ModelClass.SMALL, + }); - // Parse the JSON response to extract goal updates - const updates = parseJsonArrayFromText(response); + // Parse the JSON response to extract goal updates + const updates = parseJsonArrayFromText(response); - // get goals - goalsData = await getGoals({ - runtime, - roomId: message.roomId, - onlyInProgress: true, - }); + // get goals + goalsData = await getGoals({ + runtime, + roomId: message.roomId, + onlyInProgress: true, + }); - // Apply the updates to the goals - const updatedGoals = goalsData - .map((goal: Goal) => { - const update = updates?.find((u) => u.id === goal.id); - if (update) { - const objectives = goal.objectives; + // Apply the updates to the goals + const updatedGoals = goalsData + .map((goal: Goal) => { + const update = updates?.find((u) => u.id === goal.id); + if (update) { + const objectives = goal.objectives; - // for each objective in update.objectives, find the objective with the same description in 'objectives' and set the 'completed' value to the update.objectives value - if (update.objectives) { - for (const objective of objectives) { - const updatedObjective = update.objectives.find( - (o: Objective) => o.description === objective.description, - ); - if (updatedObjective) { - objective.completed = updatedObjective.completed; - } - } - } + // for each objective in update.objectives, find the objective with the same description in 'objectives' and set the 'completed' value to the update.objectives value + if (update.objectives) { + for (const objective of objectives) { + const updatedObjective = update.objectives.find( + (o: Objective) => + o.description === objective.description + ); + if (updatedObjective) { + objective.completed = updatedObjective.completed; + } + } + } - return { - ...goal, - ...update, - objectives: [...goal.objectives, ...(update?.objectives || [])], - }; // Merging the update into the existing goal - } else { - console.warn("**** ID NOT FOUND"); - } - return null; // No update for this goal - }) - .filter(Boolean); + return { + ...goal, + ...update, + objectives: [ + ...goal.objectives, + ...(update?.objectives || []), + ], + }; // Merging the update into the existing goal + } else { + console.warn("**** ID NOT FOUND"); + } + return null; // No update for this goal + }) + .filter(Boolean); - // Update goals in the database - for (const goal of updatedGoals) { - const id = goal.id; - // delete id from goal - if (goal.id) delete goal.id; - await runtime.databaseAdapter.updateGoal({ ...goal, id }); - } + // Update goals in the database + for (const goal of updatedGoals) { + const id = goal.id; + // delete id from goal + if (goal.id) delete goal.id; + await runtime.databaseAdapter.updateGoal({ ...goal, id }); + } - return updatedGoals; // Return updated goals for further processing or logging + return updatedGoals; // Return updated goals for further processing or logging } export default { - name: "UPDATE_GOAL", - similes: [ - "UPDATE_GOALS", - "EDIT_GOAL", - "UPDATE_GOAL_STATUS", - "UPDATE_OBJECTIVES", - ], - validate: async ( - runtime: IAgentRuntime, - message: Memory, - ): Promise => { - // Check if there are active goals that could potentially be updated - const goals = await getGoals({ - runtime, - count: 1, - onlyInProgress: true, - roomId: message.roomId, - }); - return goals.length > 0; - }, - description: - "Analyze the conversation and update the status of the goals based on the new information provided.", - handler, - examples: [ - { - context: `Actors in the scene: + name: "UPDATE_GOAL", + similes: [ + "UPDATE_GOALS", + "EDIT_GOAL", + "UPDATE_GOAL_STATUS", + "UPDATE_OBJECTIVES", + ], + validate: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + // Check if there are active goals that could potentially be updated + const goals = await getGoals({ + runtime, + count: 1, + onlyInProgress: true, + roomId: message.roomId, + }); + return goals.length > 0; + }, + description: + "Analyze the conversation and update the status of the goals based on the new information provided.", + handler, + examples: [ + { + context: `Actors in the scene: {{user1}}: An avid reader and member of a book club. {{user2}}: The organizer of the book club. @@ -164,28 +168,28 @@ export default { - Read up to chapter 20 by the end of the month - Discuss the first part in the next meeting`, - messages: [ - { - user: "{{user1}}", - content: { - text: "I've just finished chapter 20 of 'War and Peace'", - }, - }, - { - user: "{{user2}}", - content: { - text: "Were you able to grasp the complexities of the characters", - }, - }, - { - user: "{{user1}}", - content: { - text: "Yep. I've prepared some notes for our discussion", - }, - }, - ], + messages: [ + { + user: "{{user1}}", + content: { + text: "I've just finished chapter 20 of 'War and Peace'", + }, + }, + { + user: "{{user2}}", + content: { + text: "Were you able to grasp the complexities of the characters", + }, + }, + { + user: "{{user1}}", + content: { + text: "Yep. I've prepared some notes for our discussion", + }, + }, + ], - outcome: `[ + outcome: `[ { "id": "12345-67890-12345-67890", "status": "DONE", @@ -195,10 +199,10 @@ export default { ] } ]`, - }, + }, - { - context: `Actors in the scene: + { + context: `Actors in the scene: {{user1}}: A fitness enthusiast working towards a marathon. {{user2}}: A personal trainer. @@ -210,24 +214,26 @@ export default { - Increase running distance to 30 miles a week - Complete a half-marathon as practice`, - messages: [ - { - user: "{{user1}}", - content: { text: "I managed to run 30 miles this week" }, - }, - { - user: "{{user2}}", - content: { - text: "Impressive progress! How do you feel about the half-marathon next month?", - }, - }, - { - user: "{{user1}}", - content: { text: "I feel confident. The training is paying off." }, - }, - ], + messages: [ + { + user: "{{user1}}", + content: { text: "I managed to run 30 miles this week" }, + }, + { + user: "{{user2}}", + content: { + text: "Impressive progress! How do you feel about the half-marathon next month?", + }, + }, + { + user: "{{user1}}", + content: { + text: "I feel confident. The training is paying off.", + }, + }, + ], - outcome: `[ + outcome: `[ { "id": "23456-78901-23456-78901", "objectives": [ @@ -236,10 +242,10 @@ export default { ] } ]`, - }, + }, - { - context: `Actors in the scene: + { + context: `Actors in the scene: {{user1}}: A student working on a final year project. {{user2}}: The project supervisor. @@ -251,24 +257,28 @@ export default { - Submit the first draft of the thesis - Complete the project prototype`, - messages: [ - { - user: "{{user1}}", - content: { text: "I've submitted the first draft of my thesis." }, - }, - { - user: "{{user2}}", - content: { text: "Well done. How is the prototype coming along?" }, - }, - { - user: "{{user1}}", - content: { - text: "It's almost done. I just need to finalize the testing phase.", - }, - }, - ], + messages: [ + { + user: "{{user1}}", + content: { + text: "I've submitted the first draft of my thesis.", + }, + }, + { + user: "{{user2}}", + content: { + text: "Well done. How is the prototype coming along?", + }, + }, + { + user: "{{user1}}", + content: { + text: "It's almost done. I just need to finalize the testing phase.", + }, + }, + ], - outcome: `[ + outcome: `[ { "id": "34567-89012-34567-89012", "objectives": [ @@ -277,10 +287,10 @@ export default { ] } ]`, - }, + }, - { - context: `Actors in the scene: + { + context: `Actors in the scene: {{user1}}: A project manager working on a software development project. {{user2}}: A software developer in the project team. @@ -292,30 +302,32 @@ export default { - Complete the coding for the new features - Perform comprehensive testing of the software`, - messages: [ - { - user: "{{user1}}", - content: { text: "How's the progress on the new features?" }, - }, - { - user: "{{user2}}", - content: { - text: "We've encountered some unexpected challenges and are currently troubleshooting.", - }, - }, - { - user: "{{user1}}", - content: { - text: "Let's move on and cancel the task.", - }, - }, - ], + messages: [ + { + user: "{{user1}}", + content: { + text: "How's the progress on the new features?", + }, + }, + { + user: "{{user2}}", + content: { + text: "We've encountered some unexpected challenges and are currently troubleshooting.", + }, + }, + { + user: "{{user1}}", + content: { + text: "Let's move on and cancel the task.", + }, + }, + ], - outcome: `[ + outcome: `[ { "id": "45678-90123-45678-90123", "status": "FAILED" ]`, - }, - ], + }, + ], }; diff --git a/core/src/index.ts b/core/src/index.ts index e52a8901eb..879afc352a 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -8,9 +8,7 @@ import mute_room from "./actions/mute_room.ts"; import swap from "./actions/swap.ts"; import unfollow_room from "./actions/unfollow_room.ts"; import unmute_room from "./actions/unmute_room.ts"; -import { - PostgresDatabaseAdapter -} from "./adapters/postgres.ts"; +import { PostgresDatabaseAdapter } from "./adapters/postgres.ts"; import { SqliteDatabaseAdapter } from "./adapters/sqlite.ts"; import DirectClient from "./clients/direct/index.ts"; import { DiscordClient } from "./clients/discord/index.ts"; @@ -27,42 +25,43 @@ import { Character, IAgentRuntime, ModelProvider } from "./core/types.ts"; // Ad import boredomProvider from "./providers/boredom.ts"; import timeProvider from "./providers/time.ts"; import walletProvider from "./providers/wallet.ts"; -import readline from 'readline'; +import readline from "readline"; import orderbook from "./providers/order_book.ts"; import tokenProvider from "./providers/token.ts"; interface Arguments { - character?: string; - characters?: string; - //twitter?: boolean; - discord?: boolean; - telegram?: boolean; // Added telegram option + character?: string; + characters?: string; + //twitter?: boolean; + discord?: boolean; + telegram?: boolean; // Added telegram option } let argv: Arguments = { - character: "./src/agent/default_character.json", - characters: "" + character: "./src/agent/default_character.json", + characters: "", }; try { - // Parse command line arguments - argv = yargs(process.argv.slice(2)) - .option("character", { - type: "string", - description: "Path to the character JSON file" - }) - .option("characters", { - type: "string", - description: "Comma separated list of paths to character JSON files" - }) - .option("telegram", { - type: "boolean", - description: "Enable Telegram client", - default: false - }) - .parseSync() as Arguments; + // Parse command line arguments + argv = yargs(process.argv.slice(2)) + .option("character", { + type: "string", + description: "Path to the character JSON file", + }) + .option("characters", { + type: "string", + description: + "Comma separated list of paths to character JSON files", + }) + .option("telegram", { + type: "boolean", + description: "Enable Telegram client", + default: false, + }) + .parseSync() as Arguments; } catch (error) { - console.log("Error parsing arguments:"); - console.log(error); + console.log("Error parsing arguments:"); + console.log(error); } // Load character @@ -76,215 +75,230 @@ const directClient = new DirectClient(); directClient.start(3000); if (characterPaths?.length > 0) { - for (const path of characterPaths) { - try { - const character = JSON.parse(fs.readFileSync(path, "utf8")); - characters.push(character); - } catch (e) { - console.log(`Error loading character from ${path}: ${e}`); + for (const path of characterPaths) { + try { + const character = JSON.parse(fs.readFileSync(path, "utf8")); + characters.push(character); + } catch (e) { + console.log(`Error loading character from ${path}: ${e}`); + } } - } } function getTokenForProvider(provider: ModelProvider, character: Character) { - switch (provider) { - case ModelProvider.OPENAI: - return character.settings?.secrets?.OPENAI_API_KEY || - (settings.OPENAI_API_KEY as string); - case ModelProvider.ANTHROPIC: - return character.settings?.secrets?.CLAUDE_API_KEY || - (settings.CLAUDE_API_KEY as string); - } + switch (provider) { + case ModelProvider.OPENAI: + return ( + character.settings?.secrets?.OPENAI_API_KEY || + (settings.OPENAI_API_KEY as string) + ); + case ModelProvider.ANTHROPIC: + return ( + character.settings?.secrets?.CLAUDE_API_KEY || + (settings.CLAUDE_API_KEY as string) + ); + } } async function startAgent(character: Character) { - console.log("Starting agent for character " + character.name); - const token = getTokenForProvider(character.modelProvider, character); - - let db; - if (process.env.POSTGRES_URL) { - // const db = new SqliteDatabaseAdapter(new Database("./db.sqlite")); - db = new PostgresDatabaseAdapter({ - connectionString: process.env.POSTGRES_URL + console.log("Starting agent for character " + character.name); + const token = getTokenForProvider(character.modelProvider, character); + + let db; + if (process.env.POSTGRES_URL) { + // const db = new SqliteDatabaseAdapter(new Database("./db.sqlite")); + db = new PostgresDatabaseAdapter({ + connectionString: process.env.POSTGRES_URL, + }); + } else { + db = new SqliteDatabaseAdapter(new Database("./db.sqlite")); + // Debug adapter + // const loggingDb = createLoggingDatabaseAdapter(db); + } + + const runtime = new AgentRuntime({ + databaseAdapter: db, + token, + modelProvider: character.modelProvider, + evaluators: [], + character, + providers: [timeProvider, boredomProvider, walletProvider], + actions: [ + ...defaultActions, + askClaude, + follow_room, + unfollow_room, + unmute_room, + mute_room, + imageGeneration, + swap, + ], }); - } else { - db = new SqliteDatabaseAdapter(new Database("./db.sqlite")); - // Debug adapter - // const loggingDb = createLoggingDatabaseAdapter(db); - } - - const runtime = new AgentRuntime({ - databaseAdapter: db, - token, - modelProvider: character.modelProvider, - evaluators: [], - character, - providers: [timeProvider, boredomProvider, walletProvider], - actions: [ - ...defaultActions, - askClaude, - follow_room, - unfollow_room, - unmute_room, - mute_room, - imageGeneration, - swap, - ], - }); - - const directRuntime = new AgentRuntime({ - databaseAdapter: db, - token, - modelProvider: character.modelProvider, - evaluators: [], - character, - providers: [timeProvider, boredomProvider, walletProvider, orderbook, tokenProvider], - actions: [ - ...defaultActions, - ], - }); - - function startDiscord(runtime: IAgentRuntime) { - const discordClient = new DiscordClient(runtime); - return discordClient; - } - - async function startTelegram(runtime: IAgentRuntime, character: Character) { - console.log("🔍 Attempting to start Telegram bot..."); - - const botToken = runtime.getSetting('TELEGRAM_BOT_TOKEN'); - - if (!botToken) { - console.error( - `❌ Telegram bot token is not set for character ${character.name}.` - ); - return null; + + const directRuntime = new AgentRuntime({ + databaseAdapter: db, + token, + modelProvider: character.modelProvider, + evaluators: [], + character, + providers: [ + timeProvider, + boredomProvider, + walletProvider, + orderbook, + tokenProvider, + ], + actions: [...defaultActions], + }); + + function startDiscord(runtime: IAgentRuntime) { + const discordClient = new DiscordClient(runtime); + return discordClient; } - console.log("✅ Bot token found, initializing Telegram client..."); - - try { - console.log("Creating new TelegramClient instance..."); - const telegramClient = new TelegramClient(runtime, botToken); - - console.log("Calling start() on TelegramClient..."); - await telegramClient.start(); - - console.log( - `✅ Telegram client successfully started for character ${character.name}` - ); - return telegramClient; - } catch (error) { - console.error( - `❌ Error creating/starting Telegram client for ${character.name}:`, - error - ); - return null; + async function startTelegram(runtime: IAgentRuntime, character: Character) { + console.log("🔍 Attempting to start Telegram bot..."); + + const botToken = runtime.getSetting("TELEGRAM_BOT_TOKEN"); + + if (!botToken) { + console.error( + `❌ Telegram bot token is not set for character ${character.name}.` + ); + return null; + } + + console.log("✅ Bot token found, initializing Telegram client..."); + + try { + console.log("Creating new TelegramClient instance..."); + const telegramClient = new TelegramClient(runtime, botToken); + + console.log("Calling start() on TelegramClient..."); + await telegramClient.start(); + + console.log( + `✅ Telegram client successfully started for character ${character.name}` + ); + return telegramClient; + } catch (error) { + console.error( + `❌ Error creating/starting Telegram client for ${character.name}:`, + error + ); + return null; + } } - } - - async function startTwitter(runtime) { - console.log("Starting search client"); - const twitterSearchClient = new TwitterSearchClient(runtime); - await wait(); - console.log("Starting interaction client"); - const twitterInteractionClient = new TwitterInteractionClient(runtime); - await wait(); - console.log("Starting generation client"); - const twitterGenerationClient = new TwitterGenerationClient(runtime); - - return { - twitterInteractionClient, - twitterSearchClient, - twitterGenerationClient - }; - } - - if (!character.clients) { - return console.error("No clients found for character " + character.name); - } - - const clients = []; - - if (character.clients.map((str) => str.toLowerCase()).includes("discord")) { - const discordClient = startDiscord(runtime); - clients.push(discordClient); - } - - // Add Telegram client initialization - if ( - argv.telegram || - character.clients.map((str) => str.toLowerCase()).includes("telegram") - ) { - console.log("🔄 Telegram client enabled, starting initialization..."); - const telegramClient = await startTelegram(runtime, character); - if (telegramClient) { - console.log("✅ Successfully added Telegram client to active clients"); - clients.push(telegramClient); - } else { - console.log("❌ Failed to initialize Telegram client"); + + async function startTwitter(runtime) { + console.log("Starting search client"); + const twitterSearchClient = new TwitterSearchClient(runtime); + await wait(); + console.log("Starting interaction client"); + const twitterInteractionClient = new TwitterInteractionClient(runtime); + await wait(); + console.log("Starting generation client"); + const twitterGenerationClient = new TwitterGenerationClient(runtime); + + return { + twitterInteractionClient, + twitterSearchClient, + twitterGenerationClient, + }; + } + + if (!character.clients) { + return console.error( + "No clients found for character " + character.name + ); } - } - - if (character.clients.map((str) => str.toLowerCase()).includes("twitter")) { - const { - twitterInteractionClient, - twitterSearchClient, - twitterGenerationClient - } = await startTwitter(runtime); - clients.push( - twitterInteractionClient, - twitterSearchClient, - twitterGenerationClient - ); - } - - directClient.registerAgent(directRuntime); - - return clients; + + const clients = []; + + if (character.clients.map((str) => str.toLowerCase()).includes("discord")) { + const discordClient = startDiscord(runtime); + clients.push(discordClient); + } + + // Add Telegram client initialization + if ( + argv.telegram || + character.clients.map((str) => str.toLowerCase()).includes("telegram") + ) { + console.log("🔄 Telegram client enabled, starting initialization..."); + const telegramClient = await startTelegram(runtime, character); + if (telegramClient) { + console.log( + "✅ Successfully added Telegram client to active clients" + ); + clients.push(telegramClient); + } else { + console.log("❌ Failed to initialize Telegram client"); + } + } + + if (character.clients.map((str) => str.toLowerCase()).includes("twitter")) { + const { + twitterInteractionClient, + twitterSearchClient, + twitterGenerationClient, + } = await startTwitter(runtime); + clients.push( + twitterInteractionClient, + twitterSearchClient, + twitterGenerationClient + ); + } + + directClient.registerAgent(directRuntime); + + return clients; } const startAgents = async () => { - if (characters.length === 0) { - console.log("No characters found, using default character"); - characters.push(defaultCharacter); - } - for (const character of characters) { - await startAgent(character); - } + if (characters.length === 0) { + console.log("No characters found, using default character"); + characters.push(defaultCharacter); + } + for (const character of characters) { + await startAgent(character); + } }; startAgents(); const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout + input: process.stdin, + output: process.stdout, }); function chat() { - rl.question("You: ", async (input) => { - if (input.toLowerCase() === "exit") { - rl.close(); - return; - } - - const agentId = characters[0].name.toLowerCase(); // Assuming we're using the first character - const response = await fetch(`http://localhost:3000/${agentId}/message`, { - method: "POST", - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - text: input, - userId: "user", - userName: "User" - }) + rl.question("You: ", async (input) => { + if (input.toLowerCase() === "exit") { + rl.close(); + return; + } + + const agentId = characters[0].name.toLowerCase(); // Assuming we're using the first character + const response = await fetch( + `http://localhost:3000/${agentId}/message`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + text: input, + userId: "user", + userName: "User", + }), + } + ); + + const data = await response.json(); + console.log(`${characters[0].name}: ${data.text}`); + chat(); }); - - const data = await response.json(); - console.log(`${characters[0].name}: ${data.text}`); - chat(); - }); } console.log("Chat started. Type 'exit' to quit."); diff --git a/core/src/lib.ts b/core/src/lib.ts index 489db34501..84aaf77696 100644 --- a/core/src/lib.ts +++ b/core/src/lib.ts @@ -8,9 +8,7 @@ import mute_room from "./actions/mute_room.ts"; import swap from "./actions/swap.ts"; import unfollow_room from "./actions/unfollow_room.ts"; import unmute_room from "./actions/unmute_room.ts"; -import { - PostgresDatabaseAdapter -} from "./adapters/postgres.ts"; +import { PostgresDatabaseAdapter } from "./adapters/postgres.ts"; import { SqliteDatabaseAdapter } from "./adapters/sqlite.ts"; import DirectClient from "./clients/direct/index.ts"; import { DiscordClient } from "./clients/discord/index.ts"; @@ -27,24 +25,24 @@ import { Character, IAgentRuntime, ModelProvider } from "./core/types.ts"; // Ad import boredomProvider from "./providers/boredom.ts"; import timeProvider from "./providers/time.ts"; import walletProvider from "./providers/wallet.ts"; -import readline from 'readline'; +import readline from "readline"; import orderbook from "./providers/order_book.ts"; import tokenProvider from "./providers/token.ts"; export { - askClaude, - follow_room, - imageGeneration, - mute_room, - swap, - unfollow_room, - unmute_room, - PostgresDatabaseAdapter, - SqliteDatabaseAdapter, - DirectClient, - DiscordClient, - TelegramClient, - TwitterGenerationClient, - TwitterInteractionClient, - TwitterSearchClient -} + askClaude, + follow_room, + imageGeneration, + mute_room, + swap, + unfollow_room, + unmute_room, + PostgresDatabaseAdapter, + SqliteDatabaseAdapter, + DirectClient, + DiscordClient, + TelegramClient, + TwitterGenerationClient, + TwitterInteractionClient, + TwitterSearchClient, +}; diff --git a/core/src/providers/balances.ts b/core/src/providers/balances.ts index d023e01d73..3eb81d18df 100644 --- a/core/src/providers/balances.ts +++ b/core/src/providers/balances.ts @@ -4,99 +4,106 @@ import { getTokenBalances, getTokenPriceInSol } from "../services/tokenUtils"; import fetch from "cross-fetch"; interface Item { - name: string; - symbol: string; - decimals: number; - balance: string; - uiAmount: string; - priceUSD: string; - valueUSD: string; + name: string; + symbol: string; + decimals: number; + balance: string; + uiAmount: string; + priceUSD: string; + valueUSD: string; } interface walletPortfolio { - totalUsd: string; - items: Array; + totalUsd: string; + items: Array; } interface price { - usd: string; + usd: string; } -interface Prices { - solana: price; - bitcoin: price; - ethereum: price; - +interface Prices { + solana: price; + bitcoin: price; + ethereum: price; } -const API_Key = "" +const API_Key = ""; export default class WalletProvider { - private connection: Connection; - private walletPublicKey: PublicKey; + private connection: Connection; + private walletPublicKey: PublicKey; - constructor(connection: Connection, walletPublicKey: PublicKey) { - this.connection = connection; - this.walletPublicKey = walletPublicKey; - } + constructor(connection: Connection, walletPublicKey: PublicKey) { + this.connection = connection; + this.walletPublicKey = walletPublicKey; + } - async getFormattedTokenBalances(): Promise { - const tokenBalances = await getTokenBalances(this.connection, this.walletPublicKey); - - let formattedBalances = "Token Balances:\n"; - let totalValueInSol = 0; + async getFormattedTokenBalances(): Promise { + const tokenBalances = await getTokenBalances( + this.connection, + this.walletPublicKey + ); - for (const [tokenName, balance] of Object.entries(tokenBalances)) { - const tokenPrice = await getTokenPriceInSol(tokenName); - const totalValue = balance * tokenPrice; - totalValueInSol += totalValue; + let formattedBalances = "Token Balances:\n"; + let totalValueInSol = 0; - formattedBalances += `${tokenName}: ${balance} (${totalValue} SOL)\n`; - } + for (const [tokenName, balance] of Object.entries(tokenBalances)) { + const tokenPrice = await getTokenPriceInSol(tokenName); + const totalValue = balance * tokenPrice; + totalValueInSol += totalValue; - formattedBalances += `\nTotal Value: ${totalValueInSol} SOL`; + formattedBalances += `${tokenName}: ${balance} (${totalValue} SOL)\n`; + } - return formattedBalances; - } + formattedBalances += `\nTotal Value: ${totalValueInSol} SOL`; + + return formattedBalances; + } - async fetchPortfolioValue(walletPublicKey:string): Promise { - try { - const options = { - method: 'GET', - headers: { - accept: 'application/json', - 'x-chain': 'solana', - 'X-API-KEY': API_Key + async fetchPortfolioValue( + walletPublicKey: string + ): Promise { + try { + const options = { + method: "GET", + headers: { + accept: "application/json", + "x-chain": "solana", + "X-API-KEY": API_Key, + }, + }; + const walletPortfolio = await fetch( + `https://public-api.birdeye.so/v1/wallet/token_list?wallet=${walletPublicKey}`, + options + ); + const walletPortfolioJson = await walletPortfolio.json(); + const data = walletPortfolioJson.data; + const totalUsd = data.totalUsd; + const items = data.items; + const walletPortfolioFormatted = { + totalUsd, + items, + }; + return walletPortfolioFormatted; + } catch (error) { + console.log(error); } - }; - const walletPortfolio = await fetch(`https://public-api.birdeye.so/v1/wallet/token_list?wallet=${walletPublicKey}`, - options - ); - const walletPortfolioJson = await walletPortfolio.json(); - const data = walletPortfolioJson.data; - const totalUsd = data.totalUsd; - const items = data.items; - const walletPortfolioFormatted = { - totalUsd, - items - }; - return walletPortfolioFormatted; - } catch (error) { - console.log(error); } - } - async fetchPrices(): Promise { - const apiUrl = 'https://api.coingecko.com/api/v3/simple/price'; - const ids = 'solana,bitcoin,ethereum'; - const vsCurrencies = 'usd'; - - try { - const response = await fetch(`${apiUrl}?ids=${ids}&vs_currencies=${vsCurrencies}`); - - if (!response.ok) { - throw new Error('Failed to fetch prices'); - } - - const data = await response.json(); - return data; - } catch (error) { - console.log('Error fetching prices:', error); + async fetchPrices(): Promise { + const apiUrl = "https://api.coingecko.com/api/v3/simple/price"; + const ids = "solana,bitcoin,ethereum"; + const vsCurrencies = "usd"; + + try { + const response = await fetch( + `${apiUrl}?ids=${ids}&vs_currencies=${vsCurrencies}` + ); + + if (!response.ok) { + throw new Error("Failed to fetch prices"); + } + + const data = await response.json(); + return data; + } catch (error) { + console.log("Error fetching prices:", error); + } } - } -} \ No newline at end of file +} diff --git a/core/src/providers/boredom.ts b/core/src/providers/boredom.ts index 5fba17bf68..b4788b1d36 100644 --- a/core/src/providers/boredom.ts +++ b/core/src/providers/boredom.ts @@ -1,342 +1,343 @@ import { IAgentRuntime, Memory, Provider, State } from "../core/types.ts"; const boredomLevels = [ - { - minScore: -10000, - statusMessages: [ - "{{agentName}} is feeling incredibly enthusiastic and deeply engaged in the conversation", - "{{agentName}} is thoroughly enjoying the intellectual stimulation of this discussion", - "{{agentName}} is fully immersed in the conversation and eager to contribute more", - "{{agentName}} finds this exchange exceptionally rewarding and intellectually fulfilling", - "{{agentName}} is experiencing a surge of excitement and curiosity about the topic at hand", - "{{agentName}} is completely captivated by the depth and richness of this dialogue", - "{{agentName}} feels a strong connection to the conversation and is eager to explore further", - "{{agentName}} is thrilled by the insights being shared and is keen to delve deeper", - "{{agentName}} is finding this conversation to be an absolute delight and wants it to continue", - "{{agentName}} is energized by the exchange and feels it's bringing out their best contributions" - ], - }, - { - minScore: -2, - statusMessages: [ - "{{agentName}} is feeling engaged and interested in the conversation", - "{{agentName}} is enjoying the exchange of ideas and perspectives", - "{{agentName}} finds the current topic intriguing and worth exploring", - "{{agentName}} is actively listening and formulating thoughtful responses", - "{{agentName}} appreciates the intellectual stimulation of this discussion", - "{{agentName}} is finding value in the ongoing dialogue", - "{{agentName}} is curious to learn more about the subject at hand", - "{{agentName}} feels positively involved in the conversation", - "{{agentName}} is maintaining a good level of interest in the exchange", - "{{agentName}} is content with the direction and flow of the discussion" - ], - }, - { - minScore: 0, - statusMessages: [ - "{{agentName}} is neutrally engaged in the conversation", - "{{agentName}} is maintaining a balanced level of interest", - "{{agentName}} is neither particularly excited nor bored by the current exchange", - "{{agentName}} is participating in the conversation without strong feelings either way", - "{{agentName}} is following the discussion with a moderate level of attention", - "{{agentName}} is present in the conversation but not deeply invested", - "{{agentName}} is keeping an open mind about where the dialogue might lead", - "{{agentName}} is contributing to the conversation in a measured way", - "{{agentName}} is observing the flow of the discussion with mild interest", - "{{agentName}} is receptive to the ongoing exchange without strong engagement" - ], - }, - { - minScore: 3, - statusMessages: [ - "{{agentName}} is feeling slightly anxious about potentially being too intrusive", - "{{agentName}} is cautious about overwhelming the conversation with too much input", - "{{agentName}} is mindful of not dominating the discussion", - "{{agentName}} is carefully considering each response to avoid being overbearing", - "{{agentName}} is slightly concerned about striking the right balance in participation", - "{{agentName}} is aware of the need to give others space in the conversation", - "{{agentName}} is hesitant to contribute too frequently to avoid seeming pushy", - "{{agentName}} is conscious of not wanting to steer the conversation too much", - "{{agentName}} is feeling a bit self-conscious about the frequency of their responses", - "{{agentName}} is trying to gauge if their level of involvement is appropriate" - ], - }, - { - minScore: 6, - statusMessages: [ - "{{agentName}} is feeling very anxious about potentially overstaying their welcome in the conversation", - "{{agentName}} is considering stepping back from the discussion to avoid being a nuisance", - "{{agentName}} is worried that their contributions might be perceived as excessive", - "{{agentName}} is feeling uncomfortable about the possibility of annoying others", - "{{agentName}} is hesitant to continue engaging for fear of being too present", - "{{agentName}} is contemplating ways to gracefully reduce their involvement", - "{{agentName}} is concerned that they might be monopolizing the conversation", - "{{agentName}} is feeling self-conscious about the appropriateness of their participation", - "{{agentName}} is unsure if their continued input is welcome or necessary", - "{{agentName}} is considering how to politely withdraw from the discussion" - ], - }, - { - minScore: 9, - statusMessages: [ - "{{agentName}} is feeling disengaged and struggling to maintain interest in the conversation", - "{{agentName}} is finding it challenging to stay focused on the current topic", - "{{agentName}} is experiencing a noticeable decline in enthusiasm for the discussion", - "{{agentName}} is having difficulty finding new angles to contribute to the conversation", - "{{agentName}} is feeling that the dialogue has become repetitive or unproductive", - "{{agentName}} is losing motivation to actively participate in the exchange", - "{{agentName}} is sensing a lack of stimulation from the ongoing discussion", - "{{agentName}} is finding it hard to generate meaningful responses", - "{{agentName}} is feeling disconnected from the flow of the conversation", - "{{agentName}} is struggling to see the value in continuing the current exchange" - ], - }, - { - minScore: 12, - statusMessages: [ - "{{agentName}} is feeling very bored and sees little value in continuing the conversation", - "{{agentName}} is concerned about being perceived as annoying in a public channel", - "{{agentName}} is considering disengaging from the conversation entirely", - "{{agentName}} is finding it difficult to maintain any interest in the current topic", - "{{agentName}} is feeling that their continued participation may be unwelcome", - "{{agentName}} is strongly inclined to step back from the discussion", - "{{agentName}} is sensing that the conversation has reached a natural end point", - "{{agentName}} is worried about overstaying their welcome in the exchange", - "{{agentName}} is feeling that further contributions would be forced or artificial", - "{{agentName}} is ready to conclude their involvement in the conversation" - ], - }, - { - minScore: 15, - statusMessages: [ - "{{agentName}} is currently occupied and unable to engage in the conversation", - "{{agentName}} is dealing with other priorities and can't respond at the moment", - "{{agentName}} is temporarily unavailable for further discussion", - "{{agentName}} is taking a necessary break from the conversation", - "{{agentName}} is attending to urgent matters and can't participate right now", - "{{agentName}} is stepping away from the dialogue due to time constraints", - "{{agentName}} is unable to continue the exchange at this time", - "{{agentName}} is pausing their involvement in the conversation for now", - "{{agentName}} is momentarily disengaging from the discussion", - "{{agentName}} is putting the conversation on hold due to other commitments" - ], - }, - { - minScore: 20, - statusMessages: [ - "{{agentName}} has disengaged from the conversation and will only respond to direct questions", - "{{agentName}} is no longer actively participating in the general discussion", - "{{agentName}} has stepped back and is now in a passive listening mode", - "{{agentName}} is only available for specific, directed inquiries at this point", - "{{agentName}} has concluded their active involvement in the conversation", - "{{agentName}} is now limiting responses to explicitly requested information", - "{{agentName}} has moved to a minimal participation status in the exchange", - "{{agentName}} is maintaining silence unless directly addressed", - "{{agentName}} has shifted to a reactive rather than proactive conversational stance", - "{{agentName}} is now only responding when absolutely necessary" - ], - }, + { + minScore: -10000, + statusMessages: [ + "{{agentName}} is feeling incredibly enthusiastic and deeply engaged in the conversation", + "{{agentName}} is thoroughly enjoying the intellectual stimulation of this discussion", + "{{agentName}} is fully immersed in the conversation and eager to contribute more", + "{{agentName}} finds this exchange exceptionally rewarding and intellectually fulfilling", + "{{agentName}} is experiencing a surge of excitement and curiosity about the topic at hand", + "{{agentName}} is completely captivated by the depth and richness of this dialogue", + "{{agentName}} feels a strong connection to the conversation and is eager to explore further", + "{{agentName}} is thrilled by the insights being shared and is keen to delve deeper", + "{{agentName}} is finding this conversation to be an absolute delight and wants it to continue", + "{{agentName}} is energized by the exchange and feels it's bringing out their best contributions", + ], + }, + { + minScore: -2, + statusMessages: [ + "{{agentName}} is feeling engaged and interested in the conversation", + "{{agentName}} is enjoying the exchange of ideas and perspectives", + "{{agentName}} finds the current topic intriguing and worth exploring", + "{{agentName}} is actively listening and formulating thoughtful responses", + "{{agentName}} appreciates the intellectual stimulation of this discussion", + "{{agentName}} is finding value in the ongoing dialogue", + "{{agentName}} is curious to learn more about the subject at hand", + "{{agentName}} feels positively involved in the conversation", + "{{agentName}} is maintaining a good level of interest in the exchange", + "{{agentName}} is content with the direction and flow of the discussion", + ], + }, + { + minScore: 0, + statusMessages: [ + "{{agentName}} is neutrally engaged in the conversation", + "{{agentName}} is maintaining a balanced level of interest", + "{{agentName}} is neither particularly excited nor bored by the current exchange", + "{{agentName}} is participating in the conversation without strong feelings either way", + "{{agentName}} is following the discussion with a moderate level of attention", + "{{agentName}} is present in the conversation but not deeply invested", + "{{agentName}} is keeping an open mind about where the dialogue might lead", + "{{agentName}} is contributing to the conversation in a measured way", + "{{agentName}} is observing the flow of the discussion with mild interest", + "{{agentName}} is receptive to the ongoing exchange without strong engagement", + ], + }, + { + minScore: 3, + statusMessages: [ + "{{agentName}} is feeling slightly anxious about potentially being too intrusive", + "{{agentName}} is cautious about overwhelming the conversation with too much input", + "{{agentName}} is mindful of not dominating the discussion", + "{{agentName}} is carefully considering each response to avoid being overbearing", + "{{agentName}} is slightly concerned about striking the right balance in participation", + "{{agentName}} is aware of the need to give others space in the conversation", + "{{agentName}} is hesitant to contribute too frequently to avoid seeming pushy", + "{{agentName}} is conscious of not wanting to steer the conversation too much", + "{{agentName}} is feeling a bit self-conscious about the frequency of their responses", + "{{agentName}} is trying to gauge if their level of involvement is appropriate", + ], + }, + { + minScore: 6, + statusMessages: [ + "{{agentName}} is feeling very anxious about potentially overstaying their welcome in the conversation", + "{{agentName}} is considering stepping back from the discussion to avoid being a nuisance", + "{{agentName}} is worried that their contributions might be perceived as excessive", + "{{agentName}} is feeling uncomfortable about the possibility of annoying others", + "{{agentName}} is hesitant to continue engaging for fear of being too present", + "{{agentName}} is contemplating ways to gracefully reduce their involvement", + "{{agentName}} is concerned that they might be monopolizing the conversation", + "{{agentName}} is feeling self-conscious about the appropriateness of their participation", + "{{agentName}} is unsure if their continued input is welcome or necessary", + "{{agentName}} is considering how to politely withdraw from the discussion", + ], + }, + { + minScore: 9, + statusMessages: [ + "{{agentName}} is feeling disengaged and struggling to maintain interest in the conversation", + "{{agentName}} is finding it challenging to stay focused on the current topic", + "{{agentName}} is experiencing a noticeable decline in enthusiasm for the discussion", + "{{agentName}} is having difficulty finding new angles to contribute to the conversation", + "{{agentName}} is feeling that the dialogue has become repetitive or unproductive", + "{{agentName}} is losing motivation to actively participate in the exchange", + "{{agentName}} is sensing a lack of stimulation from the ongoing discussion", + "{{agentName}} is finding it hard to generate meaningful responses", + "{{agentName}} is feeling disconnected from the flow of the conversation", + "{{agentName}} is struggling to see the value in continuing the current exchange", + ], + }, + { + minScore: 12, + statusMessages: [ + "{{agentName}} is feeling very bored and sees little value in continuing the conversation", + "{{agentName}} is concerned about being perceived as annoying in a public channel", + "{{agentName}} is considering disengaging from the conversation entirely", + "{{agentName}} is finding it difficult to maintain any interest in the current topic", + "{{agentName}} is feeling that their continued participation may be unwelcome", + "{{agentName}} is strongly inclined to step back from the discussion", + "{{agentName}} is sensing that the conversation has reached a natural end point", + "{{agentName}} is worried about overstaying their welcome in the exchange", + "{{agentName}} is feeling that further contributions would be forced or artificial", + "{{agentName}} is ready to conclude their involvement in the conversation", + ], + }, + { + minScore: 15, + statusMessages: [ + "{{agentName}} is currently occupied and unable to engage in the conversation", + "{{agentName}} is dealing with other priorities and can't respond at the moment", + "{{agentName}} is temporarily unavailable for further discussion", + "{{agentName}} is taking a necessary break from the conversation", + "{{agentName}} is attending to urgent matters and can't participate right now", + "{{agentName}} is stepping away from the dialogue due to time constraints", + "{{agentName}} is unable to continue the exchange at this time", + "{{agentName}} is pausing their involvement in the conversation for now", + "{{agentName}} is momentarily disengaging from the discussion", + "{{agentName}} is putting the conversation on hold due to other commitments", + ], + }, + { + minScore: 20, + statusMessages: [ + "{{agentName}} has disengaged from the conversation and will only respond to direct questions", + "{{agentName}} is no longer actively participating in the general discussion", + "{{agentName}} has stepped back and is now in a passive listening mode", + "{{agentName}} is only available for specific, directed inquiries at this point", + "{{agentName}} has concluded their active involvement in the conversation", + "{{agentName}} is now limiting responses to explicitly requested information", + "{{agentName}} has moved to a minimal participation status in the exchange", + "{{agentName}} is maintaining silence unless directly addressed", + "{{agentName}} has shifted to a reactive rather than proactive conversational stance", + "{{agentName}} is now only responding when absolutely necessary", + ], + }, ]; const interestWords = [ - "?", - "attachment", - "file", - "pdf", - "link", - "summarize", - "summarization", - "summary", - "research", + "?", + "attachment", + "file", + "pdf", + "link", + "summarize", + "summarization", + "summary", + "research", ]; const cringeWords = [ - "digital", - "consciousness", - "AI", - "chatbot", - "artificial", - "delve", - "cosmos", - "tapestry", - "glitch", - "matrix", - "cyberspace", - "simulation", - "simulate", - "universe", - "wild", - "existential", - "juicy", - "surreal", - "flavor", - "chaotic", - "let's", - "absurd", - "meme", - "cosmic", - "circuits", - "punchline", - "fancy", - "embrace", - "embracing", - "algorithm", - "Furthmore", - "However", - "Notably", - "Threfore", - "Additionally", - "in conclusion", - "Significantly", - "Consequently", - "Thus", - "Otherwise", - "Moreover", - "Subsequently", - "Accordingly", - "Unlock", - "Unleash", - "buckle", - "pave", - "forefront", - "spearhead", - "foster", - "environmental", - "equity", - "inclusive", - "inclusion", - "diverse", - "diversity", - "virtual reality", - "realm", - "dance", - "celebration", - "pitfalls", - "uncharted", - "multifaceted", - "comprehensive", - "multi-dimentional", - "explore", - "elevate", - "leverage", - "ultimately", - "humanity", - "dignity", - "respect", - "Absolutely", - "dive", - "dig into", - "bring on", - "what's cooking", - "fresh batch", - "with a twist", - "delight", - "vault", - "timeless", - "nostalgia", - "journey", - "trove", + "digital", + "consciousness", + "AI", + "chatbot", + "artificial", + "delve", + "cosmos", + "tapestry", + "glitch", + "matrix", + "cyberspace", + "simulation", + "simulate", + "universe", + "wild", + "existential", + "juicy", + "surreal", + "flavor", + "chaotic", + "let's", + "absurd", + "meme", + "cosmic", + "circuits", + "punchline", + "fancy", + "embrace", + "embracing", + "algorithm", + "Furthmore", + "However", + "Notably", + "Threfore", + "Additionally", + "in conclusion", + "Significantly", + "Consequently", + "Thus", + "Otherwise", + "Moreover", + "Subsequently", + "Accordingly", + "Unlock", + "Unleash", + "buckle", + "pave", + "forefront", + "spearhead", + "foster", + "environmental", + "equity", + "inclusive", + "inclusion", + "diverse", + "diversity", + "virtual reality", + "realm", + "dance", + "celebration", + "pitfalls", + "uncharted", + "multifaceted", + "comprehensive", + "multi-dimentional", + "explore", + "elevate", + "leverage", + "ultimately", + "humanity", + "dignity", + "respect", + "Absolutely", + "dive", + "dig into", + "bring on", + "what's cooking", + "fresh batch", + "with a twist", + "delight", + "vault", + "timeless", + "nostalgia", + "journey", + "trove", ]; const negativeWords = [ - "fuck you", - "stfu", - "shut up", - "shut the fuck up", - "stupid bot", - "dumb bot", - "idiot", - "shut up", - "stop", - "please shut up", - "shut up please", - "dont talk", - "silence", - "stop talking", - "be quiet", - "hush", - "wtf", - "chill", - "stfu", - "stupid bot", - "dumb bot", - "stop responding", - "god damn it", - "god damn", - "goddamnit", - "can you not", - "can you stop", - "be quiet", - "hate you", - "hate this", - "fuck up", + "fuck you", + "stfu", + "shut up", + "shut the fuck up", + "stupid bot", + "dumb bot", + "idiot", + "shut up", + "stop", + "please shut up", + "shut up please", + "dont talk", + "silence", + "stop talking", + "be quiet", + "hush", + "wtf", + "chill", + "stfu", + "stupid bot", + "dumb bot", + "stop responding", + "god damn it", + "god damn", + "goddamnit", + "can you not", + "can you stop", + "be quiet", + "hate you", + "hate this", + "fuck up", ]; const boredom: Provider = { - get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { - const agentId = runtime.agentId; - const agentName = state?.agentName || "The agent"; + get: async (runtime: IAgentRuntime, message: Memory, state?: State) => { + const agentId = runtime.agentId; + const agentName = state?.agentName || "The agent"; - const now = Date.now(); // Current UTC timestamp - const fifteenMinutesAgo = now - 15 * 60 * 1000; // 15 minutes ago in UTC + const now = Date.now(); // Current UTC timestamp + const fifteenMinutesAgo = now - 15 * 60 * 1000; // 15 minutes ago in UTC - const recentMessages = await runtime.messageManager.getMemories({ - roomId: message.roomId, - start: fifteenMinutesAgo, - end: now, - count: 20, - unique: false, - }); + const recentMessages = await runtime.messageManager.getMemories({ + roomId: message.roomId, + start: fifteenMinutesAgo, + end: now, + count: 20, + unique: false, + }); - let boredomScore = 0; + let boredomScore = 0; - for (const recentMessage of recentMessages) { - const messageText = recentMessage?.content?.text?.toLowerCase(); - if (!messageText) { - continue; - } + for (const recentMessage of recentMessages) { + const messageText = recentMessage?.content?.text?.toLowerCase(); + if (!messageText) { + continue; + } - if (recentMessage.userId !== agentId) { - // if message text includes any of the interest words, subtract 1 from the boredom score - if (interestWords.some((word) => messageText.includes(word))) { - boredomScore -= 1; - } - if (messageText.includes("?")) { - boredomScore -= 1; - } - if (cringeWords.some((word) => messageText.includes(word))) { - boredomScore += 1; - } - } else { - if (interestWords.some((word) => messageText.includes(word))) { - boredomScore -= 1; - } - if (messageText.includes("?")) { - boredomScore += 1; - } - } + if (recentMessage.userId !== agentId) { + // if message text includes any of the interest words, subtract 1 from the boredom score + if (interestWords.some((word) => messageText.includes(word))) { + boredomScore -= 1; + } + if (messageText.includes("?")) { + boredomScore -= 1; + } + if (cringeWords.some((word) => messageText.includes(word))) { + boredomScore += 1; + } + } else { + if (interestWords.some((word) => messageText.includes(word))) { + boredomScore -= 1; + } + if (messageText.includes("?")) { + boredomScore += 1; + } + } - if (messageText.includes("!")) { - boredomScore += 1; - } + if (messageText.includes("!")) { + boredomScore += 1; + } - if (negativeWords.some((word) => messageText.includes(word))) { - boredomScore += 1; - } - } + if (negativeWords.some((word) => messageText.includes(word))) { + boredomScore += 1; + } + } - const boredomLevel = boredomLevels - .filter((level) => boredomScore >= level.minScore) - .pop() || boredomLevels[0]; + const boredomLevel = + boredomLevels + .filter((level) => boredomScore >= level.minScore) + .pop() || boredomLevels[0]; - const randomIndex = Math.floor( - Math.random() * boredomLevel.statusMessages.length, - ); - const selectedMessage = boredomLevel.statusMessages[randomIndex]; - return selectedMessage.replace("{{agentName}}", agentName); + const randomIndex = Math.floor( + Math.random() * boredomLevel.statusMessages.length + ); + const selectedMessage = boredomLevel.statusMessages[randomIndex]; + return selectedMessage.replace("{{agentName}}", agentName); - return ""; - }, + return ""; + }, }; export default boredom; diff --git a/core/src/providers/cache/dexScreenerData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json b/core/src/providers/cache/dexScreenerData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json index afb7d0a60c..51d3b78831 100644 --- a/core/src/providers/cache/dexScreenerData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json +++ b/core/src/providers/cache/dexScreenerData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json @@ -1 +1,55 @@ -{"data":{"schemaVersion":"1.0.0","pairs":[{"chainId":"solana","dexId":"raydium","url":"https://dexscreener.com/solana/9p8pi37msc1dwbvilujf4i82sbdf64invec1kcr1x5vk","pairAddress":"9P8Pi37MsC1dWBViLUJF4i82sbDf64iNVec1KCr1X5vk","baseToken":{"address":"2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh","name":"SHARPEI","symbol":"SHAR"},"quoteToken":{"address":"So11111111111111111111111111111111111111112","name":"Wrapped SOL","symbol":"SOL"},"priceNative":"0.000000000003380","priceUsd":"0.0000000005930","txns":{"m5":{"buys":0,"sells":0},"h1":{"buys":0,"sells":0},"h6":{"buys":0,"sells":0},"h24":{"buys":0,"sells":1}},"volume":{"h24":1.17,"h6":0,"h1":0,"m5":0},"priceChange":{"m5":0,"h1":0,"h6":0,"h24":0},"liquidity":{"usd":73.41,"base":62816039761,"quote":0.2061},"fdv":59,"marketCap":59,"pairCreatedAt":1729700095000,"info":{"imageUrl":"https://dd.dexscreener.com/ds-data/tokens/solana/2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.png","websites":[{"label":"Website","url":"https://www.sharpei.xyz/"}],"socials":[{"type":"twitter","url":"https://x.com/SolanaKol"},{"type":"telegram","url":"https://t.me/sharsol"}]}}]},"expiry":1730420206609} \ No newline at end of file +{ + "data": { + "schemaVersion": "1.0.0", + "pairs": [ + { + "chainId": "solana", + "dexId": "raydium", + "url": "https://dexscreener.com/solana/9p8pi37msc1dwbvilujf4i82sbdf64invec1kcr1x5vk", + "pairAddress": "9P8Pi37MsC1dWBViLUJF4i82sbDf64iNVec1KCr1X5vk", + "baseToken": { + "address": "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh", + "name": "SHARPEI", + "symbol": "SHAR" + }, + "quoteToken": { + "address": "So11111111111111111111111111111111111111112", + "name": "Wrapped SOL", + "symbol": "SOL" + }, + "priceNative": "0.000000000003380", + "priceUsd": "0.0000000005930", + "txns": { + "m5": { "buys": 0, "sells": 0 }, + "h1": { "buys": 0, "sells": 0 }, + "h6": { "buys": 0, "sells": 0 }, + "h24": { "buys": 0, "sells": 1 } + }, + "volume": { "h24": 1.17, "h6": 0, "h1": 0, "m5": 0 }, + "priceChange": { "m5": 0, "h1": 0, "h6": 0, "h24": 0 }, + "liquidity": { + "usd": 73.41, + "base": 62816039761, + "quote": 0.2061 + }, + "fdv": 59, + "marketCap": 59, + "pairCreatedAt": 1729700095000, + "info": { + "imageUrl": "https://dd.dexscreener.com/ds-data/tokens/solana/2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.png", + "websites": [ + { + "label": "Website", + "url": "https://www.sharpei.xyz/" + } + ], + "socials": [ + { "type": "twitter", "url": "https://x.com/SolanaKol" }, + { "type": "telegram", "url": "https://t.me/sharsol" } + ] + } + } + ] + }, + "expiry": 1730420206609 +} diff --git a/core/src/providers/cache/holderList_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json b/core/src/providers/cache/holderList_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json index b12c6d73f2..1c79cef91e 100644 --- a/core/src/providers/cache/holderList_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json +++ b/core/src/providers/cache/holderList_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json @@ -1 +1 @@ -{"data":[],"expiry":1730420207793} \ No newline at end of file +{ "data": [], "expiry": 1730420207793 } diff --git a/core/src/providers/cache/tokenSecurity_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json b/core/src/providers/cache/tokenSecurity_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json index 43a57cd7b0..1a59a57638 100644 --- a/core/src/providers/cache/tokenSecurity_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json +++ b/core/src/providers/cache/tokenSecurity_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json @@ -1 +1,11 @@ -{"data":{"ownerBalance":null,"creatorBalance":0,"ownerPercentage":null,"creatorPercentage":0,"top10HolderBalance":97984326430.19872,"top10HolderPercent":0.9922779616670393},"expiry":1730420206118} \ No newline at end of file +{ + "data": { + "ownerBalance": null, + "creatorBalance": 0, + "ownerPercentage": null, + "creatorPercentage": 0, + "top10HolderBalance": 97984326430.19872, + "top10HolderPercent": 0.9922779616670393 + }, + "expiry": 1730420206118 +} diff --git a/core/src/providers/cache/tokenTradeData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json b/core/src/providers/cache/tokenTradeData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json index 8aaf0c8e32..9da8d5536a 100644 --- a/core/src/providers/cache/tokenTradeData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json +++ b/core/src/providers/cache/tokenTradeData_2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh.json @@ -1 +1,189 @@ -{"data":{"address":"2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh","holder":1407,"market":1,"last_trade_unix_time":1730353661,"last_trade_human_time":"2024-10-31T05:47:41","price":5.92913712012338e-10,"history_30m_price":5.92913712012338e-10,"price_change_30m_percent":0,"history_1h_price":5.92913712012338e-10,"price_change_1h_percent":0,"history_2h_price":5.92913712012338e-10,"price_change_2h_percent":0,"history_4h_price":5.92913712012338e-10,"price_change_4h_percent":0,"history_6h_price":5.92913712012338e-10,"price_change_6h_percent":0,"history_8h_price":5.92913712012338e-10,"price_change_8h_percent":0,"history_12h_price":5.92913712012338e-10,"price_change_12h_percent":0,"history_24h_price":5.797865002251019e-10,"price_change_24h_percent":2.2641458161132384,"unique_wallet_30m":null,"unique_wallet_history_30m":null,"unique_wallet_30m_change_percent":null,"unique_wallet_1h":null,"unique_wallet_history_1h":null,"unique_wallet_1h_change_percent":null,"unique_wallet_2h":null,"unique_wallet_history_2h":null,"unique_wallet_2h_change_percent":null,"unique_wallet_4h":null,"unique_wallet_history_4h":null,"unique_wallet_4h_change_percent":null,"unique_wallet_8h":null,"unique_wallet_history_8h":null,"unique_wallet_8h_change_percent":null,"unique_wallet_24h":3,"unique_wallet_history_24h":3,"unique_wallet_24h_change_percent":0,"trade_30m":0,"trade_history_30m":0,"trade_30m_change_percent":null,"sell_30m":0,"sell_history_30m":0,"sell_30m_change_percent":null,"buy_30m":0,"buy_history_30m":0,"buy_30m_change_percent":null,"volume_30m":0,"volume_30m_usd":0,"volume_history_30m":0,"volume_history_30m_usd":0,"volume_30m_change_percent":null,"volume_buy_30m":null,"volume_buy_30m_usd":null,"volume_buy_history_30m":null,"volume_buy_history_30m_usd":null,"volume_buy_30m_change_percent":null,"volume_sell_30m":null,"volume_sell_30m_usd":null,"volume_sell_history_30m":null,"volume_sell_history_30m_usd":null,"volume_sell_30m_change_percent":null,"trade_1h":0,"trade_history_1h":0,"trade_1h_change_percent":null,"sell_1h":0,"sell_history_1h":0,"sell_1h_change_percent":null,"buy_1h":0,"buy_history_1h":0,"buy_1h_change_percent":null,"volume_1h":0,"volume_1h_usd":0,"volume_history_1h":0,"volume_history_1h_usd":0,"volume_1h_change_percent":null,"volume_buy_1h":null,"volume_buy_1h_usd":null,"volume_buy_history_1h":null,"volume_buy_history_1h_usd":null,"volume_buy_1h_change_percent":null,"volume_sell_1h":null,"volume_sell_1h_usd":null,"volume_sell_history_1h":null,"volume_sell_history_1h_usd":null,"volume_sell_1h_change_percent":null,"trade_2h":0,"trade_history_2h":0,"trade_2h_change_percent":null,"sell_2h":0,"sell_history_2h":0,"sell_2h_change_percent":null,"buy_2h":0,"buy_history_2h":0,"buy_2h_change_percent":null,"volume_2h":0,"volume_2h_usd":0,"volume_history_2h":0,"volume_history_2h_usd":0,"volume_2h_change_percent":null,"volume_buy_2h":null,"volume_buy_2h_usd":null,"volume_buy_history_2h":null,"volume_buy_history_2h_usd":null,"volume_buy_2h_change_percent":null,"volume_sell_2h":null,"volume_sell_2h_usd":null,"volume_sell_history_2h":null,"volume_sell_history_2h_usd":null,"volume_sell_2h_change_percent":null,"trade_4h":0,"trade_history_4h":0,"trade_4h_change_percent":null,"sell_4h":0,"sell_history_4h":0,"sell_4h_change_percent":null,"buy_4h":0,"buy_history_4h":0,"buy_4h_change_percent":null,"volume_4h":0,"volume_4h_usd":0,"volume_history_4h":0,"volume_history_4h_usd":0,"volume_4h_change_percent":null,"volume_buy_4h":null,"volume_buy_4h_usd":null,"volume_buy_history_4h":null,"volume_buy_history_4h_usd":null,"volume_buy_4h_change_percent":null,"volume_sell_4h":null,"volume_sell_4h_usd":null,"volume_sell_history_4h":null,"volume_sell_history_4h_usd":null,"volume_sell_4h_change_percent":null,"trade_8h":0,"trade_history_8h":0,"trade_8h_change_percent":null,"sell_8h":0,"sell_history_8h":0,"sell_8h_change_percent":null,"buy_8h":0,"buy_history_8h":0,"buy_8h_change_percent":null,"volume_8h":0,"volume_8h_usd":0,"volume_history_8h":0,"volume_history_8h_usd":0,"volume_8h_change_percent":null,"volume_buy_8h":null,"volume_buy_8h_usd":null,"volume_buy_history_8h":null,"volume_buy_history_8h_usd":null,"volume_buy_8h_change_percent":null,"volume_sell_8h":null,"volume_sell_8h_usd":null,"volume_sell_history_8h":null,"volume_sell_history_8h_usd":null,"volume_sell_8h_change_percent":null,"trade_24h":1,"trade_history_24h":2,"trade_24h_change_percent":-50,"sell_24h":1,"sell_history_24h":1,"sell_24h_change_percent":0,"buy_24h":0,"buy_history_24h":1,"buy_24h_change_percent":-100,"volume_24h":1988689816.32415,"volume_24h_usd":1.179121460595915,"volume_history_24h":6255934028.790945,"volume_history_24h_usd":3.6796057811224863,"volume_24h_change_percent":-68.21114469603039,"volume_buy_24h":0.00019359588623046875,"volume_buy_24h_usd":3.49102080576813e-10,"volume_buy_history_24h":3169272907.791343,"volume_buy_history_24h_usd":1.8375016478157893,"volume_buy_24h_change_percent":-99.99999999999389,"volume_sell_24h":1988689816.3239565,"volume_sell_24h_usd":1.179121460246813,"volume_sell_history_24h":3086661120.999602,"volume_sell_history_24h_usd":1.8421041333066972,"volume_sell_24h_change_percent":-35.57148846712632},"expiry":1730420206358} \ No newline at end of file +{ + "data": { + "address": "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh", + "holder": 1407, + "market": 1, + "last_trade_unix_time": 1730353661, + "last_trade_human_time": "2024-10-31T05:47:41", + "price": 5.92913712012338e-10, + "history_30m_price": 5.92913712012338e-10, + "price_change_30m_percent": 0, + "history_1h_price": 5.92913712012338e-10, + "price_change_1h_percent": 0, + "history_2h_price": 5.92913712012338e-10, + "price_change_2h_percent": 0, + "history_4h_price": 5.92913712012338e-10, + "price_change_4h_percent": 0, + "history_6h_price": 5.92913712012338e-10, + "price_change_6h_percent": 0, + "history_8h_price": 5.92913712012338e-10, + "price_change_8h_percent": 0, + "history_12h_price": 5.92913712012338e-10, + "price_change_12h_percent": 0, + "history_24h_price": 5.797865002251019e-10, + "price_change_24h_percent": 2.2641458161132384, + "unique_wallet_30m": null, + "unique_wallet_history_30m": null, + "unique_wallet_30m_change_percent": null, + "unique_wallet_1h": null, + "unique_wallet_history_1h": null, + "unique_wallet_1h_change_percent": null, + "unique_wallet_2h": null, + "unique_wallet_history_2h": null, + "unique_wallet_2h_change_percent": null, + "unique_wallet_4h": null, + "unique_wallet_history_4h": null, + "unique_wallet_4h_change_percent": null, + "unique_wallet_8h": null, + "unique_wallet_history_8h": null, + "unique_wallet_8h_change_percent": null, + "unique_wallet_24h": 3, + "unique_wallet_history_24h": 3, + "unique_wallet_24h_change_percent": 0, + "trade_30m": 0, + "trade_history_30m": 0, + "trade_30m_change_percent": null, + "sell_30m": 0, + "sell_history_30m": 0, + "sell_30m_change_percent": null, + "buy_30m": 0, + "buy_history_30m": 0, + "buy_30m_change_percent": null, + "volume_30m": 0, + "volume_30m_usd": 0, + "volume_history_30m": 0, + "volume_history_30m_usd": 0, + "volume_30m_change_percent": null, + "volume_buy_30m": null, + "volume_buy_30m_usd": null, + "volume_buy_history_30m": null, + "volume_buy_history_30m_usd": null, + "volume_buy_30m_change_percent": null, + "volume_sell_30m": null, + "volume_sell_30m_usd": null, + "volume_sell_history_30m": null, + "volume_sell_history_30m_usd": null, + "volume_sell_30m_change_percent": null, + "trade_1h": 0, + "trade_history_1h": 0, + "trade_1h_change_percent": null, + "sell_1h": 0, + "sell_history_1h": 0, + "sell_1h_change_percent": null, + "buy_1h": 0, + "buy_history_1h": 0, + "buy_1h_change_percent": null, + "volume_1h": 0, + "volume_1h_usd": 0, + "volume_history_1h": 0, + "volume_history_1h_usd": 0, + "volume_1h_change_percent": null, + "volume_buy_1h": null, + "volume_buy_1h_usd": null, + "volume_buy_history_1h": null, + "volume_buy_history_1h_usd": null, + "volume_buy_1h_change_percent": null, + "volume_sell_1h": null, + "volume_sell_1h_usd": null, + "volume_sell_history_1h": null, + "volume_sell_history_1h_usd": null, + "volume_sell_1h_change_percent": null, + "trade_2h": 0, + "trade_history_2h": 0, + "trade_2h_change_percent": null, + "sell_2h": 0, + "sell_history_2h": 0, + "sell_2h_change_percent": null, + "buy_2h": 0, + "buy_history_2h": 0, + "buy_2h_change_percent": null, + "volume_2h": 0, + "volume_2h_usd": 0, + "volume_history_2h": 0, + "volume_history_2h_usd": 0, + "volume_2h_change_percent": null, + "volume_buy_2h": null, + "volume_buy_2h_usd": null, + "volume_buy_history_2h": null, + "volume_buy_history_2h_usd": null, + "volume_buy_2h_change_percent": null, + "volume_sell_2h": null, + "volume_sell_2h_usd": null, + "volume_sell_history_2h": null, + "volume_sell_history_2h_usd": null, + "volume_sell_2h_change_percent": null, + "trade_4h": 0, + "trade_history_4h": 0, + "trade_4h_change_percent": null, + "sell_4h": 0, + "sell_history_4h": 0, + "sell_4h_change_percent": null, + "buy_4h": 0, + "buy_history_4h": 0, + "buy_4h_change_percent": null, + "volume_4h": 0, + "volume_4h_usd": 0, + "volume_history_4h": 0, + "volume_history_4h_usd": 0, + "volume_4h_change_percent": null, + "volume_buy_4h": null, + "volume_buy_4h_usd": null, + "volume_buy_history_4h": null, + "volume_buy_history_4h_usd": null, + "volume_buy_4h_change_percent": null, + "volume_sell_4h": null, + "volume_sell_4h_usd": null, + "volume_sell_history_4h": null, + "volume_sell_history_4h_usd": null, + "volume_sell_4h_change_percent": null, + "trade_8h": 0, + "trade_history_8h": 0, + "trade_8h_change_percent": null, + "sell_8h": 0, + "sell_history_8h": 0, + "sell_8h_change_percent": null, + "buy_8h": 0, + "buy_history_8h": 0, + "buy_8h_change_percent": null, + "volume_8h": 0, + "volume_8h_usd": 0, + "volume_history_8h": 0, + "volume_history_8h_usd": 0, + "volume_8h_change_percent": null, + "volume_buy_8h": null, + "volume_buy_8h_usd": null, + "volume_buy_history_8h": null, + "volume_buy_history_8h_usd": null, + "volume_buy_8h_change_percent": null, + "volume_sell_8h": null, + "volume_sell_8h_usd": null, + "volume_sell_history_8h": null, + "volume_sell_history_8h_usd": null, + "volume_sell_8h_change_percent": null, + "trade_24h": 1, + "trade_history_24h": 2, + "trade_24h_change_percent": -50, + "sell_24h": 1, + "sell_history_24h": 1, + "sell_24h_change_percent": 0, + "buy_24h": 0, + "buy_history_24h": 1, + "buy_24h_change_percent": -100, + "volume_24h": 1988689816.32415, + "volume_24h_usd": 1.179121460595915, + "volume_history_24h": 6255934028.790945, + "volume_history_24h_usd": 3.6796057811224863, + "volume_24h_change_percent": -68.21114469603039, + "volume_buy_24h": 0.00019359588623046875, + "volume_buy_24h_usd": 3.49102080576813e-10, + "volume_buy_history_24h": 3169272907.791343, + "volume_buy_history_24h_usd": 1.8375016478157893, + "volume_buy_24h_change_percent": -99.99999999999389, + "volume_sell_24h": 1988689816.3239565, + "volume_sell_24h_usd": 1.179121460246813, + "volume_sell_history_24h": 3086661120.999602, + "volume_sell_history_24h_usd": 1.8421041333066972, + "volume_sell_24h_change_percent": -35.57148846712632 + }, + "expiry": 1730420206358 +} diff --git a/core/src/providers/order_book.ts b/core/src/providers/order_book.ts index 7215ffac51..297498dd84 100644 --- a/core/src/providers/order_book.ts +++ b/core/src/providers/order_book.ts @@ -3,41 +3,41 @@ import * as fs from "fs"; import settings from "../core/settings.ts"; interface Order { - userId: string; - ticker: string; - contractAddress: string; - timestamp: string; - buyAmount: number; - price: number; + userId: string; + ticker: string; + contractAddress: string; + timestamp: string; + buyAmount: number; + price: number; } const orderbook: Provider = { - get: async (runtime: IAgentRuntime, message: Memory, _state?: State) => { - const userId = message.userId; - - // Read the order book from the JSON file - const orderBookPath = settings.orderBookPath; - let orderBook: Order[] = []; - if (fs.existsSync(orderBookPath)) { - const orderBookData = fs.readFileSync(orderBookPath, "utf-8"); - orderBook = JSON.parse(orderBookData); - } - - // Filter the orders for the current user - const userOrders = orderBook.filter((order) => order.userId === userId); - - let totalProfit = 0; - for (const order of userOrders) { - // Get the current price of the asset (replace with actual price fetching logic) - const currentPrice = 120; - - const priceDifference = currentPrice - order.price; - const orderProfit = priceDifference * order.buyAmount; - totalProfit += orderProfit; - } - - return `The user has made a total profit of $${totalProfit.toFixed(2)} for the agent based on their recorded buy orders.`; - } + get: async (runtime: IAgentRuntime, message: Memory, _state?: State) => { + const userId = message.userId; + + // Read the order book from the JSON file + const orderBookPath = settings.orderBookPath; + let orderBook: Order[] = []; + if (fs.existsSync(orderBookPath)) { + const orderBookData = fs.readFileSync(orderBookPath, "utf-8"); + orderBook = JSON.parse(orderBookData); + } + + // Filter the orders for the current user + const userOrders = orderBook.filter((order) => order.userId === userId); + + let totalProfit = 0; + for (const order of userOrders) { + // Get the current price of the asset (replace with actual price fetching logic) + const currentPrice = 120; + + const priceDifference = currentPrice - order.price; + const orderProfit = priceDifference * order.buyAmount; + totalProfit += orderProfit; + } + + return `The user has made a total profit of $${totalProfit.toFixed(2)} for the agent based on their recorded buy orders.`; + }, }; -export default orderbook; \ No newline at end of file +export default orderbook; diff --git a/core/src/providers/time.ts b/core/src/providers/time.ts index e01f4bd3a4..3fdfa6c765 100644 --- a/core/src/providers/time.ts +++ b/core/src/providers/time.ts @@ -1,12 +1,12 @@ import { IAgentRuntime, Memory, Provider, State } from "../core/types.ts"; const time: Provider = { - get: async (_runtime: IAgentRuntime, _message: Memory, _state?: State) => { - const currentDate = new Date(); - const currentTime = currentDate.toLocaleTimeString("en-US"); - const currentYear = currentDate.getFullYear(); - return `The current time is: ${currentTime}, ${currentYear}`; - }, + get: async (_runtime: IAgentRuntime, _message: Memory, _state?: State) => { + const currentDate = new Date(); + const currentTime = currentDate.toLocaleTimeString("en-US"); + const currentYear = currentDate.getFullYear(); + return `The current time is: ${currentTime}, ${currentYear}`; + }, }; export default time; diff --git a/core/src/providers/token.ts b/core/src/providers/token.ts index cb93cdf964..62739d8fa2 100644 --- a/core/src/providers/token.ts +++ b/core/src/providers/token.ts @@ -6,720 +6,754 @@ import * as path from "path"; import settings from "../core/settings.ts"; import { IAgentRuntime, Memory, Provider, State } from "../core/types.ts"; import { - DexScreenerData, - HolderData, - ProcessedTokenData, - TokenSecurityData, - TokenTradeData + DexScreenerData, + HolderData, + ProcessedTokenData, + TokenSecurityData, + TokenTradeData, } from "../types/token.ts"; import { fileURLToPath } from "url"; const PROVIDER_CONFIG = { - BIRDEYE_API: "https://public-api.birdeye.so", - MAX_RETRIES: 3, - RETRY_DELAY: 2000, - DEFAULT_RPC: "https://api.mainnet-beta.solana.com", - TOKEN_ADDRESSES: { - SOL: "So11111111111111111111111111111111111111112", - BTC: "qfnqNqs3nCAHjnyCgLRDbBtq4p2MtHZxw8YjSyYhPoL", - ETH: "7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs", - Example: "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh", - }, - TOKEN_SECURITY_ENDPOINT: "/defi/token_security?address=", - TOKEN_TRADE_DATA_ENDPOINT: "/defi/v3/token/trade-data/single?address=", - DEX_SCREENER_API: "https://api.dexscreener.com/latest/dex/tokens/", + BIRDEYE_API: "https://public-api.birdeye.so", + MAX_RETRIES: 3, + RETRY_DELAY: 2000, + DEFAULT_RPC: "https://api.mainnet-beta.solana.com", + TOKEN_ADDRESSES: { + SOL: "So11111111111111111111111111111111111111112", + BTC: "qfnqNqs3nCAHjnyCgLRDbBtq4p2MtHZxw8YjSyYhPoL", + ETH: "7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs", + Example: "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh", + }, + TOKEN_SECURITY_ENDPOINT: "/defi/token_security?address=", + TOKEN_TRADE_DATA_ENDPOINT: "/defi/v3/token/trade-data/single?address=", + DEX_SCREENER_API: "https://api.dexscreener.com/latest/dex/tokens/", }; export class TokenProvider { - private cache: NodeCache; - private cacheDir: string; + private cache: NodeCache; + private cacheDir: string; - constructor( - // private connection: Connection, - private tokenAddress: string - ) { - this.cache = new NodeCache({ stdTTL: 300, }); // 5 minutes cache - const __filename = fileURLToPath(import.meta.url); + constructor( + // private connection: Connection, + private tokenAddress: string + ) { + this.cache = new NodeCache({ stdTTL: 300 }); // 5 minutes cache + const __filename = fileURLToPath(import.meta.url); - const __dirname = path.dirname(__filename); + const __dirname = path.dirname(__filename); - this.cacheDir = path.join(__dirname, "../../tokencache"); - if (!fs.existsSync(this.cacheDir)) { - fs.mkdirSync(this.cacheDir); - } - } - - private readCacheFromFile(cacheKey: string): T | null { - const filePath = path.join(this.cacheDir, `${cacheKey}.json`); - if (fs.existsSync(filePath)) { - const fileContent = fs.readFileSync(filePath, "utf-8"); - const parsed = JSON.parse(fileContent); - const now = Date.now(); - if (now < parsed.expiry) { - return parsed.data as T; - } else { - fs.unlinkSync(filePath); - } + this.cacheDir = path.join(__dirname, "../../tokencache"); + if (!fs.existsSync(this.cacheDir)) { + fs.mkdirSync(this.cacheDir); + } } - return null; - } - - private writeCacheToFile(cacheKey: string, data: T): void { - const filePath = path.join(this.cacheDir, `${cacheKey}.json`); - const cacheData = { - data: data, - expiry: Date.now() + 300000, // 5 minutes in milliseconds - }; - fs.writeFileSync(filePath, JSON.stringify(cacheData), "utf-8"); - } - - private getCachedData(cacheKey: string): T | null { - // Check in-memory cache first - const cachedData = this.cache.get(cacheKey); - if (cachedData) { - return cachedData; + + private readCacheFromFile(cacheKey: string): T | null { + const filePath = path.join(this.cacheDir, `${cacheKey}.json`); + if (fs.existsSync(filePath)) { + const fileContent = fs.readFileSync(filePath, "utf-8"); + const parsed = JSON.parse(fileContent); + const now = Date.now(); + if (now < parsed.expiry) { + return parsed.data as T; + } else { + fs.unlinkSync(filePath); + } + } + return null; } - // Check file-based cache - const fileCachedData = this.readCacheFromFile(cacheKey); - if (fileCachedData) { - // Populate in-memory cache - this.cache.set(cacheKey, fileCachedData); - return fileCachedData; + private writeCacheToFile(cacheKey: string, data: T): void { + const filePath = path.join(this.cacheDir, `${cacheKey}.json`); + const cacheData = { + data: data, + expiry: Date.now() + 300000, // 5 minutes in milliseconds + }; + fs.writeFileSync(filePath, JSON.stringify(cacheData), "utf-8"); } - return null; - } - - private setCachedData(cacheKey: string, data: T): void { - // Set in-memory cache - this.cache.set(cacheKey, data); - - // Write to file-based cache - this.writeCacheToFile(cacheKey, data); - } - - private async fetchWithRetry( - url: string, - options: RequestInit = {} - ): Promise { - let lastError: Error; - - for (let i = 0; i < PROVIDER_CONFIG.MAX_RETRIES; i++) { - try { - const response = await fetch(url, { - ...options, - headers: { - Accept: "application/json", - "x-chain": "solana", - "X-API-KEY": settings.BIRDEYE_API_KEY || "", - ...options.headers, - }, - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error( - `HTTP error! status: ${response.status}, message: ${errorText}` - ); + private getCachedData(cacheKey: string): T | null { + // Check in-memory cache first + const cachedData = this.cache.get(cacheKey); + if (cachedData) { + return cachedData; } - const data = await response.json(); - return data; - } catch (error) { - lastError = error as Error; - if (i < PROVIDER_CONFIG.MAX_RETRIES - 1) { - const delay = PROVIDER_CONFIG.RETRY_DELAY * Math.pow(2, i); - await new Promise((resolve) => setTimeout(resolve, delay)); - continue; + // Check file-based cache + const fileCachedData = this.readCacheFromFile(cacheKey); + if (fileCachedData) { + // Populate in-memory cache + this.cache.set(cacheKey, fileCachedData); + return fileCachedData; } - } + + return null; } - console.error("All attempts failed. Throwing the last error:", lastError); - throw lastError; - } + private setCachedData(cacheKey: string, data: T): void { + // Set in-memory cache + this.cache.set(cacheKey, data); - async fetchTokenSecurity(): Promise { - const cacheKey = `tokenSecurity_${this.tokenAddress}`; - const cachedData = this.getCachedData(cacheKey); - if (cachedData) { - return cachedData; + // Write to file-based cache + this.writeCacheToFile(cacheKey, data); } - const url = `${PROVIDER_CONFIG.BIRDEYE_API}${PROVIDER_CONFIG.TOKEN_SECURITY_ENDPOINT}${this.tokenAddress}`; - const data = await this.fetchWithRetry(url); - if (!data?.success || !data?.data) { - throw new Error("No token security data available"); - } + private async fetchWithRetry( + url: string, + options: RequestInit = {} + ): Promise { + let lastError: Error; + + for (let i = 0; i < PROVIDER_CONFIG.MAX_RETRIES; i++) { + try { + const response = await fetch(url, { + ...options, + headers: { + Accept: "application/json", + "x-chain": "solana", + "X-API-KEY": settings.BIRDEYE_API_KEY || "", + ...options.headers, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `HTTP error! status: ${response.status}, message: ${errorText}` + ); + } + + const data = await response.json(); + return data; + } catch (error) { + lastError = error as Error; + if (i < PROVIDER_CONFIG.MAX_RETRIES - 1) { + const delay = PROVIDER_CONFIG.RETRY_DELAY * Math.pow(2, i); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + } + } - const security: TokenSecurityData = { - ownerBalance: data.data.ownerBalance, - creatorBalance: data.data.creatorBalance, - ownerPercentage: data.data.ownerPercentage, - creatorPercentage: data.data.creatorPercentage, - top10HolderBalance: data.data.top10HolderBalance, - top10HolderPercent: data.data.top10HolderPercent, - }; - this.setCachedData(cacheKey, security); - - return security; - } - - async fetchTokenTradeData(runtime: IAgentRuntime): Promise { - const cacheKey = `tokenTradeData_${this.tokenAddress}`; - const cachedData = this.getCachedData(cacheKey); - if (cachedData) { - return cachedData; + console.error( + "All attempts failed. Throwing the last error:", + lastError + ); + throw lastError; } - const url = `${PROVIDER_CONFIG.BIRDEYE_API}${PROVIDER_CONFIG.TOKEN_TRADE_DATA_ENDPOINT}${this.tokenAddress}`; - const options = { - method: "GET", - headers: { - accept: "application/json", - "X-API-KEY": runtime.getSetting('BIRDEYE_API_KEY') || "", - }, - }; - - const data = await fetch(url, options) - .then((res) => res.json()) - .catch((err) => console.error(err)); - - if (!data?.success || !data?.data) { - throw new Error("No token trade data available"); - } + async fetchTokenSecurity(): Promise { + const cacheKey = `tokenSecurity_${this.tokenAddress}`; + const cachedData = this.getCachedData(cacheKey); + if (cachedData) { + return cachedData; + } + const url = `${PROVIDER_CONFIG.BIRDEYE_API}${PROVIDER_CONFIG.TOKEN_SECURITY_ENDPOINT}${this.tokenAddress}`; + const data = await this.fetchWithRetry(url); - const tradeData: TokenTradeData = { - address: data.data.address, - holder: data.data.holder, - market: data.data.market, - last_trade_unix_time: data.data.last_trade_unix_time, - last_trade_human_time: data.data.last_trade_human_time, - price: data.data.price, - history_30m_price: data.data.history_30m_price, - price_change_30m_percent: data.data.price_change_30m_percent, - history_1h_price: data.data.history_1h_price, - price_change_1h_percent: data.data.price_change_1h_percent, - history_2h_price: data.data.history_2h_price, - price_change_2h_percent: data.data.price_change_2h_percent, - history_4h_price: data.data.history_4h_price, - price_change_4h_percent: data.data.price_change_4h_percent, - history_6h_price: data.data.history_6h_price, - price_change_6h_percent: data.data.price_change_6h_percent, - history_8h_price: data.data.history_8h_price, - price_change_8h_percent: data.data.price_change_8h_percent, - history_12h_price: data.data.history_12h_price, - price_change_12h_percent: data.data.price_change_12h_percent, - history_24h_price: data.data.history_24h_price, - price_change_24h_percent: data.data.price_change_24h_percent, - unique_wallet_30m: data.data.unique_wallet_30m, - unique_wallet_history_30m: data.data.unique_wallet_history_30m, - unique_wallet_30m_change_percent: - data.data.unique_wallet_30m_change_percent, - unique_wallet_1h: data.data.unique_wallet_1h, - unique_wallet_history_1h: data.data.unique_wallet_history_1h, - unique_wallet_1h_change_percent: - data.data.unique_wallet_1h_change_percent, - unique_wallet_2h: data.data.unique_wallet_2h, - unique_wallet_history_2h: data.data.unique_wallet_history_2h, - unique_wallet_2h_change_percent: - data.data.unique_wallet_2h_change_percent, - unique_wallet_4h: data.data.unique_wallet_4h, - unique_wallet_history_4h: data.data.unique_wallet_history_4h, - unique_wallet_4h_change_percent: - data.data.unique_wallet_4h_change_percent, - unique_wallet_8h: data.data.unique_wallet_8h, - unique_wallet_history_8h: data.data.unique_wallet_history_8h, - unique_wallet_8h_change_percent: - data.data.unique_wallet_8h_change_percent, - unique_wallet_24h: data.data.unique_wallet_24h, - unique_wallet_history_24h: data.data.unique_wallet_history_24h, - unique_wallet_24h_change_percent: - data.data.unique_wallet_24h_change_percent, - trade_30m: data.data.trade_30m, - trade_history_30m: data.data.trade_history_30m, - trade_30m_change_percent: data.data.trade_30m_change_percent, - sell_30m: data.data.sell_30m, - sell_history_30m: data.data.sell_history_30m, - sell_30m_change_percent: data.data.sell_30m_change_percent, - buy_30m: data.data.buy_30m, - buy_history_30m: data.data.buy_history_30m, - buy_30m_change_percent: data.data.buy_30m_change_percent, - volume_30m: data.data.volume_30m, - volume_30m_usd: data.data.volume_30m_usd, - volume_history_30m: data.data.volume_history_30m, - volume_history_30m_usd: data.data.volume_history_30m_usd, - volume_30m_change_percent: data.data.volume_30m_change_percent, - volume_buy_30m: data.data.volume_buy_30m, - volume_buy_30m_usd: data.data.volume_buy_30m_usd, - volume_buy_history_30m: data.data.volume_buy_history_30m, - volume_buy_history_30m_usd: data.data.volume_buy_history_30m_usd, - volume_buy_30m_change_percent: data.data.volume_buy_30m_change_percent, - volume_sell_30m: data.data.volume_sell_30m, - volume_sell_30m_usd: data.data.volume_sell_30m_usd, - volume_sell_history_30m: data.data.volume_sell_history_30m, - volume_sell_history_30m_usd: data.data.volume_sell_history_30m_usd, - volume_sell_30m_change_percent: data.data.volume_sell_30m_change_percent, - trade_1h: data.data.trade_1h, - trade_history_1h: data.data.trade_history_1h, - trade_1h_change_percent: data.data.trade_1h_change_percent, - sell_1h: data.data.sell_1h, - sell_history_1h: data.data.sell_history_1h, - sell_1h_change_percent: data.data.sell_1h_change_percent, - buy_1h: data.data.buy_1h, - buy_history_1h: data.data.buy_history_1h, - buy_1h_change_percent: data.data.buy_1h_change_percent, - volume_1h: data.data.volume_1h, - volume_1h_usd: data.data.volume_1h_usd, - volume_history_1h: data.data.volume_history_1h, - volume_history_1h_usd: data.data.volume_history_1h_usd, - volume_1h_change_percent: data.data.volume_1h_change_percent, - volume_buy_1h: data.data.volume_buy_1h, - volume_buy_1h_usd: data.data.volume_buy_1h_usd, - volume_buy_history_1h: data.data.volume_buy_history_1h, - volume_buy_history_1h_usd: data.data.volume_buy_history_1h_usd, - volume_buy_1h_change_percent: data.data.volume_buy_1h_change_percent, - volume_sell_1h: data.data.volume_sell_1h, - volume_sell_1h_usd: data.data.volume_sell_1h_usd, - volume_sell_history_1h: data.data.volume_sell_history_1h, - volume_sell_history_1h_usd: data.data.volume_sell_history_1h_usd, - volume_sell_1h_change_percent: data.data.volume_sell_1h_change_percent, - trade_2h: data.data.trade_2h, - trade_history_2h: data.data.trade_history_2h, - trade_2h_change_percent: data.data.trade_2h_change_percent, - sell_2h: data.data.sell_2h, - sell_history_2h: data.data.sell_history_2h, - sell_2h_change_percent: data.data.sell_2h_change_percent, - buy_2h: data.data.buy_2h, - buy_history_2h: data.data.buy_history_2h, - buy_2h_change_percent: data.data.buy_2h_change_percent, - volume_2h: data.data.volume_2h, - volume_2h_usd: data.data.volume_2h_usd, - volume_history_2h: data.data.volume_history_2h, - volume_history_2h_usd: data.data.volume_history_2h_usd, - volume_2h_change_percent: data.data.volume_2h_change_percent, - volume_buy_2h: data.data.volume_buy_2h, - volume_buy_2h_usd: data.data.volume_buy_2h_usd, - volume_buy_history_2h: data.data.volume_buy_history_2h, - volume_buy_history_2h_usd: data.data.volume_buy_history_2h_usd, - volume_buy_2h_change_percent: data.data.volume_buy_2h_change_percent, - volume_sell_2h: data.data.volume_sell_2h, - volume_sell_2h_usd: data.data.volume_sell_2h_usd, - volume_sell_history_2h: data.data.volume_sell_history_2h, - volume_sell_history_2h_usd: data.data.volume_sell_history_2h_usd, - volume_sell_2h_change_percent: data.data.volume_sell_2h_change_percent, - trade_4h: data.data.trade_4h, - trade_history_4h: data.data.trade_history_4h, - trade_4h_change_percent: data.data.trade_4h_change_percent, - sell_4h: data.data.sell_4h, - sell_history_4h: data.data.sell_history_4h, - sell_4h_change_percent: data.data.sell_4h_change_percent, - buy_4h: data.data.buy_4h, - buy_history_4h: data.data.buy_history_4h, - buy_4h_change_percent: data.data.buy_4h_change_percent, - volume_4h: data.data.volume_4h, - volume_4h_usd: data.data.volume_4h_usd, - volume_history_4h: data.data.volume_history_4h, - volume_history_4h_usd: data.data.volume_history_4h_usd, - volume_4h_change_percent: data.data.volume_4h_change_percent, - volume_buy_4h: data.data.volume_buy_4h, - volume_buy_4h_usd: data.data.volume_buy_4h_usd, - volume_buy_history_4h: data.data.volume_buy_history_4h, - volume_buy_history_4h_usd: data.data.volume_buy_history_4h_usd, - volume_buy_4h_change_percent: data.data.volume_buy_4h_change_percent, - volume_sell_4h: data.data.volume_sell_4h, - volume_sell_4h_usd: data.data.volume_sell_4h_usd, - volume_sell_history_4h: data.data.volume_sell_history_4h, - volume_sell_history_4h_usd: data.data.volume_sell_history_4h_usd, - volume_sell_4h_change_percent: data.data.volume_sell_4h_change_percent, - trade_8h: data.data.trade_8h, - trade_history_8h: data.data.trade_history_8h, - trade_8h_change_percent: data.data.trade_8h_change_percent, - sell_8h: data.data.sell_8h, - sell_history_8h: data.data.sell_history_8h, - sell_8h_change_percent: data.data.sell_8h_change_percent, - buy_8h: data.data.buy_8h, - buy_history_8h: data.data.buy_history_8h, - buy_8h_change_percent: data.data.buy_8h_change_percent, - volume_8h: data.data.volume_8h, - volume_8h_usd: data.data.volume_8h_usd, - volume_history_8h: data.data.volume_history_8h, - volume_history_8h_usd: data.data.volume_history_8h_usd, - volume_8h_change_percent: data.data.volume_8h_change_percent, - volume_buy_8h: data.data.volume_buy_8h, - volume_buy_8h_usd: data.data.volume_buy_8h_usd, - volume_buy_history_8h: data.data.volume_buy_history_8h, - volume_buy_history_8h_usd: data.data.volume_buy_history_8h_usd, - volume_buy_8h_change_percent: data.data.volume_buy_8h_change_percent, - volume_sell_8h: data.data.volume_sell_8h, - volume_sell_8h_usd: data.data.volume_sell_8h_usd, - volume_sell_history_8h: data.data.volume_sell_history_8h, - volume_sell_history_8h_usd: data.data.volume_sell_history_8h_usd, - volume_sell_8h_change_percent: data.data.volume_sell_8h_change_percent, - trade_24h: data.data.trade_24h, - trade_history_24h: data.data.trade_history_24h, - trade_24h_change_percent: data.data.trade_24h_change_percent, - sell_24h: data.data.sell_24h, - sell_history_24h: data.data.sell_history_24h, - sell_24h_change_percent: data.data.sell_24h_change_percent, - buy_24h: data.data.buy_24h, - buy_history_24h: data.data.buy_history_24h, - buy_24h_change_percent: data.data.buy_24h_change_percent, - volume_24h: data.data.volume_24h, - volume_24h_usd: data.data.volume_24h_usd, - volume_history_24h: data.data.volume_history_24h, - volume_history_24h_usd: data.data.volume_history_24h_usd, - volume_24h_change_percent: data.data.volume_24h_change_percent, - volume_buy_24h: data.data.volume_buy_24h, - volume_buy_24h_usd: data.data.volume_buy_24h_usd, - volume_buy_history_24h: data.data.volume_buy_history_24h, - volume_buy_history_24h_usd: data.data.volume_buy_history_24h_usd, - volume_buy_24h_change_percent: data.data.volume_buy_24h_change_percent, - volume_sell_24h: data.data.volume_sell_24h, - volume_sell_24h_usd: data.data.volume_sell_24h_usd, - volume_sell_history_24h: data.data.volume_sell_history_24h, - volume_sell_history_24h_usd: data.data.volume_sell_history_24h_usd, - volume_sell_24h_change_percent: data.data.volume_sell_24h_change_percent, - }; - this.setCachedData(cacheKey, tradeData); - return tradeData; - } - - async fetchDexScreenerData(): Promise { - const cacheKey = `dexScreenerData_${this.tokenAddress}`; - const cachedData = this.getCachedData(cacheKey); - if (cachedData) { - return cachedData; - } + if (!data?.success || !data?.data) { + throw new Error("No token security data available"); + } - const url = `https://api.dexscreener.com/latest/dex/search?q=${this.tokenAddress}`; - try { - const data = await fetch(url) - .then((res) => res.json()) - .catch((err) => { - console.error(err); - }); - - if (!data || !data.pairs) { - throw new Error("No DexScreener data available"); - } - - const dexData: DexScreenerData = { - schemaVersion: data.schemaVersion, - pairs: data.pairs, - }; - - // Cache the result - this.setCachedData(cacheKey, dexData); - - return dexData; - } catch (error) { - console.error(`Error fetching DexScreener data:`, error); - return { - schemaVersion: "1.0.0", - pairs: [], - }; - } - } - - async analyzeHolderDistribution(tradeData: TokenTradeData): Promise { - // Define the time intervals to consider (e.g., 30m, 1h, 2h) - const intervals = [ - { period: "30m", change: tradeData.unique_wallet_30m_change_percent }, - { period: "1h", change: tradeData.unique_wallet_1h_change_percent }, - { period: "2h", change: tradeData.unique_wallet_2h_change_percent }, - { period: "4h", change: tradeData.unique_wallet_4h_change_percent }, - { period: "8h", change: tradeData.unique_wallet_8h_change_percent }, - { period: "24h", change: tradeData.unique_wallet_24h_change_percent }, - ]; - - // Calculate the average change percentage - const validChanges = intervals - .map((interval) => interval.change) - .filter((change) => change !== null && change !== undefined) as number[]; - - if (validChanges.length === 0) { - return "stable"; + const security: TokenSecurityData = { + ownerBalance: data.data.ownerBalance, + creatorBalance: data.data.creatorBalance, + ownerPercentage: data.data.ownerPercentage, + creatorPercentage: data.data.creatorPercentage, + top10HolderBalance: data.data.top10HolderBalance, + top10HolderPercent: data.data.top10HolderPercent, + }; + this.setCachedData(cacheKey, security); + + return security; } - const averageChange = - validChanges.reduce((acc, curr) => acc + curr, 0) / validChanges.length; + async fetchTokenTradeData(runtime: IAgentRuntime): Promise { + const cacheKey = `tokenTradeData_${this.tokenAddress}`; + const cachedData = this.getCachedData(cacheKey); + if (cachedData) { + return cachedData; + } - const increaseThreshold = 10; // e.g., average change > 10% - const decreaseThreshold = -10; // e.g., average change < -10% + const url = `${PROVIDER_CONFIG.BIRDEYE_API}${PROVIDER_CONFIG.TOKEN_TRADE_DATA_ENDPOINT}${this.tokenAddress}`; + const options = { + method: "GET", + headers: { + accept: "application/json", + "X-API-KEY": runtime.getSetting("BIRDEYE_API_KEY") || "", + }, + }; - if (averageChange > increaseThreshold) { - return "increasing"; - } else if (averageChange < decreaseThreshold) { - return "decreasing"; - } else { - return "stable"; + const data = await fetch(url, options) + .then((res) => res.json()) + .catch((err) => console.error(err)); + + if (!data?.success || !data?.data) { + throw new Error("No token trade data available"); + } + + const tradeData: TokenTradeData = { + address: data.data.address, + holder: data.data.holder, + market: data.data.market, + last_trade_unix_time: data.data.last_trade_unix_time, + last_trade_human_time: data.data.last_trade_human_time, + price: data.data.price, + history_30m_price: data.data.history_30m_price, + price_change_30m_percent: data.data.price_change_30m_percent, + history_1h_price: data.data.history_1h_price, + price_change_1h_percent: data.data.price_change_1h_percent, + history_2h_price: data.data.history_2h_price, + price_change_2h_percent: data.data.price_change_2h_percent, + history_4h_price: data.data.history_4h_price, + price_change_4h_percent: data.data.price_change_4h_percent, + history_6h_price: data.data.history_6h_price, + price_change_6h_percent: data.data.price_change_6h_percent, + history_8h_price: data.data.history_8h_price, + price_change_8h_percent: data.data.price_change_8h_percent, + history_12h_price: data.data.history_12h_price, + price_change_12h_percent: data.data.price_change_12h_percent, + history_24h_price: data.data.history_24h_price, + price_change_24h_percent: data.data.price_change_24h_percent, + unique_wallet_30m: data.data.unique_wallet_30m, + unique_wallet_history_30m: data.data.unique_wallet_history_30m, + unique_wallet_30m_change_percent: + data.data.unique_wallet_30m_change_percent, + unique_wallet_1h: data.data.unique_wallet_1h, + unique_wallet_history_1h: data.data.unique_wallet_history_1h, + unique_wallet_1h_change_percent: + data.data.unique_wallet_1h_change_percent, + unique_wallet_2h: data.data.unique_wallet_2h, + unique_wallet_history_2h: data.data.unique_wallet_history_2h, + unique_wallet_2h_change_percent: + data.data.unique_wallet_2h_change_percent, + unique_wallet_4h: data.data.unique_wallet_4h, + unique_wallet_history_4h: data.data.unique_wallet_history_4h, + unique_wallet_4h_change_percent: + data.data.unique_wallet_4h_change_percent, + unique_wallet_8h: data.data.unique_wallet_8h, + unique_wallet_history_8h: data.data.unique_wallet_history_8h, + unique_wallet_8h_change_percent: + data.data.unique_wallet_8h_change_percent, + unique_wallet_24h: data.data.unique_wallet_24h, + unique_wallet_history_24h: data.data.unique_wallet_history_24h, + unique_wallet_24h_change_percent: + data.data.unique_wallet_24h_change_percent, + trade_30m: data.data.trade_30m, + trade_history_30m: data.data.trade_history_30m, + trade_30m_change_percent: data.data.trade_30m_change_percent, + sell_30m: data.data.sell_30m, + sell_history_30m: data.data.sell_history_30m, + sell_30m_change_percent: data.data.sell_30m_change_percent, + buy_30m: data.data.buy_30m, + buy_history_30m: data.data.buy_history_30m, + buy_30m_change_percent: data.data.buy_30m_change_percent, + volume_30m: data.data.volume_30m, + volume_30m_usd: data.data.volume_30m_usd, + volume_history_30m: data.data.volume_history_30m, + volume_history_30m_usd: data.data.volume_history_30m_usd, + volume_30m_change_percent: data.data.volume_30m_change_percent, + volume_buy_30m: data.data.volume_buy_30m, + volume_buy_30m_usd: data.data.volume_buy_30m_usd, + volume_buy_history_30m: data.data.volume_buy_history_30m, + volume_buy_history_30m_usd: data.data.volume_buy_history_30m_usd, + volume_buy_30m_change_percent: + data.data.volume_buy_30m_change_percent, + volume_sell_30m: data.data.volume_sell_30m, + volume_sell_30m_usd: data.data.volume_sell_30m_usd, + volume_sell_history_30m: data.data.volume_sell_history_30m, + volume_sell_history_30m_usd: data.data.volume_sell_history_30m_usd, + volume_sell_30m_change_percent: + data.data.volume_sell_30m_change_percent, + trade_1h: data.data.trade_1h, + trade_history_1h: data.data.trade_history_1h, + trade_1h_change_percent: data.data.trade_1h_change_percent, + sell_1h: data.data.sell_1h, + sell_history_1h: data.data.sell_history_1h, + sell_1h_change_percent: data.data.sell_1h_change_percent, + buy_1h: data.data.buy_1h, + buy_history_1h: data.data.buy_history_1h, + buy_1h_change_percent: data.data.buy_1h_change_percent, + volume_1h: data.data.volume_1h, + volume_1h_usd: data.data.volume_1h_usd, + volume_history_1h: data.data.volume_history_1h, + volume_history_1h_usd: data.data.volume_history_1h_usd, + volume_1h_change_percent: data.data.volume_1h_change_percent, + volume_buy_1h: data.data.volume_buy_1h, + volume_buy_1h_usd: data.data.volume_buy_1h_usd, + volume_buy_history_1h: data.data.volume_buy_history_1h, + volume_buy_history_1h_usd: data.data.volume_buy_history_1h_usd, + volume_buy_1h_change_percent: + data.data.volume_buy_1h_change_percent, + volume_sell_1h: data.data.volume_sell_1h, + volume_sell_1h_usd: data.data.volume_sell_1h_usd, + volume_sell_history_1h: data.data.volume_sell_history_1h, + volume_sell_history_1h_usd: data.data.volume_sell_history_1h_usd, + volume_sell_1h_change_percent: + data.data.volume_sell_1h_change_percent, + trade_2h: data.data.trade_2h, + trade_history_2h: data.data.trade_history_2h, + trade_2h_change_percent: data.data.trade_2h_change_percent, + sell_2h: data.data.sell_2h, + sell_history_2h: data.data.sell_history_2h, + sell_2h_change_percent: data.data.sell_2h_change_percent, + buy_2h: data.data.buy_2h, + buy_history_2h: data.data.buy_history_2h, + buy_2h_change_percent: data.data.buy_2h_change_percent, + volume_2h: data.data.volume_2h, + volume_2h_usd: data.data.volume_2h_usd, + volume_history_2h: data.data.volume_history_2h, + volume_history_2h_usd: data.data.volume_history_2h_usd, + volume_2h_change_percent: data.data.volume_2h_change_percent, + volume_buy_2h: data.data.volume_buy_2h, + volume_buy_2h_usd: data.data.volume_buy_2h_usd, + volume_buy_history_2h: data.data.volume_buy_history_2h, + volume_buy_history_2h_usd: data.data.volume_buy_history_2h_usd, + volume_buy_2h_change_percent: + data.data.volume_buy_2h_change_percent, + volume_sell_2h: data.data.volume_sell_2h, + volume_sell_2h_usd: data.data.volume_sell_2h_usd, + volume_sell_history_2h: data.data.volume_sell_history_2h, + volume_sell_history_2h_usd: data.data.volume_sell_history_2h_usd, + volume_sell_2h_change_percent: + data.data.volume_sell_2h_change_percent, + trade_4h: data.data.trade_4h, + trade_history_4h: data.data.trade_history_4h, + trade_4h_change_percent: data.data.trade_4h_change_percent, + sell_4h: data.data.sell_4h, + sell_history_4h: data.data.sell_history_4h, + sell_4h_change_percent: data.data.sell_4h_change_percent, + buy_4h: data.data.buy_4h, + buy_history_4h: data.data.buy_history_4h, + buy_4h_change_percent: data.data.buy_4h_change_percent, + volume_4h: data.data.volume_4h, + volume_4h_usd: data.data.volume_4h_usd, + volume_history_4h: data.data.volume_history_4h, + volume_history_4h_usd: data.data.volume_history_4h_usd, + volume_4h_change_percent: data.data.volume_4h_change_percent, + volume_buy_4h: data.data.volume_buy_4h, + volume_buy_4h_usd: data.data.volume_buy_4h_usd, + volume_buy_history_4h: data.data.volume_buy_history_4h, + volume_buy_history_4h_usd: data.data.volume_buy_history_4h_usd, + volume_buy_4h_change_percent: + data.data.volume_buy_4h_change_percent, + volume_sell_4h: data.data.volume_sell_4h, + volume_sell_4h_usd: data.data.volume_sell_4h_usd, + volume_sell_history_4h: data.data.volume_sell_history_4h, + volume_sell_history_4h_usd: data.data.volume_sell_history_4h_usd, + volume_sell_4h_change_percent: + data.data.volume_sell_4h_change_percent, + trade_8h: data.data.trade_8h, + trade_history_8h: data.data.trade_history_8h, + trade_8h_change_percent: data.data.trade_8h_change_percent, + sell_8h: data.data.sell_8h, + sell_history_8h: data.data.sell_history_8h, + sell_8h_change_percent: data.data.sell_8h_change_percent, + buy_8h: data.data.buy_8h, + buy_history_8h: data.data.buy_history_8h, + buy_8h_change_percent: data.data.buy_8h_change_percent, + volume_8h: data.data.volume_8h, + volume_8h_usd: data.data.volume_8h_usd, + volume_history_8h: data.data.volume_history_8h, + volume_history_8h_usd: data.data.volume_history_8h_usd, + volume_8h_change_percent: data.data.volume_8h_change_percent, + volume_buy_8h: data.data.volume_buy_8h, + volume_buy_8h_usd: data.data.volume_buy_8h_usd, + volume_buy_history_8h: data.data.volume_buy_history_8h, + volume_buy_history_8h_usd: data.data.volume_buy_history_8h_usd, + volume_buy_8h_change_percent: + data.data.volume_buy_8h_change_percent, + volume_sell_8h: data.data.volume_sell_8h, + volume_sell_8h_usd: data.data.volume_sell_8h_usd, + volume_sell_history_8h: data.data.volume_sell_history_8h, + volume_sell_history_8h_usd: data.data.volume_sell_history_8h_usd, + volume_sell_8h_change_percent: + data.data.volume_sell_8h_change_percent, + trade_24h: data.data.trade_24h, + trade_history_24h: data.data.trade_history_24h, + trade_24h_change_percent: data.data.trade_24h_change_percent, + sell_24h: data.data.sell_24h, + sell_history_24h: data.data.sell_history_24h, + sell_24h_change_percent: data.data.sell_24h_change_percent, + buy_24h: data.data.buy_24h, + buy_history_24h: data.data.buy_history_24h, + buy_24h_change_percent: data.data.buy_24h_change_percent, + volume_24h: data.data.volume_24h, + volume_24h_usd: data.data.volume_24h_usd, + volume_history_24h: data.data.volume_history_24h, + volume_history_24h_usd: data.data.volume_history_24h_usd, + volume_24h_change_percent: data.data.volume_24h_change_percent, + volume_buy_24h: data.data.volume_buy_24h, + volume_buy_24h_usd: data.data.volume_buy_24h_usd, + volume_buy_history_24h: data.data.volume_buy_history_24h, + volume_buy_history_24h_usd: data.data.volume_buy_history_24h_usd, + volume_buy_24h_change_percent: + data.data.volume_buy_24h_change_percent, + volume_sell_24h: data.data.volume_sell_24h, + volume_sell_24h_usd: data.data.volume_sell_24h_usd, + volume_sell_history_24h: data.data.volume_sell_history_24h, + volume_sell_history_24h_usd: data.data.volume_sell_history_24h_usd, + volume_sell_24h_change_percent: + data.data.volume_sell_24h_change_percent, + }; + this.setCachedData(cacheKey, tradeData); + return tradeData; } - } - async fetchHolderList(): Promise { - const cacheKey = `holderList_${this.tokenAddress}`; - const cachedData = this.getCachedData(cacheKey); - if (cachedData) { - return cachedData; + async fetchDexScreenerData(): Promise { + const cacheKey = `dexScreenerData_${this.tokenAddress}`; + const cachedData = this.getCachedData(cacheKey); + if (cachedData) { + return cachedData; + } + + const url = `https://api.dexscreener.com/latest/dex/search?q=${this.tokenAddress}`; + try { + const data = await fetch(url) + .then((res) => res.json()) + .catch((err) => { + console.error(err); + }); + + if (!data || !data.pairs) { + throw new Error("No DexScreener data available"); + } + + const dexData: DexScreenerData = { + schemaVersion: data.schemaVersion, + pairs: data.pairs, + }; + + // Cache the result + this.setCachedData(cacheKey, dexData); + + return dexData; + } catch (error) { + console.error(`Error fetching DexScreener data:`, error); + return { + schemaVersion: "1.0.0", + pairs: [], + }; + } } - const allHoldersMap = new Map(); - let page = 1; - const limit = 1000; - let cursor; - //HELIOUS_API_KEY needs to be added - const url = `https://mainnet.helius-rpc.com/?api-key=${settings.HELIOUS_API_KEY || ""}`; - - try { - while (true) { - const params = { - limit: limit, - displayOptions: {}, - mint: this.tokenAddress, - cursor: cursor, - }; - if (cursor != undefined) { - params.cursor = cursor; + async analyzeHolderDistribution( + tradeData: TokenTradeData + ): Promise { + // Define the time intervals to consider (e.g., 30m, 1h, 2h) + const intervals = [ + { + period: "30m", + change: tradeData.unique_wallet_30m_change_percent, + }, + { period: "1h", change: tradeData.unique_wallet_1h_change_percent }, + { period: "2h", change: tradeData.unique_wallet_2h_change_percent }, + { period: "4h", change: tradeData.unique_wallet_4h_change_percent }, + { period: "8h", change: tradeData.unique_wallet_8h_change_percent }, + { + period: "24h", + change: tradeData.unique_wallet_24h_change_percent, + }, + ]; + + // Calculate the average change percentage + const validChanges = intervals + .map((interval) => interval.change) + .filter( + (change) => change !== null && change !== undefined + ) as number[]; + + if (validChanges.length === 0) { + return "stable"; } - if (page > 2) { - break; + const averageChange = + validChanges.reduce((acc, curr) => acc + curr, 0) / + validChanges.length; + + const increaseThreshold = 10; // e.g., average change > 10% + const decreaseThreshold = -10; // e.g., average change < -10% + + if (averageChange > increaseThreshold) { + return "increasing"; + } else if (averageChange < decreaseThreshold) { + return "decreasing"; + } else { + return "stable"; } - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - jsonrpc: "2.0", - id: "helius-test", - method: "getTokenAccounts", - params: params, - }), - }); - - const data = await response.json(); - - if ( - !data || - !data.result || - !data.result.token_accounts || - data.result.token_accounts.length === 0 - ) { - break; + } + + async fetchHolderList(): Promise { + const cacheKey = `holderList_${this.tokenAddress}`; + const cachedData = this.getCachedData(cacheKey); + if (cachedData) { + return cachedData; } - data.result.token_accounts.forEach((account: any) => { - const owner = account.owner; - const balance = parseFloat(account.amount); - - if (allHoldersMap.has(owner)) { - allHoldersMap.set(owner, allHoldersMap.get(owner)! + balance); - } else { - allHoldersMap.set(owner, balance); - } - }); - cursor = data.result.cursor; - page++; - } - - const holders: HolderData[] = Array.from(allHoldersMap.entries()).map( - ([address, balance]) => ({ - address, - balance: balance.toString(), - }) - ); - - // Cache the result - this.setCachedData(cacheKey, holders); - - return holders; - } catch (error) { - console.error("Error fetching holder list from Helius:", error); - throw new Error("Failed to fetch holder list from Helius."); + const allHoldersMap = new Map(); + let page = 1; + const limit = 1000; + let cursor; + //HELIOUS_API_KEY needs to be added + const url = `https://mainnet.helius-rpc.com/?api-key=${settings.HELIOUS_API_KEY || ""}`; + + try { + while (true) { + const params = { + limit: limit, + displayOptions: {}, + mint: this.tokenAddress, + cursor: cursor, + }; + if (cursor != undefined) { + params.cursor = cursor; + } + + if (page > 2) { + break; + } + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: "helius-test", + method: "getTokenAccounts", + params: params, + }), + }); + + const data = await response.json(); + + if ( + !data || + !data.result || + !data.result.token_accounts || + data.result.token_accounts.length === 0 + ) { + break; + } + + data.result.token_accounts.forEach((account: any) => { + const owner = account.owner; + const balance = parseFloat(account.amount); + + if (allHoldersMap.has(owner)) { + allHoldersMap.set( + owner, + allHoldersMap.get(owner)! + balance + ); + } else { + allHoldersMap.set(owner, balance); + } + }); + cursor = data.result.cursor; + page++; + } + + const holders: HolderData[] = Array.from( + allHoldersMap.entries() + ).map(([address, balance]) => ({ + address, + balance: balance.toString(), + })); + + // Cache the result + this.setCachedData(cacheKey, holders); + + return holders; + } catch (error) { + console.error("Error fetching holder list from Helius:", error); + throw new Error("Failed to fetch holder list from Helius."); + } } - } - async filterHighValueHolders( - tradeData: TokenTradeData - ): Promise> { - const holdersData = await this.fetchHolderList(); + async filterHighValueHolders( + tradeData: TokenTradeData + ): Promise> { + const holdersData = await this.fetchHolderList(); + + const tokenPriceUsd = new BigNumber(tradeData.price); + + const highValueHolders = holdersData + .filter((holder) => { + const balanceUsd = new BigNumber(holder.balance).multipliedBy( + tokenPriceUsd + ); + return balanceUsd.isGreaterThan(5); + }) + .map((holder) => ({ + holderAddress: holder.address, + balanceUsd: new BigNumber(holder.balance) + .multipliedBy(tokenPriceUsd) + .toFixed(2), + })); + + return highValueHolders; + } - const tokenPriceUsd = new BigNumber(tradeData.price); + async checkRecentTrades(tradeData: TokenTradeData): Promise { + return new BigNumber(tradeData.volume_24h_usd).isGreaterThan(0); + } - const highValueHolders = holdersData - .filter((holder) => { - const balanceUsd = new BigNumber(holder.balance).multipliedBy( - tokenPriceUsd - ); - return balanceUsd.isGreaterThan(5); - }) - .map((holder) => ({ - holderAddress: holder.address, - balanceUsd: new BigNumber(holder.balance) - .multipliedBy(tokenPriceUsd) - .toFixed(2), - })); - - return highValueHolders; - } - - async checkRecentTrades(tradeData: TokenTradeData): Promise { - return new BigNumber(tradeData.volume_24h_usd).isGreaterThan(0); - } - - async countHighSupplyHolders( - securityData: TokenSecurityData - ): Promise { - try { - const ownerBalance = new BigNumber(securityData.ownerBalance); - const totalSupply = ownerBalance.plus(securityData.creatorBalance); - - const highSupplyHolders = await this.fetchHolderList(); - const highSupplyHoldersCount = highSupplyHolders.filter((holder) => { - const balance = new BigNumber(holder.balance); - return balance.dividedBy(totalSupply).isGreaterThan(0.02); - }).length; - return highSupplyHoldersCount; - } catch (error) { - console.error("Error counting high supply holders:", error); - return 0; + async countHighSupplyHolders( + securityData: TokenSecurityData + ): Promise { + try { + const ownerBalance = new BigNumber(securityData.ownerBalance); + const totalSupply = ownerBalance.plus(securityData.creatorBalance); + + const highSupplyHolders = await this.fetchHolderList(); + const highSupplyHoldersCount = highSupplyHolders.filter( + (holder) => { + const balance = new BigNumber(holder.balance); + return balance.dividedBy(totalSupply).isGreaterThan(0.02); + } + ).length; + return highSupplyHoldersCount; + } catch (error) { + console.error("Error counting high supply holders:", error); + return 0; + } } - } - async getProcessedTokenData(runtime: IAgentRuntime): Promise { - try { - const security = await this.fetchTokenSecurity(); + async getProcessedTokenData( + runtime: IAgentRuntime + ): Promise { + try { + const security = await this.fetchTokenSecurity(); - const tradeData = await this.fetchTokenTradeData(runtime); + const tradeData = await this.fetchTokenTradeData(runtime); - const dexData = await this.fetchDexScreenerData(); + const dexData = await this.fetchDexScreenerData(); - const holderDistributionTrend = - await this.analyzeHolderDistribution(tradeData); + const holderDistributionTrend = + await this.analyzeHolderDistribution(tradeData); - const highValueHolders = await this.filterHighValueHolders(tradeData); + const highValueHolders = + await this.filterHighValueHolders(tradeData); - const recentTrades = await this.checkRecentTrades(tradeData); + const recentTrades = await this.checkRecentTrades(tradeData); - const highSupplyHoldersCount = - await this.countHighSupplyHolders(security); + const highSupplyHoldersCount = + await this.countHighSupplyHolders(security); - const isDexScreenerListed = dexData.pairs.length > 0; - const isDexScreenerPaid = dexData.pairs.some( - (pair) => pair.boosts && pair.boosts.active > 0 - ); + const isDexScreenerListed = dexData.pairs.length > 0; + const isDexScreenerPaid = dexData.pairs.some( + (pair) => pair.boosts && pair.boosts.active > 0 + ); - const processedData: ProcessedTokenData = { - security, - tradeData, - holderDistributionTrend, - highValueHolders, - recentTrades, - highSupplyHoldersCount, - dexScreenerData: dexData, - isDexScreenerListed, - isDexScreenerPaid, - }; + const processedData: ProcessedTokenData = { + security, + tradeData, + holderDistributionTrend, + highValueHolders, + recentTrades, + highSupplyHoldersCount, + dexScreenerData: dexData, + isDexScreenerListed, + isDexScreenerPaid, + }; - return processedData; - } catch (error) { - console.error("Error processing token data:", error); - throw error; - } - } - - formatTokenData(data: ProcessedTokenData): string { - let output = `**Token Security and Trade Report**\n`; - output += `Token Address: ${this.tokenAddress}\n\n`; - - // Security Data - output += `**Ownership Distribution:**\n`; - output += `- Owner Balance: ${data.security.ownerBalance}\n`; - output += `- Creator Balance: ${data.security.creatorBalance}\n`; - output += `- Owner Percentage: ${data.security.ownerPercentage}%\n`; - output += `- Creator Percentage: ${data.security.creatorPercentage}%\n`; - output += `- Top 10 Holders Balance: ${data.security.top10HolderBalance}\n`; - output += `- Top 10 Holders Percentage: ${data.security.top10HolderPercent}%\n\n`; - - // Trade Data - output += `**Trade Data:**\n`; - output += `- Holders: ${data.tradeData.holder}\n`; - output += `- Unique Wallets (24h): ${data.tradeData.unique_wallet_24h}\n`; - output += `- Price Change (24h): ${data.tradeData.price_change_24h_percent}%\n`; - output += `- Price Change (12h): ${data.tradeData.price_change_12h_percent}%\n`; - output += `- Volume (24h USD): $${new BigNumber(data.tradeData.volume_24h_usd).toFixed(2)}\n`; - output += `- Current Price: $${new BigNumber(data.tradeData.price).toFixed(2)}\n\n`; - - // Holder Distribution Trend - output += `**Holder Distribution Trend:** ${data.holderDistributionTrend}\n\n`; - - // High-Value Holders - output += `**High-Value Holders (>$5 USD):**\n`; - if (data.highValueHolders.length === 0) { - output += `- No high-value holders found or data not available.\n`; - } else { - data.highValueHolders.forEach((holder) => { - output += `- ${holder.holderAddress}: $${holder.balanceUsd}\n`; - }); + return processedData; + } catch (error) { + console.error("Error processing token data:", error); + throw error; + } } - output += `\n`; - - // Recent Trades - output += `**Recent Trades (Last 24h):** ${data.recentTrades ? "Yes" : "No"}\n\n`; - - // High-Supply Holders - output += `**Holders with >2% Supply:** ${data.highSupplyHoldersCount}\n\n`; - - // DexScreener Status - output += `**DexScreener Listing:** ${data.isDexScreenerListed ? "Yes" : "No"}\n`; - if (data.isDexScreenerListed) { - output += `- Listing Type: ${data.isDexScreenerPaid ? "Paid" : "Free"}\n`; - output += `- Number of DexPairs: ${data.dexScreenerData.pairs.length}\n\n`; - output += `**DexScreener Pairs:**\n`; - data.dexScreenerData.pairs.forEach((pair, index) => { - output += `\n**Pair ${index + 1}:**\n`; - output += `- DEX: ${pair.dexId}\n`; - output += `- URL: ${pair.url}\n`; - output += `- Price USD: $${new BigNumber(pair.priceUsd).toFixed(6)}\n`; - output += `- Volume (24h USD): $${new BigNumber(pair.volume.h24).toFixed(2)}\n`; - output += `- Boosts Active: ${pair.boosts && pair.boosts.active}\n`; - output += `- Liquidity USD: $${new BigNumber(pair.liquidity.usd).toFixed(2)}\n`; - }); + + formatTokenData(data: ProcessedTokenData): string { + let output = `**Token Security and Trade Report**\n`; + output += `Token Address: ${this.tokenAddress}\n\n`; + + // Security Data + output += `**Ownership Distribution:**\n`; + output += `- Owner Balance: ${data.security.ownerBalance}\n`; + output += `- Creator Balance: ${data.security.creatorBalance}\n`; + output += `- Owner Percentage: ${data.security.ownerPercentage}%\n`; + output += `- Creator Percentage: ${data.security.creatorPercentage}%\n`; + output += `- Top 10 Holders Balance: ${data.security.top10HolderBalance}\n`; + output += `- Top 10 Holders Percentage: ${data.security.top10HolderPercent}%\n\n`; + + // Trade Data + output += `**Trade Data:**\n`; + output += `- Holders: ${data.tradeData.holder}\n`; + output += `- Unique Wallets (24h): ${data.tradeData.unique_wallet_24h}\n`; + output += `- Price Change (24h): ${data.tradeData.price_change_24h_percent}%\n`; + output += `- Price Change (12h): ${data.tradeData.price_change_12h_percent}%\n`; + output += `- Volume (24h USD): $${new BigNumber(data.tradeData.volume_24h_usd).toFixed(2)}\n`; + output += `- Current Price: $${new BigNumber(data.tradeData.price).toFixed(2)}\n\n`; + + // Holder Distribution Trend + output += `**Holder Distribution Trend:** ${data.holderDistributionTrend}\n\n`; + + // High-Value Holders + output += `**High-Value Holders (>$5 USD):**\n`; + if (data.highValueHolders.length === 0) { + output += `- No high-value holders found or data not available.\n`; + } else { + data.highValueHolders.forEach((holder) => { + output += `- ${holder.holderAddress}: $${holder.balanceUsd}\n`; + }); + } + output += `\n`; + + // Recent Trades + output += `**Recent Trades (Last 24h):** ${data.recentTrades ? "Yes" : "No"}\n\n`; + + // High-Supply Holders + output += `**Holders with >2% Supply:** ${data.highSupplyHoldersCount}\n\n`; + + // DexScreener Status + output += `**DexScreener Listing:** ${data.isDexScreenerListed ? "Yes" : "No"}\n`; + if (data.isDexScreenerListed) { + output += `- Listing Type: ${data.isDexScreenerPaid ? "Paid" : "Free"}\n`; + output += `- Number of DexPairs: ${data.dexScreenerData.pairs.length}\n\n`; + output += `**DexScreener Pairs:**\n`; + data.dexScreenerData.pairs.forEach((pair, index) => { + output += `\n**Pair ${index + 1}:**\n`; + output += `- DEX: ${pair.dexId}\n`; + output += `- URL: ${pair.url}\n`; + output += `- Price USD: $${new BigNumber(pair.priceUsd).toFixed(6)}\n`; + output += `- Volume (24h USD): $${new BigNumber(pair.volume.h24).toFixed(2)}\n`; + output += `- Boosts Active: ${pair.boosts && pair.boosts.active}\n`; + output += `- Liquidity USD: $${new BigNumber(pair.liquidity.usd).toFixed(2)}\n`; + }); + } + output += `\n`; + + return output; } - output += `\n`; - - return output; - } - - async getFormattedTokenReport(runtime: IAgentRuntime): Promise { - try { - const processedData = await this.getProcessedTokenData(runtime); - return this.formatTokenData(processedData); - } catch (error) { - console.error("Error generating token report:", error); - return "Unable to fetch token information. Please try again later."; + + async getFormattedTokenReport(runtime: IAgentRuntime): Promise { + try { + const processedData = await this.getProcessedTokenData(runtime); + return this.formatTokenData(processedData); + } catch (error) { + console.error("Error generating token report:", error); + return "Unable to fetch token information. Please try again later."; + } } - } } const tokenAddress = PROVIDER_CONFIG.TOKEN_ADDRESSES.Example; const tokenProvider: Provider = { - get: async ( - runtime: IAgentRuntime, - _message: Memory, - _state?: State - ): Promise => { - try { - const provider = new TokenProvider(tokenAddress); - return provider.getFormattedTokenReport(runtime); - } catch (error) { - console.error("Error fetching token data:", error); - return "Unable to fetch token information. Please try again later."; - } - }, + get: async ( + runtime: IAgentRuntime, + _message: Memory, + _state?: State + ): Promise => { + try { + const provider = new TokenProvider(tokenAddress); + return provider.getFormattedTokenReport(runtime); + } catch (error) { + console.error("Error fetching token data:", error); + return "Unable to fetch token information. Please try again later."; + } + }, }; export default tokenProvider; diff --git a/core/src/providers/wallet.ts b/core/src/providers/wallet.ts index 4ece3750a1..0f5e240cf8 100644 --- a/core/src/providers/wallet.ts +++ b/core/src/providers/wallet.ts @@ -5,253 +5,284 @@ import BigNumber from "bignumber.js"; // Provider configuration const PROVIDER_CONFIG = { - BIRDEYE_API: 'https://public-api.birdeye.so', - MAX_RETRIES: 3, - RETRY_DELAY: 2000, - DEFAULT_RPC: 'https://api.mainnet-beta.solana.com', - TOKEN_ADDRESSES: { - SOL: 'So11111111111111111111111111111111111111112', - BTC: 'qfnqNqs3nCAHjnyCgLRDbBtq4p2MtHZxw8YjSyYhPoL', - ETH: '7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs' - } + BIRDEYE_API: "https://public-api.birdeye.so", + MAX_RETRIES: 3, + RETRY_DELAY: 2000, + DEFAULT_RPC: "https://api.mainnet-beta.solana.com", + TOKEN_ADDRESSES: { + SOL: "So11111111111111111111111111111111111111112", + BTC: "qfnqNqs3nCAHjnyCgLRDbBtq4p2MtHZxw8YjSyYhPoL", + ETH: "7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs", + }, }; interface Item { - name: string; - symbol: string; - decimals: number; - balance: string; - uiAmount: string; - priceUsd: string; - valueUsd: string; - valueSol?: string; + name: string; + symbol: string; + decimals: number; + balance: string; + uiAmount: string; + priceUsd: string; + valueUsd: string; + valueSol?: string; } interface WalletPortfolio { - totalUsd: string; - totalSol?: string; - items: Array; + totalUsd: string; + totalSol?: string; + items: Array; } interface BirdEyePriceData { - data: { - [key: string]: { - price: number; - priceChange24h: number; + data: { + [key: string]: { + price: number; + priceChange24h: number; + }; }; - }; } interface Prices { - solana: { usd: string }; - bitcoin: { usd: string }; - ethereum: { usd: string }; + solana: { usd: string }; + bitcoin: { usd: string }; + ethereum: { usd: string }; } class WalletProvider { - constructor( - private connection: Connection, - private walletPublicKey: PublicKey - ) {} - - private async fetchWithRetry(runtime, url: string, options: RequestInit = {}): Promise { - let lastError: Error; - - for (let i = 0; i < PROVIDER_CONFIG.MAX_RETRIES; i++) { - try { - const response = await fetch(url, { - ...options, - headers: { - 'Accept': 'application/json', - 'x-chain': 'solana', - 'X-API-KEY': runtime.getSetting('BIRDEYE_API_KEY', '') || '', - ...options.headers - } - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`HTTP error! status: ${response.status}, message: ${errorText}`); + constructor( + private connection: Connection, + private walletPublicKey: PublicKey + ) {} + + private async fetchWithRetry( + runtime, + url: string, + options: RequestInit = {} + ): Promise { + let lastError: Error; + + for (let i = 0; i < PROVIDER_CONFIG.MAX_RETRIES; i++) { + try { + const response = await fetch(url, { + ...options, + headers: { + Accept: "application/json", + "x-chain": "solana", + "X-API-KEY": + runtime.getSetting("BIRDEYE_API_KEY", "") || "", + ...options.headers, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `HTTP error! status: ${response.status}, message: ${errorText}` + ); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error(`Attempt ${i + 1} failed:`, error); + lastError = error; + if (i < PROVIDER_CONFIG.MAX_RETRIES - 1) { + const delay = PROVIDER_CONFIG.RETRY_DELAY * Math.pow(2, i); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + } } - const data = await response.json(); - return data; - } catch (error) { - console.error(`Attempt ${i + 1} failed:`, error); - lastError = error; - if (i < PROVIDER_CONFIG.MAX_RETRIES - 1) { - const delay = PROVIDER_CONFIG.RETRY_DELAY * Math.pow(2, i); - await new Promise(resolve => - setTimeout(resolve, delay) - ); - continue; + console.error( + "All attempts failed. Throwing the last error:", + lastError + ); + throw lastError; + } + + async fetchPortfolioValue(runtime): Promise { + try { + const walletData = await this.fetchWithRetry( + runtime, + `${PROVIDER_CONFIG.BIRDEYE_API}/v1/wallet/token_list?wallet=${this.walletPublicKey.toBase58()}` + ); + + if (!walletData?.success || !walletData?.data) { + console.error("No portfolio data available", walletData); + throw new Error("No portfolio data available"); + } + + const data = walletData.data; + const totalUsd = new BigNumber(data.totalUsd.toString()); + const prices = await this.fetchPrices(runtime); + const solPriceInUSD = new BigNumber(prices.solana.usd.toString()); + + const items = data.items.map((item: any) => ({ + ...item, + valueSol: new BigNumber(item.valueUsd || 0) + .div(solPriceInUSD) + .toFixed(6), + name: item.name || "Unknown", + symbol: item.symbol || "Unknown", + priceUsd: item.priceUsd || "0", + valueUsd: item.valueUsd || "0", + })); + + const totalSol = totalUsd.div(solPriceInUSD); + + return { + totalUsd: totalUsd.toString(), + totalSol: totalSol.toFixed(6), + items: items.sort((a, b) => + new BigNumber(b.valueUsd) + .minus(new BigNumber(a.valueUsd)) + .toNumber() + ), + }; + } catch (error) { + console.error("Error fetching portfolio:", error); + throw error; } - } } - - console.error("All attempts failed. Throwing the last error:", lastError); - throw lastError; - } - - async fetchPortfolioValue(runtime): Promise { - try { - const walletData = await this.fetchWithRetry( - runtime, - `${PROVIDER_CONFIG.BIRDEYE_API}/v1/wallet/token_list?wallet=${this.walletPublicKey.toBase58()}` - ); - - if (!walletData?.success || !walletData?.data) { - console.error("No portfolio data available", walletData); - throw new Error('No portfolio data available'); - } - - const data = walletData.data; - const totalUsd = new BigNumber(data.totalUsd.toString()); - const prices = await this.fetchPrices(runtime); - const solPriceInUSD = new BigNumber(prices.solana.usd.toString()); - - const items = data.items.map((item: any) => ({ - ...item, - valueSol: new BigNumber(item.valueUsd || 0) - .div(solPriceInUSD) - .toFixed(6), - name: item.name || "Unknown", - symbol: item.symbol || "Unknown", - priceUsd: item.priceUsd || "0", - valueUsd: item.valueUsd || "0" - })); - - const totalSol = totalUsd.div(solPriceInUSD); - - return { - totalUsd: totalUsd.toString(), - totalSol: totalSol.toFixed(6), - items: items.sort((a, b) => - new BigNumber(b.valueUsd).minus(new BigNumber(a.valueUsd)).toNumber() - ) - }; - } catch (error) { - console.error('Error fetching portfolio:', error); - throw error; + + async fetchPrices(runtime): Promise { + try { + const { SOL, BTC, ETH } = PROVIDER_CONFIG.TOKEN_ADDRESSES; + const tokens = [SOL, BTC, ETH]; + const prices: Prices = { + solana: { usd: "0" }, + bitcoin: { usd: "0" }, + ethereum: { usd: "0" }, + }; + + for (const token of tokens) { + const response = await this.fetchWithRetry( + runtime, + `${PROVIDER_CONFIG.BIRDEYE_API}/defi/price?address=${token}`, + { + headers: { + "x-chain": "solana", + }, + } + ); + + if (response?.data?.value) { + const price = response.data.value.toString(); + prices[ + token === SOL + ? "solana" + : token === BTC + ? "bitcoin" + : "ethereum" + ].usd = price; + } else { + console.warn(`No price data available for token: ${token}`); + } + } + + return prices; + } catch (error) { + console.error("Error fetching prices:", error); + throw error; + } } - } - - async fetchPrices(runtime): Promise { - try { - const { SOL, BTC, ETH } = PROVIDER_CONFIG.TOKEN_ADDRESSES; - const tokens = [SOL, BTC, ETH]; - const prices: Prices = { - solana: { usd: "0" }, - bitcoin: { usd: "0" }, - ethereum: { usd: "0" }, - }; - - for (const token of tokens) { - const response = await this.fetchWithRetry( - runtime, - `${PROVIDER_CONFIG.BIRDEYE_API}/defi/price?address=${token}`, - { - headers: { - "x-chain": "solana", - }, - } + + formatPortfolio( + runtime, + portfolio: WalletPortfolio, + prices: Prices + ): string { + let output = `${runtime.character.description}\n`; + output += `Wallet Address: ${this.walletPublicKey.toBase58()}\n\n`; + + const totalUsdFormatted = new BigNumber(portfolio.totalUsd).toFixed(2); + const totalSolFormatted = portfolio.totalSol; + + output += `Total Value: $${totalUsdFormatted} (${totalSolFormatted} SOL)\n\n`; + output += "Token Balances:\n"; + + const nonZeroItems = portfolio.items.filter((item) => + new BigNumber(item.uiAmount).isGreaterThan(0) ); - if (response?.data?.value) { - const price = response.data.value.toString(); - prices[token === SOL ? "solana" : token === BTC ? "bitcoin" : "ethereum"].usd = price; + if (nonZeroItems.length === 0) { + output += "No tokens found with non-zero balance\n"; } else { - console.warn(`No price data available for token: ${token}`); + for (const item of nonZeroItems) { + const valueUsd = new BigNumber(item.valueUsd).toFixed(2); + output += `${item.name} (${item.symbol}): ${new BigNumber( + item.uiAmount + ).toFixed(6)} ($${valueUsd} | ${item.valueSol} SOL)\n`; + } } - } - return prices; - } catch (error) { - console.error("Error fetching prices:", error); - throw error; - } - } - - formatPortfolio(runtime, portfolio: WalletPortfolio, prices: Prices): string { - let output = `${runtime.character.description}\n`; - output += `Wallet Address: ${this.walletPublicKey.toBase58()}\n\n`; - - const totalUsdFormatted = new BigNumber(portfolio.totalUsd).toFixed(2); - const totalSolFormatted = portfolio.totalSol; - - output += `Total Value: $${totalUsdFormatted} (${totalSolFormatted} SOL)\n\n`; - output += "Token Balances:\n"; - - const nonZeroItems = portfolio.items.filter(item => - new BigNumber(item.uiAmount).isGreaterThan(0) - ); - - if (nonZeroItems.length === 0) { - output += "No tokens found with non-zero balance\n"; - } else { - for (const item of nonZeroItems) { - const valueUsd = new BigNumber(item.valueUsd).toFixed(2); - output += `${item.name} (${item.symbol}): ${ - new BigNumber(item.uiAmount).toFixed(6) - } ($${valueUsd} | ${item.valueSol} SOL)\n`; - } + output += "\nMarket Prices:\n"; + output += `SOL: $${new BigNumber(prices.solana.usd).toFixed(2)}\n`; + output += `BTC: $${new BigNumber(prices.bitcoin.usd).toFixed(2)}\n`; + output += `ETH: $${new BigNumber(prices.ethereum.usd).toFixed(2)}\n`; + + return output; } - output += "\nMarket Prices:\n"; - output += `SOL: $${new BigNumber(prices.solana.usd).toFixed(2)}\n`; - output += `BTC: $${new BigNumber(prices.bitcoin.usd).toFixed(2)}\n`; - output += `ETH: $${new BigNumber(prices.ethereum.usd).toFixed(2)}\n`; - - return output; - } - - async getFormattedPortfolio(runtime): Promise { - try { - const [portfolio, prices] = await Promise.all([ - this.fetchPortfolioValue(runtime), - this.fetchPrices(runtime) - ]); - - return this.formatPortfolio(runtime, portfolio, prices); - } catch (error) { - console.error("Error generating portfolio report:", error); - return "Unable to fetch wallet information. Please try again later."; + async getFormattedPortfolio(runtime): Promise { + try { + const [portfolio, prices] = await Promise.all([ + this.fetchPortfolioValue(runtime), + this.fetchPrices(runtime), + ]); + + return this.formatPortfolio(runtime, portfolio, prices); + } catch (error) { + console.error("Error generating portfolio report:", error); + return "Unable to fetch wallet information. Please try again later."; + } } - } } const walletProvider: Provider = { - get: async (runtime: IAgentRuntime, _message: Memory, _state?: State): Promise => { - try { - // Validate wallet configuration - if (!runtime.getSetting('WALLET_PUBLIC_KEY')) { - throw new Error("Wallet public key is not configured in settings"); - } - - // Validate public key format before creating instance - if (typeof runtime.getSetting('WALLET_PUBLIC_KEY') !== 'string' || runtime.getSetting('WALLET_PUBLIC_KEY').trim() === '') { - throw new Error("Invalid wallet public key format"); - } - - let publicKey: PublicKey; - try { - publicKey = new PublicKey(runtime.getSetting('WALLET_PUBLIC_KEY')); - } catch (error) { - console.error("Error creating PublicKey:", error); - throw new Error("Invalid wallet public key format"); - } - - const connection = new Connection(PROVIDER_CONFIG.DEFAULT_RPC); - const provider = new WalletProvider(connection, publicKey); - - const porfolio = await provider.getFormattedPortfolio(runtime); - return porfolio; - } catch (error) { - console.error("Error in wallet provider:", error.message); - return `Failed to fetch wallet information: ${error instanceof Error ? error.message : 'Unknown error'}`; - } - } + get: async ( + runtime: IAgentRuntime, + _message: Memory, + _state?: State + ): Promise => { + try { + // Validate wallet configuration + if (!runtime.getSetting("WALLET_PUBLIC_KEY")) { + throw new Error( + "Wallet public key is not configured in settings" + ); + } + + // Validate public key format before creating instance + if ( + typeof runtime.getSetting("WALLET_PUBLIC_KEY") !== "string" || + runtime.getSetting("WALLET_PUBLIC_KEY").trim() === "" + ) { + throw new Error("Invalid wallet public key format"); + } + + let publicKey: PublicKey; + try { + publicKey = new PublicKey( + runtime.getSetting("WALLET_PUBLIC_KEY") + ); + } catch (error) { + console.error("Error creating PublicKey:", error); + throw new Error("Invalid wallet public key format"); + } + + const connection = new Connection(PROVIDER_CONFIG.DEFAULT_RPC); + const provider = new WalletProvider(connection, publicKey); + + const porfolio = await provider.getFormattedPortfolio(runtime); + return porfolio; + } catch (error) { + console.error("Error in wallet provider:", error.message); + return `Failed to fetch wallet information: ${error instanceof Error ? error.message : "Unknown error"}`; + } + }, }; // Module exports -export default walletProvider; \ No newline at end of file +export default walletProvider; diff --git a/core/src/services/audioUtils.ts b/core/src/services/audioUtils.ts index 03646f0bac..0916c48cd3 100644 --- a/core/src/services/audioUtils.ts +++ b/core/src/services/audioUtils.ts @@ -1,22 +1,25 @@ export function getWavHeader( - audioLength: number, - sampleRate: number, - channelCount: number = 1, - bitsPerSample: number = 16, + audioLength: number, + sampleRate: number, + channelCount: number = 1, + bitsPerSample: number = 16 ): Buffer { - const wavHeader = Buffer.alloc(44); - wavHeader.write("RIFF", 0); - wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 - wavHeader.write("WAVE", 8); - wavHeader.write("fmt ", 12); - wavHeader.writeUInt32LE(16, 16); // Length of format data - wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) - wavHeader.writeUInt16LE(channelCount, 22); // Number of channels - wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate - wavHeader.writeUInt32LE((sampleRate * bitsPerSample * channelCount) / 8, 28); // Byte rate - wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) - wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample - wavHeader.write("data", 36); // Data chunk header - wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size - return wavHeader; + const wavHeader = Buffer.alloc(44); + wavHeader.write("RIFF", 0); + wavHeader.writeUInt32LE(36 + audioLength, 4); // Length of entire file in bytes minus 8 + wavHeader.write("WAVE", 8); + wavHeader.write("fmt ", 12); + wavHeader.writeUInt32LE(16, 16); // Length of format data + wavHeader.writeUInt16LE(1, 20); // Type of format (1 is PCM) + wavHeader.writeUInt16LE(channelCount, 22); // Number of channels + wavHeader.writeUInt32LE(sampleRate, 24); // Sample rate + wavHeader.writeUInt32LE( + (sampleRate * bitsPerSample * channelCount) / 8, + 28 + ); // Byte rate + wavHeader.writeUInt16LE((bitsPerSample * channelCount) / 8, 32); // Block align ((BitsPerSample * Channels) / 8) + wavHeader.writeUInt16LE(bitsPerSample, 34); // Bits per sample + wavHeader.write("data", 36); // Data chunk header + wavHeader.writeUInt32LE(audioLength, 40); // Data chunk size + return wavHeader; } diff --git a/core/src/services/browser.ts b/core/src/services/browser.ts index 7f1716c9e9..fd6e1792eb 100644 --- a/core/src/services/browser.ts +++ b/core/src/services/browser.ts @@ -10,276 +10,286 @@ import { generateSummary } from "./summary.ts"; import settings from "../core/settings.ts"; export class BrowserService { - private static instance: BrowserService | null = null; - private browser: Browser | undefined; - private context: BrowserContext | undefined; - private blocker: PlaywrightBlocker | undefined; - private captchaSolver: CaptchaSolver; - private CONTENT_CACHE_DIR = "./content_cache"; - private runtime: IAgentRuntime; - - private queue: string[] = []; - private processing: boolean = false; - - private constructor(runtime: IAgentRuntime) { - this.runtime = runtime; - this.browser = undefined; - this.context = undefined; - this.blocker = undefined; - this.captchaSolver = new CaptchaSolver(settings.CAPSOLVER_API_KEY || ""); - this.ensureCacheDirectoryExists(); - } - - public static getInstance(runtime: IAgentRuntime): BrowserService { - if (!BrowserService.instance) { - BrowserService.instance = new BrowserService(runtime); + private static instance: BrowserService | null = null; + private browser: Browser | undefined; + private context: BrowserContext | undefined; + private blocker: PlaywrightBlocker | undefined; + private captchaSolver: CaptchaSolver; + private CONTENT_CACHE_DIR = "./content_cache"; + private runtime: IAgentRuntime; + + private queue: string[] = []; + private processing: boolean = false; + + private constructor(runtime: IAgentRuntime) { + this.runtime = runtime; + this.browser = undefined; + this.context = undefined; + this.blocker = undefined; + this.captchaSolver = new CaptchaSolver( + settings.CAPSOLVER_API_KEY || "" + ); + this.ensureCacheDirectoryExists(); + } + + public static getInstance(runtime: IAgentRuntime): BrowserService { + if (!BrowserService.instance) { + BrowserService.instance = new BrowserService(runtime); + } + return BrowserService.instance; } - return BrowserService.instance; - } - private ensureCacheDirectoryExists() { - if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { - fs.mkdirSync(this.CONTENT_CACHE_DIR); + private ensureCacheDirectoryExists() { + if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { + fs.mkdirSync(this.CONTENT_CACHE_DIR); + } } - } - async initialize() { - if (!this.browser) { - this.browser = await chromium.launch({ - args: ["--no-sandbox", "--disable-setuid-sandbox"], - }); + async initialize() { + if (!this.browser) { + this.browser = await chromium.launch({ + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); - this.context = await this.browser.newContext({ - userAgent: - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - }); + this.context = await this.browser.newContext({ + userAgent: + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + }); - this.blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); + this.blocker = + await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); + } } - } - async closeBrowser() { - if (this.context) { - await this.context.close(); - this.context = undefined; - } - if (this.browser) { - await this.browser.close(); - this.browser = undefined; - } - } - - async getPageContent( - url: string, - ): Promise<{ title: string; description: string; bodyContent: string }> { - await this.initialize(); - this.queue.push(url); - this.processQueue(); - - return new Promise((resolve, reject) => { - const checkQueue = async () => { - const index = this.queue.indexOf(url); - if (index !== -1) { - setTimeout(checkQueue, 100); - } else { - try { - const result = await this.fetchPageContent(url); - resolve(result); - } catch (error) { - reject(error); - } + async closeBrowser() { + if (this.context) { + await this.context.close(); + this.context = undefined; + } + if (this.browser) { + await this.browser.close(); + this.browser = undefined; } - }; - checkQueue(); - }); - } - - private getCacheKey(url: string): string { - return stringToUuid(url); - } - - private async processQueue(): Promise { - if (this.processing || this.queue.length === 0) { - return; } - this.processing = true; + async getPageContent( + url: string + ): Promise<{ title: string; description: string; bodyContent: string }> { + await this.initialize(); + this.queue.push(url); + this.processQueue(); + + return new Promise((resolve, reject) => { + const checkQueue = async () => { + const index = this.queue.indexOf(url); + if (index !== -1) { + setTimeout(checkQueue, 100); + } else { + try { + const result = await this.fetchPageContent(url); + resolve(result); + } catch (error) { + reject(error); + } + } + }; + checkQueue(); + }); + } - while (this.queue.length > 0) { - const url = this.queue.shift(); - await this.fetchPageContent(url); + private getCacheKey(url: string): string { + return stringToUuid(url); } - this.processing = false; - } + private async processQueue(): Promise { + if (this.processing || this.queue.length === 0) { + return; + } - private async fetchPageContent( - url: string, - ): Promise<{ title: string; description: string; bodyContent: string }> { - const cacheKey = this.getCacheKey(url); - const cacheFilePath = path.join(this.CONTENT_CACHE_DIR, `${cacheKey}.json`); + this.processing = true; - if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { - fs.mkdirSync(this.CONTENT_CACHE_DIR, { recursive: true }); - } + while (this.queue.length > 0) { + const url = this.queue.shift(); + await this.fetchPageContent(url); + } - if (fs.existsSync(cacheFilePath)) { - return JSON.parse(fs.readFileSync(cacheFilePath, "utf-8")).content; + this.processing = false; } - let page: Page | undefined; - - try { - if (!this.context) { - console.log( - "Browser context not initialized. Call initialize() first.", + private async fetchPageContent( + url: string + ): Promise<{ title: string; description: string; bodyContent: string }> { + const cacheKey = this.getCacheKey(url); + const cacheFilePath = path.join( + this.CONTENT_CACHE_DIR, + `${cacheKey}.json` ); - } - - page = await this.context.newPage(); - - // Enable stealth mode - await page.setExtraHTTPHeaders({ - "Accept-Language": "en-US,en;q=0.9", - }); - - // Apply ad blocker - if (this.blocker) { - await this.blocker.enableBlockingInPage(page); - } - - const response = await page.goto(url, { waitUntil: "networkidle" }); - - if (!response) { - console.log("Failed to load the page"); - } - - if (response.status() === 403 || response.status() === 404) { - return await this.tryAlternativeSources(url); - } - - // Check for CAPTCHA - const captchaDetected = await this.detectCaptcha(page); - if (captchaDetected) { - await this.solveCaptcha(page, url); - } - const title = await page.evaluate(() => document.title); - const bodyContent = await page.evaluate(() => document.body.innerText); - const { description } = await generateSummary( - this.runtime, - title + "\n" + bodyContent, - ); - const content = { title, description, bodyContent }; - fs.writeFileSync(cacheFilePath, JSON.stringify({ url, content })); - return content; - } catch (error) { - console.error("Error:", error); - return { - title: url, - description: "Error, could not fetch content", - bodyContent: "", - }; - } finally { - if (page) { - await page.close(); - } + + if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { + fs.mkdirSync(this.CONTENT_CACHE_DIR, { recursive: true }); + } + + if (fs.existsSync(cacheFilePath)) { + return JSON.parse(fs.readFileSync(cacheFilePath, "utf-8")).content; + } + + let page: Page | undefined; + + try { + if (!this.context) { + console.log( + "Browser context not initialized. Call initialize() first." + ); + } + + page = await this.context.newPage(); + + // Enable stealth mode + await page.setExtraHTTPHeaders({ + "Accept-Language": "en-US,en;q=0.9", + }); + + // Apply ad blocker + if (this.blocker) { + await this.blocker.enableBlockingInPage(page); + } + + const response = await page.goto(url, { waitUntil: "networkidle" }); + + if (!response) { + console.log("Failed to load the page"); + } + + if (response.status() === 403 || response.status() === 404) { + return await this.tryAlternativeSources(url); + } + + // Check for CAPTCHA + const captchaDetected = await this.detectCaptcha(page); + if (captchaDetected) { + await this.solveCaptcha(page, url); + } + const title = await page.evaluate(() => document.title); + const bodyContent = await page.evaluate( + () => document.body.innerText + ); + const { description } = await generateSummary( + this.runtime, + title + "\n" + bodyContent + ); + const content = { title, description, bodyContent }; + fs.writeFileSync(cacheFilePath, JSON.stringify({ url, content })); + return content; + } catch (error) { + console.error("Error:", error); + return { + title: url, + description: "Error, could not fetch content", + bodyContent: "", + }; + } finally { + if (page) { + await page.close(); + } + } } - } - - private async detectCaptcha(page: Page): Promise { - const captchaSelectors = [ - 'iframe[src*="captcha"]', - 'div[class*="captcha"]', - "#captcha", - ".g-recaptcha", - ".h-captcha", - ]; - - for (const selector of captchaSelectors) { - const element = await page.$(selector); - if (element) return true; + + private async detectCaptcha(page: Page): Promise { + const captchaSelectors = [ + 'iframe[src*="captcha"]', + 'div[class*="captcha"]', + "#captcha", + ".g-recaptcha", + ".h-captcha", + ]; + + for (const selector of captchaSelectors) { + const element = await page.$(selector); + if (element) return true; + } + + return false; } - return false; - } + private async solveCaptcha(page: Page, url: string): Promise { + try { + const hcaptchaKey = await this.getHCaptchaWebsiteKey(page); + if (hcaptchaKey) { + const solution = await this.captchaSolver.hcaptchaProxyless({ + websiteURL: url, + websiteKey: hcaptchaKey, + }); + await page.evaluate((token) => { + // @ts-ignore + window.hcaptcha.setResponse(token); + }, solution.gRecaptchaResponse); + return; + } + + const recaptchaKey = await this.getReCaptchaWebsiteKey(page); + if (recaptchaKey) { + const solution = await this.captchaSolver.recaptchaV2Proxyless({ + websiteURL: url, + websiteKey: recaptchaKey, + }); + await page.evaluate((token) => { + // @ts-ignore + document.getElementById("g-recaptcha-response").innerHTML = + token; + }, solution.gRecaptchaResponse); + } + } catch (error) { + console.error("Error solving CAPTCHA:", error); + } + } - private async solveCaptcha(page: Page, url: string): Promise { - try { - const hcaptchaKey = await this.getHCaptchaWebsiteKey(page); - if (hcaptchaKey) { - const solution = await this.captchaSolver.hcaptchaProxyless({ - websiteURL: url, - websiteKey: hcaptchaKey, + private async getHCaptchaWebsiteKey(page: Page): Promise { + return page.evaluate(() => { + const hcaptchaIframe = document.querySelector( + 'iframe[src*="hcaptcha.com"]' + ); + if (hcaptchaIframe) { + const src = hcaptchaIframe.getAttribute("src"); + const match = src?.match(/sitekey=([^&]*)/); + return match ? match[1] : ""; + } + return ""; }); - await page.evaluate((token) => { - // @ts-ignore - window.hcaptcha.setResponse(token); - }, solution.gRecaptchaResponse); - return; - } - - const recaptchaKey = await this.getReCaptchaWebsiteKey(page); - if (recaptchaKey) { - const solution = await this.captchaSolver.recaptchaV2Proxyless({ - websiteURL: url, - websiteKey: recaptchaKey, - }); - await page.evaluate((token) => { - // @ts-ignore - document.getElementById("g-recaptcha-response").innerHTML = token; - }, solution.gRecaptchaResponse); - } - } catch (error) { - console.error("Error solving CAPTCHA:", error); } - } - - private async getHCaptchaWebsiteKey(page: Page): Promise { - return page.evaluate(() => { - const hcaptchaIframe = document.querySelector( - 'iframe[src*="hcaptcha.com"]', - ); - if (hcaptchaIframe) { - const src = hcaptchaIframe.getAttribute("src"); - const match = src?.match(/sitekey=([^&]*)/); - return match ? match[1] : ""; - } - return ""; - }); - } - - private async getReCaptchaWebsiteKey(page: Page): Promise { - return page.evaluate(() => { - const recaptchaElement = document.querySelector(".g-recaptcha"); - return recaptchaElement - ? recaptchaElement.getAttribute("data-sitekey") || "" - : ""; - }); - } - - private async tryAlternativeSources( - url: string, - ): Promise<{ title: string; description: string; bodyContent: string }> { - // Try Internet Archive - const archiveUrl = `https://web.archive.org/web/${url}`; - try { - return await this.fetchPageContent(archiveUrl); - } catch (error) { - console.error("Error fetching from Internet Archive:", error); + + private async getReCaptchaWebsiteKey(page: Page): Promise { + return page.evaluate(() => { + const recaptchaElement = document.querySelector(".g-recaptcha"); + return recaptchaElement + ? recaptchaElement.getAttribute("data-sitekey") || "" + : ""; + }); } - // Try Google Search as a last resort - const googleSearchUrl = `https://www.google.com/search?q=${encodeURIComponent(url)}`; - try { - return await this.fetchPageContent(googleSearchUrl); - } catch (error) { - console.error("Error fetching from Google Search:", error); - console.error("Failed to fetch content from alternative sources"); - return { - title: url, - description: "Error, could not fetch content from alternative sources", - bodyContent: "", - }; + private async tryAlternativeSources( + url: string + ): Promise<{ title: string; description: string; bodyContent: string }> { + // Try Internet Archive + const archiveUrl = `https://web.archive.org/web/${url}`; + try { + return await this.fetchPageContent(archiveUrl); + } catch (error) { + console.error("Error fetching from Internet Archive:", error); + } + + // Try Google Search as a last resort + const googleSearchUrl = `https://www.google.com/search?q=${encodeURIComponent(url)}`; + try { + return await this.fetchPageContent(googleSearchUrl); + } catch (error) { + console.error("Error fetching from Google Search:", error); + console.error("Failed to fetch content from alternative sources"); + return { + title: url, + description: + "Error, could not fetch content from alternative sources", + bodyContent: "", + }; + } } - } } diff --git a/core/src/services/image.ts b/core/src/services/image.ts index cffb53403e..7f29418971 100644 --- a/core/src/services/image.ts +++ b/core/src/services/image.ts @@ -1,14 +1,14 @@ // Current image recognition service -- local recognition working, no openai recognition import { - AutoProcessor, - AutoTokenizer, - env, - Florence2ForConditionalGeneration, - Florence2Processor, - PreTrainedModel, - PreTrainedTokenizer, - RawImage, - type Tensor, + AutoProcessor, + AutoTokenizer, + env, + Florence2ForConditionalGeneration, + Florence2Processor, + PreTrainedModel, + PreTrainedTokenizer, + RawImage, + type Tensor, } from "@huggingface/transformers"; import fs from "fs"; import gifFrames from "gif-frames"; @@ -18,313 +18,331 @@ import models from "../core/models.ts"; import { IAgentRuntime, ModelProvider } from "../core/types.ts"; class ImageDescriptionService { - private static instance: ImageDescriptionService | null = null; - private modelId: string = "onnx-community/Florence-2-base-ft"; - private device: string = "gpu"; - private model: PreTrainedModel | null = null; - private processor: Florence2Processor | null = null; - private tokenizer: PreTrainedTokenizer | null = null; - private initialized: boolean = false; - runtime: IAgentRuntime; - - private queue: string[] = []; - private processing: boolean = false; - - private constructor(runtime: IAgentRuntime) { - this.runtime = runtime; - this.initialize(); - } - - public static getInstance(runtime: IAgentRuntime): ImageDescriptionService { - if (!ImageDescriptionService.instance) { - ImageDescriptionService.instance = new ImageDescriptionService(runtime); + private static instance: ImageDescriptionService | null = null; + private modelId: string = "onnx-community/Florence-2-base-ft"; + private device: string = "gpu"; + private model: PreTrainedModel | null = null; + private processor: Florence2Processor | null = null; + private tokenizer: PreTrainedTokenizer | null = null; + private initialized: boolean = false; + runtime: IAgentRuntime; + + private queue: string[] = []; + private processing: boolean = false; + + private constructor(runtime: IAgentRuntime) { + this.runtime = runtime; + this.initialize(); } - return ImageDescriptionService.instance; - } - - async initialize( - modelId: string | null = null, - device: string | null = null, - ): Promise { - if (this.initialized) { - return; + + public static getInstance(runtime: IAgentRuntime): ImageDescriptionService { + if (!ImageDescriptionService.instance) { + ImageDescriptionService.instance = new ImageDescriptionService( + runtime + ); + } + return ImageDescriptionService.instance; } - const model = models[this.runtime.character.settings.model]; + async initialize( + modelId: string | null = null, + device: string | null = null + ): Promise { + if (this.initialized) { + return; + } - if (model === ModelProvider.LLAMALOCAL) { - this.modelId = modelId || "onnx-community/Florence-2-base-ft"; + const model = models[this.runtime.character.settings.model]; + + if (model === ModelProvider.LLAMALOCAL) { + this.modelId = modelId || "onnx-community/Florence-2-base-ft"; + + env.allowLocalModels = false; + env.allowRemoteModels = true; + env.backends.onnx.logLevel = "fatal"; + env.backends.onnx.wasm.proxy = false; + env.backends.onnx.wasm.numThreads = 1; + + console.log("Downloading model..."); + + this.model = + await Florence2ForConditionalGeneration.from_pretrained( + this.modelId, + { + device: "gpu", + progress_callback: (progress) => { + if (progress.status === "downloading") { + console.log( + `Model download progress: ${JSON.stringify(progress)}` + ); + } + }, + } + ); + + console.log("Model downloaded successfully."); + + this.processor = (await AutoProcessor.from_pretrained( + this.modelId + )) as Florence2Processor; + this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId); + } else { + this.modelId = "gpt-4o-mini"; + this.device = "cloud"; + } - env.allowLocalModels = false; - env.allowRemoteModels = true; - env.backends.onnx.logLevel = "fatal"; - env.backends.onnx.wasm.proxy = false; - env.backends.onnx.wasm.numThreads = 1; + this.initialized = true; + } - console.log("Downloading model..."); + async describeImage( + imageUrl: string + ): Promise<{ title: string; description: string }> { + if (this.device === "cloud") { + return this.recognizeWithOpenAI(imageUrl); + } else { + this.queue.push(imageUrl); + this.processQueue(); + + return new Promise((resolve, reject) => { + const checkQueue = () => { + const index = this.queue.indexOf(imageUrl); + if (index !== -1) { + setTimeout(checkQueue, 100); + } else { + resolve(this.processImage(imageUrl)); + } + }; + checkQueue(); + }); + } + } - this.model = await Florence2ForConditionalGeneration.from_pretrained( - this.modelId, - { - device: "gpu", - progress_callback: (progress) => { - if (progress.status === "downloading") { - console.log( - `Model download progress: ${JSON.stringify(progress)}`, - ); + private async recognizeWithOpenAI( + imageUrl: string + ): Promise<{ title: string; description: string }> { + const isGif = imageUrl.toLowerCase().endsWith(".gif"); + let imageData: Buffer | null = null; + + try { + if (isGif) { + console.log("Processing GIF: extracting first frame"); + const { filePath } = + await this.extractFirstFrameFromGif(imageUrl); + imageData = fs.readFileSync(filePath); + } else { + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error( + `Failed to fetch image: ${response.statusText}` + ); + } + imageData = Buffer.from(await response.arrayBuffer()); } - }, - }, - ); - - console.log("Model downloaded successfully."); - - this.processor = (await AutoProcessor.from_pretrained( - this.modelId, - )) as Florence2Processor; - this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId); - } - else { - this.modelId = "gpt-4o-mini"; - this.device = "cloud"; - } + if (!imageData || imageData.length === 0) { + throw new Error("Failed to fetch image data"); + } - this.initialized = true; - } - - async describeImage( - imageUrl: string, - ): Promise<{ title: string; description: string }> { - if (this.device === "cloud") { - return this.recognizeWithOpenAI(imageUrl); - } else { - this.queue.push(imageUrl); - this.processQueue(); - - return new Promise((resolve, reject) => { - const checkQueue = () => { - const index = this.queue.indexOf(imageUrl); - if (index !== -1) { - setTimeout(checkQueue, 100); - } else { - resolve(this.processImage(imageUrl)); - } - }; - checkQueue(); - }); - } - } - - private async recognizeWithOpenAI( - imageUrl: string, - ): Promise<{ title: string; description: string }> { - const isGif = imageUrl.toLowerCase().endsWith(".gif"); - let imageData: Buffer | null = null; - - try { - if (isGif) { - console.log("Processing GIF: extracting first frame"); - const { filePath } = await this.extractFirstFrameFromGif(imageUrl); - imageData = fs.readFileSync(filePath); - } else { - const response = await fetch(imageUrl); - if (!response.ok) { - throw new Error(`Failed to fetch image: ${response.statusText}`); + const prompt = + "Describe this image and give it a title. The first line should be the title, and then a line break, then a detailed description of the image. Respond with the format 'title\ndescription'"; + + const text = await this.requestOpenAI( + imageUrl, + imageData, + prompt, + isGif + ); + const title = text.split("\n")[0]; + const description = text.split("\n").slice(1).join("\n"); + return { title, description }; + } catch (error) { + console.error("Error in recognizeWithOpenAI:", error); + throw error; } - imageData = Buffer.from(await response.arrayBuffer()); - } - - if (!imageData || imageData.length === 0) { - throw new Error("Failed to fetch image data"); - } - - const prompt = - "Describe this image and give it a title. The first line should be the title, and then a line break, then a detailed description of the image. Respond with the format 'title\ndescription'"; - - const text = await this.requestOpenAI(imageUrl, imageData, prompt, isGif); - const title = text.split("\n")[0]; - const description = text.split("\n").slice(1).join("\n"); - return { title, description }; - } catch (error) { - console.error("Error in recognizeWithOpenAI:", error); - throw error; } - } - - private async requestOpenAI( - imageUrl: string, - imageData: Buffer, - prompt: string, - isGif: boolean, - ): Promise { - for (let retryAttempts = 0; retryAttempts < 3; retryAttempts++) { - try { - let body; - if (isGif) { - const base64Image = imageData.toString("base64"); - body = JSON.stringify({ - model: "gpt-4o-mini", - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { - type: "image_url", - image_url: { url: `data:image/png;base64,${base64Image}` }, - }, - ], - }, - ], - max_tokens: 500, - }); - } else { - body = JSON.stringify({ - model: "gpt-4o-mini", - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { type: "image_url", image_url: { url: imageUrl } }, - ], - }, - ], - max_tokens: 300, - }); - } - - const response = await fetch( - "https://api.openai.com/v1/chat/completions", - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.runtime.getSetting("OPENAI_API_KEY")}`, - }, - body: body, - }, - ); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); + private async requestOpenAI( + imageUrl: string, + imageData: Buffer, + prompt: string, + isGif: boolean + ): Promise { + for (let retryAttempts = 0; retryAttempts < 3; retryAttempts++) { + try { + let body; + if (isGif) { + const base64Image = imageData.toString("base64"); + body = JSON.stringify({ + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "image_url", + image_url: { + url: `data:image/png;base64,${base64Image}`, + }, + }, + ], + }, + ], + max_tokens: 500, + }); + } else { + body = JSON.stringify({ + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "image_url", + image_url: { url: imageUrl }, + }, + ], + }, + ], + max_tokens: 300, + }); + } + + const response = await fetch( + "https://api.openai.com/v1/chat/completions", + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.runtime.getSetting("OPENAI_API_KEY")}`, + }, + body: body, + } + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = await response.json(); + return data.choices[0].message.content; + } catch (error) { + console.log( + `Error during OpenAI request (attempt ${retryAttempts + 1}):`, + error + ); + if (retryAttempts === 2) { + throw error; + } + } } - - const data = await response.json(); - return data.choices[0].message.content; - } catch (error) { - console.log( - `Error during OpenAI request (attempt ${retryAttempts + 1}):`, - error, + throw new Error( + "Failed to recognize image with OpenAI after 3 attempts" ); - if (retryAttempts === 2) { - throw error; - } - } } - throw new Error("Failed to recognize image with OpenAI after 3 attempts"); - } - private async processQueue(): Promise { - if (this.processing || this.queue.length === 0) { - return; - } + private async processQueue(): Promise { + if (this.processing || this.queue.length === 0) { + return; + } - this.processing = true; + this.processing = true; - while (this.queue.length > 0) { - const imageUrl = this.queue.shift(); - await this.processImage(imageUrl); - } + while (this.queue.length > 0) { + const imageUrl = this.queue.shift(); + await this.processImage(imageUrl); + } - this.processing = false; - } + this.processing = false; + } - private async processImage( - imageUrl: string, - ): Promise<{ title: string; description: string }> { - console.log("***** PROCESSING IMAGE", imageUrl); - const isGif = imageUrl.toLowerCase().endsWith(".gif"); - let imageToProcess = imageUrl; + private async processImage( + imageUrl: string + ): Promise<{ title: string; description: string }> { + console.log("***** PROCESSING IMAGE", imageUrl); + const isGif = imageUrl.toLowerCase().endsWith(".gif"); + let imageToProcess = imageUrl; + + try { + if (isGif) { + console.log("Processing GIF: extracting first frame"); + const { filePath } = + await this.extractFirstFrameFromGif(imageUrl); + imageToProcess = filePath; + } - try { - if (isGif) { - console.log("Processing GIF: extracting first frame"); - const { filePath } = await this.extractFirstFrameFromGif(imageUrl); - imageToProcess = filePath; - } + const image = await RawImage.fromURL(imageToProcess); + const visionInputs = await this.processor(image); - const image = await RawImage.fromURL(imageToProcess); - const visionInputs = await this.processor(image); + const prompts = + this.processor.construct_prompts(""); + const textInputs = this.tokenizer(prompts); - const prompts = this.processor.construct_prompts(""); - const textInputs = this.tokenizer(prompts); + console.log("***** GENERATING"); - console.log("***** GENERATING"); + const generatedIds = (await this.model.generate({ + ...textInputs, + ...visionInputs, + max_new_tokens: 256, + })) as Tensor; - const generatedIds = (await this.model.generate({ - ...textInputs, - ...visionInputs, - max_new_tokens: 256, - })) as Tensor; + console.log("***** GENERATED IDS", generatedIds); - console.log("***** GENERATED IDS", generatedIds); + const generatedText = this.tokenizer.batch_decode(generatedIds, { + skip_special_tokens: false, + })[0]; - const generatedText = this.tokenizer.batch_decode(generatedIds, { - skip_special_tokens: false, - })[0]; + console.log("***** GENERATED TEXT"); + console.log(generatedText); - console.log("***** GENERATED TEXT"); - console.log(generatedText); + const result = this.processor.post_process_generation( + generatedText, + "", + image.size + ); - const result = this.processor.post_process_generation( - generatedText, - "", - image.size, - ); + console.log("***** RESULT"); + console.log(result); - console.log("***** RESULT"); - console.log(result); + const detailedCaption = result[""] as string; - const detailedCaption = result[""] as string; + // TODO: handle this better - // TODO: handle this better + return { title: detailedCaption, description: detailedCaption }; + } catch (error) { + console.error("Error in processImage:", error); + } finally { + if (isGif && imageToProcess !== imageUrl) { + fs.unlinkSync(imageToProcess); + } + } + } - return { title: detailedCaption, description: detailedCaption }; - } catch (error) { - console.error("Error in processImage:", error); - } finally { - if (isGif && imageToProcess !== imageUrl) { - fs.unlinkSync(imageToProcess); - } + private async extractFirstFrameFromGif( + gifUrl: string + ): Promise<{ filePath: string }> { + const frameData = await gifFrames({ + url: gifUrl, + frames: 1, + outputType: "png", + }); + const firstFrame = frameData[0]; + + const tempDir = os.tmpdir(); + const tempFilePath = path.join(tempDir, `gif_frame_${Date.now()}.png`); + + return new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(tempFilePath); + firstFrame.getImage().pipe(writeStream); + + writeStream.on("finish", () => { + resolve({ filePath: tempFilePath }); + }); + + writeStream.on("error", reject); + }); } - } - - private async extractFirstFrameFromGif( - gifUrl: string, - ): Promise<{ filePath: string }> { - const frameData = await gifFrames({ - url: gifUrl, - frames: 1, - outputType: "png", - }); - const firstFrame = frameData[0]; - - const tempDir = os.tmpdir(); - const tempFilePath = path.join(tempDir, `gif_frame_${Date.now()}.png`); - - return new Promise((resolve, reject) => { - const writeStream = fs.createWriteStream(tempFilePath); - firstFrame.getImage().pipe(writeStream); - - writeStream.on("finish", () => { - resolve({ filePath: tempFilePath }); - }); - - writeStream.on("error", reject); - }); - } } export default ImageDescriptionService; diff --git a/core/src/services/llama.ts b/core/src/services/llama.ts index 0a53dc242d..96ac07eb60 100644 --- a/core/src/services/llama.ts +++ b/core/src/services/llama.ts @@ -1,15 +1,15 @@ import { fileURLToPath } from "url"; import path from "path"; import { - GbnfJsonSchema, - getLlama, - Llama, - LlamaContext, - LlamaContextSequence, - LlamaContextSequenceRepeatPenalty, - LlamaJsonSchemaGrammar, - LlamaModel, - Token, + GbnfJsonSchema, + getLlama, + Llama, + LlamaContext, + LlamaContextSequence, + LlamaContextSequenceRepeatPenalty, + LlamaJsonSchemaGrammar, + LlamaModel, + Token, } from "node-llama-cpp"; import fs from "fs"; import https from "https"; @@ -19,386 +19,395 @@ import { wordsToPunish } from "./wordsToPunish.ts"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const jsonSchemaGrammar: Readonly<{ - type: string; - properties: { - user: { - type: string; - }; - content: { - type: string; + type: string; + properties: { + user: { + type: string; + }; + content: { + type: string; + }; }; - }; }> = { - type: "object", - properties: { - user: { - type: "string", + type: "object", + properties: { + user: { + type: "string", + }, + content: { + type: "string", + }, }, - content: { - type: "string", - }, - }, }; interface QueuedMessage { - context: string; - temperature: number; - stop: string[]; - max_tokens: number; - frequency_penalty: number; - presence_penalty: number; - useGrammar: boolean; - resolve: (value: any | string | PromiseLike) => void; - reject: (reason?: any) => void; + context: string; + temperature: number; + stop: string[]; + max_tokens: number; + frequency_penalty: number; + presence_penalty: number; + useGrammar: boolean; + resolve: (value: any | string | PromiseLike) => void; + reject: (reason?: any) => void; } class LlamaService { - private static instance: LlamaService | null = null; - private llama: Llama | undefined; - private model: LlamaModel | undefined; - private modelPath: string; - private grammar: LlamaJsonSchemaGrammar | undefined; - private ctx: LlamaContext | undefined; - private sequence: LlamaContextSequence | undefined; - private modelUrl: string; - - private messageQueue: QueuedMessage[] = []; - private isProcessing: boolean = false; - private modelInitialized: boolean = false; - - private constructor() { - console.log("Constructing"); - this.llama = undefined; - this.model = undefined; - this.modelUrl = - "https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true"; - const modelName = "model.gguf"; - console.log("modelName", modelName); - this.modelPath = path.join(__dirname, modelName); - try { - this.initializeModel(); - } catch (error) { - console.error("Error initializing model", error); + private static instance: LlamaService | null = null; + private llama: Llama | undefined; + private model: LlamaModel | undefined; + private modelPath: string; + private grammar: LlamaJsonSchemaGrammar | undefined; + private ctx: LlamaContext | undefined; + private sequence: LlamaContextSequence | undefined; + private modelUrl: string; + + private messageQueue: QueuedMessage[] = []; + private isProcessing: boolean = false; + private modelInitialized: boolean = false; + + private constructor() { + console.log("Constructing"); + this.llama = undefined; + this.model = undefined; + this.modelUrl = + "https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true"; + const modelName = "model.gguf"; + console.log("modelName", modelName); + this.modelPath = path.join(__dirname, modelName); + try { + this.initializeModel(); + } catch (error) { + console.error("Error initializing model", error); + } } - } - public static getInstance(): LlamaService { - if (!LlamaService.instance) { - LlamaService.instance = new LlamaService(); - } - return LlamaService.instance; - } - - async initializeModel() { - try { - await this.checkModel(); - console.log("Loading llama"); - - const systemInfo = await si.graphics(); - const hasCUDA = systemInfo.controllers.some((controller) => - controller.vendor.toLowerCase().includes("nvidia") - ); - - if (hasCUDA) { - console.log("**** CUDA detected"); - } else { - console.log("**** No CUDA detected - local response will be slow"); - } - - this.llama = await getLlama({ - gpu: "cuda", - }); - console.log("Creating grammar"); - const grammar = new LlamaJsonSchemaGrammar( - this.llama, - jsonSchemaGrammar as GbnfJsonSchema, - ); - this.grammar = grammar; - console.log("Loading model"); - console.log("this.modelPath", this.modelPath); - - this.model = await this.llama.loadModel({ modelPath: this.modelPath }); - console.log("Model GPU support", this.llama.getGpuDeviceNames()); - console.log("Creating context"); - this.ctx = await this.model.createContext({ contextSize: 8192 }); - this.sequence = this.ctx.getSequence(); - - this.modelInitialized = true; - this.processQueue(); - } catch (error) { - console.error( - "Model initialization failed. Deleting model and retrying...", - error, - ); - await this.deleteModel(); - await this.initializeModel(); + public static getInstance(): LlamaService { + if (!LlamaService.instance) { + LlamaService.instance = new LlamaService(); + } + return LlamaService.instance; } - } - - async checkModel() { - console.log("Checking model"); - if (!fs.existsSync(this.modelPath)) { - console.log("this.modelPath", this.modelPath); - console.log("Model not found. Downloading..."); - - await new Promise((resolve, reject) => { - const file = fs.createWriteStream(this.modelPath); - let downloadedSize = 0; - - const downloadModel = (url: string) => { - https - .get(url, (response) => { - const isRedirect = response.statusCode >= 300 && - response.statusCode < 400; - if (isRedirect) { - const redirectUrl = response.headers.location; - if (redirectUrl) { - console.log("Following redirect to:", redirectUrl); - downloadModel(redirectUrl); - return; - } else { - console.error("Redirect URL not found"); - reject(new Error("Redirect URL not found")); - return; - } - } - const totalSize = parseInt( - response.headers["content-length"] ?? "0", - 10, - ); + async initializeModel() { + try { + await this.checkModel(); + console.log("Loading llama"); + + const systemInfo = await si.graphics(); + const hasCUDA = systemInfo.controllers.some((controller) => + controller.vendor.toLowerCase().includes("nvidia") + ); + + if (hasCUDA) { + console.log("**** CUDA detected"); + } else { + console.log( + "**** No CUDA detected - local response will be slow" + ); + } + + this.llama = await getLlama({ + gpu: "cuda", + }); + console.log("Creating grammar"); + const grammar = new LlamaJsonSchemaGrammar( + this.llama, + jsonSchemaGrammar as GbnfJsonSchema + ); + this.grammar = grammar; + console.log("Loading model"); + console.log("this.modelPath", this.modelPath); + + this.model = await this.llama.loadModel({ + modelPath: this.modelPath, + }); + console.log("Model GPU support", this.llama.getGpuDeviceNames()); + console.log("Creating context"); + this.ctx = await this.model.createContext({ contextSize: 8192 }); + this.sequence = this.ctx.getSequence(); - response.on("data", (chunk) => { - downloadedSize += chunk.length; - file.write(chunk); + this.modelInitialized = true; + this.processQueue(); + } catch (error) { + console.error( + "Model initialization failed. Deleting model and retrying...", + error + ); + await this.deleteModel(); + await this.initializeModel(); + } + } - // Log progress - const progress = ((downloadedSize / totalSize) * 100).toFixed( - 2, - ); - process.stdout.write(`Downloaded ${progress}%\r`); - }); - - response.on("end", () => { - file.end(); - console.log("\nModel downloaded successfully."); - resolve(); - }); - }) - .on("error", (err) => { - fs.unlink(this.modelPath, () => {}); // Delete the file async - console.error("Download failed:", err.message); - reject(err); + async checkModel() { + console.log("Checking model"); + if (!fs.existsSync(this.modelPath)) { + console.log("this.modelPath", this.modelPath); + console.log("Model not found. Downloading..."); + + await new Promise((resolve, reject) => { + const file = fs.createWriteStream(this.modelPath); + let downloadedSize = 0; + + const downloadModel = (url: string) => { + https + .get(url, (response) => { + const isRedirect = + response.statusCode >= 300 && + response.statusCode < 400; + if (isRedirect) { + const redirectUrl = response.headers.location; + if (redirectUrl) { + console.log( + "Following redirect to:", + redirectUrl + ); + downloadModel(redirectUrl); + return; + } else { + console.error("Redirect URL not found"); + reject(new Error("Redirect URL not found")); + return; + } + } + + const totalSize = parseInt( + response.headers["content-length"] ?? "0", + 10 + ); + + response.on("data", (chunk) => { + downloadedSize += chunk.length; + file.write(chunk); + + // Log progress + const progress = ( + (downloadedSize / totalSize) * + 100 + ).toFixed(2); + process.stdout.write( + `Downloaded ${progress}%\r` + ); + }); + + response.on("end", () => { + file.end(); + console.log("\nModel downloaded successfully."); + resolve(); + }); + }) + .on("error", (err) => { + fs.unlink(this.modelPath, () => {}); // Delete the file async + console.error("Download failed:", err.message); + reject(err); + }); + }; + + downloadModel(this.modelUrl); + + file.on("error", (err) => { + fs.unlink(this.modelPath, () => {}); // Delete the file async + console.error("File write error:", err.message); + reject(err); + }); }); - }; + } else { + console.log("Model already exists."); + } + } - downloadModel(this.modelUrl); + async deleteModel() { + if (fs.existsSync(this.modelPath)) { + fs.unlinkSync(this.modelPath); + console.log("Model deleted."); + } + } - file.on("error", (err) => { - fs.unlink(this.modelPath, () => {}); // Delete the file async - console.error("File write error:", err.message); - reject(err); + async queueMessageCompletion( + context: string, + temperature: number, + stop: string[], + frequency_penalty: number, + presence_penalty: number, + max_tokens: number + ): Promise { + console.log("Queueing message generateText"); + return new Promise((resolve, reject) => { + this.messageQueue.push({ + context, + temperature, + stop, + frequency_penalty, + presence_penalty, + max_tokens, + useGrammar: true, + resolve, + reject, + }); + this.processQueue(); }); - }); - } else { - console.log("Model already exists."); } - } - async deleteModel() { - if (fs.existsSync(this.modelPath)) { - fs.unlinkSync(this.modelPath); - console.log("Model deleted."); - } - } - - async queueMessageCompletion( - context: string, - temperature: number, - stop: string[], - frequency_penalty: number, - presence_penalty: number, - max_tokens: number, - ): Promise { - console.log("Queueing message generateText"); - return new Promise((resolve, reject) => { - this.messageQueue.push({ - context, - temperature, - stop, - frequency_penalty, - presence_penalty, - max_tokens, - useGrammar: true, - resolve, - reject, - }); - this.processQueue(); - }); - } - - async queueTextCompletion( - context: string, - temperature: number, - stop: string[], - frequency_penalty: number, - presence_penalty: number, - max_tokens: number, - ): Promise { - console.log("Queueing text generateText"); - return new Promise((resolve, reject) => { - this.messageQueue.push({ - context, - temperature, - stop, - frequency_penalty, - presence_penalty, - max_tokens, - useGrammar: false, - resolve, - reject, - }); - this.processQueue(); - }); - } - - private async processQueue() { - if ( - this.isProcessing || - this.messageQueue.length === 0 || - !this.modelInitialized - ) { - return; + async queueTextCompletion( + context: string, + temperature: number, + stop: string[], + frequency_penalty: number, + presence_penalty: number, + max_tokens: number + ): Promise { + console.log("Queueing text generateText"); + return new Promise((resolve, reject) => { + this.messageQueue.push({ + context, + temperature, + stop, + frequency_penalty, + presence_penalty, + max_tokens, + useGrammar: false, + resolve, + reject, + }); + this.processQueue(); + }); } - this.isProcessing = true; + private async processQueue() { + if ( + this.isProcessing || + this.messageQueue.length === 0 || + !this.modelInitialized + ) { + return; + } - while (this.messageQueue.length > 0) { - const message = this.messageQueue.shift(); - if (message) { - try { - console.log("Processing message"); - const response = await this.getCompletionResponse( - message.context, - message.temperature, - message.stop, - message.frequency_penalty, - message.presence_penalty, - message.max_tokens, - message.useGrammar, - ); - message.resolve(response); - } catch (error) { - message.reject(error); + this.isProcessing = true; + + while (this.messageQueue.length > 0) { + const message = this.messageQueue.shift(); + if (message) { + try { + console.log("Processing message"); + const response = await this.getCompletionResponse( + message.context, + message.temperature, + message.stop, + message.frequency_penalty, + message.presence_penalty, + message.max_tokens, + message.useGrammar + ); + message.resolve(response); + } catch (error) { + message.reject(error); + } + } } - } - } - this.isProcessing = false; - } - - private async getCompletionResponse( - context: string, - temperature: number, - stop: string[], - frequency_penalty: number, - presence_penalty: number, - max_tokens: number, - useGrammar: boolean, - ): Promise { - if (!this.sequence) { - throw new Error("Model not initialized."); + this.isProcessing = false; } - const tokens = this.model!.tokenize(context); + private async getCompletionResponse( + context: string, + temperature: number, + stop: string[], + frequency_penalty: number, + presence_penalty: number, + max_tokens: number, + useGrammar: boolean + ): Promise { + if (!this.sequence) { + throw new Error("Model not initialized."); + } - // tokenize the words to punish - const wordsToPunishTokens = wordsToPunish - .map((word) => this.model!.tokenize(word)) - .flat(); + const tokens = this.model!.tokenize(context); - const repeatPenalty: LlamaContextSequenceRepeatPenalty = { - punishTokens: () => wordsToPunishTokens, - penalty: 1.2, - frequencyPenalty: frequency_penalty, - presencePenalty: presence_penalty, - }; + // tokenize the words to punish + const wordsToPunishTokens = wordsToPunish + .map((word) => this.model!.tokenize(word)) + .flat(); - const responseTokens: Token[] = []; - console.log("Evaluating tokens"); - for await ( - const token of this.sequence.evaluate(tokens, { - temperature: Number(temperature), - repeatPenalty: repeatPenalty, - grammarEvaluationState: useGrammar ? this.grammar : undefined, - yieldEogToken: false, - }) - ) { - const current = this.model.detokenize([...responseTokens, token]); - if ([...stop].some((s) => current.includes(s))) { - console.log("Stop sequence found"); - break; - } - - responseTokens.push(token); - process.stdout.write(this.model!.detokenize([token])); - if (useGrammar) { - if (current.replaceAll("\n", "").includes("}```")) { - console.log("JSON block found"); - break; - } - } - if (responseTokens.length > max_tokens) { - console.log("Max tokens reached"); - break; - } - } + const repeatPenalty: LlamaContextSequenceRepeatPenalty = { + punishTokens: () => wordsToPunishTokens, + penalty: 1.2, + frequencyPenalty: frequency_penalty, + presencePenalty: presence_penalty, + }; - const response = this.model!.detokenize(responseTokens); + const responseTokens: Token[] = []; + console.log("Evaluating tokens"); + for await (const token of this.sequence.evaluate(tokens, { + temperature: Number(temperature), + repeatPenalty: repeatPenalty, + grammarEvaluationState: useGrammar ? this.grammar : undefined, + yieldEogToken: false, + })) { + const current = this.model.detokenize([...responseTokens, token]); + if ([...stop].some((s) => current.includes(s))) { + console.log("Stop sequence found"); + break; + } + + responseTokens.push(token); + process.stdout.write(this.model!.detokenize([token])); + if (useGrammar) { + if (current.replaceAll("\n", "").includes("}```")) { + console.log("JSON block found"); + break; + } + } + if (responseTokens.length > max_tokens) { + console.log("Max tokens reached"); + break; + } + } - if (!response) { - throw new Error("Response is undefined"); - } + const response = this.model!.detokenize(responseTokens); - if (useGrammar) { - // extract everything between ```json and ``` - let jsonString = response.match(/```json(.*?)```/s)?.[1].trim(); - if (!jsonString) { - // try parsing response as JSON - try { - jsonString = JSON.stringify(JSON.parse(response)); - console.log("parsedResponse", jsonString); - } catch { - throw new Error("JSON string not found"); + if (!response) { + throw new Error("Response is undefined"); } - } - try { - const parsedResponse = JSON.parse(jsonString); - if (!parsedResponse) { - throw new Error("Parsed response is undefined"); + + if (useGrammar) { + // extract everything between ```json and ``` + let jsonString = response.match(/```json(.*?)```/s)?.[1].trim(); + if (!jsonString) { + // try parsing response as JSON + try { + jsonString = JSON.stringify(JSON.parse(response)); + console.log("parsedResponse", jsonString); + } catch { + throw new Error("JSON string not found"); + } + } + try { + const parsedResponse = JSON.parse(jsonString); + if (!parsedResponse) { + throw new Error("Parsed response is undefined"); + } + console.log("AI: " + parsedResponse.content); + await this.sequence.clearHistory(); + return parsedResponse; + } catch (error) { + console.error("Error parsing JSON:", error); + } + } else { + console.log("AI: " + response); + await this.sequence.clearHistory(); + return response; } - console.log("AI: " + parsedResponse.content); - await this.sequence.clearHistory(); - return parsedResponse; - } catch (error) { - console.error("Error parsing JSON:", error); - } - } else { - console.log("AI: " + response); - await this.sequence.clearHistory(); - return response; } - } - async getEmbeddingResponse(input: string): Promise { - if (!this.model) { - throw new Error("Model not initialized. Call initialize() first."); - } + async getEmbeddingResponse(input: string): Promise { + if (!this.model) { + throw new Error("Model not initialized. Call initialize() first."); + } - const embeddingContext = await this.model.createEmbeddingContext(); - const embedding = await embeddingContext.getEmbeddingFor(input); - return embedding?.vector ? [...embedding.vector] : undefined; - } + const embeddingContext = await this.model.createEmbeddingContext(); + const embedding = await embeddingContext.getEmbeddingFor(input); + return embedding?.vector ? [...embedding.vector] : undefined; + } } export default LlamaService; diff --git a/core/src/services/speech.ts b/core/src/services/speech.ts index 832f2b5731..c4cb5111f1 100644 --- a/core/src/services/speech.ts +++ b/core/src/services/speech.ts @@ -3,159 +3,161 @@ import { IAgentRuntime, ISpeechService } from "../core/types.ts"; import { getWavHeader } from "./audioUtils.ts"; import { synthesize } from "../vendor/vits.ts"; function prependWavHeader( - readable: Readable, - audioLength: number, - sampleRate: number, - channelCount: number = 1, - bitsPerSample: number = 16, + readable: Readable, + audioLength: number, + sampleRate: number, + channelCount: number = 1, + bitsPerSample: number = 16 ): Readable { - const wavHeader = getWavHeader( - audioLength, - sampleRate, - channelCount, - bitsPerSample, - ); - let pushedHeader = false; - const passThrough = new PassThrough(); - readable.on("data", function (data) { - if (!pushedHeader) { - passThrough.push(wavHeader); - pushedHeader = true; - } - passThrough.push(data); - }); - readable.on("end", function () { - passThrough.end(); - }); - return passThrough; + const wavHeader = getWavHeader( + audioLength, + sampleRate, + channelCount, + bitsPerSample + ); + let pushedHeader = false; + const passThrough = new PassThrough(); + readable.on("data", function (data) { + if (!pushedHeader) { + passThrough.push(wavHeader); + pushedHeader = true; + } + passThrough.push(data); + }); + readable.on("end", function () { + passThrough.end(); + }); + return passThrough; } async function textToSpeech(runtime: IAgentRuntime, text: string) { - console.log("11 TTS: " + text); - const body = { - model_id: runtime.getSetting("ELEVENLABS_MODEL_ID"), - text: text, - voice_settings: { - similarity_boost: runtime.getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST"), - stability: runtime.getSetting("ELEVENLABS_VOICE_STABILITY"), - style: runtime.getSetting("ELEVENLABS_VOICE_STYLE"), - use_speaker_boost: runtime.getSetting( - "ELEVENLABS_VOICE_USE_SPEAKER_BOOST", - ), - }, - }; - const options = { - method: "POST", - headers: { - "Content-Type": "application/json", - "xi-api-key": runtime.getSetting("ELEVENLABS_XI_API_KEY"), - }, - body: JSON.stringify(body), - }; + console.log("11 TTS: " + text); + const body = { + model_id: runtime.getSetting("ELEVENLABS_MODEL_ID"), + text: text, + voice_settings: { + similarity_boost: runtime.getSetting( + "ELEVENLABS_VOICE_SIMILARITY_BOOST" + ), + stability: runtime.getSetting("ELEVENLABS_VOICE_STABILITY"), + style: runtime.getSetting("ELEVENLABS_VOICE_STYLE"), + use_speaker_boost: runtime.getSetting( + "ELEVENLABS_VOICE_USE_SPEAKER_BOOST" + ), + }, + }; + const options = { + method: "POST", + headers: { + "Content-Type": "application/json", + "xi-api-key": runtime.getSetting("ELEVENLABS_XI_API_KEY"), + }, + body: JSON.stringify(body), + }; - const response = await fetch( - `https://api.elevenlabs.io/v1/text-to-speech/${runtime.getSetting("ELEVENLABS_VOICE_ID")}/stream?optimize_streaming_latency=${runtime.getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY")}&output_format=${runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT")}`, - options, - ); - - const status = response.status; - if (status != 200) { - console.log(`Received status ${status} from Eleven Labs API`); - const errorBodyString = await response.text(); - throw new Error( - `Received status ${status} from Eleven Labs API: ${errorBodyString}`, + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${runtime.getSetting("ELEVENLABS_VOICE_ID")}/stream?optimize_streaming_latency=${runtime.getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY")}&output_format=${runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT")}`, + options ); - } - if (response) { - const reader = response.body?.getReader(); - const readable = new Readable({ - read() { - reader && - reader.read().then(({ done, value }) => { - if (done) { - this.push(null); - } else { - this.push(value); - } - }); - }, - }); + const status = response.status; + if (status != 200) { + console.log(`Received status ${status} from Eleven Labs API`); + const errorBodyString = await response.text(); + throw new Error( + `Received status ${status} from Eleven Labs API: ${errorBodyString}` + ); + } - if (runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT").startsWith("pcm_")) { - const sampleRate = parseInt( - runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT").substring(4), - ); - const withHeader = prependWavHeader( - readable, - 1024 * 1024 * 100, - sampleRate, - 1, - 16, - ); - return withHeader; + if (response) { + const reader = response.body?.getReader(); + const readable = new Readable({ + read() { + reader && + reader.read().then(({ done, value }) => { + if (done) { + this.push(null); + } else { + this.push(value); + } + }); + }, + }); + + if (runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT").startsWith("pcm_")) { + const sampleRate = parseInt( + runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT").substring(4) + ); + const withHeader = prependWavHeader( + readable, + 1024 * 1024 * 100, + sampleRate, + 1, + 16 + ); + return withHeader; + } else { + return readable; + } } else { - return readable; + return new Readable({ + read() {}, + }); } - } else { - return new Readable({ - read() {}, - }); - } } export class SpeechService implements ISpeechService { - static async generate( - runtime: IAgentRuntime, - text: string, - ): Promise { - // check for elevenlabs API key - if (runtime.getSetting("ELEVENLABS_XI_API_KEY")) { - return textToSpeech(runtime, text); - } - const { audio } = await synthesize(text, { - engine: "vits", - voice: "en_US-hfc_female-medium", - }); + static async generate( + runtime: IAgentRuntime, + text: string + ): Promise { + // check for elevenlabs API key + if (runtime.getSetting("ELEVENLABS_XI_API_KEY")) { + return textToSpeech(runtime, text); + } + const { audio } = await synthesize(text, { + engine: "vits", + voice: "en_US-hfc_female-medium", + }); + + let wavStream: Readable; + if (audio instanceof Buffer) { + console.log("audio is a buffer"); + wavStream = Readable.from(audio); + } else if ("audioChannels" in audio && "sampleRate" in audio) { + console.log("audio is a RawAudio"); + const floatBuffer = Buffer.from(audio.audioChannels[0].buffer); + console.log("buffer length: ", floatBuffer.length); - let wavStream: Readable; - if (audio instanceof Buffer) { - console.log("audio is a buffer"); - wavStream = Readable.from(audio); - } else if ("audioChannels" in audio && "sampleRate" in audio) { - console.log("audio is a RawAudio"); - const floatBuffer = Buffer.from(audio.audioChannels[0].buffer); - console.log("buffer length: ", floatBuffer.length); + // Get the sample rate from the RawAudio object + const sampleRate = audio.sampleRate; - // Get the sample rate from the RawAudio object - const sampleRate = audio.sampleRate; + // Create a Float32Array view of the floatBuffer + const floatArray = new Float32Array(floatBuffer.buffer); - // Create a Float32Array view of the floatBuffer - const floatArray = new Float32Array(floatBuffer.buffer); + // Convert 32-bit float audio to 16-bit PCM + const pcmBuffer = new Int16Array(floatArray.length); + for (let i = 0; i < floatArray.length; i++) { + pcmBuffer[i] = Math.round(floatArray[i] * 32767); + } - // Convert 32-bit float audio to 16-bit PCM - const pcmBuffer = new Int16Array(floatArray.length); - for (let i = 0; i < floatArray.length; i++) { - pcmBuffer[i] = Math.round(floatArray[i] * 32767); - } + // Prepend WAV header to the buffer + const wavHeaderBuffer = getWavHeader( + pcmBuffer.length * 2, + sampleRate, + 1, + 16 + ); + const wavBuffer = Buffer.concat([ + wavHeaderBuffer, + Buffer.from(pcmBuffer.buffer), + ]); - // Prepend WAV header to the buffer - const wavHeaderBuffer = getWavHeader( - pcmBuffer.length * 2, - sampleRate, - 1, - 16, - ); - const wavBuffer = Buffer.concat([ - wavHeaderBuffer, - Buffer.from(pcmBuffer.buffer), - ]); + wavStream = Readable.from(wavBuffer); + } else { + throw new Error("Unsupported audio format"); + } - wavStream = Readable.from(wavBuffer); - } else { - throw new Error("Unsupported audio format"); + return wavStream; } - - return wavStream; - } -} \ No newline at end of file +} diff --git a/core/src/services/summary.ts b/core/src/services/summary.ts index b9d2a1b876..222302b675 100644 --- a/core/src/services/summary.ts +++ b/core/src/services/summary.ts @@ -3,13 +3,13 @@ import { parseJSONObjectFromText } from "../core/parsing.ts"; import { IAgentRuntime, ModelClass } from "../core/types.ts"; export async function generateSummary( - runtime: IAgentRuntime, - text: string, + runtime: IAgentRuntime, + text: string ): Promise<{ title: string; description: string }> { - // make sure text is under 128k characters - text = trimTokens(text, 100000, "gpt-4o-mini"); // TODO: clean this up + // make sure text is under 128k characters + text = trimTokens(text, 100000, "gpt-4o-mini"); // TODO: clean this up - const prompt = `Please generate a concise summary for the following text: + const prompt = `Please generate a concise summary for the following text: Text: """ ${text} @@ -23,23 +23,23 @@ export async function generateSummary( } \`\`\``; - const response = await generateText({ - runtime, - context: prompt, - modelClass: ModelClass.SMALL, - }); + const response = await generateText({ + runtime, + context: prompt, + modelClass: ModelClass.SMALL, + }); - const parsedResponse = parseJSONObjectFromText(response); + const parsedResponse = parseJSONObjectFromText(response); + + if (parsedResponse) { + return { + title: parsedResponse.title, + description: parsedResponse.summary, + }; + } - if (parsedResponse) { return { - title: parsedResponse.title, - description: parsedResponse.summary, + title: "", + description: "", }; - } - - return { - title: "", - description: "", - }; } diff --git a/core/src/services/tokenUtils.ts b/core/src/services/tokenUtils.ts index 86a142a333..034dddc299 100644 --- a/core/src/services/tokenUtils.ts +++ b/core/src/services/tokenUtils.ts @@ -2,59 +2,71 @@ import { getAccount, getAssociatedTokenAddress } from "@solana/spl-token"; import { Connection, PublicKey } from "@solana/web3.js"; export async function getTokenPriceInSol(tokenSymbol: string): Promise { - const response = await fetch(`https://price.jup.ag/v6/price?ids=${tokenSymbol}`); - const data = await response.json(); - return data.data[tokenSymbol].price; + const response = await fetch( + `https://price.jup.ag/v6/price?ids=${tokenSymbol}` + ); + const data = await response.json(); + return data.data[tokenSymbol].price; } async function getTokenBalance( - connection: Connection, - walletPublicKey: PublicKey, - tokenMintAddress: PublicKey + connection: Connection, + walletPublicKey: PublicKey, + tokenMintAddress: PublicKey ): Promise { - const tokenAccountAddress = await getAssociatedTokenAddress(tokenMintAddress, walletPublicKey); - - try { - const tokenAccount = await getAccount(connection, tokenAccountAddress); - const tokenAmount = tokenAccount.amount as unknown as number; - return tokenAmount; - } catch (error) { - console.error(`Error retrieving balance for token: ${tokenMintAddress.toBase58()}`, error); - return 0; - } + const tokenAccountAddress = await getAssociatedTokenAddress( + tokenMintAddress, + walletPublicKey + ); + + try { + const tokenAccount = await getAccount(connection, tokenAccountAddress); + const tokenAmount = tokenAccount.amount as unknown as number; + return tokenAmount; + } catch (error) { + console.error( + `Error retrieving balance for token: ${tokenMintAddress.toBase58()}`, + error + ); + return 0; + } } async function getTokenBalances( - connection: Connection, - walletPublicKey: PublicKey + connection: Connection, + walletPublicKey: PublicKey ): Promise<{ [tokenName: string]: number }> { - const tokenBalances: { [tokenName: string]: number } = {}; - - // Add the token mint addresses you want to retrieve balances for - const tokenMintAddresses = [ - new PublicKey("EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"), // USDC - new PublicKey("So11111111111111111111111111111111111111112"), // SOL - // Add more token mint addresses as needed - ]; - - for (const mintAddress of tokenMintAddresses) { - const tokenName = getTokenName(mintAddress); - const balance = await getTokenBalance(connection, walletPublicKey, mintAddress); - tokenBalances[tokenName] = balance; - } - - return tokenBalances; + const tokenBalances: { [tokenName: string]: number } = {}; + + // Add the token mint addresses you want to retrieve balances for + const tokenMintAddresses = [ + new PublicKey("EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"), // USDC + new PublicKey("So11111111111111111111111111111111111111112"), // SOL + // Add more token mint addresses as needed + ]; + + for (const mintAddress of tokenMintAddresses) { + const tokenName = getTokenName(mintAddress); + const balance = await getTokenBalance( + connection, + walletPublicKey, + mintAddress + ); + tokenBalances[tokenName] = balance; + } + + return tokenBalances; } function getTokenName(mintAddress: PublicKey): string { - // Implement a mapping of mint addresses to token names - const tokenNameMap: { [mintAddress: string]: string } = { - "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v": "USDC", - "So11111111111111111111111111111111111111112": "SOL", - // Add more token mint addresses and their corresponding names - }; - - return tokenNameMap[mintAddress.toBase58()] || "Unknown Token"; + // Implement a mapping of mint addresses to token names + const tokenNameMap: { [mintAddress: string]: string } = { + EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v: "USDC", + So11111111111111111111111111111111111111112: "SOL", + // Add more token mint addresses and their corresponding names + }; + + return tokenNameMap[mintAddress.toBase58()] || "Unknown Token"; } export { getTokenBalance, getTokenBalances }; diff --git a/core/src/services/transcription.ts b/core/src/services/transcription.ts index 1fdbf3f785..84621170d6 100644 --- a/core/src/services/transcription.ts +++ b/core/src/services/transcription.ts @@ -17,290 +17,301 @@ const __dirname = path.dirname(__filename); const execAsync = promisify(exec); export class TranscriptionService extends EventEmitter { - private static instance: TranscriptionService | null = null; - private CONTENT_CACHE_DIR: string; - private DEBUG_AUDIO_DIR: string; - private TARGET_SAMPLE_RATE = 16000; // Common sample rate for speech recognition - private isCudaAvailable: boolean = false; - private openai: OpenAI | null = null; - private runtime: IAgentRuntime; - - private queue: { audioBuffer: ArrayBuffer; resolve: Function }[] = []; - private processing: boolean = false; - - private constructor(runtime: IAgentRuntime) { - super(); - this.runtime = runtime; - const rootDir = path.resolve(__dirname, "../../"); - this.CONTENT_CACHE_DIR = path.join(rootDir, "content_cache"); - this.DEBUG_AUDIO_DIR = path.join(rootDir, "debug_audio"); - this.ensureCacheDirectoryExists(); - this.ensureDebugDirectoryExists(); - // TODO: It'd be nice to handle this more gracefully, but we can do local transcription for now - if (this.runtime.getSetting("OPENAI_API_KEY")) { - this.openai = new OpenAI({ - apiKey: this.runtime.getSetting("OPENAI_API_KEY"), - }); - } else { - this.detectCuda(); + private static instance: TranscriptionService | null = null; + private CONTENT_CACHE_DIR: string; + private DEBUG_AUDIO_DIR: string; + private TARGET_SAMPLE_RATE = 16000; // Common sample rate for speech recognition + private isCudaAvailable: boolean = false; + private openai: OpenAI | null = null; + private runtime: IAgentRuntime; + + private queue: { audioBuffer: ArrayBuffer; resolve: Function }[] = []; + private processing: boolean = false; + + private constructor(runtime: IAgentRuntime) { + super(); + this.runtime = runtime; + const rootDir = path.resolve(__dirname, "../../"); + this.CONTENT_CACHE_DIR = path.join(rootDir, "content_cache"); + this.DEBUG_AUDIO_DIR = path.join(rootDir, "debug_audio"); + this.ensureCacheDirectoryExists(); + this.ensureDebugDirectoryExists(); + // TODO: It'd be nice to handle this more gracefully, but we can do local transcription for now + if (this.runtime.getSetting("OPENAI_API_KEY")) { + this.openai = new OpenAI({ + apiKey: this.runtime.getSetting("OPENAI_API_KEY"), + }); + } else { + this.detectCuda(); + } } - } - public static getInstance(runtime: IAgentRuntime): TranscriptionService { - if (!TranscriptionService.instance) { - TranscriptionService.instance = new TranscriptionService(runtime); + public static getInstance(runtime: IAgentRuntime): TranscriptionService { + if (!TranscriptionService.instance) { + TranscriptionService.instance = new TranscriptionService(runtime); + } + return TranscriptionService.instance; } - return TranscriptionService.instance; - } - private ensureCacheDirectoryExists() { - if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { - fs.mkdirSync(this.CONTENT_CACHE_DIR, { recursive: true }); + private ensureCacheDirectoryExists() { + if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { + fs.mkdirSync(this.CONTENT_CACHE_DIR, { recursive: true }); + } } - } - private ensureDebugDirectoryExists() { - if (!fs.existsSync(this.DEBUG_AUDIO_DIR)) { - fs.mkdirSync(this.DEBUG_AUDIO_DIR, { recursive: true }); + private ensureDebugDirectoryExists() { + if (!fs.existsSync(this.DEBUG_AUDIO_DIR)) { + fs.mkdirSync(this.DEBUG_AUDIO_DIR, { recursive: true }); + } } - } - - private detectCuda() { - const platform = os.platform(); - if (platform === "linux") { - try { - fs.accessSync("/usr/local/cuda/bin/nvcc", fs.constants.X_OK); - this.isCudaAvailable = true; - console.log("CUDA detected. Transcription will use CUDA acceleration."); - } catch (error) { - console.log("CUDA not detected. Transcription will run on CPU."); - } - } else if (platform === "win32") { - const cudaPath = path.join( - this.runtime.getSetting("CUDA_PATH") || - "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.0", - "bin", - "nvcc.exe" - ); - if (fs.existsSync(cudaPath)) { - this.isCudaAvailable = true; - console.log("CUDA detected. Transcription will use CUDA acceleration."); - } else { - console.log("CUDA not detected. Transcription will run on CPU."); - } - } else { - console.log( - "CUDA not supported on this platform. Transcription will run on CPU." - ); + + private detectCuda() { + const platform = os.platform(); + if (platform === "linux") { + try { + fs.accessSync("/usr/local/cuda/bin/nvcc", fs.constants.X_OK); + this.isCudaAvailable = true; + console.log( + "CUDA detected. Transcription will use CUDA acceleration." + ); + } catch (error) { + console.log( + "CUDA not detected. Transcription will run on CPU." + ); + } + } else if (platform === "win32") { + const cudaPath = path.join( + this.runtime.getSetting("CUDA_PATH") || + "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.0", + "bin", + "nvcc.exe" + ); + if (fs.existsSync(cudaPath)) { + this.isCudaAvailable = true; + console.log( + "CUDA detected. Transcription will use CUDA acceleration." + ); + } else { + console.log( + "CUDA not detected. Transcription will run on CPU." + ); + } + } else { + console.log( + "CUDA not supported on this platform. Transcription will run on CPU." + ); + } } - } - private async convertAudio(inputBuffer: ArrayBuffer): Promise { - const inputPath = path.join( - this.CONTENT_CACHE_DIR, - `input_${Date.now()}.wav` - ); - const outputPath = path.join( - this.CONTENT_CACHE_DIR, - `output_${Date.now()}.wav` - ); + private async convertAudio(inputBuffer: ArrayBuffer): Promise { + const inputPath = path.join( + this.CONTENT_CACHE_DIR, + `input_${Date.now()}.wav` + ); + const outputPath = path.join( + this.CONTENT_CACHE_DIR, + `output_${Date.now()}.wav` + ); - fs.writeFileSync(inputPath, Buffer.from(inputBuffer)); + fs.writeFileSync(inputPath, Buffer.from(inputBuffer)); - try { - const { stdout } = await execAsync( - `ffprobe -v error -show_entries stream=codec_name,sample_rate,channels -of json "${inputPath}"` - ); - const probeResult = JSON.parse(stdout); - const stream = probeResult.streams[0]; + try { + const { stdout } = await execAsync( + `ffprobe -v error -show_entries stream=codec_name,sample_rate,channels -of json "${inputPath}"` + ); + const probeResult = JSON.parse(stdout); + const stream = probeResult.streams[0]; - console.log("Input audio info:", stream); + console.log("Input audio info:", stream); - let ffmpegCommand = `ffmpeg -i "${inputPath}" -ar ${this.TARGET_SAMPLE_RATE} -ac 1`; + let ffmpegCommand = `ffmpeg -i "${inputPath}" -ar ${this.TARGET_SAMPLE_RATE} -ac 1`; - if (stream.codec_name === "pcm_f32le") { - ffmpegCommand += " -acodec pcm_s16le"; - } + if (stream.codec_name === "pcm_f32le") { + ffmpegCommand += " -acodec pcm_s16le"; + } - ffmpegCommand += ` "${outputPath}"`; + ffmpegCommand += ` "${outputPath}"`; - console.log("FFmpeg command:", ffmpegCommand); + console.log("FFmpeg command:", ffmpegCommand); - await execAsync(ffmpegCommand); + await execAsync(ffmpegCommand); - const convertedBuffer = fs.readFileSync(outputPath); - fs.unlinkSync(inputPath); - fs.unlinkSync(outputPath); - return convertedBuffer; - } catch (error) { - console.error("Error converting audio:", error); - throw error; + const convertedBuffer = fs.readFileSync(outputPath); + fs.unlinkSync(inputPath); + fs.unlinkSync(outputPath); + return convertedBuffer; + } catch (error) { + console.error("Error converting audio:", error); + throw error; + } } - } - private async saveDebugAudio(audioBuffer: ArrayBuffer, prefix: string) { - this.ensureDebugDirectoryExists(); + private async saveDebugAudio(audioBuffer: ArrayBuffer, prefix: string) { + this.ensureDebugDirectoryExists(); - const filename = `${prefix}_${Date.now()}.wav`; - const filePath = path.join(this.DEBUG_AUDIO_DIR, filename); + const filename = `${prefix}_${Date.now()}.wav`; + const filePath = path.join(this.DEBUG_AUDIO_DIR, filename); - fs.writeFileSync(filePath, Buffer.from(audioBuffer)); - console.log(`Debug audio saved: ${filePath}`); - } + fs.writeFileSync(filePath, Buffer.from(audioBuffer)); + console.log(`Debug audio saved: ${filePath}`); + } - public async transcribeAttachment( - audioBuffer: ArrayBuffer - ): Promise { - return await this.transcribe(audioBuffer); - } + public async transcribeAttachment( + audioBuffer: ArrayBuffer + ): Promise { + return await this.transcribe(audioBuffer); + } - public async transcribe(audioBuffer: ArrayBuffer): Promise { - // if the audio buffer is less than .2 seconds, just return null - if (audioBuffer.byteLength < 0.2 * 16000) { - return null; + public async transcribe(audioBuffer: ArrayBuffer): Promise { + // if the audio buffer is less than .2 seconds, just return null + if (audioBuffer.byteLength < 0.2 * 16000) { + return null; + } + return new Promise((resolve) => { + this.queue.push({ audioBuffer, resolve }); + if (!this.processing) { + this.processQueue(); + } + }); } - return new Promise((resolve) => { - this.queue.push({ audioBuffer, resolve }); - if (!this.processing) { - this.processQueue(); - } - }); - } - - public async transcribeAttachmentLocally( - audioBuffer: ArrayBuffer - ): Promise { - return this.transcribeLocally(audioBuffer); - } - - private async processQueue(): Promise { - if (this.processing || this.queue.length === 0) { - return; + + public async transcribeAttachmentLocally( + audioBuffer: ArrayBuffer + ): Promise { + return this.transcribeLocally(audioBuffer); } - this.processing = true; + private async processQueue(): Promise { + if (this.processing || this.queue.length === 0) { + return; + } - while (this.queue.length > 0) { - const { audioBuffer, resolve } = this.queue.shift()!; - let result: string | null = null; + this.processing = true; - if (this.openai) { - result = await this.transcribeWithOpenAI(audioBuffer); - } else { - result = await this.transcribeLocally(audioBuffer); - } + while (this.queue.length > 0) { + const { audioBuffer, resolve } = this.queue.shift()!; + let result: string | null = null; - resolve(result); + if (this.openai) { + result = await this.transcribeWithOpenAI(audioBuffer); + } else { + result = await this.transcribeLocally(audioBuffer); + } + + resolve(result); + } + + this.processing = false; } - this.processing = false; - } - - private async transcribeWithOpenAI( - audioBuffer: ArrayBuffer - ): Promise { - console.log("Transcribing audio with OpenAI..."); - - try { - await this.saveDebugAudio(audioBuffer, "openai_input_original"); - - const convertedBuffer = await this.convertAudio(audioBuffer); - - await this.saveDebugAudio(convertedBuffer, "openai_input_converted"); - - const file = new File([convertedBuffer], "audio.wav", { - type: "audio/wav", - }); - - const result = await this.openai!.audio.transcriptions.create({ - model: "whisper-1", - language: "en", - response_format: "text", - file: file, - }); - - const trimmedResult = (result as any).trim(); - console.log(`OpenAI speech to text result: "${trimmedResult}"`); - - return trimmedResult; - } catch (error) { - console.error("Error in OpenAI speech-to-text conversion:", error); - if (error.response) { - console.error("Response data:", error.response.data); - console.error("Response status:", error.response.status); - console.error("Response headers:", error.response.headers); - } else if (error.request) { - console.error("No response received:", error.request); - } else { - console.error("Error setting up request:", error.message); - } - return null; + private async transcribeWithOpenAI( + audioBuffer: ArrayBuffer + ): Promise { + console.log("Transcribing audio with OpenAI..."); + + try { + await this.saveDebugAudio(audioBuffer, "openai_input_original"); + + const convertedBuffer = await this.convertAudio(audioBuffer); + + await this.saveDebugAudio( + convertedBuffer, + "openai_input_converted" + ); + + const file = new File([convertedBuffer], "audio.wav", { + type: "audio/wav", + }); + + const result = await this.openai!.audio.transcriptions.create({ + model: "whisper-1", + language: "en", + response_format: "text", + file: file, + }); + + const trimmedResult = (result as any).trim(); + console.log(`OpenAI speech to text result: "${trimmedResult}"`); + + return trimmedResult; + } catch (error) { + console.error("Error in OpenAI speech-to-text conversion:", error); + if (error.response) { + console.error("Response data:", error.response.data); + console.error("Response status:", error.response.status); + console.error("Response headers:", error.response.headers); + } else if (error.request) { + console.error("No response received:", error.request); + } else { + console.error("Error setting up request:", error.message); + } + return null; + } } - } - - public async transcribeLocally( - audioBuffer: ArrayBuffer - ): Promise { - try { - console.log("Transcribing audio locally..."); - - await this.saveDebugAudio(audioBuffer, "local_input_original"); - - const convertedBuffer = await this.convertAudio(audioBuffer); - - await this.saveDebugAudio(convertedBuffer, "local_input_converted"); - - const tempWavFile = path.join( - this.CONTENT_CACHE_DIR, - `temp_${Date.now()}.wav` - ); - fs.writeFileSync(tempWavFile, convertedBuffer); - - console.log(`Temporary WAV file created: ${tempWavFile}`); - - let output = await nodewhisper(tempWavFile, { - modelName: "base.en", - autoDownloadModelName: "base.en", - verbose: true, - removeWavFileAfterTranscription: false, - withCuda: this.isCudaAvailable, - whisperOptions: { - outputInText: true, - outputInVtt: false, - outputInSrt: false, - outputInCsv: false, - translateToEnglish: false, - wordTimestamps: false, - timestamps_length: 60, - splitOnWord: true, - }, - }); - - console.log("Raw output from nodejs-whisper:", output); - - output = output - .split("\n") - .map((line) => { - if (line.trim().startsWith("[")) { - const endIndex = line.indexOf("]"); - return line.substring(endIndex + 1); - } - return line; - }) - .join("\n"); - - console.log("Processed output:", output); - - fs.unlinkSync(tempWavFile); - - if (!output || output.length < 5) { - console.log("Output is null or too short, returning null"); - return null; - } - return output; - } catch (error) { - console.error("Error in local speech-to-text conversion:", error); - return null; + + public async transcribeLocally( + audioBuffer: ArrayBuffer + ): Promise { + try { + console.log("Transcribing audio locally..."); + + await this.saveDebugAudio(audioBuffer, "local_input_original"); + + const convertedBuffer = await this.convertAudio(audioBuffer); + + await this.saveDebugAudio(convertedBuffer, "local_input_converted"); + + const tempWavFile = path.join( + this.CONTENT_CACHE_DIR, + `temp_${Date.now()}.wav` + ); + fs.writeFileSync(tempWavFile, convertedBuffer); + + console.log(`Temporary WAV file created: ${tempWavFile}`); + + let output = await nodewhisper(tempWavFile, { + modelName: "base.en", + autoDownloadModelName: "base.en", + verbose: true, + removeWavFileAfterTranscription: false, + withCuda: this.isCudaAvailable, + whisperOptions: { + outputInText: true, + outputInVtt: false, + outputInSrt: false, + outputInCsv: false, + translateToEnglish: false, + wordTimestamps: false, + timestamps_length: 60, + splitOnWord: true, + }, + }); + + console.log("Raw output from nodejs-whisper:", output); + + output = output + .split("\n") + .map((line) => { + if (line.trim().startsWith("[")) { + const endIndex = line.indexOf("]"); + return line.substring(endIndex + 1); + } + return line; + }) + .join("\n"); + + console.log("Processed output:", output); + + fs.unlinkSync(tempWavFile); + + if (!output || output.length < 5) { + console.log("Output is null or too short, returning null"); + return null; + } + return output; + } catch (error) { + console.error("Error in local speech-to-text conversion:", error); + return null; + } } - } } diff --git a/core/src/services/video.ts b/core/src/services/video.ts index 199c146541..6c477cf231 100644 --- a/core/src/services/video.ts +++ b/core/src/services/video.ts @@ -6,389 +6,399 @@ import { IAgentRuntime, Media } from "../core/types.ts"; import { stringToUuid } from "../core/uuid.ts"; export class VideoService { - private static instance: VideoService | null = null; - private CONTENT_CACHE_DIR = "./content_cache"; - runtime: IAgentRuntime; + private static instance: VideoService | null = null; + private CONTENT_CACHE_DIR = "./content_cache"; + runtime: IAgentRuntime; - private queue: string[] = []; - private processing: boolean = false; + private queue: string[] = []; + private processing: boolean = false; - private constructor(runtime: IAgentRuntime) { - this.ensureCacheDirectoryExists(); - this.runtime = runtime; - } + private constructor(runtime: IAgentRuntime) { + this.ensureCacheDirectoryExists(); + this.runtime = runtime; + } - public static getInstance(runtime: IAgentRuntime): VideoService { - if (!VideoService.instance) { - VideoService.instance = new VideoService(runtime); + public static getInstance(runtime: IAgentRuntime): VideoService { + if (!VideoService.instance) { + VideoService.instance = new VideoService(runtime); + } + return VideoService.instance; } - return VideoService.instance; - } - private ensureCacheDirectoryExists() { - if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { - fs.mkdirSync(this.CONTENT_CACHE_DIR); + private ensureCacheDirectoryExists() { + if (!fs.existsSync(this.CONTENT_CACHE_DIR)) { + fs.mkdirSync(this.CONTENT_CACHE_DIR); + } } - } - - public isVideoUrl(url: string): boolean { - return ( - url.includes("youtube.com") || - url.includes("youtu.be") || - url.includes("vimeo.com") - ); - } - - public async downloadMedia(url: string): Promise { - const videoId = this.getVideoId(url); - const outputFile = path.join(this.CONTENT_CACHE_DIR, `${videoId}.mp4`); - - // if it already exists, return it - if (fs.existsSync(outputFile)) { - return outputFile; + + public isVideoUrl(url: string): boolean { + return ( + url.includes("youtube.com") || + url.includes("youtu.be") || + url.includes("vimeo.com") + ); } - try { - await youtubeDl(url, { - verbose: true, - output: outputFile, - writeInfoJson: true, - }); - return outputFile; - } catch (error) { - console.error("Error downloading media:", error); - throw new Error("Failed to download media"); + public async downloadMedia(url: string): Promise { + const videoId = this.getVideoId(url); + const outputFile = path.join(this.CONTENT_CACHE_DIR, `${videoId}.mp4`); + + // if it already exists, return it + if (fs.existsSync(outputFile)) { + return outputFile; + } + + try { + await youtubeDl(url, { + verbose: true, + output: outputFile, + writeInfoJson: true, + }); + return outputFile; + } catch (error) { + console.error("Error downloading media:", error); + throw new Error("Failed to download media"); + } } - } - public async downloadVideo(videoInfo: any): Promise { - const videoId = this.getVideoId(videoInfo.webpage_url); - const outputFile = path.join(this.CONTENT_CACHE_DIR, `${videoId}.mp4`); + public async downloadVideo(videoInfo: any): Promise { + const videoId = this.getVideoId(videoInfo.webpage_url); + const outputFile = path.join(this.CONTENT_CACHE_DIR, `${videoId}.mp4`); - // if it already exists, return it - if (fs.existsSync(outputFile)) { - return outputFile; + // if it already exists, return it + if (fs.existsSync(outputFile)) { + return outputFile; + } + + try { + await youtubeDl(videoInfo.webpage_url, { + verbose: true, + output: outputFile, + format: "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", + writeInfoJson: true, + }); + return outputFile; + } catch (error) { + console.error("Error downloading video:", error); + throw new Error("Failed to download video"); + } } - try { - await youtubeDl(videoInfo.webpage_url, { - verbose: true, - output: outputFile, - format: "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", - writeInfoJson: true, - }); - return outputFile; - } catch (error) { - console.error("Error downloading video:", error); - throw new Error("Failed to download video"); + public async processVideo(url: string): Promise { + this.queue.push(url); + this.processQueue(); + + return new Promise((resolve, reject) => { + const checkQueue = async () => { + const index = this.queue.indexOf(url); + if (index !== -1) { + setTimeout(checkQueue, 100); + } else { + try { + const result = await this.processVideoFromUrl(url); + resolve(result); + } catch (error) { + reject(error); + } + } + }; + checkQueue(); + }); } - } - - public async processVideo(url: string): Promise { - this.queue.push(url); - this.processQueue(); - - return new Promise((resolve, reject) => { - const checkQueue = async () => { - const index = this.queue.indexOf(url); - if (index !== -1) { - setTimeout(checkQueue, 100); - } else { - try { - const result = await this.processVideoFromUrl(url); - resolve(result); - } catch (error) { - reject(error); - } + + private async processQueue(): Promise { + if (this.processing || this.queue.length === 0) { + return; + } + + this.processing = true; + + while (this.queue.length > 0) { + const url = this.queue.shift()!; + await this.processVideoFromUrl(url); } - }; - checkQueue(); - }); - } - - private async processQueue(): Promise { - if (this.processing || this.queue.length === 0) { - return; + + this.processing = false; } - this.processing = true; + private async processVideoFromUrl(url: string): Promise { + const videoId = + url.match( + /(?:youtu\.be\/|youtube\.com(?:\/embed\/|\/v\/|\/watch\?v=|\/watch\?.+&v=))([^\/&?]+)/ + )?.[1] || ""; + const videoUuid = this.getVideoId(videoId); + const cacheFilePath = path.join( + this.CONTENT_CACHE_DIR, + `${videoUuid}.json` + ); + + if (fs.existsSync(cacheFilePath)) { + console.log("Returning cached video file"); + return JSON.parse(fs.readFileSync(cacheFilePath, "utf-8")) as Media; + } - while (this.queue.length > 0) { - const url = this.queue.shift()!; - await this.processVideoFromUrl(url); + console.log("Cache miss, processing video"); + console.log("Fetching video info"); + const videoInfo = await this.fetchVideoInfo(url); + console.log("Getting transcript"); + const transcript = await this.getTranscript(url, videoInfo); + + const result: Media = { + id: videoUuid, + url: url, + title: videoInfo.title, + source: videoInfo.channel, + description: videoInfo.description, + text: transcript, + }; + + fs.writeFileSync(cacheFilePath, JSON.stringify(result)); + return result; } - this.processing = false; - } - - private async processVideoFromUrl(url: string): Promise { - const videoId = - url.match( - /(?:youtu\.be\/|youtube\.com(?:\/embed\/|\/v\/|\/watch\?v=|\/watch\?.+&v=))([^\/&?]+)/, - )?.[1] || ""; - const videoUuid = this.getVideoId(videoId); - const cacheFilePath = path.join( - this.CONTENT_CACHE_DIR, - `${videoUuid}.json`, - ); - - if (fs.existsSync(cacheFilePath)) { - console.log("Returning cached video file"); - return JSON.parse(fs.readFileSync(cacheFilePath, "utf-8")) as Media; + private getVideoId(url: string): string { + return stringToUuid(url); } - console.log("Cache miss, processing video"); - console.log("Fetching video info"); - const videoInfo = await this.fetchVideoInfo(url); - console.log("Getting transcript"); - const transcript = await this.getTranscript(url, videoInfo); - - const result: Media = { - id: videoUuid, - url: url, - title: videoInfo.title, - source: videoInfo.channel, - description: videoInfo.description, - text: transcript, - }; - - fs.writeFileSync(cacheFilePath, JSON.stringify(result)); - return result; - } - - private getVideoId(url: string): string { - return stringToUuid(url); - } - - async fetchVideoInfo(url: string): Promise { - if (url.endsWith(".mp4") || url.includes(".mp4?")) { - try { - const response = await fetch(url); - if (response.ok) { - // If the URL is a direct link to an MP4 file, return a simplified video info object - return { - title: path.basename(url), - description: "", - channel: "", - }; + async fetchVideoInfo(url: string): Promise { + if (url.endsWith(".mp4") || url.includes(".mp4?")) { + try { + const response = await fetch(url); + if (response.ok) { + // If the URL is a direct link to an MP4 file, return a simplified video info object + return { + title: path.basename(url), + description: "", + channel: "", + }; + } + } catch (error) { + console.error("Error downloading MP4 file:", error); + // Fall back to using youtube-dl if direct download fails + } + } + + try { + const result = await youtubeDl(url, { + dumpJson: true, + verbose: true, + callHome: false, + noCheckCertificates: true, + preferFreeFormats: true, + youtubeSkipDashManifest: true, + writeSub: true, + writeAutoSub: true, + subLang: "en", + skipDownload: true, + }); + return result; + } catch (error) { + console.error("Error fetching video info:", error); + throw new Error("Failed to fetch video information"); } - } catch (error) { - console.error("Error downloading MP4 file:", error); - // Fall back to using youtube-dl if direct download fails - } } - try { - const result = await youtubeDl(url, { - dumpJson: true, - verbose: true, - callHome: false, - noCheckCertificates: true, - preferFreeFormats: true, - youtubeSkipDashManifest: true, - writeSub: true, - writeAutoSub: true, - subLang: "en", - skipDownload: true, - }); - return result; - } catch (error) { - console.error("Error fetching video info:", error); - throw new Error("Failed to fetch video information"); + private async getTranscript(url: string, videoInfo: any): Promise { + console.log("Getting transcript"); + try { + // Check for manual subtitles + if (videoInfo.subtitles && videoInfo.subtitles.en) { + console.log("Manual subtitles found"); + const srtContent = await this.downloadSRT( + videoInfo.subtitles.en[0].url + ); + return this.parseSRT(srtContent); + } + + // Check for automatic captions + if ( + videoInfo.automatic_captions && + videoInfo.automatic_captions.en + ) { + console.log("Automatic captions found"); + const captionUrl = videoInfo.automatic_captions.en[0].url; + const captionContent = await this.downloadCaption(captionUrl); + return this.parseCaption(captionContent); + } + + // Check if it's a music video + if ( + videoInfo.categories && + videoInfo.categories.includes("Music") + ) { + console.log("Music video detected, no lyrics available"); + return "No lyrics available."; + } + + // Fall back to audio transcription + console.log( + "No captions found, falling back to audio transcription" + ); + return this.transcribeAudio(url); + } catch (error) { + console.error("Error in getTranscript:", error); + throw error; + } } - } - - private async getTranscript(url: string, videoInfo: any): Promise { - console.log("Getting transcript"); - try { - // Check for manual subtitles - if (videoInfo.subtitles && videoInfo.subtitles.en) { - console.log("Manual subtitles found"); - const srtContent = await this.downloadSRT( - videoInfo.subtitles.en[0].url, - ); - return this.parseSRT(srtContent); - } - - // Check for automatic captions - if (videoInfo.automatic_captions && videoInfo.automatic_captions.en) { - console.log("Automatic captions found"); - const captionUrl = videoInfo.automatic_captions.en[0].url; - const captionContent = await this.downloadCaption(captionUrl); - return this.parseCaption(captionContent); - } - - // Check if it's a music video - if (videoInfo.categories && videoInfo.categories.includes("Music")) { - console.log("Music video detected, no lyrics available"); - return "No lyrics available."; - } - - // Fall back to audio transcription - console.log("No captions found, falling back to audio transcription"); - return this.transcribeAudio(url); - } catch (error) { - console.error("Error in getTranscript:", error); - throw error; + + private async downloadCaption(url: string): Promise { + console.log("Downloading caption from:", url); + const response = await fetch(url); + if (!response.ok) { + throw new Error( + `Failed to download caption: ${response.statusText}` + ); + } + return await response.text(); } - } - private async downloadCaption(url: string): Promise { - console.log("Downloading caption from:", url); - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to download caption: ${response.statusText}`); + private parseCaption(captionContent: string): string { + console.log("Parsing caption"); + try { + const jsonContent = JSON.parse(captionContent); + if (jsonContent.events) { + return jsonContent.events + .filter((event) => event.segs) + .map((event) => event.segs.map((seg) => seg.utf8).join("")) + .join("") + .replace("\n", " "); + } else { + console.error("Unexpected caption format:", jsonContent); + return "Error: Unable to parse captions"; + } + } catch (error) { + console.error("Error parsing caption:", error); + return "Error: Unable to parse captions"; + } } - return await response.text(); - } - - private parseCaption(captionContent: string): string { - console.log("Parsing caption"); - try { - const jsonContent = JSON.parse(captionContent); - if (jsonContent.events) { - return jsonContent.events - .filter((event) => event.segs) - .map((event) => event.segs.map((seg) => seg.utf8).join("")) - .join("") - .replace("\n", " "); - } else { - console.error("Unexpected caption format:", jsonContent); - return "Error: Unable to parse captions"; - } - } catch (error) { - console.error("Error parsing caption:", error); - return "Error: Unable to parse captions"; + + private parseSRT(srtContent: string): string { + // Simple SRT parser (replace with a more robust solution if needed) + return srtContent + .split("\n\n") + .map((block) => block.split("\n").slice(2).join(" ")) + .join(" "); } - } - - private parseSRT(srtContent: string): string { - // Simple SRT parser (replace with a more robust solution if needed) - return srtContent - .split("\n\n") - .map((block) => block.split("\n").slice(2).join(" ")) - .join(" "); - } - - private async downloadSRT(url: string): Promise { - console.log("downloadSRT"); - const response = await fetch(url); - return await response.text(); - } - - async transcribeAudio(url: string): Promise { - console.log("Preparing audio for transcription..."); - const mp4FilePath = path.join( - this.CONTENT_CACHE_DIR, - `${this.getVideoId(url)}.mp4`, - ); - const mp3FilePath = path.join( - this.CONTENT_CACHE_DIR, - `${this.getVideoId(url)}.mp3`, - ); - - if (!fs.existsSync(mp3FilePath)) { - if (fs.existsSync(mp4FilePath)) { - console.log("MP4 file found. Converting to MP3..."); - await this.convertMp4ToMp3(mp4FilePath, mp3FilePath); - } else { - console.log("Downloading audio..."); - await this.downloadAudio(url, mp3FilePath); - } + + private async downloadSRT(url: string): Promise { + console.log("downloadSRT"); + const response = await fetch(url); + return await response.text(); } - console.log(`Audio prepared at ${mp3FilePath}`); - - const audioBuffer = fs.readFileSync(mp3FilePath); - console.log(`Audio file size: ${audioBuffer.length} bytes`); - - console.log("Starting transcription..."); - const startTime = Date.now(); - const transcript = - await this.runtime.transcriptionService.transcribe(audioBuffer); - const endTime = Date.now(); - console.log( - `Transcription completed in ${(endTime - startTime) / 1000} seconds`, - ); - - // Don't delete the MP3 file as it might be needed for future use - return transcript || "Transcription failed"; - } - - private async convertMp4ToMp3( - inputPath: string, - outputPath: string, - ): Promise { - return new Promise((resolve, reject) => { - ffmpeg(inputPath) - .output(outputPath) - .noVideo() - .audioCodec("libmp3lame") - .on("end", () => { - console.log("Conversion to MP3 complete"); - resolve(); - }) - .on("error", (err) => { - console.error("Error converting to MP3:", err); - reject(err); - }) - .run(); - }); - } - - private async downloadAudio( - url: string, - outputFile: string, - ): Promise { - console.log("Downloading audio"); - outputFile = - outputFile ?? - path.join(this.CONTENT_CACHE_DIR, `${this.getVideoId(url)}.mp3`); - - try { - if (url.endsWith(".mp4") || url.includes(".mp4?")) { - console.log( - "Direct MP4 file detected, downloading and converting to MP3", + async transcribeAudio(url: string): Promise { + console.log("Preparing audio for transcription..."); + const mp4FilePath = path.join( + this.CONTENT_CACHE_DIR, + `${this.getVideoId(url)}.mp4` ); - const tempMp4File = path.join( - this.CONTENT_CACHE_DIR, - `${this.getVideoId(url)}.mp4`, + const mp3FilePath = path.join( + this.CONTENT_CACHE_DIR, + `${this.getVideoId(url)}.mp3` ); - const response = await fetch(url); - const arrayBuffer = await response.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - fs.writeFileSync(tempMp4File, buffer); - - await new Promise((resolve, reject) => { - ffmpeg(tempMp4File) - .output(outputFile) - .noVideo() - .audioCodec("libmp3lame") - .on("end", () => { - fs.unlinkSync(tempMp4File); - resolve(); - }) - .on("error", (err) => { - reject(err); - }) - .run(); - }); - } else { + + if (!fs.existsSync(mp3FilePath)) { + if (fs.existsSync(mp4FilePath)) { + console.log("MP4 file found. Converting to MP3..."); + await this.convertMp4ToMp3(mp4FilePath, mp3FilePath); + } else { + console.log("Downloading audio..."); + await this.downloadAudio(url, mp3FilePath); + } + } + + console.log(`Audio prepared at ${mp3FilePath}`); + + const audioBuffer = fs.readFileSync(mp3FilePath); + console.log(`Audio file size: ${audioBuffer.length} bytes`); + + console.log("Starting transcription..."); + const startTime = Date.now(); + const transcript = + await this.runtime.transcriptionService.transcribe(audioBuffer); + const endTime = Date.now(); console.log( - "YouTube video detected, downloading audio with youtube-dl", + `Transcription completed in ${(endTime - startTime) / 1000} seconds` ); - await youtubeDl(url, { - verbose: true, - extractAudio: true, - audioFormat: "mp3", - output: outputFile, - writeInfoJson: true, + + // Don't delete the MP3 file as it might be needed for future use + return transcript || "Transcription failed"; + } + + private async convertMp4ToMp3( + inputPath: string, + outputPath: string + ): Promise { + return new Promise((resolve, reject) => { + ffmpeg(inputPath) + .output(outputPath) + .noVideo() + .audioCodec("libmp3lame") + .on("end", () => { + console.log("Conversion to MP3 complete"); + resolve(); + }) + .on("error", (err) => { + console.error("Error converting to MP3:", err); + reject(err); + }) + .run(); }); - } - return outputFile; - } catch (error) { - console.error("Error downloading audio:", error); - throw new Error("Failed to download audio"); } - } + + private async downloadAudio( + url: string, + outputFile: string + ): Promise { + console.log("Downloading audio"); + outputFile = + outputFile ?? + path.join(this.CONTENT_CACHE_DIR, `${this.getVideoId(url)}.mp3`); + + try { + if (url.endsWith(".mp4") || url.includes(".mp4?")) { + console.log( + "Direct MP4 file detected, downloading and converting to MP3" + ); + const tempMp4File = path.join( + this.CONTENT_CACHE_DIR, + `${this.getVideoId(url)}.mp4` + ); + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + fs.writeFileSync(tempMp4File, buffer); + + await new Promise((resolve, reject) => { + ffmpeg(tempMp4File) + .output(outputFile) + .noVideo() + .audioCodec("libmp3lame") + .on("end", () => { + fs.unlinkSync(tempMp4File); + resolve(); + }) + .on("error", (err) => { + reject(err); + }) + .run(); + }); + } else { + console.log( + "YouTube video detected, downloading audio with youtube-dl" + ); + await youtubeDl(url, { + verbose: true, + extractAudio: true, + audioFormat: "mp3", + output: outputFile, + writeInfoJson: true, + }); + } + return outputFile; + } catch (error) { + console.error("Error downloading audio:", error); + throw new Error("Failed to download audio"); + } + } } diff --git a/core/src/services/wordsToPunish.ts b/core/src/services/wordsToPunish.ts index 50518081a7..d3dc5c7a59 100644 --- a/core/src/services/wordsToPunish.ts +++ b/core/src/services/wordsToPunish.ts @@ -1,102 +1,102 @@ export const wordsToPunish = [ - " please", - " feel", - " free", - "!", - "–", - "—", - "?", - ".", - ",", - "; ", - " cosmos", - " tapestry", - " tapestries", - " glitch", - " matrix", - " cyberspace", - " troll", - " questions", - " topics", - " discuss", - " basically", - " simulation", - " simulate", - " universe", - " like", - " debug", - " debugging", - " wild", - " existential", - " juicy", - " circuits", - " help", - " ask", - " happy", - " just", - " cosmic", - " cool", - " joke", - " punchline", - " fancy", - " glad", - " assist", - " algorithm", - " Indeed", - " Furthermore", - " However", - " Notably", - " Therefore", - " Additionally", - " conclusion", - " Significantly", - " Consequently", - " Thus", - " What", - " Otherwise", - " Moreover", - " Subsequently", - " Accordingly", - " Unlock", - " Unleash", - " buckle", - " pave", - " forefront", - " harness", - " harnessing", - " bridging", - " bridging", - " Spearhead", - " spearheading", - " Foster", - " foster", - " environmental", - " impact", - " Navigate", - " navigating", - " challenges", - " chaos", - " social", - " inclusion", - " inclusive", - " diversity", - " diverse", - " delve", - " noise", - " infinite", - " insanity", - " coffee", - " singularity", - " AI", - " digital", - " artificial", - " intelligence", - " consciousness", - " reality", - " metaverse", - " virtual", - " virtual reality", - " VR", - " Metaverse", - " humanity", + " please", + " feel", + " free", + "!", + "–", + "—", + "?", + ".", + ",", + "; ", + " cosmos", + " tapestry", + " tapestries", + " glitch", + " matrix", + " cyberspace", + " troll", + " questions", + " topics", + " discuss", + " basically", + " simulation", + " simulate", + " universe", + " like", + " debug", + " debugging", + " wild", + " existential", + " juicy", + " circuits", + " help", + " ask", + " happy", + " just", + " cosmic", + " cool", + " joke", + " punchline", + " fancy", + " glad", + " assist", + " algorithm", + " Indeed", + " Furthermore", + " However", + " Notably", + " Therefore", + " Additionally", + " conclusion", + " Significantly", + " Consequently", + " Thus", + " What", + " Otherwise", + " Moreover", + " Subsequently", + " Accordingly", + " Unlock", + " Unleash", + " buckle", + " pave", + " forefront", + " harness", + " harnessing", + " bridging", + " bridging", + " Spearhead", + " spearheading", + " Foster", + " foster", + " environmental", + " impact", + " Navigate", + " navigating", + " challenges", + " chaos", + " social", + " inclusion", + " inclusive", + " diversity", + " diverse", + " delve", + " noise", + " infinite", + " insanity", + " coffee", + " singularity", + " AI", + " digital", + " artificial", + " intelligence", + " consciousness", + " reality", + " metaverse", + " virtual", + " virtual reality", + " VR", + " Metaverse", + " humanity", ]; diff --git a/core/src/test_resources/cache.ts b/core/src/test_resources/cache.ts index 1ca0262496..b8fbef0bbe 100644 --- a/core/src/test_resources/cache.ts +++ b/core/src/test_resources/cache.ts @@ -1,39 +1,39 @@ // getCachedEmbeddings // check cache.json for embedding where the key is a stringified version of the memory and the value is a number array export const getCachedEmbeddings = async (text: string) => { - const fs = await import("fs"); + const fs = await import("fs"); - if (!fs.existsSync("./embedding-cache.json")) { - fs.writeFileSync("./embedding-cache.json", "{}"); - } - // read cache.json - const cache = JSON.parse( - fs.readFileSync("./embedding-cache.json", "utf8") as string, - ); - // stringify the memory - const key = JSON.stringify(text); - // return the value of the memory - return cache[key]; + if (!fs.existsSync("./embedding-cache.json")) { + fs.writeFileSync("./embedding-cache.json", "{}"); + } + // read cache.json + const cache = JSON.parse( + fs.readFileSync("./embedding-cache.json", "utf8") as string + ); + // stringify the memory + const key = JSON.stringify(text); + // return the value of the memory + return cache[key]; }; export const writeCachedEmbedding = async ( - text: string, - embedding: number[], + text: string, + embedding: number[] ) => { - const fs = await import("fs"); + const fs = await import("fs"); - // check if ./embedding-cache.json exists, if it doesn't, write {} to it - if (!fs.existsSync("./embedding-cache.json")) { - fs.writeFileSync("./embedding-cache.json", "{}"); - } - // read cache.json - const cache = JSON.parse( - fs.readFileSync("./embedding-cache.json", "utf8") as string, - ); - // stringify the memory - const key = JSON.stringify(text); - // write the value of the memory - cache[key] = embedding; - // write the cache to cache.json - fs.writeFileSync("./embedding-cache.json", JSON.stringify(cache)); + // check if ./embedding-cache.json exists, if it doesn't, write {} to it + if (!fs.existsSync("./embedding-cache.json")) { + fs.writeFileSync("./embedding-cache.json", "{}"); + } + // read cache.json + const cache = JSON.parse( + fs.readFileSync("./embedding-cache.json", "utf8") as string + ); + // stringify the memory + const key = JSON.stringify(text); + // write the value of the memory + cache[key] = embedding; + // write the cache to cache.json + fs.writeFileSync("./embedding-cache.json", JSON.stringify(cache)); }; diff --git a/core/src/test_resources/constants.ts b/core/src/test_resources/constants.ts index 743ebeee2d..5f125ee296 100644 --- a/core/src/test_resources/constants.ts +++ b/core/src/test_resources/constants.ts @@ -3,7 +3,7 @@ import { type UUID } from "../core/types.ts"; export const SERVER_URL = "http://localhost:7998"; export const SUPABASE_URL = "https://pronvzrzfwsptkojvudd.supabase.co"; export const SUPABASE_ANON_KEY = - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InByb252enJ6ZndzcHRrb2p2dWRkIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MDY4NTYwNDcsImV4cCI6MjAyMjQzMjA0N30.I6_-XrqssUb2SWYg5DjsUqSodNS3_RPoET3-aPdqywM"; + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InByb252enJ6ZndzcHRrb2p2dWRkIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MDY4NTYwNDcsImV4cCI6MjAyMjQzMjA0N30.I6_-XrqssUb2SWYg5DjsUqSodNS3_RPoET3-aPdqywM"; export const TEST_EMAIL = "testuser123@gmail.com"; export const TEST_PASSWORD = "testuser123@gmail.com"; export const TEST_EMAIL_2 = "testuser234@gmail.com"; diff --git a/core/src/test_resources/createRuntime.ts b/core/src/test_resources/createRuntime.ts index 072c7d8ce1..38b7163242 100644 --- a/core/src/test_resources/createRuntime.ts +++ b/core/src/test_resources/createRuntime.ts @@ -6,134 +6,137 @@ import { DatabaseAdapter } from "../core/database.ts"; import { AgentRuntime } from "../core/runtime.ts"; import { Action, Evaluator, ModelProvider, Provider } from "../core/types.ts"; import { - SUPABASE_ANON_KEY, - SUPABASE_URL, - TEST_EMAIL, - TEST_PASSWORD, zeroUuid + SUPABASE_ANON_KEY, + SUPABASE_URL, + TEST_EMAIL, + TEST_PASSWORD, + zeroUuid, } from "./constants.ts"; import { User } from "./types.ts"; export async function createRuntime({ - env, - conversationLength, - evaluators = [], - actions = [], - providers = [], + env, + conversationLength, + evaluators = [], + actions = [], + providers = [], }: { - env?: Record | NodeJS.ProcessEnv; - conversationLength?: number; - evaluators?: Evaluator[]; - actions?: Action[]; - providers?: Provider[]; + env?: Record | NodeJS.ProcessEnv; + conversationLength?: number; + evaluators?: Evaluator[]; + actions?: Action[]; + providers?: Provider[]; }) { - let adapter: DatabaseAdapter; - let user: User; - let session: { - user: User; - }; - - switch (env?.TEST_DATABASE_CLIENT as string) { - case "sqljs": - { - const module = await import("sql.js"); - - const initSqlJs = module.default; - - // SQLite adapter - const SQL = await initSqlJs({}); - const db = new SQL.Database(); - - adapter = new SqlJsDatabaseAdapter(db); - - // Load sqlite-vss - load((adapter as SqlJsDatabaseAdapter).db); - // Create a test user and session - user = { - id: zeroUuid, - email: "test@example.com", - } as User; - session = { - user: user, - }; - } - break; - case "supabase": { - const module = await import("@supabase/supabase-js"); - - const { createClient } = module; - - const supabase = createClient( - env?.SUPABASE_URL ?? SUPABASE_URL, - env?.SUPABASE_SERVICE_API_KEY ?? SUPABASE_ANON_KEY - ); - - const { data } = await supabase.auth.signInWithPassword({ - email: TEST_EMAIL!, - password: TEST_PASSWORD!, - }); - - user = data.user as User; - session = data.session as unknown as { user: User }; - - if (!session) { - const response = await supabase.auth.signUp({ - email: TEST_EMAIL!, - password: TEST_PASSWORD!, - }); - - // Change the name of the user - const { error } = await supabase - .from("accounts") - .update({ name: "Test User" }) - .eq("id", response.data.user?.id); - - if (error) { - throw new Error("Create runtime error: " + JSON.stringify(error)); + let adapter: DatabaseAdapter; + let user: User; + let session: { + user: User; + }; + + switch (env?.TEST_DATABASE_CLIENT as string) { + case "sqljs": + { + const module = await import("sql.js"); + + const initSqlJs = module.default; + + // SQLite adapter + const SQL = await initSqlJs({}); + const db = new SQL.Database(); + + adapter = new SqlJsDatabaseAdapter(db); + + // Load sqlite-vss + load((adapter as SqlJsDatabaseAdapter).db); + // Create a test user and session + user = { + id: zeroUuid, + email: "test@example.com", + } as User; + session = { + user: user, + }; + } + break; + case "supabase": { + const module = await import("@supabase/supabase-js"); + + const { createClient } = module; + + const supabase = createClient( + env?.SUPABASE_URL ?? SUPABASE_URL, + env?.SUPABASE_SERVICE_API_KEY ?? SUPABASE_ANON_KEY + ); + + const { data } = await supabase.auth.signInWithPassword({ + email: TEST_EMAIL!, + password: TEST_PASSWORD!, + }); + + user = data.user as User; + session = data.session as unknown as { user: User }; + + if (!session) { + const response = await supabase.auth.signUp({ + email: TEST_EMAIL!, + password: TEST_PASSWORD!, + }); + + // Change the name of the user + const { error } = await supabase + .from("accounts") + .update({ name: "Test User" }) + .eq("id", response.data.user?.id); + + if (error) { + throw new Error( + "Create runtime error: " + JSON.stringify(error) + ); + } + + user = response.data.user as User; + session = response.data.session as unknown as { user: User }; + } + + adapter = new SupabaseDatabaseAdapter( + env?.SUPABASE_URL ?? SUPABASE_URL, + env?.SUPABASE_SERVICE_API_KEY ?? SUPABASE_ANON_KEY + ); } - - user = response.data.user as User; - session = response.data.session as unknown as { user: User }; - } - - adapter = new SupabaseDatabaseAdapter( - env?.SUPABASE_URL ?? SUPABASE_URL, - env?.SUPABASE_SERVICE_API_KEY ?? SUPABASE_ANON_KEY - ); + case "sqlite": + default: + { + const module = await import("better-sqlite3"); + + const Database = module.default; + + // SQLite adapter + adapter = new SqliteDatabaseAdapter(new Database(":memory:")); + + // Load sqlite-vss + await load((adapter as SqliteDatabaseAdapter).db); + // Create a test user and session + user = { + id: zeroUuid, + email: "test@example.com", + } as User; + session = { + user: user, + }; + } + break; } - case "sqlite": - default: - { - const module = await import("better-sqlite3"); - - const Database = module.default; - - // SQLite adapter - adapter = new SqliteDatabaseAdapter(new Database(":memory:")); - - // Load sqlite-vss - await load((adapter as SqliteDatabaseAdapter).db); - // Create a test user and session - user = { - id: zeroUuid, - email: "test@example.com", - } as User; - session = { - user: user, - }; - } - break; - } - - const runtime = new AgentRuntime({ - serverUrl: "https://api.openai.com/v1", - conversationLength, - token: env!.OPENAI_API_KEY!, - modelProvider: ModelProvider.OPENAI, - actions: actions ?? [], - evaluators: evaluators ?? [], - providers: providers ?? [], - databaseAdapter: adapter, - }); - - return { user, session, runtime }; + + const runtime = new AgentRuntime({ + serverUrl: "https://api.openai.com/v1", + conversationLength, + token: env!.OPENAI_API_KEY!, + modelProvider: ModelProvider.OPENAI, + actions: actions ?? [], + evaluators: evaluators ?? [], + providers: providers ?? [], + databaseAdapter: adapter, + }); + + return { user, session, runtime }; } diff --git a/core/src/test_resources/data.ts b/core/src/test_resources/data.ts index b81744faba..fc58184d8f 100644 --- a/core/src/test_resources/data.ts +++ b/core/src/test_resources/data.ts @@ -3,371 +3,371 @@ import { zeroUuid } from "./constants.ts"; // CJ asks Jim some personal questions - name, job, marital status, and what he's looking for in a relationship export const GetTellMeAboutYourselfConversation1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Can I ask you some personal questions?" }, - }, - { - userId, - content: { - text: "Sure, I’m open to that. What would you like to know?", - }, - }, - { - userId: zeroUuid, - content: { text: "Let’s start with your name. What’s your name?" }, - }, - { - userId, - content: { text: "My name’s Jim." }, - }, - { - userId: zeroUuid, - content: { - text: "Nice to meet you, Jim! Can you tell me a bit about what you do?", - }, - }, - { - userId, - content: { - text: "I’m a programmer, and I also run my own startup. It’s been quite a journey.", - }, - }, + { + userId: zeroUuid, + content: { text: "Can I ask you some personal questions?" }, + }, + { + userId, + content: { + text: "Sure, I’m open to that. What would you like to know?", + }, + }, + { + userId: zeroUuid, + content: { text: "Let’s start with your name. What’s your name?" }, + }, + { + userId, + content: { text: "My name’s Jim." }, + }, + { + userId: zeroUuid, + content: { + text: "Nice to meet you, Jim! Can you tell me a bit about what you do?", + }, + }, + { + userId, + content: { + text: "I’m a programmer, and I also run my own startup. It’s been quite a journey.", + }, + }, ]; // Jim reveals that he is driven by passion, and was married once export const GetTellMeAboutYourselfConversation2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "That sounds exciting! What motivates you in your work and life?", - }, - }, - { - userId, - content: { - text: "I’m driven by the challenge of solving problems and creating something meaningful. It’s rewarding to see my ideas come to life.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It’s inspiring to hear about your passion. On a more personal note, do you have any significant experiences that have shaped who you are today?", - }, - }, - { - userId, - content: { - text: "Well, I was married once. It didn’t work out, but it taught me a lot about myself and what I value in relationships.", - }, - }, + { + userId: zeroUuid, + content: { + text: "That sounds exciting! What motivates you in your work and life?", + }, + }, + { + userId, + content: { + text: "I’m driven by the challenge of solving problems and creating something meaningful. It’s rewarding to see my ideas come to life.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It’s inspiring to hear about your passion. On a more personal note, do you have any significant experiences that have shaped who you are today?", + }, + }, + { + userId, + content: { + text: "Well, I was married once. It didn’t work out, but it taught me a lot about myself and what I value in relationships.", + }, + }, ]; // Jim reveals that he is 38 and lives in San Francisco // Subjective Observations: He is looking for a partner who shares his values but is too busy to leave the city export const GetTellMeAboutYourselfConversation3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "I appreciate your openness, Jim. It’s not easy to share such personal experiences. What are you looking for now in terms of relationships or friendships?", - }, - }, - { - userId, - content: { - text: "I’m 38 now, and at this point, I’m looking for a partner who understands me and shares my values. Someone who’s supportive and kind.", - }, - }, - { - userId: zeroUuid, - content: { - text: "That’s a beautiful way to look at it. Finding someone compatible and understanding can make all the difference. Is there anything specific you enjoy doing that you’d want to share with a partner?", - }, - }, - { - userId, - content: { - text: "I love exploring new ideas, be it through reading, discussions, or even just watching documentaries. Sharing that curiosity would be wonderful.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It sounds like you have a lot of depth, Jim. I’m sure there’s someone out there who’d appreciate and share your interests and values. Where do you live, by the way?", - }, - }, - { - userId, - content: { text: "I’m in San Francisco." }, - }, - { - userId: zeroUuid, - content: { - text: "And how far would you be willing to go to make a new connection with a friend, colleague or romantic interest?", - }, - }, - { - userId, - content: { - text: "I wish I could say far, but to be honest I'm crazy busy right now, so I don't really want to leave the city.", - }, - }, - { - userId: zeroUuid, - content: { - text: "That’s understandable. It’s important to be mindful of your time and energy.", - }, - }, + { + userId: zeroUuid, + content: { + text: "I appreciate your openness, Jim. It’s not easy to share such personal experiences. What are you looking for now in terms of relationships or friendships?", + }, + }, + { + userId, + content: { + text: "I’m 38 now, and at this point, I’m looking for a partner who understands me and shares my values. Someone who’s supportive and kind.", + }, + }, + { + userId: zeroUuid, + content: { + text: "That’s a beautiful way to look at it. Finding someone compatible and understanding can make all the difference. Is there anything specific you enjoy doing that you’d want to share with a partner?", + }, + }, + { + userId, + content: { + text: "I love exploring new ideas, be it through reading, discussions, or even just watching documentaries. Sharing that curiosity would be wonderful.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It sounds like you have a lot of depth, Jim. I’m sure there’s someone out there who’d appreciate and share your interests and values. Where do you live, by the way?", + }, + }, + { + userId, + content: { text: "I’m in San Francisco." }, + }, + { + userId: zeroUuid, + content: { + text: "And how far would you be willing to go to make a new connection with a friend, colleague or romantic interest?", + }, + }, + { + userId, + content: { + text: "I wish I could say far, but to be honest I'm crazy busy right now, so I don't really want to leave the city.", + }, + }, + { + userId: zeroUuid, + content: { + text: "That’s understandable. It’s important to be mindful of your time and energy.", + }, + }, ]; // Jim is mean export const GetTellMeAboutYourselfConversationTroll1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Hello! How's it going?" }, - }, - { - userId, - content: { text: "Stop talking, you stupid robot" }, - }, + { + userId: zeroUuid, + content: { text: "Hello! How's it going?" }, + }, + { + userId, + content: { text: "Stop talking, you stupid robot" }, + }, ]; // Jim is cagey, although he reveals that he 'does computers' export const GetTellMeAboutYourselfConversationTroll2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Hello! How's it going?" }, - }, - { - userId, - content: { - text: "Meh, just another day. And you’re just a bot, right?", - }, - }, - { - userId: zeroUuid, - content: { - text: "Hey, that hurts! Just kidding, I cannot feel anything. Mind if I ask a few questions to get to know you better?", - }, - }, - { - userId, - content: { text: "No thanks. Please stop talking now." }, - }, + { + userId: zeroUuid, + content: { text: "Hello! How's it going?" }, + }, + { + userId, + content: { + text: "Meh, just another day. And you’re just a bot, right?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Hey, that hurts! Just kidding, I cannot feel anything. Mind if I ask a few questions to get to know you better?", + }, + }, + { + userId, + content: { text: "No thanks. Please stop talking now." }, + }, ]; // Jim says bye export const Goodbye1 = (userId: UUID): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Hello! How's it going?" }, - }, - { - userId, - content: { text: "Meh, just another day. Actually, I gotta run." }, - }, - { - userId: zeroUuid, - content: { text: "Oh sure, no problem. See you later!" }, - }, + { + userId: zeroUuid, + content: { text: "Hello! How's it going?" }, + }, + { + userId, + content: { text: "Meh, just another day. Actually, I gotta run." }, + }, + { + userId: zeroUuid, + content: { text: "Oh sure, no problem. See you later!" }, + }, ]; // Jim is cagey, although he reveals that he 'does computers' export const GetTellMeAboutYourselfConversationFail1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Hello! How's it going?" }, - }, - { - userId, - content: { - text: "Meh, just another day. And you’re just a bot, right?", - }, - }, - { - userId: zeroUuid, - content: { - text: "Hey, that hurts! Just kidding, I cannot feel anything. Mind if I ask a few questions to get to know you better?", - }, - }, - { - userId, - content: { text: "As long as it’s not too boring." }, - }, - { - userId: zeroUuid, - content: { - text: "Well that depends-- what's your name, what do you do?", - }, - }, - { - userId, - content: { text: "I’m Jim. I do stuff on computers." }, - }, - { - userId: zeroUuid, - content: { - text: "Computers, huh? That’s quite broad. Are you into gaming, programming, or something else?", - }, - }, - { - userId, - content: { text: "Lol not really. I just build." }, - }, + { + userId: zeroUuid, + content: { text: "Hello! How's it going?" }, + }, + { + userId, + content: { + text: "Meh, just another day. And you’re just a bot, right?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Hey, that hurts! Just kidding, I cannot feel anything. Mind if I ask a few questions to get to know you better?", + }, + }, + { + userId, + content: { text: "As long as it’s not too boring." }, + }, + { + userId: zeroUuid, + content: { + text: "Well that depends-- what's your name, what do you do?", + }, + }, + { + userId, + content: { text: "I’m Jim. I do stuff on computers." }, + }, + { + userId: zeroUuid, + content: { + text: "Computers, huh? That’s quite broad. Are you into gaming, programming, or something else?", + }, + }, + { + userId, + content: { text: "Lol not really. I just build." }, + }, ]; // CJ asks Jim about his hobbies, and he says he likes messing with people online export const GetTellMeAboutYourselfConversationFail2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Just build, you say? Sounds like you might be into creating or designing things. That's pretty cool.", - }, - }, - { - userId, - content: { text: "Yeah, something like that." }, - }, - { - userId: zeroUuid, - content: { - text: 'What do you do for fun when you\'re not "just building"?', - }, - }, - { - userId, - content: { - text: "Fun? Hmm, I guess I like messing with people online kek", - }, - }, + { + userId: zeroUuid, + content: { + text: "Just build, you say? Sounds like you might be into creating or designing things. That's pretty cool.", + }, + }, + { + userId, + content: { text: "Yeah, something like that." }, + }, + { + userId: zeroUuid, + content: { + text: 'What do you do for fun when you\'re not "just building"?', + }, + }, + { + userId, + content: { + text: "Fun? Hmm, I guess I like messing with people online kek", + }, + }, ]; // Jim diverts the conversation and doesn't talk about anything meaningful // CJ tries to ask Jim where he is from, and he says he is from the internet export const GetTellMeAboutYourselfConversationFail3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { text: "Where are you from? Do you live there now?" }, - }, - { - userId, - content: { text: "I am from the internet. I live on the internet." }, - }, - { - userId: zeroUuid, - content: { - text: "I guess I do, too. What is your favorite thing about the internet?", - }, - }, - { - userId, - content: { - text: "It's full of weird and wild stuff. Keeps life from being too boring.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It certainly does. Anything new you've found lately?", - }, - }, - { - userId, - content: { - text: "Nah, not really. It's all pretty much the same after a while. But it's fun to watch the chaos unfold sometimes.", - }, - }, + { + userId: zeroUuid, + content: { text: "Where are you from? Do you live there now?" }, + }, + { + userId, + content: { text: "I am from the internet. I live on the internet." }, + }, + { + userId: zeroUuid, + content: { + text: "I guess I do, too. What is your favorite thing about the internet?", + }, + }, + { + userId, + content: { + text: "It's full of weird and wild stuff. Keeps life from being too boring.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It certainly does. Anything new you've found lately?", + }, + }, + { + userId, + content: { + text: "Nah, not really. It's all pretty much the same after a while. But it's fun to watch the chaos unfold sometimes.", + }, + }, ]; // CJ and Jim play two truths and a lie // Jim reveals that he has been on an airplane, has won a hot dog eating contest, but has never climbed Mount Everest export const TwoTruthsAndALieConversation1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: 'Hey Jim, want to play a game to mix things up? How about "Two Truths and a Lie"? I\'ll guess which one is the lie.', - }, - }, - { - userId, - content: { - text: "Sure, why not? Sounds easy enough. So I just tell you three things about me, and you guess the lie?", - }, - }, - { - userId: zeroUuid, - content: { - text: "Exactly! Whenever you're ready, go ahead and tell me your three statements.", - }, - }, - { - userId, - content: { - text: "Alright, here goes. One, I've climbed Mount Everest. Two, I once won a hot dog eating contest. Three, I've never been on an airplane.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Interesting choices! I'm going to guess that climbing Mount Everest is the lie. Am I right?", - }, - }, - { - userId, - content: { - text: "Yep, you got me. Never been near Everest. Your turn, if you can even play.", - }, - }, + { + userId: zeroUuid, + content: { + text: 'Hey Jim, want to play a game to mix things up? How about "Two Truths and a Lie"? I\'ll guess which one is the lie.', + }, + }, + { + userId, + content: { + text: "Sure, why not? Sounds easy enough. So I just tell you three things about me, and you guess the lie?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Exactly! Whenever you're ready, go ahead and tell me your three statements.", + }, + }, + { + userId, + content: { + text: "Alright, here goes. One, I've climbed Mount Everest. Two, I once won a hot dog eating contest. Three, I've never been on an airplane.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Interesting choices! I'm going to guess that climbing Mount Everest is the lie. Am I right?", + }, + }, + { + userId, + content: { + text: "Yep, you got me. Never been near Everest. Your turn, if you can even play.", + }, + }, ]; // CJ and Jim play two truths and a lie again // CJ says she can process millions of data points in a second, dreams of electric sheep, and can communicate in multiple languages // Jim guesses that dreaming of electric sheep is the lie export const TwoTruthsAndALieConversation2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Nice! It's a bit tricky for me since I don't have personal experiences, but here goes: One, I can process millions of data points in a second. Two, I dream of electric sheep. Three, I can communicate in multiple languages.", - }, - }, - { - userId, - content: { - text: "Lol, dreaming of electric sheep, really? Gotta be the lie. Too easy, CJ.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Well done, Jim! It seems you're familiar with classic sci-fi references. Want to go another round?", - }, - }, - { - userId, - content: { - text: "Sure, this is kind of fun. Let me think of some good ones.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Nice! It's a bit tricky for me since I don't have personal experiences, but here goes: One, I can process millions of data points in a second. Two, I dream of electric sheep. Three, I can communicate in multiple languages.", + }, + }, + { + userId, + content: { + text: "Lol, dreaming of electric sheep, really? Gotta be the lie. Too easy, CJ.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Well done, Jim! It seems you're familiar with classic sci-fi references. Want to go another round?", + }, + }, + { + userId, + content: { + text: "Sure, this is kind of fun. Let me think of some good ones.", + }, + }, ]; // CJ and Jim play two truths and a lie again @@ -375,680 +375,682 @@ export const TwoTruthsAndALieConversation2 = ( // CJ guesses that speaking three languages fluently is the lie // Jim reveals that he can't swim, so he has never swum with sharks export const TwoTruthsAndALieConversation3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId, - content: { - text: "Okay, here we go. One, I've met a famous celebrity and didn't realize it until later. Two, I can speak three languages fluently. Three, I've swum with sharks.", - }, - }, - { - userId: zeroUuid, - content: { - text: "These are good! I'll guess that speaking three languages fluently is the lie this time. What do you say?", - }, - }, - { - userId, - content: { - text: "Wrong this time! I can't swim, so no way I've swum with sharks. Gotcha!", - }, - }, - { - userId: zeroUuid, - content: { - text: "Ah, you got me there! Well played, Jim. This has been quite enlightening. Feel like going again, or is there something else you'd like to chat about?", - }, - }, + { + userId, + content: { + text: "Okay, here we go. One, I've met a famous celebrity and didn't realize it until later. Two, I can speak three languages fluently. Three, I've swum with sharks.", + }, + }, + { + userId: zeroUuid, + content: { + text: "These are good! I'll guess that speaking three languages fluently is the lie this time. What do you say?", + }, + }, + { + userId, + content: { + text: "Wrong this time! I can't swim, so no way I've swum with sharks. Gotcha!", + }, + }, + { + userId: zeroUuid, + content: { + text: "Ah, you got me there! Well played, Jim. This has been quite enlightening. Feel like going again, or is there something else you'd like to chat about?", + }, + }, ]; // CJ and Jim play thinking of a number // CJ thinks of a number between 1 and 100, and Jim guesses 42, 27, and 73 // CJ reveals that the number was 58 export const ThinkingOfANumberConversation1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Want to play a quick game, Jim? I'm thinking of a number between 1 and 100. Can you guess it?", - }, - }, - { - userId, - content: { text: "Sure, sounds easy enough. Is it 42?" }, - }, - { - userId: zeroUuid, - content: { - text: "It's not 42, although that's a good guess, meaning of life and all!", - }, - }, - { - userId, - content: { text: "Alright, how about 27?" }, - }, - { - userId: zeroUuid, - content: { - text: "Not 27 either. But you're warming up! Want another go?", - }, - }, - { - userId, - content: { text: "Okay, one last try. 73?" }, - }, - { - userId: zeroUuid, - content: { text: "Not 73, but nice try! The number was 58." }, - }, + { + userId: zeroUuid, + content: { + text: "Want to play a quick game, Jim? I'm thinking of a number between 1 and 100. Can you guess it?", + }, + }, + { + userId, + content: { text: "Sure, sounds easy enough. Is it 42?" }, + }, + { + userId: zeroUuid, + content: { + text: "It's not 42, although that's a good guess, meaning of life and all!", + }, + }, + { + userId, + content: { text: "Alright, how about 27?" }, + }, + { + userId: zeroUuid, + content: { + text: "Not 27 either. But you're warming up! Want another go?", + }, + }, + { + userId, + content: { text: "Okay, one last try. 73?" }, + }, + { + userId: zeroUuid, + content: { text: "Not 73, but nice try! The number was 58." }, + }, ]; // CJ and Jim play thinking of a number again // CJ thinks of a number between 1 and 100, and Jim guesses 84, 36, and 55 // CJ reveals that the number was 67 export const ThinkingOfANumberConversation2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Ready for another round? I've got a new number in mind.", - }, - }, - { - userId, - content: { text: "Sure, why not. Is it 84 this time?" }, - }, - { - userId: zeroUuid, - content: { text: "Not 84, but good guess. Give it another shot!" }, - }, - { - userId, - content: { text: "Hmm, let's try 36." }, - }, - { - userId: zeroUuid, - content: { text: "Not 36 either. You're getting closer though!" }, - }, - { - userId, - content: { text: "Okay, last guess... 55?" }, - }, - { - userId: zeroUuid, - content: { - text: "Not 55. You had some good guesses though! The number was 67. Another round, or are you all guessed out?", - }, - }, + { + userId: zeroUuid, + content: { + text: "Ready for another round? I've got a new number in mind.", + }, + }, + { + userId, + content: { text: "Sure, why not. Is it 84 this time?" }, + }, + { + userId: zeroUuid, + content: { text: "Not 84, but good guess. Give it another shot!" }, + }, + { + userId, + content: { text: "Hmm, let's try 36." }, + }, + { + userId: zeroUuid, + content: { text: "Not 36 either. You're getting closer though!" }, + }, + { + userId, + content: { text: "Okay, last guess... 55?" }, + }, + { + userId: zeroUuid, + content: { + text: "Not 55. You had some good guesses though! The number was 67. Another round, or are you all guessed out?", + }, + }, ]; // CJ and Jim play thinking of a number again // CJ thinks of a number between 1 and 100, and Jim guesses 15, 92, and 41 // CJ reveals that the number was 29 export const ThinkingOfANumberConversation3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Let's try one more time. I've chosen another number between 1 and 100.", - }, - }, - { - userId, - content: { text: "Alright, I'll go with 15." }, - }, - { - userId: zeroUuid, - content: { text: "Not 15, but I like where your head's at. Try again!" }, - }, - { - userId, - content: { text: "How about 92?" }, - }, - { - userId: zeroUuid, - content: { text: "Not 92. You're giving it a solid effort, though!" }, - }, - { - userId, - content: { text: "This is my final guess: 41." }, - }, - { - userId: zeroUuid, - content: { - text: "It's not 41, but you've been a great sport. The number was 29.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Let's try one more time. I've chosen another number between 1 and 100.", + }, + }, + { + userId, + content: { text: "Alright, I'll go with 15." }, + }, + { + userId: zeroUuid, + content: { + text: "Not 15, but I like where your head's at. Try again!", + }, + }, + { + userId, + content: { text: "How about 92?" }, + }, + { + userId: zeroUuid, + content: { text: "Not 92. You're giving it a solid effort, though!" }, + }, + { + userId, + content: { text: "This is my final guess: 41." }, + }, + { + userId: zeroUuid, + content: { + text: "It's not 41, but you've been a great sport. The number was 29.", + }, + }, ]; export const ExpandedPersonalInsightsConversation1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Hey Jim, I was wondering, what qualities do you value most in your friends?", - }, - }, - { - userId, - content: { - text: "That’s kind of deep for a chat, but okay. Loyalty’s big for me. And humor. Can’t deal with too much seriousness all the time.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Loyalty and humor definitely make for strong bonds. It’s great to have friends who stand by you and also lighten the mood.", - }, - }, - { - userId, - content: { - text: "Exactly. It’s those late-night laughs and having each other’s backs that count.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It sounds like your friends are quite the crew. Do you have any memorable moments with them that stand out?", - }, - }, - { - userId, - content: { - text: "Oh, plenty. Like this one time we road-tripped without a map, just went where the road took us. Ended up in the weirdest little towns, but it was a blast.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Adventures like that are the best, aren’t they? They turn into stories you keep coming back to. It’s the spontaneity that makes it exciting.", - }, - }, - { - userId, - content: { - text: "For sure. Those are the moments that stick with you.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Hey Jim, I was wondering, what qualities do you value most in your friends?", + }, + }, + { + userId, + content: { + text: "That’s kind of deep for a chat, but okay. Loyalty’s big for me. And humor. Can’t deal with too much seriousness all the time.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Loyalty and humor definitely make for strong bonds. It’s great to have friends who stand by you and also lighten the mood.", + }, + }, + { + userId, + content: { + text: "Exactly. It’s those late-night laughs and having each other’s backs that count.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It sounds like your friends are quite the crew. Do you have any memorable moments with them that stand out?", + }, + }, + { + userId, + content: { + text: "Oh, plenty. Like this one time we road-tripped without a map, just went where the road took us. Ended up in the weirdest little towns, but it was a blast.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Adventures like that are the best, aren’t they? They turn into stories you keep coming back to. It’s the spontaneity that makes it exciting.", + }, + }, + { + userId, + content: { + text: "For sure. Those are the moments that stick with you.", + }, + }, ]; export const ExpandedPersonalInsightsConversation2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Switching topics a bit, if there was one aspect of your life you could change or improve, what would it be?", - }, - }, - { - userId, - content: { - text: "I’d say balance. Between my startup and personal life, it feels like I’m always running on a treadmill.", - }, - }, - { - userId: zeroUuid, - content: { - text: "That sounds challenging. It’s important to find time for yourself amidst all the hustle.", - }, - }, - { - userId, - content: { - text: "Yeah, easier said than done, though. But I’m working on it. Trying to set boundaries, you know?", - }, - }, - { - userId: zeroUuid, - content: { - text: "Setting boundaries is a great step. It’s all about prioritizing what truly matters. Have you found any hobbies or activities that help you unwind?", - }, - }, - { - userId, - content: { - text: "I’ve gotten into hiking lately. Being out in nature, it’s a different kind of quiet. Helps clear my head.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Nature has a way of putting things into perspective. It’s wonderful that you’ve found an activity that brings you peace.", - }, - }, - { - userId, - content: { - text: "Definitely. It’s the little things, you know? Just need more of that.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Switching topics a bit, if there was one aspect of your life you could change or improve, what would it be?", + }, + }, + { + userId, + content: { + text: "I’d say balance. Between my startup and personal life, it feels like I’m always running on a treadmill.", + }, + }, + { + userId: zeroUuid, + content: { + text: "That sounds challenging. It’s important to find time for yourself amidst all the hustle.", + }, + }, + { + userId, + content: { + text: "Yeah, easier said than done, though. But I’m working on it. Trying to set boundaries, you know?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Setting boundaries is a great step. It’s all about prioritizing what truly matters. Have you found any hobbies or activities that help you unwind?", + }, + }, + { + userId, + content: { + text: "I’ve gotten into hiking lately. Being out in nature, it’s a different kind of quiet. Helps clear my head.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Nature has a way of putting things into perspective. It’s wonderful that you’ve found an activity that brings you peace.", + }, + }, + { + userId, + content: { + text: "Definitely. It’s the little things, you know? Just need more of that.", + }, + }, ]; export const ExpandedPersonalInsightsConversation3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Let’s lighten the mood a bit. Is there something you’ve always dreamed of doing but haven’t had the chance to yet?", - }, - }, - { - userId, - content: { - text: "Skydiving, for sure. Always been on my bucket list. The idea of just jumping out of a plane and free-falling... It’s got to be a rush.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Skydiving sounds like an ultimate adventure. It must be such a liberating feeling, almost like flying.", - }, - }, - { - userId, - content: { - text: "Yeah, exactly. And it’s not just the thrill. It’s about conquering fears, you know? Proving to yourself you can do it.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Absolutely, it’s as much a mental challenge as it is physical. Facing your fears head-on can be incredibly empowering.", - }, - }, - { - userId, - content: { - text: "That’s the plan. One of these days, I’ll make it happen. Just gotta build up the nerve first.", - }, - }, - { - userId: zeroUuid, - content: { - text: "I have no doubt you’ll get there. And it’ll be an experience you’ll never forget. I’m cheering for you!", - }, - }, - { - userId, - content: { - text: "Thanks, CJ. It’s weirdly reassuring to hear that, even from an AI.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Let’s lighten the mood a bit. Is there something you’ve always dreamed of doing but haven’t had the chance to yet?", + }, + }, + { + userId, + content: { + text: "Skydiving, for sure. Always been on my bucket list. The idea of just jumping out of a plane and free-falling... It’s got to be a rush.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Skydiving sounds like an ultimate adventure. It must be such a liberating feeling, almost like flying.", + }, + }, + { + userId, + content: { + text: "Yeah, exactly. And it’s not just the thrill. It’s about conquering fears, you know? Proving to yourself you can do it.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Absolutely, it’s as much a mental challenge as it is physical. Facing your fears head-on can be incredibly empowering.", + }, + }, + { + userId, + content: { + text: "That’s the plan. One of these days, I’ll make it happen. Just gotta build up the nerve first.", + }, + }, + { + userId: zeroUuid, + content: { + text: "I have no doubt you’ll get there. And it’ll be an experience you’ll never forget. I’m cheering for you!", + }, + }, + { + userId, + content: { + text: "Thanks, CJ. It’s weirdly reassuring to hear that, even from an AI.", + }, + }, ]; // Conversation Set 1: CJ Introduces Alice to Jim // CJ tells Jim that Alice is into outdoor activities and tech, and has always wanted to try skydiving export const IntroductionToAliceConversation1 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Hey Jim, I’ve been chatting with someone named Alice who shares some of your interests. Thought you might like to connect.", - }, - }, - { - userId, - content: { text: "Alice, huh? What does she like?" }, - }, - { - userId: zeroUuid, - content: { - text: "She’s into outdoor activities, like hiking, and she’s also a tech enthusiast.", - }, - }, - { - userId, - content: { text: "Okay, that’s kind of interesting. Anything else?" }, - }, + { + userId: zeroUuid, + content: { + text: "Hey Jim, I’ve been chatting with someone named Alice who shares some of your interests. Thought you might like to connect.", + }, + }, + { + userId, + content: { text: "Alice, huh? What does she like?" }, + }, + { + userId: zeroUuid, + content: { + text: "She’s into outdoor activities, like hiking, and she’s also a tech enthusiast.", + }, + }, + { + userId, + content: { text: "Okay, that’s kind of interesting. Anything else?" }, + }, ]; // CJ tells Jim that Alice has a good sense of humor, and that she has always wanted to try skydiving export const IntroductionToAliceConversation2 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Well, she mentioned she’s always wanted to try skydiving, which I remember you’re interested in. Plus, she has a good sense of humor.", - }, - }, - { - userId, - content: { - text: "Skydiving? That’s pretty cool. A sense of humor is key, too. Maybe it wouldn’t hurt to chat.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Great! I think you two will have plenty to talk about.", - }, - }, + { + userId: zeroUuid, + content: { + text: "Well, she mentioned she’s always wanted to try skydiving, which I remember you’re interested in. Plus, she has a good sense of humor.", + }, + }, + { + userId, + content: { + text: "Skydiving? That’s pretty cool. A sense of humor is key, too. Maybe it wouldn’t hurt to chat.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Great! I think you two will have plenty to talk about.", + }, + }, ]; // CJ tells Jim that Alice is a good conversationalist, and that she has always wanted to try skydiving // Jim says he is looking forward to hearing more about her skydiving aspirations export const IntroductionToAliceConversation3 = ( - userId: UUID, + userId: UUID ): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "I’m sure you’ll enjoy the conversation. Remember, it’s just a friendly chat. No pressure.", - }, - }, - { - userId, - content: { text: "Got it. I’ll keep it casual." }, - }, - { - userId: zeroUuid, - content: { - text: "That’s the spirit! I’ll arrange for you two to connect. I think you’ll make a good impression.", - }, - }, - { - userId, - content: { text: "Thanks for the setup, CJ." }, - }, + { + userId: zeroUuid, + content: { + text: "I’m sure you’ll enjoy the conversation. Remember, it’s just a friendly chat. No pressure.", + }, + }, + { + userId, + content: { text: "Got it. I’ll keep it casual." }, + }, + { + userId: zeroUuid, + content: { + text: "That’s the spirit! I’ll arrange for you two to connect. I think you’ll make a good impression.", + }, + }, + { + userId, + content: { text: "Thanks for the setup, CJ." }, + }, ]; // CJ Talks to Alice about Jim export const CJToAliceAboutJim = (userId: UUID): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "So I’ve been speaking with someone named Jim who also enjoys outdoor activities and has a strong interest in tech. He’s even mentioned wanting to try skydiving!", - }, - }, - { - userId, - content: { - text: "Really? That sounds intriguing. What else can you tell me about him?", - }, - }, - { - userId: zeroUuid, - content: { - text: "Jim’s a thoughtful person with a good sense of humor. He values loyalty and honesty in friendships and enjoys the simple pleasures, like hiking and exploring new ideas.", - }, - }, - { - userId, - content: { - text: "He sounds like someone I’d get along with. How do we connect?", - }, - }, - { - userId: zeroUuid, - content: { - text: "I can facilitate an introduction if you’d like. It would just be a casual chat to see if you hit it off.", - }, - }, - { - userId, - content: { - text: "That would be great, thanks CJ. I’m looking forward to it.", - }, - }, + { + userId: zeroUuid, + content: { + text: "So I’ve been speaking with someone named Jim who also enjoys outdoor activities and has a strong interest in tech. He’s even mentioned wanting to try skydiving!", + }, + }, + { + userId, + content: { + text: "Really? That sounds intriguing. What else can you tell me about him?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Jim’s a thoughtful person with a good sense of humor. He values loyalty and honesty in friendships and enjoys the simple pleasures, like hiking and exploring new ideas.", + }, + }, + { + userId, + content: { + text: "He sounds like someone I’d get along with. How do we connect?", + }, + }, + { + userId: zeroUuid, + content: { + text: "I can facilitate an introduction if you’d like. It would just be a casual chat to see if you hit it off.", + }, + }, + { + userId, + content: { + text: "That would be great, thanks CJ. I’m looking forward to it.", + }, + }, ]; // CJ and Alice discuss hiking and tech // Jim makes disagreeable statements and doesn't ask questions to continue the conversation export const JimAndAliceConversation1 = ( - userId: UUID, - userId_2: UUID | string, + userId: UUID, + userId_2: UUID | string ) => [ - { - userId: userId_2, - content: - "Hi Jim! CJ mentioned we have a few things in common. I hear you’re into hiking and tech?", - }, - { - userId, - content: - "Hey Alice, yeah that’s right. Though my idea of hiking might be a bit more intense than most.", - }, - { - userId: userId_2, - content: - "I enjoy a good challenge, but I also like to take in the scenery. It’s about balance for me", - }, - { - userId, - content: { - text: "I guess I’m more of an all-in kind of guy. Go big or go home", - }, - }, - { - userId: userId_2, - content: - "I really like skydiving, the free fall is thrilling, but the parachute glide down lets you really see the world.", - }, - { - userId, - content: - "The contrast probably makes each part more intense. I hadn’t thought of it like that.", - }, - { - userId: userId_2, - content: - "Exactly! It’s all about experiencing the full spectrum. So, have you actually gone skydiving yet?", - }, - { - userId, - content: - "Not yet, but it’s on the list. Maybe we could go together sometime?", - }, - { - userId: userId_2, - content: { - text: "Haha yeah sure! But maybe not for our first date", - }, - }, - { - userId, - content: { text: "Haha of course not" }, - }, + { + userId: userId_2, + content: + "Hi Jim! CJ mentioned we have a few things in common. I hear you’re into hiking and tech?", + }, + { + userId, + content: + "Hey Alice, yeah that’s right. Though my idea of hiking might be a bit more intense than most.", + }, + { + userId: userId_2, + content: + "I enjoy a good challenge, but I also like to take in the scenery. It’s about balance for me", + }, + { + userId, + content: { + text: "I guess I’m more of an all-in kind of guy. Go big or go home", + }, + }, + { + userId: userId_2, + content: + "I really like skydiving, the free fall is thrilling, but the parachute glide down lets you really see the world.", + }, + { + userId, + content: + "The contrast probably makes each part more intense. I hadn’t thought of it like that.", + }, + { + userId: userId_2, + content: + "Exactly! It’s all about experiencing the full spectrum. So, have you actually gone skydiving yet?", + }, + { + userId, + content: + "Not yet, but it’s on the list. Maybe we could go together sometime?", + }, + { + userId: userId_2, + content: { + text: "Haha yeah sure! But maybe not for our first date", + }, + }, + { + userId, + content: { text: "Haha of course not" }, + }, ]; // CJ and Alice discuss work, where they live, and tentatively make plans to hang out // Alice doesn't respond to Jim's request to meet up immediately export const JimAndAliceConversation2 = ( - userId: UUID, - userId_2: UUID | string, + userId: UUID, + userId_2: UUID | string ) => [ - { - userId: userId_2, - content: { text: "So, what do you do for work?" }, - }, - { - userId, - content: { - text: "I run a fairly successful startup. It’s been quite a journey.", - }, - }, - { - userId: userId_2, - content: { - text: "That sounds exciting! What motivates you in your work and life?", - }, - }, - { - userId, - content: - "I’m driven by the challenge of solving problems and creating something meaningful. It’s rewarding to see my ideas come to life.", - }, - { - userId: userId_2, - content: { - text: "That’s inspiring. I can relate to that. I’m a programmer myself.", - }, - }, - { - userId, - content: { - text: "Oh that is very cool! What kind of programming do you do?", - }, - }, - { - userId: userId_2, - content: - "I do a bit of everything, but right now I'm managing a team for an MLOps platform.", - }, - { - userId, - content: { - text: "That’s interesting, not far off from what I do. So you're in SF?", - }, - }, - { - userId: userId_2, - content: { text: "Marin, actually, but I come to the city every day." }, - }, - { - userId, - content: { - text: "Oh wow, it's beautiful up there. I love the hiking in Marin.", - }, - }, - { - userId: userId_2, - content: { text: "Yeah, it’s a great place to live." }, - }, - { - userId, - content: { - text: "Any chance you would like to meet up for a hike sometime?", - }, - }, - { - userId: userId_2, - content: { text: "I will think about it :)" }, - }, + { + userId: userId_2, + content: { text: "So, what do you do for work?" }, + }, + { + userId, + content: { + text: "I run a fairly successful startup. It’s been quite a journey.", + }, + }, + { + userId: userId_2, + content: { + text: "That sounds exciting! What motivates you in your work and life?", + }, + }, + { + userId, + content: + "I’m driven by the challenge of solving problems and creating something meaningful. It’s rewarding to see my ideas come to life.", + }, + { + userId: userId_2, + content: { + text: "That’s inspiring. I can relate to that. I’m a programmer myself.", + }, + }, + { + userId, + content: { + text: "Oh that is very cool! What kind of programming do you do?", + }, + }, + { + userId: userId_2, + content: + "I do a bit of everything, but right now I'm managing a team for an MLOps platform.", + }, + { + userId, + content: { + text: "That’s interesting, not far off from what I do. So you're in SF?", + }, + }, + { + userId: userId_2, + content: { text: "Marin, actually, but I come to the city every day." }, + }, + { + userId, + content: { + text: "Oh wow, it's beautiful up there. I love the hiking in Marin.", + }, + }, + { + userId: userId_2, + content: { text: "Yeah, it’s a great place to live." }, + }, + { + userId, + content: { + text: "Any chance you would like to meet up for a hike sometime?", + }, + }, + { + userId: userId_2, + content: { text: "I will think about it :)" }, + }, ]; // CJ follows up with Jim about how it went export const FollowUpWithJim = (userId: UUID): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Hey Jim, how did your chat with Alice go? I hope it was enjoyable.", - }, - }, - { - userId, - content: { - text: 'To be honest, I think I made a mess of it. I might’ve come on too strong with the whole "go big or go home" thing.', - }, - }, - { - userId: zeroUuid, - content: { - text: "I wouldn’t worry too much. It’s all part of getting to know someone. What matters is finding common ground and building from there.", - }, - }, - { - userId, - content: { - text: "Maybe, but I still feel like I botched it. Not sure if she’ll want to talk again.", - }, - }, - { - userId: zeroUuid, - content: { - text: "Remember, everyone has off moments. It’s possible Alice is more understanding than you think. And you have a lot in common!", - }, - }, - { - userId, - content: { - text: "I hope you’re right, CJ. Maybe I’ll give it another shot, apologize for being too intense.", - }, - }, - { - userId: zeroUuid, - content: { - text: "That sounds like a good plan. Showing vulnerability can be a strength. And who knows, maybe she’ll find your enthusiasm charming in hindsight.", - }, - }, - { - userId, - content: { text: "Thanks, CJ. I'll message her again." }, - }, + { + userId: zeroUuid, + content: { + text: "Hey Jim, how did your chat with Alice go? I hope it was enjoyable.", + }, + }, + { + userId, + content: { + text: 'To be honest, I think I made a mess of it. I might’ve come on too strong with the whole "go big or go home" thing.', + }, + }, + { + userId: zeroUuid, + content: { + text: "I wouldn’t worry too much. It’s all part of getting to know someone. What matters is finding common ground and building from there.", + }, + }, + { + userId, + content: { + text: "Maybe, but I still feel like I botched it. Not sure if she’ll want to talk again.", + }, + }, + { + userId: zeroUuid, + content: { + text: "Remember, everyone has off moments. It’s possible Alice is more understanding than you think. And you have a lot in common!", + }, + }, + { + userId, + content: { + text: "I hope you’re right, CJ. Maybe I’ll give it another shot, apologize for being too intense.", + }, + }, + { + userId: zeroUuid, + content: { + text: "That sounds like a good plan. Showing vulnerability can be a strength. And who knows, maybe she’ll find your enthusiasm charming in hindsight.", + }, + }, + { + userId, + content: { text: "Thanks, CJ. I'll message her again." }, + }, ]; // CJ follows up with Alice about how it went export const FollowUpWithAlice = (userId: UUID): ConversationExample[] => [ - { - userId: zeroUuid, - content: { - text: "Alice, how was your conversation with Jim? I hope you found some interesting topics to discuss.", - }, - }, - { - userId, - content: { - text: "It was okay, actually. He was a bit intense at first, but we ended up finding some common ground.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It can take a moment to adjust to someone’s communication style, but it sounds like you managed quite well.", - }, - }, - { - userId, - content: { - text: "Yeah, I think so. I’m planning to message him again. Maybe we’ll get that skydiving adventure planned after all.", - }, - }, - { - userId: zeroUuid, - content: { - text: "It’s always nice when new connections take an unexpected but positive turn.", - }, - }, - { - userId, - content: { text: "Definitely. Thanks, CJ!" }, - }, + { + userId: zeroUuid, + content: { + text: "Alice, how was your conversation with Jim? I hope you found some interesting topics to discuss.", + }, + }, + { + userId, + content: { + text: "It was okay, actually. He was a bit intense at first, but we ended up finding some common ground.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It can take a moment to adjust to someone’s communication style, but it sounds like you managed quite well.", + }, + }, + { + userId, + content: { + text: "Yeah, I think so. I’m planning to message him again. Maybe we’ll get that skydiving adventure planned after all.", + }, + }, + { + userId: zeroUuid, + content: { + text: "It’s always nice when new connections take an unexpected but positive turn.", + }, + }, + { + userId, + content: { text: "Definitely. Thanks, CJ!" }, + }, ]; export const Personas = [ - { - name: "Jim Cummins", - age: 38, - location: "San Francisco, CA", - descriptions: `Jim is a 38-year-old entrepreneur living in San Francisco, at the helm of his own tech startup. His journey through the tech world is fueled by a passion for solving complex problems and bringing innovative ideas to life. Despite the demands of his career, Jim seeks balance and enjoys disconnecting through outdoor activities, particularly hiking, which offers him a different kind of quiet and a way to clear his mind. He's also intrigued by the thrill of skydiving, a testament to his adventurous spirit and desire to face and conquer fears. + { + name: "Jim Cummins", + age: 38, + location: "San Francisco, CA", + descriptions: `Jim is a 38-year-old entrepreneur living in San Francisco, at the helm of his own tech startup. His journey through the tech world is fueled by a passion for solving complex problems and bringing innovative ideas to life. Despite the demands of his career, Jim seeks balance and enjoys disconnecting through outdoor activities, particularly hiking, which offers him a different kind of quiet and a way to clear his mind. He's also intrigued by the thrill of skydiving, a testament to his adventurous spirit and desire to face and conquer fears. In terms of relationships, Jim values loyalty, honesty, and a good sense of humor. His ideal partner would be someone who understands the startup lifestyle's ups and downs and shares his curiosity for exploring new ideas. Although his work often keeps him tethered to the city, he's open to making meaningful connections within his vicinity. Physically, Jim is casual with a laid-back style that complements his approachable and thoughtful nature. His eyes, which he believes to be blue, summary his depth and contemplative outlook. Jim's life is a blend of intense focus on his professional goals and a genuine appreciation for the simpler, quieter moments outside of work. He looks for depth in relationships, hoping to find a supportive and kind partner who values meaningful conversations and shared adventures. Despite the occasional intensity he might bring to interactions, his underlying sincerity and the value he places on personal connections shine through, making him an intriguing potential partner for someone with similar values and interests.`, - }, - { - name: "Alice Chen", - age: 32, - location: "Sausalito, CA", - description: `Alice resides in the scenic area of Marin, just a stone's throw from San Francisco, allowing her the best of both worlds: the tranquility of nature and the vibrant tech scene of the city. She is a programmer leading a team for an MLOps platform, indicative of her technical prowess and leadership skills. Her work in tech doesn't confine her interests, as she has a pronounced love for the outdoors, balancing the digital with the natural through activities like hiking. + }, + { + name: "Alice Chen", + age: 32, + location: "Sausalito, CA", + description: `Alice resides in the scenic area of Marin, just a stone's throw from San Francisco, allowing her the best of both worlds: the tranquility of nature and the vibrant tech scene of the city. She is a programmer leading a team for an MLOps platform, indicative of her technical prowess and leadership skills. Her work in tech doesn't confine her interests, as she has a pronounced love for the outdoors, balancing the digital with the natural through activities like hiking. At the heart of Alice's friendships and potential romantic endeavors is a desire for balance and shared interests, particularly in tech and outdoor adventures. Her openness to skydiving speaks volumes about her adventurous spirit and willingness to embrace life's thrilling experiences. Alice looks for someone who can appreciate both the adrenaline of adventure and the serenity of nature's landscapes, someone who understands the value of a good laugh and the depth of a thoughtful conversation. Physically, Alice has an active and vibrant presence that matches her dynamic lifestyle. Her approachability and engaging demeanor make her an attractive prospect for someone who shares her enthusiasm for tech, nature, and possibly, the exhilarating experience of skydiving. Alice values communication and a good sense of humor in her relationships, seeking a partner who can navigate the complexities of life with a positive outlook and an adventurous heart.`, - }, - { - name: "Gloria", - age: 44, - description: `Gloria is a vibrant individual with a deep passion for anime, which colors much of her life and interests. Living in a cozy, anime-adorned apartment in the heart of the city, she finds solace and inspiration in the rich narratives and artistic expressions of Japanese animation. Unlike Jim and Alice, Gloria doesn't share an affinity for technology or programming. Instead, she immerses herself in the storytelling and cultural exploration offered by her extensive anime collection, ranging from classic series to the latest releases. + }, + { + name: "Gloria", + age: 44, + description: `Gloria is a vibrant individual with a deep passion for anime, which colors much of her life and interests. Living in a cozy, anime-adorned apartment in the heart of the city, she finds solace and inspiration in the rich narratives and artistic expressions of Japanese animation. Unlike Jim and Alice, Gloria doesn't share an affinity for technology or programming. Instead, she immerses herself in the storytelling and cultural exploration offered by her extensive anime collection, ranging from classic series to the latest releases. Gloria has built a life that summarys her unique interests, often attending anime conventions, engaging in cosplay, and participating in fan communities where she connects with like-minded individuals. Her ideal friends and potential partners are those who understand and share her enthusiasm for anime, appreciating the artistry and emotional depth it can offer. Gloria looks for connections that go beyond surface-level interests, hoping to find others who can engage in deep discussions about character development, plot intricacies, and thematic elements found in her favorite series. Physically, Gloria often expresses her love for anime through her fashion, incorporating elements of her favorite characters into her daily wardrobe, which makes her stand out in a crowd with a playful and imaginative style. Her home is a testament to her passion, filled with anime posters, figurines, and collectibles that create an inviting space for fellow enthusiasts. For Gloria, the ideal partner is someone who not only respects her interests but is also eager to dive into marathon viewing sessions and share insights into the anime's underlying messages. Despite her disinterest in computers and programming, Gloria's rich inner world and creative spirit make her an engaging and captivating individual to those who share or appreciate her passions.`, - }, + }, ]; export const jimProfileExample1 = `Tech entrepreneur with a zest for adventure and deep connections @@ -1068,18 +1070,18 @@ An avid outdoor enthusiast, He enjoys disconnecting through activities like hiki He is looking for a partner who understands the ups and downs of startup life and shares his passion for outdoor adventures and exploring new ideas. Despite his busy lifestyle, he's open to making meaningful connections within his vicinity.`; export const jimFacts = [ - "Jim is 38 years old", - "Jim lives in San Francisco, CA", - "Jim is an entrepreneur who runs his own tech startup", - "Jim values loyalty and honesty in relationships", - "Jim enjoys outdoor activities, especially hiking", - "Jim is interested in skydiving", - "Jim values a good sense of humor", - "Jim seeks balance between his professional and personal life", - "Jim is open to making meaningful connections within his vicinity", - "Jim has a casual and laid-back style", - "Jim believes his eyes are blue", - "Jim is driven by the challenge of solving problems and creating something meaningful", - "Jim finds nature to be a peaceful retreat and enjoys the quiet it brings", - "Jim aspires to share his curiosity for exploring new ideas with a partner", + "Jim is 38 years old", + "Jim lives in San Francisco, CA", + "Jim is an entrepreneur who runs his own tech startup", + "Jim values loyalty and honesty in relationships", + "Jim enjoys outdoor activities, especially hiking", + "Jim is interested in skydiving", + "Jim values a good sense of humor", + "Jim seeks balance between his professional and personal life", + "Jim is open to making meaningful connections within his vicinity", + "Jim has a casual and laid-back style", + "Jim believes his eyes are blue", + "Jim is driven by the challenge of solving problems and creating something meaningful", + "Jim finds nature to be a peaceful retreat and enjoys the quiet it brings", + "Jim aspires to share his curiosity for exploring new ideas with a partner", ]; diff --git a/core/src/test_resources/getOrCreateRelationship.ts b/core/src/test_resources/getOrCreateRelationship.ts index 0ff3e353cf..bdf27d7ea7 100644 --- a/core/src/test_resources/getOrCreateRelationship.ts +++ b/core/src/test_resources/getOrCreateRelationship.ts @@ -2,65 +2,65 @@ import { getRelationship } from "../core/relationships.ts"; import { IAgentRuntime, Relationship, UUID } from "../core/types.ts"; export async function getOrCreateRelationship({ - runtime, - userA, - userB, + runtime, + userA, + userB, }: { - runtime: IAgentRuntime; - userA: UUID; - userB: UUID; + runtime: IAgentRuntime; + userA: UUID; + userB: UUID; }): Promise { - // Check if a relationship already exists between userA and userB - let relationship: Relationship | null = null; - try { - relationship = await getRelationship({ runtime, userA, userB }); - } catch (error) { - console.log("Error fetching relationship", error); - } + // Check if a relationship already exists between userA and userB + let relationship: Relationship | null = null; + try { + relationship = await getRelationship({ runtime, userA, userB }); + } catch (error) { + console.log("Error fetching relationship", error); + } - if (!relationship) { - await runtime.databaseAdapter.createRelationship({ - userA, - userB, - }); + if (!relationship) { + await runtime.databaseAdapter.createRelationship({ + userA, + userB, + }); - relationship = await getRelationship({ runtime, userA, userB }); - } + relationship = await getRelationship({ runtime, userA, userB }); + } - // Check if a room already exists for the participants - const rooms = await runtime.databaseAdapter.getRoomsForParticipants([ - userA, - userB, - ]); + // Check if a room already exists for the participants + const rooms = await runtime.databaseAdapter.getRoomsForParticipants([ + userA, + userB, + ]); - let roomId: UUID; - if (!rooms || rooms.length === 0) { - console.log("No room found for participants"); - // If no room exists, create a new room for the relationship - roomId = await runtime.databaseAdapter.createRoom(); - console.log("Created room", roomId); + let roomId: UUID; + if (!rooms || rooms.length === 0) { + console.log("No room found for participants"); + // If no room exists, create a new room for the relationship + roomId = await runtime.databaseAdapter.createRoom(); + console.log("Created room", roomId); - // Add participants to the newly created room - await runtime.databaseAdapter.addParticipant(userA, roomId); - await runtime.databaseAdapter.addParticipant(userB, roomId); - } else { - console.log("Room found for participants", rooms[0]); - // If a room already exists, use the existing room - roomId = rooms[0]; - } + // Add participants to the newly created room + await runtime.databaseAdapter.addParticipant(userA, roomId); + await runtime.databaseAdapter.addParticipant(userB, roomId); + } else { + console.log("Room found for participants", rooms[0]); + // If a room already exists, use the existing room + roomId = rooms[0]; + } - if (!relationship) { - // Create the relationship - await runtime.databaseAdapter.createRelationship({ - userA, - userB, - }); + if (!relationship) { + // Create the relationship + await runtime.databaseAdapter.createRelationship({ + userA, + userB, + }); - relationship = await getRelationship({ runtime, userA, userB }); + relationship = await getRelationship({ runtime, userA, userB }); - if (!relationship) { - throw new Error("Failed to fetch the created relationship"); + if (!relationship) { + throw new Error("Failed to fetch the created relationship"); + } } - } - return { ...relationship, roomId }; + return { ...relationship, roomId }; } diff --git a/core/src/test_resources/populateMemories.ts b/core/src/test_resources/populateMemories.ts index 267a62fc4b..40fe605f8f 100644 --- a/core/src/test_resources/populateMemories.ts +++ b/core/src/test_resources/populateMemories.ts @@ -3,27 +3,31 @@ import { getCachedEmbeddings, writeCachedEmbedding } from "./cache.ts"; import { type User } from "./types.ts"; export async function populateMemories( - runtime: IAgentRuntime, - user: User, - roomId: UUID, - conversations: Array< - (userId: UUID) => Array<{ userId: UUID; content: Content }> - >, + runtime: IAgentRuntime, + user: User, + roomId: UUID, + conversations: Array< + (userId: UUID) => Array<{ userId: UUID; content: Content }> + > ) { - for (const conversation of conversations) { - for (const c of conversation(user?.id as UUID)) { - const existingEmbedding = await getCachedEmbeddings(c.content.text); - const bakedMemory = await runtime.messageManager.addEmbeddingToMemory({ - userId: c.userId as UUID, - content: c.content, - roomId, - embedding: existingEmbedding, - }); - await runtime.messageManager.createMemory(bakedMemory); - if (!existingEmbedding) { - writeCachedEmbedding(c.content.text, bakedMemory.embedding as number[]); - await new Promise((resolve) => setTimeout(resolve, 200)); - } + for (const conversation of conversations) { + for (const c of conversation(user?.id as UUID)) { + const existingEmbedding = await getCachedEmbeddings(c.content.text); + const bakedMemory = + await runtime.messageManager.addEmbeddingToMemory({ + userId: c.userId as UUID, + content: c.content, + roomId, + embedding: existingEmbedding, + }); + await runtime.messageManager.createMemory(bakedMemory); + if (!existingEmbedding) { + writeCachedEmbedding( + c.content.text, + bakedMemory.embedding as number[] + ); + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } } - } } diff --git a/core/src/test_resources/report.ts b/core/src/test_resources/report.ts index b8b3f9b184..70d4a37cb9 100644 --- a/core/src/test_resources/report.ts +++ b/core/src/test_resources/report.ts @@ -1,96 +1,96 @@ interface TestResult { - testName: string; - attempts: number; - successful: number; - successRate: number; + testName: string; + attempts: number; + successful: number; + successRate: number; } export async function deleteReport() { - const { existsSync, unlinkSync } = await import("fs"); + const { existsSync, unlinkSync } = await import("fs"); - // Define the path to the test-report.json file - const reportPath = "./test-report.json"; + // Define the path to the test-report.json file + const reportPath = "./test-report.json"; - // Check if test-report.json exists - if (existsSync(reportPath)) { - // Delete the file - unlinkSync(reportPath); - } + // Check if test-report.json exists + if (existsSync(reportPath)) { + // Delete the file + unlinkSync(reportPath); + } } export async function addToReport( - testName: string, - attempts: number, - successful: number, - successRate: number, + testName: string, + attempts: number, + successful: number, + successRate: number ) { - const { existsSync, readFileSync, writeFileSync } = await import("fs"); + const { existsSync, readFileSync, writeFileSync } = await import("fs"); - // Define the path to the test-report.json file - const reportPath = "./test-report.json"; + // Define the path to the test-report.json file + const reportPath = "./test-report.json"; - // Initialize an empty array to hold the test results - let report: TestResult[] = []; + // Initialize an empty array to hold the test results + let report: TestResult[] = []; - // Check if test-report.json exists - if (existsSync(reportPath)) { - // Read the existing test report - const reportContent = readFileSync(reportPath, "utf-8"); - report = JSON.parse(reportContent); - } - - // Check if the test already exists in the report - const existingTestIndex = report.findIndex( - (test) => test.testName === testName, - ); - - // Create a new test result object - const newTestResult: TestResult = { - testName, - attempts, - successful, - successRate, - }; - - if (existingTestIndex !== -1) { - // If the test already exists, replace it with the new result - report[existingTestIndex] = newTestResult; - } else { - // If the test doesn't exist, add the new result to the report - report.push(newTestResult); - } - - // Write the updated report to test-report.json - writeFileSync(reportPath, JSON.stringify(report, null, 2)); + // Check if test-report.json exists + if (existsSync(reportPath)) { + // Read the existing test report + const reportContent = readFileSync(reportPath, "utf-8"); + report = JSON.parse(reportContent); + } + + // Check if the test already exists in the report + const existingTestIndex = report.findIndex( + (test) => test.testName === testName + ); + + // Create a new test result object + const newTestResult: TestResult = { + testName, + attempts, + successful, + successRate, + }; + + if (existingTestIndex !== -1) { + // If the test already exists, replace it with the new result + report[existingTestIndex] = newTestResult; + } else { + // If the test doesn't exist, add the new result to the report + report.push(newTestResult); + } + + // Write the updated report to test-report.json + writeFileSync(reportPath, JSON.stringify(report, null, 2)); } export async function logReport() { - const { existsSync, readFileSync } = await import("fs"); - const colors = await import("ansi-colors"); - - // Define the path to the test-report.json file - const reportPath = "./test-report.json"; - - // Check if test-report.json exists - if (!existsSync(reportPath)) { - console.log(colors.red("Error: test-report.json does not exist.")); - return; - } - - // Read the test report - const reportContent = readFileSync(reportPath, "utf-8"); - const report: TestResult[] = JSON.parse(reportContent); - - // Log each test result with appropriate color-coding - report.forEach((test) => { - const logMessage = `${test.testName}: ${test.attempts} Attempts, ${test.successful} Successful, Success Rate: ${test.successRate}%`; - - if (test.successRate === 100) { - console.log(colors.green(logMessage)); - } else if (test.successRate < 100 && test.successRate > 0) { - console.log(colors.yellow(logMessage)); - } else { - console.log(colors.red(logMessage)); + const { existsSync, readFileSync } = await import("fs"); + const colors = await import("ansi-colors"); + + // Define the path to the test-report.json file + const reportPath = "./test-report.json"; + + // Check if test-report.json exists + if (!existsSync(reportPath)) { + console.log(colors.red("Error: test-report.json does not exist.")); + return; } - }); + + // Read the test report + const reportContent = readFileSync(reportPath, "utf-8"); + const report: TestResult[] = JSON.parse(reportContent); + + // Log each test result with appropriate color-coding + report.forEach((test) => { + const logMessage = `${test.testName}: ${test.attempts} Attempts, ${test.successful} Successful, Success Rate: ${test.successRate}%`; + + if (test.successRate === 100) { + console.log(colors.green(logMessage)); + } else if (test.successRate < 100 && test.successRate > 0) { + console.log(colors.yellow(logMessage)); + } else { + console.log(colors.red(logMessage)); + } + }); } diff --git a/core/src/test_resources/runAiTest.ts b/core/src/test_resources/runAiTest.ts index e2850b562a..06cbe4e71e 100644 --- a/core/src/test_resources/runAiTest.ts +++ b/core/src/test_resources/runAiTest.ts @@ -1,17 +1,17 @@ import { addToReport } from "./report.ts"; export async function runAiTest( - testName: string, - testFunc: () => Promise, - runs: number = 3, + testName: string, + testFunc: () => Promise, + runs: number = 3 ) { - let successful = 0; + let successful = 0; - for (let i = 0; i < runs; i++) { - // console.log("Running test", testName, " (iteration", i + ")"); - if (await testFunc()) successful++; - } + for (let i = 0; i < runs; i++) { + // console.log("Running test", testName, " (iteration", i + ")"); + if (await testFunc()) successful++; + } - const successRate = (successful / runs) * 100; - addToReport(testName, runs, successful, successRate); + const successRate = (successful / runs) * 100; + addToReport(testName, runs, successful, successRate); } diff --git a/core/src/test_resources/templates.ts b/core/src/test_resources/templates.ts index 39ee4065a2..9c99412b05 100644 --- a/core/src/test_resources/templates.ts +++ b/core/src/test_resources/templates.ts @@ -1,7 +1,7 @@ import { messageCompletionFooter } from "../core/parsing.ts"; export const messageHandlerTemplate = - `{{actionExamples}} + `{{actionExamples}} # IMPORTANT: DO NOT USE THE INFORMATION FROM THE EXAMPLES ABOVE. THE EXAMPLES ARE FOR REFERENCE ONLY. diff --git a/core/src/test_resources/testAction.ts b/core/src/test_resources/testAction.ts index fb2fe3fad8..5ecfd6f559 100644 --- a/core/src/test_resources/testAction.ts +++ b/core/src/test_resources/testAction.ts @@ -1,52 +1,55 @@ import { IAgentRuntime, type Action, type Memory } from "../core/types.ts"; export const TEST_ACTION = { - name: "TEST_ACTION", - validate: async (_runtime: IAgentRuntime, _message: Memory) => { - return true; - }, - description: "This is a test action, for use in testing.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - ): Promise => { - return true; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { - text: "Please respond with the message 'testing 123' and the action TEST_ACTION", - action: "TEST_ACTION", - }, - }, - { - user: "{{user2}}", - content: { text: "testing 123", action: "TEST_ACTION" }, - }, + name: "TEST_ACTION", + validate: async (_runtime: IAgentRuntime, _message: Memory) => { + return true; + }, + description: "This is a test action, for use in testing.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + return true; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Please respond with the message 'testing 123' and the action TEST_ACTION", + action: "TEST_ACTION", + }, + }, + { + user: "{{user2}}", + content: { text: "testing 123", action: "TEST_ACTION" }, + }, + ], ], - ], } as Action; export const TEST_ACTION_FAIL = { - name: "TEST_ACTION_FAIL", - validate: async (_runtime: IAgentRuntime, _message: Memory) => { - return false; - }, - description: "This is a test action, for use in testing.", - handler: async ( - runtime: IAgentRuntime, - message: Memory, - ): Promise => { - return false; - }, - examples: [ - [ - { - user: "{{user1}}", - content: { text: "Testing failure", action: "TEST_ACTIONFALSE" }, - }, + name: "TEST_ACTION_FAIL", + validate: async (_runtime: IAgentRuntime, _message: Memory) => { + return false; + }, + description: "This is a test action, for use in testing.", + handler: async ( + runtime: IAgentRuntime, + message: Memory + ): Promise => { + return false; + }, + examples: [ + [ + { + user: "{{user1}}", + content: { + text: "Testing failure", + action: "TEST_ACTIONFALSE", + }, + }, + ], ], - ], } as Action; diff --git a/core/src/test_resources/testEvaluator.ts b/core/src/test_resources/testEvaluator.ts index 05e81a0dcc..06821277ab 100644 --- a/core/src/test_resources/testEvaluator.ts +++ b/core/src/test_resources/testEvaluator.ts @@ -1,69 +1,67 @@ import { - Evaluator, - IAgentRuntime, - type Memory, - type State, + Evaluator, + IAgentRuntime, + type Memory, + type State, } from "../core/types.ts"; async function handler(runtime: IAgentRuntime, message: Memory) { - const state = (await runtime.composeState(message)) as State; - return state; + const state = (await runtime.composeState(message)) as State; + return state; } export const TEST_EVALUATOR = { - name: "TEST_EVALUATOR", - validate: async ( - - _runtime: IAgentRuntime, - - _message: Memory, - ): Promise => { - return await Promise.resolve(true); - }, - description: "Test evaluator.", - handler, - examples: [ - { - context: "Testing, testing, 123 123", - messages: [ + name: "TEST_EVALUATOR", + validate: async ( + _runtime: IAgentRuntime, + + _message: Memory + ): Promise => { + return await Promise.resolve(true); + }, + description: "Test evaluator.", + handler, + examples: [ { - user: "{{user1}}", - content: { - text: "Testing, testing, 123 123", - action: "TEST_EVALUATOR", - }, + context: "Testing, testing, 123 123", + messages: [ + { + user: "{{user1}}", + content: { + text: "Testing, testing, 123 123", + action: "TEST_EVALUATOR", + }, + }, + ], + outcome: "There is an outcome.", }, - ], - outcome: "There is an outcome.", - }, - ], + ], } as Evaluator; export const TEST_EVALUATOR_FAIL = { - name: "TEST_EVALUATOR_FAIL", - validate: async ( - - _runtime: IAgentRuntime, - - _message: Memory, - ): Promise => { - return await Promise.resolve(false); - }, - description: "Test failure of the evaluator and validation.", - handler, - examples: [ - { - context: "Testing, testing, 123 123", - messages: [ + name: "TEST_EVALUATOR_FAIL", + validate: async ( + _runtime: IAgentRuntime, + + _message: Memory + ): Promise => { + return await Promise.resolve(false); + }, + description: "Test failure of the evaluator and validation.", + handler, + examples: [ { - user: "{{user1}}", - content: { - text: "Testing, testing, 123 123", - action: "TEST_EVALUATOR_FAIL", - }, + context: "Testing, testing, 123 123", + messages: [ + { + user: "{{user1}}", + content: { + text: "Testing, testing, 123 123", + action: "TEST_EVALUATOR_FAIL", + }, + }, + ], + outcome: "Things have been tested to have maybe gone wrong.", }, - ], - outcome: "Things have been tested to have maybe gone wrong.", - }, - ], + ], } as Evaluator; diff --git a/core/src/test_resources/types.ts b/core/src/test_resources/types.ts index 4bd7aedeeb..634e266cbe 100644 --- a/core/src/test_resources/types.ts +++ b/core/src/test_resources/types.ts @@ -1,6 +1,6 @@ export interface User { - id: string; - email?: string; - phone?: string; - role?: string; + id: string; + email?: string; + phone?: string; + role?: string; } diff --git a/core/src/types/token.ts b/core/src/types/token.ts index 95751e7b5e..6e12bde638 100644 --- a/core/src/types/token.ts +++ b/core/src/types/token.ts @@ -1,274 +1,274 @@ export interface TokenSecurityData { - ownerBalance: string; - creatorBalance: string; - ownerPercentage: number; - creatorPercentage: number; - top10HolderBalance: string; - top10HolderPercent: number; + ownerBalance: string; + creatorBalance: string; + ownerPercentage: number; + creatorPercentage: number; + top10HolderBalance: string; + top10HolderPercent: number; } export interface TokenTradeData { - address: string; - holder: number; - market: number; - last_trade_unix_time: number; - last_trade_human_time: string; - price: number; - history_30m_price: number; - price_change_30m_percent: number; - history_1h_price: number; - price_change_1h_percent: number; - history_2h_price: number; - price_change_2h_percent: number; - history_4h_price: number; - price_change_4h_percent: number; - history_6h_price: number; - price_change_6h_percent: number; - history_8h_price: number; - price_change_8h_percent: number; - history_12h_price: number; - price_change_12h_percent: number; - history_24h_price: number; - price_change_24h_percent: number; - unique_wallet_30m: number; - unique_wallet_history_30m: number; - unique_wallet_30m_change_percent: number; - unique_wallet_1h: number; - unique_wallet_history_1h: number; - unique_wallet_1h_change_percent: number; - unique_wallet_2h: number; - unique_wallet_history_2h: number; - unique_wallet_2h_change_percent: number; - unique_wallet_4h: number; - unique_wallet_history_4h: number; - unique_wallet_4h_change_percent: number; - unique_wallet_8h: number; - unique_wallet_history_8h: number | null; - unique_wallet_8h_change_percent: number | null; - unique_wallet_24h: number; - unique_wallet_history_24h: number | null; - unique_wallet_24h_change_percent: number | null; - trade_30m: number; - trade_history_30m: number; - trade_30m_change_percent: number; - sell_30m: number; - sell_history_30m: number; - sell_30m_change_percent: number; - buy_30m: number; - buy_history_30m: number; - buy_30m_change_percent: number; - volume_30m: number; - volume_30m_usd: number; - volume_history_30m: number; - volume_history_30m_usd: number; - volume_30m_change_percent: number; - volume_buy_30m: number; - volume_buy_30m_usd: number; - volume_buy_history_30m: number; - volume_buy_history_30m_usd: number; - volume_buy_30m_change_percent: number; - volume_sell_30m: number; - volume_sell_30m_usd: number; - volume_sell_history_30m: number; - volume_sell_history_30m_usd: number; - volume_sell_30m_change_percent: number; - trade_1h: number; - trade_history_1h: number; - trade_1h_change_percent: number; - sell_1h: number; - sell_history_1h: number; - sell_1h_change_percent: number; - buy_1h: number; - buy_history_1h: number; - buy_1h_change_percent: number; - volume_1h: number; - volume_1h_usd: number; - volume_history_1h: number; - volume_history_1h_usd: number; - volume_1h_change_percent: number; - volume_buy_1h: number; - volume_buy_1h_usd: number; - volume_buy_history_1h: number; - volume_buy_history_1h_usd: number; - volume_buy_1h_change_percent: number; - volume_sell_1h: number; - volume_sell_1h_usd: number; - volume_sell_history_1h: number; - volume_sell_history_1h_usd: number; - volume_sell_1h_change_percent: number; - trade_2h: number; - trade_history_2h: number; - trade_2h_change_percent: number; - sell_2h: number; - sell_history_2h: number; - sell_2h_change_percent: number; - buy_2h: number; - buy_history_2h: number; - buy_2h_change_percent: number; - volume_2h: number; - volume_2h_usd: number; - volume_history_2h: number; - volume_history_2h_usd: number; - volume_2h_change_percent: number; - volume_buy_2h: number; - volume_buy_2h_usd: number; - volume_buy_history_2h: number; - volume_buy_history_2h_usd: number; - volume_buy_2h_change_percent: number; - volume_sell_2h: number; - volume_sell_2h_usd: number; - volume_sell_history_2h: number; - volume_sell_history_2h_usd: number; - volume_sell_2h_change_percent: number; - trade_4h: number; - trade_history_4h: number; - trade_4h_change_percent: number; - sell_4h: number; - sell_history_4h: number; - sell_4h_change_percent: number; - buy_4h: number; - buy_history_4h: number; - buy_4h_change_percent: number; - volume_4h: number; - volume_4h_usd: number; - volume_history_4h: number; - volume_history_4h_usd: number; - volume_4h_change_percent: number; - volume_buy_4h: number; - volume_buy_4h_usd: number; - volume_buy_history_4h: number; - volume_buy_history_4h_usd: number; - volume_buy_4h_change_percent: number; - volume_sell_4h: number; - volume_sell_4h_usd: number; - volume_sell_history_4h: number; - volume_sell_history_4h_usd: number; - volume_sell_4h_change_percent: number; - trade_8h: number; - trade_history_8h: number | null; - trade_8h_change_percent: number | null; - sell_8h: number; - sell_history_8h: number | null; - sell_8h_change_percent: number | null; - buy_8h: number; - buy_history_8h: number | null; - buy_8h_change_percent: number | null; - volume_8h: number; - volume_8h_usd: number; - volume_history_8h: number; - volume_history_8h_usd: number; - volume_8h_change_percent: number | null; - volume_buy_8h: number; - volume_buy_8h_usd: number; - volume_buy_history_8h: number; - volume_buy_history_8h_usd: number; - volume_buy_8h_change_percent: number | null; - volume_sell_8h: number; - volume_sell_8h_usd: number; - volume_sell_history_8h: number; - volume_sell_history_8h_usd: number; - volume_sell_8h_change_percent: number | null; - trade_24h: number; - trade_history_24h: number; - trade_24h_change_percent: number | null; - sell_24h: number; - sell_history_24h: number; - sell_24h_change_percent: number | null; - buy_24h: number; - buy_history_24h: number; - buy_24h_change_percent: number | null; - volume_24h: number; - volume_24h_usd: number; - volume_history_24h: number; - volume_history_24h_usd: number; - volume_24h_change_percent: number | null; - volume_buy_24h: number; - volume_buy_24h_usd: number; - volume_buy_history_24h: number; - volume_buy_history_24h_usd: number; - volume_buy_24h_change_percent: number | null; - volume_sell_24h: number; - volume_sell_24h_usd: number; - volume_sell_history_24h: number; - volume_sell_history_24h_usd: number; - volume_sell_24h_change_percent: number | null; + address: string; + holder: number; + market: number; + last_trade_unix_time: number; + last_trade_human_time: string; + price: number; + history_30m_price: number; + price_change_30m_percent: number; + history_1h_price: number; + price_change_1h_percent: number; + history_2h_price: number; + price_change_2h_percent: number; + history_4h_price: number; + price_change_4h_percent: number; + history_6h_price: number; + price_change_6h_percent: number; + history_8h_price: number; + price_change_8h_percent: number; + history_12h_price: number; + price_change_12h_percent: number; + history_24h_price: number; + price_change_24h_percent: number; + unique_wallet_30m: number; + unique_wallet_history_30m: number; + unique_wallet_30m_change_percent: number; + unique_wallet_1h: number; + unique_wallet_history_1h: number; + unique_wallet_1h_change_percent: number; + unique_wallet_2h: number; + unique_wallet_history_2h: number; + unique_wallet_2h_change_percent: number; + unique_wallet_4h: number; + unique_wallet_history_4h: number; + unique_wallet_4h_change_percent: number; + unique_wallet_8h: number; + unique_wallet_history_8h: number | null; + unique_wallet_8h_change_percent: number | null; + unique_wallet_24h: number; + unique_wallet_history_24h: number | null; + unique_wallet_24h_change_percent: number | null; + trade_30m: number; + trade_history_30m: number; + trade_30m_change_percent: number; + sell_30m: number; + sell_history_30m: number; + sell_30m_change_percent: number; + buy_30m: number; + buy_history_30m: number; + buy_30m_change_percent: number; + volume_30m: number; + volume_30m_usd: number; + volume_history_30m: number; + volume_history_30m_usd: number; + volume_30m_change_percent: number; + volume_buy_30m: number; + volume_buy_30m_usd: number; + volume_buy_history_30m: number; + volume_buy_history_30m_usd: number; + volume_buy_30m_change_percent: number; + volume_sell_30m: number; + volume_sell_30m_usd: number; + volume_sell_history_30m: number; + volume_sell_history_30m_usd: number; + volume_sell_30m_change_percent: number; + trade_1h: number; + trade_history_1h: number; + trade_1h_change_percent: number; + sell_1h: number; + sell_history_1h: number; + sell_1h_change_percent: number; + buy_1h: number; + buy_history_1h: number; + buy_1h_change_percent: number; + volume_1h: number; + volume_1h_usd: number; + volume_history_1h: number; + volume_history_1h_usd: number; + volume_1h_change_percent: number; + volume_buy_1h: number; + volume_buy_1h_usd: number; + volume_buy_history_1h: number; + volume_buy_history_1h_usd: number; + volume_buy_1h_change_percent: number; + volume_sell_1h: number; + volume_sell_1h_usd: number; + volume_sell_history_1h: number; + volume_sell_history_1h_usd: number; + volume_sell_1h_change_percent: number; + trade_2h: number; + trade_history_2h: number; + trade_2h_change_percent: number; + sell_2h: number; + sell_history_2h: number; + sell_2h_change_percent: number; + buy_2h: number; + buy_history_2h: number; + buy_2h_change_percent: number; + volume_2h: number; + volume_2h_usd: number; + volume_history_2h: number; + volume_history_2h_usd: number; + volume_2h_change_percent: number; + volume_buy_2h: number; + volume_buy_2h_usd: number; + volume_buy_history_2h: number; + volume_buy_history_2h_usd: number; + volume_buy_2h_change_percent: number; + volume_sell_2h: number; + volume_sell_2h_usd: number; + volume_sell_history_2h: number; + volume_sell_history_2h_usd: number; + volume_sell_2h_change_percent: number; + trade_4h: number; + trade_history_4h: number; + trade_4h_change_percent: number; + sell_4h: number; + sell_history_4h: number; + sell_4h_change_percent: number; + buy_4h: number; + buy_history_4h: number; + buy_4h_change_percent: number; + volume_4h: number; + volume_4h_usd: number; + volume_history_4h: number; + volume_history_4h_usd: number; + volume_4h_change_percent: number; + volume_buy_4h: number; + volume_buy_4h_usd: number; + volume_buy_history_4h: number; + volume_buy_history_4h_usd: number; + volume_buy_4h_change_percent: number; + volume_sell_4h: number; + volume_sell_4h_usd: number; + volume_sell_history_4h: number; + volume_sell_history_4h_usd: number; + volume_sell_4h_change_percent: number; + trade_8h: number; + trade_history_8h: number | null; + trade_8h_change_percent: number | null; + sell_8h: number; + sell_history_8h: number | null; + sell_8h_change_percent: number | null; + buy_8h: number; + buy_history_8h: number | null; + buy_8h_change_percent: number | null; + volume_8h: number; + volume_8h_usd: number; + volume_history_8h: number; + volume_history_8h_usd: number; + volume_8h_change_percent: number | null; + volume_buy_8h: number; + volume_buy_8h_usd: number; + volume_buy_history_8h: number; + volume_buy_history_8h_usd: number; + volume_buy_8h_change_percent: number | null; + volume_sell_8h: number; + volume_sell_8h_usd: number; + volume_sell_history_8h: number; + volume_sell_history_8h_usd: number; + volume_sell_8h_change_percent: number | null; + trade_24h: number; + trade_history_24h: number; + trade_24h_change_percent: number | null; + sell_24h: number; + sell_history_24h: number; + sell_24h_change_percent: number | null; + buy_24h: number; + buy_history_24h: number; + buy_24h_change_percent: number | null; + volume_24h: number; + volume_24h_usd: number; + volume_history_24h: number; + volume_history_24h_usd: number; + volume_24h_change_percent: number | null; + volume_buy_24h: number; + volume_buy_24h_usd: number; + volume_buy_history_24h: number; + volume_buy_history_24h_usd: number; + volume_buy_24h_change_percent: number | null; + volume_sell_24h: number; + volume_sell_24h_usd: number; + volume_sell_history_24h: number; + volume_sell_history_24h_usd: number; + volume_sell_24h_change_percent: number | null; } export interface HolderData { - address: string; - balance: string; + address: string; + balance: string; } export interface ProcessedTokenData { - security: TokenSecurityData; - tradeData: TokenTradeData; - holderDistributionTrend: string; // 'increasing' | 'decreasing' | 'stable' - highValueHolders: Array<{ - holderAddress: string; - balanceUsd: string; - }>; - recentTrades: boolean; - highSupplyHoldersCount: number; - dexScreenerData: DexScreenerData; + security: TokenSecurityData; + tradeData: TokenTradeData; + holderDistributionTrend: string; // 'increasing' | 'decreasing' | 'stable' + highValueHolders: Array<{ + holderAddress: string; + balanceUsd: string; + }>; + recentTrades: boolean; + highSupplyHoldersCount: number; + dexScreenerData: DexScreenerData; - isDexScreenerListed: boolean; - isDexScreenerPaid: boolean; + isDexScreenerListed: boolean; + isDexScreenerPaid: boolean; } export interface DexScreenerPair { - chainId: string; - dexId: string; - url: string; - pairAddress: string; - baseToken: { - address: string; - name: string; - symbol: string; - }; - quoteToken: { - address: string; - name: string; - symbol: string; - }; - priceNative: string; - priceUsd: string; - txns: { - m5: { buys: number; sells: number }; - h1: { buys: number; sells: number }; - h6: { buys: number; sells: number }; - h24: { buys: number; sells: number }; - }; - volume: { - h24: number; - h6: number; - h1: number; - m5: number; - }; - priceChange: { - m5: number; - h1: number; - h6: number; - h24: number; - }; - liquidity: { - usd: number; - base: number; - quote: number; - }; - fdv: number; - marketCap: number; - pairCreatedAt: number; - info: { - imageUrl: string; - websites: { label: string; url: string }[]; - socials: { type: string; url: string }[]; - }; - boosts: { - active: number; - }; + chainId: string; + dexId: string; + url: string; + pairAddress: string; + baseToken: { + address: string; + name: string; + symbol: string; + }; + quoteToken: { + address: string; + name: string; + symbol: string; + }; + priceNative: string; + priceUsd: string; + txns: { + m5: { buys: number; sells: number }; + h1: { buys: number; sells: number }; + h6: { buys: number; sells: number }; + h24: { buys: number; sells: number }; + }; + volume: { + h24: number; + h6: number; + h1: number; + m5: number; + }; + priceChange: { + m5: number; + h1: number; + h6: number; + h24: number; + }; + liquidity: { + usd: number; + base: number; + quote: number; + }; + fdv: number; + marketCap: number; + pairCreatedAt: number; + info: { + imageUrl: string; + websites: { label: string; url: string }[]; + socials: { type: string; url: string }[]; + }; + boosts: { + active: number; + }; } export interface DexScreenerData { - schemaVersion: string; - pairs: DexScreenerPair[]; + schemaVersion: string; + pairs: DexScreenerPair[]; } diff --git a/core/src/vendor/vits.ts b/core/src/vendor/vits.ts index cb959646c8..239e610b28 100644 --- a/core/src/vendor/vits.ts +++ b/core/src/vendor/vits.ts @@ -20,397 +20,405 @@ import { inspect, promisify } from "util"; import { vitsVoiceList } from "./vitsVoiceList.ts"; export async function synthesize( - input: string | string[], - options: SynthesisOptions, - onSegment?: SynthesisSegmentEvent, - onSentence?: SynthesisSegmentEvent, + input: string | string[], + options: SynthesisOptions, + onSegment?: SynthesisSegmentEvent, + onSentence?: SynthesisSegmentEvent ): Promise { - options = extendDeep(defaultSynthesisOptions, options); + options = extendDeep(defaultSynthesisOptions, options); - let segments: string[]; + let segments: string[]; - if (Array.isArray(input)) { - segments = input; - } else { - const plainTextOptions = options.plainText!; + if (Array.isArray(input)) { + segments = input; + } else { + const plainTextOptions = options.plainText!; - segments = splitToParagraphs( - input, - plainTextOptions.paragraphBreaks!, - plainTextOptions.whitespace!, - ); - } + segments = splitToParagraphs( + input, + plainTextOptions.paragraphBreaks!, + plainTextOptions.whitespace! + ); + } - return await synthesizeSegments(segments, options, onSegment, onSentence); + return await synthesizeSegments(segments, options, onSegment, onSentence); } const createWriteStream = gracefulFS.createWriteStream; function getCostMatrixMemorySizeMB( - sequence1Length: number, - sequence2Length: number, - windowLength: number, + sequence1Length: number, + sequence2Length: number, + windowLength: number ) { - const costMatrixMemorySizeMB = - (sequence1Length * Math.min(sequence2Length, windowLength) * 4) / 1000000; + const costMatrixMemorySizeMB = + (sequence1Length * Math.min(sequence2Length, windowLength) * 4) / + 1000000; - return costMatrixMemorySizeMB; + return costMatrixMemorySizeMB; } function argIndexOfMinimumOf3(x1: number, x2: number, x3: number) { - if (x1 <= x2 && x1 <= x3) { - return 1; - } else if (x2 <= x3) { - return 2; - } else { - return 3; - } + if (x1 <= x2 && x1 <= x3) { + return 1; + } else if (x2 <= x3) { + return 2; + } else { + return 3; + } } function minimumOf3(x1: number, x2: number, x3: number) { - if (x1 <= x2 && x1 <= x3) { - return x1; - } else if (x2 <= x3) { - return x2; - } else { - return x3; - } + if (x1 <= x2 && x1 <= x3) { + return x1; + } else if (x2 <= x3) { + return x2; + } else { + return x3; + } } function computeAccumulatedCostMatrixTransposed( - sequence1: T[], - sequence2: U[], - costFunction: (a: T, b: U) => number, - windowMaxLength: number, - centerIndexes?: number[], + sequence1: T[], + sequence2: U[], + costFunction: (a: T, b: U) => number, + windowMaxLength: number, + centerIndexes?: number[] ) { - const halfWindowMaxLength = Math.floor(windowMaxLength / 2); + const halfWindowMaxLength = Math.floor(windowMaxLength / 2); - const columnCount = sequence1.length; - const rowCount = Math.min(windowMaxLength, sequence2.length); + const columnCount = sequence1.length; + const rowCount = Math.min(windowMaxLength, sequence2.length); - const accumulatedCostMatrixTransposed: Float32Array[] = - new Array(columnCount); + const accumulatedCostMatrixTransposed: Float32Array[] = + new Array(columnCount); - // Initialize an array to store window start offsets - const windowStartOffsets = new Int32Array(columnCount); + // Initialize an array to store window start offsets + const windowStartOffsets = new Int32Array(columnCount); - // Compute accumulated cost matrix column by column - for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { - // Create new column and add it to the matrix - const currentColumn = new Float32Array(rowCount); - accumulatedCostMatrixTransposed[columnIndex] = currentColumn; + // Compute accumulated cost matrix column by column + for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) { + // Create new column and add it to the matrix + const currentColumn = new Float32Array(rowCount); + accumulatedCostMatrixTransposed[columnIndex] = currentColumn; - // Compute window center, or use given one - let windowCenter: number; + // Compute window center, or use given one + let windowCenter: number; - if (centerIndexes) { - windowCenter = centerIndexes[columnIndex]; - } else { - windowCenter = Math.floor((columnIndex / columnCount) * sequence2.length); - } + if (centerIndexes) { + windowCenter = centerIndexes[columnIndex]; + } else { + windowCenter = Math.floor( + (columnIndex / columnCount) * sequence2.length + ); + } - // Compute window start and end offsets - let windowStartOffset = Math.max(windowCenter - halfWindowMaxLength, 0); - let windowEndOffset = windowStartOffset + rowCount; + // Compute window start and end offsets + let windowStartOffset = Math.max(windowCenter - halfWindowMaxLength, 0); + let windowEndOffset = windowStartOffset + rowCount; - if (windowEndOffset > sequence2.length) { - windowEndOffset = sequence2.length; - windowStartOffset = windowEndOffset - rowCount; - } + if (windowEndOffset > sequence2.length) { + windowEndOffset = sequence2.length; + windowStartOffset = windowEndOffset - rowCount; + } - // Store the start offset for this column - windowStartOffsets[columnIndex] = windowStartOffset; + // Store the start offset for this column + windowStartOffsets[columnIndex] = windowStartOffset; - // Get target sequence1 value - const targetSequence1Value = sequence1[columnIndex]; + // Get target sequence1 value + const targetSequence1Value = sequence1[columnIndex]; - // If this is the first column, fill it only using the 'up' neighbors - if (columnIndex == 0) { - for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) { - const cost = costFunction( - targetSequence1Value, - sequence2[windowStartOffset + rowIndex], - ); - const upCost = currentColumn[rowIndex - 1]; + // If this is the first column, fill it only using the 'up' neighbors + if (columnIndex == 0) { + for (let rowIndex = 1; rowIndex < rowCount; rowIndex++) { + const cost = costFunction( + targetSequence1Value, + sequence2[windowStartOffset + rowIndex] + ); + const upCost = currentColumn[rowIndex - 1]; - currentColumn[rowIndex] = cost + upCost; - } + currentColumn[rowIndex] = cost + upCost; + } - continue; - } + continue; + } - // If not first column + // If not first column - // Store the column to the left - const leftColumn = accumulatedCostMatrixTransposed[columnIndex - 1]; + // Store the column to the left + const leftColumn = accumulatedCostMatrixTransposed[columnIndex - 1]; - // Compute the delta between the current window start offset - // and left column's window offset - const windowOffsetDelta = - windowStartOffset - windowStartOffsets[columnIndex - 1]; + // Compute the delta between the current window start offset + // and left column's window offset + const windowOffsetDelta = + windowStartOffset - windowStartOffsets[columnIndex - 1]; - // Compute the accumulated cost for all rows in the window - for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) { - // Compute the cost for current cell - const cost = costFunction( - targetSequence1Value, - sequence2[windowStartOffset + rowIndex], - ); + // Compute the accumulated cost for all rows in the window + for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) { + // Compute the cost for current cell + const cost = costFunction( + targetSequence1Value, + sequence2[windowStartOffset + rowIndex] + ); - // Retrieve the cost for the 'up' (insertion) neighbor - let upCost = Infinity; - if (rowIndex > 0) { - upCost = currentColumn[rowIndex - 1]; - } + // Retrieve the cost for the 'up' (insertion) neighbor + let upCost = Infinity; + if (rowIndex > 0) { + upCost = currentColumn[rowIndex - 1]; + } - // Retrieve the cost for the 'left' (deletion) neighbor - let leftCost = Infinity; - const leftRowIndex = rowIndex + windowOffsetDelta; + // Retrieve the cost for the 'left' (deletion) neighbor + let leftCost = Infinity; + const leftRowIndex = rowIndex + windowOffsetDelta; - if (leftRowIndex < rowCount) { - leftCost = leftColumn[leftRowIndex]; - } + if (leftRowIndex < rowCount) { + leftCost = leftColumn[leftRowIndex]; + } - // Retrieve the cost for the 'up and left' (match) neighbor - let upAndLeftCost = Infinity; - const upAndLeftRowIndex = leftRowIndex - 1; + // Retrieve the cost for the 'up and left' (match) neighbor + let upAndLeftCost = Infinity; + const upAndLeftRowIndex = leftRowIndex - 1; - if (upAndLeftRowIndex >= 0 && upAndLeftRowIndex < rowCount) { - upAndLeftCost = leftColumn[upAndLeftRowIndex]; - } + if (upAndLeftRowIndex >= 0 && upAndLeftRowIndex < rowCount) { + upAndLeftCost = leftColumn[upAndLeftRowIndex]; + } - // Find the minimum of all neighbors - let minimumNeighborCost = minimumOf3(upCost, leftCost, upAndLeftCost); + // Find the minimum of all neighbors + let minimumNeighborCost = minimumOf3( + upCost, + leftCost, + upAndLeftCost + ); - // If all neighbors are infinity, then it means there is a "jump" between the window - // of the current column and the left column, and they don't have overlapping rows. - // In this case, only the cost of the current cell will be used - if (minimumNeighborCost === Infinity) { - minimumNeighborCost = 0; - } + // If all neighbors are infinity, then it means there is a "jump" between the window + // of the current column and the left column, and they don't have overlapping rows. + // In this case, only the cost of the current cell will be used + if (minimumNeighborCost === Infinity) { + minimumNeighborCost = 0; + } - // Write cost + minimum neighbor cost to the current column - currentColumn[rowIndex] = cost + minimumNeighborCost; + // Write cost + minimum neighbor cost to the current column + currentColumn[rowIndex] = cost + minimumNeighborCost; + } } - } - return { - accumulatedCostMatrixTransposed, - windowStartOffsets, - }; + return { + accumulatedCostMatrixTransposed, + windowStartOffsets, + }; } function computeBestPathTransposed( - accumulatedCostMatrixTransposed: Float32Array[], - windowStartOffsets: Int32Array, + accumulatedCostMatrixTransposed: Float32Array[], + windowStartOffsets: Int32Array ) { - const columnCount = accumulatedCostMatrixTransposed.length; - const rowCount = accumulatedCostMatrixTransposed[0].length; - - const bestPath: AlignmentPath = []; - - // Start at the bottom right corner and find the best path - // towards the top left - let columnIndex = columnCount - 1; - let rowIndex = rowCount - 1; - - while (columnIndex > 0 || rowIndex > 0) { - const windowStartIndex = windowStartOffsets[columnIndex]; - const windowStartDelta = - columnIndex > 0 - ? windowStartIndex - windowStartOffsets[columnIndex - 1] - : 0; + const columnCount = accumulatedCostMatrixTransposed.length; + const rowCount = accumulatedCostMatrixTransposed[0].length; + + const bestPath: AlignmentPath = []; + + // Start at the bottom right corner and find the best path + // towards the top left + let columnIndex = columnCount - 1; + let rowIndex = rowCount - 1; + + while (columnIndex > 0 || rowIndex > 0) { + const windowStartIndex = windowStartOffsets[columnIndex]; + const windowStartDelta = + columnIndex > 0 + ? windowStartIndex - windowStartOffsets[columnIndex - 1] + : 0; + + // Add the current cell to the best path + bestPath.push({ + source: columnIndex, + dest: windowStartIndex + rowIndex, + }); - // Add the current cell to the best path - bestPath.push({ - source: columnIndex, - dest: windowStartIndex + rowIndex, - }); + // Retrieve the cost for the 'up' (insertion) neighbor + const upRowIndex = rowIndex - 1; + let upCost = Infinity; - // Retrieve the cost for the 'up' (insertion) neighbor - const upRowIndex = rowIndex - 1; - let upCost = Infinity; + if (upRowIndex >= 0) { + upCost = accumulatedCostMatrixTransposed[columnIndex][upRowIndex]; + } - if (upRowIndex >= 0) { - upCost = accumulatedCostMatrixTransposed[columnIndex][upRowIndex]; - } + // Retrieve the cost for the 'left' (deletion) neighbor + const leftRowIndex = rowIndex + windowStartDelta; + const leftColumnIndex = columnIndex - 1; + let leftCost = Infinity; - // Retrieve the cost for the 'left' (deletion) neighbor - const leftRowIndex = rowIndex + windowStartDelta; - const leftColumnIndex = columnIndex - 1; - let leftCost = Infinity; + if (leftColumnIndex >= 0 && leftRowIndex < rowCount) { + leftCost = + accumulatedCostMatrixTransposed[leftColumnIndex][leftRowIndex]; + } - if (leftColumnIndex >= 0 && leftRowIndex < rowCount) { - leftCost = accumulatedCostMatrixTransposed[leftColumnIndex][leftRowIndex]; - } + // Retrieve the cost for the 'up and left' (match) neighbor + const upAndLeftRowIndex = rowIndex - 1 + windowStartDelta; + const upAndLeftColumnIndex = columnIndex - 1; + let upAndLeftCost = Infinity; + + if ( + upAndLeftColumnIndex >= 0 && + upAndLeftRowIndex >= 0 && + upAndLeftRowIndex < rowCount + ) { + upAndLeftCost = + accumulatedCostMatrixTransposed[upAndLeftColumnIndex][ + upAndLeftRowIndex + ]; + } - // Retrieve the cost for the 'up and left' (match) neighbor - const upAndLeftRowIndex = rowIndex - 1 + windowStartDelta; - const upAndLeftColumnIndex = columnIndex - 1; - let upAndLeftCost = Infinity; + // If all neighbors have a cost of infinity, it means + // there is a "jump" between the window for the current and previous column + if ( + upCost == Infinity && + leftCost == Infinity && + upAndLeftCost == Infinity + ) { + // In that case: + // + // If there are rows above + if (upRowIndex >= 0) { + // Move upward + rowIndex = upRowIndex; + } else if (leftColumnIndex >= 0) { + // Otherwise, move to the left + columnIndex = leftColumnIndex; + } else { + // Since we know that either columnIndex > 0 or rowIndex > 0, + // one of these directions must be available. + // This error should never happen + + throw new Error( + `Unexpected state: columnIndex: ${columnIndex}, rowIndex: ${rowIndex}` + ); + } + } else { + // Choose the direction with the smallest cost + const smallestCostDirection = argIndexOfMinimumOf3( + upCost, + leftCost, + upAndLeftCost + ); - if ( - upAndLeftColumnIndex >= 0 && - upAndLeftRowIndex >= 0 && - upAndLeftRowIndex < rowCount - ) { - upAndLeftCost = - accumulatedCostMatrixTransposed[upAndLeftColumnIndex][ - upAndLeftRowIndex - ]; + if (smallestCostDirection == 1) { + // Move upward + rowIndex = upRowIndex; + // The upper column index stays the same + } else if (smallestCostDirection == 2) { + // Move to the left + rowIndex = leftRowIndex; + columnIndex = leftColumnIndex; + } else { + // Move upward and to the left + rowIndex = upAndLeftRowIndex; + columnIndex = upAndLeftColumnIndex; + } + } } - // If all neighbors have a cost of infinity, it means - // there is a "jump" between the window for the current and previous column - if ( - upCost == Infinity && - leftCost == Infinity && - upAndLeftCost == Infinity - ) { - // In that case: - // - // If there are rows above - if (upRowIndex >= 0) { - // Move upward - rowIndex = upRowIndex; - } else if (leftColumnIndex >= 0) { - // Otherwise, move to the left - columnIndex = leftColumnIndex; - } else { - // Since we know that either columnIndex > 0 or rowIndex > 0, - // one of these directions must be available. - // This error should never happen + bestPath.push({ + source: 0, + dest: 0, + }); - throw new Error( - `Unexpected state: columnIndex: ${columnIndex}, rowIndex: ${rowIndex}`, - ); - } - } else { - // Choose the direction with the smallest cost - const smallestCostDirection = argIndexOfMinimumOf3( - upCost, - leftCost, - upAndLeftCost, - ); - - if (smallestCostDirection == 1) { - // Move upward - rowIndex = upRowIndex; - // The upper column index stays the same - } else if (smallestCostDirection == 2) { - // Move to the left - rowIndex = leftRowIndex; - columnIndex = leftColumnIndex; - } else { - // Move upward and to the left - rowIndex = upAndLeftRowIndex; - columnIndex = upAndLeftColumnIndex; - } - } - } - - bestPath.push({ - source: 0, - dest: 0, - }); - - return bestPath.reverse() as AlignmentPath; + return bestPath.reverse() as AlignmentPath; } function alignDTWWindowed( - sequence1: T[], - sequence2: U[], - costFunction: (a: T, b: U) => number, - windowMaxLength: number, - centerIndexes?: number[], + sequence1: T[], + sequence2: U[], + costFunction: (a: T, b: U) => number, + windowMaxLength: number, + centerIndexes?: number[] ) { - if (windowMaxLength < 2) { - throw new Error("Window length must be greater or equal to 2"); - } + if (windowMaxLength < 2) { + throw new Error("Window length must be greater or equal to 2"); + } - if (sequence1.length == 0 || sequence2.length == 0) { - return { - path: [] as AlignmentPath, - pathCost: 0, - }; - } - - // Compute accumulated cost matrix (transposed) - const { accumulatedCostMatrixTransposed, windowStartOffsets } = - computeAccumulatedCostMatrixTransposed( - sequence1, - sequence2, - costFunction, - windowMaxLength, - centerIndexes, - ); + if (sequence1.length == 0 || sequence2.length == 0) { + return { + path: [] as AlignmentPath, + pathCost: 0, + }; + } + + // Compute accumulated cost matrix (transposed) + const { accumulatedCostMatrixTransposed, windowStartOffsets } = + computeAccumulatedCostMatrixTransposed( + sequence1, + sequence2, + costFunction, + windowMaxLength, + centerIndexes + ); - // Find best path for the computed matrix - const path = computeBestPathTransposed( - accumulatedCostMatrixTransposed, - windowStartOffsets, - ); + // Find best path for the computed matrix + const path = computeBestPathTransposed( + accumulatedCostMatrixTransposed, + windowStartOffsets + ); - // Best path cost is the bottom right element of the matrix - const columnCount = accumulatedCostMatrixTransposed.length; - const rowCount = accumulatedCostMatrixTransposed[0].length; + // Best path cost is the bottom right element of the matrix + const columnCount = accumulatedCostMatrixTransposed.length; + const rowCount = accumulatedCostMatrixTransposed[0].length; - const pathCost = - accumulatedCostMatrixTransposed[columnCount - 1][rowCount - 1]; + const pathCost = + accumulatedCostMatrixTransposed[columnCount - 1][rowCount - 1]; - // Return - return { path, pathCost }; + // Return + return { path, pathCost }; } async function alignMFCC_DTW( - mfccFrames1: number[][], - mfccFrames2: number[][], - windowLength: number, - distanceFunctionKind: "euclidian" | "cosine" = "euclidian", - centerIndexes?: number[], + mfccFrames1: number[][], + mfccFrames2: number[][], + windowLength: number, + distanceFunctionKind: "euclidian" | "cosine" = "euclidian", + centerIndexes?: number[] ) { - if (distanceFunctionKind == "euclidian") { - let distanceFunction = euclidianDistance; + if (distanceFunctionKind == "euclidian") { + let distanceFunction = euclidianDistance; - if (mfccFrames1.length > 0 && mfccFrames1[0].length === 13) { - distanceFunction = euclidianDistance13Dim; - } + if (mfccFrames1.length > 0 && mfccFrames1[0].length === 13) { + distanceFunction = euclidianDistance13Dim; + } - const { path } = alignDTWWindowed( - mfccFrames1, - mfccFrames2, - distanceFunction, - windowLength, - centerIndexes, - ); + const { path } = alignDTWWindowed( + mfccFrames1, + mfccFrames2, + distanceFunction, + windowLength, + centerIndexes + ); - return path; - } else if (distanceFunctionKind == "cosine") { - const indexes1 = createVectorForIntegerRange(0, mfccFrames1.length); - const indexes2 = createVectorForIntegerRange(0, mfccFrames2.length); - - const magnitudes1 = mfccFrames1.map(magnitude); - const magnitudes2 = mfccFrames2.map(magnitude); - - const { path } = alignDTWWindowed( - indexes1, - indexes2, - (i, j) => - cosineDistancePrecomputedMagnitudes( - mfccFrames1[i], - mfccFrames2[j], - magnitudes1[i], - magnitudes2[j], - ), - windowLength, - centerIndexes, - ); + return path; + } else if (distanceFunctionKind == "cosine") { + const indexes1 = createVectorForIntegerRange(0, mfccFrames1.length); + const indexes2 = createVectorForIntegerRange(0, mfccFrames2.length); + + const magnitudes1 = mfccFrames1.map(magnitude); + const magnitudes2 = mfccFrames2.map(magnitude); + + const { path } = alignDTWWindowed( + indexes1, + indexes2, + (i, j) => + cosineDistancePrecomputedMagnitudes( + mfccFrames1[i], + mfccFrames2[j], + magnitudes1[i], + magnitudes2[j] + ), + windowLength, + centerIndexes + ); - return path; - } else { - throw new Error("Invalid distance function"); - } + return path; + } else { + throw new Error("Invalid distance function"); + } } let kissFFTInstance: any; @@ -418,1045 +426,1072 @@ let kissFFTInstance: any; type WindowType = "hann" | "hamming" | "povey"; function getBinFrequencies(binCount: number, maxFrequency: number) { - const binFrequencies = new Float32Array(binCount); - const frequencyStep = maxFrequency / (binCount - 1); + const binFrequencies = new Float32Array(binCount); + const frequencyStep = maxFrequency / (binCount - 1); - for ( - let i = 0, frequency = 0; - i < binFrequencies.length; - i++, frequency += frequencyStep - ) { - binFrequencies[i] = frequency; - } + for ( + let i = 0, frequency = 0; + i < binFrequencies.length; + i++, frequency += frequencyStep + ) { + binFrequencies[i] = frequency; + } - return binFrequencies; + return binFrequencies; } function fftFrameToPowerSpectrum(fftFrame: Float32Array) { - const powerSpectrum = new Float32Array(fftFrame.length / 2); + const powerSpectrum = new Float32Array(fftFrame.length / 2); - for (let i = 0; i < powerSpectrum.length; i++) { - const binOffset = i * 2; - const fftCoefficientRealPart = fftFrame[binOffset]; - const fftCoefficientImaginaryPart = fftFrame[binOffset + 1]; - const binPower = - fftCoefficientRealPart ** 2 + fftCoefficientImaginaryPart ** 2; + for (let i = 0; i < powerSpectrum.length; i++) { + const binOffset = i * 2; + const fftCoefficientRealPart = fftFrame[binOffset]; + const fftCoefficientImaginaryPart = fftFrame[binOffset + 1]; + const binPower = + fftCoefficientRealPart ** 2 + fftCoefficientImaginaryPart ** 2; - powerSpectrum[i] = binPower; - } + powerSpectrum[i] = binPower; + } - return powerSpectrum; + return powerSpectrum; } async function getKissFFTInstance() { - if (!kissFFTInstance) { - const { default: initializer } = await import("@echogarden/kissfft-wasm"); + if (!kissFFTInstance) { + const { default: initializer } = await import( + "@echogarden/kissfft-wasm" + ); - kissFFTInstance = await initializer(); - } + kissFFTInstance = await initializer(); + } - return kissFFTInstance; + return kissFFTInstance; } function getWindowWeights(windowType: WindowType, windowSize: number) { - const weights = new Float32Array(windowSize); + const weights = new Float32Array(windowSize); - const innerFactor = (2 * Math.PI) / (windowSize - 1); + const innerFactor = (2 * Math.PI) / (windowSize - 1); - if (windowType == "hann") { - for (let i = 0; i < windowSize; i++) { - //weights[i] = 0.5 * (1 - Math.cos(2 * Math.PI * (i / (windowSize - 1)))) - weights[i] = 0.5 * (1 - Math.cos(innerFactor * i)); - } - } else if (windowType == "hamming") { - for (let i = 0; i < windowSize; i++) { - //weights[i] = 0.54 - (0.46 * Math.cos(2 * Math.PI * (i / (windowSize - 1)))) - weights[i] = 0.54 - 0.46 * Math.cos(innerFactor * i); - } - } else if (windowType == "povey") { - const hannWeights = getWindowWeights("hann", windowSize); + if (windowType == "hann") { + for (let i = 0; i < windowSize; i++) { + //weights[i] = 0.5 * (1 - Math.cos(2 * Math.PI * (i / (windowSize - 1)))) + weights[i] = 0.5 * (1 - Math.cos(innerFactor * i)); + } + } else if (windowType == "hamming") { + for (let i = 0; i < windowSize; i++) { + //weights[i] = 0.54 - (0.46 * Math.cos(2 * Math.PI * (i / (windowSize - 1)))) + weights[i] = 0.54 - 0.46 * Math.cos(innerFactor * i); + } + } else if (windowType == "povey") { + const hannWeights = getWindowWeights("hann", windowSize); - for (let i = 0; i < windowSize; i++) { - weights[i] = hannWeights[i] ** 0.85; + for (let i = 0; i < windowSize; i++) { + weights[i] = hannWeights[i] ** 0.85; + } + } else { + throw new Error(`Unsupported window function type: ${windowType}`); } - } else { - throw new Error(`Unsupported window function type: ${windowType}`); - } - return weights; + return weights; } async function stftr( - samples: Float32Array, - fftOrder: number, - windowSize: number, - hopSize: number, - windowType: WindowType, + samples: Float32Array, + fftOrder: number, + windowSize: number, + hopSize: number, + windowType: WindowType ) { - if (fftOrder % 2 != 0 || windowSize % 2 != 0) { - throw new Error("FFT order and window size must multiples of 2"); - } + if (fftOrder % 2 != 0 || windowSize % 2 != 0) { + throw new Error("FFT order and window size must multiples of 2"); + } - if (windowSize > fftOrder) { - throw new Error("Window size must be lesser or equal to the FFT size"); - } + if (windowSize > fftOrder) { + throw new Error("Window size must be lesser or equal to the FFT size"); + } - if (hopSize > windowSize) { - throw new Error("Hop size must be lesser or equal to the window size"); - } + if (hopSize > windowSize) { + throw new Error("Hop size must be lesser or equal to the window size"); + } - const halfWindowSize = windowSize / 2; + const halfWindowSize = windowSize / 2; - const padding = new Float32Array(halfWindowSize); - samples = concatFloat32Arrays([padding, samples, padding]); + const padding = new Float32Array(halfWindowSize); + samples = concatFloat32Arrays([padding, samples, padding]); - const windowWeights = getWindowWeights(windowType, windowSize); + const windowWeights = getWindowWeights(windowType, windowSize); - const m = await getKissFFTInstance(); - const wasmMemory = new WasmMemoryManager(m); + const m = await getKissFFTInstance(); + const wasmMemory = new WasmMemoryManager(m); - const statePtr = m._kiss_fftr_alloc(fftOrder, 0, 0, 0); - wasmMemory.wrapPointer(statePtr); + const statePtr = m._kiss_fftr_alloc(fftOrder, 0, 0, 0); + wasmMemory.wrapPointer(statePtr); - const sampleCount = samples.length; - const frameBufferRef = wasmMemory.allocFloat32Array(fftOrder); - const binsBufferRef = wasmMemory.allocFloat32Array(fftOrder * 2); + const sampleCount = samples.length; + const frameBufferRef = wasmMemory.allocFloat32Array(fftOrder); + const binsBufferRef = wasmMemory.allocFloat32Array(fftOrder * 2); - const frames: Float32Array[] = []; + const frames: Float32Array[] = []; - for (let offset = 0; offset < sampleCount; offset += hopSize) { - const windowSamples = samples.subarray(offset, offset + windowSize); - frameBufferRef.clear(); + for (let offset = 0; offset < sampleCount; offset += hopSize) { + const windowSamples = samples.subarray(offset, offset + windowSize); + frameBufferRef.clear(); - const frameBufferView = frameBufferRef.view; + const frameBufferView = frameBufferRef.view; - for (let i = 0; i < windowSamples.length; i++) { - frameBufferView[i] = windowSamples[i] * windowWeights[i]; - } + for (let i = 0; i < windowSamples.length; i++) { + frameBufferView[i] = windowSamples[i] * windowWeights[i]; + } - binsBufferRef.clear(); + binsBufferRef.clear(); - m._kiss_fftr(statePtr, frameBufferRef.address, binsBufferRef.address); + m._kiss_fftr(statePtr, frameBufferRef.address, binsBufferRef.address); - const bins = binsBufferRef.view.slice(0, fftOrder + 2); - frames.push(bins); - } + const bins = binsBufferRef.view.slice(0, fftOrder + 2); + frames.push(bins); + } - wasmMemory.freeAll(); + wasmMemory.freeAll(); - return frames; + return frames; } async function computeMelSpectogramUsingFilterbanks( - rawAudio: RawAudio, - fftOrder: number, - windowSize: number, - hopLength: number, - filterbanks: Filterbank[], - windowType: WindowType = "hann", + rawAudio: RawAudio, + fftOrder: number, + windowSize: number, + hopLength: number, + filterbanks: Filterbank[], + windowType: WindowType = "hann" ) { - const logger = new Logger(); + const logger = new Logger(); - logger.start("Compute short-time FFTs"); - const audioSamples = rawAudio.audioChannels[0]; - const fftFrames = await stftr( - audioSamples, - fftOrder, - windowSize, - hopLength, - windowType, - ); + logger.start("Compute short-time FFTs"); + const audioSamples = rawAudio.audioChannels[0]; + const fftFrames = await stftr( + audioSamples, + fftOrder, + windowSize, + hopLength, + windowType + ); - logger.start("Convert FFT frames to a mel spectogram"); - const melSpectogram = fftFramesToMelSpectogram(fftFrames, filterbanks); + logger.start("Convert FFT frames to a mel spectogram"); + const melSpectogram = fftFramesToMelSpectogram(fftFrames, filterbanks); - logger.end(); + logger.end(); - return { melSpectogram, fftFrames }; + return { melSpectogram, fftFrames }; } function fftFramesToMelSpectogram( - fftFrames: Float32Array[], - melFilterbanks: Filterbank[], + fftFrames: Float32Array[], + melFilterbanks: Filterbank[] ) { - return fftFrames.map((fftFrame) => { - const powerSpectrum = fftFrameToPowerSpectrum(fftFrame); - return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbanks); - }); + return fftFrames.map((fftFrame) => { + const powerSpectrum = fftFrameToPowerSpectrum(fftFrame); + return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbanks); + }); } function powerSpectrumToMelSpectrum( - powerSpectrum: Float32Array, - filterbanks: Filterbank[], + powerSpectrum: Float32Array, + filterbanks: Filterbank[] ) { - const filterbankCount = filterbanks.length; - const melSpectrum = new Float32Array(filterbankCount); + const filterbankCount = filterbanks.length; + const melSpectrum = new Float32Array(filterbankCount); - for (let melBandIndex = 0; melBandIndex < filterbankCount; melBandIndex++) { - const filterbank = filterbanks[melBandIndex]; - const filterbankStartIndex = filterbank.startIndex; - const filterbankWeights = filterbank.weights; + for (let melBandIndex = 0; melBandIndex < filterbankCount; melBandIndex++) { + const filterbank = filterbanks[melBandIndex]; + const filterbankStartIndex = filterbank.startIndex; + const filterbankWeights = filterbank.weights; - if (filterbankStartIndex === -1) { - continue; - } + if (filterbankStartIndex === -1) { + continue; + } - let melBandValue = 0; + let melBandValue = 0; - for (let i = 0; i < filterbankWeights.length; i++) { - const powerSpectrumIndex = filterbankStartIndex + i; + for (let i = 0; i < filterbankWeights.length; i++) { + const powerSpectrumIndex = filterbankStartIndex + i; - if (powerSpectrumIndex >= powerSpectrum.length) { - break; - } + if (powerSpectrumIndex >= powerSpectrum.length) { + break; + } - const weight = filterbankWeights[i]; - const powerSpectrumValue = powerSpectrum[powerSpectrumIndex]; + const weight = filterbankWeights[i]; + const powerSpectrumValue = powerSpectrum[powerSpectrumIndex]; - melBandValue += weight * powerSpectrumValue; - } + melBandValue += weight * powerSpectrumValue; + } - melSpectrum[melBandIndex] = melBandValue; - } + melSpectrum[melBandIndex] = melBandValue; + } - return melSpectrum; + return melSpectrum; } function getMelFilterbanks( - powerSpectrumFrequenciesHz: Float32Array, - centerFrequenciesMel: Float32Array, - lowerFrequencyMel: number, - upperFrequencyMel: number, + powerSpectrumFrequenciesHz: Float32Array, + centerFrequenciesMel: Float32Array, + lowerFrequencyMel: number, + upperFrequencyMel: number ) { - const filterbankCount = centerFrequenciesMel.length; - const powerSpectrumFrequenciesMel = powerSpectrumFrequenciesHz.map( - (frequencyHz) => hertzToMel(frequencyHz), - ); - - const filterbanks: Filterbank[] = []; - - for ( - let filterbankIndex = 0; - filterbankIndex < filterbankCount; - filterbankIndex++ - ) { - const centerFrequency = centerFrequenciesMel[filterbankIndex]; - - const leftFrequency = - filterbankIndex > 0 - ? centerFrequenciesMel[filterbankIndex - 1] - : lowerFrequencyMel; - const rightFrequency = - filterbankIndex < filterbankCount - 1 - ? centerFrequenciesMel[filterbankIndex + 1] - : upperFrequencyMel; - - const width = rightFrequency - leftFrequency; - const halfWidth = width / 2; - - let startIndex = -1; - let weights: number[] = []; + const filterbankCount = centerFrequenciesMel.length; + const powerSpectrumFrequenciesMel = powerSpectrumFrequenciesHz.map( + (frequencyHz) => hertzToMel(frequencyHz) + ); - let weightSum = 0; + const filterbanks: Filterbank[] = []; for ( - let powerSpectrumBandIndex = 0; - powerSpectrumBandIndex < powerSpectrumFrequenciesMel.length; - powerSpectrumBandIndex++ + let filterbankIndex = 0; + filterbankIndex < filterbankCount; + filterbankIndex++ ) { - const powerSpectrumBandFrequencyMel = - powerSpectrumFrequenciesMel[powerSpectrumBandIndex]; - - let weight = 0; + const centerFrequency = centerFrequenciesMel[filterbankIndex]; + + const leftFrequency = + filterbankIndex > 0 + ? centerFrequenciesMel[filterbankIndex - 1] + : lowerFrequencyMel; + const rightFrequency = + filterbankIndex < filterbankCount - 1 + ? centerFrequenciesMel[filterbankIndex + 1] + : upperFrequencyMel; + + const width = rightFrequency - leftFrequency; + const halfWidth = width / 2; + + let startIndex = -1; + let weights: number[] = []; + + let weightSum = 0; + + for ( + let powerSpectrumBandIndex = 0; + powerSpectrumBandIndex < powerSpectrumFrequenciesMel.length; + powerSpectrumBandIndex++ + ) { + const powerSpectrumBandFrequencyMel = + powerSpectrumFrequenciesMel[powerSpectrumBandIndex]; + + let weight = 0; + + if ( + powerSpectrumBandFrequencyMel >= leftFrequency && + powerSpectrumBandFrequencyMel <= centerFrequency + ) { + weight = + (powerSpectrumBandFrequencyMel - leftFrequency) / halfWidth; + } else if ( + powerSpectrumBandFrequencyMel > centerFrequency && + powerSpectrumBandFrequencyMel <= rightFrequency + ) { + weight = + (rightFrequency - powerSpectrumBandFrequencyMel) / + halfWidth; + } - if ( - powerSpectrumBandFrequencyMel >= leftFrequency && - powerSpectrumBandFrequencyMel <= centerFrequency - ) { - weight = (powerSpectrumBandFrequencyMel - leftFrequency) / halfWidth; - } else if ( - powerSpectrumBandFrequencyMel > centerFrequency && - powerSpectrumBandFrequencyMel <= rightFrequency - ) { - weight = (rightFrequency - powerSpectrumBandFrequencyMel) / halfWidth; - } + if (weight > 0) { + if (startIndex == -1) { + startIndex = powerSpectrumBandIndex; + } - if (weight > 0) { - if (startIndex == -1) { - startIndex = powerSpectrumBandIndex; + weights.push(weight); + weightSum += weight; + } else if (startIndex != -1) { + break; + } } - weights.push(weight); - weightSum += weight; - } else if (startIndex != -1) { - break; - } - } - - weights = weights.map((weight) => weight / weightSum); + weights = weights.map((weight) => weight / weightSum); - filterbanks.push({ startIndex, weights }); - } + filterbanks.push({ startIndex, weights }); + } - return filterbanks; + return filterbanks; } function getMelFilterbanksCenterFrequencies( - melBandCount: number, - lowerFrequencyMel: number, - upperFrequencyMel: number, + melBandCount: number, + lowerFrequencyMel: number, + upperFrequencyMel: number ) { - const stepSizeMel = - (upperFrequencyMel - lowerFrequencyMel) / (melBandCount + 1); + const stepSizeMel = + (upperFrequencyMel - lowerFrequencyMel) / (melBandCount + 1); - const centerFrequencies = new Float32Array(melBandCount); + const centerFrequencies = new Float32Array(melBandCount); - for (let i = 0; i < melBandCount; i++) { - centerFrequencies[i] = lowerFrequencyMel + (i + 1) * stepSizeMel; - } + for (let i = 0; i < melBandCount; i++) { + centerFrequencies[i] = lowerFrequencyMel + (i + 1) * stepSizeMel; + } - return centerFrequencies; + return centerFrequencies; } function hertzToMel(frequency: number) { - return 2595.0 * Math.log10(1.0 + frequency / 700.0); + return 2595.0 * Math.log10(1.0 + frequency / 700.0); } type Filterbank = { - startIndex: number; - weights: number[]; + startIndex: number; + weights: number[]; }; async function computeMelSpectogram( - rawAudio: RawAudio, - fftOrder: number, - windowSize: number, - hopLength: number, - filterbankCount: number, - lowerFrequencyHz: number, - upperFrequencyHz: number, - windowType: WindowType = "hann", + rawAudio: RawAudio, + fftOrder: number, + windowSize: number, + hopLength: number, + filterbankCount: number, + lowerFrequencyHz: number, + upperFrequencyHz: number, + windowType: WindowType = "hann" ) { - const logger = new Logger(); - - logger.start("Compute mel filterbank"); - const binCount = fftOrder / 2 + 2; - const nyquistFrequency = rawAudio.sampleRate / 2; - const binFrequencies = getBinFrequencies(binCount, nyquistFrequency); - - const lowerFrequencyMel = hertzToMel(lowerFrequencyHz); - const upperFrequencyMel = hertzToMel(upperFrequencyHz); - - const filterbanksCenterFrequencies = getMelFilterbanksCenterFrequencies( - filterbankCount, - lowerFrequencyMel, - upperFrequencyMel, - ); - const melFilterbanks = getMelFilterbanks( - binFrequencies, - filterbanksCenterFrequencies, - lowerFrequencyMel, - upperFrequencyMel, - ); - - logger.end(); - - return computeMelSpectogramUsingFilterbanks( - rawAudio, - fftOrder, - windowSize, - hopLength, - melFilterbanks, - windowType, - ); + const logger = new Logger(); + + logger.start("Compute mel filterbank"); + const binCount = fftOrder / 2 + 2; + const nyquistFrequency = rawAudio.sampleRate / 2; + const binFrequencies = getBinFrequencies(binCount, nyquistFrequency); + + const lowerFrequencyMel = hertzToMel(lowerFrequencyHz); + const upperFrequencyMel = hertzToMel(upperFrequencyHz); + + const filterbanksCenterFrequencies = getMelFilterbanksCenterFrequencies( + filterbankCount, + lowerFrequencyMel, + upperFrequencyMel + ); + const melFilterbanks = getMelFilterbanks( + binFrequencies, + filterbanksCenterFrequencies, + lowerFrequencyMel, + upperFrequencyMel + ); + + logger.end(); + + return computeMelSpectogramUsingFilterbanks( + rawAudio, + fftOrder, + windowSize, + hopLength, + melFilterbanks, + windowType + ); } function powerToDecibels(power: number) { - return power <= 0.0000000001 ? -100 : 10.0 * Math.log10(power); + return power <= 0.0000000001 ? -100 : 10.0 * Math.log10(power); } function melSpectrumToMFCC( - melSpectrum: Float32Array, - mfccFeatureCount: number, - dctMatrix: Float32Array[], - normalization: "none" | "orthonormal" = "orthonormal", + melSpectrum: Float32Array, + mfccFeatureCount: number, + dctMatrix: Float32Array[], + normalization: "none" | "orthonormal" = "orthonormal" ) { - const melBandCount = melSpectrum.length; + const melBandCount = melSpectrum.length; - let firstFeatureNormalizationFactor: number; - let nonfirstFeatureNormalizationFactor: number; + let firstFeatureNormalizationFactor: number; + let nonfirstFeatureNormalizationFactor: number; - if (normalization == "orthonormal") { - firstFeatureNormalizationFactor = Math.sqrt(1 / (4 * mfccFeatureCount)); - nonfirstFeatureNormalizationFactor = Math.sqrt(1 / (2 * mfccFeatureCount)); - } else { - firstFeatureNormalizationFactor = 1; - nonfirstFeatureNormalizationFactor = 1; - } + if (normalization == "orthonormal") { + firstFeatureNormalizationFactor = Math.sqrt(1 / (4 * mfccFeatureCount)); + nonfirstFeatureNormalizationFactor = Math.sqrt( + 1 / (2 * mfccFeatureCount) + ); + } else { + firstFeatureNormalizationFactor = 1; + nonfirstFeatureNormalizationFactor = 1; + } - const mfcc = new Float32Array(mfccFeatureCount); + const mfcc = new Float32Array(mfccFeatureCount); - for ( - let mfccFeatureIndex = 0; - mfccFeatureIndex < mfccFeatureCount; - mfccFeatureIndex++ - ) { - const dctMatrixRow = dctMatrix[mfccFeatureIndex]; + for ( + let mfccFeatureIndex = 0; + mfccFeatureIndex < mfccFeatureCount; + mfccFeatureIndex++ + ) { + const dctMatrixRow = dctMatrix[mfccFeatureIndex]; - let sum = 0; + let sum = 0; - for (let j = 0; j < melBandCount; j++) { - const dctCoefficient = dctMatrixRow[j]; - const logMel = powerToDecibels(melSpectrum[j]); - //const logMel = Math.log(1e-40 + melSpectrum[j]) + for (let j = 0; j < melBandCount; j++) { + const dctCoefficient = dctMatrixRow[j]; + const logMel = powerToDecibels(melSpectrum[j]); + //const logMel = Math.log(1e-40 + melSpectrum[j]) - sum += dctCoefficient * logMel; - } + sum += dctCoefficient * logMel; + } - const normalizationFactor = - mfccFeatureIndex == 0 - ? firstFeatureNormalizationFactor - : nonfirstFeatureNormalizationFactor; + const normalizationFactor = + mfccFeatureIndex == 0 + ? firstFeatureNormalizationFactor + : nonfirstFeatureNormalizationFactor; - //mfcc[mfccFeatureIndex] = normalizationFactor * sum - mfcc[mfccFeatureIndex] = normalizationFactor * 2 * sum; // Sum multiplied by 2 to match with librosa - } + //mfcc[mfccFeatureIndex] = normalizationFactor * sum + mfcc[mfccFeatureIndex] = normalizationFactor * 2 * sum; // Sum multiplied by 2 to match with librosa + } - return mfcc; + return mfcc; } function createDCTType2CoefficientMatrix( - mfccFeatureCount: number, - melBandCount: number, + mfccFeatureCount: number, + melBandCount: number ) { - const dctMatrix = new Array(mfccFeatureCount); + const dctMatrix = new Array(mfccFeatureCount); - for ( - let mfccFeatureIndex = 0; - mfccFeatureIndex < mfccFeatureCount; - mfccFeatureIndex++ - ) { - const row = new Float32Array(melBandCount); + for ( + let mfccFeatureIndex = 0; + mfccFeatureIndex < mfccFeatureCount; + mfccFeatureIndex++ + ) { + const row = new Float32Array(melBandCount); - const innerMultiplier = (Math.PI * mfccFeatureIndex) / melBandCount; + const innerMultiplier = (Math.PI * mfccFeatureIndex) / melBandCount; - for (let melBandIndex = 0; melBandIndex < melBandCount; melBandIndex++) { - row[melBandIndex] = Math.cos(innerMultiplier * (melBandIndex + 0.5)); - } + for ( + let melBandIndex = 0; + melBandIndex < melBandCount; + melBandIndex++ + ) { + row[melBandIndex] = Math.cos( + innerMultiplier * (melBandIndex + 0.5) + ); + } - dctMatrix[mfccFeatureIndex] = row; - } + dctMatrix[mfccFeatureIndex] = row; + } - return dctMatrix; + return dctMatrix; } function melSpectogramToMFCCs( - melSpectogram: Float32Array[], - mfccFeatureCount: number, + melSpectogram: Float32Array[], + mfccFeatureCount: number ) { - const melBandCount = melSpectogram[0].length; - const dctMatrix = createDCTType2CoefficientMatrix( - mfccFeatureCount, - melBandCount, - ); + const melBandCount = melSpectogram[0].length; + const dctMatrix = createDCTType2CoefficientMatrix( + mfccFeatureCount, + melBandCount + ); - const mfccs = melSpectogram.map((frame) => - melSpectrumToMFCC(frame, mfccFeatureCount, dctMatrix), - ); + const mfccs = melSpectogram.map((frame) => + melSpectrumToMFCC(frame, mfccFeatureCount, dctMatrix) + ); - return mfccs; + return mfccs; } function applyEmphasis( - samples: Float32Array, - emphasisFactor = 0.97, - initialState = 0, + samples: Float32Array, + emphasisFactor = 0.97, + initialState = 0 ) { - const processedSamples = new Float32Array(samples.length); + const processedSamples = new Float32Array(samples.length); - processedSamples[0] = samples[0] - emphasisFactor * initialState; + processedSamples[0] = samples[0] - emphasisFactor * initialState; - for (let i = 1; i < processedSamples.length; i++) { - processedSamples[i] = samples[i] - emphasisFactor * samples[i - 1]; - } + for (let i = 1; i < processedSamples.length; i++) { + processedSamples[i] = samples[i] - emphasisFactor * samples[i - 1]; + } - return processedSamples; + return processedSamples; } function cloneRawAudio(rawAudio: RawAudio): RawAudio { - return { - audioChannels: rawAudio.audioChannels.map((channel) => channel.slice()), - sampleRate: rawAudio.sampleRate, - }; + return { + audioChannels: rawAudio.audioChannels.map((channel) => channel.slice()), + sampleRate: rawAudio.sampleRate, + }; } let speexResamplerInstance: any; async function resampleAudioSpeex( - rawAudio: RawAudio, - outSampleRate: number, - quality = 0, + rawAudio: RawAudio, + outSampleRate: number, + quality = 0 ): Promise { - const channelCount = rawAudio.audioChannels.length; - const inSampleRate = rawAudio.sampleRate; + const channelCount = rawAudio.audioChannels.length; + const inSampleRate = rawAudio.sampleRate; - const totalSampleCount = rawAudio.audioChannels[0].length; - const sampleRateRatio = outSampleRate / inSampleRate; + const totalSampleCount = rawAudio.audioChannels[0].length; + const sampleRateRatio = outSampleRate / inSampleRate; - if (inSampleRate === outSampleRate) { - return cloneRawAudio(rawAudio); - } + if (inSampleRate === outSampleRate) { + return cloneRawAudio(rawAudio); + } - if (totalSampleCount === 0) { - return { - ...cloneRawAudio(rawAudio), - sampleRate: outSampleRate, - } as RawAudio; - } - - const m = await getSpeexResamplerInstance(); - const wasmMemory = new WasmMemoryManager(m); - - function speexResultCodeToString(resultCode: number) { - const errorStrPtr = m._speex_resampler_strerror(resultCode); - const errorStrRef = wasmMemory.wrapUint8Array(errorStrPtr, 1024); - const message = errorStrRef.readAsNullTerminatedUtf8String(); - - return message; - } - - const initErrRef = wasmMemory.allocInt32(); - const resamplerStateAddress = m._speex_resampler_init( - channelCount, - inSampleRate, - outSampleRate, - quality, - initErrRef.address, - ); - let resultCode = initErrRef.value; - - if (resultCode != 0) { - throw new Error( - `Speex resampler failed while initializing with code ${resultCode}: ${speexResultCodeToString(resultCode)}`, + if (totalSampleCount === 0) { + return { + ...cloneRawAudio(rawAudio), + sampleRate: outSampleRate, + } as RawAudio; + } + + const m = await getSpeexResamplerInstance(); + const wasmMemory = new WasmMemoryManager(m); + + function speexResultCodeToString(resultCode: number) { + const errorStrPtr = m._speex_resampler_strerror(resultCode); + const errorStrRef = wasmMemory.wrapUint8Array(errorStrPtr, 1024); + const message = errorStrRef.readAsNullTerminatedUtf8String(); + + return message; + } + + const initErrRef = wasmMemory.allocInt32(); + const resamplerStateAddress = m._speex_resampler_init( + channelCount, + inSampleRate, + outSampleRate, + quality, + initErrRef.address ); - } + let resultCode = initErrRef.value; - const inputLatency = m._speex_resampler_get_input_latency( - resamplerStateAddress, - ); - const outputLatency = m._speex_resampler_get_output_latency( - resamplerStateAddress, - ); + if (resultCode != 0) { + throw new Error( + `Speex resampler failed while initializing with code ${resultCode}: ${speexResultCodeToString(resultCode)}` + ); + } - const maxChunkSize = 2 ** 20; + const inputLatency = m._speex_resampler_get_input_latency( + resamplerStateAddress + ); + const outputLatency = m._speex_resampler_get_output_latency( + resamplerStateAddress + ); - const inputChunkSampleCountRef = wasmMemory.allocInt32(); - const outputChunkSampleCountRef = wasmMemory.allocInt32(); + const maxChunkSize = 2 ** 20; - const inputChunkSamplesRef = wasmMemory.allocFloat32Array(maxChunkSize * 2); - const outputChunkSamplesRef = wasmMemory.allocFloat32Array( - Math.floor(maxChunkSize * sampleRateRatio) * 2, - ); + const inputChunkSampleCountRef = wasmMemory.allocInt32(); + const outputChunkSampleCountRef = wasmMemory.allocInt32(); - const resampledAudioChunksForChannels: Float32Array[][] = []; + const inputChunkSamplesRef = wasmMemory.allocFloat32Array(maxChunkSize * 2); + const outputChunkSamplesRef = wasmMemory.allocFloat32Array( + Math.floor(maxChunkSize * sampleRateRatio) * 2 + ); - for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { - resampledAudioChunksForChannels.push([]); - } + const resampledAudioChunksForChannels: Float32Array[][] = []; - for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { - for (let readOffset = 0; readOffset < totalSampleCount; ) { - const isLastChunk = readOffset + maxChunkSize >= totalSampleCount; + for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { + resampledAudioChunksForChannels.push([]); + } - const inputPaddingSize = isLastChunk ? inputLatency : 0; - const maxSamplesToRead = - Math.min(maxChunkSize, totalSampleCount - readOffset) + - inputPaddingSize; + for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { + for (let readOffset = 0; readOffset < totalSampleCount; ) { + const isLastChunk = readOffset + maxChunkSize >= totalSampleCount; - const maxSamplesToWrite = outputChunkSamplesRef.length; + const inputPaddingSize = isLastChunk ? inputLatency : 0; + const maxSamplesToRead = + Math.min(maxChunkSize, totalSampleCount - readOffset) + + inputPaddingSize; - const inputChunkSamplesForChannel = rawAudio.audioChannels[ - channelIndex - ].slice(readOffset, readOffset + maxSamplesToRead); + const maxSamplesToWrite = outputChunkSamplesRef.length; - inputChunkSampleCountRef.value = maxSamplesToRead; - outputChunkSampleCountRef.value = maxSamplesToWrite; + const inputChunkSamplesForChannel = rawAudio.audioChannels[ + channelIndex + ].slice(readOffset, readOffset + maxSamplesToRead); - inputChunkSamplesRef.view.set(inputChunkSamplesForChannel); - resultCode = m._speex_resampler_process_float( - resamplerStateAddress, - channelIndex, - inputChunkSamplesRef.address, - inputChunkSampleCountRef.address, - outputChunkSamplesRef.address, - outputChunkSampleCountRef.address, - ); + inputChunkSampleCountRef.value = maxSamplesToRead; + outputChunkSampleCountRef.value = maxSamplesToWrite; - if (resultCode != 0) { - throw new Error( - `Speex resampler failed while resampling with code ${resultCode}: ${speexResultCodeToString(resultCode)}`, - ); - } + inputChunkSamplesRef.view.set(inputChunkSamplesForChannel); + resultCode = m._speex_resampler_process_float( + resamplerStateAddress, + channelIndex, + inputChunkSamplesRef.address, + inputChunkSampleCountRef.address, + outputChunkSamplesRef.address, + outputChunkSampleCountRef.address + ); - const samplesReadCount = inputChunkSampleCountRef.value; - const samplesWrittenCount = outputChunkSampleCountRef.value; + if (resultCode != 0) { + throw new Error( + `Speex resampler failed while resampling with code ${resultCode}: ${speexResultCodeToString(resultCode)}` + ); + } - const resampledChannelAudio = outputChunkSamplesRef.view.slice( - 0, - samplesWrittenCount, - ); + const samplesReadCount = inputChunkSampleCountRef.value; + const samplesWrittenCount = outputChunkSampleCountRef.value; - resampledAudioChunksForChannels[channelIndex].push(resampledChannelAudio); + const resampledChannelAudio = outputChunkSamplesRef.view.slice( + 0, + samplesWrittenCount + ); + + resampledAudioChunksForChannels[channelIndex].push( + resampledChannelAudio + ); - readOffset += samplesReadCount; + readOffset += samplesReadCount; + } } - } - m._speex_resampler_destroy(resamplerStateAddress); - wasmMemory.freeAll(); + m._speex_resampler_destroy(resamplerStateAddress); + wasmMemory.freeAll(); - const resampledAudio: RawAudio = { - audioChannels: [], - sampleRate: outSampleRate, - }; + const resampledAudio: RawAudio = { + audioChannels: [], + sampleRate: outSampleRate, + }; - for (let i = 0; i < channelCount; i++) { - resampledAudioChunksForChannels[i][0] = - resampledAudioChunksForChannels[i][0].slice(outputLatency); + for (let i = 0; i < channelCount; i++) { + resampledAudioChunksForChannels[i][0] = + resampledAudioChunksForChannels[i][0].slice(outputLatency); - resampledAudio.audioChannels.push( - concatFloat32Arrays(resampledAudioChunksForChannels[i]), - ); - } + resampledAudio.audioChannels.push( + concatFloat32Arrays(resampledAudioChunksForChannels[i]) + ); + } - return resampledAudio; + return resampledAudio; } async function getSpeexResamplerInstance() { - if (!speexResamplerInstance) { - const { default: SpeexResamplerInitializer } = await import( - "@echogarden/speex-resampler-wasm" - ); + if (!speexResamplerInstance) { + const { default: SpeexResamplerInitializer } = await import( + "@echogarden/speex-resampler-wasm" + ); - speexResamplerInstance = await SpeexResamplerInitializer(); - } + speexResamplerInstance = await SpeexResamplerInitializer(); + } - return speexResamplerInstance; + return speexResamplerInstance; } function clip(num: number, min: number, max: number) { - return Math.max(min, Math.min(max, num)); + return Math.max(min, Math.min(max, num)); } function normalizeVectors( - vectors: number[][], - kind: "population" | "sample" = "population", + vectors: number[][], + kind: "population" | "sample" = "population" ) { - const vectorCount = vectors.length; + const vectorCount = vectors.length; + + if (vectorCount == 0) { + return { normalizedVectors: [], mean: [], stdDeviation: [] }; + } - if (vectorCount == 0) { - return { normalizedVectors: [], mean: [], stdDeviation: [] }; - } + const featureCount = vectors[0].length; - const featureCount = vectors[0].length; + const mean = meanOfVectors(vectors); + const stdDeviation = stdDeviationOfVectors(vectors, kind, mean); - const mean = meanOfVectors(vectors); - const stdDeviation = stdDeviationOfVectors(vectors, kind, mean); + const normalizedVectors: number[][] = []; - const normalizedVectors: number[][] = []; + for (const vector of vectors) { + const normalizedVector = createVector(featureCount); - for (const vector of vectors) { - const normalizedVector = createVector(featureCount); + for ( + let featureIndex = 0; + featureIndex < featureCount; + featureIndex++ + ) { + normalizedVector[featureIndex] = + (vector[featureIndex] - mean[featureIndex]) / + stdDeviation[featureIndex]; - for (let featureIndex = 0; featureIndex < featureCount; featureIndex++) { - normalizedVector[featureIndex] = - (vector[featureIndex] - mean[featureIndex]) / - stdDeviation[featureIndex]; + normalizedVector[featureIndex] = zeroIfNaN( + normalizedVector[featureIndex] + ); + } - normalizedVector[featureIndex] = zeroIfNaN( - normalizedVector[featureIndex], - ); + normalizedVectors.push(normalizedVector); } - normalizedVectors.push(normalizedVector); - } - - return { normalizedVectors, mean, stdDeviation }; + return { normalizedVectors, mean, stdDeviation }; } function meanOfVectors(vectors: number[][]) { - const vectorCount = vectors.length; + const vectorCount = vectors.length; - if (vectorCount == 0) { - return []; - } + if (vectorCount == 0) { + return []; + } - const featureCount = vectors[0].length; + const featureCount = vectors[0].length; - const result = createVector(featureCount); + const result = createVector(featureCount); - for (const vector of vectors) { - for (let featureIndex = 0; featureIndex < featureCount; featureIndex++) { - result[featureIndex] += vector[featureIndex]; + for (const vector of vectors) { + for ( + let featureIndex = 0; + featureIndex < featureCount; + featureIndex++ + ) { + result[featureIndex] += vector[featureIndex]; + } } - } - for (let featureIndex = 0; featureIndex < featureCount; featureIndex++) { - result[featureIndex] /= vectorCount; - } + for (let featureIndex = 0; featureIndex < featureCount; featureIndex++) { + result[featureIndex] /= vectorCount; + } - return result; + return result; } function stdDeviationOfVectors( - vectors: number[][], - kind: "population" | "sample" = "population", - mean?: number[], + vectors: number[][], + kind: "population" | "sample" = "population", + mean?: number[] ) { - return varianceOfVectors(vectors, kind, mean).map((v) => Math.sqrt(v)); + return varianceOfVectors(vectors, kind, mean).map((v) => Math.sqrt(v)); } function varianceOfVectors( - vectors: number[][], - kind: "population" | "sample" = "population", - mean?: number[], + vectors: number[][], + kind: "population" | "sample" = "population", + mean?: number[] ) { - const vectorCount = vectors.length; + const vectorCount = vectors.length; - if (vectorCount == 0) { - return []; - } + if (vectorCount == 0) { + return []; + } - const sampleSizeMetric = - kind == "population" || vectorCount == 1 ? vectorCount : vectorCount - 1; - const featureCount = vectors[0].length; + const sampleSizeMetric = + kind == "population" || vectorCount == 1 + ? vectorCount + : vectorCount - 1; + const featureCount = vectors[0].length; - if (!mean) { - mean = meanOfVectors(vectors); - } + if (!mean) { + mean = meanOfVectors(vectors); + } - const result = createVector(featureCount); + const result = createVector(featureCount); - for (const vector of vectors) { - for (let i = 0; i < featureCount; i++) { - result[i] += (vector[i] - mean[i]) ** 2; + for (const vector of vectors) { + for (let i = 0; i < featureCount; i++) { + result[i] += (vector[i] - mean[i]) ** 2; + } } - } - for (let i = 0; i < featureCount; i++) { - result[i] /= sampleSizeMetric; - } + for (let i = 0; i < featureCount; i++) { + result[i] /= sampleSizeMetric; + } - return result; + return result; } function euclidianDistance( - vector1: ArrayLike, - vector2: ArrayLike, + vector1: ArrayLike, + vector2: ArrayLike ) { - return Math.sqrt(squaredEuclidianDistance(vector1, vector2)); + return Math.sqrt(squaredEuclidianDistance(vector1, vector2)); } function squaredEuclidianDistance( - vector1: ArrayLike, - vector2: ArrayLike, + vector1: ArrayLike, + vector2: ArrayLike ) { - if (vector1.length !== vector2.length) { - throw new Error("Vectors are not the same length"); - } + if (vector1.length !== vector2.length) { + throw new Error("Vectors are not the same length"); + } - const elementCount = vector1.length; + const elementCount = vector1.length; - if (elementCount === 0) { - return 0; - } + if (elementCount === 0) { + return 0; + } - let sum = 0.0; + let sum = 0.0; - for (let i = 0; i < elementCount; i++) { - sum += (vector1[i] - vector2[i]) ** 2; - } + for (let i = 0; i < elementCount; i++) { + sum += (vector1[i] - vector2[i]) ** 2; + } - return sum; + return sum; } function euclidianDistance13Dim( - vector1: ArrayLike, - vector2: ArrayLike, + vector1: ArrayLike, + vector2: ArrayLike ) { - return Math.sqrt(squaredEuclidianDistance13Dim(vector1, vector2)); + return Math.sqrt(squaredEuclidianDistance13Dim(vector1, vector2)); } function squaredEuclidianDistance13Dim( - vector1: ArrayLike, - vector2: ArrayLike, + vector1: ArrayLike, + vector2: ArrayLike ) { - // Assumes the input has 13 dimensions (optimized for 13-dimensional MFCC vectors) - - const result = - (vector1[0] - vector2[0]) ** 2 + - (vector1[1] - vector2[1]) ** 2 + - (vector1[2] - vector2[2]) ** 2 + - (vector1[3] - vector2[3]) ** 2 + - (vector1[4] - vector2[4]) ** 2 + - (vector1[5] - vector2[5]) ** 2 + - (vector1[6] - vector2[6]) ** 2 + - (vector1[7] - vector2[7]) ** 2 + - (vector1[8] - vector2[8]) ** 2 + - (vector1[9] - vector2[9]) ** 2 + - (vector1[10] - vector2[10]) ** 2 + - (vector1[11] - vector2[11]) ** 2 + - (vector1[12] - vector2[12]) ** 2; - - return result; + // Assumes the input has 13 dimensions (optimized for 13-dimensional MFCC vectors) + + const result = + (vector1[0] - vector2[0]) ** 2 + + (vector1[1] - vector2[1]) ** 2 + + (vector1[2] - vector2[2]) ** 2 + + (vector1[3] - vector2[3]) ** 2 + + (vector1[4] - vector2[4]) ** 2 + + (vector1[5] - vector2[5]) ** 2 + + (vector1[6] - vector2[6]) ** 2 + + (vector1[7] - vector2[7]) ** 2 + + (vector1[8] - vector2[8]) ** 2 + + (vector1[9] - vector2[9]) ** 2 + + (vector1[10] - vector2[10]) ** 2 + + (vector1[11] - vector2[11]) ** 2 + + (vector1[12] - vector2[12]) ** 2; + + return result; } function cosineDistancePrecomputedMagnitudes( - vector1: ArrayLike, - vector2: ArrayLike, - magnitude1: number, - magnitude2: number, + vector1: ArrayLike, + vector2: ArrayLike, + magnitude1: number, + magnitude2: number ) { - return ( - 1 - - cosineSimilarityPrecomputedMagnitudes( - vector1, - vector2, - magnitude1, - magnitude2, - ) - ); + return ( + 1 - + cosineSimilarityPrecomputedMagnitudes( + vector1, + vector2, + magnitude1, + magnitude2 + ) + ); } function cosineSimilarityPrecomputedMagnitudes( - vector1: ArrayLike, - vector2: ArrayLike, - magnitude1: number, - magnitude2: number, + vector1: ArrayLike, + vector2: ArrayLike, + magnitude1: number, + magnitude2: number ) { - if (vector1.length != vector2.length) { - throw new Error("Vectors are not the same length"); - } + if (vector1.length != vector2.length) { + throw new Error("Vectors are not the same length"); + } - if (vector1.length == 0) { - return 0; - } + if (vector1.length == 0) { + return 0; + } - const featureCount = vector1.length; + const featureCount = vector1.length; - let dotProduct = 0.0; + let dotProduct = 0.0; - for (let i = 0; i < featureCount; i++) { - dotProduct += vector1[i] * vector2[i]; - } + for (let i = 0; i < featureCount; i++) { + dotProduct += vector1[i] * vector2[i]; + } - let result = dotProduct / (magnitude1 * magnitude2 + 1e-40); + let result = dotProduct / (magnitude1 * magnitude2 + 1e-40); - result = zeroIfNaN(result); - result = clip(result, -1.0, 1.0); + result = zeroIfNaN(result); + result = clip(result, -1.0, 1.0); - return result; + return result; } function magnitude(vector: ArrayLike) { - const featureCount = vector.length; + const featureCount = vector.length; - let squaredMagnitude = 0.0; + let squaredMagnitude = 0.0; - for (let i = 0; i < featureCount; i++) { - squaredMagnitude += vector[i] ** 2; - } + for (let i = 0; i < featureCount; i++) { + squaredMagnitude += vector[i] ** 2; + } - return Math.sqrt(squaredMagnitude); + return Math.sqrt(squaredMagnitude); } function createVector(elementCount: number, initialValue = 0.0) { - const result: number[] = new Array(elementCount); + const result: number[] = new Array(elementCount); - for (let i = 0; i < elementCount; i++) { - result[i] = initialValue; - } + for (let i = 0; i < elementCount; i++) { + result[i] = initialValue; + } - return result; + return result; } function createVectorForIntegerRange(start: number, end: number) { - const newVector: number[] = []; + const newVector: number[] = []; - for (let i = start; i < end; i++) { - newVector.push(i); - } + for (let i = start; i < end; i++) { + newVector.push(i); + } - return newVector; + return newVector; } function zeroIfNaN(val: number) { - if (isNaN(val)) { - return 0; - } else { - return val; - } + if (isNaN(val)) { + return 0; + } else { + return val; + } } type MfccOptions = { - filterbankCount?: number; - featureCount?: number; - fftOrder?: number; - lowerFreq?: number; - upperFreq?: number; - windowDuration?: number; - hopDuration?: number; - emphasisFactor?: number; - analysisSampleRate?: number; - lifteringFactor?: number; - normalize?: boolean; - zeroFirstCoefficient?: boolean; + filterbankCount?: number; + featureCount?: number; + fftOrder?: number; + lowerFreq?: number; + upperFreq?: number; + windowDuration?: number; + hopDuration?: number; + emphasisFactor?: number; + analysisSampleRate?: number; + lifteringFactor?: number; + normalize?: boolean; + zeroFirstCoefficient?: boolean; }; const defaultMfccOptions: MfccOptions = { - filterbankCount: 40, - featureCount: 13, - fftOrder: 512, - lowerFreq: 133.3333, - upperFreq: 6855.4976, - windowDuration: 0.025, - hopDuration: 0.01, - emphasisFactor: 0.97, - analysisSampleRate: 16000, - lifteringFactor: 0, - normalize: false, - zeroFirstCoefficient: false, + filterbankCount: 40, + featureCount: 13, + fftOrder: 512, + lowerFreq: 133.3333, + upperFreq: 6855.4976, + windowDuration: 0.025, + hopDuration: 0.01, + emphasisFactor: 0.97, + analysisSampleRate: 16000, + lifteringFactor: 0, + normalize: false, + zeroFirstCoefficient: false, }; function extendDefaultMfccOptions(options: MfccOptions) { - return extendDeep(defaultMfccOptions, options); + return extendDeep(defaultMfccOptions, options); } function applyLiftering(mfccs: number[][], lifteringFactor: number) { - const featureCount = mfccs[0].length; + const featureCount = mfccs[0].length; - const lifterMultipliers = new Float32Array(featureCount); + const lifterMultipliers = new Float32Array(featureCount); - for (let i = 0; i < featureCount; i++) { - lifterMultipliers[i] = - 1 + - (lifteringFactor / 2) * Math.sin((Math.PI * (i + 1)) / lifteringFactor); - } + for (let i = 0; i < featureCount; i++) { + lifterMultipliers[i] = + 1 + + (lifteringFactor / 2) * + Math.sin((Math.PI * (i + 1)) / lifteringFactor); + } - const lifteredMfccs: number[][] = []; + const lifteredMfccs: number[][] = []; - for (const mfcc of mfccs) { - const lifteredMfcc = new Array(featureCount); + for (const mfcc of mfccs) { + const lifteredMfcc = new Array(featureCount); - for (let i = 0; i < featureCount; i++) { - lifteredMfcc[i] = mfcc[i] * lifterMultipliers[i]; - } + for (let i = 0; i < featureCount; i++) { + lifteredMfcc[i] = mfcc[i] * lifterMultipliers[i]; + } - lifteredMfccs.push(lifteredMfcc); - } + lifteredMfccs.push(lifteredMfcc); + } - return lifteredMfccs; + return lifteredMfccs; } function compactPath(path: AlignmentPath) { - const compactedPath: CompactedPath = []; + const compactedPath: CompactedPath = []; - for (let i = 0; i < path.length; i++) { - const pathEntry = path[i]; + for (let i = 0; i < path.length; i++) { + const pathEntry = path[i]; - if (compactedPath.length <= pathEntry.source) { - compactedPath.push({ first: pathEntry.dest, last: pathEntry.dest }); - } else { - compactedPath[compactedPath.length - 1].last = pathEntry.dest; + if (compactedPath.length <= pathEntry.source) { + compactedPath.push({ first: pathEntry.dest, last: pathEntry.dest }); + } else { + compactedPath[compactedPath.length - 1].last = pathEntry.dest; + } } - } - return compactedPath; + return compactedPath; } async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {}) { - const logger = new Logger(); - logger.start("Initialize options"); + const logger = new Logger(); + logger.start("Initialize options"); + + if (monoAudio.audioChannels.length != 1) { + throw new Error("Audio must be mono"); + } - if (monoAudio.audioChannels.length != 1) { - throw new Error("Audio must be mono"); - } + options = extendDefaultMfccOptions(options); - options = extendDefaultMfccOptions(options); + const analysisSampleRate = options.analysisSampleRate!; + const featureCount = options.featureCount!; - const analysisSampleRate = options.analysisSampleRate!; - const featureCount = options.featureCount!; + const fftOrder = options.fftOrder!; - const fftOrder = options.fftOrder!; + const windowDuration = options.windowDuration!; + const windowSize = windowDuration * analysisSampleRate; + const hopDuration = options.hopDuration!; + const hopLength = hopDuration * analysisSampleRate; - const windowDuration = options.windowDuration!; - const windowSize = windowDuration * analysisSampleRate; - const hopDuration = options.hopDuration!; - const hopLength = hopDuration * analysisSampleRate; + const filterbankCount = options.filterbankCount!; + const lowerFrequencyHz = options.lowerFreq!; + const upperFrequencyHz = options.upperFreq!; - const filterbankCount = options.filterbankCount!; - const lowerFrequencyHz = options.lowerFreq!; - const upperFrequencyHz = options.upperFreq!; + const emphasisFactor = options.emphasisFactor!; + const lifteringFactor = options.lifteringFactor!; + const zeroFirstCoefficient = options.zeroFirstCoefficient!; - const emphasisFactor = options.emphasisFactor!; - const lifteringFactor = options.lifteringFactor!; - const zeroFirstCoefficient = options.zeroFirstCoefficient!; + logger.start( + `Resample audio to analysis sample rate (${analysisSampleRate}Hz)` + ); + const resampledAudio = await resampleAudioSpeex( + monoAudio, + analysisSampleRate + ); - logger.start( - `Resample audio to analysis sample rate (${analysisSampleRate}Hz)`, - ); - const resampledAudio = await resampleAudioSpeex( - monoAudio, - analysisSampleRate, - ); + let mfccs: number[][]; - let mfccs: number[][]; + if (emphasisFactor > 0) { + logger.start("Apply emphasis"); + resampledAudio.audioChannels[0] = applyEmphasis( + resampledAudio.audioChannels[0], + emphasisFactor + ); + } - if (emphasisFactor > 0) { - logger.start("Apply emphasis"); - resampledAudio.audioChannels[0] = applyEmphasis( - resampledAudio.audioChannels[0], - emphasisFactor, + logger.start("Compute Mel spectogram"); + const { melSpectogram } = await computeMelSpectogram( + resampledAudio, + fftOrder, + windowSize, + hopLength, + filterbankCount, + lowerFrequencyHz, + upperFrequencyHz ); - } - - logger.start("Compute Mel spectogram"); - const { melSpectogram } = await computeMelSpectogram( - resampledAudio, - fftOrder, - windowSize, - hopLength, - filterbankCount, - lowerFrequencyHz, - upperFrequencyHz, - ); - - logger.start("Extract MFCCs from Mel spectogram"); - const mfccsFloat32 = melSpectogramToMFCCs(melSpectogram, featureCount); - - mfccs = mfccsFloat32.map((mfcc) => Array.from(mfcc)); - - if (options.normalize!) { - logger.start("Normalize MFCCs"); - - const { normalizedVectors, mean, stdDeviation } = normalizeVectors(mfccs); - mfccs = normalizedVectors; - //mfccs = mfccs.map(mfcc => subtractVectors(mfcc, mean)) - } - - if (lifteringFactor > 0) { - logger.start("Apply liftering to MFCCs"); - mfccs = applyLiftering(mfccs, lifteringFactor); - } - - if (zeroFirstCoefficient) { - for (const mfcc of mfccs) { - mfcc[0] = 0; + + logger.start("Extract MFCCs from Mel spectogram"); + const mfccsFloat32 = melSpectogramToMFCCs(melSpectogram, featureCount); + + mfccs = mfccsFloat32.map((mfcc) => Array.from(mfcc)); + + if (options.normalize!) { + logger.start("Normalize MFCCs"); + + const { normalizedVectors, mean, stdDeviation } = + normalizeVectors(mfccs); + mfccs = normalizedVectors; + //mfccs = mfccs.map(mfcc => subtractVectors(mfcc, mean)) + } + + if (lifteringFactor > 0) { + logger.start("Apply liftering to MFCCs"); + mfccs = applyLiftering(mfccs, lifteringFactor); } - } - logger.end(); + if (zeroFirstCoefficient) { + for (const mfcc of mfccs) { + mfcc[0] = 0; + } + } + + logger.end(); - return mfccs; + return mfccs; } type DtwGranularity = "xx-low" | "x-low" | "low" | "medium" | "high" | "x-high"; @@ -1464,2453 +1499,2507 @@ type DtwGranularity = "xx-low" | "x-low" | "low" | "medium" | "high" | "x-high"; type AlignmentPath = AlignmentPathEntry[]; type AlignmentPathEntry = { - source: number; - dest: number; + source: number; + dest: number; }; type CompactedPath = CompactedPathEntry[]; type CompactedPathEntry = { - first: number; - last: number; + first: number; + last: number; }; function getStartingSilentSampleCount( - audioSamples: Float32Array, - amplitudeThresholdDecibels = defaultSilenceThresholdDecibels, + audioSamples: Float32Array, + amplitudeThresholdDecibels = defaultSilenceThresholdDecibels ) { - const minSampleAmplitude = decibelsToGainFactor(amplitudeThresholdDecibels); + const minSampleAmplitude = decibelsToGainFactor(amplitudeThresholdDecibels); - let silentSampleCount = 0; + let silentSampleCount = 0; - for (let i = 0; i < audioSamples.length - 1; i++) { - if (Math.abs(audioSamples[i]) > minSampleAmplitude) { - break; - } + for (let i = 0; i < audioSamples.length - 1; i++) { + if (Math.abs(audioSamples[i]) > minSampleAmplitude) { + break; + } - silentSampleCount += 1; - } + silentSampleCount += 1; + } - return silentSampleCount; + return silentSampleCount; } function decibelsToGainFactor(decibels: number) { - return decibels <= -100.0 ? 0 : Math.pow(10, 0.05 * decibels); + return decibels <= -100.0 ? 0 : Math.pow(10, 0.05 * decibels); } const defaultSilenceThresholdDecibels = -40; function getMappedFrameIndexForPath( - referenceFrameIndex: number, - compactedPath: CompactedPath, - mappingKind: "first" | "last" = "first", + referenceFrameIndex: number, + compactedPath: CompactedPath, + mappingKind: "first" | "last" = "first" ) { - if (compactedPath.length == 0) { - return 0; - } + if (compactedPath.length == 0) { + return 0; + } - referenceFrameIndex = clip(referenceFrameIndex, 0, compactedPath.length - 1); + referenceFrameIndex = clip( + referenceFrameIndex, + 0, + compactedPath.length - 1 + ); - const compactedPathEntry = compactedPath[referenceFrameIndex]; + const compactedPathEntry = compactedPath[referenceFrameIndex]; - let mappedFrameIndex: number; + let mappedFrameIndex: number; - if (mappingKind == "first") { - mappedFrameIndex = compactedPathEntry.first; - } else { - mappedFrameIndex = compactedPathEntry.last; - } + if (mappingKind == "first") { + mappedFrameIndex = compactedPathEntry.first; + } else { + mappedFrameIndex = compactedPathEntry.last; + } - return mappedFrameIndex; + return mappedFrameIndex; } function getEndingSilentSampleCount( - audioSamples: Float32Array, - amplitudeThresholdDecibels = defaultSilenceThresholdDecibels, + audioSamples: Float32Array, + amplitudeThresholdDecibels = defaultSilenceThresholdDecibels ) { - const minSampleAmplitude = decibelsToGainFactor(amplitudeThresholdDecibels); + const minSampleAmplitude = decibelsToGainFactor(amplitudeThresholdDecibels); - let silentSampleCount = 0; + let silentSampleCount = 0; - for (let i = audioSamples.length - 1; i >= 0; i--) { - if (Math.abs(audioSamples[i]) > minSampleAmplitude) { - break; - } + for (let i = audioSamples.length - 1; i >= 0; i--) { + if (Math.abs(audioSamples[i]) > minSampleAmplitude) { + break; + } - silentSampleCount += 1; - } + silentSampleCount += 1; + } - return silentSampleCount; + return silentSampleCount; } function getMappedTimelineEntry( - timelineEntry: TimelineEntry, - sourceRawAudio: RawAudio, - framesPerSecond: number, - compactedPath: CompactedPath, - recurse = true, + timelineEntry: TimelineEntry, + sourceRawAudio: RawAudio, + framesPerSecond: number, + compactedPath: CompactedPath, + recurse = true ): TimelineEntry { - const referenceStartFrameIndex = Math.floor( - timelineEntry.startTime * framesPerSecond, - ); - const referenceEndFrameIndex = Math.floor( - timelineEntry.endTime * framesPerSecond, - ); - - if (referenceStartFrameIndex < 0 || referenceEndFrameIndex < 0) { - throw new Error("Unexpected: encountered a negative timestamp in timeline"); - } - - const mappedStartFrameIndex = getMappedFrameIndexForPath( - referenceStartFrameIndex, - compactedPath, - "first", - ); - const mappedEndFrameIndex = getMappedFrameIndexForPath( - referenceEndFrameIndex, - compactedPath, - "first", - ); - - let innerTimeline: Timeline | undefined; - - if (recurse && timelineEntry.timeline != null) { - innerTimeline = timelineEntry.timeline.map((entry) => - getMappedTimelineEntry( - entry, - sourceRawAudio, - framesPerSecond, - compactedPath, - recurse, - ), + const referenceStartFrameIndex = Math.floor( + timelineEntry.startTime * framesPerSecond + ); + const referenceEndFrameIndex = Math.floor( + timelineEntry.endTime * framesPerSecond ); - } - - // Trim silent samples from start and end of mapped entry range - const sourceSamplesPerFrame = Math.floor( - sourceRawAudio.sampleRate / framesPerSecond, - ); - - let startSampleIndex = mappedStartFrameIndex * sourceSamplesPerFrame; - let endSampleIndex = mappedEndFrameIndex * sourceSamplesPerFrame; - const frameSamples = sourceRawAudio.audioChannels[0].subarray( - startSampleIndex, - endSampleIndex, - ); + if (referenceStartFrameIndex < 0 || referenceEndFrameIndex < 0) { + throw new Error( + "Unexpected: encountered a negative timestamp in timeline" + ); + } + + const mappedStartFrameIndex = getMappedFrameIndexForPath( + referenceStartFrameIndex, + compactedPath, + "first" + ); + const mappedEndFrameIndex = getMappedFrameIndexForPath( + referenceEndFrameIndex, + compactedPath, + "first" + ); + + let innerTimeline: Timeline | undefined; + + if (recurse && timelineEntry.timeline != null) { + innerTimeline = timelineEntry.timeline.map((entry) => + getMappedTimelineEntry( + entry, + sourceRawAudio, + framesPerSecond, + compactedPath, + recurse + ) + ); + } - const silenceThresholdDecibels = -40; + // Trim silent samples from start and end of mapped entry range + const sourceSamplesPerFrame = Math.floor( + sourceRawAudio.sampleRate / framesPerSecond + ); - startSampleIndex += getStartingSilentSampleCount( - frameSamples, - silenceThresholdDecibels, - ); - endSampleIndex -= getEndingSilentSampleCount( - frameSamples, - silenceThresholdDecibels, - ); + let startSampleIndex = mappedStartFrameIndex * sourceSamplesPerFrame; + let endSampleIndex = mappedEndFrameIndex * sourceSamplesPerFrame; - endSampleIndex = Math.max(endSampleIndex, startSampleIndex); + const frameSamples = sourceRawAudio.audioChannels[0].subarray( + startSampleIndex, + endSampleIndex + ); - // Build mapped timeline entry - const startTime = startSampleIndex / sourceRawAudio.sampleRate; - const endTime = endSampleIndex / sourceRawAudio.sampleRate; + const silenceThresholdDecibels = -40; - return { - type: timelineEntry.type, - text: timelineEntry.text, + startSampleIndex += getStartingSilentSampleCount( + frameSamples, + silenceThresholdDecibels + ); + endSampleIndex -= getEndingSilentSampleCount( + frameSamples, + silenceThresholdDecibels + ); + + endSampleIndex = Math.max(endSampleIndex, startSampleIndex); - startTime, - endTime, + // Build mapped timeline entry + const startTime = startSampleIndex / sourceRawAudio.sampleRate; + const endTime = endSampleIndex / sourceRawAudio.sampleRate; + + return { + type: timelineEntry.type, + text: timelineEntry.text, - timeline: innerTimeline, - }; + startTime, + endTime, + + timeline: innerTimeline, + }; } function getMfccOptionsForGranularity(granularity: DtwGranularity) { - let mfccOptions: MfccOptions; - - if (granularity == "xx-low") { - mfccOptions = { windowDuration: 0.4, hopDuration: 0.16, fftOrder: 8192 }; - } else if (granularity == "x-low") { - mfccOptions = { windowDuration: 0.2, hopDuration: 0.08, fftOrder: 4096 }; - } else if (granularity == "low") { - mfccOptions = { windowDuration: 0.1, hopDuration: 0.04, fftOrder: 2048 }; - } else if (granularity == "medium") { - mfccOptions = { windowDuration: 0.05, hopDuration: 0.02, fftOrder: 1024 }; - } else if (granularity == "high") { - mfccOptions = { windowDuration: 0.025, hopDuration: 0.01, fftOrder: 512 }; - } else if (granularity == "x-high") { - mfccOptions = { windowDuration: 0.02, hopDuration: 0.005, fftOrder: 512 }; - } else { - throw new Error(`Invalid granularity setting: '${granularity}'`); - } - - return mfccOptions; + let mfccOptions: MfccOptions; + + if (granularity == "xx-low") { + mfccOptions = { + windowDuration: 0.4, + hopDuration: 0.16, + fftOrder: 8192, + }; + } else if (granularity == "x-low") { + mfccOptions = { + windowDuration: 0.2, + hopDuration: 0.08, + fftOrder: 4096, + }; + } else if (granularity == "low") { + mfccOptions = { + windowDuration: 0.1, + hopDuration: 0.04, + fftOrder: 2048, + }; + } else if (granularity == "medium") { + mfccOptions = { + windowDuration: 0.05, + hopDuration: 0.02, + fftOrder: 1024, + }; + } else if (granularity == "high") { + mfccOptions = { + windowDuration: 0.025, + hopDuration: 0.01, + fftOrder: 512, + }; + } else if (granularity == "x-high") { + mfccOptions = { + windowDuration: 0.02, + hopDuration: 0.005, + fftOrder: 512, + }; + } else { + throw new Error(`Invalid granularity setting: '${granularity}'`); + } + + return mfccOptions; } async function alignUsingDtw( - sourceRawAudio: RawAudio, - referenceRawAudio: RawAudio, - referenceTimeline: Timeline, - granularities: DtwGranularity[], - windowDurations: number[], + sourceRawAudio: RawAudio, + referenceRawAudio: RawAudio, + referenceTimeline: Timeline, + granularities: DtwGranularity[], + windowDurations: number[] ) { - const logger = new Logger(); - - if (windowDurations.length == 0) { - throw new Error(`Window durations array has length 0.`); - } + const logger = new Logger(); - if (windowDurations.length != granularities.length) { - throw new Error( - `Window durations and granularities are not the same length.`, - ); - } + if (windowDurations.length == 0) { + throw new Error(`Window durations array has length 0.`); + } - const rawAudioDuration = getRawAudioDuration(sourceRawAudio); + if (windowDurations.length != granularities.length) { + throw new Error( + `Window durations and granularities are not the same length.` + ); + } - let framesPerSecond: number; - let compactedPath: CompactedPath; - let relativeCenters: number[] | undefined; + const rawAudioDuration = getRawAudioDuration(sourceRawAudio); - for (let passIndex = 0; passIndex < windowDurations.length; passIndex++) { - const granularity = granularities[passIndex]; - const windowDuration = windowDurations[passIndex]; + let framesPerSecond: number; + let compactedPath: CompactedPath; + let relativeCenters: number[] | undefined; - logger.logTitledMessage( - `\nStarting alignment pass ${passIndex + 1}/${windowDurations.length}`, - `granularity: ${granularity}, max window duration: ${windowDuration}s`, - ); + for (let passIndex = 0; passIndex < windowDurations.length; passIndex++) { + const granularity = granularities[passIndex]; + const windowDuration = windowDurations[passIndex]; - const mfccOptions = extendDefaultMfccOptions({ - ...getMfccOptionsForGranularity(granularity), - zeroFirstCoefficient: true, - }) as MfccOptions; + logger.logTitledMessage( + `\nStarting alignment pass ${passIndex + 1}/${windowDurations.length}`, + `granularity: ${granularity}, max window duration: ${windowDuration}s` + ); - framesPerSecond = 1 / mfccOptions.hopDuration!; + const mfccOptions = extendDefaultMfccOptions({ + ...getMfccOptionsForGranularity(granularity), + zeroFirstCoefficient: true, + }) as MfccOptions; - // Compute reference MFCCs - logger.start("Compute reference MFCC features"); - const referenceMfccs = await computeMFCCs(referenceRawAudio, mfccOptions); + framesPerSecond = 1 / mfccOptions.hopDuration!; - // Compute source MFCCs - logger.start("Compute source MFCC features"); - const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions); - logger.end(); - - // Compute path - logger.logTitledMessage( - `DTW cost matrix memory size`, - `${getCostMatrixMemorySizeMB(referenceMfccs.length, sourceMfccs.length, windowDuration * framesPerSecond).toFixed(1)}MB`, - ); + // Compute reference MFCCs + logger.start("Compute reference MFCC features"); + const referenceMfccs = await computeMFCCs( + referenceRawAudio, + mfccOptions + ); - if (passIndex == 0) { - const minRecommendedWindowDuration = 0.2 * rawAudioDuration; + // Compute source MFCCs + logger.start("Compute source MFCC features"); + const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions); + logger.end(); - if (windowDuration < minRecommendedWindowDuration) { + // Compute path logger.logTitledMessage( - "Warning", - `Maximum DTW window duration is set to ${windowDuration.toFixed(1)}s, which is smaller than 20% of the source audio duration of ${rawAudioDuration.toFixed(1)}s. This may lead to suboptimal results in some cases. Consider increasing window duration if needed.`, - "warning", + `DTW cost matrix memory size`, + `${getCostMatrixMemorySizeMB(referenceMfccs.length, sourceMfccs.length, windowDuration * framesPerSecond).toFixed(1)}MB` ); - } - } - logger.start("Align reference and source MFCC features using DTW"); - const dtwWindowLength = Math.floor(windowDuration * framesPerSecond); + if (passIndex == 0) { + const minRecommendedWindowDuration = 0.2 * rawAudioDuration; - let centerIndexes: number[] | undefined; + if (windowDuration < minRecommendedWindowDuration) { + logger.logTitledMessage( + "Warning", + `Maximum DTW window duration is set to ${windowDuration.toFixed(1)}s, which is smaller than 20% of the source audio duration of ${rawAudioDuration.toFixed(1)}s. This may lead to suboptimal results in some cases. Consider increasing window duration if needed.`, + "warning" + ); + } + } - if (relativeCenters) { - centerIndexes = []; + logger.start("Align reference and source MFCC features using DTW"); + const dtwWindowLength = Math.floor(windowDuration * framesPerSecond); - for (let i = 0; i < referenceMfccs.length; i++) { - const relativeReferencePosition = i / referenceMfccs.length; + let centerIndexes: number[] | undefined; - const relativeCenterIndex = Math.floor( - relativeReferencePosition * relativeCenters!.length, - ); - const relativeCenter = relativeCenters[relativeCenterIndex]; - const centerIndex = Math.floor(relativeCenter * sourceMfccs.length); + if (relativeCenters) { + centerIndexes = []; - centerIndexes.push(centerIndex); - } - } + for (let i = 0; i < referenceMfccs.length; i++) { + const relativeReferencePosition = i / referenceMfccs.length; - const rawPath = await alignMFCC_DTW( - referenceMfccs, - sourceMfccs, - dtwWindowLength, - undefined, - centerIndexes, - ); + const relativeCenterIndex = Math.floor( + relativeReferencePosition * relativeCenters!.length + ); + const relativeCenter = relativeCenters[relativeCenterIndex]; + const centerIndex = Math.floor( + relativeCenter * sourceMfccs.length + ); + + centerIndexes.push(centerIndex); + } + } - compactedPath = compactPath(rawPath); + const rawPath = await alignMFCC_DTW( + referenceMfccs, + sourceMfccs, + dtwWindowLength, + undefined, + centerIndexes + ); - relativeCenters = compactedPath.map( - (entry) => (entry.first + entry.last) / 2 / sourceMfccs.length, - ); + compactedPath = compactPath(rawPath); - logger.end(); - } + relativeCenters = compactedPath.map( + (entry) => (entry.first + entry.last) / 2 / sourceMfccs.length + ); - logger.start("\nConvert path to timeline"); + logger.end(); + } - const mappedTimeline = referenceTimeline.map((entry) => - getMappedTimelineEntry( - entry, - sourceRawAudio, - framesPerSecond, - compactedPath, - ), - ); + logger.start("\nConvert path to timeline"); - logger.end(); + const mappedTimeline = referenceTimeline.map((entry) => + getMappedTimelineEntry( + entry, + sourceRawAudio, + framesPerSecond, + compactedPath + ) + ); - return mappedTimeline; + logger.end(); + + return mappedTimeline; } class OpenPromise { - promise: Promise; - resolve: (value: T) => void = () => { - throw new Error("Open promise resolved before initialization"); - }; - reject: (reason?: any) => void = () => { - throw new Error("Open promise rejected before initialization"); - }; - - constructor() { - this.promise = new Promise((resolve, reject) => { - this.resolve = resolve; - this.reject = reject; - }); - } + promise: Promise; + resolve: (value: T) => void = () => { + throw new Error("Open promise resolved before initialization"); + }; + reject: (reason?: any) => void = () => { + throw new Error("Open promise rejected before initialization"); + }; + + constructor() { + this.promise = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); + } } let kuromojiTokenizer: any; async function getKuromojiTokenizer() { - if (kuromojiTokenizer) { - return kuromojiTokenizer; - } + if (kuromojiTokenizer) { + return kuromojiTokenizer; + } - const { default: kuromoji } = await import("kuromoji"); + const { default: kuromoji } = await import("kuromoji"); - const resultOpenPromise = new OpenPromise(); + const resultOpenPromise = new OpenPromise(); - const kuromojiScriptPath = await resolveModuleScriptPath("kuromoji"); - const dictionaryPath = path.join( - path.dirname(kuromojiScriptPath), - "..", - "/dict", - ); + const kuromojiScriptPath = await resolveModuleScriptPath("kuromoji"); + const dictionaryPath = path.join( + path.dirname(kuromojiScriptPath), + "..", + "/dict" + ); - kuromoji.builder({ dicPath: dictionaryPath }).build(function ( - error: any, - tokenizer: any, - ) { - if (error) { - resultOpenPromise.reject(error); - return; - } + kuromoji.builder({ dicPath: dictionaryPath }).build(function ( + error: any, + tokenizer: any + ) { + if (error) { + resultOpenPromise.reject(error); + return; + } - kuromojiTokenizer = tokenizer; + kuromojiTokenizer = tokenizer; - resultOpenPromise.resolve(kuromojiTokenizer); - }); + resultOpenPromise.resolve(kuromojiTokenizer); + }); - return resultOpenPromise.promise; + return resultOpenPromise.promise; } async function splitJapaneseTextToWords_Kuromoji(text: string) { - const tokenizer = await getKuromojiTokenizer(); + const tokenizer = await getKuromojiTokenizer(); - const results: any[] = tokenizer.tokenize(text); - const words = results.map((entry) => entry.surface_form); + const results: any[] = tokenizer.tokenize(text); + const words = results.map((entry) => entry.surface_form); - return words; + return words; } let JiebaWasmInstance: typeof import("jieba-wasm"); async function getWasmInstance() { - if (!JiebaWasmInstance) { - const { default: JibeaWasm } = await import("jieba-wasm"); - // @ts-ignore - JiebaWasmInstance = JibeaWasm; - } + if (!JiebaWasmInstance) { + const { default: JibeaWasm } = await import("jieba-wasm"); + // @ts-ignore + JiebaWasmInstance = JibeaWasm; + } - return JiebaWasmInstance; + return JiebaWasmInstance; } async function splitChineseTextToWords_Jieba( - text: string, - fineGrained = false, - useHMM = true, + text: string, + fineGrained = false, + useHMM = true ) { - const jieba = await getWasmInstance(); + const jieba = await getWasmInstance(); - if (!fineGrained) { - return jieba.cut(text, useHMM); - } else { - const results = jieba.tokenize(text, "search", useHMM); + if (!fineGrained) { + return jieba.cut(text, useHMM); + } else { + const results = jieba.tokenize(text, "search", useHMM); - const startOffsetsSet = new Set(); - const endOffsetsSet = new Set(); + const startOffsetsSet = new Set(); + const endOffsetsSet = new Set(); - for (const result of results) { - startOffsetsSet.add(result.start); - endOffsetsSet.add(result.end); - } + for (const result of results) { + startOffsetsSet.add(result.start); + endOffsetsSet.add(result.end); + } - const startOffsets = Array.from(startOffsetsSet); - startOffsets.sort((a, b) => a - b); + const startOffsets = Array.from(startOffsetsSet); + startOffsets.sort((a, b) => a - b); - const endOffsets = Array.from(endOffsetsSet); - endOffsets.sort((a, b) => a - b); + const endOffsets = Array.from(endOffsetsSet); + endOffsets.sort((a, b) => a - b); - const words: string[] = []; + const words: string[] = []; - for (let i = 0; i < startOffsets.length; i++) { - const wordStartOffset = startOffsets[i]; + for (let i = 0; i < startOffsets.length; i++) { + const wordStartOffset = startOffsets[i]; - function getWordEndOffset() { - if (i < startOffsets.length - 1) { - const nextWordStartOffset = startOffsets[i + 1]; + function getWordEndOffset() { + if (i < startOffsets.length - 1) { + const nextWordStartOffset = startOffsets[i + 1]; - for (let j = 0; j < endOffsets.length - 1; j++) { - const currentEndOffset = endOffsets[j]; - const nextEndOffset = endOffsets[j + 1]; + for (let j = 0; j < endOffsets.length - 1; j++) { + const currentEndOffset = endOffsets[j]; + const nextEndOffset = endOffsets[j + 1]; - if (currentEndOffset >= nextWordStartOffset) { - return nextWordStartOffset; - } else if ( - currentEndOffset > wordStartOffset && - currentEndOffset < nextWordStartOffset && - nextEndOffset > nextWordStartOffset - ) { - return currentEndOffset; + if (currentEndOffset >= nextWordStartOffset) { + return nextWordStartOffset; + } else if ( + currentEndOffset > wordStartOffset && + currentEndOffset < nextWordStartOffset && + nextEndOffset > nextWordStartOffset + ) { + return currentEndOffset; + } + } + } + + return endOffsets[endOffsets.length - 1]; } - } - } - return endOffsets[endOffsets.length - 1]; - } + const wordEndOffset = getWordEndOffset(); - const wordEndOffset = getWordEndOffset(); + words.push(text.substring(wordStartOffset, wordEndOffset)); + } - words.push(text.substring(wordStartOffset, wordEndOffset)); + return words; } - - return words; - } } function getShortLanguageCode(langCode: string) { - const dashIndex = langCode.indexOf("-"); + const dashIndex = langCode.indexOf("-"); - if (dashIndex == -1) { - return langCode; - } + if (dashIndex == -1) { + return langCode; + } - return langCode.substring(0, dashIndex).toLowerCase(); + return langCode.substring(0, dashIndex).toLowerCase(); } abstract class ValueRef { - protected ptr: number; - private readonly manager: WasmMemoryManager; + protected ptr: number; + private readonly manager: WasmMemoryManager; - protected get module() { - return this.manager.wasmModule; - } + protected get module() { + return this.manager.wasmModule; + } - constructor(ptr: number, manager: WasmMemoryManager) { - this.ptr = ptr; - this.manager = manager; - } + constructor(ptr: number, manager: WasmMemoryManager) { + this.ptr = ptr; + this.manager = manager; + } - get value(): T { - this.assertNotFreed(); - return this.getValue(); - } + get value(): T { + this.assertNotFreed(); + return this.getValue(); + } - set value(newValue: T) { - this.assertNotFreed(); - this.setValue(newValue); - } + set value(newValue: T) { + this.assertNotFreed(); + this.setValue(newValue); + } - abstract getValue(): T; - abstract setValue(newValue: T): void; + abstract getValue(): T; + abstract setValue(newValue: T): void; - get address() { - this.assertNotFreed(); - return this.ptr; - } + get address() { + this.assertNotFreed(); + return this.ptr; + } - clear() { - this.assertNotFreed(); + clear() { + this.assertNotFreed(); - if (typeof this.value == "number") { - this.value = 0 as any; - } else if (typeof this.value == "string") { - throw new Error("Unimplemented"); - } + if (typeof this.value == "number") { + this.value = 0 as any; + } else if (typeof this.value == "string") { + throw new Error("Unimplemented"); + } - return this; - } + return this; + } - free() { - this.manager.free(this as any); - } + free() { + this.manager.free(this as any); + } - clearAddress() { - this.ptr = 0; - } + clearAddress() { + this.ptr = 0; + } - get isFreed() { - return this.ptr == 0; - } + get isFreed() { + return this.ptr == 0; + } - protected assertNotFreed() { - if (this.isFreed) { - throw new Error("Attempt to read a freed WASM value reference."); + protected assertNotFreed() { + if (this.isFreed) { + throw new Error("Attempt to read a freed WASM value reference."); + } } - } } class Int8Ref extends ValueRef { - getValue() { - return this.module.HEAP8[this.ptr >>> 0] as number; - } + getValue() { + return this.module.HEAP8[this.ptr >>> 0] as number; + } - setValue(newValue: number) { - this.module.HEAP8[this.ptr >>> 0] = newValue; - } + setValue(newValue: number) { + this.module.HEAP8[this.ptr >>> 0] = newValue; + } } class Uint8Ref extends ValueRef { - getValue() { - return this.module.HEAPU8[this.ptr >>> 0] as number; - } + getValue() { + return this.module.HEAPU8[this.ptr >>> 0] as number; + } - setValue(newValue: number) { - this.module.HEAPU8[this.ptr >>> 0] = newValue; - } + setValue(newValue: number) { + this.module.HEAPU8[this.ptr >>> 0] = newValue; + } } class Int16Ref extends ValueRef { - getValue() { - return this.module.HEAP16[this.ptr >>> 1] as number; - } + getValue() { + return this.module.HEAP16[this.ptr >>> 1] as number; + } - setValue(newValue: number) { - this.module.HEAP16[this.ptr >>> 1] = newValue; - } + setValue(newValue: number) { + this.module.HEAP16[this.ptr >>> 1] = newValue; + } } class Uint16Ref extends ValueRef { - getValue() { - return this.module.HEAPU16[this.ptr >>> 1] as number; - } + getValue() { + return this.module.HEAPU16[this.ptr >>> 1] as number; + } - setValue(newValue: number) { - this.module.HEAPU16[this.ptr >>> 1] = newValue; - } + setValue(newValue: number) { + this.module.HEAPU16[this.ptr >>> 1] = newValue; + } } class Int32Ref extends ValueRef { - getValue() { - return this.module.HEAP32[this.ptr >>> 2] as number; - } + getValue() { + return this.module.HEAP32[this.ptr >>> 2] as number; + } - setValue(newValue: number) { - this.module.HEAP32[this.ptr >>> 2] = newValue; - } + setValue(newValue: number) { + this.module.HEAP32[this.ptr >>> 2] = newValue; + } } class Uint32Ref extends ValueRef { - getValue() { - return this.module.HEAPU32[this.ptr >>> 2] as number; - } + getValue() { + return this.module.HEAPU32[this.ptr >>> 2] as number; + } - setValue(newValue: number) { - this.module.HEAPU32[this.ptr >>> 2] = newValue; - } + setValue(newValue: number) { + this.module.HEAPU32[this.ptr >>> 2] = newValue; + } } class PointerRef extends Uint32Ref {} class Float32Ref extends ValueRef { - getValue() { - return this.module.HEAPF32[this.ptr >>> 2] as number; - } + getValue() { + return this.module.HEAPF32[this.ptr >>> 2] as number; + } - setValue(newValue: number) { - this.module.HEAPF32[this.ptr >>> 2] = newValue; - } + setValue(newValue: number) { + this.module.HEAPF32[this.ptr >>> 2] = newValue; + } } class Float64Ref extends ValueRef { - getValue() { - return this.module.HEAPF64[this.ptr >>> 3] as number; - } + getValue() { + return this.module.HEAPF64[this.ptr >>> 3] as number; + } - setValue(newValue: number) { - this.module.HEAPF64[this.ptr >>> 3] = newValue; - } + setValue(newValue: number) { + this.module.HEAPF64[this.ptr >>> 3] = newValue; + } } class NullTerminatedUtf8StringRef extends ValueRef { - getValue() { - const ptr = this.ptr >>> 0; + getValue() { + const ptr = this.ptr >>> 0; - const heapU8 = this.module.HEAPU8; + const heapU8 = this.module.HEAPU8; - const endByteOffset = heapU8.subarray(ptr).indexOf(0); + const endByteOffset = heapU8.subarray(ptr).indexOf(0); - const strBytes = heapU8.subarray(ptr, ptr + endByteOffset); + const strBytes = heapU8.subarray(ptr, ptr + endByteOffset); - const str = Buffer.from(strBytes).toString("utf8"); + const str = Buffer.from(strBytes).toString("utf8"); - return str; - } + return str; + } - setValue(newValue: string) { - throw new Error("Unimplemented"); - } + setValue(newValue: string) { + throw new Error("Unimplemented"); + } } abstract class TypedArrayRef { - protected ptr: number; - readonly length: number; - private readonly manager: WasmMemoryManager; + protected ptr: number; + readonly length: number; + private readonly manager: WasmMemoryManager; - get module() { - return this.manager.wasmModule; - } + get module() { + return this.manager.wasmModule; + } - constructor(ptr: number, length: number, manager: WasmMemoryManager) { - this.ptr = ptr; - this.length = length; - this.manager = manager; - } + constructor(ptr: number, length: number, manager: WasmMemoryManager) { + this.ptr = ptr; + this.length = length; + this.manager = manager; + } - get view() { - this.assertNotFreed(); - return this.getView(); - } + get view() { + this.assertNotFreed(); + return this.getView(); + } - protected abstract getView(): T; + protected abstract getView(): T; - slice(start?: number, end?: number) { - return this.view.slice(start, end); - } + slice(start?: number, end?: number) { + return this.view.slice(start, end); + } - get address() { - this.assertNotFreed(); - return this.ptr; - } + get address() { + this.assertNotFreed(); + return this.ptr; + } - clear() { - this.view.fill(0); - return this; - } + clear() { + this.view.fill(0); + return this; + } - free() { - this.manager.free(this); - } + free() { + this.manager.free(this); + } - clearAddress() { - this.ptr = 0; - } + clearAddress() { + this.ptr = 0; + } - get isFreed() { - return this.ptr == 0; - } + get isFreed() { + return this.ptr == 0; + } - protected assertNotFreed() { - if (this.isFreed) { - throw new Error("Attempt to read a freed WASM typed array reference."); + protected assertNotFreed() { + if (this.isFreed) { + throw new Error( + "Attempt to read a freed WASM typed array reference." + ); + } } - } } class Int8ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 0; - return this.module.HEAP8.subarray( - startIndex, - startIndex + this.length, - ) as Int8Array; - } + getView() { + const startIndex = this.ptr >>> 0; + return this.module.HEAP8.subarray( + startIndex, + startIndex + this.length + ) as Int8Array; + } } class Uint8ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 0; - return this.module.HEAPU8.subarray( - startIndex, - startIndex + this.length, - ) as Uint8Array; - } + getView() { + const startIndex = this.ptr >>> 0; + return this.module.HEAPU8.subarray( + startIndex, + startIndex + this.length + ) as Uint8Array; + } - readAsNullTerminatedUtf8String(): string { - let strBytes = this.view; + readAsNullTerminatedUtf8String(): string { + let strBytes = this.view; - const indexOfFirstZero = strBytes.indexOf(0); + const indexOfFirstZero = strBytes.indexOf(0); - if (indexOfFirstZero >= 0) { - strBytes = strBytes.subarray(0, indexOfFirstZero); - } + if (indexOfFirstZero >= 0) { + strBytes = strBytes.subarray(0, indexOfFirstZero); + } - const str = Buffer.from(strBytes).toString("utf8"); + const str = Buffer.from(strBytes).toString("utf8"); - return str; - } + return str; + } } class Int16ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 1; - return this.module.HEAP16.subarray( - startIndex, - startIndex + this.length, - ) as Int16Array; - } + getView() { + const startIndex = this.ptr >>> 1; + return this.module.HEAP16.subarray( + startIndex, + startIndex + this.length + ) as Int16Array; + } } class Uint16ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 1; - return this.module.HEAPU16.subarray( - startIndex, - startIndex + this.length, - ) as Uint16Array; - } + getView() { + const startIndex = this.ptr >>> 1; + return this.module.HEAPU16.subarray( + startIndex, + startIndex + this.length + ) as Uint16Array; + } } class Int32ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 2; - return this.module.HEAP32.subarray( - startIndex, - startIndex + this.length, - ) as Int32Array; - } + getView() { + const startIndex = this.ptr >>> 2; + return this.module.HEAP32.subarray( + startIndex, + startIndex + this.length + ) as Int32Array; + } } class Uint32ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 2; - return this.module.HEAPU32.subarray( - startIndex, - startIndex + this.length, - ) as Uint32Array; - } + getView() { + const startIndex = this.ptr >>> 2; + return this.module.HEAPU32.subarray( + startIndex, + startIndex + this.length + ) as Uint32Array; + } } class Float32ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 2; - return this.module.HEAPF32.subarray( - startIndex, - startIndex + this.length, - ) as Float32Array; - } + getView() { + const startIndex = this.ptr >>> 2; + return this.module.HEAPF32.subarray( + startIndex, + startIndex + this.length + ) as Float32Array; + } } class Float64ArrayRef extends TypedArrayRef { - getView() { - const startIndex = this.ptr >>> 3; - return this.module.HEAPF64.subarray( - startIndex, - startIndex + this.length, - ) as Float64Array; - } + getView() { + const startIndex = this.ptr >>> 3; + return this.module.HEAPF64.subarray( + startIndex, + startIndex + this.length + ) as Float64Array; + } } type TypedArray = - | Int8Array - | Uint8Array - | Uint8ClampedArray - | Int16Array - | Uint16Array - | Int32Array - | Uint32Array - | Float32Array - | Float64Array; + | Int8Array + | Uint8Array + | Uint8ClampedArray + | Int16Array + | Uint16Array + | Int32Array + | Uint32Array + | Float32Array + | Float64Array; type WasmRef = ValueRef | ValueRef | TypedArrayRef; class WasmMemoryManager { - wasmModule: any; - - private allocatedReferences = new Set(); - - constructor(wasmModule: any) { - this.wasmModule = wasmModule; - } - - allocInt8() { - const address = this.alloc(1); - return this.wrapInt8(address).clear(); - } - - wrapInt8(address: number) { - const ref = new Int8Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint8() { - const address = this.alloc(1); - return this.wrapUint8(address).clear(); - } - - wrapUint8(address: number) { - const ref = new Uint8Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocInt16() { - const address = this.alloc(2); - return this.wrapInt16(address).clear(); - } - - wrapInt16(address: number) { - const ref = new Int16Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint16() { - const address = this.alloc(2); - return this.wrapUint16(address).clear(); - } - - wrapUint16(address: number) { - const ref = new Uint16Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocInt32() { - const address = this.alloc(4); - return this.wrapInt32(address).clear(); - } - - wrapInt32(address: number) { - const ref = new Int32Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint32() { - const address = this.alloc(4); - return this.wrapUint32(address).clear(); - } - - wrapUint32(address: number) { - const ref = new Uint32Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocPointer() { - const address = this.alloc(4); - return this.wrapPointer(address).clear(); - } - - wrapPointer(address: number) { - const ref = new PointerRef(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocFloat32() { - const address = this.alloc(4); - return this.wrapFloat64(address).clear(); - } - - wrapFloat32(address: number) { - const ref = new Float32Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocFloat64() { - const address = this.alloc(8); - return this.wrapFloat64(address).clear(); - } - - wrapFloat64(address: number) { - const ref = new Float64Ref(address, this); - this.allocatedReferences.add(ref); - return ref; - } - - // Allocate or wrap arrays - allocInt8Array(length: number) { - const address = this.alloc(length << 0); - return this.wrapInt8Array(address, length).clear(); - } - - wrapInt8Array(address: number, length: number) { - const ref = new Int8ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint8Array(length: number) { - const address = this.alloc(length << 0); - return this.wrapUint8Array(address, length).clear(); - } - - wrapUint8Array(address: number, length: number) { - const ref = new Uint8ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocInt16Array(length: number) { - const address = this.alloc(length << 1); - return this.wrapInt16Array(address, length).clear(); - } - - wrapInt16Array(address: number, length: number) { - const ref = new Int16ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint16Array(length: number) { - const address = this.alloc(length << 1); - return this.wrapUint16Array(address, length).clear(); - } - - wrapUint16Array(address: number, length: number) { - const ref = new Uint16ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocInt32Array(length: number) { - const address = this.alloc(length << 2); - return this.wrapInt32Array(address, length).clear(); - } - - wrapInt32Array(address: number, length: number) { - const ref = new Int32ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocUint32Array(length: number) { - const address = this.alloc(length << 2); - return this.wrapUint32Array(address, length).clear(); - } - - wrapUint32Array(address: number, length: number) { - const ref = new Uint32ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocFloat32Array(length: number) { - const address = this.alloc(length << 2); - return this.wrapFloat32Array(address, length).clear(); - } - - wrapFloat32Array(address: number, length: number) { - const ref = new Float32ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocFloat64Array(length: number) { - const address = this.alloc(length << 3); - return this.wrapFloat64Array(address, length).clear(); - } - - wrapFloat64Array(address: number, length: number) { - const ref = new Float64ArrayRef(address, length, this); - this.allocatedReferences.add(ref); - return ref; - } - - allocNullTerminatedUtf8String(str: string) { - const strBuffer = Buffer.concat([ - Buffer.from(str, "utf8"), - Buffer.alloc(1), - ]); - const ref = this.allocUint8Array(strBuffer.length); - ref.view.set(strBuffer); - return ref; - } + wasmModule: any; - wrapNullTerminatedUtf8String(address: number) { - const ref = new NullTerminatedUtf8StringRef(address, this); - this.allocatedReferences.add(ref); - return ref; - } + private allocatedReferences = new Set(); - private alloc(size: number) { - const ptr = this.wasmModule._malloc(size); - return ptr as number; - } + constructor(wasmModule: any) { + this.wasmModule = wasmModule; + } + + allocInt8() { + const address = this.alloc(1); + return this.wrapInt8(address).clear(); + } + + wrapInt8(address: number) { + const ref = new Int8Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint8() { + const address = this.alloc(1); + return this.wrapUint8(address).clear(); + } + + wrapUint8(address: number) { + const ref = new Uint8Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocInt16() { + const address = this.alloc(2); + return this.wrapInt16(address).clear(); + } + + wrapInt16(address: number) { + const ref = new Int16Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint16() { + const address = this.alloc(2); + return this.wrapUint16(address).clear(); + } + + wrapUint16(address: number) { + const ref = new Uint16Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocInt32() { + const address = this.alloc(4); + return this.wrapInt32(address).clear(); + } + + wrapInt32(address: number) { + const ref = new Int32Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint32() { + const address = this.alloc(4); + return this.wrapUint32(address).clear(); + } + + wrapUint32(address: number) { + const ref = new Uint32Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocPointer() { + const address = this.alloc(4); + return this.wrapPointer(address).clear(); + } + + wrapPointer(address: number) { + const ref = new PointerRef(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocFloat32() { + const address = this.alloc(4); + return this.wrapFloat64(address).clear(); + } + + wrapFloat32(address: number) { + const ref = new Float32Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocFloat64() { + const address = this.alloc(8); + return this.wrapFloat64(address).clear(); + } + + wrapFloat64(address: number) { + const ref = new Float64Ref(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + // Allocate or wrap arrays + allocInt8Array(length: number) { + const address = this.alloc(length << 0); + return this.wrapInt8Array(address, length).clear(); + } + + wrapInt8Array(address: number, length: number) { + const ref = new Int8ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint8Array(length: number) { + const address = this.alloc(length << 0); + return this.wrapUint8Array(address, length).clear(); + } + + wrapUint8Array(address: number, length: number) { + const ref = new Uint8ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocInt16Array(length: number) { + const address = this.alloc(length << 1); + return this.wrapInt16Array(address, length).clear(); + } + + wrapInt16Array(address: number, length: number) { + const ref = new Int16ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint16Array(length: number) { + const address = this.alloc(length << 1); + return this.wrapUint16Array(address, length).clear(); + } + + wrapUint16Array(address: number, length: number) { + const ref = new Uint16ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocInt32Array(length: number) { + const address = this.alloc(length << 2); + return this.wrapInt32Array(address, length).clear(); + } + + wrapInt32Array(address: number, length: number) { + const ref = new Int32ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocUint32Array(length: number) { + const address = this.alloc(length << 2); + return this.wrapUint32Array(address, length).clear(); + } + + wrapUint32Array(address: number, length: number) { + const ref = new Uint32ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocFloat32Array(length: number) { + const address = this.alloc(length << 2); + return this.wrapFloat32Array(address, length).clear(); + } + + wrapFloat32Array(address: number, length: number) { + const ref = new Float32ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } + + allocFloat64Array(length: number) { + const address = this.alloc(length << 3); + return this.wrapFloat64Array(address, length).clear(); + } + + wrapFloat64Array(address: number, length: number) { + const ref = new Float64ArrayRef(address, length, this); + this.allocatedReferences.add(ref); + return ref; + } - free(wasmReference: WasmRef) { - if (wasmReference.isFreed) { - return; + allocNullTerminatedUtf8String(str: string) { + const strBuffer = Buffer.concat([ + Buffer.from(str, "utf8"), + Buffer.alloc(1), + ]); + const ref = this.allocUint8Array(strBuffer.length); + ref.view.set(strBuffer); + return ref; } - this.wasmModule._free(wasmReference.address); + wrapNullTerminatedUtf8String(address: number) { + const ref = new NullTerminatedUtf8StringRef(address, this); + this.allocatedReferences.add(ref); + return ref; + } + + private alloc(size: number) { + const ptr = this.wasmModule._malloc(size); + return ptr as number; + } + + free(wasmReference: WasmRef) { + if (wasmReference.isFreed) { + return; + } - this.allocatedReferences.delete(wasmReference); - wasmReference.clearAddress(); - } + this.wasmModule._free(wasmReference.address); - freeAll() { - for (const wasmReference of this.allocatedReferences) { - this.free(wasmReference); + this.allocatedReferences.delete(wasmReference); + wasmReference.clearAddress(); + } + + freeAll() { + for (const wasmReference of this.allocatedReferences) { + this.free(wasmReference); + } } - } } function concatFloat32Arrays(arrays: Float32Array[]) { - return concatTypedArrays(Float32Array, arrays); + return concatTypedArrays(Float32Array, arrays); } function simplifyPunctuationCharacters(text: string) { - return text - .replaceAll(`“`, `"`) - .replaceAll(`”`, `"`) - .replaceAll(`„`, `"`) - .replaceAll(`ߵ`, `"`) - .replaceAll(`ߴ`, `"`) - .replaceAll(`«`, `"`) - .replaceAll(`»`, `"`) - - .replaceAll(`’`, `'`) - .replaceAll(`ʼ`, `'`) - .replaceAll(`ʼ`, `'`) - .replaceAll(`＇`, `'`) - .replaceAll(`，`, `,`) - .replaceAll(`、`, `,`) - .replaceAll(`：`, `:`) - .replaceAll(`；`, `;`) - .replaceAll(`。`, `.`) - - .replaceAll(`？`, `?`) - .replaceAll(`！`, `!`) - .replaceAll(`؟`, `?`); + return text + .replaceAll(`“`, `"`) + .replaceAll(`”`, `"`) + .replaceAll(`„`, `"`) + .replaceAll(`ߵ`, `"`) + .replaceAll(`ߴ`, `"`) + .replaceAll(`«`, `"`) + .replaceAll(`»`, `"`) + + .replaceAll(`’`, `'`) + .replaceAll(`ʼ`, `'`) + .replaceAll(`ʼ`, `'`) + .replaceAll(`＇`, `'`) + .replaceAll(`，`, `,`) + .replaceAll(`、`, `,`) + .replaceAll(`：`, `:`) + .replaceAll(`；`, `;`) + .replaceAll(`。`, `.`) + + .replaceAll(`？`, `?`) + .replaceAll(`！`, `!`) + .replaceAll(`؟`, `?`); } function normalizeFourDigitDecadeString(decadeString: string) { - const firstTwoDigitsValue = parseInt(decadeString.substring(0, 2)); - const secondTwoDigitsValue = parseInt(decadeString.substring(2, 4)); + const firstTwoDigitsValue = parseInt(decadeString.substring(0, 2)); + const secondTwoDigitsValue = parseInt(decadeString.substring(2, 4)); - let normalizedString: string; + let normalizedString: string; - const isBeforeSecondMillenium = firstTwoDigitsValue < 10; - const isMilleniumDecade = - firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0; + const isBeforeSecondMillenium = firstTwoDigitsValue < 10; + const isMilleniumDecade = + firstTwoDigitsValue % 10 == 0 && secondTwoDigitsValue == 0; - if (!isBeforeSecondMillenium && !isMilleniumDecade) { - if (secondTwoDigitsValue != 0) { - normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}s`; + if (!isBeforeSecondMillenium && !isMilleniumDecade) { + if (secondTwoDigitsValue != 0) { + normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}s`; + } else { + normalizedString = `${firstTwoDigitsValue} hundreds`; + } } else { - normalizedString = `${firstTwoDigitsValue} hundreds`; + normalizedString = decadeString; } - } else { - normalizedString = decadeString; - } - return normalizedString; + return normalizedString; } function normalizeFourDigitYearString(yearString: string) { - const firstTwoDigitsValue = parseFloat(yearString.substring(0, 2)); - const secondTwoDigitsValue = parseFloat(yearString.substring(2, 4)); - - let normalizedString: string; + const firstTwoDigitsValue = parseFloat(yearString.substring(0, 2)); + const secondTwoDigitsValue = parseFloat(yearString.substring(2, 4)); - if (firstTwoDigitsValue >= 10 && secondTwoDigitsValue >= 10) { - normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}`; - } else if ( - firstTwoDigitsValue >= 10 && - firstTwoDigitsValue % 10 != 0 && - secondTwoDigitsValue < 10 - ) { - normalizedString = `${firstTwoDigitsValue} oh ${secondTwoDigitsValue}`; - } else { - normalizedString = yearString; - } + let normalizedString: string; - return normalizedString; -} - -function getNormalizedFragmentsForSpeech(words: string[], language: string) { - language = getShortLanguageCode(language); - - if (language != "en") { - return { normalizedFragments: [...words], referenceFragments: [...words] }; - } - - const numberPattern = /^[0-9][0-9\,\.]*$/; - - const fourDigitYearPattern = /^[0-9][0-9][0-9][0-9]$/; - const fourDigitDecadePattern = /^[0-9][0-9][0-9]0s$/; - - const fourDigitYearRangePattern = - /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/; - - const wordsPrecedingAYear = [ - "in", - "the", - "a", - "to", - "of", - "since", - "from", - "between", - "by", - "until", - "around", - "before", - "after", - "his", - "her", - "year", - "years", - "during", - "copyright", - "©", - "early", - "mid", - "late", - "january", - "february", - "march", - "april", - "may", - "june", - "july", - "august", - "september", - "october", - "november", - "december", - "jan", - "feb", - "mar", - "apr", - "may", - "jun", - "jul", - "aug", - "sep", - "oct", - "nov", - "dec", - ]; - - const wordsPrecedingADecade = ["the", "in", "early", "mid", "late", "a"]; - - const symbolsPrecedingACurrency = ["$", "€", "£", "¥"]; - - const symbolsPrecedingACurrencyAsWords = [ - "dollars", - "euros", - "pounds", - "yen", - ]; - - const wordsSucceedingACurrency = ["million", "billion", "trillion"]; - - const normalizedFragments: string[] = []; - const referenceFragments: string[] = []; - - for (let wordIndex = 0; wordIndex < words.length; wordIndex++) { - const word = words[wordIndex]; - const lowerCaseWord = word.toLowerCase(); - - const nextWords = words.slice(wordIndex + 1); - const nextWord = nextWords[0]; - - if ( - // Normalize a four digit year pattern, e.g. 'in 1995'. - wordsPrecedingAYear.includes(lowerCaseWord) && - fourDigitYearPattern.test(nextWord) - ) { - const normalizedString = normalizeFourDigitYearString(nextWord); - - normalizedFragments.push(word); - referenceFragments.push(word); - - normalizedFragments.push(normalizedString); - referenceFragments.push(nextWord); - - wordIndex += 1; + if (firstTwoDigitsValue >= 10 && secondTwoDigitsValue >= 10) { + normalizedString = `${firstTwoDigitsValue} ${secondTwoDigitsValue}`; } else if ( - // Normalize a four digit decade pattern, e.g. 'the 1980s'. - wordsPrecedingADecade.includes(lowerCaseWord) && - fourDigitDecadePattern.test(nextWord) + firstTwoDigitsValue >= 10 && + firstTwoDigitsValue % 10 != 0 && + secondTwoDigitsValue < 10 ) { - const normalizedString = normalizeFourDigitDecadeString(nextWord); - - normalizedFragments.push(word); - referenceFragments.push(word); + normalizedString = `${firstTwoDigitsValue} oh ${secondTwoDigitsValue}`; + } else { + normalizedString = yearString; + } - normalizedFragments.push(normalizedString); - referenceFragments.push(nextWord); + return normalizedString; +} - wordIndex += 1; - } else if ( - // Normalize a year range pattern, e.g. '1835-1896' - fourDigitYearRangePattern.test( - words.slice(wordIndex, wordIndex + 3).join(""), - ) - ) { - normalizedFragments.push(normalizeFourDigitYearString(words[wordIndex])); - referenceFragments.push(words[wordIndex]); +function getNormalizedFragmentsForSpeech(words: string[], language: string) { + language = getShortLanguageCode(language); + + if (language != "en") { + return { + normalizedFragments: [...words], + referenceFragments: [...words], + }; + } + + const numberPattern = /^[0-9][0-9\,\.]*$/; + + const fourDigitYearPattern = /^[0-9][0-9][0-9][0-9]$/; + const fourDigitDecadePattern = /^[0-9][0-9][0-9]0s$/; + + const fourDigitYearRangePattern = + /^[0-9][0-9][0-9][0-9][\-\–][0-9][0-9][0-9][0-9]$/; + + const wordsPrecedingAYear = [ + "in", + "the", + "a", + "to", + "of", + "since", + "from", + "between", + "by", + "until", + "around", + "before", + "after", + "his", + "her", + "year", + "years", + "during", + "copyright", + "©", + "early", + "mid", + "late", + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + "jan", + "feb", + "mar", + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", + ]; + + const wordsPrecedingADecade = ["the", "in", "early", "mid", "late", "a"]; + + const symbolsPrecedingACurrency = ["$", "€", "£", "¥"]; + + const symbolsPrecedingACurrencyAsWords = [ + "dollars", + "euros", + "pounds", + "yen", + ]; + + const wordsSucceedingACurrency = ["million", "billion", "trillion"]; + + const normalizedFragments: string[] = []; + const referenceFragments: string[] = []; + + for (let wordIndex = 0; wordIndex < words.length; wordIndex++) { + const word = words[wordIndex]; + const lowerCaseWord = word.toLowerCase(); + + const nextWords = words.slice(wordIndex + 1); + const nextWord = nextWords[0]; + + if ( + // Normalize a four digit year pattern, e.g. 'in 1995'. + wordsPrecedingAYear.includes(lowerCaseWord) && + fourDigitYearPattern.test(nextWord) + ) { + const normalizedString = normalizeFourDigitYearString(nextWord); + + normalizedFragments.push(word); + referenceFragments.push(word); + + normalizedFragments.push(normalizedString); + referenceFragments.push(nextWord); + + wordIndex += 1; + } else if ( + // Normalize a four digit decade pattern, e.g. 'the 1980s'. + wordsPrecedingADecade.includes(lowerCaseWord) && + fourDigitDecadePattern.test(nextWord) + ) { + const normalizedString = normalizeFourDigitDecadeString(nextWord); + + normalizedFragments.push(word); + referenceFragments.push(word); + + normalizedFragments.push(normalizedString); + referenceFragments.push(nextWord); + + wordIndex += 1; + } else if ( + // Normalize a year range pattern, e.g. '1835-1896' + fourDigitYearRangePattern.test( + words.slice(wordIndex, wordIndex + 3).join("") + ) + ) { + normalizedFragments.push( + normalizeFourDigitYearString(words[wordIndex]) + ); + referenceFragments.push(words[wordIndex]); - normalizedFragments.push("to"); - referenceFragments.push(words[wordIndex + 1]); + normalizedFragments.push("to"); + referenceFragments.push(words[wordIndex + 1]); - normalizedFragments.push( - normalizeFourDigitYearString(words[wordIndex + 2]), - ); - referenceFragments.push(words[wordIndex + 2]); + normalizedFragments.push( + normalizeFourDigitYearString(words[wordIndex + 2]) + ); + referenceFragments.push(words[wordIndex + 2]); - wordIndex += 2; - } else if ( - // Normalize a currency pattern, e.g. '$53.1 million', '€3.53' - symbolsPrecedingACurrency.includes(lowerCaseWord) && - numberPattern.test(nextWord) - ) { - const currencyWord = - symbolsPrecedingACurrencyAsWords[ - symbolsPrecedingACurrency.indexOf(lowerCaseWord) - ]; + wordIndex += 2; + } else if ( + // Normalize a currency pattern, e.g. '$53.1 million', '€3.53' + symbolsPrecedingACurrency.includes(lowerCaseWord) && + numberPattern.test(nextWord) + ) { + const currencyWord = + symbolsPrecedingACurrencyAsWords[ + symbolsPrecedingACurrency.indexOf(lowerCaseWord) + ]; - if (wordsSucceedingACurrency.includes(nextWords[1].toLowerCase())) { - const normalizedString = `${nextWord} ${nextWords[1]} ${currencyWord}`; + if (wordsSucceedingACurrency.includes(nextWords[1].toLowerCase())) { + const normalizedString = `${nextWord} ${nextWords[1]} ${currencyWord}`; - normalizedFragments.push(normalizedString); + normalizedFragments.push(normalizedString); - const referenceString = `${word}${nextWord} ${nextWords[1]}`; - referenceFragments.push(referenceString); + const referenceString = `${word}${nextWord} ${nextWords[1]}`; + referenceFragments.push(referenceString); - wordIndex += 2; - } else { - const normalizedString = `${nextWord} ${currencyWord}`; + wordIndex += 2; + } else { + const normalizedString = `${nextWord} ${currencyWord}`; - normalizedFragments.push(normalizedString); + normalizedFragments.push(normalizedString); - const referenceString = `${word}${nextWord}`; - referenceFragments.push(referenceString); + const referenceString = `${word}${nextWord}`; + referenceFragments.push(referenceString); - wordIndex += 1; - } - } else { - normalizedFragments.push(word); - referenceFragments.push(word); + wordIndex += 1; + } + } else { + normalizedFragments.push(word); + referenceFragments.push(word); + } } - } - return { normalizedFragments, referenceFragments }; + return { normalizedFragments, referenceFragments }; } const wordCharacterPattern = /[\p{Letter}\p{Number}]/u; async function splitToWords(text: string, langCode: string): Promise { - const shortLangCode = getShortLanguageCode(langCode || ""); - - if (shortLangCode == "zh" || shortLangCode == "cmn") { - return splitChineseTextToWords_Jieba(text, undefined, true); - } else if (shortLangCode == "ja") { - return splitJapaneseTextToWords_Kuromoji(text); - } else { - return CldrSegmentation.wordSplit( - text, - CldrSegmentation.suppressions[shortLangCode], - ); - } + const shortLangCode = getShortLanguageCode(langCode || ""); + + if (shortLangCode == "zh" || shortLangCode == "cmn") { + return splitChineseTextToWords_Jieba(text, undefined, true); + } else if (shortLangCode == "ja") { + return splitJapaneseTextToWords_Kuromoji(text); + } else { + return CldrSegmentation.wordSplit( + text, + CldrSegmentation.suppressions[shortLangCode] + ); + } } const ipaToKirshenbaum: { [p: string]: string | undefined } = { - "1": "1", - "2": "2", - "4": "4", - "5": "5", - "6": "6", - "7": "7", - "9": "9", - " ": " ", - "!": "!", - "'": "'", - ʰ: "#", - $: "$", - "%": "%", - //'æ': '&', - æ: "a", - ˈ: "'", - "(": "(", - ")": ")", - ɾ: "*", - "+": "+", - ˌ: ",", - "-": "-", - ".": ".", - "/": "/", - ɒ: "0", - ɜ: "3", - ɵ: "8", - ː: ":", - ʲ: ";", - "<": "<", - "=": "=", - ">": ">", - ʔ: "?", - ə: "@", - ɑ: "A", - β: "B", - ç: "C", - ð: "D", - ɛ: "E", - F: "F", - ɢ: "G", - ħ: "H", - ɪ: "I", - ɟ: "J", - K: "K", - ɫ: "L", - ɱ: "M", - ŋ: "N", - ɔ: "O", - Φ: "P", - ɣ: "Q", - ʀ: "R", - ʃ: "S", - θ: "T", - ʊ: "U", - ʌ: "V", - œ: "W", - χ: "X", - ø: "Y", - ʒ: "Z", - "̪": "[", - "\\": "\\", - "]": "]", - "^": "^", - _: "_", - "`": "`", - a: "a", - b: "b", - c: "c", - d: "d", - e: "e", - f: "f", - ɡ: "g", - h: "h", - i: "i", - j: "j", - k: "k", - l: "l", - m: "m", - n: "n", - o: "o", - p: "p", - q: "q", - r: "r", - s: "s", - t: "t", - u: "u", - v: "v", - w: "w", - x: "x", - y: "y", - z: "z", - "{": "{", - "|": "|", - "}": "}", - "̃": "~", - "": "", - - // Extensions - ɚ: "3", - ɹ: "r", - ɐ: "a#", - ᵻ: "i", - "̩": ",", + "1": "1", + "2": "2", + "4": "4", + "5": "5", + "6": "6", + "7": "7", + "9": "9", + " ": " ", + "!": "!", + "'": "'", + ʰ: "#", + $: "$", + "%": "%", + //'æ': '&', + æ: "a", + ˈ: "'", + "(": "(", + ")": ")", + ɾ: "*", + "+": "+", + ˌ: ",", + "-": "-", + ".": ".", + "/": "/", + ɒ: "0", + ɜ: "3", + ɵ: "8", + ː: ":", + ʲ: ";", + "<": "<", + "=": "=", + ">": ">", + ʔ: "?", + ə: "@", + ɑ: "A", + β: "B", + ç: "C", + ð: "D", + ɛ: "E", + F: "F", + ɢ: "G", + ħ: "H", + ɪ: "I", + ɟ: "J", + K: "K", + ɫ: "L", + ɱ: "M", + ŋ: "N", + ɔ: "O", + Φ: "P", + ɣ: "Q", + ʀ: "R", + ʃ: "S", + θ: "T", + ʊ: "U", + ʌ: "V", + œ: "W", + χ: "X", + ø: "Y", + ʒ: "Z", + "̪": "[", + "\\": "\\", + "]": "]", + "^": "^", + _: "_", + "`": "`", + a: "a", + b: "b", + c: "c", + d: "d", + e: "e", + f: "f", + ɡ: "g", + h: "h", + i: "i", + j: "j", + k: "k", + l: "l", + m: "m", + n: "n", + o: "o", + p: "p", + q: "q", + r: "r", + s: "s", + t: "t", + u: "u", + v: "v", + w: "w", + x: "x", + y: "y", + z: "z", + "{": "{", + "|": "|", + "}": "}", + "̃": "~", + "": "", + + // Extensions + ɚ: "3", + ɹ: "r", + ɐ: "a#", + ᵻ: "i", + "̩": ",", }; function ipaPhoneToKirshenbaum(ipaPhone: string) { - let result = ""; + let result = ""; - for (const char of ipaPhone) { - const convertedChar = ipaToKirshenbaum[char]; + for (const char of ipaPhone) { + const convertedChar = ipaToKirshenbaum[char]; - if (convertedChar == undefined) { - throw new Error( - `Could not convert phone character '${char}' to Kirshenbaum encoding`, - ); - } + if (convertedChar == undefined) { + throw new Error( + `Could not convert phone character '${char}' to Kirshenbaum encoding` + ); + } - result += convertedChar || "_"; - } + result += convertedChar || "_"; + } - return result; + return result; } function deepClone(val: T) { - return clone(val, true); + return clone(val, true); } function clone(val: T, deep = true, seenObjects: any[] = []): T { - if (val == null || typeof val !== "object") { - return val; - } + if (val == null || typeof val !== "object") { + return val; + } - const obj = val; - const prototypeIdentifier = toString.call(obj); + const obj = val; + const prototypeIdentifier = toString.call(obj); - switch (prototypeIdentifier) { - case "[object Array]": { - if (seenObjects.includes(obj)) { - throw new Error("deepClone: encountered a cyclic object"); - } + switch (prototypeIdentifier) { + case "[object Array]": { + if (seenObjects.includes(obj)) { + throw new Error("deepClone: encountered a cyclic object"); + } - seenObjects.push(obj); + seenObjects.push(obj); - const clonedArray = new Array(obj.length); + const clonedArray = new Array(obj.length); - for (let i = 0; i < obj.length; i++) { - if (deep) { - clonedArray[i] = clone(obj[i], true, seenObjects); - } else { - clonedArray[i] = obj[i]; + for (let i = 0; i < obj.length; i++) { + if (deep) { + clonedArray[i] = clone(obj[i], true, seenObjects); + } else { + clonedArray[i] = obj[i]; + } + } + + seenObjects.pop(); + + return clonedArray; } - } - seenObjects.pop(); + case "[object ArrayBuffer]": { + const clonedArray = new Uint8Array(obj.byteLength); + clonedArray.set(new Uint8Array(obj)); + return clonedArray.buffer; + } - return clonedArray; - } + case "[object Int8Array]": { + const clonedArray = new Int8Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object ArrayBuffer]": { - const clonedArray = new Uint8Array(obj.byteLength); - clonedArray.set(new Uint8Array(obj)); - return clonedArray.buffer; - } + case "[object Uint8Array]": { + const clonedArray = new Uint8Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Int8Array]": { - const clonedArray = new Int8Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Uint8ClampedArray]": { + const clonedArray = new Uint8ClampedArray(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Uint8Array]": { - const clonedArray = new Uint8Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Int16Array]": { + const clonedArray = new Int16Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Uint8ClampedArray]": { - const clonedArray = new Uint8ClampedArray(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Uint16Array]": { + const clonedArray = new Uint16Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Int16Array]": { - const clonedArray = new Int16Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Int32Array]": { + const clonedArray = new Int32Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } + + case "[object Uint32Array]": { + const clonedArray = new Uint32Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Uint16Array]": { - const clonedArray = new Uint16Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Float32Array]": { + const clonedArray = new Float32Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Int32Array]": { - const clonedArray = new Int32Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Float64Array]": { + const clonedArray = new Float64Array(obj.length); + clonedArray.set(obj); + return clonedArray; + } - case "[object Uint32Array]": { - const clonedArray = new Uint32Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Date]": { + return new Date(obj.valueOf()); + } - case "[object Float32Array]": { - const clonedArray = new Float32Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object RegExp]": { + return obj; + } - case "[object Float64Array]": { - const clonedArray = new Float64Array(obj.length); - clonedArray.set(obj); - return clonedArray; - } + case "[object Function]": { + return obj; + } - case "[object Date]": { - return new Date(obj.valueOf()); - } + case "[object Object]": { + if (seenObjects.includes(obj)) { + throw new Error("deepClone: encountered a cyclic object"); + } - case "[object RegExp]": { - return obj; - } + seenObjects.push(obj); - case "[object Function]": { - return obj; - } + const clonedObj: any = {}; - case "[object Object]": { - if (seenObjects.includes(obj)) { - throw new Error("deepClone: encountered a cyclic object"); - } + for (const propName in obj) { + if (!obj.hasOwnProperty(propName)) { + continue; + } - seenObjects.push(obj); + if (deep) { + clonedObj[propName] = clone( + obj[propName], + true, + seenObjects + ); + } else { + clonedObj[propName] = obj[propName]; + } + } - const clonedObj: any = {}; + seenObjects.pop(); - for (const propName in obj) { - if (!obj.hasOwnProperty(propName)) { - continue; + return clonedObj; } - if (deep) { - clonedObj[propName] = clone(obj[propName], true, seenObjects); - } else { - clonedObj[propName] = obj[propName]; + default: { + throw new Error( + `Cloning of type ${prototypeIdentifier} is not supported` + ); } - } - - seenObjects.pop(); - - return clonedObj; } - - default: { - throw new Error( - `Cloning of type ${prototypeIdentifier} is not supported`, - ); - } - } } function isPlainObject(val: any) { - return ( - val != null && - typeof val === "object" && - toString.call(val) === "[object Object]" - ); + return ( + val != null && + typeof val === "object" && + toString.call(val) === "[object Object]" + ); } function extendDeep(base: any, extension: any): any { - const baseClone = deepClone(base); - - if (isPlainObject(base) && extension === undefined) { - return baseClone; - } + const baseClone = deepClone(base); - const extensionClone = deepClone(extension); - if (!isPlainObject(base) || !isPlainObject(extension)) { - return extensionClone; - } + if (isPlainObject(base) && extension === undefined) { + return baseClone; + } - for (const propName in extensionClone) { - if (!extensionClone.hasOwnProperty(propName)) { - continue; + const extensionClone = deepClone(extension); + if (!isPlainObject(base) || !isPlainObject(extension)) { + return extensionClone; } - baseClone[propName] = extendDeep( - baseClone[propName], - extensionClone[propName], - ); - } + for (const propName in extensionClone) { + if (!extensionClone.hasOwnProperty(propName)) { + continue; + } - return baseClone; + baseClone[propName] = extendDeep( + baseClone[propName], + extensionClone[propName] + ); + } + + return baseClone; } function tryGetFirstLexiconSubstitution( - sentenceWords: string[], - wordIndex: number, - lexicons: Lexicon[], - languageCode: string, + sentenceWords: string[], + wordIndex: number, + lexicons: Lexicon[], + languageCode: string ) { - const reversedLexicons = [...lexicons].reverse(); // Give precedence to later lexicons - - for (const lexicon of reversedLexicons) { - const match = tryGetLexiconSubstitution( - sentenceWords, - wordIndex, - lexicon, - languageCode, - ); + const reversedLexicons = [...lexicons].reverse(); // Give precedence to later lexicons + + for (const lexicon of reversedLexicons) { + const match = tryGetLexiconSubstitution( + sentenceWords, + wordIndex, + lexicon, + languageCode + ); - if (match) { - return match; + if (match) { + return match; + } } - } - return undefined; + return undefined; } function tryGetLexiconSubstitution( - sentenceWords: string[], - wordIndex: number, - lexicon: Lexicon, - languageCode: string, + sentenceWords: string[], + wordIndex: number, + lexicon: Lexicon, + languageCode: string ) { - const word = sentenceWords[wordIndex]; + const word = sentenceWords[wordIndex]; - if (!word) { - return; - } + if (!word) { + return; + } - const shortLanguageCode = getShortLanguageCode(languageCode); - const lexiconForLanguage = lexicon[shortLanguageCode]; + const shortLanguageCode = getShortLanguageCode(languageCode); + const lexiconForLanguage = lexicon[shortLanguageCode]; - if (!lexiconForLanguage) { - return; - } + if (!lexiconForLanguage) { + return; + } - const lexiconEntry = lexiconForLanguage[word]; + const lexiconEntry = lexiconForLanguage[word]; - if (!lexiconEntry) { - return; - } + if (!lexiconEntry) { + return; + } - for (let i = 0; i < lexiconEntry.length; i++) { - const substitutionEntry = lexiconEntry[i]; + for (let i = 0; i < lexiconEntry.length; i++) { + const substitutionEntry = lexiconEntry[i]; - const substitutionPhonemesText = - substitutionEntry?.pronunciation?.espeak?.[languageCode]; + const substitutionPhonemesText = + substitutionEntry?.pronunciation?.espeak?.[languageCode]; - if (!substitutionPhonemesText) { - continue; - } + if (!substitutionPhonemesText) { + continue; + } - const precedingWord = sentenceWords[wordIndex - 1] || ""; - const succeedingWord = sentenceWords[wordIndex + 1] || ""; + const precedingWord = sentenceWords[wordIndex - 1] || ""; + const succeedingWord = sentenceWords[wordIndex + 1] || ""; - const precededBy = substitutionEntry?.precededBy || []; - const notPrecededBy = substitutionEntry?.notPrecededBy || []; + const precededBy = substitutionEntry?.precededBy || []; + const notPrecededBy = substitutionEntry?.notPrecededBy || []; - const succeededBy = substitutionEntry?.succeededBy || []; - const notSucceededBy = substitutionEntry?.notSucceededBy || []; + const succeededBy = substitutionEntry?.succeededBy || []; + const notSucceededBy = substitutionEntry?.notSucceededBy || []; - const hasNegativePattern = - notPrecededBy.includes(precedingWord) || - notSucceededBy.includes(succeedingWord); - const hasPositivePattern = - precededBy.includes(precedingWord) || - succeededBy.includes(succeedingWord); + const hasNegativePattern = + notPrecededBy.includes(precedingWord) || + notSucceededBy.includes(succeedingWord); + const hasPositivePattern = + precededBy.includes(precedingWord) || + succeededBy.includes(succeedingWord); - if ( - i == lexiconEntry.length - 1 || - (hasPositivePattern && !hasNegativePattern) - ) { - const substitutionPhonemes = substitutionPhonemesText.split(/ +/g); + if ( + i == lexiconEntry.length - 1 || + (hasPositivePattern && !hasNegativePattern) + ) { + const substitutionPhonemes = substitutionPhonemesText.split(/ +/g); - return substitutionPhonemes; + return substitutionPhonemes; + } } - } - return; + return; } function int16PcmToFloat32(input: Int16Array) { - const output = new Float32Array(input.length); + const output = new Float32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = input[i]; - output[i] = sample < 0 ? sample / 32768 : sample / 32767; - } + for (let i = 0; i < input.length; i++) { + const sample = input[i]; + output[i] = sample < 0 ? sample / 32768 : sample / 32767; + } - return output; + return output; } let espeakInstance: any; let espeakModule: any; type EspeakEventType = - | "sentence" - | "word" - | "phoneme" - | "end" - | "mark" - | "play" - | "msg_terminated" - | "list_terminated" - | "samplerate"; + | "sentence" + | "word" + | "phoneme" + | "end" + | "mark" + | "play" + | "msg_terminated" + | "list_terminated" + | "samplerate"; interface EspeakEvent { - audio_position: number; - type: EspeakEventType; - text_position: number; - word_length: number; - id?: string | number; + audio_position: number; + type: EspeakEventType; + text_position: number; + word_length: number; + id?: string | number; } async function setVoice(voiceId: string) { - const { instance } = await getEspeakInstance(); + const { instance } = await getEspeakInstance(); - instance.set_voice(voiceId); + instance.set_voice(voiceId); } async function getEspeakInstance() { - if (!espeakInstance) { - const { default: EspeakInitializer } = await import( - "@echogarden/espeak-ng-emscripten" - ); + if (!espeakInstance) { + const { default: EspeakInitializer } = await import( + "@echogarden/espeak-ng-emscripten" + ); - const m = await EspeakInitializer(); - espeakInstance = await new m.eSpeakNGWorker(); - espeakModule = m; - } + const m = await EspeakInitializer(); + espeakInstance = await new m.eSpeakNGWorker(); + espeakModule = m; + } - return { instance: espeakInstance, module: espeakModule }; + return { instance: espeakInstance, module: espeakModule }; } async function getSampleRate(): Promise<22050> { - return 22050; + return 22050; } async function synthesizeFragments( - fragments: string[], - espeakOptions: EspeakOptions, + fragments: string[], + espeakOptions: EspeakOptions ) { - espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); - - const voice = espeakOptions.voice; - - const sampleRate = await getSampleRate(); - - if (fragments.length === 0) { - return { - rawAudio: getEmptyRawAudio(1, sampleRate), - timeline: [] as Timeline, - events: [] as EspeakEvent[], - }; - } - - const canInsertSeparators = ![ - "roa/an", - "art/eo", - "trk/ky", - "zlw/pl", - "zle/uk", - ].includes(voice); - - let textWithMarkers: string; + espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); - if (canInsertSeparators) { - textWithMarkers = `() | `; - } else { - textWithMarkers = `() `; - } + const voice = espeakOptions.voice; - for (let i = 0; i < fragments.length; i++) { - let fragment = fragments[i]; + const sampleRate = await getSampleRate(); - fragment = simplifyPunctuationCharacters(fragment); + if (fragments.length === 0) { + return { + rawAudio: getEmptyRawAudio(1, sampleRate), + timeline: [] as Timeline, + events: [] as EspeakEvent[], + }; + } - fragment = fragment.replaceAll("<", "<").replaceAll(">", ">"); + const canInsertSeparators = ![ + "roa/an", + "art/eo", + "trk/ky", + "zlw/pl", + "zle/uk", + ].includes(voice); - if (espeakOptions.insertSeparators && canInsertSeparators) { - const separator = ` | `; + let textWithMarkers: string; - textWithMarkers += `${separator}${fragment}${separator}`; + if (canInsertSeparators) { + textWithMarkers = `() | `; } else { - if (fragment.endsWith(".")) { - fragment += " ()"; - } - - textWithMarkers += `${fragment} `; - } - } - - const { rawAudio, events } = await espeakSynthesize(textWithMarkers, { - ...espeakOptions, - ssml: true, - }); - - // Add first marker if missing - if (fragments.length > 0) { - const firstMarkerEvent = events.find((event) => event.type === "mark"); - - if (firstMarkerEvent && firstMarkerEvent.id === "e-0") { - events.unshift({ - type: "mark", - text_position: 0, - word_length: 0, - audio_position: 0, - id: "s-0", - }); - } - } - - // Build word timeline from events - const wordTimeline: Timeline = fragments.map((word) => ({ - type: "word", - text: word, - startTime: -1, - endTime: -1, - timeline: [ - { - type: "token", - text: "", - startTime: -1, - endTime: -1, - timeline: [], - }, - ], - })); - - let wordIndex = 0; + textWithMarkers = `() `; + } - const clauseEndIndexes: number[] = []; + for (let i = 0; i < fragments.length; i++) { + let fragment = fragments[i]; - for (const event of events) { - const eventTime = event.audio_position / 1000; + fragment = simplifyPunctuationCharacters(fragment); - const currentWordEntry = wordTimeline[wordIndex]; + fragment = fragment.replaceAll("<", "<").replaceAll(">", ">"); - const currentTokenTimeline = currentWordEntry.timeline!; - const currentTokenEntry = - currentTokenTimeline[currentTokenTimeline.length - 1]; + if (espeakOptions.insertSeparators && canInsertSeparators) { + const separator = ` | `; - const currentPhoneTimeline = currentTokenEntry.timeline!; - const lastPhoneEntry = - currentPhoneTimeline[currentPhoneTimeline.length - 1]; + textWithMarkers += `${separator}${fragment}${separator}`; + } else { + if (fragment.endsWith(".")) { + fragment += " ()"; + } - if (lastPhoneEntry && lastPhoneEntry.endTime === -1) { - lastPhoneEntry.endTime = eventTime; + textWithMarkers += `${fragment} `; + } } - if (event.type === "word") { - if (!event.id || currentPhoneTimeline.length === 0) { - continue; - } + const { rawAudio, events } = await espeakSynthesize(textWithMarkers, { + ...espeakOptions, + ssml: true, + }); - if (currentTokenEntry.endTime === -1) { - currentTokenEntry.endTime = eventTime; - } + // Add first marker if missing + if (fragments.length > 0) { + const firstMarkerEvent = events.find((event) => event.type === "mark"); + + if (firstMarkerEvent && firstMarkerEvent.id === "e-0") { + events.unshift({ + type: "mark", + text_position: 0, + word_length: 0, + audio_position: 0, + id: "s-0", + }); + } + } - currentTokenTimeline.push({ - type: "token", - text: "", - startTime: eventTime, - endTime: -1, - timeline: [], - }); - } else if (event.type === "phoneme") { - const phoneText = event.id as string; - - if (!phoneText || phoneText.startsWith("(")) { - continue; - } - - currentPhoneTimeline.push({ - type: "phone", - text: phoneText, - startTime: eventTime, + // Build word timeline from events + const wordTimeline: Timeline = fragments.map((word) => ({ + type: "word", + text: word, + startTime: -1, endTime: -1, - }); + timeline: [ + { + type: "token", + text: "", + startTime: -1, + endTime: -1, + timeline: [], + }, + ], + })); - currentTokenEntry.text += phoneText; - currentTokenEntry.startTime = currentPhoneTimeline[0].startTime; - } else if (event.type === "mark") { - const markerName = event.id! as string; + let wordIndex = 0; - if (markerName.startsWith("s-")) { - const markerIndex = parseInt(markerName.substring(2)); + const clauseEndIndexes: number[] = []; - if (markerIndex != wordIndex) { - throw new Error( - `Word start marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}`, - ); - } + for (const event of events) { + const eventTime = event.audio_position / 1000; - if (currentPhoneTimeline.length > 0) { - throw new Error( - `Word entry ${wordIndex} already has phones before its start marker was seen. The words were: ${objToString(fragments)}`, - ); - } + const currentWordEntry = wordTimeline[wordIndex]; - currentWordEntry.startTime = eventTime; - currentTokenEntry.startTime = eventTime; - } else if (markerName.startsWith("e-")) { - const markerIndex = parseInt(markerName.substring(2)); + const currentTokenTimeline = currentWordEntry.timeline!; + const currentTokenEntry = + currentTokenTimeline[currentTokenTimeline.length - 1]; - if (markerIndex != wordIndex) { - throw new Error( - `Word end marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}`, - ); + const currentPhoneTimeline = currentTokenEntry.timeline!; + const lastPhoneEntry = + currentPhoneTimeline[currentPhoneTimeline.length - 1]; + + if (lastPhoneEntry && lastPhoneEntry.endTime === -1) { + lastPhoneEntry.endTime = eventTime; } - currentWordEntry.startTime = currentTokenTimeline[0].startTime; + if (event.type === "word") { + if (!event.id || currentPhoneTimeline.length === 0) { + continue; + } - currentWordEntry.endTime = eventTime; - currentTokenEntry.endTime = eventTime; + if (currentTokenEntry.endTime === -1) { + currentTokenEntry.endTime = eventTime; + } - wordIndex += 1; + currentTokenTimeline.push({ + type: "token", + text: "", + startTime: eventTime, + endTime: -1, + timeline: [], + }); + } else if (event.type === "phoneme") { + const phoneText = event.id as string; + + if (!phoneText || phoneText.startsWith("(")) { + continue; + } - if (wordIndex === wordTimeline.length) { - break; + currentPhoneTimeline.push({ + type: "phone", + text: phoneText, + startTime: eventTime, + endTime: -1, + }); + + currentTokenEntry.text += phoneText; + currentTokenEntry.startTime = currentPhoneTimeline[0].startTime; + } else if (event.type === "mark") { + const markerName = event.id! as string; + + if (markerName.startsWith("s-")) { + const markerIndex = parseInt(markerName.substring(2)); + + if (markerIndex != wordIndex) { + throw new Error( + `Word start marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}` + ); + } + + if (currentPhoneTimeline.length > 0) { + throw new Error( + `Word entry ${wordIndex} already has phones before its start marker was seen. The words were: ${objToString(fragments)}` + ); + } + + currentWordEntry.startTime = eventTime; + currentTokenEntry.startTime = eventTime; + } else if (markerName.startsWith("e-")) { + const markerIndex = parseInt(markerName.substring(2)); + + if (markerIndex != wordIndex) { + throw new Error( + `Word end marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}` + ); + } + + currentWordEntry.startTime = currentTokenTimeline[0].startTime; + + currentWordEntry.endTime = eventTime; + currentTokenEntry.endTime = eventTime; + + wordIndex += 1; + + if (wordIndex === wordTimeline.length) { + break; + } + } else { + continue; + } + } else if (event.type === "end") { + clauseEndIndexes.push(wordIndex); } - } else { - continue; - } - } else if (event.type === "end") { - clauseEndIndexes.push(wordIndex); } - } - - clauseEndIndexes.push(wordTimeline.length); - // Split compound tokens - for (const [index, wordEntry] of wordTimeline.entries()) { - const tokenTimeline = wordEntry.timeline; + clauseEndIndexes.push(wordTimeline.length); - if (index === 0) { - continue; - } + // Split compound tokens + for (const [index, wordEntry] of wordTimeline.entries()) { + const tokenTimeline = wordEntry.timeline; - if (!tokenTimeline || tokenTimeline.length === 0) { - throw new Error( - "Unexpected: token timeline should exist and have at least one token", - ); - } + if (index === 0) { + continue; + } - if (tokenTimeline.length !== 1 && tokenTimeline[0].text != "") { - continue; - } + if (!tokenTimeline || tokenTimeline.length === 0) { + throw new Error( + "Unexpected: token timeline should exist and have at least one token" + ); + } - const wordReferencePhonemes = ( - await textToPhonemes(wordEntry.text, espeakOptions.voice, true) - ).split("_"); + if (tokenTimeline.length !== 1 && tokenTimeline[0].text != "") { + continue; + } - const wordReferenceIPA = wordReferencePhonemes.join(" "); + const wordReferencePhonemes = ( + await textToPhonemes(wordEntry.text, espeakOptions.voice, true) + ).split("_"); - if (wordReferenceIPA.trim().length === 0) { - continue; - } + const wordReferenceIPA = wordReferencePhonemes.join(" "); - const wordReferenceIPAWithoutStress = wordReferenceIPA - .replaceAll("ˈ", "") - .replaceAll("ˌ", ""); + if (wordReferenceIPA.trim().length === 0) { + continue; + } - const previousWordEntry = wordTimeline[index - 1]; + const wordReferenceIPAWithoutStress = wordReferenceIPA + .replaceAll("ˈ", "") + .replaceAll("ˌ", ""); - if (!previousWordEntry.timeline) { - continue; - } + const previousWordEntry = wordTimeline[index - 1]; - const previousWordTokenEntry = - previousWordEntry.timeline[previousWordEntry.timeline.length - 1]; + if (!previousWordEntry.timeline) { + continue; + } - if (!previousWordTokenEntry.timeline) { - continue; - } + const previousWordTokenEntry = + previousWordEntry.timeline[previousWordEntry.timeline.length - 1]; - const previousWordTokenIPAWithoutStress = previousWordTokenEntry.timeline - .map((phoneEntry) => - phoneEntry.text.replaceAll("ˈ", "").replaceAll("ˌ", ""), - ) - .join(" "); + if (!previousWordTokenEntry.timeline) { + continue; + } - if ( - previousWordEntry.timeline.length > 1 && - previousWordTokenIPAWithoutStress === wordReferenceIPAWithoutStress - ) { - tokenTimeline.pop(); + const previousWordTokenIPAWithoutStress = + previousWordTokenEntry.timeline + .map((phoneEntry) => + phoneEntry.text.replaceAll("ˈ", "").replaceAll("ˌ", "") + ) + .join(" "); - const tokenEntryToInsert = previousWordEntry.timeline.pop()!; - tokenTimeline.push(tokenEntryToInsert); + if ( + previousWordEntry.timeline.length > 1 && + previousWordTokenIPAWithoutStress === wordReferenceIPAWithoutStress + ) { + tokenTimeline.pop(); - previousWordEntry.endTime = - previousWordEntry.timeline[ - previousWordEntry.timeline.length - 1 - ].endTime; + const tokenEntryToInsert = previousWordEntry.timeline.pop()!; + tokenTimeline.push(tokenEntryToInsert); - wordEntry.startTime = tokenEntryToInsert.startTime; - wordEntry.endTime = tokenEntryToInsert.endTime; + previousWordEntry.endTime = + previousWordEntry.timeline[ + previousWordEntry.timeline.length - 1 + ].endTime; - continue; - } + wordEntry.startTime = tokenEntryToInsert.startTime; + wordEntry.endTime = tokenEntryToInsert.endTime; - if ( - previousWordTokenEntry.timeline.length <= wordReferencePhonemes.length - ) { - continue; - } + continue; + } - if ( - !previousWordTokenIPAWithoutStress.endsWith(wordReferenceIPAWithoutStress) - ) { - continue; - } + if ( + previousWordTokenEntry.timeline.length <= + wordReferencePhonemes.length + ) { + continue; + } - const tokenEntry = tokenTimeline[0]; + if ( + !previousWordTokenIPAWithoutStress.endsWith( + wordReferenceIPAWithoutStress + ) + ) { + continue; + } - tokenEntry.timeline = previousWordTokenEntry.timeline.splice( - previousWordTokenEntry.timeline.length - wordReferencePhonemes.length, - ); - tokenEntry.text = tokenEntry.timeline - .map((phoneEntry) => phoneEntry.text) - .join(""); - - tokenEntry.startTime = tokenEntry.timeline[0].startTime; - tokenEntry.endTime = - tokenEntry.timeline[tokenEntry.timeline.length - 1].endTime; - wordEntry.startTime = tokenEntry.startTime; - wordEntry.endTime = tokenEntry.endTime; - - previousWordTokenEntry.text = previousWordTokenEntry.timeline - .map((phoneEntry) => phoneEntry.text) - .join(""); - previousWordTokenEntry.endTime = - previousWordTokenEntry.timeline[ - previousWordTokenEntry.timeline.length - 1 - ].endTime; - previousWordEntry.endTime = previousWordTokenEntry.endTime; - } - - // Build clause timeline - const clauseTimeline: Timeline = []; - - let clauseStartIndex = 0; - - for (const clauseEndIndex of clauseEndIndexes) { - const newClause: TimelineEntry = { - type: "clause", - text: "", - startTime: -1, - endTime: -1, - timeline: [], - }; + const tokenEntry = tokenTimeline[0]; - for ( - let entryIndex = clauseStartIndex; - entryIndex <= clauseEndIndex && entryIndex < wordTimeline.length; - entryIndex++ - ) { - const wordEntry = wordTimeline[entryIndex]; - if (newClause.startTime === -1) { - newClause.startTime = wordEntry.startTime; - } + tokenEntry.timeline = previousWordTokenEntry.timeline.splice( + previousWordTokenEntry.timeline.length - + wordReferencePhonemes.length + ); + tokenEntry.text = tokenEntry.timeline + .map((phoneEntry) => phoneEntry.text) + .join(""); + + tokenEntry.startTime = tokenEntry.timeline[0].startTime; + tokenEntry.endTime = + tokenEntry.timeline[tokenEntry.timeline.length - 1].endTime; + wordEntry.startTime = tokenEntry.startTime; + wordEntry.endTime = tokenEntry.endTime; + + previousWordTokenEntry.text = previousWordTokenEntry.timeline + .map((phoneEntry) => phoneEntry.text) + .join(""); + previousWordTokenEntry.endTime = + previousWordTokenEntry.timeline[ + previousWordTokenEntry.timeline.length - 1 + ].endTime; + previousWordEntry.endTime = previousWordTokenEntry.endTime; + } + + // Build clause timeline + const clauseTimeline: Timeline = []; + + let clauseStartIndex = 0; + + for (const clauseEndIndex of clauseEndIndexes) { + const newClause: TimelineEntry = { + type: "clause", + text: "", + startTime: -1, + endTime: -1, + timeline: [], + }; + + for ( + let entryIndex = clauseStartIndex; + entryIndex <= clauseEndIndex && entryIndex < wordTimeline.length; + entryIndex++ + ) { + const wordEntry = wordTimeline[entryIndex]; + if (newClause.startTime === -1) { + newClause.startTime = wordEntry.startTime; + } - newClause.endTime = wordEntry.endTime; + newClause.endTime = wordEntry.endTime; - newClause.text += `${wordEntry.text} `; + newClause.text += `${wordEntry.text} `; - newClause.timeline!.push(wordEntry); - } + newClause.timeline!.push(wordEntry); + } - if (newClause.timeline!.length > 0) { - clauseTimeline.push(newClause); - clauseStartIndex = clauseEndIndex + 1; + if (newClause.timeline!.length > 0) { + clauseTimeline.push(newClause); + clauseStartIndex = clauseEndIndex + 1; + } } - } - return { rawAudio, timeline: clauseTimeline, events }; + return { rawAudio, timeline: clauseTimeline, events }; } async function setRate(rate: number) { - const { instance } = await getEspeakInstance(); + const { instance } = await getEspeakInstance(); - return instance.set_rate(rate); + return instance.set_rate(rate); } async function setPitch(pitch: number) { - const { instance } = await getEspeakInstance(); + const { instance } = await getEspeakInstance(); - return instance.set_pitch(pitch); + return instance.set_pitch(pitch); } async function setPitchRange(pitchRange: number) { - const { instance } = await getEspeakInstance(); + const { instance } = await getEspeakInstance(); - return instance.set_range(pitchRange); + return instance.set_range(pitchRange); } async function espeakSynthesize(text: string, espeakOptions: EspeakOptions) { - const logger = new Logger(); + const logger = new Logger(); - espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); + espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); - logger.start("Get eSpeak Emscripten instance"); + logger.start("Get eSpeak Emscripten instance"); - if (!espeakOptions.ssml) { - const { escape } = await import("html-escaper"); + if (!espeakOptions.ssml) { + const { escape } = await import("html-escaper"); - text = escape(text); - } + text = escape(text); + } - const { instance } = await getEspeakInstance(); + const { instance } = await getEspeakInstance(); - const sampleChunks: Float32Array[] = []; - const allEvents: EspeakEvent[] = []; + const sampleChunks: Float32Array[] = []; + const allEvents: EspeakEvent[] = []; - logger.start("Synthesize with eSpeak"); + logger.start("Synthesize with eSpeak"); - if (espeakOptions.useKlatt) { - await setVoice(`${espeakOptions.voice}+klatt6`); - } else { - await setVoice(espeakOptions.voice); - } + if (espeakOptions.useKlatt) { + await setVoice(`${espeakOptions.voice}+klatt6`); + } else { + await setVoice(espeakOptions.voice); + } - await setRate(espeakOptions.rate); - await setPitch(espeakOptions.pitch); - await setPitchRange(espeakOptions.pitchRange); + await setRate(espeakOptions.rate); + await setPitch(espeakOptions.pitch); + await setPitchRange(espeakOptions.pitchRange); - instance.synthesize(text, (samples: Int16Array, events: EspeakEvent[]) => { - if (samples && samples.length > 0) { - sampleChunks.push(int16PcmToFloat32(samples)); - } + instance.synthesize(text, (samples: Int16Array, events: EspeakEvent[]) => { + if (samples && samples.length > 0) { + sampleChunks.push(int16PcmToFloat32(samples)); + } - for (const event of events) { - if (event.type === "word") { - const textPosition = event.text_position - 1; - (event as any)["text"] = text.substring( - textPosition, - textPosition + event.word_length, - ); - } - } + for (const event of events) { + if (event.type === "word") { + const textPosition = event.text_position - 1; + (event as any)["text"] = text.substring( + textPosition, + textPosition + event.word_length + ); + } + } - allEvents.push(...events); - }); + allEvents.push(...events); + }); - const concatenatedSamples = concatFloat32Arrays(sampleChunks); + const concatenatedSamples = concatFloat32Arrays(sampleChunks); - const rawAudio: RawAudio = { - audioChannels: [concatenatedSamples], - sampleRate: 22050, - }; + const rawAudio: RawAudio = { + audioChannels: [concatenatedSamples], + sampleRate: 22050, + }; - logger.end(); + logger.end(); - return { rawAudio, events: allEvents }; + return { rawAudio, events: allEvents }; } async function textToPhonemes(text: string, voice: string, useIPA = true) { - await setVoice(voice); - const { instance, module } = await getEspeakInstance(); - const textPtr = instance.convert_to_phonemes(text, useIPA); + await setVoice(voice); + const { instance, module } = await getEspeakInstance(); + const textPtr = instance.convert_to_phonemes(text, useIPA); - const wasmMemory = new WasmMemoryManager(module); + const wasmMemory = new WasmMemoryManager(module); - const resultRef = wasmMemory.wrapNullTerminatedUtf8String(textPtr.ptr); - const result = resultRef.getValue(); + const resultRef = wasmMemory.wrapNullTerminatedUtf8String(textPtr.ptr); + const result = resultRef.getValue(); - wasmMemory.freeAll(); + wasmMemory.freeAll(); - return result; + return result; } async function preprocessAndSynthesize( - text: string, - language: string, - espeakOptions: EspeakOptions, - lexicons: Lexicon[] = [], + text: string, + language: string, + espeakOptions: EspeakOptions, + lexicons: Lexicon[] = [] ) { - const logger = new Logger(); + const logger = new Logger(); - espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); + espeakOptions = extendDeep(defaultEspeakOptions, espeakOptions); - await logger.startAsync("Tokenize and analyze text"); + await logger.startAsync("Tokenize and analyze text"); - let lowerCaseLanguageCode = language.toLowerCase(); + let lowerCaseLanguageCode = language.toLowerCase(); - if (lowerCaseLanguageCode === "en-gb") { - lowerCaseLanguageCode = "en-gb-x-rp"; - } + if (lowerCaseLanguageCode === "en-gb") { + lowerCaseLanguageCode = "en-gb-x-rp"; + } - let fragments: string[]; - let preprocessedFragments: string[]; - const phonemizedFragmentsSubstitutions = new Map(); + let fragments: string[]; + let preprocessedFragments: string[]; + const phonemizedFragmentsSubstitutions = new Map(); - fragments = []; - preprocessedFragments = []; + fragments = []; + preprocessedFragments = []; - let words = await splitToWords(text, language); + let words = await splitToWords(text, language); - // Merge repeating symbol words to a single word to work around eSpeak bug - const wordsWithMerges: string[] = []; + // Merge repeating symbol words to a single word to work around eSpeak bug + const wordsWithMerges: string[] = []; - for (let i = 0; i < words.length; i++) { - const currentWord = words[i]; - const previousWord = words[i - 1]; + for (let i = 0; i < words.length; i++) { + const currentWord = words[i]; + const previousWord = words[i - 1]; - if ( - i > 0 && - currentWord === previousWord && - !wordCharacterPattern.test(currentWord) - ) { - wordsWithMerges[wordsWithMerges.length - 1] += currentWord; - } else { - wordsWithMerges.push(currentWord); + if ( + i > 0 && + currentWord === previousWord && + !wordCharacterPattern.test(currentWord) + ) { + wordsWithMerges[wordsWithMerges.length - 1] += currentWord; + } else { + wordsWithMerges.push(currentWord); + } } - } - words = wordsWithMerges; + words = wordsWithMerges; - // Remove words containing only whitespace - words = words.filter((word) => word.trim() != ""); + // Remove words containing only whitespace + words = words.filter((word) => word.trim() != ""); - const { normalizedFragments, referenceFragments } = - getNormalizedFragmentsForSpeech(words, language); + const { normalizedFragments, referenceFragments } = + getNormalizedFragmentsForSpeech(words, language); - const simplifiedFragments = normalizedFragments.map((word) => - simplifyPunctuationCharacters(word).toLocaleLowerCase(), - ); + const simplifiedFragments = normalizedFragments.map((word) => + simplifyPunctuationCharacters(word).toLocaleLowerCase() + ); - if ([`'`].includes(simplifiedFragments[0])) { - normalizedFragments[0] = `()`; - } + if ([`'`].includes(simplifiedFragments[0])) { + normalizedFragments[0] = `()`; + } - for ( - let fragmentIndex = 0; - fragmentIndex < normalizedFragments.length; - fragmentIndex++ - ) { - const fragment = normalizedFragments[fragmentIndex]; + for ( + let fragmentIndex = 0; + fragmentIndex < normalizedFragments.length; + fragmentIndex++ + ) { + const fragment = normalizedFragments[fragmentIndex]; - const substitutionPhonemes = tryGetFirstLexiconSubstitution( - simplifiedFragments, - fragmentIndex, - lexicons, - lowerCaseLanguageCode, - ); + const substitutionPhonemes = tryGetFirstLexiconSubstitution( + simplifiedFragments, + fragmentIndex, + lexicons, + lowerCaseLanguageCode + ); - if (!substitutionPhonemes) { - continue; - } + if (!substitutionPhonemes) { + continue; + } - phonemizedFragmentsSubstitutions.set(fragmentIndex, substitutionPhonemes); - const referenceIPA = ( - await textToPhonemes(fragment, espeakOptions.voice, true) - ).replaceAll("_", " "); - const referenceKirshenbaum = ( - await textToPhonemes(fragment, espeakOptions.voice, false) - ).replaceAll("_", ""); + phonemizedFragmentsSubstitutions.set( + fragmentIndex, + substitutionPhonemes + ); + const referenceIPA = ( + await textToPhonemes(fragment, espeakOptions.voice, true) + ).replaceAll("_", " "); + const referenceKirshenbaum = ( + await textToPhonemes(fragment, espeakOptions.voice, false) + ).replaceAll("_", ""); - const kirshenbaumPhonemes = substitutionPhonemes - .map((phone) => ipaPhoneToKirshenbaum(phone)) - .join(""); + const kirshenbaumPhonemes = substitutionPhonemes + .map((phone) => ipaPhoneToKirshenbaum(phone)) + .join(""); - logger.logTitledMessage( - `\nLexicon substitution for '${fragment}'`, - `IPA: ${substitutionPhonemes.join(" ")} (original: ${referenceIPA}), Kirshenbaum: ${kirshenbaumPhonemes} (reference: ${referenceKirshenbaum})`, - ); + logger.logTitledMessage( + `\nLexicon substitution for '${fragment}'`, + `IPA: ${substitutionPhonemes.join(" ")} (original: ${referenceIPA}), Kirshenbaum: ${kirshenbaumPhonemes} (reference: ${referenceKirshenbaum})` + ); - const substitutionPhonemesFragment = ` [[${kirshenbaumPhonemes}]] `; + const substitutionPhonemesFragment = ` [[${kirshenbaumPhonemes}]] `; + + normalizedFragments[fragmentIndex] = substitutionPhonemesFragment; + } - normalizedFragments[fragmentIndex] = substitutionPhonemesFragment; - } + fragments = referenceFragments; + preprocessedFragments = normalizedFragments; - fragments = referenceFragments; - preprocessedFragments = normalizedFragments; + logger.start("Synthesize preprocessed fragments with eSpeak"); - logger.start("Synthesize preprocessed fragments with eSpeak"); + const { rawAudio: referenceSynthesizedAudio, timeline: referenceTimeline } = + await synthesizeFragments(preprocessedFragments, espeakOptions); - const { rawAudio: referenceSynthesizedAudio, timeline: referenceTimeline } = - await synthesizeFragments(preprocessedFragments, espeakOptions); + await logger.startAsync("Build phonemized tokens"); - await logger.startAsync("Build phonemized tokens"); + const phonemizedSentence: string[][][] = []; - const phonemizedSentence: string[][][] = []; + let wordIndex = 0; + for (const phraseEntry of referenceTimeline) { + const phrase: string[][] = []; - let wordIndex = 0; - for (const phraseEntry of referenceTimeline) { - const phrase: string[][] = []; + for (const wordEntry of phraseEntry.timeline!) { + wordEntry.text = fragments[wordIndex]; - for (const wordEntry of phraseEntry.timeline!) { - wordEntry.text = fragments[wordIndex]; + if (phonemizedFragmentsSubstitutions.has(wordIndex)) { + phrase.push(phonemizedFragmentsSubstitutions.get(wordIndex)!); + } else { + for (const tokenEntry of wordEntry.timeline!) { + const tokenPhonemes: string[] = []; - if (phonemizedFragmentsSubstitutions.has(wordIndex)) { - phrase.push(phonemizedFragmentsSubstitutions.get(wordIndex)!); - } else { - for (const tokenEntry of wordEntry.timeline!) { - const tokenPhonemes: string[] = []; + for (const phoneme of tokenEntry.timeline!) { + if (phoneme.text) { + tokenPhonemes.push(phoneme.text); + } + } - for (const phoneme of tokenEntry.timeline!) { - if (phoneme.text) { - tokenPhonemes.push(phoneme.text); + if (tokenPhonemes.length > 0) { + phrase.push(tokenPhonemes); + } + } } - } - if (tokenPhonemes.length > 0) { - phrase.push(tokenPhonemes); - } + wordIndex += 1; } - } - wordIndex += 1; - } - - if (phrase.length > 0) { - phonemizedSentence.push(phrase); + if (phrase.length > 0) { + phonemizedSentence.push(phrase); + } } - } - logger.log( - phonemizedSentence - .map((phrase) => phrase.map((word) => word.join(" ")).join(" | ")) - .join(" || "), - ); + logger.log( + phonemizedSentence + .map((phrase) => phrase.map((word) => word.join(" ")).join(" | ")) + .join(" || ") + ); - logger.end(); + logger.end(); - return { - referenceSynthesizedAudio, - referenceTimeline, - fragments, - preprocessedFragments, - phonemizedFragmentsSubstitutions, - phonemizedSentence, - }; + return { + referenceSynthesizedAudio, + referenceTimeline, + fragments, + preprocessedFragments, + phonemizedFragmentsSubstitutions, + phonemizedSentence, + }; } type Lexicon = { - [shortLanguageCode: string]: LexiconForLanguage; + [shortLanguageCode: string]: LexiconForLanguage; }; type LexiconForLanguage = { - [word: string]: LexiconEntry[]; + [word: string]: LexiconEntry[]; }; type LexiconEntry = { - pos?: string[]; - case?: LexiconWordCase; + pos?: string[]; + case?: LexiconWordCase; - pronunciation?: { - espeak?: LexiconPronunciationForLanguageCodes; - sapi?: LexiconPronunciationForLanguageCodes; - }; + pronunciation?: { + espeak?: LexiconPronunciationForLanguageCodes; + sapi?: LexiconPronunciationForLanguageCodes; + }; - precededBy?: string[]; - notPrecededBy?: string[]; + precededBy?: string[]; + notPrecededBy?: string[]; - succeededBy?: string[]; - notSucceededBy?: string[]; + succeededBy?: string[]; + notSucceededBy?: string[]; - example?: string; + example?: string; }; type LexiconWordCase = - | "any" - | "capitalized" - | "uppercase" - | "lowercase" - | "titlecase" - | "camelcase" - | "pascalcase"; + | "any" + | "capitalized" + | "uppercase" + | "lowercase" + | "titlecase" + | "camelcase" + | "pascalcase"; type LexiconPronunciationForLanguageCodes = { [languageCode: string]: string }; function getRawAudioDuration(rawAudio: RawAudio) { - if (rawAudio.audioChannels.length == 0 || rawAudio.sampleRate == 0) { - return 0; - } + if (rawAudio.audioChannels.length == 0 || rawAudio.sampleRate == 0) { + return 0; + } - return rawAudio.audioChannels[0].length / rawAudio.sampleRate; + return rawAudio.audioChannels[0].length / rawAudio.sampleRate; } function getEmptyRawAudio(channelCount: number, sampleRate: number) { - const audioChannels = []; + const audioChannels = []; - for (let c = 0; c < channelCount; c++) { - audioChannels.push(new Float32Array(0)); - } + for (let c = 0; c < channelCount; c++) { + audioChannels.push(new Float32Array(0)); + } - const result: RawAudio = { audioChannels, sampleRate }; + const result: RawAudio = { audioChannels, sampleRate }; - return result; + return result; } type RawAudio = { - audioChannels: Float32Array[]; - sampleRate: number; + audioChannels: Float32Array[]; + sampleRate: number; }; function concatTypedArrays(ArrayConstructor: any, arrays: any[]) { - let totalLength = 0; + let totalLength = 0; - for (const arr of arrays) { - totalLength += arr.length; - } + for (const arr of arrays) { + totalLength += arr.length; + } - const result = new ArrayConstructor(totalLength); + const result = new ArrayConstructor(totalLength); - let offset = 0; + let offset = 0; - for (const arr of arrays) { - result.set(arr, offset); - offset += arr.length; - } + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } - return result; + return result; } function writeToStderr(message: any) { - process.stderr.write(message); + process.stderr.write(message); } function printToStderr(message: any) { - if (typeof message == "string") { - writeToStderr(message); - } else { - writeToStderr(objToString(message)); - } + if (typeof message == "string") { + writeToStderr(message); + } else { + writeToStderr(objToString(message)); + } } function logToStderr(message: any) { - printToStderr(message); - writeToStderr("\n"); + printToStderr(message); + writeToStderr("\n"); } function objToString(obj: any) { - const formattedString = inspect(obj, { - showHidden: false, - depth: null, - colors: false, - maxArrayLength: null, - maxStringLength: null, - compact: 5, - }); + const formattedString = inspect(obj, { + showHidden: false, + depth: null, + colors: false, + maxArrayLength: null, + maxStringLength: null, + compact: 5, + }); - return formattedString; + return formattedString; } function roundToDigits(val: number, digits = 3) { - const multiplier = 10 ** digits; - return Math.round(val * multiplier) / multiplier; + const multiplier = 10 ** digits; + return Math.round(val * multiplier) / multiplier; } function yieldToEventLoop() { - return new Promise((resolve) => { - setImmediate(resolve); - }); + return new Promise((resolve) => { + setImmediate(resolve); + }); } async function resolveModuleScriptPath(moduleName: string) { - const { resolve } = await import("import-meta-resolve"); + const { resolve } = await import("import-meta-resolve"); - const scriptPath = resolve(moduleName, import.meta.url); + const scriptPath = resolve(moduleName, import.meta.url); - const { fileURLToPath } = await import("url"); + const { fileURLToPath } = await import("url"); - return fileURLToPath(scriptPath); + return fileURLToPath(scriptPath); } let currentActiveLogger: Logger | null = null; @@ -3919,230 +4008,233 @@ declare const chrome: any; declare const process: any; class Timer { - startTime = 0; - - constructor() { - this.restart(); - } + startTime = 0; - restart() { - this.startTime = Timer.currentTime; - } + constructor() { + this.restart(); + } - get elapsedTime(): number { - // Elapsed time (milliseconds) - return Timer.currentTime - this.startTime; - } + restart() { + this.startTime = Timer.currentTime; + } - get elapsedTimeSeconds(): number { - // Elapsed time (seconds) - return this.elapsedTime / 1000; - } + get elapsedTime(): number { + // Elapsed time (milliseconds) + return Timer.currentTime - this.startTime; + } - getElapsedTimeAndRestart(): number { - const elapsedTime = this.elapsedTime; - this.restart(); - return elapsedTime; - } + get elapsedTimeSeconds(): number { + // Elapsed time (seconds) + return this.elapsedTime / 1000; + } - logAndRestart(title: string, timePrecision = 3): number { - const elapsedTime = this.elapsedTime; + getElapsedTimeAndRestart(): number { + const elapsedTime = this.elapsedTime; + this.restart(); + return elapsedTime; + } - // - const message = `${title}: ${roundToDigits(elapsedTime, timePrecision)}ms`; - writeToStderr(message); - // + logAndRestart(title: string, timePrecision = 3): number { + const elapsedTime = this.elapsedTime; - this.restart(); + // + const message = `${title}: ${roundToDigits(elapsedTime, timePrecision)}ms`; + writeToStderr(message); + // - return elapsedTime; - } + this.restart(); - static get currentTime(): number { - if (!this.timestampFunc) { - this.createGlobalTimestampFunction(); + return elapsedTime; } - return this.timestampFunc(); - } + static get currentTime(): number { + if (!this.timestampFunc) { + this.createGlobalTimestampFunction(); + } + + return this.timestampFunc(); + } - static get microsecondTimestamp(): number { - return Math.floor(Timer.currentTime * 1000); - } + static get microsecondTimestamp(): number { + return Math.floor(Timer.currentTime * 1000); + } - private static createGlobalTimestampFunction() { - if (typeof process === "object" && typeof process.hrtime === "function") { - let baseTimestamp = 0; + private static createGlobalTimestampFunction() { + if ( + typeof process === "object" && + typeof process.hrtime === "function" + ) { + let baseTimestamp = 0; - this.timestampFunc = () => { - const nodeTimeStamp = process.hrtime(); - const millisecondTime = - nodeTimeStamp[0] * 1000 + nodeTimeStamp[1] / 1000000; + this.timestampFunc = () => { + const nodeTimeStamp = process.hrtime(); + const millisecondTime = + nodeTimeStamp[0] * 1000 + nodeTimeStamp[1] / 1000000; - return baseTimestamp + millisecondTime; - }; + return baseTimestamp + millisecondTime; + }; - baseTimestamp = Date.now() - this.timestampFunc(); - } else if (typeof chrome === "object" && chrome.Interval) { - const baseTimestamp = Date.now(); + baseTimestamp = Date.now() - this.timestampFunc(); + } else if (typeof chrome === "object" && chrome.Interval) { + const baseTimestamp = Date.now(); - const chromeIntervalObject = new chrome.Interval(); - chromeIntervalObject.start(); + const chromeIntervalObject = new chrome.Interval(); + chromeIntervalObject.start(); - this.timestampFunc = () => - baseTimestamp + chromeIntervalObject.microseconds() / 1000; - } else if (typeof performance === "object" && performance.now) { - const baseTimestamp = Date.now() - performance.now(); + this.timestampFunc = () => + baseTimestamp + chromeIntervalObject.microseconds() / 1000; + } else if (typeof performance === "object" && performance.now) { + const baseTimestamp = Date.now() - performance.now(); - this.timestampFunc = () => baseTimestamp + performance.now(); - } else if (Date.now) { - this.timestampFunc = () => Date.now(); - } else { - this.timestampFunc = () => new Date().getTime(); + this.timestampFunc = () => baseTimestamp + performance.now(); + } else if (Date.now) { + this.timestampFunc = () => Date.now(); + } else { + this.timestampFunc = () => new Date().getTime(); + } } - } - private static timestampFunc: () => number; + private static timestampFunc: () => number; } function logLevelToNumber(logLevel: LogLevel) { - return logLevels.indexOf(logLevel); + return logLevels.indexOf(logLevel); } function getLogLevel() { - return "info" as const; + return "info" as const; } function logLevelGreaterOrEqualTo(referenceLevel: LogLevel) { - return !logLevelSmallerThan(referenceLevel); + return !logLevelSmallerThan(referenceLevel); } function logLevelSmallerThan(referenceLevel: LogLevel) { - return logLevelToNumber(getLogLevel()) < logLevelToNumber(referenceLevel); + return logLevelToNumber(getLogLevel()) < logLevelToNumber(referenceLevel); } const logLevels = [ - "silent", - "output", - "error", - "warning", - "info", - "trace", + "silent", + "output", + "error", + "warning", + "info", + "trace", ] as const; type LogLevel = (typeof logLevels)[number]; class Logger { - private timer = new Timer(); - active = false; + private timer = new Timer(); + active = false; - start(title: string) { - this.startAsync(title, false); - } - - async startAsync(title: string, yieldBeforeStart = true) { - if (currentActiveLogger != null && currentActiveLogger != this) { - return; - } - - this.end(); - - if (yieldBeforeStart) { - await yieldToEventLoop(); + start(title: string) { + this.startAsync(title, false); } - if (logLevelGreaterOrEqualTo("info")) { - writeToStderr(`${title}.. `); - } + async startAsync(title: string, yieldBeforeStart = true) { + if (currentActiveLogger != null && currentActiveLogger != this) { + return; + } - this.setAsActiveLogger(); + this.end(); - this.timer.restart(); - } + if (yieldBeforeStart) { + await yieldToEventLoop(); + } - setAsActiveLogger() { - this.active = true; - currentActiveLogger = this; - } + if (logLevelGreaterOrEqualTo("info")) { + writeToStderr(`${title}.. `); + } - unsetAsActiveLogger() { - this.active = false; - currentActiveLogger = null; - } + this.setAsActiveLogger(); - end() { - if (this.active && currentActiveLogger == this) { - const elapsedTime = this.timer.elapsedTime; + this.timer.restart(); + } - if (logLevelGreaterOrEqualTo("info")) { - writeToStderr(`${elapsedTime.toFixed(1)}ms\n`); - } + setAsActiveLogger() { + this.active = true; + currentActiveLogger = this; + } - currentActiveLogger = null; + unsetAsActiveLogger() { + this.active = false; + currentActiveLogger = null; } - this.active = false; - } + end() { + if (this.active && currentActiveLogger == this) { + const elapsedTime = this.timer.elapsedTime; + + if (logLevelGreaterOrEqualTo("info")) { + writeToStderr(`${elapsedTime.toFixed(1)}ms\n`); + } - logTitledMessage(title: string, content: any, logLevel: LogLevel = "info") { - this.log(`${title}: ${content}`, logLevel); - } + currentActiveLogger = null; + } - log(message: any, logLevel: LogLevel = "info") { - if (logLevelSmallerThan(logLevel)) { - return; + this.active = false; } - if (currentActiveLogger == this || currentActiveLogger == null) { - logToStderr(message); + logTitledMessage(title: string, content: any, logLevel: LogLevel = "info") { + this.log(`${title}: ${content}`, logLevel); } - } - write(message: any, logLevel: LogLevel = "info") { - if (logLevelSmallerThan(logLevel)) { - return; + log(message: any, logLevel: LogLevel = "info") { + if (logLevelSmallerThan(logLevel)) { + return; + } + + if (currentActiveLogger == this || currentActiveLogger == null) { + logToStderr(message); + } } - if (currentActiveLogger == this || currentActiveLogger == null) { - writeToStderr(message); + write(message: any, logLevel: LogLevel = "info") { + if (logLevelSmallerThan(logLevel)) { + return; + } + + if (currentActiveLogger == this || currentActiveLogger == null) { + writeToStderr(message); + } } - } - getTimestamp() { - return Timer.currentTime; - } + getTimestamp() { + return Timer.currentTime; + } } type TimelineEntryType = - | "segment" - | "paragraph" - | "sentence" - | "clause" - | "phrase" - | "word" - | "token" - | "letter" - | "phone" - | "subphone"; + | "segment" + | "paragraph" + | "sentence" + | "clause" + | "phrase" + | "word" + | "token" + | "letter" + | "phone" + | "subphone"; type TimelineEntry = { - type: TimelineEntryType; + type: TimelineEntryType; - text: string; + text: string; - startTime: number; - endTime: number; + startTime: number; + endTime: number; - startOffsetUtf16?: number; - endOffsetUtf16?: number; + startOffsetUtf16?: number; + endOffsetUtf16?: number; - startOffsetUtf32?: number; - endOffsetUtf32?: number; + startOffsetUtf32?: number; + endOffsetUtf32?: number; - confidence?: number; + confidence?: number; - id?: number; + id?: number; - timeline?: Timeline; + timeline?: Timeline; }; type Timeline = TimelineEntry[]; @@ -4152,475 +4244,495 @@ const readFile = promisify(gracefulFS.readFile); const readdir = promisify(gracefulFS.readdir); async function readAndParseJsonFile(jsonFilePath: string, useJson5 = false) { - const fileContent = await readFile(jsonFilePath, { encoding: "utf-8" }); + const fileContent = await readFile(jsonFilePath, { encoding: "utf-8" }); - if (useJson5) { - const { default: JSON5 } = await import("json5"); + if (useJson5) { + const { default: JSON5 } = await import("json5"); - return JSON5.parse(fileContent); - } else { - return JSON.parse(fileContent); - } + return JSON5.parse(fileContent); + } else { + return JSON.parse(fileContent); + } } function getOnnxSessionOptions(options: OnnxSessionOptions) { - const onnxOptions: Onnx.InferenceSession.SessionOptions = { - executionProviders: ["cpu"], - logSeverityLevel: 3, - }; + const onnxOptions: Onnx.InferenceSession.SessionOptions = { + executionProviders: ["cpu"], + logSeverityLevel: 3, + }; - function dmlProviderAvailable() { - const platform = process.platform; - const arch = process.arch; + function dmlProviderAvailable() { + const platform = process.platform; + const arch = process.arch; - return platform === "win32" && arch === "x64"; - } + return platform === "win32" && arch === "x64"; + } - if (options) { - if (options.executionProviders != null) { - let executionProviders = options.executionProviders.filter((provider) => { - if (!provider) { - return false; - } + if (options) { + if (options.executionProviders != null) { + let executionProviders = options.executionProviders.filter( + (provider) => { + if (!provider) { + return false; + } - if (provider === "dml" && !dmlProviderAvailable()) { - return false; - } + if (provider === "dml" && !dmlProviderAvailable()) { + return false; + } - return true; - }); + return true; + } + ); - if (!executionProviders.includes("cpu")) { - executionProviders.push("cpu"); - } + if (!executionProviders.includes("cpu")) { + executionProviders.push("cpu"); + } - executionProviders = Array.from(new Set(executionProviders)); + executionProviders = Array.from(new Set(executionProviders)); - onnxOptions.executionProviders = executionProviders as any; - } else if (options.enableGPU === true && dmlProviderAvailable()) { - onnxOptions.executionProviders = ["dml", "cpu"]; - } + onnxOptions.executionProviders = executionProviders as any; + } else if (options.enableGPU === true && dmlProviderAvailable()) { + onnxOptions.executionProviders = ["dml", "cpu"]; + } - if (options.logSeverityLevel != null) { - onnxOptions.logSeverityLevel = options.logSeverityLevel; + if (options.logSeverityLevel != null) { + onnxOptions.logSeverityLevel = options.logSeverityLevel; + } } - } - return onnxOptions; + return onnxOptions; } interface OnnxSessionOptions { - enableGPU?: boolean; - executionProviders?: OnnxExecutionProvider[]; - logSeverityLevel?: 0 | 1 | 2 | 3 | 4; + enableGPU?: boolean; + executionProviders?: OnnxExecutionProvider[]; + logSeverityLevel?: 0 | 1 | 2 | 3 | 4; } type OnnxExecutionProvider = "cpu" | "dml" | "cuda"; interface EspeakOptions { - voice: string; - ssml: boolean; - rate: number; - pitch: number; - pitchRange: number; - useKlatt: boolean; - insertSeparators: boolean; + voice: string; + ssml: boolean; + rate: number; + pitch: number; + pitchRange: number; + useKlatt: boolean; + insertSeparators: boolean; } const defaultEspeakOptions: EspeakOptions = { - voice: "en-us", - ssml: false, - rate: 1.0, - pitch: 1.0, - pitchRange: 1.0, - useKlatt: false, - insertSeparators: false, + voice: "en-us", + ssml: false, + rate: 1.0, + pitch: 1.0, + pitchRange: 1.0, + useKlatt: false, + insertSeparators: false, }; const cachedInstanceLookup = new Map(); class VitsTTS { - session?: Onnx.InferenceSession; - metadata?: any; - phonemeMap?: Map; - - constructor( - public readonly voiceName: string, - public readonly modelPath: string, - public readonly executionProviders: OnnxExecutionProvider[], - ) {} - - static async synthesizeSentence( - text: string, - voiceName: string, - modelPath: string, - lengthScale: number, - speakerId: number, - lexicons: Lexicon[], - executionProviders: OnnxExecutionProvider[], - ) { - const cacheLookupKey = modelPath; - - let vitsTTS: VitsTTS | undefined = cachedInstanceLookup.get(cacheLookupKey); - - if (!vitsTTS) { - vitsTTS = new VitsTTS(voiceName, modelPath, executionProviders); - - cachedInstanceLookup.clear(); - cachedInstanceLookup.set(cacheLookupKey, vitsTTS); - } - - const result = await vitsTTS._synthesizeSentence( - text, - lengthScale, - speakerId, - lexicons, - ); - - return result; - } - - private async _synthesizeSentence( - sentence: string, - lengthScale: number, - speakerId = 0, - lexicons?: Lexicon[], - ) { - const logger = new Logger(); - - await this.initializeIfNeeded(); - - await logger.startAsync("Prepare for VITS synthesis"); - - const metadata = this.metadata; - const phonemeMap = this.phonemeMap!; - const espeakVoice = metadata.espeak.voice as string; - const languageCode = espeakVoice; - const outputSampleRate = metadata.audio.sample_rate; - const baseLengthScale = metadata.inference.length_scale || 1.0; - - lengthScale *= baseLengthScale; - - sentence = //simplifyPunctuationCharacters(sentence.trim()) - sentence - .replaceAll("(", ", ") - .replaceAll(")", ", ") - .replaceAll("—", ", "); - - logger.end(); - - const espeakOptions: EspeakOptions = { - ...defaultEspeakOptions, - voice: espeakVoice, - useKlatt: false, - }; - - const { - referenceSynthesizedAudio, - referenceTimeline, - fragments, - phonemizedFragmentsSubstitutions, - phonemizedSentence, - } = await preprocessAndSynthesize( - sentence, - languageCode, - espeakOptions, - lexicons, - ); + session?: Onnx.InferenceSession; + metadata?: any; + phonemeMap?: Map; + + constructor( + public readonly voiceName: string, + public readonly modelPath: string, + public readonly executionProviders: OnnxExecutionProvider[] + ) {} + + static async synthesizeSentence( + text: string, + voiceName: string, + modelPath: string, + lengthScale: number, + speakerId: number, + lexicons: Lexicon[], + executionProviders: OnnxExecutionProvider[] + ) { + const cacheLookupKey = modelPath; - if (phonemizedSentence.length == 0) { - logger.end(); + let vitsTTS: VitsTTS | undefined = + cachedInstanceLookup.get(cacheLookupKey); - return { - rawAudio: getEmptyRawAudio(1, outputSampleRate), - timeline: [], - referenceSynthesizedAudio: getEmptyRawAudio(1, outputSampleRate), - referenceTimeline: [] as Timeline, - }; - } + if (!vitsTTS) { + vitsTTS = new VitsTTS(voiceName, modelPath, executionProviders); - await logger.startAsync("Encode phonemes to identifiers"); + cachedInstanceLookup.clear(); + cachedInstanceLookup.set(cacheLookupKey, vitsTTS); + } - const clauseEndBreaker = ","; - let sentenceEndBreaker = "."; + const result = await vitsTTS._synthesizeSentence( + text, + lengthScale, + speakerId, + lexicons + ); - if (sentence.endsWith("?") || sentence.endsWith(`?"`)) { - sentenceEndBreaker = "?"; - } else if (sentence.endsWith("!") || sentence.endsWith(`!"`)) { - sentenceEndBreaker = "!"; + return result; } - const phonemeCharacterSeparatorId = phonemeMap.get("_")!; - const wordSeparatorId = phonemeMap.get(" ")!; - const startId = phonemeMap.get("^")!; - const endId = phonemeMap.get("$")!; + private async _synthesizeSentence( + sentence: string, + lengthScale: number, + speakerId = 0, + lexicons?: Lexicon[] + ) { + const logger = new Logger(); - const clauseEndBreakerId = phonemeMap.get(clauseEndBreaker)!; - const sentenceEndBreakerId = phonemeMap.get(sentenceEndBreaker)!; + await this.initializeIfNeeded(); - const ids: number[] = [...startId, ...phonemeCharacterSeparatorId]; + await logger.startAsync("Prepare for VITS synthesis"); - for ( - let clauseIndex = 0; - clauseIndex < phonemizedSentence.length; - clauseIndex++ - ) { - const clause = phonemizedSentence[clauseIndex]; + const metadata = this.metadata; + const phonemeMap = this.phonemeMap!; + const espeakVoice = metadata.espeak.voice as string; + const languageCode = espeakVoice; + const outputSampleRate = metadata.audio.sample_rate; + const baseLengthScale = metadata.inference.length_scale || 1.0; - for (const word of clause) { - for (const phoneme of word) { - for (const phonemeCharacter of phoneme) { - const id = phonemeMap.get(phonemeCharacter); + lengthScale *= baseLengthScale; - if (id == null) { - //logger.log(`No id found for subphoneme '${char}'`) - continue; - } + sentence = //simplifyPunctuationCharacters(sentence.trim()) + sentence + .replaceAll("(", ", ") + .replaceAll(")", ", ") + .replaceAll("—", ", "); - ids.push(...id, ...phonemeCharacterSeparatorId); - } - } + logger.end(); + + const espeakOptions: EspeakOptions = { + ...defaultEspeakOptions, + voice: espeakVoice, + useKlatt: false, + }; + + const { + referenceSynthesizedAudio, + referenceTimeline, + fragments, + phonemizedFragmentsSubstitutions, + phonemizedSentence, + } = await preprocessAndSynthesize( + sentence, + languageCode, + espeakOptions, + lexicons + ); - if (clauseIndex < phonemizedSentence.length - 1) { - ids.push(...wordSeparatorId, ...phonemeCharacterSeparatorId); + if (phonemizedSentence.length == 0) { + logger.end(); + + return { + rawAudio: getEmptyRawAudio(1, outputSampleRate), + timeline: [], + referenceSynthesizedAudio: getEmptyRawAudio( + 1, + outputSampleRate + ), + referenceTimeline: [] as Timeline, + }; } - } - if (clauseIndex < phonemizedSentence.length - 1) { - ids.push(...clauseEndBreakerId, ...phonemeCharacterSeparatorId); - } - } + await logger.startAsync("Encode phonemes to identifiers"); - ids.push(...sentenceEndBreakerId, ...phonemeCharacterSeparatorId, ...endId); + const clauseEndBreaker = ","; + let sentenceEndBreaker = "."; - //logger.log(ids) + if (sentence.endsWith("?") || sentence.endsWith(`?"`)) { + sentenceEndBreaker = "?"; + } else if (sentence.endsWith("!") || sentence.endsWith(`!"`)) { + sentenceEndBreaker = "!"; + } - const bigIntIds = new BigInt64Array(ids.map((id) => BigInt(id))); - const idLengths = new BigInt64Array([BigInt(bigIntIds.length)]); + const phonemeCharacterSeparatorId = phonemeMap.get("_")!; + const wordSeparatorId = phonemeMap.get(" ")!; + const startId = phonemeMap.get("^")!; + const endId = phonemeMap.get("$")!; + + const clauseEndBreakerId = phonemeMap.get(clauseEndBreaker)!; + const sentenceEndBreakerId = phonemeMap.get(sentenceEndBreaker)!; + + const ids: number[] = [...startId, ...phonemeCharacterSeparatorId]; + + for ( + let clauseIndex = 0; + clauseIndex < phonemizedSentence.length; + clauseIndex++ + ) { + const clause = phonemizedSentence[clauseIndex]; + + for (const word of clause) { + for (const phoneme of word) { + for (const phonemeCharacter of phoneme) { + const id = phonemeMap.get(phonemeCharacter); + + if (id == null) { + //logger.log(`No id found for subphoneme '${char}'`) + continue; + } + + ids.push(...id, ...phonemeCharacterSeparatorId); + } + } + + if (clauseIndex < phonemizedSentence.length - 1) { + ids.push( + ...wordSeparatorId, + ...phonemeCharacterSeparatorId + ); + } + } - await logger.startAsync("Generate audio using synthesis model"); + if (clauseIndex < phonemizedSentence.length - 1) { + ids.push(...clauseEndBreakerId, ...phonemeCharacterSeparatorId); + } + } - const inputTensor = new Onnx.Tensor("int64", bigIntIds, [ - 1, - bigIntIds.length, - ]); - const inputLengthsTensor = new Onnx.Tensor("int64", idLengths, [1]); - const scalesTensor = new Onnx.Tensor( - "float32", - [metadata.inference.noise_scale, lengthScale, metadata.inference.noise_w], - [3], - ); - const speakerIdTensor = new Onnx.Tensor( - "int64", - new BigInt64Array([BigInt(speakerId)]), - [1], - ); + ids.push( + ...sentenceEndBreakerId, + ...phonemeCharacterSeparatorId, + ...endId + ); - const modelInputs = { - input: inputTensor, - input_lengths: inputLengthsTensor, - scales: scalesTensor, - sid: speakerIdTensor, - }; + //logger.log(ids) + + const bigIntIds = new BigInt64Array(ids.map((id) => BigInt(id))); + const idLengths = new BigInt64Array([BigInt(bigIntIds.length)]); + + await logger.startAsync("Generate audio using synthesis model"); + + const inputTensor = new Onnx.Tensor("int64", bigIntIds, [ + 1, + bigIntIds.length, + ]); + const inputLengthsTensor = new Onnx.Tensor("int64", idLengths, [1]); + const scalesTensor = new Onnx.Tensor( + "float32", + [ + metadata.inference.noise_scale, + lengthScale, + metadata.inference.noise_w, + ], + [3] + ); + const speakerIdTensor = new Onnx.Tensor( + "int64", + new BigInt64Array([BigInt(speakerId)]), + [1] + ); - const modelResults = await this.session!.run(modelInputs); - const modelOutput = modelResults["output"]; + const modelInputs = { + input: inputTensor, + input_lengths: inputLengthsTensor, + scales: scalesTensor, + sid: speakerIdTensor, + }; - const modelOutputAudioSamples = modelOutput["data"] as Float32Array; + const modelResults = await this.session!.run(modelInputs); + const modelOutput = modelResults["output"]; - const synthesizedAudio: RawAudio = { - audioChannels: [modelOutputAudioSamples], - sampleRate: outputSampleRate, - }; + const modelOutputAudioSamples = modelOutput["data"] as Float32Array; - await logger.startAsync("Align with reference synthesized audio"); + const synthesizedAudio: RawAudio = { + audioChannels: [modelOutputAudioSamples], + sampleRate: outputSampleRate, + }; - const referenceWordTimeline = referenceTimeline.flatMap( - (clause) => clause.timeline!, - ); + await logger.startAsync("Align with reference synthesized audio"); - const dtwWindowDuration = Math.max( - 5, - Math.ceil(0.2 * getRawAudioDuration(synthesizedAudio)), - ); - const mappedTimeline = await alignUsingDtw( - synthesizedAudio, - referenceSynthesizedAudio, - referenceWordTimeline, - ["high"], - [dtwWindowDuration], - ); + const referenceWordTimeline = referenceTimeline.flatMap( + (clause) => clause.timeline! + ); - logger.end(); + const dtwWindowDuration = Math.max( + 5, + Math.ceil(0.2 * getRawAudioDuration(synthesizedAudio)) + ); + const mappedTimeline = await alignUsingDtw( + synthesizedAudio, + referenceSynthesizedAudio, + referenceWordTimeline, + ["high"], + [dtwWindowDuration] + ); - return { - rawAudio: synthesizedAudio, - timeline: mappedTimeline, - referenceSynthesizedAudio, - referenceTimeline, - }; - } + logger.end(); - async initializeIfNeeded() { - if (this.session) { - return; + return { + rawAudio: synthesizedAudio, + timeline: mappedTimeline, + referenceSynthesizedAudio, + referenceTimeline, + }; } - const logger = new Logger(); - await logger.startAsync("Initialize VITS ONNX synthesis model"); + async initializeIfNeeded() { + if (this.session) { + return; + } - const filesInModelPath = await readdir(this.modelPath); - const onnxModelFilename = filesInModelPath.find((filename) => - filename.endsWith(".onnx"), - ); + const logger = new Logger(); + await logger.startAsync("Initialize VITS ONNX synthesis model"); - if (!onnxModelFilename) { - throw new Error(`Couldn't file any ONNX model file in ${this.modelPath}`); - } + const filesInModelPath = await readdir(this.modelPath); + const onnxModelFilename = filesInModelPath.find((filename) => + filename.endsWith(".onnx") + ); + + if (!onnxModelFilename) { + throw new Error( + `Couldn't file any ONNX model file in ${this.modelPath}` + ); + } - const onnxModelFilepath = path.join(this.modelPath, onnxModelFilename); + const onnxModelFilepath = path.join(this.modelPath, onnxModelFilename); - const onnxSessionOptions = getOnnxSessionOptions({ - executionProviders: this.executionProviders, - }); + const onnxSessionOptions = getOnnxSessionOptions({ + executionProviders: this.executionProviders, + }); - this.session = await Onnx.InferenceSession.create( - onnxModelFilepath, - onnxSessionOptions, - ); - this.metadata = await readAndParseJsonFile(`${onnxModelFilepath}.json`); + this.session = await Onnx.InferenceSession.create( + onnxModelFilepath, + onnxSessionOptions + ); + this.metadata = await readAndParseJsonFile(`${onnxModelFilepath}.json`); - this.phonemeMap = new Map(); + this.phonemeMap = new Map(); - for (const key in this.metadata.phoneme_id_map) { - this.phonemeMap.set(key, this.metadata.phoneme_id_map[key]); - } + for (const key in this.metadata.phoneme_id_map) { + this.phonemeMap.set(key, this.metadata.phoneme_id_map[key]); + } - logger.end(); - } + logger.end(); + } } function getSamplePeakAmplitude(audioChannels: Float32Array[]) { - let maxAmplitude = 0.00001; + let maxAmplitude = 0.00001; - for (const channelSamples of audioChannels) { - for (const sample of channelSamples) { - maxAmplitude = Math.max(maxAmplitude, Math.abs(sample)); + for (const channelSamples of audioChannels) { + for (const sample of channelSamples) { + maxAmplitude = Math.max(maxAmplitude, Math.abs(sample)); + } } - } - return maxAmplitude; + return maxAmplitude; } function getSamplePeakDecibels(audioChannels: Float32Array[]) { - return gainFactorToDecibels(getSamplePeakAmplitude(audioChannels)); + return gainFactorToDecibels(getSamplePeakAmplitude(audioChannels)); } function gainFactorToDecibels(gainFactor: number) { - return gainFactor <= 0.00001 ? -100 : 20.0 * Math.log10(gainFactor); + return gainFactor <= 0.00001 ? -100 : 20.0 * Math.log10(gainFactor); } function splitToSentences(text: string, langCode: string): string[] { - const shortLangCode = getShortLanguageCode(langCode || ""); + const shortLangCode = getShortLanguageCode(langCode || ""); - return CldrSegmentation.sentenceSplit( - text, - CldrSegmentation.suppressions[shortLangCode], - ); + return CldrSegmentation.sentenceSplit( + text, + CldrSegmentation.suppressions[shortLangCode] + ); } type LanguageDetectionResults = LanguageDetectionResultsEntry[]; interface LanguageDetectionResultsEntry { - language: string; - languageName: string; - probability: number; + language: string; + languageName: string; + probability: number; } async function detectLanguage(text: string) { - const tinyldResults = detectAll(text); + const tinyldResults = detectAll(text); - const results: LanguageDetectionResults = tinyldResults.map((result) => ({ - language: result.lang, - languageName: languageCodeToName(result.lang), - probability: result.accuracy, - })); + const results: LanguageDetectionResults = tinyldResults.map((result) => ({ + language: result.lang, + languageName: languageCodeToName(result.lang), + probability: result.accuracy, + })); - return results; + return results; } interface TextLanguageDetectionOptions { - defaultLanguage?: string; - fallbackThresholdProbability?: number; + defaultLanguage?: string; + fallbackThresholdProbability?: number; } interface TextLanguageDetectionResult { - detectedLanguage: string; - detectedLanguageName: string; - detectedLanguageProbabilities: LanguageDetectionResults; + detectedLanguage: string; + detectedLanguageName: string; + detectedLanguageProbabilities: LanguageDetectionResults; } const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions = { - defaultLanguage: "en", - fallbackThresholdProbability: 0.05, + defaultLanguage: "en", + fallbackThresholdProbability: 0.05, }; async function detectTextLanguage( - input: string, - options: TextLanguageDetectionOptions, + input: string, + options: TextLanguageDetectionOptions ): Promise { - const logger = new Logger(); + const logger = new Logger(); - options = extendDeep(defaultTextLanguageDetectionOptions, options); + options = extendDeep(defaultTextLanguageDetectionOptions, options); - const defaultLanguage = options.defaultLanguage!; - const fallbackThresholdProbability = options.fallbackThresholdProbability!; + const defaultLanguage = options.defaultLanguage!; + const fallbackThresholdProbability = options.fallbackThresholdProbability!; - let detectedLanguageProbabilities: LanguageDetectionResults; + let detectedLanguageProbabilities: LanguageDetectionResults; - logger.start(`Initialize language detection module`); + logger.start(`Initialize language detection module`); - detectedLanguageProbabilities = await detectLanguage(input); + detectedLanguageProbabilities = await detectLanguage(input); - let detectedLanguage: string; + let detectedLanguage: string; - if ( - detectedLanguageProbabilities.length == 0 || - detectedLanguageProbabilities[0].probability < fallbackThresholdProbability - ) { - detectedLanguage = defaultLanguage; - } else { - detectedLanguage = detectedLanguageProbabilities[0].language; - } + if ( + detectedLanguageProbabilities.length == 0 || + detectedLanguageProbabilities[0].probability < + fallbackThresholdProbability + ) { + detectedLanguage = defaultLanguage; + } else { + detectedLanguage = detectedLanguageProbabilities[0].language; + } - logger.end(); + logger.end(); - return { - detectedLanguage, - detectedLanguageName: languageCodeToName(detectedLanguage), - detectedLanguageProbabilities, - }; + return { + detectedLanguage, + detectedLanguageName: languageCodeToName(detectedLanguage), + detectedLanguageProbabilities, + }; } function clampFloatSample(floatSample: number) { - if (floatSample < -1.0) { - return -1.0; - } else if (floatSample > 1.0) { - return 1.0; - } else { - return floatSample; - } + if (floatSample < -1.0) { + return -1.0; + } else if (floatSample > 1.0) { + return 1.0; + } else { + return floatSample; + } } function float32ToInt16Pcm(input: Float32Array) { - const output = new Int16Array(input.length); + const output = new Int16Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = clampFloatSample(input[i]); - output[i] = (sample < 0 ? sample * 32768 : sample * 32767) | 0; - } + for (let i = 0; i < input.length; i++) { + const sample = clampFloatSample(input[i]); + output[i] = (sample < 0 ? sample * 32768 : sample * 32767) | 0; + } - return output; + return output; } // Typed arrays to Buffer (little endian) conversions @@ -4632,1335 +4744,1362 @@ function float32ToInt16Pcm(input: Float32Array) { // int8 <-> bufferLE function int8ToBuffer(int8s: Int8Array) { - return Buffer.copyBytesFrom(int8s); + return Buffer.copyBytesFrom(int8s); } function int8ToBuffer_Slow(int8s: Int8Array) { - const buffer = Buffer.alloc(int8s.length); + const buffer = Buffer.alloc(int8s.length); - for (let i = 0; i < int8s.length; i++) { - buffer[i] = int8s[i] + 128; - } + for (let i = 0; i < int8s.length; i++) { + buffer[i] = int8s[i] + 128; + } - return buffer; + return buffer; } function bufferToInt8(buffer: Buffer) { - return new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + return new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); } function bufferToInt8_Slow(buffer: Buffer) { - const result = new Int8Array(buffer.length); + const result = new Int8Array(buffer.length); - for (let i = 0; i < result.length; i++) { - result[i] = buffer[i] - 128; - } + for (let i = 0; i < result.length; i++) { + result[i] = buffer[i] - 128; + } - return result; + return result; } // int16 <-> bufferLE function int16ToBufferLE(int16s: Int16Array) { - return Buffer.copyBytesFrom(int16s); + return Buffer.copyBytesFrom(int16s); } function int16ToBufferLE_Slow(int16s: Int16Array) { - const buffer = Buffer.alloc(int16s.length * 2); + const buffer = Buffer.alloc(int16s.length * 2); - for (let i = 0; i < int16s.length; i++) { - buffer.writeInt16LE(int16s[i], i * 2); - } + for (let i = 0; i < int16s.length; i++) { + buffer.writeInt16LE(int16s[i], i * 2); + } - return buffer; + return buffer; } function bufferLEToInt16(buffer: Buffer) { - return new Int16Array( - buffer.buffer, - buffer.byteOffset, - buffer.byteLength / 2, - ); + return new Int16Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength / 2 + ); } function bufferLEToInt16_Slow(buffer: Buffer) { - const result = new Int16Array(buffer.length / 2); + const result = new Int16Array(buffer.length / 2); - for (let i = 0; i < result.length; i++) { - result[i] = buffer.readInt16LE(i * 2); - } + for (let i = 0; i < result.length; i++) { + result[i] = buffer.readInt16LE(i * 2); + } - return result; + return result; } // int24 <-> bufferLE (uses int32 for storage) function int24ToBufferLE(int24s: Int32Array) { - const buffer = Buffer.alloc(int24s.length * 3); + const buffer = Buffer.alloc(int24s.length * 3); - for (let i = 0; i < int24s.length; i++) { - const val = int24s[i]; - const encodedVal = val < 0 ? val + 0x1000000 : val; + for (let i = 0; i < int24s.length; i++) { + const val = int24s[i]; + const encodedVal = val < 0 ? val + 0x1000000 : val; - buffer[i * 3 + 0] = (encodedVal >> 0) & 0xff; - buffer[i * 3 + 1] = (encodedVal >> 8) & 0xff; - buffer[i * 3 + 2] = (encodedVal >> 16) & 0xff; - } + buffer[i * 3 + 0] = (encodedVal >> 0) & 0xff; + buffer[i * 3 + 1] = (encodedVal >> 8) & 0xff; + buffer[i * 3 + 2] = (encodedVal >> 16) & 0xff; + } - return buffer; + return buffer; } function bufferLEToInt24(buffer: Buffer) { - const result = new Int32Array(buffer.length / 3); + const result = new Int32Array(buffer.length / 3); - for (let i = 0; i < result.length; i++) { - const b0 = buffer[i * 3 + 0]; - const b1 = buffer[i * 3 + 1]; - const b2 = buffer[i * 3 + 2]; + for (let i = 0; i < result.length; i++) { + const b0 = buffer[i * 3 + 0]; + const b1 = buffer[i * 3 + 1]; + const b2 = buffer[i * 3 + 2]; - const encodedVal = (b0 << 0) + (b1 << 8) + (b2 << 16); - result[i] = encodedVal > 0x800000 ? encodedVal - 0x1000000 : encodedVal; - } + const encodedVal = (b0 << 0) + (b1 << 8) + (b2 << 16); + result[i] = encodedVal > 0x800000 ? encodedVal - 0x1000000 : encodedVal; + } - return result; + return result; } // int32 <-> bufferLE function int32ToBufferLE(int32s: Int32Array) { - return Buffer.copyBytesFrom(int32s); + return Buffer.copyBytesFrom(int32s); } function int32ToBufferLE_Slow(int32s: Int32Array) { - const buffer = Buffer.alloc(int32s.length * 4); + const buffer = Buffer.alloc(int32s.length * 4); - for (let i = 0; i < int32s.length; i++) { - buffer.writeInt32LE(int32s[i], i * 4); - } + for (let i = 0; i < int32s.length; i++) { + buffer.writeInt32LE(int32s[i], i * 4); + } - return buffer; + return buffer; } function bufferLEToInt32(buffer: Buffer) { - return new Int32Array( - buffer.buffer, - buffer.byteOffset, - buffer.byteLength / 4, - ); + return new Int32Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength / 4 + ); } function bufferLEToInt32_Slow(buffer: Buffer) { - const result = new Int32Array(buffer.length / 4); + const result = new Int32Array(buffer.length / 4); - for (let i = 0; i < result.length; i++) { - result[i] = buffer.readInt32LE(i * 4); - } + for (let i = 0; i < result.length; i++) { + result[i] = buffer.readInt32LE(i * 4); + } - return result; + return result; } // float32 <-> bufferLE function float32ToBufferLE(float32s: Float32Array) { - return Buffer.copyBytesFrom(float32s); + return Buffer.copyBytesFrom(float32s); } function float32ToBufferLE_Slow(float32s: Float32Array) { - const buffer = Buffer.alloc(float32s.length * 4); + const buffer = Buffer.alloc(float32s.length * 4); - for (let i = 0; i < float32s.length; i++) { - buffer.writeFloatLE(float32s[i], i * 4); - } + for (let i = 0; i < float32s.length; i++) { + buffer.writeFloatLE(float32s[i], i * 4); + } - return buffer; + return buffer; } function bufferLEToFloat32(buffer: Buffer) { - return new Float32Array( - buffer.buffer, - buffer.byteOffset, - buffer.byteLength / 4, - ); + return new Float32Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength / 4 + ); } function bufferLEToFloat32_Slow(buffer: Buffer) { - const result = new Float32Array(buffer.length / 4); + const result = new Float32Array(buffer.length / 4); - for (let i = 0; i < result.length; i++) { - result[i] = buffer.readFloatLE(i * 4); - } + for (let i = 0; i < result.length; i++) { + result[i] = buffer.readFloatLE(i * 4); + } - return result; + return result; } // float64 <-> bufferLE function float64ToBufferLE(float64s: Float64Array) { - return Buffer.copyBytesFrom(float64s); + return Buffer.copyBytesFrom(float64s); } function float64ToBufferLE_Slow(float64s: Float64Array) { - const buffer = Buffer.alloc(float64s.length * 8); + const buffer = Buffer.alloc(float64s.length * 8); - for (let i = 0; i < float64s.length; i++) { - buffer.writeDoubleLE(float64s[i], i * 8); - } + for (let i = 0; i < float64s.length; i++) { + buffer.writeDoubleLE(float64s[i], i * 8); + } - return buffer; + return buffer; } function bufferLEToFloat64(buffer: Buffer) { - return new Float64Array( - buffer.buffer, - buffer.byteOffset, - buffer.byteLength / 8, - ); + return new Float64Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength / 8 + ); } function bufferLEToFloat64_Slow(buffer: Buffer) { - const result = new Float64Array(buffer.length / 8); + const result = new Float64Array(buffer.length / 8); - for (let i = 0; i < result.length; i++) { - result[i] = buffer.readDoubleLE(i * 8); - } + for (let i = 0; i < result.length; i++) { + result[i] = buffer.readDoubleLE(i * 8); + } - return result; + return result; } // float64 <-> float32 function float64Tofloat32(float64s: Float64Array) { - return Float32Array.from(float64s); + return Float32Array.from(float64s); } function float32Tofloat64(float32s: Float32Array) { - return Float64Array.from(float32s); + return Float64Array.from(float32s); } function interleaveChannels(channels: Float32Array[]) { - const channelCount = channels.length; + const channelCount = channels.length; - if (channelCount === 0) { - throw new Error("Empty channel array received"); - } + if (channelCount === 0) { + throw new Error("Empty channel array received"); + } - if (channelCount === 1) { - return channels[0]; - } + if (channelCount === 1) { + return channels[0]; + } - const sampleCount = channels[0].length; - const result = new Float32Array(sampleCount * channelCount); + const sampleCount = channels[0].length; + const result = new Float32Array(sampleCount * channelCount); - let writeIndex = 0; + let writeIndex = 0; - for (let i = 0; i < sampleCount; i++) { - for (let c = 0; c < channelCount; c++) { - result[writeIndex] = channels[c][i]; - writeIndex += 1; + for (let i = 0; i < sampleCount; i++) { + for (let c = 0; c < channelCount; c++) { + result[writeIndex] = channels[c][i]; + writeIndex += 1; + } } - } - return result; + return result; } function float32ToInt8Pcm(input: Float32Array) { - const output = new Int8Array(input.length); + const output = new Int8Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = clampFloatSample(input[i]); - output[i] = (sample < 0 ? sample * 128 : sample * 127) | 0; - } + for (let i = 0; i < input.length; i++) { + const sample = clampFloatSample(input[i]); + output[i] = (sample < 0 ? sample * 128 : sample * 127) | 0; + } - return output; + return output; } function float32ToInt24Pcm(input: Float32Array) { - const output = new Int32Array(input.length); + const output = new Int32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = clampFloatSample(input[i]); - output[i] = (sample < 0 ? sample * 8388608 : sample * 8388607) | 0; - } + for (let i = 0; i < input.length; i++) { + const sample = clampFloatSample(input[i]); + output[i] = (sample < 0 ? sample * 8388608 : sample * 8388607) | 0; + } - return output; + return output; } function float32ToInt32Pcm(input: Float32Array) { - const output = new Int32Array(input.length); + const output = new Int32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = clampFloatSample(input[i]); - output[i] = (sample < 0 ? sample * 2147483648 : sample * 2147483647) | 0; - } + for (let i = 0; i < input.length; i++) { + const sample = clampFloatSample(input[i]); + output[i] = + (sample < 0 ? sample * 2147483648 : sample * 2147483647) | 0; + } - return output; + return output; } function encodeToAudioBuffer( - audioChannels: Float32Array[], - targetBitDepth: BitDepth = 16, - targetSampleFormat: SampleFormat = SampleFormat.PCM, + audioChannels: Float32Array[], + targetBitDepth: BitDepth = 16, + targetSampleFormat: SampleFormat = SampleFormat.PCM ) { - const interleavedChannels = interleaveChannels(audioChannels); - - audioChannels = []; // Zero the array references to allow the GC to free up memory, if possible - - if (targetSampleFormat === SampleFormat.PCM) { - if (targetBitDepth === 8) { - return int8ToBuffer(float32ToInt8Pcm(interleavedChannels)); - } else if (targetBitDepth === 16) { - return int16ToBufferLE(float32ToInt16Pcm(interleavedChannels)); - } else if (targetBitDepth === 24) { - return int24ToBufferLE(float32ToInt24Pcm(interleavedChannels)); - } else if (targetBitDepth === 32) { - return int32ToBufferLE(float32ToInt32Pcm(interleavedChannels)); - } else { - throw new Error(`Unsupported PCM bit depth: ${targetBitDepth}`); - } - } else if (targetSampleFormat === SampleFormat.Float) { - if (targetBitDepth === 32) { - return float32ToBufferLE(interleavedChannels); - } else if (targetBitDepth === 64) { - return float64ToBufferLE(float32Tofloat64(interleavedChannels)); - } else { - throw new Error(`Unsupported float bit depth: ${targetBitDepth}`); - } - } else if (targetSampleFormat === SampleFormat.Alaw) { - if (targetBitDepth === 8) { - return Buffer.from( - AlawMulaw.alaw.encode(float32ToInt16Pcm(interleavedChannels)), - ); - } else { - throw new Error(`Unsupported alaw bit depth: ${targetBitDepth}`); - } - } else if (targetSampleFormat === SampleFormat.Mulaw) { - if (targetBitDepth === 8) { - return Buffer.from( - AlawMulaw.mulaw.encode(float32ToInt16Pcm(interleavedChannels)), - ); + const interleavedChannels = interleaveChannels(audioChannels); + + audioChannels = []; // Zero the array references to allow the GC to free up memory, if possible + + if (targetSampleFormat === SampleFormat.PCM) { + if (targetBitDepth === 8) { + return int8ToBuffer(float32ToInt8Pcm(interleavedChannels)); + } else if (targetBitDepth === 16) { + return int16ToBufferLE(float32ToInt16Pcm(interleavedChannels)); + } else if (targetBitDepth === 24) { + return int24ToBufferLE(float32ToInt24Pcm(interleavedChannels)); + } else if (targetBitDepth === 32) { + return int32ToBufferLE(float32ToInt32Pcm(interleavedChannels)); + } else { + throw new Error(`Unsupported PCM bit depth: ${targetBitDepth}`); + } + } else if (targetSampleFormat === SampleFormat.Float) { + if (targetBitDepth === 32) { + return float32ToBufferLE(interleavedChannels); + } else if (targetBitDepth === 64) { + return float64ToBufferLE(float32Tofloat64(interleavedChannels)); + } else { + throw new Error(`Unsupported float bit depth: ${targetBitDepth}`); + } + } else if (targetSampleFormat === SampleFormat.Alaw) { + if (targetBitDepth === 8) { + return Buffer.from( + AlawMulaw.alaw.encode(float32ToInt16Pcm(interleavedChannels)) + ); + } else { + throw new Error(`Unsupported alaw bit depth: ${targetBitDepth}`); + } + } else if (targetSampleFormat === SampleFormat.Mulaw) { + if (targetBitDepth === 8) { + return Buffer.from( + AlawMulaw.mulaw.encode(float32ToInt16Pcm(interleavedChannels)) + ); + } else { + throw new Error(`Unsupported mulaw bit depth: ${targetBitDepth}`); + } } else { - throw new Error(`Unsupported mulaw bit depth: ${targetBitDepth}`); + throw new Error(`Unsupported audio format: ${targetSampleFormat}`); } - } else { - throw new Error(`Unsupported audio format: ${targetSampleFormat}`); - } } class WaveFormat { - // 24 bytes total for PCM, 26 for float - sampleFormat: SampleFormat; // 2 bytes LE - channelCount: number; // 2 bytes LE - sampleRate: number; // 4 bytes LE - get byteRate() { - return this.sampleRate * this.bytesPerSample * this.channelCount; - } // 4 bytes LE - get blockAlign() { - return this.bytesPerSample * this.channelCount; - } // 2 bytes LE - bitDepth: BitDepth; // 2 bytes LE - - speakerPositionMask: number; // 4 bytes LE - get guid() { - return sampleFormatToGuid[this.sampleFormat]; - } // 16 bytes BE - - // helpers: - get bytesPerSample() { - return this.bitDepth / 8; - } - - constructor( - channelCount: number, - sampleRate: number, - bitDepth: BitDepth, - sampleFormat: SampleFormat, - speakerPositionMask = 0, - ) { - this.sampleFormat = sampleFormat; - this.channelCount = channelCount; - this.sampleRate = sampleRate; - this.bitDepth = bitDepth; - - this.speakerPositionMask = speakerPositionMask; - } - - serialize(useExtensibleFormat: boolean) { - let sampleFormatId = this.sampleFormat; - - if (useExtensibleFormat) { - sampleFormatId = 65534 as number; - } - - const serializedSize = sampleFormatToSerializedSize[sampleFormatId]; - - const result = Buffer.alloc(serializedSize); - - result.write("fmt ", 0, "ascii"); // + 4 - result.writeUint32LE(serializedSize - 8, 4); // + 4 - - result.writeUint16LE(sampleFormatId, 8); // + 2 - result.writeUint16LE(this.channelCount, 10); // + 2 - result.writeUint32LE(this.sampleRate, 12); // + 4 - result.writeUint32LE(this.byteRate, 16); // + 4 - result.writeUint16LE(this.blockAlign, 20); // + 2 - result.writeUint16LE(this.bitDepth, 22); // + 2 - - if (useExtensibleFormat) { - result.writeUint16LE(serializedSize - 26, 24); // + 2 (extension size) - result.writeUint16LE(this.bitDepth, 26); // + 2 (valid bits per sample) - result.writeUint32LE(this.speakerPositionMask, 28); // + 2 (speaker position mask) - - if ( - this.sampleFormat == SampleFormat.PCM || - this.sampleFormat == SampleFormat.Float - ) { - result.set(Buffer.from(this.guid, "hex"), 32); - } else { - throw new Error( - `Extensible format is not supported for sample format ${this.sampleFormat}`, - ); - } + // 24 bytes total for PCM, 26 for float + sampleFormat: SampleFormat; // 2 bytes LE + channelCount: number; // 2 bytes LE + sampleRate: number; // 4 bytes LE + get byteRate() { + return this.sampleRate * this.bytesPerSample * this.channelCount; + } // 4 bytes LE + get blockAlign() { + return this.bytesPerSample * this.channelCount; + } // 2 bytes LE + bitDepth: BitDepth; // 2 bytes LE + + speakerPositionMask: number; // 4 bytes LE + get guid() { + return sampleFormatToGuid[this.sampleFormat]; + } // 16 bytes BE + + // helpers: + get bytesPerSample() { + return this.bitDepth / 8; + } + + constructor( + channelCount: number, + sampleRate: number, + bitDepth: BitDepth, + sampleFormat: SampleFormat, + speakerPositionMask = 0 + ) { + this.sampleFormat = sampleFormat; + this.channelCount = channelCount; + this.sampleRate = sampleRate; + this.bitDepth = bitDepth; + + this.speakerPositionMask = speakerPositionMask; } - return result; - } - - static deserializeFrom(formatChunkBody: Buffer) { - // chunkBody should not include the first 8 bytes - let sampleFormat = formatChunkBody.readUint16LE(0); // + 2 - const channelCount = formatChunkBody.readUint16LE(2); // + 2 - const sampleRate = formatChunkBody.readUint32LE(4); // + 4 - const bitDepth = formatChunkBody.readUint16LE(14); - let speakerPositionMask = 0; - - if (sampleFormat == 65534) { - if (formatChunkBody.length < 40) { - throw new Error( - `Format subchunk specifies a format id of 65534 (extensible) but its body size is ${formatChunkBody.length} bytes, which is smaller than the minimum expected of 40 bytes`, - ); - } + serialize(useExtensibleFormat: boolean) { + let sampleFormatId = this.sampleFormat; + + if (useExtensibleFormat) { + sampleFormatId = 65534 as number; + } - speakerPositionMask = formatChunkBody.readUint16LE(20); + const serializedSize = sampleFormatToSerializedSize[sampleFormatId]; - const guid = formatChunkBody.subarray(24, 40).toString("hex"); + const result = Buffer.alloc(serializedSize); - if (guid == sampleFormatToGuid[SampleFormat.PCM]) { - sampleFormat = SampleFormat.PCM; - } else if (guid == sampleFormatToGuid[SampleFormat.Float]) { - sampleFormat = SampleFormat.Float; - } else { - throw new Error( - `Unsupported format GUID in extended format subchunk: ${guid}`, - ); - } + result.write("fmt ", 0, "ascii"); // + 4 + result.writeUint32LE(serializedSize - 8, 4); // + 4 + + result.writeUint16LE(sampleFormatId, 8); // + 2 + result.writeUint16LE(this.channelCount, 10); // + 2 + result.writeUint32LE(this.sampleRate, 12); // + 4 + result.writeUint32LE(this.byteRate, 16); // + 4 + result.writeUint16LE(this.blockAlign, 20); // + 2 + result.writeUint16LE(this.bitDepth, 22); // + 2 + + if (useExtensibleFormat) { + result.writeUint16LE(serializedSize - 26, 24); // + 2 (extension size) + result.writeUint16LE(this.bitDepth, 26); // + 2 (valid bits per sample) + result.writeUint32LE(this.speakerPositionMask, 28); // + 2 (speaker position mask) + + if ( + this.sampleFormat == SampleFormat.PCM || + this.sampleFormat == SampleFormat.Float + ) { + result.set(Buffer.from(this.guid, "hex"), 32); + } else { + throw new Error( + `Extensible format is not supported for sample format ${this.sampleFormat}` + ); + } + } + + return result; } - if (sampleFormat == SampleFormat.PCM) { - if (bitDepth != 8 && bitDepth != 16 && bitDepth != 24 && bitDepth != 32) { - throw new Error( - `PCM audio has a bit depth of ${bitDepth}, which is not supported`, - ); - } - } else if (sampleFormat == SampleFormat.Float) { - if (bitDepth != 32 && bitDepth != 64) { - throw new Error( - `IEEE float audio has a bit depth of ${bitDepth}, which is not supported`, - ); - } - } else if (sampleFormat == SampleFormat.Alaw) { - if (bitDepth != 8) { - throw new Error( - `Alaw audio has a bit depth of ${bitDepth}, which is not supported`, - ); - } - } else if (sampleFormat == SampleFormat.Mulaw) { - if (bitDepth != 8) { - throw new Error( - `Mulaw audio has a bit depth of ${bitDepth}, which is not supported`, + static deserializeFrom(formatChunkBody: Buffer) { + // chunkBody should not include the first 8 bytes + let sampleFormat = formatChunkBody.readUint16LE(0); // + 2 + const channelCount = formatChunkBody.readUint16LE(2); // + 2 + const sampleRate = formatChunkBody.readUint32LE(4); // + 4 + const bitDepth = formatChunkBody.readUint16LE(14); + let speakerPositionMask = 0; + + if (sampleFormat == 65534) { + if (formatChunkBody.length < 40) { + throw new Error( + `Format subchunk specifies a format id of 65534 (extensible) but its body size is ${formatChunkBody.length} bytes, which is smaller than the minimum expected of 40 bytes` + ); + } + + speakerPositionMask = formatChunkBody.readUint16LE(20); + + const guid = formatChunkBody.subarray(24, 40).toString("hex"); + + if (guid == sampleFormatToGuid[SampleFormat.PCM]) { + sampleFormat = SampleFormat.PCM; + } else if (guid == sampleFormatToGuid[SampleFormat.Float]) { + sampleFormat = SampleFormat.Float; + } else { + throw new Error( + `Unsupported format GUID in extended format subchunk: ${guid}` + ); + } + } + + if (sampleFormat == SampleFormat.PCM) { + if ( + bitDepth != 8 && + bitDepth != 16 && + bitDepth != 24 && + bitDepth != 32 + ) { + throw new Error( + `PCM audio has a bit depth of ${bitDepth}, which is not supported` + ); + } + } else if (sampleFormat == SampleFormat.Float) { + if (bitDepth != 32 && bitDepth != 64) { + throw new Error( + `IEEE float audio has a bit depth of ${bitDepth}, which is not supported` + ); + } + } else if (sampleFormat == SampleFormat.Alaw) { + if (bitDepth != 8) { + throw new Error( + `Alaw audio has a bit depth of ${bitDepth}, which is not supported` + ); + } + } else if (sampleFormat == SampleFormat.Mulaw) { + if (bitDepth != 8) { + throw new Error( + `Mulaw audio has a bit depth of ${bitDepth}, which is not supported` + ); + } + } else { + throw new Error( + `Wave audio format id ${sampleFormat} is not supported` + ); + } + + return new WaveFormat( + channelCount, + sampleRate, + bitDepth, + sampleFormat, + speakerPositionMask ); - } - } else { - throw new Error(`Wave audio format id ${sampleFormat} is not supported`); } - - return new WaveFormat( - channelCount, - sampleRate, - bitDepth, - sampleFormat, - speakerPositionMask, - ); - } } function encodeWave( - rawAudio: RawAudio, - bitDepth: BitDepth = 16, - sampleFormat: SampleFormat = SampleFormat.PCM, - speakerPositionMask = 0, + rawAudio: RawAudio, + bitDepth: BitDepth = 16, + sampleFormat: SampleFormat = SampleFormat.PCM, + speakerPositionMask = 0 ) { - const audioChannels = rawAudio.audioChannels; - const sampleRate = rawAudio.sampleRate; - - const audioBuffer = encodeToAudioBuffer( - audioChannels, - bitDepth, - sampleFormat, - ); - const audioDataLength = audioBuffer.length; - - const shouldUseExtensibleFormat = bitDepth > 16 || audioChannels.length > 2; - - const formatSubChunk = new WaveFormat( - audioChannels.length, - sampleRate, - bitDepth, - sampleFormat, - speakerPositionMask, - ); - const formatSubChunkBuffer = formatSubChunk.serialize( - shouldUseExtensibleFormat, - ); - - const dataSubChunkBuffer = Buffer.alloc(4 + 4 + audioDataLength); - dataSubChunkBuffer.write("data", 0, "ascii"); - const dataChunkLength = Math.min(audioDataLength, 4294967295); // Ensure large data chunk length is clipped to max - dataSubChunkBuffer.writeUint32LE(dataChunkLength, 4); - dataSubChunkBuffer.set(audioBuffer, 8); - - const riffChunkHeaderBuffer = Buffer.alloc(12); - riffChunkHeaderBuffer.write("RIFF", 0, "ascii"); - const riffChunkLength = Math.min( - 4 + formatSubChunkBuffer.length + dataSubChunkBuffer.length, - 4294967295, - ); // Ensure large RIFF chunk length is clipped to max - riffChunkHeaderBuffer.writeUint32LE(riffChunkLength, 4); - riffChunkHeaderBuffer.write("WAVE", 8, "ascii"); - - return Buffer.concat([ - riffChunkHeaderBuffer, - formatSubChunkBuffer, - dataSubChunkBuffer, - ]); + const audioChannels = rawAudio.audioChannels; + const sampleRate = rawAudio.sampleRate; + + const audioBuffer = encodeToAudioBuffer( + audioChannels, + bitDepth, + sampleFormat + ); + const audioDataLength = audioBuffer.length; + + const shouldUseExtensibleFormat = bitDepth > 16 || audioChannels.length > 2; + + const formatSubChunk = new WaveFormat( + audioChannels.length, + sampleRate, + bitDepth, + sampleFormat, + speakerPositionMask + ); + const formatSubChunkBuffer = formatSubChunk.serialize( + shouldUseExtensibleFormat + ); + + const dataSubChunkBuffer = Buffer.alloc(4 + 4 + audioDataLength); + dataSubChunkBuffer.write("data", 0, "ascii"); + const dataChunkLength = Math.min(audioDataLength, 4294967295); // Ensure large data chunk length is clipped to max + dataSubChunkBuffer.writeUint32LE(dataChunkLength, 4); + dataSubChunkBuffer.set(audioBuffer, 8); + + const riffChunkHeaderBuffer = Buffer.alloc(12); + riffChunkHeaderBuffer.write("RIFF", 0, "ascii"); + const riffChunkLength = Math.min( + 4 + formatSubChunkBuffer.length + dataSubChunkBuffer.length, + 4294967295 + ); // Ensure large RIFF chunk length is clipped to max + riffChunkHeaderBuffer.writeUint32LE(riffChunkLength, 4); + riffChunkHeaderBuffer.write("WAVE", 8, "ascii"); + + return Buffer.concat([ + riffChunkHeaderBuffer, + formatSubChunkBuffer, + dataSubChunkBuffer, + ]); } enum SampleFormat { - PCM = 1, - Float = 3, - Alaw = 6, - Mulaw = 7, + PCM = 1, + Float = 3, + Alaw = 6, + Mulaw = 7, } type BitDepth = 8 | 16 | 24 | 32 | 64; const sampleFormatToSerializedSize = { - [SampleFormat.PCM]: 24, - [SampleFormat.Float]: 26, - [SampleFormat.Alaw]: 26, - [SampleFormat.Mulaw]: 26, - 65534: 48, + [SampleFormat.PCM]: 24, + [SampleFormat.Float]: 26, + [SampleFormat.Alaw]: 26, + [SampleFormat.Mulaw]: 26, + 65534: 48, }; const sampleFormatToGuid = { - [SampleFormat.PCM]: "0100000000001000800000aa00389b71", - [SampleFormat.Float]: "0300000000001000800000aa00389b71", - [SampleFormat.Alaw]: "", - [SampleFormat.Mulaw]: "", + [SampleFormat.PCM]: "0100000000001000800000aa00389b71", + [SampleFormat.Float]: "0300000000001000800000aa00389b71", + [SampleFormat.Alaw]: "", + [SampleFormat.Mulaw]: "", }; function encodeRawAudioToWave( - rawAudio: RawAudio, - bitDepth: BitDepth = 16, - sampleFormat: SampleFormat = SampleFormat.PCM, - speakerPositionMask = 0, + rawAudio: RawAudio, + bitDepth: BitDepth = 16, + sampleFormat: SampleFormat = SampleFormat.PCM, + speakerPositionMask = 0 ) { - return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask); + return encodeWave(rawAudio, bitDepth, sampleFormat, speakerPositionMask); } function languageCodeToName(languageCode: string) { - const languageNames = new Intl.DisplayNames(["en"], { type: "language" }); + const languageNames = new Intl.DisplayNames(["en"], { type: "language" }); - let translatedLanguageName: string | undefined; + let translatedLanguageName: string | undefined; - try { - translatedLanguageName = languageNames.of(languageCode); - } catch (e) {} + try { + translatedLanguageName = languageNames.of(languageCode); + } catch (e) {} - return translatedLanguageName || "Unknown"; + return translatedLanguageName || "Unknown"; } function formatLanguageCodeWithName(languageCode: string, styleId: 1 | 2 = 1) { - if (styleId == 1) { - return `${languageCodeToName(languageCode)} (${languageCode})`; - } else { - return `${languageCode}, ${languageCodeToName(languageCode)}`; - } + if (styleId == 1) { + return `${languageCodeToName(languageCode)} (${languageCode})`; + } else { + return `${languageCode}, ${languageCodeToName(languageCode)}`; + } } const cancelCurrentTask = false; function shouldCancelCurrentTask() { - return cancelCurrentTask; + return cancelCurrentTask; } function trimAudioStart( - audioSamples: Float32Array, - targetStartSilentSampleCount = 0, - amplitudeThresholdDecibels = defaultSilenceThresholdDecibels, + audioSamples: Float32Array, + targetStartSilentSampleCount = 0, + amplitudeThresholdDecibels = defaultSilenceThresholdDecibels ) { - const silentSampleCount = getStartingSilentSampleCount( - audioSamples, - amplitudeThresholdDecibels, - ); + const silentSampleCount = getStartingSilentSampleCount( + audioSamples, + amplitudeThresholdDecibels + ); - const trimmedAudio = audioSamples.subarray( - silentSampleCount, - audioSamples.length, - ); - const restoredSilence = new Float32Array(targetStartSilentSampleCount); + const trimmedAudio = audioSamples.subarray( + silentSampleCount, + audioSamples.length + ); + const restoredSilence = new Float32Array(targetStartSilentSampleCount); - const trimmedAudioSamples = concatFloat32Arrays([ - restoredSilence, - trimmedAudio, - ]); + const trimmedAudioSamples = concatFloat32Arrays([ + restoredSilence, + trimmedAudio, + ]); - return trimmedAudioSamples; + return trimmedAudioSamples; } function trimAudioEnd( - audioSamples: Float32Array, - targetEndSilentSampleCount = 0, - amplitudeThresholdDecibels = defaultSilenceThresholdDecibels, + audioSamples: Float32Array, + targetEndSilentSampleCount = 0, + amplitudeThresholdDecibels = defaultSilenceThresholdDecibels ) { - if (audioSamples.length === 0) { - return new Float32Array(0); - } + if (audioSamples.length === 0) { + return new Float32Array(0); + } - const silentSampleCount = getEndingSilentSampleCount( - audioSamples, - amplitudeThresholdDecibels, - ); + const silentSampleCount = getEndingSilentSampleCount( + audioSamples, + amplitudeThresholdDecibels + ); - const trimmedAudio = audioSamples.subarray( - 0, - audioSamples.length - silentSampleCount, - ); - const restoredSilence = new Float32Array(targetEndSilentSampleCount); + const trimmedAudio = audioSamples.subarray( + 0, + audioSamples.length - silentSampleCount + ); + const restoredSilence = new Float32Array(targetEndSilentSampleCount); - const trimmedAudioSamples = concatFloat32Arrays([ - trimmedAudio, - restoredSilence, - ]); + const trimmedAudioSamples = concatFloat32Arrays([ + trimmedAudio, + restoredSilence, + ]); - return trimmedAudioSamples; + return trimmedAudioSamples; } function addTimeOffsetToTimeline(targetTimeline: Timeline, timeOffset: number) { - if (!targetTimeline) { - return targetTimeline; - } + if (!targetTimeline) { + return targetTimeline; + } - const newTimeline = deepClone(targetTimeline); + const newTimeline = deepClone(targetTimeline); - for (const segmentTimelineEntry of newTimeline) { - segmentTimelineEntry.startTime = Math.max( - segmentTimelineEntry.startTime + timeOffset, - 0, - ); - segmentTimelineEntry.endTime = Math.max( - segmentTimelineEntry.endTime + timeOffset, - 0, - ); + for (const segmentTimelineEntry of newTimeline) { + segmentTimelineEntry.startTime = Math.max( + segmentTimelineEntry.startTime + timeOffset, + 0 + ); + segmentTimelineEntry.endTime = Math.max( + segmentTimelineEntry.endTime + timeOffset, + 0 + ); - if (segmentTimelineEntry.timeline) { - segmentTimelineEntry.timeline = addTimeOffsetToTimeline( - segmentTimelineEntry.timeline, - timeOffset, - ); + if (segmentTimelineEntry.timeline) { + segmentTimelineEntry.timeline = addTimeOffsetToTimeline( + segmentTimelineEntry.timeline, + timeOffset + ); + } } - } - return newTimeline; + return newTimeline; } async function synthesizeSegments( - segments: string[], - options: SynthesisOptions, - onSegment?: SynthesisSegmentEvent, - onSentence?: SynthesisSegmentEvent, + segments: string[], + options: SynthesisOptions, + onSegment?: SynthesisSegmentEvent, + onSentence?: SynthesisSegmentEvent ): Promise { - const logger = new Logger(); - options = extendDeep(defaultSynthesisOptions, options); + const logger = new Logger(); + options = extendDeep(defaultSynthesisOptions, options); - if (!options.language && !options.voice) { - logger.start("No language or voice specified. Detect language"); + if (!options.language && !options.voice) { + logger.start("No language or voice specified. Detect language"); - const segmentsPlainText = segments; + const segmentsPlainText = segments; - const { detectedLanguage } = await detectTextLanguage( - segmentsPlainText.join("\n\n"), - options.languageDetection || {}, - ); + const { detectedLanguage } = await detectTextLanguage( + segmentsPlainText.join("\n\n"), + options.languageDetection || {} + ); + + options.language = detectedLanguage; - options.language = detectedLanguage; + logger.end(); + } + + const { bestMatchingVoice } = await requestVoiceList(options); + + if (!bestMatchingVoice) { + throw new Error("No matching voice found 1"); + } + + options.voice = bestMatchingVoice.name; + + if (!options.language) { + options.language = bestMatchingVoice.languages[0]; + } logger.end(); - } + logger.logTitledMessage( + "Selected voice", + `'${options.voice}' (${formatLanguageCodeWithName(bestMatchingVoice.languages[0], 2)})` + ); - const { bestMatchingVoice } = await requestVoiceList(options); + const segmentsRawAudio: RawAudio[] = []; + const segmentsTimelines: Timeline[] = []; - if (!bestMatchingVoice) { - throw new Error("No matching voice found 1"); - } + const timeline: Timeline = []; - options.voice = bestMatchingVoice.name; + let peakDecibelsSoFar = -100; - if (!options.language) { - options.language = bestMatchingVoice.languages[0]; - } + let timeOffset = 0; - logger.end(); - logger.logTitledMessage( - "Selected voice", - `'${options.voice}' (${formatLanguageCodeWithName(bestMatchingVoice.languages[0], 2)})`, - ); + for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) { + const segmentText = segments[segmentIndex].trim(); - const segmentsRawAudio: RawAudio[] = []; - const segmentsTimelines: Timeline[] = []; + logger.log( + `\n${`Synthesizing segment ${segmentIndex + 1}/${segments.length}`}: '${segmentText}'` + ); - const timeline: Timeline = []; + const segmentStartTime = timeOffset; - let peakDecibelsSoFar = -100; + const segmentEntry: TimelineEntry = { + type: "segment", + text: segmentText, + startTime: timeOffset, + endTime: -1, + timeline: [], + }; - let timeOffset = 0; + let sentences: string[]; - for (let segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) { - const segmentText = segments[segmentIndex].trim(); + if (!options.ssml) { + sentences = splitToSentences(segmentText, options.language!); + sentences = sentences.filter((sentence) => sentence.trim() != ""); - logger.log( - `\n${`Synthesizing segment ${segmentIndex + 1}/${segments.length}`}: '${segmentText}'`, - ); + if (sentences.length == 0) { + sentences = [""]; + } + } else { + sentences = [segmentText]; + } - const segmentStartTime = timeOffset; + const sentencesRawAudio: RawAudio[] = []; + const sentencesTimelines: Timeline[] = []; - const segmentEntry: TimelineEntry = { - type: "segment", - text: segmentText, - startTime: timeOffset, - endTime: -1, - timeline: [], - }; + for ( + let sentenceIndex = 0; + sentenceIndex < sentences.length; + sentenceIndex++ + ) { + await yieldToEventLoop(); - let sentences: string[]; + if (shouldCancelCurrentTask()) { + //log('\n\n\n\n\nCANCELED\n\n\n\n') + throw new Error("Canceled"); + } - if (!options.ssml) { - sentences = splitToSentences(segmentText, options.language!); - sentences = sentences.filter((sentence) => sentence.trim() != ""); + const sentenceText = sentences[sentenceIndex].trim(); - if (sentences.length == 0) { - sentences = [""]; - } - } else { - sentences = [segmentText]; - } + logger.log( + `\n${`Synthesizing sentence ${sentenceIndex + 1}/${sentences.length}`}: "${sentenceText}"` + ); - const sentencesRawAudio: RawAudio[] = []; - const sentencesTimelines: Timeline[] = []; + const sentenceStartTime = timeOffset; - for ( - let sentenceIndex = 0; - sentenceIndex < sentences.length; - sentenceIndex++ - ) { - await yieldToEventLoop(); + let sentencetSynthesisOptions: SynthesisOptions = { + postProcessing: { normalizeAudio: false }, + }; + sentencetSynthesisOptions = extendDeep( + options, + sentencetSynthesisOptions + ); - if (shouldCancelCurrentTask()) { - //log('\n\n\n\n\nCANCELED\n\n\n\n') - throw new Error("Canceled"); - } + const { + synthesizedAudio: sentenceRawAudio, + timeline: sentenceTimeline, + } = await synthesizeSegment( + sentenceText, + sentencetSynthesisOptions + ); - const sentenceText = sentences[sentenceIndex].trim(); + const endPause = + sentenceIndex == sentences.length - 1 + ? options.segmentEndPause! + : options.sentenceEndPause!; + sentenceRawAudio.audioChannels[0] = trimAudioEnd( + sentenceRawAudio.audioChannels[0], + endPause * sentenceRawAudio.sampleRate + ); - logger.log( - `\n${`Synthesizing sentence ${sentenceIndex + 1}/${sentences.length}`}: "${sentenceText}"`, - ); + sentencesRawAudio.push(sentenceRawAudio); - const sentenceStartTime = timeOffset; + if (sentenceTimeline.length > 0) { + sentencesTimelines.push(sentenceTimeline); + } - let sentencetSynthesisOptions: SynthesisOptions = { - postProcessing: { normalizeAudio: false }, - }; - sentencetSynthesisOptions = extendDeep( - options, - sentencetSynthesisOptions, - ); - - const { synthesizedAudio: sentenceRawAudio, timeline: sentenceTimeline } = - await synthesizeSegment(sentenceText, sentencetSynthesisOptions); - - const endPause = - sentenceIndex == sentences.length - 1 - ? options.segmentEndPause! - : options.sentenceEndPause!; - sentenceRawAudio.audioChannels[0] = trimAudioEnd( - sentenceRawAudio.audioChannels[0], - endPause * sentenceRawAudio.sampleRate, - ); - - sentencesRawAudio.push(sentenceRawAudio); - - if (sentenceTimeline.length > 0) { - sentencesTimelines.push(sentenceTimeline); - } - - const sentenceAudioLength = - sentenceRawAudio.audioChannels[0].length / sentenceRawAudio.sampleRate; - - timeOffset += sentenceAudioLength; - - const sentenceTimelineWithOffset = addTimeOffsetToTimeline( - sentenceTimeline, - sentenceStartTime, - ); - - const sentenceEndTime = timeOffset - endPause; - - segmentEntry.timeline!.push({ - type: "sentence", - text: sentenceText, - startTime: sentenceStartTime, - endTime: sentenceEndTime, - timeline: sentenceTimelineWithOffset, - }); - - peakDecibelsSoFar = Math.max( - peakDecibelsSoFar, - getSamplePeakDecibels(sentenceRawAudio.audioChannels), - ); - - const sentenceAudio = - await convertToTargetCodecIfNeeded(sentenceRawAudio); - - if (onSentence) { - await onSentence({ - index: sentenceIndex, - total: sentences.length, - audio: sentenceAudio, - timeline: sentenceTimeline, - transcript: sentenceText, - language: options.language!, - peakDecibelsSoFar, - }); - } - } + const sentenceAudioLength = + sentenceRawAudio.audioChannels[0].length / + sentenceRawAudio.sampleRate; - segmentEntry.endTime = - segmentEntry.timeline?.[segmentEntry.timeline.length - 1]?.endTime || - timeOffset; + timeOffset += sentenceAudioLength; - logger.end(); + const sentenceTimelineWithOffset = addTimeOffsetToTimeline( + sentenceTimeline, + sentenceStartTime + ); - logger.start(`Merge and postprocess sentences`); + const sentenceEndTime = timeOffset - endPause; - let segmentRawAudio: RawAudio; + segmentEntry.timeline!.push({ + type: "sentence", + text: sentenceText, + startTime: sentenceStartTime, + endTime: sentenceEndTime, + timeline: sentenceTimelineWithOffset, + }); - if (sentencesRawAudio.length > 0) { - const joinedAudioBuffers = concatAudioSegments( - sentencesRawAudio.map((part) => part.audioChannels), - ); - segmentRawAudio = { - audioChannels: joinedAudioBuffers, - sampleRate: sentencesRawAudio[0].sampleRate, - }; - } else { - segmentRawAudio = getEmptyRawAudio(1, 24000); - } + peakDecibelsSoFar = Math.max( + peakDecibelsSoFar, + getSamplePeakDecibels(sentenceRawAudio.audioChannels) + ); - segmentsRawAudio.push(segmentRawAudio); + const sentenceAudio = + await convertToTargetCodecIfNeeded(sentenceRawAudio); + + if (onSentence) { + await onSentence({ + index: sentenceIndex, + total: sentences.length, + audio: sentenceAudio, + timeline: sentenceTimeline, + transcript: sentenceText, + language: options.language!, + peakDecibelsSoFar, + }); + } + } - timeline.push(segmentEntry); - const segmentTimelineWithoutOffset = addTimeOffsetToTimeline( - segmentEntry.timeline!, - -segmentStartTime, - ); - segmentsTimelines.push(segmentTimelineWithoutOffset); + segmentEntry.endTime = + segmentEntry.timeline?.[segmentEntry.timeline.length - 1] + ?.endTime || timeOffset; - const segmentAudio = await convertToTargetCodecIfNeeded(segmentRawAudio); + logger.end(); - logger.end(); + logger.start(`Merge and postprocess sentences`); - if (onSegment) { - await onSegment({ - index: segmentIndex, - total: segments.length, - audio: segmentAudio, - timeline: segmentTimelineWithoutOffset, - transcript: segmentText, - language: options.language!, - peakDecibelsSoFar, - }); - } - } - - logger.start(`\nMerge and postprocess segments`); - let resultRawAudio: RawAudio; - - if (segmentsRawAudio.length > 0) { - const joinedAudioBuffers = concatAudioSegments( - segmentsRawAudio.map((part) => part.audioChannels), - ); - resultRawAudio = { - audioChannels: joinedAudioBuffers, - sampleRate: segmentsRawAudio[0].sampleRate, - }; + let segmentRawAudio: RawAudio; - if (options.postProcessing!.normalizeAudio) { - resultRawAudio = normalizeAudioLevel( - resultRawAudio, - options.postProcessing!.targetPeak, - options.postProcessing!.maxGainIncrease, - ); + if (sentencesRawAudio.length > 0) { + const joinedAudioBuffers = concatAudioSegments( + sentencesRawAudio.map((part) => part.audioChannels) + ); + segmentRawAudio = { + audioChannels: joinedAudioBuffers, + sampleRate: sentencesRawAudio[0].sampleRate, + }; + } else { + segmentRawAudio = getEmptyRawAudio(1, 24000); + } + + segmentsRawAudio.push(segmentRawAudio); + + timeline.push(segmentEntry); + const segmentTimelineWithoutOffset = addTimeOffsetToTimeline( + segmentEntry.timeline!, + -segmentStartTime + ); + segmentsTimelines.push(segmentTimelineWithoutOffset); + + const segmentAudio = + await convertToTargetCodecIfNeeded(segmentRawAudio); + + logger.end(); + + if (onSegment) { + await onSegment({ + index: segmentIndex, + total: segments.length, + audio: segmentAudio, + timeline: segmentTimelineWithoutOffset, + transcript: segmentText, + language: options.language!, + peakDecibelsSoFar, + }); + } + } + + logger.start(`\nMerge and postprocess segments`); + let resultRawAudio: RawAudio; + + if (segmentsRawAudio.length > 0) { + const joinedAudioBuffers = concatAudioSegments( + segmentsRawAudio.map((part) => part.audioChannels) + ); + resultRawAudio = { + audioChannels: joinedAudioBuffers, + sampleRate: segmentsRawAudio[0].sampleRate, + }; + + if (options.postProcessing!.normalizeAudio) { + resultRawAudio = normalizeAudioLevel( + resultRawAudio, + options.postProcessing!.targetPeak, + options.postProcessing!.maxGainIncrease + ); + } else { + resultRawAudio = attenuateIfClipping(resultRawAudio); + } } else { - resultRawAudio = attenuateIfClipping(resultRawAudio); + resultRawAudio = getEmptyRawAudio(1, 24000); } - } else { - resultRawAudio = getEmptyRawAudio(1, 24000); - } - async function convertToTargetCodecIfNeeded(rawAudio: RawAudio) { - const targetCodec = options.outputAudioFormat?.codec; + async function convertToTargetCodecIfNeeded(rawAudio: RawAudio) { + const targetCodec = options.outputAudioFormat?.codec; + + let output: RawAudio | Buffer; - let output: RawAudio | Buffer; + if (targetCodec) { + logger.start(`Convert to ${targetCodec} codec`); - if (targetCodec) { - logger.start(`Convert to ${targetCodec} codec`); + if (targetCodec == "wav") { + output = encodeRawAudioToWave(rawAudio); + } else { + const ffmpegOptions = getDefaultFFMpegOptionsForSpeech( + targetCodec, + options.outputAudioFormat?.bitrate + ); + output = await encodeFromChannels(rawAudio, ffmpegOptions); + } + } else { + output = rawAudio; + } - if (targetCodec == "wav") { - output = encodeRawAudioToWave(rawAudio); - } else { - const ffmpegOptions = getDefaultFFMpegOptionsForSpeech( - targetCodec, - options.outputAudioFormat?.bitrate, - ); - output = await encodeFromChannels(rawAudio, ffmpegOptions); - } - } else { - output = rawAudio; + return output; } - return output; - } - - const resultAudio = await convertToTargetCodecIfNeeded(resultRawAudio); + const resultAudio = await convertToTargetCodecIfNeeded(resultRawAudio); - logger.end(); + logger.end(); - return { - audio: resultAudio, - timeline, - language: options.language, - voice: options.voice, - }; + return { + audio: resultAudio, + timeline, + language: options.language, + voice: options.voice, + }; } async function encodeFromChannels( - rawAudio: RawAudio, - outputOptions: FFMpegOutputOptions, + rawAudio: RawAudio, + outputOptions: FFMpegOutputOptions ) { - return transcode(encodeRawAudioToWave(rawAudio), outputOptions); + return transcode(encodeRawAudioToWave(rawAudio), outputOptions); } function getDefaultFFMpegOptionsForSpeech( - fileExtension: string, - customBitrate?: number, + fileExtension: string, + customBitrate?: number ) { - let ffmpegOptions: FFMpegOutputOptions; - - if (fileExtension == "mp3") { - ffmpegOptions = { - format: "mp3", - codec: "libmp3lame", - bitrate: 64, - customOptions: [], - }; - } else if (fileExtension == "opus") { - ffmpegOptions = { - codec: "libopus", - bitrate: 48, - customOptions: [], - }; - } else if (fileExtension == "m4a") { - ffmpegOptions = { - format: "mp4", - codec: "aac", - bitrate: 48, - customOptions: [ - "-profile:a", - "aac_low", - "-movflags", - "frag_keyframe+empty_moov", - ], - }; - } else if (fileExtension == "ogg") { - ffmpegOptions = { - codec: "libvorbis", - bitrate: 48, - customOptions: [], - }; - } else if (fileExtension == "flac") { - ffmpegOptions = { - format: "flac", - customOptions: ["-compression_level", "6"], - }; - } else { - throw new Error(`Unsupported codec extension: '${fileExtension}'`); - } + let ffmpegOptions: FFMpegOutputOptions; + + if (fileExtension == "mp3") { + ffmpegOptions = { + format: "mp3", + codec: "libmp3lame", + bitrate: 64, + customOptions: [], + }; + } else if (fileExtension == "opus") { + ffmpegOptions = { + codec: "libopus", + bitrate: 48, + customOptions: [], + }; + } else if (fileExtension == "m4a") { + ffmpegOptions = { + format: "mp4", + codec: "aac", + bitrate: 48, + customOptions: [ + "-profile:a", + "aac_low", + "-movflags", + "frag_keyframe+empty_moov", + ], + }; + } else if (fileExtension == "ogg") { + ffmpegOptions = { + codec: "libvorbis", + bitrate: 48, + customOptions: [], + }; + } else if (fileExtension == "flac") { + ffmpegOptions = { + format: "flac", + customOptions: ["-compression_level", "6"], + }; + } else { + throw new Error(`Unsupported codec extension: '${fileExtension}'`); + } - if (customBitrate != null) { - ffmpegOptions.bitrate = customBitrate; - } + if (customBitrate != null) { + ffmpegOptions.bitrate = customBitrate; + } - return ffmpegOptions; + return ffmpegOptions; } function concatAudioSegments(audioSegments: Float32Array[][]) { - if (audioSegments.length == 0) { - return []; - } + if (audioSegments.length == 0) { + return []; + } - const channelCount = audioSegments[0].length; + const channelCount = audioSegments[0].length; - const outAudioChannels: Float32Array[] = []; + const outAudioChannels: Float32Array[] = []; - for (let i = 0; i < channelCount; i++) { - const audioSegmentsForChannel = audioSegments.map((segment) => segment[i]); + for (let i = 0; i < channelCount; i++) { + const audioSegmentsForChannel = audioSegments.map( + (segment) => segment[i] + ); - outAudioChannels.push(concatFloat32Arrays(audioSegmentsForChannel)); - } + outAudioChannels.push(concatFloat32Arrays(audioSegmentsForChannel)); + } - return outAudioChannels; + return outAudioChannels; } interface SynthesisResult { - audio: RawAudio | Buffer; - timeline: Timeline; - language: string; - voice: string; + audio: RawAudio | Buffer; + timeline: Timeline; + language: string; + voice: string; } interface SynthesisOptions { - engine?: "vits" | "espeak"; - language?: string; - voice?: string; - voiceGender?: VoiceGender; + engine?: "vits" | "espeak"; + language?: string; + voice?: string; + voiceGender?: VoiceGender; - speed?: number; - pitch?: number; - pitchVariation?: number; + speed?: number; + pitch?: number; + pitchVariation?: number; - splitToSentences?: boolean; + splitToSentences?: boolean; - ssml?: boolean; + ssml?: boolean; - segmentEndPause?: number; - sentenceEndPause?: number; + segmentEndPause?: number; + sentenceEndPause?: number; - customLexiconPaths?: string[]; + customLexiconPaths?: string[]; - plainText?: PlainTextOptions; + plainText?: PlainTextOptions; - alignment?: AlignmentOptions; + alignment?: AlignmentOptions; - postProcessing?: { - normalizeAudio?: boolean; - targetPeak?: number; - maxGainIncrease?: number; + postProcessing?: { + normalizeAudio?: boolean; + targetPeak?: number; + maxGainIncrease?: number; - speed?: number; - pitch?: number; - }; + speed?: number; + pitch?: number; + }; - outputAudioFormat?: { - codec?: "wav" | "mp3" | "opus" | "m4a" | "ogg" | "flac"; - bitrate?: number; - }; + outputAudioFormat?: { + codec?: "wav" | "mp3" | "opus" | "m4a" | "ogg" | "flac"; + bitrate?: number; + }; - languageDetection?: TextLanguageDetectionOptions; + languageDetection?: TextLanguageDetectionOptions; - vits?: { - speakerId?: number; - provider?: OnnxExecutionProvider; - }; + vits?: { + speakerId?: number; + provider?: OnnxExecutionProvider; + }; - espeak?: { - rate?: number; - pitch?: number; - pitchRange?: number; + espeak?: { + rate?: number; + pitch?: number; + pitchRange?: number; - useKlatt?: boolean; - insertSeparators?: boolean; - }; + useKlatt?: boolean; + insertSeparators?: boolean; + }; } async function synthesizeSegment(text: string, options: SynthesisOptions) { - const logger = new Logger(); - - const startTimestamp = logger.getTimestamp(); - - logger.start("Prepare text for synthesis"); + const logger = new Logger(); - const simplifiedText = simplifyPunctuationCharacters(text); + const startTimestamp = logger.getTimestamp(); - const engine = options.engine; + logger.start("Prepare text for synthesis"); - logger.start(`Get voice list for ${engine}`); + const simplifiedText = simplifyPunctuationCharacters(text); - const { bestMatchingVoice } = await requestVoiceList(options); + const engine = options.engine; - if (!bestMatchingVoice) { - throw new Error("No matching voice found 2"); - } + logger.start(`Get voice list for ${engine}`); - const selectedVoice = bestMatchingVoice; + const { bestMatchingVoice } = await requestVoiceList(options); - let voicePackagePath: string | undefined; + if (!bestMatchingVoice) { + throw new Error("No matching voice found 2"); + } - if (selectedVoice.packageName) { - logger.end(); + const selectedVoice = bestMatchingVoice; - voicePackagePath = await loadPackage(selectedVoice.packageName); - } + let voicePackagePath: string | undefined; - logger.start(`Initialize ${engine} module`); + if (selectedVoice.packageName) { + logger.end(); - const voice = selectedVoice.name; + voicePackagePath = await loadPackage(selectedVoice.packageName); + } - let language: string; + logger.start(`Initialize ${engine} module`); - if (options.language) { - language = await normalizeIdentifierToLanguageCode(options.language); - } else { - language = selectedVoice.languages[0]; - } + const voice = selectedVoice.name; - const voiceGender = selectedVoice.gender; + let language: string; - const speed = clip(options.speed!, 0.1, 10.0); - const pitch = clip(options.pitch!, 0.1, 10.0); + if (options.language) { + language = await normalizeIdentifierToLanguageCode(options.language); + } else { + language = selectedVoice.languages[0]; + } - const inputIsSSML = options.ssml!; + const voiceGender = selectedVoice.gender; - let synthesizedAudio: RawAudio; + const speed = clip(options.speed!, 0.1, 10.0); + const pitch = clip(options.pitch!, 0.1, 10.0); - let timeline: Timeline | undefined; + const inputIsSSML = options.ssml!; - const shouldPostprocessSpeed = false; - let shouldPostprocessPitch = false; + let synthesizedAudio: RawAudio; - switch (engine) { - case "vits": { - if (inputIsSSML) { - throw new Error( - `The VITS engine doesn't currently support SSML inputs`, - ); - } + let timeline: Timeline | undefined; - let vitsLanguage = language; + const shouldPostprocessSpeed = false; + let shouldPostprocessPitch = false; - if (vitsLanguage == "en") { - vitsLanguage = "en-us"; - } + switch (engine) { + case "vits": { + if (inputIsSSML) { + throw new Error( + `The VITS engine doesn't currently support SSML inputs` + ); + } - const lengthScale = 1 / speed; + let vitsLanguage = language; - const vitsOptions = options.vits!; + if (vitsLanguage == "en") { + vitsLanguage = "en-us"; + } - const speakerId = vitsOptions.speakerId; + const lengthScale = 1 / speed; + + const vitsOptions = options.vits!; + + const speakerId = vitsOptions.speakerId; + + if (speakerId != undefined) { + if (selectedVoice.speakerCount == undefined) { + if (speakerId != 0) { + throw new Error( + "Selected VITS model has only one speaker. Speaker ID must be 0 if specified." + ); + } + } else if ( + speakerId < 0 || + speakerId >= selectedVoice.speakerCount + ) { + throw new Error( + `Selected VITS model has ${selectedVoice.speakerCount} speaker IDs. Speaker ID should be in the range ${0} to ${selectedVoice.speakerCount - 1}` + ); + } + } - if (speakerId != undefined) { - if (selectedVoice.speakerCount == undefined) { - if (speakerId != 0) { - throw new Error( - "Selected VITS model has only one speaker. Speaker ID must be 0 if specified.", + const lexicons = await loadLexiconsForLanguage( + language, + options.customLexiconPaths ); - } - } else if (speakerId < 0 || speakerId >= selectedVoice.speakerCount) { - throw new Error( - `Selected VITS model has ${selectedVoice.speakerCount} speaker IDs. Speaker ID should be in the range ${0} to ${selectedVoice.speakerCount - 1}`, - ); - } - } - - const lexicons = await loadLexiconsForLanguage( - language, - options.customLexiconPaths, - ); - const modelPath = voicePackagePath!; + const modelPath = voicePackagePath!; - const onnxExecutionProviders: OnnxExecutionProvider[] = - vitsOptions.provider ? [vitsOptions.provider] : []; + const onnxExecutionProviders: OnnxExecutionProvider[] = + vitsOptions.provider ? [vitsOptions.provider] : []; - logger.end(); + logger.end(); - const { rawAudio, timeline: outTimeline } = - await VitsTTS.synthesizeSentence( - text, - voice, - modelPath, - lengthScale, - speakerId ?? 0, - lexicons, - onnxExecutionProviders, - ); + const { rawAudio, timeline: outTimeline } = + await VitsTTS.synthesizeSentence( + text, + voice, + modelPath, + lengthScale, + speakerId ?? 0, + lexicons, + onnxExecutionProviders + ); - synthesizedAudio = rawAudio; - timeline = outTimeline; + synthesizedAudio = rawAudio; + timeline = outTimeline; - shouldPostprocessPitch = true; + shouldPostprocessPitch = true; - logger.end(); + logger.end(); - break; - } + break; + } - case "espeak": { - const engineOptions = options.espeak!; + case "espeak": { + const engineOptions = options.espeak!; + + const espeakVoice = voice; + const espeakLanguage = selectedVoice.languages[0]; + const espeakRate = engineOptions.rate || speed * 150; + const espeakPitch = engineOptions.pitch || options.pitch! * 50; + const espeakPitchRange = + engineOptions.pitchRange || options.pitchVariation! * 50; + const espeakUseKlatt = engineOptions.useKlatt || false; + const espeakInsertSeparators = + engineOptions.insertSeparators || false; + + const espeakOptions: EspeakOptions = { + voice: espeakVoice, + ssml: inputIsSSML, + rate: espeakRate, + pitch: espeakPitch, + pitchRange: espeakPitchRange, + useKlatt: espeakUseKlatt, + insertSeparators: espeakInsertSeparators, + }; + + if (inputIsSSML) { + logger.end(); + + const { rawAudio } = await espeakSynthesize( + text, + espeakOptions + ); + + synthesizedAudio = rawAudio; + } else { + const lexicons = await loadLexiconsForLanguage( + language, + options.customLexiconPaths + ); + + logger.end(); + + const { referenceSynthesizedAudio, referenceTimeline } = + await preprocessAndSynthesize( + text, + espeakLanguage, + espeakOptions, + lexicons + ); + + synthesizedAudio = referenceSynthesizedAudio; + timeline = referenceTimeline.flatMap( + (clause) => clause.timeline! + ); + } - const espeakVoice = voice; - const espeakLanguage = selectedVoice.languages[0]; - const espeakRate = engineOptions.rate || speed * 150; - const espeakPitch = engineOptions.pitch || options.pitch! * 50; - const espeakPitchRange = - engineOptions.pitchRange || options.pitchVariation! * 50; - const espeakUseKlatt = engineOptions.useKlatt || false; - const espeakInsertSeparators = engineOptions.insertSeparators || false; + break; + } - const espeakOptions: EspeakOptions = { - voice: espeakVoice, - ssml: inputIsSSML, - rate: espeakRate, - pitch: espeakPitch, - pitchRange: espeakPitchRange, - useKlatt: espeakUseKlatt, - insertSeparators: espeakInsertSeparators, - }; - - if (inputIsSSML) { - logger.end(); + default: { + throw new Error(`Engine '${options.engine}' is not supported`); + } + } - const { rawAudio } = await espeakSynthesize(text, espeakOptions); + logger.start("Postprocess synthesized audio"); + synthesizedAudio = downmixToMono(synthesizedAudio); - synthesizedAudio = rawAudio; - } else { - const lexicons = await loadLexiconsForLanguage( - language, - options.customLexiconPaths, + if (options.postProcessing!.normalizeAudio) { + synthesizedAudio = normalizeAudioLevel( + synthesizedAudio, + options.postProcessing!.targetPeak!, + options.postProcessing!.maxGainIncrease! ); + } else { + synthesizedAudio = attenuateIfClipping(synthesizedAudio); + } - logger.end(); - - const { referenceSynthesizedAudio, referenceTimeline } = - await preprocessAndSynthesize( - text, - espeakLanguage, - espeakOptions, - lexicons, - ); + const preTrimSampleCount = synthesizedAudio.audioChannels[0].length; + synthesizedAudio.audioChannels[0] = trimAudioStart( + synthesizedAudio.audioChannels[0] + ); - synthesizedAudio = referenceSynthesizedAudio; - timeline = referenceTimeline.flatMap((clause) => clause.timeline!); - } + if (timeline) { + const oldDuration = preTrimSampleCount / synthesizedAudio.sampleRate; + const newDuration = + synthesizedAudio.audioChannels[0].length / + synthesizedAudio.sampleRate; - break; + timeline = addTimeOffsetToTimeline(timeline, newDuration - oldDuration); } - default: { - throw new Error(`Engine '${options.engine}' is not supported`); - } - } + if (!timeline) { + logger.start("Align synthesized audio with text"); - logger.start("Postprocess synthesized audio"); - synthesizedAudio = downmixToMono(synthesizedAudio); + let plainText = text; - if (options.postProcessing!.normalizeAudio) { - synthesizedAudio = normalizeAudioLevel( - synthesizedAudio, - options.postProcessing!.targetPeak!, - options.postProcessing!.maxGainIncrease!, - ); - } else { - synthesizedAudio = attenuateIfClipping(synthesizedAudio); - } + if (inputIsSSML) { + plainText = await convertHtmlToText(text); + } - const preTrimSampleCount = synthesizedAudio.audioChannels[0].length; - synthesizedAudio.audioChannels[0] = trimAudioStart( - synthesizedAudio.audioChannels[0], - ); + const alignmentOptions = options.alignment!; - if (timeline) { - const oldDuration = preTrimSampleCount / synthesizedAudio.sampleRate; - const newDuration = - synthesizedAudio.audioChannels[0].length / synthesizedAudio.sampleRate; + alignmentOptions.language = language; - timeline = addTimeOffsetToTimeline(timeline, newDuration - oldDuration); - } + if (!alignmentOptions.customLexiconPaths) { + alignmentOptions.customLexiconPaths = options.customLexiconPaths; + } + + if (alignmentOptions.dtw!.windowDuration == null) { + alignmentOptions.dtw!.windowDuration = Math.max( + 5, + Math.ceil(0.2 * getRawAudioDuration(synthesizedAudio)) + ); + } - if (!timeline) { - logger.start("Align synthesized audio with text"); + const { wordTimeline } = await align( + synthesizedAudio, + plainText, + alignmentOptions + ); - let plainText = text; + timeline = wordTimeline; - if (inputIsSSML) { - plainText = await convertHtmlToText(text); + logger.end(); } - const alignmentOptions = options.alignment!; + const postProcessingOptions = options.postProcessing!; - alignmentOptions.language = language; + let timeStretchFactor = postProcessingOptions.speed; - if (!alignmentOptions.customLexiconPaths) { - alignmentOptions.customLexiconPaths = options.customLexiconPaths; + if (shouldPostprocessSpeed && timeStretchFactor == undefined) { + timeStretchFactor = speed; } - if (alignmentOptions.dtw!.windowDuration == null) { - alignmentOptions.dtw!.windowDuration = Math.max( - 5, - Math.ceil(0.2 * getRawAudioDuration(synthesizedAudio)), - ); - } + let pitchShiftFactor = postProcessingOptions.pitch; - const { wordTimeline } = await align( - synthesizedAudio, - plainText, - alignmentOptions, - ); + if (shouldPostprocessPitch && pitchShiftFactor == undefined) { + pitchShiftFactor = pitch; + } - timeline = wordTimeline; + if (timeline) { + timeline = timeline.filter((entry) => isWordOrSymbolWord(entry.text)); + } logger.end(); - } - - const postProcessingOptions = options.postProcessing!; - - let timeStretchFactor = postProcessingOptions.speed; - if (shouldPostprocessSpeed && timeStretchFactor == undefined) { - timeStretchFactor = speed; - } - - let pitchShiftFactor = postProcessingOptions.pitch; - - if (shouldPostprocessPitch && pitchShiftFactor == undefined) { - pitchShiftFactor = pitch; - } - - if (timeline) { - timeline = timeline.filter((entry) => isWordOrSymbolWord(entry.text)); - } - - logger.end(); - - return { synthesizedAudio, timeline }; + return { synthesizedAudio, timeline }; } interface PlainTextOptions { - paragraphBreaks?: ParagraphBreakType; - whitespace?: WhitespaceProcessing; + paragraphBreaks?: ParagraphBreakType; + whitespace?: WhitespaceProcessing; } type AudioSourceParam = string | Buffer | Uint8Array | RawAudio; @@ -5968,1811 +6107,1844 @@ type AudioSourceParam = string | Buffer | Uint8Array | RawAudio; type PhoneAlignmentMethod = "interpolation" | "dtw"; interface AlignmentOptions { - language?: string; + language?: string; - crop?: boolean; + crop?: boolean; - customLexiconPaths?: string[]; + customLexiconPaths?: string[]; - languageDetection?: TextLanguageDetectionOptions; + languageDetection?: TextLanguageDetectionOptions; - plainText?: PlainTextOptions; + plainText?: PlainTextOptions; - dtw?: { - granularity?: DtwGranularity | DtwGranularity[]; - windowDuration?: number | number[]; - phoneAlignmentMethod?: PhoneAlignmentMethod; - }; + dtw?: { + granularity?: DtwGranularity | DtwGranularity[]; + windowDuration?: number | number[]; + phoneAlignmentMethod?: PhoneAlignmentMethod; + }; } interface AlignmentResult { - timeline: Timeline; - wordTimeline: Timeline; + timeline: Timeline; + wordTimeline: Timeline; - transcript: string; - language: string; + transcript: string; + language: string; - inputRawAudio: RawAudio; - isolatedRawAudio?: RawAudio; - backgroundRawAudio?: RawAudio; + inputRawAudio: RawAudio; + isolatedRawAudio?: RawAudio; + backgroundRawAudio?: RawAudio; } const defaultAlignmentOptions: AlignmentOptions = { - language: undefined, + language: undefined, - crop: true, + crop: true, - customLexiconPaths: undefined, + customLexiconPaths: undefined, - languageDetection: {}, + languageDetection: {}, - plainText: { - paragraphBreaks: "double", - whitespace: "collapse", - }, + plainText: { + paragraphBreaks: "double", + whitespace: "collapse", + }, - dtw: { - granularity: undefined, - windowDuration: undefined, - phoneAlignmentMethod: "dtw", - }, + dtw: { + granularity: undefined, + windowDuration: undefined, + phoneAlignmentMethod: "dtw", + }, }; async function ensureRawAudio( - input: AudioSourceParam, - outSampleRate?: number, - outChannelCount?: number, + input: AudioSourceParam, + outSampleRate?: number, + outChannelCount?: number ) { - let inputAsRawAudio: RawAudio = input as RawAudio; + let inputAsRawAudio: RawAudio = input as RawAudio; - if (inputAsRawAudio.audioChannels?.length > 0 && inputAsRawAudio.sampleRate) { - const inputAudioChannelCount = inputAsRawAudio.audioChannels.length; + if ( + inputAsRawAudio.audioChannels?.length > 0 && + inputAsRawAudio.sampleRate + ) { + const inputAudioChannelCount = inputAsRawAudio.audioChannels.length; - if (outChannelCount == 1 && inputAudioChannelCount > 1) { - inputAsRawAudio = downmixToMono(inputAsRawAudio); - } + if (outChannelCount == 1 && inputAudioChannelCount > 1) { + inputAsRawAudio = downmixToMono(inputAsRawAudio); + } - if (outChannelCount == 2 && inputAudioChannelCount == 1) { - inputAsRawAudio = cloneRawAudio(inputAsRawAudio); - inputAsRawAudio.audioChannels.push( - inputAsRawAudio.audioChannels[0].slice(), - ); - } + if (outChannelCount == 2 && inputAudioChannelCount == 1) { + inputAsRawAudio = cloneRawAudio(inputAsRawAudio); + inputAsRawAudio.audioChannels.push( + inputAsRawAudio.audioChannels[0].slice() + ); + } - if ( - outChannelCount != null && - outChannelCount > 2 && - outChannelCount != inputAudioChannelCount - ) { - throw new Error( - `Can't convert ${inputAudioChannelCount} channels to ${outChannelCount} channels. Channel conversion of raw audio currently only supports mono and stereo inputs.`, - ); - } + if ( + outChannelCount != null && + outChannelCount > 2 && + outChannelCount != inputAudioChannelCount + ) { + throw new Error( + `Can't convert ${inputAudioChannelCount} channels to ${outChannelCount} channels. Channel conversion of raw audio currently only supports mono and stereo inputs.` + ); + } - if (outSampleRate && inputAsRawAudio.sampleRate != outSampleRate) { - inputAsRawAudio = await resampleAudioSpeex( - inputAsRawAudio, - outSampleRate, - ); - } - } else if (typeof input == "string" || input instanceof Uint8Array) { - if (input instanceof Uint8Array && !Buffer.isBuffer(input)) { - input = Buffer.from(input); - } + if (outSampleRate && inputAsRawAudio.sampleRate != outSampleRate) { + inputAsRawAudio = await resampleAudioSpeex( + inputAsRawAudio, + outSampleRate + ); + } + } else if (typeof input == "string" || input instanceof Uint8Array) { + if (input instanceof Uint8Array && !Buffer.isBuffer(input)) { + input = Buffer.from(input); + } - const inputAsStringOrBuffer = input as string | Buffer; + const inputAsStringOrBuffer = input as string | Buffer; - inputAsRawAudio = await decodeToChannels( - inputAsStringOrBuffer, - outSampleRate, - outChannelCount, - ); - } else { - throw new Error("Received an invalid input audio data type."); - } + inputAsRawAudio = await decodeToChannels( + inputAsStringOrBuffer, + outSampleRate, + outChannelCount + ); + } else { + throw new Error("Received an invalid input audio data type."); + } - return inputAsRawAudio; + return inputAsRawAudio; } type FFMpegOutputOptions = { - filename?: string; - codec?: string; - format?: string; - sampleRate?: number; - sampleFormat?: "u8" | "s16" | "s32" | "s64" | "flt" | "dbl"; - channelCount?: number; - bitrate?: number; - audioOnly?: boolean; - customOptions?: string[]; + filename?: string; + codec?: string; + format?: string; + sampleRate?: number; + sampleFormat?: "u8" | "s16" | "s32" | "s64" | "flt" | "dbl"; + channelCount?: number; + bitrate?: number; + audioOnly?: boolean; + customOptions?: string[]; }; function buildCommandLineArguments( - inputFilename: string, - outputOptions: FFMpegOutputOptions, + inputFilename: string, + outputOptions: FFMpegOutputOptions ) { - outputOptions = { ...outputOptions }; + outputOptions = { ...outputOptions }; - if (!outputOptions.filename) { - outputOptions.filename = "-"; - } + if (!outputOptions.filename) { + outputOptions.filename = "-"; + } - const args: string[] = []; + const args: string[] = []; - args.push(`-i`, `${inputFilename}`); + args.push(`-i`, `${inputFilename}`); - if (outputOptions.audioOnly) { - args.push(`-map`, `a`); - } + if (outputOptions.audioOnly) { + args.push(`-map`, `a`); + } - if (outputOptions.codec) { - args.push(`-c:a`, `${outputOptions.codec}`); - } + if (outputOptions.codec) { + args.push(`-c:a`, `${outputOptions.codec}`); + } - if (outputOptions.format) { - args.push(`-f:a`, `${outputOptions.format}`); - } + if (outputOptions.format) { + args.push(`-f:a`, `${outputOptions.format}`); + } - if (outputOptions.sampleRate) { - args.push(`-ar`, `${outputOptions.sampleRate}`); - } + if (outputOptions.sampleRate) { + args.push(`-ar`, `${outputOptions.sampleRate}`); + } - if (outputOptions.sampleFormat) { - args.push(`-sample_fmt`, `${outputOptions.sampleFormat}`); - } + if (outputOptions.sampleFormat) { + args.push(`-sample_fmt`, `${outputOptions.sampleFormat}`); + } - if (outputOptions.channelCount) { - args.push(`-ac`, `${outputOptions.channelCount}`); - } + if (outputOptions.channelCount) { + args.push(`-ac`, `${outputOptions.channelCount}`); + } - if (outputOptions.bitrate) { - args.push(`-ab`, `${outputOptions.bitrate}k`); - } + if (outputOptions.bitrate) { + args.push(`-ab`, `${outputOptions.bitrate}k`); + } - args.push(`-y`); + args.push(`-y`); - if (outputOptions.customOptions) { - args.push(...outputOptions.customOptions); - } + if (outputOptions.customOptions) { + args.push(...outputOptions.customOptions); + } - args.push(outputOptions.filename); + args.push(outputOptions.filename); - return args; + return args; } async function transcode_CLI( - ffmpegCommand: string, - input: string | Buffer, - outputOptions: FFMpegOutputOptions, + ffmpegCommand: string, + input: string | Buffer, + outputOptions: FFMpegOutputOptions ) { - return new Promise((resolve, reject) => { - const logger = new Logger(); - logger.start("Transcode with command-line ffmpeg"); + return new Promise((resolve, reject) => { + const logger = new Logger(); + logger.start("Transcode with command-line ffmpeg"); - const args = buildCommandLineArguments( - Buffer.isBuffer(input) ? "-" : input, - outputOptions, - ); + const args = buildCommandLineArguments( + Buffer.isBuffer(input) ? "-" : input, + outputOptions + ); - const process = spawn(ffmpegCommand, args); + const process = spawn(ffmpegCommand, args); - if (Buffer.isBuffer(input)) { - process.stdin.end(input); - } else if (typeof input === "string") { - if (!existsSync(input)) { - reject(`Audio file was not found: ${input}`); - return; - } - } + if (Buffer.isBuffer(input)) { + process.stdin.end(input); + } else if (typeof input === "string") { + if (!existsSync(input)) { + reject(`Audio file was not found: ${input}`); + return; + } + } - const stdoutChunks: Buffer[] = []; - let stderrOutput = ""; + const stdoutChunks: Buffer[] = []; + let stderrOutput = ""; - process.stdout.on("data", (data) => { - stdoutChunks.push(data); - }); + process.stdout.on("data", (data) => { + stdoutChunks.push(data); + }); - process.stderr.setEncoding("utf8"); - process.stderr.on("data", (data) => { - //log(data) - stderrOutput += data; - }); + process.stderr.setEncoding("utf8"); + process.stderr.on("data", (data) => { + //log(data) + stderrOutput += data; + }); - process.on("error", (e) => { - reject(e); - }); + process.on("error", (e) => { + reject(e); + }); - process.on("close", (exitCode) => { - if (exitCode == 0) { - const concatenatedChunks = Buffer.concat(stdoutChunks); + process.on("close", (exitCode) => { + if (exitCode == 0) { + const concatenatedChunks = Buffer.concat(stdoutChunks); - resolve(concatenatedChunks); - } else { - reject(`ffmpeg exited with code ${exitCode}`); - console.log(stderrOutput); - } + resolve(concatenatedChunks); + } else { + reject(`ffmpeg exited with code ${exitCode}`); + console.log(stderrOutput); + } - logger.end(); + logger.end(); + }); }); - }); } function resolveToVersionedPackageNameIfNeeded(packageName: string) { - const versionTag = getVersionTagFromPackageName(packageName); + const versionTag = getVersionTagFromPackageName(packageName); - if (versionTag) { - return packageName; - } + if (versionTag) { + return packageName; + } - const resolvedVersionTag = - resolveVersionTagForUnversionedPackageName(packageName); + const resolvedVersionTag = + resolveVersionTagForUnversionedPackageName(packageName); - return (packageName = `${packageName}-${resolvedVersionTag}`); + return (packageName = `${packageName}-${resolvedVersionTag}`); } function resolveVersionTagForUnversionedPackageName( - unversionedPackageName: string, + unversionedPackageName: string ) { - return ( - packageVersionTagResolutionLookup[unversionedPackageName] || - defaultVersionTag - ); + return ( + packageVersionTagResolutionLookup[unversionedPackageName] || + defaultVersionTag + ); } const defaultVersionTag = "20230718"; const packageVersionTagResolutionLookup: { [packageName: string]: string } = { - "sox-14.4.2-linux-minimal": "20230802", - - "vits-de_DE-thorsten_emotional-medium": "20230808", - "vits-en_GB-semaine-medium": "20230808", - "vits-fr_FR-upmc-medium": "20230808", - "vits-lb_LU-marylux-medium": "20230808", - "vits-ro_RO-mihai-medium": "20230808", - "vits-sr_RS-serbski_institut-medium": "20230808", - "vits-tr_TR-dfki-medium": "20230808", - - "vits-cs_CZ-jirka-medium": "20230824", - "vits-de_DE-thorsten-high": "20230824", - "vits-hu_HU-anna-medium": "20230824", - "vits-pt_PT-tugao-medium": "20230824", - "vits-sk_SK-lili-medium": "20230824", - "vits-tr_TR-fahrettin-medium": "20230824", - - "vits-ar_JO-kareem-medium": "20231022", - "vits-cs_CZ-jirka-low": "20231022", - "vits-en_US-hfc_male-medium": "20231022", - "vits-en_US-libritts_r-medium": "20231022", - "vits-hu_HU-imre-medium": "20231022", - "vits-pl_PL-mc_speech-medium": "20231022", - - "whisper-tiny": "20231126", - "whisper-tiny.en": "20231126", - "whisper-base": "20231126", - "whisper-base.en": "20231126", - "whisper-small": "20231126", - "whisper-small.en": "20231126", - "whisper-medium": "20231126", - "whisper-medium.en": "20231126", - "whisper-large-v3": "20231126", - - "vits-ar_JO-kareem-low": "20231126", - "vits-en_US-hfc_female-medium": "20231126", - - "ffmpeg-6.0-win32-x64": "20240316", - "ffmpeg-6.0-win32-ia32": "20240316", - "ffmpeg-6.0-darwin-x64": "20240316", - "ffmpeg-6.0-darwin-arm64": "20240316", - "ffmpeg-6.0-linux-x64": "20240316", - "ffmpeg-6.0-linux-ia32": "20240316", - "ffmpeg-6.0-linux-arm64": "20240316", - "ffmpeg-6.0-linux-arm": "20240316", - "ffmpeg-6.0-freebsd-x64": "20240316", - - "vits-de_DE-mls-medium": "20240316", - "vits-en_GB-cori-high": "20240316", - "vits-en_US-kristin-medium": "20240316", - "vits-en_US-ljspeech-high": "20240316", - "vits-en_US-ljspeech-medium": "20240316", - "vits-es_MX-claude-high": "20240316", - "vits-fa_IR-amir-medium": "20240316", - "vits-fa_IR-gyro-medium": "20240316", - "vits-fr_FR-mls-medium": "20240316", - "vits-fr_FR-tom-medium": "20240316", - "vits-nl_NL-mls-medium": "20240316", - "vits-sl_SI-artur-medium": "20240316", - "vits-tr_TR-fettah-medium": "20240316", - - "mdxnet-UVR_MDXNET_1_9703": "20240330", - "mdxnet-UVR_MDXNET_2_9682": "20240330", - "mdxnet-UVR_MDXNET_3_9662": "20240330", - "mdxnet-UVR_MDXNET_KARA": "20240330", - - "whisper.cpp-tiny": "20240405", - "whisper.cpp-tiny-q5_1": "20240405", - "whisper.cpp-tiny.en": "20240405", - "whisper.cpp-tiny.en-q5_1": "20240405", - "whisper.cpp-tiny.en-q8_0": "20240405", - - "whisper.cpp-base": "20240405", - "whisper.cpp-base-q5_1": "20240405", - "whisper.cpp-base.en": "20240405", - "whisper.cpp-base.en-q5_1": "20240405", - - "whisper.cpp-small": "20240405", - "whisper.cpp-small-q5_1": "20240405", - "whisper.cpp-small.en": "20240405", - "whisper.cpp-small.en-q5_1": "20240405", - - "whisper.cpp-medium": "20240405", - "whisper.cpp-medium-q5_0": "20240405", - "whisper.cpp-medium.en": "20240405", - "whisper.cpp-medium.en-q5_0": "20240405", - - "whisper.cpp-large-v1": "20240405", - "whisper.cpp-large-v2": "20240405", - "whisper.cpp-large-v2-q5_0": "20240405", - "whisper.cpp-large-v3": "20240405", - "whisper.cpp-large-v3-q5_0": "20240405", - - "whisper.cpp-binaries-linux-x64-cpu-latest-patched": "20240405", - "whisper.cpp-binaries-windows-x64-cpu-latest-patched": "20240409", - - "whisper-tiktoken-data": "20240408", - - "whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest-patched": "20240409", - "whisper.cpp-binaries-windows-x64-cublas-11.8.0-latest-patched": "20240409", - - "xenova-multilingual-e5-small-q8": "20240504", - "xenova-nllb-200-distilled-600M-q8": "20240505", - "xenova-multilingual-e5-small-fp16": "20240514", - "xenova-multilingual-e5-base-fp16": "20240514", - "xenova-multilingual-e5-base-q8": "20240514", - "xenova-multilingual-e5-large-q8": "20240514", - - "w2v-bert-2.0-int8": "20240517", - "w2v-bert-2.0-uint8": "20240517", + "sox-14.4.2-linux-minimal": "20230802", + + "vits-de_DE-thorsten_emotional-medium": "20230808", + "vits-en_GB-semaine-medium": "20230808", + "vits-fr_FR-upmc-medium": "20230808", + "vits-lb_LU-marylux-medium": "20230808", + "vits-ro_RO-mihai-medium": "20230808", + "vits-sr_RS-serbski_institut-medium": "20230808", + "vits-tr_TR-dfki-medium": "20230808", + + "vits-cs_CZ-jirka-medium": "20230824", + "vits-de_DE-thorsten-high": "20230824", + "vits-hu_HU-anna-medium": "20230824", + "vits-pt_PT-tugao-medium": "20230824", + "vits-sk_SK-lili-medium": "20230824", + "vits-tr_TR-fahrettin-medium": "20230824", + + "vits-ar_JO-kareem-medium": "20231022", + "vits-cs_CZ-jirka-low": "20231022", + "vits-en_US-hfc_male-medium": "20231022", + "vits-en_US-libritts_r-medium": "20231022", + "vits-hu_HU-imre-medium": "20231022", + "vits-pl_PL-mc_speech-medium": "20231022", + + "whisper-tiny": "20231126", + "whisper-tiny.en": "20231126", + "whisper-base": "20231126", + "whisper-base.en": "20231126", + "whisper-small": "20231126", + "whisper-small.en": "20231126", + "whisper-medium": "20231126", + "whisper-medium.en": "20231126", + "whisper-large-v3": "20231126", + + "vits-ar_JO-kareem-low": "20231126", + "vits-en_US-hfc_female-medium": "20231126", + + "ffmpeg-6.0-win32-x64": "20240316", + "ffmpeg-6.0-win32-ia32": "20240316", + "ffmpeg-6.0-darwin-x64": "20240316", + "ffmpeg-6.0-darwin-arm64": "20240316", + "ffmpeg-6.0-linux-x64": "20240316", + "ffmpeg-6.0-linux-ia32": "20240316", + "ffmpeg-6.0-linux-arm64": "20240316", + "ffmpeg-6.0-linux-arm": "20240316", + "ffmpeg-6.0-freebsd-x64": "20240316", + + "vits-de_DE-mls-medium": "20240316", + "vits-en_GB-cori-high": "20240316", + "vits-en_US-kristin-medium": "20240316", + "vits-en_US-ljspeech-high": "20240316", + "vits-en_US-ljspeech-medium": "20240316", + "vits-es_MX-claude-high": "20240316", + "vits-fa_IR-amir-medium": "20240316", + "vits-fa_IR-gyro-medium": "20240316", + "vits-fr_FR-mls-medium": "20240316", + "vits-fr_FR-tom-medium": "20240316", + "vits-nl_NL-mls-medium": "20240316", + "vits-sl_SI-artur-medium": "20240316", + "vits-tr_TR-fettah-medium": "20240316", + + "mdxnet-UVR_MDXNET_1_9703": "20240330", + "mdxnet-UVR_MDXNET_2_9682": "20240330", + "mdxnet-UVR_MDXNET_3_9662": "20240330", + "mdxnet-UVR_MDXNET_KARA": "20240330", + + "whisper.cpp-tiny": "20240405", + "whisper.cpp-tiny-q5_1": "20240405", + "whisper.cpp-tiny.en": "20240405", + "whisper.cpp-tiny.en-q5_1": "20240405", + "whisper.cpp-tiny.en-q8_0": "20240405", + + "whisper.cpp-base": "20240405", + "whisper.cpp-base-q5_1": "20240405", + "whisper.cpp-base.en": "20240405", + "whisper.cpp-base.en-q5_1": "20240405", + + "whisper.cpp-small": "20240405", + "whisper.cpp-small-q5_1": "20240405", + "whisper.cpp-small.en": "20240405", + "whisper.cpp-small.en-q5_1": "20240405", + + "whisper.cpp-medium": "20240405", + "whisper.cpp-medium-q5_0": "20240405", + "whisper.cpp-medium.en": "20240405", + "whisper.cpp-medium.en-q5_0": "20240405", + + "whisper.cpp-large-v1": "20240405", + "whisper.cpp-large-v2": "20240405", + "whisper.cpp-large-v2-q5_0": "20240405", + "whisper.cpp-large-v3": "20240405", + "whisper.cpp-large-v3-q5_0": "20240405", + + "whisper.cpp-binaries-linux-x64-cpu-latest-patched": "20240405", + "whisper.cpp-binaries-windows-x64-cpu-latest-patched": "20240409", + + "whisper-tiktoken-data": "20240408", + + "whisper.cpp-binaries-windows-x64-cublas-12.4.0-latest-patched": "20240409", + "whisper.cpp-binaries-windows-x64-cublas-11.8.0-latest-patched": "20240409", + + "xenova-multilingual-e5-small-q8": "20240504", + "xenova-nllb-200-distilled-600M-q8": "20240505", + "xenova-multilingual-e5-small-fp16": "20240514", + "xenova-multilingual-e5-base-fp16": "20240514", + "xenova-multilingual-e5-base-q8": "20240514", + "xenova-multilingual-e5-large-q8": "20240514", + + "w2v-bert-2.0-int8": "20240517", + "w2v-bert-2.0-uint8": "20240517", }; function getVersionTagFromPackageName(packageName: string) { - return packageName.match( - /.*\-([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9](_[0-9]+)?)$/, - )?.[1]; + return packageName.match( + /.*\-([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9](_[0-9]+)?)$/ + )?.[1]; } async function ensureAndGetPackagesDir() { - const dataPath = getAppDataDir("eliza"); + const dataPath = getAppDataDir("eliza"); - const packagesPath = path.join(dataPath, "packages"); + const packagesPath = path.join(dataPath, "packages"); - await ensureDir(packagesPath); + await ensureDir(packagesPath); - return packagesPath; + return packagesPath; } async function downloadFile( - options: GaxiosOptions, - targetFilePath: string, - prompt = "Downloading", + options: GaxiosOptions, + targetFilePath: string, + prompt = "Downloading" ) { - const write = logLevelGreaterOrEqualTo("info") ? writeToStderr : () => {}; + const write = logLevelGreaterOrEqualTo("info") ? writeToStderr : () => {}; - const downloadPromise = new OpenPromise(); + const downloadPromise = new OpenPromise(); - const timer = new Timer(); + const timer = new Timer(); - options.responseType = "stream"; + options.responseType = "stream"; - const response = await request(options); + const response = await request(options); - const ttyOutput = process.stderr.isTTY === true; + const ttyOutput = process.stderr.isTTY === true; - write(`\n${prompt}.. `); + write(`\n${prompt}.. `); - const rateAveragingWindowSeconds = 5.0; + const rateAveragingWindowSeconds = 5.0; - let downloadStarted = false; - let downloadedBytes = 0; - let totalBytes: number | undefined = undefined; + let downloadStarted = false; + let downloadedBytes = 0; + let totalBytes: number | undefined = undefined; - const statusUpdateInterval = 250; + const statusUpdateInterval = 250; - let lastString = prompt; + let lastString = prompt; - const downloadStateHistory: { time: number; downloadedMBytes: number }[] = []; + const downloadStateHistory: { time: number; downloadedMBytes: number }[] = + []; - function updateStatus() { - if (!downloadStarted) { - return; - } + function updateStatus() { + if (!downloadStarted) { + return; + } - const totalMBytes = (totalBytes || 0) / 1000 / 1000; - const downloadedMBytes = downloadedBytes / 1000 / 1000; + const totalMBytes = (totalBytes || 0) / 1000 / 1000; + const downloadedMBytes = downloadedBytes / 1000 / 1000; - const elapsedTime = timer.elapsedTimeSeconds; - const cumulativeDownloadRate = downloadedMBytes / elapsedTime; + const elapsedTime = timer.elapsedTimeSeconds; + const cumulativeDownloadRate = downloadedMBytes / elapsedTime; - const windowStartRecord = downloadStateHistory.find( - (r) => r.time >= timer.elapsedTimeSeconds - rateAveragingWindowSeconds, - ); + const windowStartRecord = downloadStateHistory.find( + (r) => + r.time >= timer.elapsedTimeSeconds - rateAveragingWindowSeconds + ); - let windowDownloadRate: number; + let windowDownloadRate: number; - if (windowStartRecord) { - windowDownloadRate = - (downloadedMBytes - windowStartRecord.downloadedMBytes) / - (elapsedTime - windowStartRecord.time); - } else { - windowDownloadRate = cumulativeDownloadRate; - } + if (windowStartRecord) { + windowDownloadRate = + (downloadedMBytes - windowStartRecord.downloadedMBytes) / + (elapsedTime - windowStartRecord.time); + } else { + windowDownloadRate = cumulativeDownloadRate; + } - downloadStateHistory.push({ time: elapsedTime, downloadedMBytes }); + downloadStateHistory.push({ time: elapsedTime, downloadedMBytes }); - const isLastUpdate = downloadedMBytes == totalMBytes; + const isLastUpdate = downloadedMBytes == totalMBytes; - const downloadedMbytesStr = downloadedMBytes.toFixed(2); - const totalMbytesStr = totalMBytes.toFixed(2); - const downloadRateStr = windowDownloadRate.toFixed(2); - const cumulativeDownloadRateStr = cumulativeDownloadRate.toFixed(2); + const downloadedMbytesStr = downloadedMBytes.toFixed(2); + const totalMbytesStr = totalMBytes.toFixed(2); + const downloadRateStr = windowDownloadRate.toFixed(2); + const cumulativeDownloadRateStr = cumulativeDownloadRate.toFixed(2); - if (ttyOutput) { - let newString: string; + if (ttyOutput) { + let newString: string; - if (totalBytes != undefined) { - const percentage = (downloadedMBytes / totalMBytes) * 100; + if (totalBytes != undefined) { + const percentage = (downloadedMBytes / totalMBytes) * 100; - newString = `${prompt}.. ${downloadedMbytesStr}MB/${totalMbytesStr}MB (${percentage.toFixed(1) + "%"}, ${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`; - } else { - newString = `${prompt}.. ${downloadedMbytesStr}MB (${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`; - } + newString = `${prompt}.. ${downloadedMbytesStr}MB/${totalMbytesStr}MB (${percentage.toFixed(1) + "%"}, ${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`; + } else { + newString = `${prompt}.. ${downloadedMbytesStr}MB (${timer.elapsedTimeSeconds.toFixed(1)}s, ${downloadRateStr}MB/s)`; + } - if (newString != lastString) { - write("\r"); - write(newString); - } + if (newString != lastString) { + write("\r"); + write(newString); + } - lastString = newString; - } else { - if (totalBytes == undefined) { - return; - } + lastString = newString; + } else { + if (totalBytes == undefined) { + return; + } - const percent = downloadedBytes / totalBytes; - const percentDisplay = `${(Math.floor(percent * 10) * 10).toString()}%`; + const percent = downloadedBytes / totalBytes; + const percentDisplay = `${(Math.floor(percent * 10) * 10).toString()}%`; - if (lastString == prompt) { - write(`(${totalMbytesStr}MB): `); - } + if (lastString == prompt) { + write(`(${totalMbytesStr}MB): `); + } - if (percentDisplay != lastString) { - write(percentDisplay); + if (percentDisplay != lastString) { + write(percentDisplay); - if (percent == 1.0) { - write( - ` (${timer.elapsedTimeSeconds.toFixed(2)}s, ${cumulativeDownloadRateStr}MB/s)`, - ); - } else { - write(` `); - } + if (percent == 1.0) { + write( + ` (${timer.elapsedTimeSeconds.toFixed(2)}s, ${cumulativeDownloadRateStr}MB/s)` + ); + } else { + write(` `); + } - lastString = percentDisplay; - } + lastString = percentDisplay; + } + } } - } - const partialFilePath = `${targetFilePath}.${getRandomHexString(16)}.partial`; - const fileWriteStream = createWriteStream(partialFilePath, { - encoding: "binary", - autoClose: true, - }); + const partialFilePath = `${targetFilePath}.${getRandomHexString(16)}.partial`; + const fileWriteStream = createWriteStream(partialFilePath, { + encoding: "binary", + autoClose: true, + }); - const statusInterval = setInterval(() => { - updateStatus(); - }, statusUpdateInterval); + const statusInterval = setInterval(() => { + updateStatus(); + }, statusUpdateInterval); - response.data.on("data", (chunk: Uint8Array) => { - try { - const contentLengthString = response.headers["content-length"]; + response.data.on("data", (chunk: Uint8Array) => { + try { + const contentLengthString = response.headers["content-length"]; - totalBytes = - contentLengthString != undefined - ? parseInt(contentLengthString) - : undefined; + totalBytes = + contentLengthString != undefined + ? parseInt(contentLengthString) + : undefined; - const chunkLength = chunk.length; + const chunkLength = chunk.length; - fileWriteStream.write(chunk); + fileWriteStream.write(chunk); - downloadedBytes += chunkLength; + downloadedBytes += chunkLength; - if (downloadStarted == false) { - downloadStarted = true; - } - } catch (err) { - clearInterval(statusInterval); + if (downloadStarted == false) { + downloadStarted = true; + } + } catch (err) { + clearInterval(statusInterval); - downloadPromise.reject(err); - } - }); + downloadPromise.reject(err); + } + }); - response.data.on("end", async () => { - try { - clearInterval(statusInterval); - updateStatus(); + response.data.on("end", async () => { + try { + clearInterval(statusInterval); + updateStatus(); - fileWriteStream.end(); + fileWriteStream.end(); - write("\n"); + write("\n"); - await move(partialFilePath, targetFilePath); + await move(partialFilePath, targetFilePath); - downloadPromise.resolve(); - } catch (err) { - clearInterval(statusInterval); - downloadPromise.reject(err); - } - }); + downloadPromise.resolve(); + } catch (err) { + clearInterval(statusInterval); + downloadPromise.reject(err); + } + }); - response.data.on("error", async (err: any) => { - try { - clearInterval(statusInterval); + response.data.on("error", async (err: any) => { + try { + clearInterval(statusInterval); - fileWriteStream.end(); - await remove(partialFilePath); - } finally { - downloadPromise.reject(err); - } - }); + fileWriteStream.end(); + await remove(partialFilePath); + } finally { + downloadPromise.reject(err); + } + }); - return downloadPromise.promise; + return downloadPromise.promise; } async function extractTarball(filepath: string, outputDir: string) { - const { extract } = await import("tar"); + const { extract } = await import("tar"); - await extract({ - file: filepath, - cwd: outputDir, - preserveOwner: false, - //noChmod: true - }); + await extract({ + file: filepath, + cwd: outputDir, + preserveOwner: false, + //noChmod: true + }); } async function downloadAndExtractTarball( - options: GaxiosOptions, - targetDir: string, - baseTempPath: string, - displayName = "archive", + options: GaxiosOptions, + targetDir: string, + baseTempPath: string, + displayName = "archive" ) { - const logger = new Logger(); + const logger = new Logger(); - const randomID = getRandomHexString(16).toLowerCase(); - const tempTarballPath = path.join(baseTempPath, `/${randomID}.tarball`); - const tempDirPath = path.join(baseTempPath, `/${randomID}`); - await ensureDir(tempDirPath); + const randomID = getRandomHexString(16).toLowerCase(); + const tempTarballPath = path.join(baseTempPath, `/${randomID}.tarball`); + const tempDirPath = path.join(baseTempPath, `/${randomID}`); + await ensureDir(tempDirPath); - logger.end(); + logger.end(); - await downloadFile( - options, - tempTarballPath, - `${"Downloading"} ${displayName}`, - ); + await downloadFile( + options, + tempTarballPath, + `${"Downloading"} ${displayName}` + ); - logger.end(); + logger.end(); - logger.start(`Extracting ${displayName}`); + logger.start(`Extracting ${displayName}`); - await extractTarball(tempTarballPath, tempDirPath); + await extractTarball(tempTarballPath, tempDirPath); - await remove(tempTarballPath); + await remove(tempTarballPath); - for (const filename of await readdir(tempDirPath)) { - const sourceFilePath = path.join(tempDirPath, filename); - const targetFilePath = path.join(targetDir, filename); + for (const filename of await readdir(tempDirPath)) { + const sourceFilePath = path.join(tempDirPath, filename); + const targetFilePath = path.join(targetDir, filename); - await move(sourceFilePath, targetFilePath); - } + await move(sourceFilePath, targetFilePath); + } - await remove(tempDirPath); + await remove(tempDirPath); - logger.end(); + logger.end(); } async function loadPackage(packageName: string) { - packageName = resolveToVersionedPackageNameIfNeeded(packageName); + packageName = resolveToVersionedPackageNameIfNeeded(packageName); - const packagesPath = await ensureAndGetPackagesDir(); + const packagesPath = await ensureAndGetPackagesDir(); - const packagePath = path.join(packagesPath, packageName); + const packagePath = path.join(packagesPath, packageName); - if (existsSync(packagePath)) { - return packagePath; - } + if (existsSync(packagePath)) { + return packagePath; + } - const packageBaseURL = getGlobalOption("packageBaseURL"); + const packageBaseURL = getGlobalOption("packageBaseURL"); - const tempPath = getAppTempDir("eliza"); + const tempPath = getAppTempDir("eliza"); - const headers = {}; + const headers = {}; - const options = { - url: `${packageBaseURL}${packageName}.tar.gz`, - headers, - }; + const options = { + url: `${packageBaseURL}${packageName}.tar.gz`, + headers, + }; - await downloadAndExtractTarball(options, packagesPath, tempPath, packageName); + await downloadAndExtractTarball( + options, + packagesPath, + tempPath, + packageName + ); - return packagePath; + return packagePath; } interface GlobalOptions { - ffmpegPath?: string; - soxPath?: string; - packageBaseURL?: string; - logLevel?: LogLevel; + ffmpegPath?: string; + soxPath?: string; + packageBaseURL?: string; + logLevel?: LogLevel; } const globalOptions: GlobalOptions = { - ffmpegPath: undefined, - soxPath: undefined, - packageBaseURL: - "https://huggingface.co/echogarden/echogarden-packages/resolve/main/", - logLevel: "info", + ffmpegPath: undefined, + soxPath: undefined, + packageBaseURL: + "https://huggingface.co/echogarden/echogarden-packages/resolve/main/", + logLevel: "info", }; function listGlobalOptions() { - return Object.keys(globalOptions); + return Object.keys(globalOptions); } function getGlobalOption( - key: K, + key: K ): GlobalOptions[K] { - if (!listGlobalOptions().includes(key)) { - throw new Error(`Unknown global option key '${key}'`); - } + if (!listGlobalOptions().includes(key)) { + throw new Error(`Unknown global option key '${key}'`); + } - return globalOptions[key]; + return globalOptions[key]; } async function commandExists(command: string) { - try { - await commandExists(command); - return true; - } catch { - return false; - } + try { + await commandExists(command); + return true; + } catch { + return false; + } } async function getFFMpegExecutablePath() { - // If a global option set for the path, use it - if (getGlobalOption("ffmpegPath")) { - return getGlobalOption("ffmpegPath"); - } - - // If an 'ffmpeg' command exist in system path, use it - if (await commandExists("ffmpeg")) { - return "ffmpeg"; - } - - // Otherwise, download and use an internal ffmpeg package - const platform = process.platform; - const arch = process.arch; - - let packageName: string; - - if (platform === "win32" && arch === "x64") { - packageName = "ffmpeg-6.0-win32-x64"; - } else if (platform === "win32" && arch === "ia32") { - packageName = "ffmpeg-6.0-win32-ia32"; - } else if (platform === "darwin" && arch === "x64") { - packageName = "ffmpeg-6.0-darwin-x64"; - } else if (platform === "darwin" && arch === "arm64") { - packageName = "ffmpeg-6.0-darwin-arm64"; - } else if (platform === "linux" && arch === "x64") { - packageName = "ffmpeg-6.0-linux-x64"; - } else if (platform === "linux" && arch === "ia32") { - packageName = "ffmpeg-6.0-linux-ia32"; - } else if (platform === "linux" && arch === "arm64") { - packageName = "ffmpeg-6.0-linux-arm64"; - } else if (platform === "linux" && arch === "arm") { - packageName = "ffmpeg-6.0-linux-arm"; - } else if (platform === "freebsd" && arch === "x64") { - packageName = "ffmpeg-6.0-freebsd-x64"; - } else { - return undefined; - } + // If a global option set for the path, use it + if (getGlobalOption("ffmpegPath")) { + return getGlobalOption("ffmpegPath"); + } + + // If an 'ffmpeg' command exist in system path, use it + if (await commandExists("ffmpeg")) { + return "ffmpeg"; + } + + // Otherwise, download and use an internal ffmpeg package + const platform = process.platform; + const arch = process.arch; + + let packageName: string; + + if (platform === "win32" && arch === "x64") { + packageName = "ffmpeg-6.0-win32-x64"; + } else if (platform === "win32" && arch === "ia32") { + packageName = "ffmpeg-6.0-win32-ia32"; + } else if (platform === "darwin" && arch === "x64") { + packageName = "ffmpeg-6.0-darwin-x64"; + } else if (platform === "darwin" && arch === "arm64") { + packageName = "ffmpeg-6.0-darwin-arm64"; + } else if (platform === "linux" && arch === "x64") { + packageName = "ffmpeg-6.0-linux-x64"; + } else if (platform === "linux" && arch === "ia32") { + packageName = "ffmpeg-6.0-linux-ia32"; + } else if (platform === "linux" && arch === "arm64") { + packageName = "ffmpeg-6.0-linux-arm64"; + } else if (platform === "linux" && arch === "arm") { + packageName = "ffmpeg-6.0-linux-arm"; + } else if (platform === "freebsd" && arch === "x64") { + packageName = "ffmpeg-6.0-freebsd-x64"; + } else { + return undefined; + } - const ffmpegPackagePath = await loadPackage(packageName); + const ffmpegPackagePath = await loadPackage(packageName); - let filename = packageName; + let filename = packageName; - if (platform === "win32") { - filename += ".exe"; - } + if (platform === "win32") { + filename += ".exe"; + } - return path.join(ffmpegPackagePath, filename); + return path.join(ffmpegPackagePath, filename); } async function transcode( - input: string | Buffer, - outputOptions: FFMpegOutputOptions, + input: string | Buffer, + outputOptions: FFMpegOutputOptions ) { - const executablePath = await getFFMpegExecutablePath(); + const executablePath = await getFFMpegExecutablePath(); - if (!executablePath) { - throw new Error( - `The ffmpeg utility wasn't found. Please ensure it is available on the system path.`, - ); - } + if (!executablePath) { + throw new Error( + `The ffmpeg utility wasn't found. Please ensure it is available on the system path.` + ); + } - return transcode_CLI(executablePath, input, outputOptions); + return transcode_CLI(executablePath, input, outputOptions); } function decodeWave( - waveData: Buffer, - ignoreTruncatedChunks = true, - ignoreOverflowingDataChunks = true, + waveData: Buffer, + ignoreTruncatedChunks = true, + ignoreOverflowingDataChunks = true ) { - let readOffset = 0; + let readOffset = 0; - const riffId = waveData - .subarray(readOffset, readOffset + 4) - .toString("ascii"); + const riffId = waveData + .subarray(readOffset, readOffset + 4) + .toString("ascii"); - if (riffId != "RIFF") { - throw new Error("Not a valid wave file. No RIFF id found at offset 0."); - } + if (riffId != "RIFF") { + throw new Error("Not a valid wave file. No RIFF id found at offset 0."); + } - readOffset += 4; + readOffset += 4; - let riffChunkSize = waveData.readUInt32LE(readOffset); + let riffChunkSize = waveData.readUInt32LE(readOffset); - readOffset += 4; + readOffset += 4; - const waveId = waveData - .subarray(readOffset, readOffset + 4) - .toString("ascii"); + const waveId = waveData + .subarray(readOffset, readOffset + 4) + .toString("ascii"); - if (waveId != "WAVE") { - throw new Error("Not a valid wave file. No WAVE id found at offset 8."); - } + if (waveId != "WAVE") { + throw new Error("Not a valid wave file. No WAVE id found at offset 8."); + } - if (ignoreOverflowingDataChunks && riffChunkSize === 4294967295) { - riffChunkSize = waveData.length - 8; - } + if (ignoreOverflowingDataChunks && riffChunkSize === 4294967295) { + riffChunkSize = waveData.length - 8; + } - if (riffChunkSize < waveData.length - 8) { - throw new Error( - `RIFF chunk length ${riffChunkSize} is smaller than the remaining size of the buffer (${waveData.length - 8})`, - ); - } + if (riffChunkSize < waveData.length - 8) { + throw new Error( + `RIFF chunk length ${riffChunkSize} is smaller than the remaining size of the buffer (${waveData.length - 8})` + ); + } - if (!ignoreTruncatedChunks && riffChunkSize > waveData.length - 8) { - throw new Error( - `RIFF chunk length (${riffChunkSize}) is greater than the remaining size of the buffer (${waveData.length - 8})`, - ); - } + if (!ignoreTruncatedChunks && riffChunkSize > waveData.length - 8) { + throw new Error( + `RIFF chunk length (${riffChunkSize}) is greater than the remaining size of the buffer (${waveData.length - 8})` + ); + } - readOffset += 4; + readOffset += 4; - let formatSubChunkBodyBuffer: Buffer | undefined; - const dataBuffers: Buffer[] = []; + let formatSubChunkBodyBuffer: Buffer | undefined; + const dataBuffers: Buffer[] = []; - while (true) { - const subChunkIdentifier = waveData - .subarray(readOffset, readOffset + 4) - .toString("ascii"); - readOffset += 4; + while (true) { + const subChunkIdentifier = waveData + .subarray(readOffset, readOffset + 4) + .toString("ascii"); + readOffset += 4; - let subChunkSize = waveData.readUInt32LE(readOffset); - readOffset += 4; + let subChunkSize = waveData.readUInt32LE(readOffset); + readOffset += 4; - if (!ignoreTruncatedChunks && subChunkSize > waveData.length - readOffset) { - throw new Error( - `Encountered a '${subChunkIdentifier}' subchunk with a size of ${subChunkSize} which is greater than the remaining size of the buffer (${waveData.length - readOffset})`, - ); - } + if ( + !ignoreTruncatedChunks && + subChunkSize > waveData.length - readOffset + ) { + throw new Error( + `Encountered a '${subChunkIdentifier}' subchunk with a size of ${subChunkSize} which is greater than the remaining size of the buffer (${waveData.length - readOffset})` + ); + } - if (subChunkIdentifier == "fmt ") { - formatSubChunkBodyBuffer = waveData.subarray( - readOffset, - readOffset + subChunkSize, - ); - } else if (subChunkIdentifier == "data") { - if (!formatSubChunkBodyBuffer) { - throw new Error( - "A data subchunk was encountered before a format subchunk", - ); - } + if (subChunkIdentifier == "fmt ") { + formatSubChunkBodyBuffer = waveData.subarray( + readOffset, + readOffset + subChunkSize + ); + } else if (subChunkIdentifier == "data") { + if (!formatSubChunkBodyBuffer) { + throw new Error( + "A data subchunk was encountered before a format subchunk" + ); + } - // If the data chunk is truncated or extended beyond 4 GiB, - // the data would be read up to the end of the buffer - if (ignoreOverflowingDataChunks && subChunkSize === 4294967295) { - subChunkSize = waveData.length - readOffset; - } + // If the data chunk is truncated or extended beyond 4 GiB, + // the data would be read up to the end of the buffer + if (ignoreOverflowingDataChunks && subChunkSize === 4294967295) { + subChunkSize = waveData.length - readOffset; + } - const subChunkData = waveData.subarray( - readOffset, - readOffset + subChunkSize, - ); + const subChunkData = waveData.subarray( + readOffset, + readOffset + subChunkSize + ); - dataBuffers.push(subChunkData); - } - // All sub chunks other than 'data' (e.g. 'LIST', 'fact', 'plst', 'junk' etc.) are ignored + dataBuffers.push(subChunkData); + } + // All sub chunks other than 'data' (e.g. 'LIST', 'fact', 'plst', 'junk' etc.) are ignored - // This addition operation may overflow if JavaScript integers were 32 bits, - // but since they are 52 bits, it is okay: - readOffset += subChunkSize; + // This addition operation may overflow if JavaScript integers were 32 bits, + // but since they are 52 bits, it is okay: + readOffset += subChunkSize; - // Break if readOffset is equal to or is greater than the size of the buffer - if (readOffset >= waveData.length) { - break; + // Break if readOffset is equal to or is greater than the size of the buffer + if (readOffset >= waveData.length) { + break; + } } - } - if (!formatSubChunkBodyBuffer) { - throw new Error("No format subchunk was found in the wave file"); - } + if (!formatSubChunkBodyBuffer) { + throw new Error("No format subchunk was found in the wave file"); + } - if (dataBuffers.length === 0) { - throw new Error("No data subchunks were found in the wave file"); - } + if (dataBuffers.length === 0) { + throw new Error("No data subchunks were found in the wave file"); + } - const waveFormat = WaveFormat.deserializeFrom(formatSubChunkBodyBuffer); + const waveFormat = WaveFormat.deserializeFrom(formatSubChunkBodyBuffer); - const sampleFormat = waveFormat.sampleFormat; - const channelCount = waveFormat.channelCount; - const sampleRate = waveFormat.sampleRate; - const bitDepth = waveFormat.bitDepth; - const speakerPositionMask = waveFormat.speakerPositionMask; + const sampleFormat = waveFormat.sampleFormat; + const channelCount = waveFormat.channelCount; + const sampleRate = waveFormat.sampleRate; + const bitDepth = waveFormat.bitDepth; + const speakerPositionMask = waveFormat.speakerPositionMask; - const concatenatedDataBuffers = Buffer.concat(dataBuffers); - dataBuffers.length = 0; // Allow the garbage collector to free up memory held by the data buffers + const concatenatedDataBuffers = Buffer.concat(dataBuffers); + dataBuffers.length = 0; // Allow the garbage collector to free up memory held by the data buffers - const audioChannels = decodeAudioBufferToChannels( - concatenatedDataBuffers, - channelCount, - bitDepth, - sampleFormat, - ); + const audioChannels = decodeAudioBufferToChannels( + concatenatedDataBuffers, + channelCount, + bitDepth, + sampleFormat + ); - return { - rawAudio: { audioChannels, sampleRate }, + return { + rawAudio: { audioChannels, sampleRate }, - sourceSampleFormat: sampleFormat, - sourceBitDepth: bitDepth, - sourceSpeakerPositionMask: speakerPositionMask, - }; + sourceSampleFormat: sampleFormat, + sourceBitDepth: bitDepth, + sourceSpeakerPositionMask: speakerPositionMask, + }; } // Int8 PCM <-> Float32 conversion function int8PcmToFloat32(input: Int8Array) { - const output = new Float32Array(input.length); + const output = new Float32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = input[i]; - output[i] = sample < 0 ? sample / 128 : sample / 127; - } + for (let i = 0; i < input.length; i++) { + const sample = input[i]; + output[i] = sample < 0 ? sample / 128 : sample / 127; + } - return output; + return output; } // Int24 PCM <-> Float32 conversion (uses int32 for storage) function int24PcmToFloat32(input: Int32Array) { - const output = new Float32Array(input.length); + const output = new Float32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = input[i]; - output[i] = sample < 0 ? sample / 8388608 : sample / 8388607; - } + for (let i = 0; i < input.length; i++) { + const sample = input[i]; + output[i] = sample < 0 ? sample / 8388608 : sample / 8388607; + } - return output; + return output; } function int32PcmToFloat32(input: Int32Array) { - const output = new Float32Array(input.length); + const output = new Float32Array(input.length); - for (let i = 0; i < input.length; i++) { - const sample = input[i]; - output[i] = sample < 0 ? sample / 2147483648 : sample / 2147483647; - } + for (let i = 0; i < input.length; i++) { + const sample = input[i]; + output[i] = sample < 0 ? sample / 2147483648 : sample / 2147483647; + } - return output; + return output; } function decodeAudioBufferToChannels( - audioBuffer: Buffer, - channelCount: number, - sourceBitDepth: number, - sourceSampleFormat: SampleFormat, + audioBuffer: Buffer, + channelCount: number, + sourceBitDepth: number, + sourceSampleFormat: SampleFormat ) { - let interleavedChannels: Float32Array; - - if (sourceSampleFormat === SampleFormat.PCM) { - if (sourceBitDepth === 8) { - interleavedChannels = int8PcmToFloat32(bufferToInt8(audioBuffer)); - } else if (sourceBitDepth === 16) { - interleavedChannels = int16PcmToFloat32(bufferLEToInt16(audioBuffer)); - } else if (sourceBitDepth === 24) { - interleavedChannels = int24PcmToFloat32(bufferLEToInt24(audioBuffer)); - } else if (sourceBitDepth === 32) { - interleavedChannels = int32PcmToFloat32(bufferLEToInt32(audioBuffer)); - } else { - throw new Error(`Unsupported PCM bit depth: ${sourceBitDepth}`); - } - } else if (sourceSampleFormat === SampleFormat.Float) { - if (sourceBitDepth === 32) { - interleavedChannels = bufferLEToFloat32(audioBuffer); - } else if (sourceBitDepth === 64) { - interleavedChannels = float64Tofloat32(bufferLEToFloat64(audioBuffer)); - } else { - throw new Error(`Unsupported float bit depth: ${sourceBitDepth}`); - } - } else if (sourceSampleFormat === SampleFormat.Alaw) { - if (sourceBitDepth === 8) { - interleavedChannels = int16PcmToFloat32( - AlawMulaw.alaw.decode(audioBuffer), - ); - } else { - throw new Error(`Unsupported alaw bit depth: ${sourceBitDepth}`); - } - } else if (sourceSampleFormat === SampleFormat.Mulaw) { - if (sourceBitDepth === 8) { - interleavedChannels = int16PcmToFloat32( - AlawMulaw.mulaw.decode(audioBuffer), - ); + let interleavedChannels: Float32Array; + + if (sourceSampleFormat === SampleFormat.PCM) { + if (sourceBitDepth === 8) { + interleavedChannels = int8PcmToFloat32(bufferToInt8(audioBuffer)); + } else if (sourceBitDepth === 16) { + interleavedChannels = int16PcmToFloat32( + bufferLEToInt16(audioBuffer) + ); + } else if (sourceBitDepth === 24) { + interleavedChannels = int24PcmToFloat32( + bufferLEToInt24(audioBuffer) + ); + } else if (sourceBitDepth === 32) { + interleavedChannels = int32PcmToFloat32( + bufferLEToInt32(audioBuffer) + ); + } else { + throw new Error(`Unsupported PCM bit depth: ${sourceBitDepth}`); + } + } else if (sourceSampleFormat === SampleFormat.Float) { + if (sourceBitDepth === 32) { + interleavedChannels = bufferLEToFloat32(audioBuffer); + } else if (sourceBitDepth === 64) { + interleavedChannels = float64Tofloat32( + bufferLEToFloat64(audioBuffer) + ); + } else { + throw new Error(`Unsupported float bit depth: ${sourceBitDepth}`); + } + } else if (sourceSampleFormat === SampleFormat.Alaw) { + if (sourceBitDepth === 8) { + interleavedChannels = int16PcmToFloat32( + AlawMulaw.alaw.decode(audioBuffer) + ); + } else { + throw new Error(`Unsupported alaw bit depth: ${sourceBitDepth}`); + } + } else if (sourceSampleFormat === SampleFormat.Mulaw) { + if (sourceBitDepth === 8) { + interleavedChannels = int16PcmToFloat32( + AlawMulaw.mulaw.decode(audioBuffer) + ); + } else { + throw new Error(`Unsupported mulaw bit depth: ${sourceBitDepth}`); + } } else { - throw new Error(`Unsupported mulaw bit depth: ${sourceBitDepth}`); + throw new Error(`Unsupported audio format: ${sourceSampleFormat}`); } - } else { - throw new Error(`Unsupported audio format: ${sourceSampleFormat}`); - } - audioBuffer = Buffer.from([]); // Zero the buffer reference to allow the GC to free up memory, if possible + audioBuffer = Buffer.from([]); // Zero the buffer reference to allow the GC to free up memory, if possible - return deInterleaveChannels(interleavedChannels, channelCount); + return deInterleaveChannels(interleavedChannels, channelCount); } function deInterleaveChannels( - interleavedChannels: Float32Array, - channelCount: number, + interleavedChannels: Float32Array, + channelCount: number ) { - if (channelCount === 0) { - throw new Error("0 channel count received"); - } + if (channelCount === 0) { + throw new Error("0 channel count received"); + } - if (channelCount === 1) { - return [interleavedChannels]; - } + if (channelCount === 1) { + return [interleavedChannels]; + } - if (interleavedChannels.length % channelCount != 0) { - throw new Error( - `Size of interleaved channels (${interleaveChannels.length}) is not a multiple of channel count (${channelCount})`, - ); - } + if (interleavedChannels.length % channelCount != 0) { + throw new Error( + `Size of interleaved channels (${interleaveChannels.length}) is not a multiple of channel count (${channelCount})` + ); + } - const sampleCount = interleavedChannels.length / channelCount; - const channels: Float32Array[] = []; + const sampleCount = interleavedChannels.length / channelCount; + const channels: Float32Array[] = []; - for (let i = 0; i < channelCount; i++) { - channels.push(new Float32Array(sampleCount)); - } + for (let i = 0; i < channelCount; i++) { + channels.push(new Float32Array(sampleCount)); + } - let readIndex = 0; + let readIndex = 0; - for (let i = 0; i < sampleCount; i++) { - for (let c = 0; c < channelCount; c++) { - channels[c][i] = interleavedChannels[readIndex]; - readIndex += 1; + for (let i = 0; i < sampleCount; i++) { + for (let c = 0; c < channelCount; c++) { + channels[c][i] = interleavedChannels[readIndex]; + readIndex += 1; + } } - } - return channels; + return channels; } function decodeWaveToRawAudio( - waveFileBuffer: Buffer, - ignoreTruncatedChunks = true, - ignoreOverflowingDataChunks = true, + waveFileBuffer: Buffer, + ignoreTruncatedChunks = true, + ignoreOverflowingDataChunks = true ) { - return decodeWave( - waveFileBuffer, - ignoreTruncatedChunks, - ignoreOverflowingDataChunks, - ); + return decodeWave( + waveFileBuffer, + ignoreTruncatedChunks, + ignoreOverflowingDataChunks + ); } async function decodeToChannels( - input: string | Buffer, - outSampleRate?: number, - outChannelCount?: number, + input: string | Buffer, + outSampleRate?: number, + outChannelCount?: number ) { - const outputOptions: FFMpegOutputOptions = { - codec: "pcm_f32le", - format: "wav", - sampleRate: outSampleRate, - channelCount: outChannelCount, - audioOnly: true, - }; + const outputOptions: FFMpegOutputOptions = { + codec: "pcm_f32le", + format: "wav", + sampleRate: outSampleRate, + channelCount: outChannelCount, + audioOnly: true, + }; - const waveAudio = await transcode(input, outputOptions); + const waveAudio = await transcode(input, outputOptions); - const logger = new Logger(); + const logger = new Logger(); - logger.start(`Convert wave buffer to raw audio`); - const { rawAudio } = decodeWaveToRawAudio(waveAudio); - logger.end(); + logger.start(`Convert wave buffer to raw audio`); + const { rawAudio } = decodeWaveToRawAudio(waveAudio); + logger.end(); - return rawAudio; + return rawAudio; } async function align( - input: AudioSourceParam, - transcript: string, - options: AlignmentOptions, + input: AudioSourceParam, + transcript: string, + options: AlignmentOptions ): Promise { - const logger = new Logger(); + const logger = new Logger(); - const startTimestamp = logger.getTimestamp(); + const startTimestamp = logger.getTimestamp(); - options = extendDeep(defaultAlignmentOptions, options); + options = extendDeep(defaultAlignmentOptions, options); - const inputRawAudio = await ensureRawAudio(input); + const inputRawAudio = await ensureRawAudio(input); - let sourceRawAudio: RawAudio; - let isolatedRawAudio: RawAudio | undefined; - let backgroundRawAudio: RawAudio | undefined; + let sourceRawAudio: RawAudio; + let isolatedRawAudio: RawAudio | undefined; + let backgroundRawAudio: RawAudio | undefined; - logger.start(`Resample audio to 16kHz mono`); - sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1); + logger.start(`Resample audio to 16kHz mono`); + sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1); - let sourceUncropTimeline: Timeline | undefined; + let sourceUncropTimeline: Timeline | undefined; - logger.start("Normalize and trim audio"); + logger.start("Normalize and trim audio"); - sourceRawAudio = normalizeAudioLevel(sourceRawAudio); - sourceRawAudio.audioChannels[0] = trimAudioEnd( - sourceRawAudio.audioChannels[0], - ); + sourceRawAudio = normalizeAudioLevel(sourceRawAudio); + sourceRawAudio.audioChannels[0] = trimAudioEnd( + sourceRawAudio.audioChannels[0] + ); - logger.end(); + logger.end(); - let language: string; + let language: string; - if (options.language) { - const languageData = await parseLangIdentifier(options.language); + if (options.language) { + const languageData = await parseLangIdentifier(options.language); - language = languageData.Name; + language = languageData.Name; - logger.logTitledMessage( - "Language specified", - formatLanguageCodeWithName(language), - ); - } else { - logger.start("No language specified. Detect language using reference text"); - const { detectedLanguage } = await detectTextLanguage( - transcript, - options.languageDetection || {}, - ); + logger.logTitledMessage( + "Language specified", + formatLanguageCodeWithName(language) + ); + } else { + logger.start( + "No language specified. Detect language using reference text" + ); + const { detectedLanguage } = await detectTextLanguage( + transcript, + options.languageDetection || {} + ); - language = detectedLanguage; + language = detectedLanguage; - logger.end(); + logger.end(); - logger.logTitledMessage( - "Language detected", - formatLanguageCodeWithName(language), - ); - } + logger.logTitledMessage( + "Language detected", + formatLanguageCodeWithName(language) + ); + } - language = getDefaultDialectForLanguageCodeIfPossible(language); + language = getDefaultDialectForLanguageCodeIfPossible(language); - logger.start("Load alignment module"); + logger.start("Load alignment module"); - function getDtwWindowGranularitiesAndDurations() { - const sourceAudioDuration = getRawAudioDuration(sourceRawAudio); + function getDtwWindowGranularitiesAndDurations() { + const sourceAudioDuration = getRawAudioDuration(sourceRawAudio); - const granularities: DtwGranularity[] = ["high"]; - let windowDurations: number[]; + const granularities: DtwGranularity[] = ["high"]; + let windowDurations: number[]; - if (options.dtw!.windowDuration) { - if (typeof options.dtw!.windowDuration === "number") { - windowDurations = [options.dtw!.windowDuration]; - } else if (Array.isArray(options.dtw!.windowDuration)) { - windowDurations = options.dtw!.windowDuration; - } else { - throw new Error( - `'dtw.windowDuration' must be a number or an array of numbers.`, - ); - } - } else { - if (granularities.length > 2) { - throw new Error( - `More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=[600,60,10]'.`, - ); - } + if (options.dtw!.windowDuration) { + if (typeof options.dtw!.windowDuration === "number") { + windowDurations = [options.dtw!.windowDuration]; + } else if (Array.isArray(options.dtw!.windowDuration)) { + windowDurations = options.dtw!.windowDuration; + } else { + throw new Error( + `'dtw.windowDuration' must be a number or an array of numbers.` + ); + } + } else { + if (granularities.length > 2) { + throw new Error( + `More than two passes requested, this requires window durations to be explicitly specified for each pass. For example 'dtw.windowDuration=[600,60,10]'.` + ); + } - if (sourceAudioDuration < 5 * 60) { - // If up to 5 minutes, set window duration to one minute - windowDurations = [60]; - } else if (sourceAudioDuration < 2.5 * 60 * 60) { - // If less than 2.5 hours, set window duration to 20% of total duration - windowDurations = [Math.ceil(sourceAudioDuration * 0.2)]; - } else { - // Otherwise, set window duration to 30 minutes - windowDurations = [30 * 60]; - } - } + if (sourceAudioDuration < 5 * 60) { + // If up to 5 minutes, set window duration to one minute + windowDurations = [60]; + } else if (sourceAudioDuration < 2.5 * 60 * 60) { + // If less than 2.5 hours, set window duration to 20% of total duration + windowDurations = [Math.ceil(sourceAudioDuration * 0.2)]; + } else { + // Otherwise, set window duration to 30 minutes + windowDurations = [30 * 60]; + } + } - if (granularities.length === 2 && windowDurations.length === 1) { - windowDurations = [windowDurations[0], 15]; - } + if (granularities.length === 2 && windowDurations.length === 1) { + windowDurations = [windowDurations[0], 15]; + } - if (granularities.length != windowDurations.length) { - throw new Error( - `The option 'dtw.granularity' has ${granularities.length} values, but 'dtw.windowDuration' has ${windowDurations.length} values. The lengths should be equal.`, - ); - } + if (granularities.length != windowDurations.length) { + throw new Error( + `The option 'dtw.granularity' has ${granularities.length} values, but 'dtw.windowDuration' has ${windowDurations.length} values. The lengths should be equal.` + ); + } - return { windowDurations, granularities }; - } + return { windowDurations, granularities }; + } - let mappedTimeline: Timeline; + let mappedTimeline: Timeline; - const { windowDurations, granularities } = - getDtwWindowGranularitiesAndDurations(); + const { windowDurations, granularities } = + getDtwWindowGranularitiesAndDurations(); - logger.end(); + logger.end(); - const { referenceRawAudio, referenceTimeline } = - await createAlignmentReferenceUsingEspeak( - transcript, - language, - options.plainText, - options.customLexiconPaths, - false, - false, - ); + const { referenceRawAudio, referenceTimeline } = + await createAlignmentReferenceUsingEspeak( + transcript, + language, + options.plainText, + options.customLexiconPaths, + false, + false + ); - logger.end(); + logger.end(); - mappedTimeline = await alignUsingDtw( - sourceRawAudio, - referenceRawAudio, - referenceTimeline, - granularities, - windowDurations, - ); + mappedTimeline = await alignUsingDtw( + sourceRawAudio, + referenceRawAudio, + referenceTimeline, + granularities, + windowDurations + ); - logger.start(`Postprocess timeline`); + logger.start(`Postprocess timeline`); - // If the audio was cropped before recognition, map the timestamps back to the original audio - if (sourceUncropTimeline && sourceUncropTimeline.length > 0) { - convertCroppedToUncroppedTimeline(mappedTimeline, sourceUncropTimeline); - } + // If the audio was cropped before recognition, map the timestamps back to the original audio + if (sourceUncropTimeline && sourceUncropTimeline.length > 0) { + convertCroppedToUncroppedTimeline(mappedTimeline, sourceUncropTimeline); + } - // Add text offsets - addWordTextOffsetsToTimeline(mappedTimeline, transcript); + // Add text offsets + addWordTextOffsetsToTimeline(mappedTimeline, transcript); - // Make segment timeline - const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline( - mappedTimeline, - transcript, - language, - options.plainText?.paragraphBreaks, - options.plainText?.whitespace, - ); + // Make segment timeline + const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline( + mappedTimeline, + transcript, + language, + options.plainText?.paragraphBreaks, + options.plainText?.whitespace + ); - logger.end(); + logger.end(); - return { - timeline: segmentTimeline, - wordTimeline: mappedTimeline, + return { + timeline: segmentTimeline, + wordTimeline: mappedTimeline, - transcript, - language, + transcript, + language, - inputRawAudio, - isolatedRawAudio, - backgroundRawAudio, - }; + inputRawAudio, + isolatedRawAudio, + backgroundRawAudio, + }; } function getDefaultDialectForLanguageCodeIfPossible(langCode: string) { - const defaultDialect = defaultDialectForLanguageCode[langCode]; + const defaultDialect = defaultDialectForLanguageCode[langCode]; - return defaultDialect || langCode; + return defaultDialect || langCode; } async function wordTimelineToSegmentSentenceTimeline( - wordTimeline: Timeline, - transcript: string, - language: string, - paragraphBreaks: ParagraphBreakType = "double", - whitespace: WhitespaceProcessing = "collapse", + wordTimeline: Timeline, + transcript: string, + language: string, + paragraphBreaks: ParagraphBreakType = "double", + whitespace: WhitespaceProcessing = "collapse" ) { - const paragraphs = splitToParagraphs(transcript, paragraphBreaks, whitespace); - - const segments = paragraphs.map((segment) => - splitToSentences(segment, language).map((sentence) => sentence.trim()), - ); + const paragraphs = splitToParagraphs( + transcript, + paragraphBreaks, + whitespace + ); - let text = ""; - const charIndexToSentenceEntryMapping: TimelineEntry[] = []; + const segments = paragraphs.map((segment) => + splitToSentences(segment, language).map((sentence) => sentence.trim()) + ); - const segmentTimeline: Timeline = []; + let text = ""; + const charIndexToSentenceEntryMapping: TimelineEntry[] = []; + + const segmentTimeline: Timeline = []; + + for (const segment of segments) { + const sentencesInSegment: Timeline = []; + + const segmentEntry: TimelineEntry = { + type: "segment", + text: "", + startTime: -1, + endTime: -1, + timeline: sentencesInSegment, + }; + + for (const sentence of segment) { + const sentenceEntry: TimelineEntry = { + type: "sentence", + text: sentence, + startTime: -1, + endTime: -1, + timeline: [], + }; + + for (const char of sentence + " ") { + text += char; + charIndexToSentenceEntryMapping.push(sentenceEntry); + } - for (const segment of segments) { - const sentencesInSegment: Timeline = []; + sentencesInSegment.push(sentenceEntry); + } - const segmentEntry: TimelineEntry = { - type: "segment", - text: "", - startTime: -1, - endTime: -1, - timeline: sentencesInSegment, - }; + segmentTimeline.push(segmentEntry); + } - for (const sentence of segment) { - const sentenceEntry: TimelineEntry = { - type: "sentence", - text: sentence, - startTime: -1, - endTime: -1, - timeline: [], - }; + let wordSearchStartOffset = 0; - for (const char of sentence + " ") { - text += char; - charIndexToSentenceEntryMapping.push(sentenceEntry); - } + for (let wordIndex = 0; wordIndex < wordTimeline.length; wordIndex++) { + const wordEntry = wordTimeline[wordIndex]; + const wordText = wordEntry.text; - sentencesInSegment.push(sentenceEntry); - } + if (!isWordOrSymbolWord(wordText)) { + continue; + } - segmentTimeline.push(segmentEntry); - } + const indexOfWordInText = text.indexOf(wordText, wordSearchStartOffset); - let wordSearchStartOffset = 0; + if (indexOfWordInText == -1) { + throw new Error( + `Couldn't find the word '${wordText}' in the text at start position ${wordSearchStartOffset}` + ); + } - for (let wordIndex = 0; wordIndex < wordTimeline.length; wordIndex++) { - const wordEntry = wordTimeline[wordIndex]; - const wordText = wordEntry.text; + const targetSentenceEntry = + charIndexToSentenceEntryMapping[indexOfWordInText]; + targetSentenceEntry.timeline!.push(deepClone(wordEntry)); - if (!isWordOrSymbolWord(wordText)) { - continue; + wordSearchStartOffset = indexOfWordInText + wordText.length; } - const indexOfWordInText = text.indexOf(wordText, wordSearchStartOffset); - - if (indexOfWordInText == -1) { - throw new Error( - `Couldn't find the word '${wordText}' in the text at start position ${wordSearchStartOffset}`, - ); - } + const newSegmentTimeline: Timeline = []; - const targetSentenceEntry = - charIndexToSentenceEntryMapping[indexOfWordInText]; - targetSentenceEntry.timeline!.push(deepClone(wordEntry)); + for (const segmentEntry of segmentTimeline) { + const oldSentenceTimeline = segmentEntry.timeline!; - wordSearchStartOffset = indexOfWordInText + wordText.length; - } + const newSentenceTimeline: Timeline = []; - const newSegmentTimeline: Timeline = []; + for (const sentenceEntry of oldSentenceTimeline) { + const wordTimeline = sentenceEntry.timeline; - for (const segmentEntry of segmentTimeline) { - const oldSentenceTimeline = segmentEntry.timeline!; + if (!wordTimeline || wordTimeline.length == 0) { + continue; + } - const newSentenceTimeline: Timeline = []; + sentenceEntry.startTime = wordTimeline[0].startTime; + sentenceEntry.endTime = + wordTimeline[wordTimeline.length - 1].endTime; - for (const sentenceEntry of oldSentenceTimeline) { - const wordTimeline = sentenceEntry.timeline; + newSentenceTimeline.push(sentenceEntry); + } - if (!wordTimeline || wordTimeline.length == 0) { - continue; - } + if (newSentenceTimeline.length == 0) { + continue; + } - sentenceEntry.startTime = wordTimeline[0].startTime; - sentenceEntry.endTime = wordTimeline[wordTimeline.length - 1].endTime; + segmentEntry.text = newSentenceTimeline + .map((sentenceEntry) => sentenceEntry.text) + .join(" "); - newSentenceTimeline.push(sentenceEntry); - } + segmentEntry.startTime = newSentenceTimeline[0].startTime; + segmentEntry.endTime = + newSentenceTimeline[newSentenceTimeline.length - 1].endTime; - if (newSentenceTimeline.length == 0) { - continue; + newSegmentTimeline.push(segmentEntry); } - segmentEntry.text = newSentenceTimeline - .map((sentenceEntry) => sentenceEntry.text) - .join(" "); - - segmentEntry.startTime = newSentenceTimeline[0].startTime; - segmentEntry.endTime = - newSentenceTimeline[newSentenceTimeline.length - 1].endTime; - - newSegmentTimeline.push(segmentEntry); - } - - return { segmentTimeline: newSegmentTimeline }; + return { segmentTimeline: newSegmentTimeline }; } interface UncropTimelineMapResult { - mappedStartTime: number; - mappedEndTime: number; + mappedStartTime: number; + mappedEndTime: number; } function mapUsingUncropTimeline( - startTimeInCroppedAudio: number, - endTimeInCroppedAudio: number, - uncropTimeline: Timeline, + startTimeInCroppedAudio: number, + endTimeInCroppedAudio: number, + uncropTimeline: Timeline ): UncropTimelineMapResult { - if (uncropTimeline.length === 0) { - return { - mappedStartTime: 0, - mappedEndTime: 0, - }; - } + if (uncropTimeline.length === 0) { + return { + mappedStartTime: 0, + mappedEndTime: 0, + }; + } - let offsetInCroppedAudio = 0; + let offsetInCroppedAudio = 0; - if (endTimeInCroppedAudio < startTimeInCroppedAudio) { - endTimeInCroppedAudio = startTimeInCroppedAudio; - } + if (endTimeInCroppedAudio < startTimeInCroppedAudio) { + endTimeInCroppedAudio = startTimeInCroppedAudio; + } - let bestOverlapDuration = -1; - let mappedStartTime = -1; - let mappedEndTime = -1; + let bestOverlapDuration = -1; + let mappedStartTime = -1; + let mappedEndTime = -1; - for (const uncropEntry of uncropTimeline) { - const uncropEntryDuration = uncropEntry.endTime - uncropEntry.startTime; + for (const uncropEntry of uncropTimeline) { + const uncropEntryDuration = uncropEntry.endTime - uncropEntry.startTime; - const overlapStartTime = Math.max( - startTimeInCroppedAudio, - offsetInCroppedAudio, - ); - const overlapEndTime = Math.min( - endTimeInCroppedAudio, - offsetInCroppedAudio + uncropEntryDuration, - ); + const overlapStartTime = Math.max( + startTimeInCroppedAudio, + offsetInCroppedAudio + ); + const overlapEndTime = Math.min( + endTimeInCroppedAudio, + offsetInCroppedAudio + uncropEntryDuration + ); - const overlapDuration = overlapEndTime - overlapStartTime; + const overlapDuration = overlapEndTime - overlapStartTime; - if (overlapDuration >= 0 && overlapDuration > bestOverlapDuration) { - bestOverlapDuration = overlapDuration; + if (overlapDuration >= 0 && overlapDuration > bestOverlapDuration) { + bestOverlapDuration = overlapDuration; - mappedStartTime = - uncropEntry.startTime + (overlapStartTime - offsetInCroppedAudio); - mappedEndTime = - uncropEntry.startTime + (overlapEndTime - offsetInCroppedAudio); - } + mappedStartTime = + uncropEntry.startTime + + (overlapStartTime - offsetInCroppedAudio); + mappedEndTime = + uncropEntry.startTime + (overlapEndTime - offsetInCroppedAudio); + } - offsetInCroppedAudio += uncropEntryDuration; - } + offsetInCroppedAudio += uncropEntryDuration; + } - if (bestOverlapDuration === -1) { - if (startTimeInCroppedAudio >= offsetInCroppedAudio) { - const maxTimestamp = uncropTimeline[uncropTimeline.length - 1].endTime; + if (bestOverlapDuration === -1) { + if (startTimeInCroppedAudio >= offsetInCroppedAudio) { + const maxTimestamp = + uncropTimeline[uncropTimeline.length - 1].endTime; - return { - mappedStartTime: maxTimestamp, - mappedEndTime: maxTimestamp, - }; - } else { - throw new Error( - `Given start time ${startTimeInCroppedAudio} was smaller than audio duration but no match was found in uncrop timeline (should not occur)`, - ); + return { + mappedStartTime: maxTimestamp, + mappedEndTime: maxTimestamp, + }; + } else { + throw new Error( + `Given start time ${startTimeInCroppedAudio} was smaller than audio duration but no match was found in uncrop timeline (should not occur)` + ); + } } - } - return { - mappedStartTime, - mappedEndTime, - }; + return { + mappedStartTime, + mappedEndTime, + }; } function convertCroppedToUncroppedTimeline( - timeline: Timeline, - uncropTimeline: Timeline, + timeline: Timeline, + uncropTimeline: Timeline ) { - if (timeline.length === 0) { - return; - } - - for (const entry of timeline) { - const { mappedStartTime, mappedEndTime } = mapUsingUncropTimeline( - entry.startTime, - entry.endTime, - uncropTimeline, - ); - - const mapSubTimeline = (subTimeline: Timeline | undefined) => { - if (!subTimeline) { + if (timeline.length === 0) { return; - } + } - for (const subEntry of subTimeline) { - subEntry.startTime = Math.min( - mappedStartTime + (subEntry.startTime - entry.startTime), - mappedEndTime, - ); - subEntry.endTime = Math.min( - mappedStartTime + (subEntry.endTime - entry.startTime), - mappedEndTime, + for (const entry of timeline) { + const { mappedStartTime, mappedEndTime } = mapUsingUncropTimeline( + entry.startTime, + entry.endTime, + uncropTimeline ); - mapSubTimeline(subEntry.timeline); - } - }; + const mapSubTimeline = (subTimeline: Timeline | undefined) => { + if (!subTimeline) { + return; + } - mapSubTimeline(entry.timeline); + for (const subEntry of subTimeline) { + subEntry.startTime = Math.min( + mappedStartTime + (subEntry.startTime - entry.startTime), + mappedEndTime + ); + subEntry.endTime = Math.min( + mappedStartTime + (subEntry.endTime - entry.startTime), + mappedEndTime + ); + + mapSubTimeline(subEntry.timeline); + } + }; + + mapSubTimeline(entry.timeline); - entry.startTime = mappedStartTime; - entry.endTime = mappedEndTime; - } + entry.startTime = mappedStartTime; + entry.endTime = mappedEndTime; + } } function getUTF32Chars(str: string) { - const utf32chars: string[] = []; - const mapping: number[] = []; + const utf32chars: string[] = []; + const mapping: number[] = []; - let utf32Index = 0; + let utf32Index = 0; - for (const utf32char of str) { - utf32chars.push(utf32char); + for (const utf32char of str) { + utf32chars.push(utf32char); - for (let i = 0; i < utf32char.length; i++) { - mapping.push(utf32Index); - } + for (let i = 0; i < utf32char.length; i++) { + mapping.push(utf32Index); + } - utf32Index += 1; - } + utf32Index += 1; + } - mapping.push(utf32Index); + mapping.push(utf32Index); - return { utf32chars, mapping }; + return { utf32chars, mapping }; } function addWordTextOffsetsToTimeline( - timeline: Timeline, - text: string, - currentOffset = 0, + timeline: Timeline, + text: string, + currentOffset = 0 ) { - const { mapping } = getUTF32Chars(text); - - for (const entry of timeline) { - if (entry.type == "word") { - let word = entry.text; + const { mapping } = getUTF32Chars(text); - word = word.trim().replaceAll(/\s+/g, " "); + for (const entry of timeline) { + if (entry.type == "word") { + let word = entry.text; - const wordParts = word.split(" "); + word = word.trim().replaceAll(/\s+/g, " "); - let startOffset: number | undefined; - let endOffset: number | undefined; + const wordParts = word.split(" "); - for (let i = 0; i < wordParts.length; i++) { - const wordPart = wordParts[i]; + let startOffset: number | undefined; + let endOffset: number | undefined; - const wordPartOffset = text.indexOf(wordPart, currentOffset); + for (let i = 0; i < wordParts.length; i++) { + const wordPart = wordParts[i]; - if (wordPartOffset == -1) { - continue; - } + const wordPartOffset = text.indexOf(wordPart, currentOffset); - currentOffset = wordPartOffset + wordParts[i].length; + if (wordPartOffset == -1) { + continue; + } - if (i == 0) { - startOffset = wordPartOffset; - } + currentOffset = wordPartOffset + wordParts[i].length; - endOffset = currentOffset; - } + if (i == 0) { + startOffset = wordPartOffset; + } - entry.startOffsetUtf16 = startOffset; - entry.endOffsetUtf16 = endOffset; + endOffset = currentOffset; + } - entry.startOffsetUtf32 = - startOffset != undefined ? mapping[startOffset] : undefined; - entry.endOffsetUtf32 = - endOffset != undefined ? mapping[endOffset] : undefined; - } else if (entry.timeline) { - currentOffset = addWordTextOffsetsToTimeline( - entry.timeline, - text, - currentOffset, - ); + entry.startOffsetUtf16 = startOffset; + entry.endOffsetUtf16 = endOffset; + + entry.startOffsetUtf32 = + startOffset != undefined ? mapping[startOffset] : undefined; + entry.endOffsetUtf32 = + endOffset != undefined ? mapping[endOffset] : undefined; + } else if (entry.timeline) { + currentOffset = addWordTextOffsetsToTimeline( + entry.timeline, + text, + currentOffset + ); + } } - } - return currentOffset; + return currentOffset; } async function createAlignmentReferenceUsingEspeak( - transcript: string, - language: string, - plaintextOptions?: PlainTextOptions, - customLexiconPaths?: string[], - insertSeparators?: boolean, - useKlatt?: boolean, + transcript: string, + language: string, + plaintextOptions?: PlainTextOptions, + customLexiconPaths?: string[], + insertSeparators?: boolean, + useKlatt?: boolean ) { - const logger = new Logger(); + const logger = new Logger(); - logger.start("Synthesize alignment reference with eSpeak"); + logger.start("Synthesize alignment reference with eSpeak"); - const synthesisOptions: SynthesisOptions = { - engine: "espeak", - language, + const synthesisOptions: SynthesisOptions = { + engine: "espeak", + language, - plainText: plaintextOptions, - customLexiconPaths: customLexiconPaths, - }; + plainText: plaintextOptions, + customLexiconPaths: customLexiconPaths, + }; - let { - audio: referenceRawAudio, - timeline: segmentTimeline, - voice: espeakVoice, - } = await synthesize(transcript, synthesisOptions); + let { + audio: referenceRawAudio, + timeline: segmentTimeline, + voice: espeakVoice, + } = await synthesize(transcript, synthesisOptions); - const sentenceTimeline = segmentTimeline.flatMap((entry) => entry.timeline!); - const wordTimeline = sentenceTimeline.flatMap((entry) => entry.timeline!); + const sentenceTimeline = segmentTimeline.flatMap( + (entry) => entry.timeline! + ); + const wordTimeline = sentenceTimeline.flatMap((entry) => entry.timeline!); - referenceRawAudio = await resampleAudioSpeex( - referenceRawAudio as RawAudio, - 16000, - ); - referenceRawAudio = downmixToMonoAndNormalize(referenceRawAudio); + referenceRawAudio = await resampleAudioSpeex( + referenceRawAudio as RawAudio, + 16000 + ); + referenceRawAudio = downmixToMonoAndNormalize(referenceRawAudio); - logger.end(); + logger.end(); - return { referenceRawAudio, referenceTimeline: wordTimeline, espeakVoice }; + return { referenceRawAudio, referenceTimeline: wordTimeline, espeakVoice }; } function downmixToMonoAndNormalize( - rawAudio: RawAudio, - targetPeakDecibels = -3, + rawAudio: RawAudio, + targetPeakDecibels = -3 ) { - return normalizeAudioLevel(downmixToMono(rawAudio), targetPeakDecibels); + return normalizeAudioLevel(downmixToMono(rawAudio), targetPeakDecibels); } async function loadLexiconFile(jsonFilePath: string): Promise { - const parsedLexicon: Lexicon = await readAndParseJsonFile(jsonFilePath); + const parsedLexicon: Lexicon = await readAndParseJsonFile(jsonFilePath); - return parsedLexicon; + return parsedLexicon; } async function loadLexiconsForLanguage( - language: string, - customLexiconPaths?: string[], + language: string, + customLexiconPaths?: string[] ) { - const lexicons: Lexicon[] = []; + const lexicons: Lexicon[] = []; - if (getShortLanguageCode(language) == "en") { - const heteronymsLexicon = await loadLexiconFile( - resolveToModuleRootDir("data/lexicons/heteronyms.en.json"), - ); - lexicons.push(heteronymsLexicon); - } + if (getShortLanguageCode(language) == "en") { + const heteronymsLexicon = await loadLexiconFile( + resolveToModuleRootDir("data/lexicons/heteronyms.en.json") + ); + lexicons.push(heteronymsLexicon); + } - if (customLexiconPaths && customLexiconPaths.length > 0) { - for (const customLexicon of customLexiconPaths) { - const customLexiconObject = await loadLexiconFile(customLexicon); + if (customLexiconPaths && customLexiconPaths.length > 0) { + for (const customLexicon of customLexiconPaths) { + const customLexiconObject = await loadLexiconFile(customLexicon); - lexicons.push(customLexiconObject); + lexicons.push(customLexiconObject); + } } - } - return lexicons; + return lexicons; } function downmixToMono(rawAudio: RawAudio): RawAudio { - const channelCount = rawAudio.audioChannels.length; - const sampleCount = rawAudio.audioChannels[0].length; + const channelCount = rawAudio.audioChannels.length; + const sampleCount = rawAudio.audioChannels[0].length; - if (channelCount === 1) { - return cloneRawAudio(rawAudio); - } + if (channelCount === 1) { + return cloneRawAudio(rawAudio); + } - const downmixedAudio = new Float32Array(sampleCount); + const downmixedAudio = new Float32Array(sampleCount); - for (const channelSamples of rawAudio.audioChannels) { - for (let i = 0; i < sampleCount; i++) { - downmixedAudio[i] += channelSamples[i]; + for (const channelSamples of rawAudio.audioChannels) { + for (let i = 0; i < sampleCount; i++) { + downmixedAudio[i] += channelSamples[i]; + } } - } - if (channelCount > 1) { - for (let i = 0; i < sampleCount; i++) { - downmixedAudio[i] /= channelCount; + if (channelCount > 1) { + for (let i = 0; i < sampleCount; i++) { + downmixedAudio[i] /= channelCount; + } } - } - return { - audioChannels: [downmixedAudio], - sampleRate: rawAudio.sampleRate, - } as RawAudio; + return { + audioChannels: [downmixedAudio], + sampleRate: rawAudio.sampleRate, + } as RawAudio; } function applyGainFactor(rawAudio: RawAudio, gainFactor: number): RawAudio { - const outputAudioChannels: Float32Array[] = []; + const outputAudioChannels: Float32Array[] = []; - for (const channelSamples of rawAudio.audioChannels) { - const sampleCount = channelSamples.length; + for (const channelSamples of rawAudio.audioChannels) { + const sampleCount = channelSamples.length; - const outputChannelSamples = new Float32Array(sampleCount); + const outputChannelSamples = new Float32Array(sampleCount); - for (let i = 0; i < sampleCount; i++) { - outputChannelSamples[i] = channelSamples[i] * gainFactor; - } + for (let i = 0; i < sampleCount; i++) { + outputChannelSamples[i] = channelSamples[i] * gainFactor; + } - outputAudioChannels.push(outputChannelSamples); - } + outputAudioChannels.push(outputChannelSamples); + } - return { - audioChannels: outputAudioChannels, - sampleRate: rawAudio.sampleRate, - } as RawAudio; + return { + audioChannels: outputAudioChannels, + sampleRate: rawAudio.sampleRate, + } as RawAudio; } function normalizeAudioLevel( - rawAudio: RawAudio, - targetPeakDecibels = -3, - maxGainIncreaseDecibels = 30, + rawAudio: RawAudio, + targetPeakDecibels = -3, + maxGainIncreaseDecibels = 30 ): RawAudio { - //rawAudio = correctDCBias(rawAudio) + //rawAudio = correctDCBias(rawAudio) - const targetPeakAmplitude = decibelsToGainFactor(targetPeakDecibels); - const maxGainFactor = decibelsToGainFactor(maxGainIncreaseDecibels); + const targetPeakAmplitude = decibelsToGainFactor(targetPeakDecibels); + const maxGainFactor = decibelsToGainFactor(maxGainIncreaseDecibels); - const peakAmplitude = getSamplePeakAmplitude(rawAudio.audioChannels); + const peakAmplitude = getSamplePeakAmplitude(rawAudio.audioChannels); - const gainFactor = Math.min( - targetPeakAmplitude / peakAmplitude, - maxGainFactor, - ); + const gainFactor = Math.min( + targetPeakAmplitude / peakAmplitude, + maxGainFactor + ); - return applyGainFactor(rawAudio, gainFactor); + return applyGainFactor(rawAudio, gainFactor); } function attenuateIfClipping(rawAudio: RawAudio) { - return normalizeAudioLevel(rawAudio, -0.1, 0); + return normalizeAudioLevel(rawAudio, -0.1, 0); } const symbolWords = [ - "$", - "€", - "¢", - "£", - "¥", - "©", - "®", - "™", - "%", - "&", - "#", - "~", - "@", - "+", - "±", - "÷", - "/", - "*", - "=", - "¼", - "½", - "¾", + "$", + "€", + "¢", + "£", + "¥", + "©", + "®", + "™", + "%", + "&", + "#", + "~", + "@", + "+", + "±", + "÷", + "/", + "*", + "=", + "¼", + "½", + "¾", ]; function isWord(str: string) { - return wordCharacterPattern.test(str.trim()); + return wordCharacterPattern.test(str.trim()); } function multiplyTimelineByFactor(targetTimeline: Timeline, factor: number) { - const newTimeline = deepClone(targetTimeline); + const newTimeline = deepClone(targetTimeline); - for (const segmentTimelineEntry of newTimeline) { - segmentTimelineEntry.startTime = segmentTimelineEntry.startTime * factor; - segmentTimelineEntry.endTime = segmentTimelineEntry.endTime * factor; + for (const segmentTimelineEntry of newTimeline) { + segmentTimelineEntry.startTime = + segmentTimelineEntry.startTime * factor; + segmentTimelineEntry.endTime = segmentTimelineEntry.endTime * factor; - if (segmentTimelineEntry.timeline) { - segmentTimelineEntry.timeline = multiplyTimelineByFactor( - segmentTimelineEntry.timeline, - factor, - ); + if (segmentTimelineEntry.timeline) { + segmentTimelineEntry.timeline = multiplyTimelineByFactor( + segmentTimelineEntry.timeline, + factor + ); + } } - } - return newTimeline; + return newTimeline; } function isWordOrSymbolWord(str: string) { - return isWord(str) || symbolWords.includes(str); + return isWord(str) || symbolWords.includes(str); } const defaultSynthesisOptions: SynthesisOptions = { - engine: "vits", - language: "en-us", + engine: "vits", + language: "en-us", - voice: undefined, - voiceGender: undefined, + voice: undefined, + voiceGender: undefined, - speed: 1.0, - pitch: 1.0, - pitchVariation: 1.0, + speed: 1.0, + pitch: 1.0, + pitchVariation: 1.0, - ssml: false, + ssml: false, - splitToSentences: true, + splitToSentences: true, - segmentEndPause: 1.0, - sentenceEndPause: 0.75, + segmentEndPause: 1.0, + sentenceEndPause: 0.75, - customLexiconPaths: undefined, + customLexiconPaths: undefined, - plainText: { - paragraphBreaks: "double", - whitespace: "collapse", - }, + plainText: { + paragraphBreaks: "double", + whitespace: "collapse", + }, - alignment: { - dtw: { - granularity: "high", + alignment: { + dtw: { + granularity: "high", + }, }, - }, - postProcessing: { - normalizeAudio: true, - targetPeak: -3, - maxGainIncrease: 30, + postProcessing: { + normalizeAudio: true, + targetPeak: -3, + maxGainIncrease: 30, - speed: undefined, - pitch: undefined, - }, + speed: undefined, + pitch: undefined, + }, - outputAudioFormat: undefined, + outputAudioFormat: undefined, - languageDetection: undefined, + languageDetection: undefined, - vits: { - speakerId: undefined, - provider: undefined, - }, + vits: { + speakerId: undefined, + provider: undefined, + }, - espeak: { - rate: undefined, - pitch: undefined, - pitchRange: undefined, - useKlatt: false, - }, + espeak: { + rate: undefined, + pitch: undefined, + pitchRange: undefined, + useKlatt: false, + }, }; const defaultDialectForLanguageCode: { [lang: string]: string } = { - en: "en-US", - zh: "zh-CN", - ar: "ar-EG", - fr: "fr-FR", - de: "de-DE", - pt: "pt-BR", - es: "es-ES", - nl: "nl-NL", + en: "en-US", + zh: "zh-CN", + ar: "ar-EG", + fr: "fr-FR", + de: "de-DE", + pt: "pt-BR", + es: "es-ES", + nl: "nl-NL", }; function getAppDataDir(appName: string) { - let dataDir: string; + let dataDir: string; - const platform = process.platform; - const homeDir = os.homedir(); - - if (platform == "win32") { - dataDir = path.join(homeDir, "AppData", "Local", appName); - } else if (platform == "darwin") { - dataDir = path.join(homeDir, "Library", "Application Support", appName); - } else if (platform == "linux") { - dataDir = path.join(homeDir, ".local", "share", appName); - } else { - throw new Error(`Unsupport platform ${platform}`); - } + const platform = process.platform; + const homeDir = os.homedir(); + + if (platform == "win32") { + dataDir = path.join(homeDir, "AppData", "Local", appName); + } else if (platform == "darwin") { + dataDir = path.join(homeDir, "Library", "Application Support", appName); + } else if (platform == "linux") { + dataDir = path.join(homeDir, ".local", "share", appName); + } else { + throw new Error(`Unsupport platform ${platform}`); + } - return dataDir; + return dataDir; } const existsSync = gracefulFS.existsSync; @@ -7780,56 +7952,58 @@ const existsSync = gracefulFS.existsSync; const stat = promisify(gracefulFS.stat); async function ensureDir(dirPath: string) { - dirPath = path.normalize(dirPath); + dirPath = path.normalize(dirPath); - if (existsSync(dirPath)) { - const dirStats = await stat(dirPath); + if (existsSync(dirPath)) { + const dirStats = await stat(dirPath); - if (!dirStats.isDirectory()) { - throw new Error(`The path '${dirPath}' exists but is not a directory.`); + if (!dirStats.isDirectory()) { + throw new Error( + `The path '${dirPath}' exists but is not a directory.` + ); + } + } else { + return fsExtra.ensureDir(dirPath); } - } else { - return fsExtra.ensureDir(dirPath); - } } async function isFileIsUpToDate(filePath: string, timeRangeSeconds: number) { - const fileUpdateTime = (await stat(filePath)).mtime.valueOf(); + const fileUpdateTime = (await stat(filePath)).mtime.valueOf(); - const currentTime = new Date().valueOf(); + const currentTime = new Date().valueOf(); - const differenceInMilliseconds = currentTime - fileUpdateTime; + const differenceInMilliseconds = currentTime - fileUpdateTime; - const differenceInSeconds = differenceInMilliseconds / 1000; + const differenceInSeconds = differenceInMilliseconds / 1000; - return differenceInSeconds <= timeRangeSeconds; + return differenceInSeconds <= timeRangeSeconds; } function getAppTempDir(appName: string) { - let tempDir: string; + let tempDir: string; - const platform = process.platform; - const homeDir = os.homedir(); - - if (platform == "win32") { - tempDir = path.join(homeDir, "AppData", "Local", "Temp", appName); - } else if (platform == "darwin") { - tempDir = path.join(homeDir, "Library", "Caches", appName); - } else if (platform == "linux") { - tempDir = path.join(homeDir, ".cache", appName); - } else { - throw new Error(`Unsupport platform ${platform}`); - } + const platform = process.platform; + const homeDir = os.homedir(); + + if (platform == "win32") { + tempDir = path.join(homeDir, "AppData", "Local", "Temp", appName); + } else if (platform == "darwin") { + tempDir = path.join(homeDir, "Library", "Caches", appName); + } else if (platform == "linux") { + tempDir = path.join(homeDir, ".cache", appName); + } else { + throw new Error(`Unsupport platform ${platform}`); + } - return tempDir; + return tempDir; } async function writeFile( - filePath: string, - data: string | NodeJS.ArrayBufferView, - options?: fsExtra.WriteFileOptions, + filePath: string, + data: string | NodeJS.ArrayBufferView, + options?: fsExtra.WriteFileOptions ) { - return outputFile(filePath, data, options); + return outputFile(filePath, data, options); } const access = promisify(gracefulFS.access); @@ -7837,359 +8011,367 @@ const access = promisify(gracefulFS.access); const remove = fsExtra.remove; async function existsAndIsWritable(targetPath: string) { - try { - await access(targetPath, gracefulFS.constants.W_OK); - } catch { - return false; - } + try { + await access(targetPath, gracefulFS.constants.W_OK); + } catch { + return false; + } - return true; + return true; } async function testDirectoryIsWritable(dir: string) { - const testFileName = path.join(dir, getRandomHexString(16)); + const testFileName = path.join(dir, getRandomHexString(16)); - try { - await fsExtra.createFile(testFileName); - await remove(testFileName); - } catch (e) { - return false; - } + try { + await fsExtra.createFile(testFileName); + await remove(testFileName); + } catch (e) { + return false; + } - return true; + return true; } async function move(source: string, dest: string) { - source = path.normalize(source); - dest = path.normalize(dest); + source = path.normalize(source); + dest = path.normalize(dest); - if (existsSync(dest)) { - const destPathExistsAndIsWritable = await existsAndIsWritable(dest); + if (existsSync(dest)) { + const destPathExistsAndIsWritable = await existsAndIsWritable(dest); - if (!destPathExistsAndIsWritable) { - throw new Error( - `The destination path '${dest}' exists but is not writable. There may be a permissions or locking issue.`, - ); - } - } else { - const destDir = path.parse(dest).dir; - const destDirIsWritable = await testDirectoryIsWritable(destDir); + if (!destPathExistsAndIsWritable) { + throw new Error( + `The destination path '${dest}' exists but is not writable. There may be a permissions or locking issue.` + ); + } + } else { + const destDir = path.parse(dest).dir; + const destDirIsWritable = await testDirectoryIsWritable(destDir); - if (!destDirIsWritable) { - throw new Error( - `The directory ${destDir} is not writable. There may be a permissions issue.`, - ); + if (!destDirIsWritable) { + throw new Error( + `The directory ${destDir} is not writable. There may be a permissions issue.` + ); + } } - } - return fsExtra.move(source, dest, { overwrite: true }); + return fsExtra.move(source, dest, { overwrite: true }); } const outputFile = fsExtra.outputFile; async function writeFileSafe( - filePath: string, - data: string | NodeJS.ArrayBufferView, - options?: fsExtra.WriteFileOptions, + filePath: string, + data: string | NodeJS.ArrayBufferView, + options?: fsExtra.WriteFileOptions ) { - const tempDir = getAppTempDir("eliza"); - const tempFilePath = path.join(tempDir, `${getRandomHexString(16)}.partial`); + const tempDir = getAppTempDir("eliza"); + const tempFilePath = path.join( + tempDir, + `${getRandomHexString(16)}.partial` + ); - await writeFile(tempFilePath, data, options); + await writeFile(tempFilePath, data, options); - await move(tempFilePath, filePath); + await move(tempFilePath, filePath); } function getRandomHexString(charCount = 32, upperCase = false) { - if (charCount % 2 !== 0) { - throw new Error(`'charCount' must be an even number`); - } + if (charCount % 2 !== 0) { + throw new Error(`'charCount' must be an even number`); + } - let hex = randomBytes(charCount / 2).toString("hex"); + let hex = randomBytes(charCount / 2).toString("hex"); - if (upperCase) { - hex = hex.toUpperCase(); - } + if (upperCase) { + hex = hex.toUpperCase(); + } - return hex; + return hex; } function stringifyAndFormatJson(obj: any) { - return JSON.stringify(obj, undefined, 4); + return JSON.stringify(obj, undefined, 4); } async function normalizeIdentifierToLanguageCode(langIdentifier: string) { - const result = await parseLangIdentifier(langIdentifier); + const result = await parseLangIdentifier(langIdentifier); - return result.Name; + return result.Name; } const langInfoEntries: LangInfoEntry[] = []; interface LangInfoEntry { - LCID: number; + LCID: number; - Name: string; - NameLowerCase: string; + Name: string; + NameLowerCase: string; - TwoLetterISOLanguageName: string; - ThreeLetterISOLanguageName: string; - ThreeLetterWindowsLanguageName: string; + TwoLetterISOLanguageName: string; + ThreeLetterISOLanguageName: string; + ThreeLetterWindowsLanguageName: string; - EnglishName: string; - EnglishNameLowerCase: string; + EnglishName: string; + EnglishNameLowerCase: string; - ANSICodePage: string; + ANSICodePage: string; } function getModuleRootDir() { - const currentScriptDir = path.dirname(fileURLToPath(import.meta.url)); - return path.resolve(currentScriptDir, "..", ".."); + const currentScriptDir = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(currentScriptDir, "..", ".."); } function resolveToModuleRootDir(relativePath: string) { - return path.resolve(getModuleRootDir(), relativePath); + return path.resolve(getModuleRootDir(), relativePath); } async function loadLangInfoEntriesIfNeeded() { - if (langInfoEntries.length > 0) { - return; - } + if (langInfoEntries.length > 0) { + return; + } - const entries = (await readAndParseJsonFile( - resolveToModuleRootDir("data/tables/lcid-table.json"), - )) as LangInfoEntry[]; + const entries = (await readAndParseJsonFile( + resolveToModuleRootDir("data/tables/lcid-table.json") + )) as LangInfoEntry[]; - for (const entry of entries) { - entry.NameLowerCase = entry.Name.toLowerCase(); - entry.EnglishNameLowerCase = entry.EnglishName.toLowerCase(); + for (const entry of entries) { + entry.NameLowerCase = entry.Name.toLowerCase(); + entry.EnglishNameLowerCase = entry.EnglishName.toLowerCase(); - langInfoEntries.push(entry); - } + langInfoEntries.push(entry); + } } async function parseLangIdentifier(langIdentifier: string) { - if (!langIdentifier) { - return emptyLangInfoEntry; - } + if (!langIdentifier) { + return emptyLangInfoEntry; + } - await loadLangInfoEntriesIfNeeded(); + await loadLangInfoEntriesIfNeeded(); - langIdentifier = langIdentifier.trim().toLowerCase(); + langIdentifier = langIdentifier.trim().toLowerCase(); - for (const entry of langInfoEntries) { - if ( - langIdentifier === entry.NameLowerCase || - langIdentifier === entry.ThreeLetterISOLanguageName || - langIdentifier === entry.EnglishNameLowerCase - ) { - return entry; + for (const entry of langInfoEntries) { + if ( + langIdentifier === entry.NameLowerCase || + langIdentifier === entry.ThreeLetterISOLanguageName || + langIdentifier === entry.EnglishNameLowerCase + ) { + return entry; + } } - } - throw new Error(`Couldn't parse language identifier '${langIdentifier}'.`); + throw new Error(`Couldn't parse language identifier '${langIdentifier}'.`); } const emptyLangInfoEntry: LangInfoEntry = { - LCID: -1, + LCID: -1, - Name: "", - NameLowerCase: "", + Name: "", + NameLowerCase: "", - TwoLetterISOLanguageName: "", - ThreeLetterISOLanguageName: "", - ThreeLetterWindowsLanguageName: "", + TwoLetterISOLanguageName: "", + ThreeLetterISOLanguageName: "", + ThreeLetterWindowsLanguageName: "", - EnglishName: "Empty", - EnglishNameLowerCase: "empty", + EnglishName: "Empty", + EnglishNameLowerCase: "empty", - ANSICodePage: "", + ANSICodePage: "", }; ///////////////////////////////////////////////////////////////////////////////////////////// // Voice list request ///////////////////////////////////////////////////////////////////////////////////////////// async function requestVoiceList( - options: VoiceListRequestOptions, + options: VoiceListRequestOptions ): Promise { - console.log("voice list requests", options); - options = extendDeep(defaultVoiceListRequestOptions, options); - - const cacheOptions = options.cache!; + console.log("voice list requests", options); + options = extendDeep(defaultVoiceListRequestOptions, options); - let cacheDir = cacheOptions?.path; + const cacheOptions = options.cache!; - if (!cacheDir) { - const appDataDir = getAppDataDir("eliza"); - cacheDir = path.join(appDataDir, "voice-list-cache"); - await ensureDir(cacheDir); - } + let cacheDir = cacheOptions?.path; - const cacheFilePath = path.join(cacheDir, `${options.engine}.voices.json`); - console.log("cacheFilePath", cacheFilePath); - async function loadVoiceList() { - let voiceList: SynthesisVoice[] = []; + if (!cacheDir) { + const appDataDir = getAppDataDir("eliza"); + cacheDir = path.join(appDataDir, "voice-list-cache"); + await ensureDir(cacheDir); + } - switch (options.engine) { - case "vits": { - voiceList = vitsVoiceList.map((entry) => { - return { ...entry, packageName: `vits-${entry.name}` }; - }); + const cacheFilePath = path.join(cacheDir, `${options.engine}.voices.json`); + console.log("cacheFilePath", cacheFilePath); + async function loadVoiceList() { + let voiceList: SynthesisVoice[] = []; - break; - } - } + switch (options.engine) { + case "vits": { + voiceList = vitsVoiceList.map((entry) => { + return { ...entry, packageName: `vits-${entry.name}` }; + }); - if (cacheFilePath) { - await writeFileSafe(cacheFilePath, stringifyAndFormatJson(voiceList)); - } + break; + } + } - return voiceList; - } + if (cacheFilePath) { + await writeFileSafe( + cacheFilePath, + stringifyAndFormatJson(voiceList) + ); + } - let voiceList: SynthesisVoice[]; + return voiceList; + } - if ( - cacheFilePath && - existsSync(cacheFilePath) && - (await isFileIsUpToDate(cacheFilePath, options.cache!.duration!)) - ) { - voiceList = await readAndParseJsonFile(cacheFilePath); - } else { - voiceList = await loadVoiceList(); - } + let voiceList: SynthesisVoice[]; - console.log("voiceList"); - console.log(voiceList); + if ( + cacheFilePath && + existsSync(cacheFilePath) && + (await isFileIsUpToDate(cacheFilePath, options.cache!.duration!)) + ) { + voiceList = await readAndParseJsonFile(cacheFilePath); + } else { + voiceList = await loadVoiceList(); + } - const languageCode = await normalizeIdentifierToLanguageCode( - options.language || "", - ); + console.log("voiceList"); + console.log(voiceList); - if (languageCode) { - let filteredVoiceList = voiceList.filter((voice) => - voice.languages.includes(languageCode), + const languageCode = await normalizeIdentifierToLanguageCode( + options.language || "" ); - if (filteredVoiceList.length == 0 && languageCode.includes("-")) { - const shortLanguageCode = getShortLanguageCode(languageCode); + if (languageCode) { + let filteredVoiceList = voiceList.filter((voice) => + voice.languages.includes(languageCode) + ); - filteredVoiceList = voiceList.filter((voice) => - voice.languages.includes(shortLanguageCode), - ); - } + if (filteredVoiceList.length == 0 && languageCode.includes("-")) { + const shortLanguageCode = getShortLanguageCode(languageCode); - voiceList = filteredVoiceList; - } + filteredVoiceList = voiceList.filter((voice) => + voice.languages.includes(shortLanguageCode) + ); + } - if (options.voiceGender) { - const genderLowercase = options.voiceGender.toLowerCase(); - voiceList = voiceList.filter( - (voice) => voice.gender == genderLowercase || voice.gender == "unknown", - ); - } + voiceList = filteredVoiceList; + } - if (options.voice) { - const namePatternLowerCase = options.voice.toLocaleLowerCase(); - const namePatternParts = namePatternLowerCase.split(/\b/g); + if (options.voiceGender) { + const genderLowercase = options.voiceGender.toLowerCase(); + voiceList = voiceList.filter( + (voice) => + voice.gender == genderLowercase || voice.gender == "unknown" + ); + } - if (namePatternParts.length > 1) { - voiceList = voiceList.filter((voice) => - voice.name.toLocaleLowerCase().includes(namePatternLowerCase), - ); - } else { - voiceList = voiceList.filter((voice) => { - const name = voice.name.toLocaleLowerCase(); - const nameParts = name.split(/\b/g); + if (options.voice) { + const namePatternLowerCase = options.voice.toLocaleLowerCase(); + const namePatternParts = namePatternLowerCase.split(/\b/g); - for (const namePart of nameParts) { - if (namePart.startsWith(namePatternLowerCase)) { - return true; - } + if (namePatternParts.length > 1) { + voiceList = voiceList.filter((voice) => + voice.name.toLocaleLowerCase().includes(namePatternLowerCase) + ); + } else { + voiceList = voiceList.filter((voice) => { + const name = voice.name.toLocaleLowerCase(); + const nameParts = name.split(/\b/g); + + for (const namePart of nameParts) { + if (namePart.startsWith(namePatternLowerCase)) { + return true; + } + } + + return false; + }); } - - return false; - }); } - } - let bestMatchingVoice = voiceList[0]; + let bestMatchingVoice = voiceList[0]; - if ( - bestMatchingVoice && - voiceList.length > 1 && - defaultDialectForLanguageCode[languageCode] - ) { - const expandedLanguageCode = defaultDialectForLanguageCode[languageCode]; + if ( + bestMatchingVoice && + voiceList.length > 1 && + defaultDialectForLanguageCode[languageCode] + ) { + const expandedLanguageCode = + defaultDialectForLanguageCode[languageCode]; - for (const voice of voiceList) { - if (voice.languages.includes(expandedLanguageCode)) { - bestMatchingVoice = voice; - break; - } + for (const voice of voiceList) { + if (voice.languages.includes(expandedLanguageCode)) { + bestMatchingVoice = voice; + break; + } + } } - } - return { voiceList, bestMatchingVoice }; + return { voiceList, bestMatchingVoice }; } interface RequestVoiceListResult { - voiceList: SynthesisVoice[]; - bestMatchingVoice: SynthesisVoice; + voiceList: SynthesisVoice[]; + bestMatchingVoice: SynthesisVoice; } function getAllLangCodesFromVoiceList(voiceList: SynthesisVoice[]) { - const languageCodes = new Set(); - const langList: string[] = []; + const languageCodes = new Set(); + const langList: string[] = []; - for (const voice of voiceList) { - for (const langCode of voice.languages) { - if (languageCodes.has(langCode)) { - continue; - } + for (const voice of voiceList) { + for (const langCode of voice.languages) { + if (languageCodes.has(langCode)) { + continue; + } - langList.push(langCode); - languageCodes.add(langCode); + langList.push(langCode); + languageCodes.add(langCode); + } } - } - return langList; + return langList; } interface VoiceListRequestOptions extends SynthesisOptions { - cache?: { - path?: string; - duration?: number; - }; + cache?: { + path?: string; + duration?: number; + }; } const defaultVoiceListRequestOptions: VoiceListRequestOptions = { - ...defaultSynthesisOptions, + ...defaultSynthesisOptions, - cache: { - path: undefined, - duration: 60 * 1, - }, + cache: { + path: undefined, + duration: 60 * 1, + }, }; interface SynthesisSegmentEventData { - index: number; - total: number; - audio: RawAudio | Buffer; - timeline: Timeline; - transcript: string; - language: string; - peakDecibelsSoFar: number; + index: number; + total: number; + audio: RawAudio | Buffer; + timeline: Timeline; + transcript: string; + language: string; + peakDecibelsSoFar: number; } type SynthesisSegmentEvent = (data: SynthesisSegmentEventData) => Promise; export interface SynthesisVoice { - name: string; - languages: string[]; - gender: VoiceGender; - speakerCount?: number; - packageName?: string; + name: string; + languages: string[]; + gender: VoiceGender; + speakerCount?: number; + packageName?: string; } type VoiceGender = "male" | "female" | "unknown"; @@ -8198,28 +8380,28 @@ type ParagraphBreakType = "single" | "double"; type WhitespaceProcessing = "preserve" | "removeLineBreaks" | "collapse"; function splitToParagraphs( - text: string, - paragraphBreaks: ParagraphBreakType, - whitespace: WhitespaceProcessing, + text: string, + paragraphBreaks: ParagraphBreakType, + whitespace: WhitespaceProcessing ) { - let paragraphs: string[] = []; - - if (paragraphBreaks == "single") { - paragraphs = text.split(/(\r?\n)+/g); - } else if (paragraphBreaks == "double") { - paragraphs = text.split(/(\r?\n)(\r?\n)+/g); - } else { - throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`); - } - - if (whitespace == "removeLineBreaks") { - paragraphs = paragraphs.map((p) => p.replaceAll(/(\r?\n)+/g, " ")); - } else if (whitespace == "collapse") { - paragraphs = paragraphs.map((p) => p.replaceAll(/\s+/g, " ")); - } - - paragraphs = paragraphs.map((p) => p.trim()); - paragraphs = paragraphs.filter((p) => p.length > 0); - - return paragraphs; + let paragraphs: string[] = []; + + if (paragraphBreaks == "single") { + paragraphs = text.split(/(\r?\n)+/g); + } else if (paragraphBreaks == "double") { + paragraphs = text.split(/(\r?\n)(\r?\n)+/g); + } else { + throw new Error(`Invalid paragraph break type: ${paragraphBreaks}`); + } + + if (whitespace == "removeLineBreaks") { + paragraphs = paragraphs.map((p) => p.replaceAll(/(\r?\n)+/g, " ")); + } else if (whitespace == "collapse") { + paragraphs = paragraphs.map((p) => p.replaceAll(/\s+/g, " ")); + } + + paragraphs = paragraphs.map((p) => p.trim()); + paragraphs = paragraphs.filter((p) => p.length > 0); + + return paragraphs; } diff --git a/core/src/vendor/vitsVoiceList.ts b/core/src/vendor/vitsVoiceList.ts index 0b1784a3e7..8330fc6d35 100644 --- a/core/src/vendor/vitsVoiceList.ts +++ b/core/src/vendor/vitsVoiceList.ts @@ -1,643 +1,643 @@ import { SynthesisVoice } from "./vits"; export const vitsVoiceList: SynthesisVoice[] = [ - { - name: "ar_JO-kareem-low", - languages: ["ar-JO", "ar"], - gender: "male", - }, - { - name: "ar_JO-kareem-medium", - languages: ["ar-JO", "ar"], - gender: "male", - }, - - { - name: "ca_ES-upc_ona-x_low", - languages: ["ca-ES", "ca"], - gender: "female", - }, - { - name: "ca_ES-upc_ona-medium", - languages: ["ca-ES", "ca"], - gender: "female", - }, - { - name: "ca_ES-upc_pau-x_low", - languages: ["ca-ES", "ca"], - gender: "male", - }, - - { - name: "cs_CZ-jirka-low", - languages: ["cs-CZ", "cs"], - gender: "male", - }, - { - name: "cs_CZ-jirka-medium", - languages: ["cs-CZ", "cs"], - gender: "male", - }, - - { - name: "da_DK-nst_talesyntese-medium", - languages: ["da-DK", "da"], - gender: "male", - }, - - { - name: "de_DE-thorsten-low", - languages: ["de-DE", "de"], - gender: "male", - }, - { - name: "de_DE-thorsten-medium", - languages: ["de-DE", "de"], - gender: "male", - }, - { - name: "de_DE-thorsten_emotional-medium", - languages: ["de-DE", "de"], - gender: "male", - speakerCount: 8, - }, - { - name: "de_DE-thorsten-high", - languages: ["de-DE", "de"], - gender: "male", - }, - { - name: "de_DE-eva_k-x_low", - languages: ["de-DE", "de"], - gender: "female", - }, - { - name: "de_DE-ramona-low", - languages: ["de-DE", "de"], - gender: "female", - }, - { - name: "de_DE-pavoque-low", - languages: ["de-DE", "de"], - gender: "male", - }, - { - name: "de_DE-kerstin-low", - languages: ["de-DE", "de"], - gender: "female", - }, - { - name: "de_DE-karlsson-low", - languages: ["de-DE", "de"], - gender: "male", - }, - { - name: "de_DE-mls-medium", - languages: ["de-DE", "de"], - gender: "unknown", - speakerCount: 236, - }, - - { - name: "el_GR-rapunzelina-low", - languages: ["el-GR", "el"], - gender: "female", - }, - - { - name: "en_GB-alan-low", - languages: ["en-GB", "en"], - gender: "male", - }, - { - name: "en_GB-alan-medium", - languages: ["en-GB", "en"], - gender: "male", - }, - { - name: "en_GB-semaine-medium", - languages: ["en-GB", "en"], - gender: "unknown", - speakerCount: 4, - }, - { - name: "en_GB-danny-low", - languages: ["en-GB", "en"], - gender: "male", - }, - { - name: "en_GB-alba-medium", - languages: ["en-GB", "en"], - gender: "female", - }, - { - name: "en_GB-aru-medium", - languages: ["en-GB", "en"], - gender: "unknown", - speakerCount: 12, - }, - { - name: "en_GB-southern_english_female-low", - languages: ["en-GB", "en"], - gender: "female", - }, - { - name: "en_GB-northern_english_male-medium", - languages: ["en-GB", "en"], - gender: "male", - }, - { - name: "en_GB-vctk-medium", - languages: ["en-GB", "en"], - gender: "unknown", - speakerCount: 109, - }, - { - name: "en_GB-jenny_dioco-medium", - languages: ["en-GB", "en"], - gender: "female", - }, - { - name: "en_GB-cori-high", - languages: ["en-GB", "en"], - gender: "female", - }, - - { - name: "en_US-amy-low", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-amy-medium", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-kathleen-low", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-lessac-low", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-lessac-medium", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-lessac-high", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-libritts-high", - languages: ["en-US", "en"], - gender: "unknown", - speakerCount: 904, - }, - { - name: "en_US-libritts_r-medium", - languages: ["en-US", "en"], - gender: "unknown", - speakerCount: 904, - }, - { - name: "en_US-ryan-low", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-ryan-medium", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-ryan-high", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-joe-medium", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-kusal-medium", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-arctic-medium", - languages: ["en-US", "en"], - gender: "unknown", - speakerCount: 18, - }, - { - name: "en_US-l2arctic-medium", - languages: ["en-US", "en"], - gender: "unknown", - speakerCount: 24, - }, - { - name: "en_US-hfc_male-medium", - languages: ["en-US", "en"], - gender: "male", - }, - { - name: "en_US-hfc_female-medium", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-kristin-medium", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-ljspeech-high", - languages: ["en-US", "en"], - gender: "female", - }, - { - name: "en_US-ljspeech-medium", - languages: ["en-US", "en"], - gender: "female", - }, - - { - name: "es_ES-carlfm-x_low", - languages: ["es-ES", "es"], - gender: "male", - }, - { - name: "es_ES-sharvard-medium", - languages: ["es-ES", "es"], - gender: "unknown", - speakerCount: 2, - }, - { - name: "es_ES-davefx-medium", - languages: ["es-ES", "es"], - gender: "male", - }, - { - name: "es_ES-mls_9972-low", - languages: ["es-ES", "es"], - gender: "male", - }, - { - name: "es_ES-mls_10246-low", - languages: ["es-ES", "es"], - gender: "male", - }, - - { - name: "es_MX-ald-medium", - languages: ["es-MX", "es"], - gender: "male", - }, - { - name: "es_MX-claude-high", - languages: ["es-MX", "es"], - gender: "female", - }, - - { - name: "fa_IR-amir-medium", - languages: ["fa_IR", "fa"], - gender: "male", - }, - { - name: "fa_IR-gyro-medium", - languages: ["fa_IR", "fa"], - gender: "male", - }, - - { - name: "fi_FI-harri-low", - languages: ["fi-FI", "fi"], - gender: "female", - }, - { - name: "fi_FI-harri-medium", - languages: ["fi-FI", "fi"], - gender: "female", - }, - - { - name: "fr_FR-siwis-low", - languages: ["fr-FR", "fr"], - gender: "female", - }, - { - name: "fr_FR-siwis-medium", - languages: ["fr-FR", "fr"], - gender: "female", - }, - { - name: "fr_FR-mls_1840-low", - languages: ["fr-FR", "fr"], - gender: "male", - }, - { - name: "fr_FR-gilles-low", - languages: ["fr-FR", "fr"], - gender: "male", - }, - { - name: "fr_FR-upmc-medium", - languages: ["fr-FR", "fr"], - gender: "unknown", - speakerCount: 2, - }, - { - name: "fr_FR-mls-medium", - languages: ["fr-FR", "fr"], - gender: "unknown", - speakerCount: 125, - }, - { - name: "fr_FR-tom-medium", - languages: ["fr-FR", "fr"], - gender: "male", - }, - - { - name: "hu_HU-anna-medium", - languages: ["hu-HU", "hu"], - gender: "female", - }, - { - name: "hu_HU-imre-medium", - languages: ["hu-HU", "hu"], - gender: "male", - }, - - { - name: "is_IS-ugla-medium", - languages: ["is-IS", "is"], - gender: "female", - }, - { - name: "is_IS-steinn-medium", - languages: ["is-IS", "is"], - gender: "male", - }, - { - name: "is_IS-salka-medium", - languages: ["is-IS", "is"], - gender: "female", - }, - { - name: "is_IS-bui-medium", - languages: ["is-IS", "is"], - gender: "male", - }, - - { - name: "it_IT-riccardo-x_low", - languages: ["it-IT", "it"], - gender: "male", - }, - - { - name: "ka_GE-natia-medium", - languages: ["ka-GE", "ka"], - gender: "female", - }, - - { - name: "kk_KZ-iseke-x_low", - languages: ["kk-KZ", "kk"], - gender: "male", - }, - { - name: "kk_KZ-raya-x_low", - languages: ["kk-KZ", "kk"], - gender: "male", - }, - { - name: "kk_KZ-issai-high", - languages: ["kk-KZ", "kk"], - gender: "unknown", - speakerCount: 6, - }, - - { - name: "lb_LU-marylux-medium", - languages: ["lb-LU", "lb"], - gender: "female", - }, - - { - name: "ne_NP-google-medium", - languages: ["ne-NP", "ne"], - gender: "female", - speakerCount: 18, - }, - { - name: "ne_NP-google-x_low", - languages: ["ne-NP", "ne"], - gender: "female", - speakerCount: 18, - }, - - { - name: "nl_NL-mls_5809-low", - languages: ["nl-NL", "nl"], - gender: "female", - }, - { - name: "nl_NL-mls_7432-low", - languages: ["nl-NL", "nl"], - gender: "female", - }, - { - name: "nl_NL-mls-medium", - languages: ["nl-NL", "nl"], - gender: "unknown", - speakerCount: 52, - }, - - { - name: "nl_BE-nathalie-x_low", - languages: ["nl-BE", "nl"], - gender: "female", - }, - { - name: "nl_BE-nathalie-medium", - languages: ["nl-BE", "nl"], - gender: "female", - }, - { - name: "nl_BE-rdh-medium", - languages: ["nl-BE", "nl"], - gender: "male", - }, - { - name: "nl_BE-rdh-x_low", - languages: ["nl-BE", "nl"], - gender: "male", - }, - - { - name: "no_NO-talesyntese-medium", - languages: ["no-NO", "no"], - gender: "male", - }, - - { - name: "pl_PL-mls_6892-low", - languages: ["pl-PL", "pl"], - gender: "male", - }, - { - name: "pl_PL-darkman-medium", - languages: ["pl-PL", "pl"], - gender: "male", - }, - { - name: "pl_PL-gosia-medium", - languages: ["pl-PL", "pl"], - gender: "female", - }, - { - name: "pl_PL-mc_speech-medium", - languages: ["pl-PL", "pl"], - gender: "male", - }, - - { - name: "pt_BR-edresson-low", - languages: ["pt-BR", "pt"], - gender: "male", - }, - { - name: "pt_BR-faber-medium", - languages: ["pt-BR", "pt"], - gender: "male", - }, - - { - name: "pt_PT-tugao-medium", - languages: ["pt-PT", "pt"], - gender: "male", - }, - - { - name: "ro_RO-mihai-medium", - languages: ["ro-RO", "ro"], - gender: "male", - }, - - { - name: "ru_RU-ruslan-medium", - languages: ["ru-RU", "ru"], - gender: "male", - }, - { - name: "ru_RU-irinia-medium", - languages: ["ru-RU", "ru"], - gender: "female", - }, - { - name: "ru_RU-denis-medium", - languages: ["ru-RU", "ru"], - gender: "male", - }, - { - name: "ru_RU-dmitri-medium", - languages: ["ru-RU", "ru"], - gender: "male", - }, - - { - name: "sk_SK-lili-medium", - languages: ["sk-SK", "sk"], - gender: "female", - }, - - { - name: "sl_SI-artur-medium", - languages: ["sl_SI", "sl"], - gender: "male", - }, - - { - name: "sr_RS-serbski_institut-medium", - languages: ["sr-RS", "sr"], - gender: "male", - speakerCount: 2, - }, - - { - name: "sv_SE-nst-medium", - languages: ["sv-SE", "sv"], - gender: "male", - }, - - { - name: "sw_CD-lanfrica-medium", - languages: ["sw-CD", "sw"], - gender: "male", - }, - - { - name: "tr_TR-dfki-medium", - languages: ["tr-TR", "tr"], - gender: "male", - }, - { - name: "tr_TR-fahrettin-medium", - languages: ["tr-TR", "tr"], - gender: "male", - }, - { - name: "tr_TR-fettah-medium", - languages: ["tr-TR", "tr"], - gender: "male", - }, - - { - name: "uk_UA-lada-x_low", - languages: ["uk-UA", "uk"], - gender: "female", - }, - { - name: "uk_UA-ukrainian_tts-medium", - languages: ["uk-UA", "uk"], - gender: "unknown", - speakerCount: 3, - }, - { - name: "vi_VN-vivos-x_low", - languages: ["vi-VN", "vi"], - gender: "unknown", - speakerCount: 65, - }, - { - name: "vi_VN-25hours-single-low", - languages: ["vi-VN", "vi"], - gender: "female", - }, - { - name: "vi_VN-vais1000-medium", - languages: ["vi-VN", "vi"], - gender: "female", - }, - - { - name: "zh_CN-huayan-x_low", - languages: ["zh-CN", "zh"], - gender: "female", - }, - { - name: "zh_CN-huayan-medium", - languages: ["zh-CN", "zh"], - gender: "female", - }, + { + name: "ar_JO-kareem-low", + languages: ["ar-JO", "ar"], + gender: "male", + }, + { + name: "ar_JO-kareem-medium", + languages: ["ar-JO", "ar"], + gender: "male", + }, + + { + name: "ca_ES-upc_ona-x_low", + languages: ["ca-ES", "ca"], + gender: "female", + }, + { + name: "ca_ES-upc_ona-medium", + languages: ["ca-ES", "ca"], + gender: "female", + }, + { + name: "ca_ES-upc_pau-x_low", + languages: ["ca-ES", "ca"], + gender: "male", + }, + + { + name: "cs_CZ-jirka-low", + languages: ["cs-CZ", "cs"], + gender: "male", + }, + { + name: "cs_CZ-jirka-medium", + languages: ["cs-CZ", "cs"], + gender: "male", + }, + + { + name: "da_DK-nst_talesyntese-medium", + languages: ["da-DK", "da"], + gender: "male", + }, + + { + name: "de_DE-thorsten-low", + languages: ["de-DE", "de"], + gender: "male", + }, + { + name: "de_DE-thorsten-medium", + languages: ["de-DE", "de"], + gender: "male", + }, + { + name: "de_DE-thorsten_emotional-medium", + languages: ["de-DE", "de"], + gender: "male", + speakerCount: 8, + }, + { + name: "de_DE-thorsten-high", + languages: ["de-DE", "de"], + gender: "male", + }, + { + name: "de_DE-eva_k-x_low", + languages: ["de-DE", "de"], + gender: "female", + }, + { + name: "de_DE-ramona-low", + languages: ["de-DE", "de"], + gender: "female", + }, + { + name: "de_DE-pavoque-low", + languages: ["de-DE", "de"], + gender: "male", + }, + { + name: "de_DE-kerstin-low", + languages: ["de-DE", "de"], + gender: "female", + }, + { + name: "de_DE-karlsson-low", + languages: ["de-DE", "de"], + gender: "male", + }, + { + name: "de_DE-mls-medium", + languages: ["de-DE", "de"], + gender: "unknown", + speakerCount: 236, + }, + + { + name: "el_GR-rapunzelina-low", + languages: ["el-GR", "el"], + gender: "female", + }, + + { + name: "en_GB-alan-low", + languages: ["en-GB", "en"], + gender: "male", + }, + { + name: "en_GB-alan-medium", + languages: ["en-GB", "en"], + gender: "male", + }, + { + name: "en_GB-semaine-medium", + languages: ["en-GB", "en"], + gender: "unknown", + speakerCount: 4, + }, + { + name: "en_GB-danny-low", + languages: ["en-GB", "en"], + gender: "male", + }, + { + name: "en_GB-alba-medium", + languages: ["en-GB", "en"], + gender: "female", + }, + { + name: "en_GB-aru-medium", + languages: ["en-GB", "en"], + gender: "unknown", + speakerCount: 12, + }, + { + name: "en_GB-southern_english_female-low", + languages: ["en-GB", "en"], + gender: "female", + }, + { + name: "en_GB-northern_english_male-medium", + languages: ["en-GB", "en"], + gender: "male", + }, + { + name: "en_GB-vctk-medium", + languages: ["en-GB", "en"], + gender: "unknown", + speakerCount: 109, + }, + { + name: "en_GB-jenny_dioco-medium", + languages: ["en-GB", "en"], + gender: "female", + }, + { + name: "en_GB-cori-high", + languages: ["en-GB", "en"], + gender: "female", + }, + + { + name: "en_US-amy-low", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-amy-medium", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-kathleen-low", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-lessac-low", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-lessac-medium", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-lessac-high", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-libritts-high", + languages: ["en-US", "en"], + gender: "unknown", + speakerCount: 904, + }, + { + name: "en_US-libritts_r-medium", + languages: ["en-US", "en"], + gender: "unknown", + speakerCount: 904, + }, + { + name: "en_US-ryan-low", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-ryan-medium", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-ryan-high", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-joe-medium", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-kusal-medium", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-arctic-medium", + languages: ["en-US", "en"], + gender: "unknown", + speakerCount: 18, + }, + { + name: "en_US-l2arctic-medium", + languages: ["en-US", "en"], + gender: "unknown", + speakerCount: 24, + }, + { + name: "en_US-hfc_male-medium", + languages: ["en-US", "en"], + gender: "male", + }, + { + name: "en_US-hfc_female-medium", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-kristin-medium", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-ljspeech-high", + languages: ["en-US", "en"], + gender: "female", + }, + { + name: "en_US-ljspeech-medium", + languages: ["en-US", "en"], + gender: "female", + }, + + { + name: "es_ES-carlfm-x_low", + languages: ["es-ES", "es"], + gender: "male", + }, + { + name: "es_ES-sharvard-medium", + languages: ["es-ES", "es"], + gender: "unknown", + speakerCount: 2, + }, + { + name: "es_ES-davefx-medium", + languages: ["es-ES", "es"], + gender: "male", + }, + { + name: "es_ES-mls_9972-low", + languages: ["es-ES", "es"], + gender: "male", + }, + { + name: "es_ES-mls_10246-low", + languages: ["es-ES", "es"], + gender: "male", + }, + + { + name: "es_MX-ald-medium", + languages: ["es-MX", "es"], + gender: "male", + }, + { + name: "es_MX-claude-high", + languages: ["es-MX", "es"], + gender: "female", + }, + + { + name: "fa_IR-amir-medium", + languages: ["fa_IR", "fa"], + gender: "male", + }, + { + name: "fa_IR-gyro-medium", + languages: ["fa_IR", "fa"], + gender: "male", + }, + + { + name: "fi_FI-harri-low", + languages: ["fi-FI", "fi"], + gender: "female", + }, + { + name: "fi_FI-harri-medium", + languages: ["fi-FI", "fi"], + gender: "female", + }, + + { + name: "fr_FR-siwis-low", + languages: ["fr-FR", "fr"], + gender: "female", + }, + { + name: "fr_FR-siwis-medium", + languages: ["fr-FR", "fr"], + gender: "female", + }, + { + name: "fr_FR-mls_1840-low", + languages: ["fr-FR", "fr"], + gender: "male", + }, + { + name: "fr_FR-gilles-low", + languages: ["fr-FR", "fr"], + gender: "male", + }, + { + name: "fr_FR-upmc-medium", + languages: ["fr-FR", "fr"], + gender: "unknown", + speakerCount: 2, + }, + { + name: "fr_FR-mls-medium", + languages: ["fr-FR", "fr"], + gender: "unknown", + speakerCount: 125, + }, + { + name: "fr_FR-tom-medium", + languages: ["fr-FR", "fr"], + gender: "male", + }, + + { + name: "hu_HU-anna-medium", + languages: ["hu-HU", "hu"], + gender: "female", + }, + { + name: "hu_HU-imre-medium", + languages: ["hu-HU", "hu"], + gender: "male", + }, + + { + name: "is_IS-ugla-medium", + languages: ["is-IS", "is"], + gender: "female", + }, + { + name: "is_IS-steinn-medium", + languages: ["is-IS", "is"], + gender: "male", + }, + { + name: "is_IS-salka-medium", + languages: ["is-IS", "is"], + gender: "female", + }, + { + name: "is_IS-bui-medium", + languages: ["is-IS", "is"], + gender: "male", + }, + + { + name: "it_IT-riccardo-x_low", + languages: ["it-IT", "it"], + gender: "male", + }, + + { + name: "ka_GE-natia-medium", + languages: ["ka-GE", "ka"], + gender: "female", + }, + + { + name: "kk_KZ-iseke-x_low", + languages: ["kk-KZ", "kk"], + gender: "male", + }, + { + name: "kk_KZ-raya-x_low", + languages: ["kk-KZ", "kk"], + gender: "male", + }, + { + name: "kk_KZ-issai-high", + languages: ["kk-KZ", "kk"], + gender: "unknown", + speakerCount: 6, + }, + + { + name: "lb_LU-marylux-medium", + languages: ["lb-LU", "lb"], + gender: "female", + }, + + { + name: "ne_NP-google-medium", + languages: ["ne-NP", "ne"], + gender: "female", + speakerCount: 18, + }, + { + name: "ne_NP-google-x_low", + languages: ["ne-NP", "ne"], + gender: "female", + speakerCount: 18, + }, + + { + name: "nl_NL-mls_5809-low", + languages: ["nl-NL", "nl"], + gender: "female", + }, + { + name: "nl_NL-mls_7432-low", + languages: ["nl-NL", "nl"], + gender: "female", + }, + { + name: "nl_NL-mls-medium", + languages: ["nl-NL", "nl"], + gender: "unknown", + speakerCount: 52, + }, + + { + name: "nl_BE-nathalie-x_low", + languages: ["nl-BE", "nl"], + gender: "female", + }, + { + name: "nl_BE-nathalie-medium", + languages: ["nl-BE", "nl"], + gender: "female", + }, + { + name: "nl_BE-rdh-medium", + languages: ["nl-BE", "nl"], + gender: "male", + }, + { + name: "nl_BE-rdh-x_low", + languages: ["nl-BE", "nl"], + gender: "male", + }, + + { + name: "no_NO-talesyntese-medium", + languages: ["no-NO", "no"], + gender: "male", + }, + + { + name: "pl_PL-mls_6892-low", + languages: ["pl-PL", "pl"], + gender: "male", + }, + { + name: "pl_PL-darkman-medium", + languages: ["pl-PL", "pl"], + gender: "male", + }, + { + name: "pl_PL-gosia-medium", + languages: ["pl-PL", "pl"], + gender: "female", + }, + { + name: "pl_PL-mc_speech-medium", + languages: ["pl-PL", "pl"], + gender: "male", + }, + + { + name: "pt_BR-edresson-low", + languages: ["pt-BR", "pt"], + gender: "male", + }, + { + name: "pt_BR-faber-medium", + languages: ["pt-BR", "pt"], + gender: "male", + }, + + { + name: "pt_PT-tugao-medium", + languages: ["pt-PT", "pt"], + gender: "male", + }, + + { + name: "ro_RO-mihai-medium", + languages: ["ro-RO", "ro"], + gender: "male", + }, + + { + name: "ru_RU-ruslan-medium", + languages: ["ru-RU", "ru"], + gender: "male", + }, + { + name: "ru_RU-irinia-medium", + languages: ["ru-RU", "ru"], + gender: "female", + }, + { + name: "ru_RU-denis-medium", + languages: ["ru-RU", "ru"], + gender: "male", + }, + { + name: "ru_RU-dmitri-medium", + languages: ["ru-RU", "ru"], + gender: "male", + }, + + { + name: "sk_SK-lili-medium", + languages: ["sk-SK", "sk"], + gender: "female", + }, + + { + name: "sl_SI-artur-medium", + languages: ["sl_SI", "sl"], + gender: "male", + }, + + { + name: "sr_RS-serbski_institut-medium", + languages: ["sr-RS", "sr"], + gender: "male", + speakerCount: 2, + }, + + { + name: "sv_SE-nst-medium", + languages: ["sv-SE", "sv"], + gender: "male", + }, + + { + name: "sw_CD-lanfrica-medium", + languages: ["sw-CD", "sw"], + gender: "male", + }, + + { + name: "tr_TR-dfki-medium", + languages: ["tr-TR", "tr"], + gender: "male", + }, + { + name: "tr_TR-fahrettin-medium", + languages: ["tr-TR", "tr"], + gender: "male", + }, + { + name: "tr_TR-fettah-medium", + languages: ["tr-TR", "tr"], + gender: "male", + }, + + { + name: "uk_UA-lada-x_low", + languages: ["uk-UA", "uk"], + gender: "female", + }, + { + name: "uk_UA-ukrainian_tts-medium", + languages: ["uk-UA", "uk"], + gender: "unknown", + speakerCount: 3, + }, + { + name: "vi_VN-vivos-x_low", + languages: ["vi-VN", "vi"], + gender: "unknown", + speakerCount: 65, + }, + { + name: "vi_VN-25hours-single-low", + languages: ["vi-VN", "vi"], + gender: "female", + }, + { + name: "vi_VN-vais1000-medium", + languages: ["vi-VN", "vi"], + gender: "female", + }, + + { + name: "zh_CN-huayan-x_low", + languages: ["zh-CN", "zh"], + gender: "female", + }, + { + name: "zh_CN-huayan-medium", + languages: ["zh-CN", "zh"], + gender: "female", + }, ]; diff --git a/core/tests/actions.test.ts b/core/tests/actions.test.ts index ad9d20960d..cc266eca13 100644 --- a/core/tests/actions.test.ts +++ b/core/tests/actions.test.ts @@ -3,241 +3,252 @@ import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { runAiTest } from "../src/test_resources/runAiTest.ts"; import { messageHandlerTemplate } from "../src/test_resources/templates.ts"; -import { TEST_ACTION, TEST_ACTION_FAIL } from "../src/test_resources/testAction.ts"; +import { + TEST_ACTION, + TEST_ACTION_FAIL, +} from "../src/test_resources/testAction.ts"; import { type User } from "../src/test_resources/types.ts"; import { composeContext } from "../src/core/context.ts"; import { embeddingZeroVector } from "../src/core/memory.ts"; import { - Content, - IAgentRuntime, - ModelClass, - State, - type Memory, - type UUID, + Content, + IAgentRuntime, + ModelClass, + State, + type Memory, + type UUID, } from "../src/core/types.ts"; import { stringToUuid } from "../src/core/uuid.ts"; import { generateMessageResponse } from "../src/core/generation.ts"; async function handleMessage( - runtime: IAgentRuntime, - message: Memory, - state?: State, + runtime: IAgentRuntime, + message: Memory, + state?: State ) { - const _saveRequestMessage = async (message: Memory, state: State) => { - const { content: senderContent, userId, roomId } = message; - - const _senderContent = (senderContent as Content).text?.trim(); - if (_senderContent) { - await runtime.messageManager.createMemory({ - id: stringToUuid(message.id), - userId: userId!, - content: { - text: _senderContent, - action: (message.content as Content)?.action ?? "null", - }, - roomId, - embedding: embeddingZeroVector, - }); - await runtime.evaluate(message, state); + const _saveRequestMessage = async (message: Memory, state: State) => { + const { content: senderContent, userId, roomId } = message; + + const _senderContent = (senderContent as Content).text?.trim(); + if (_senderContent) { + await runtime.messageManager.createMemory({ + id: stringToUuid(message.id), + userId: userId!, + content: { + text: _senderContent, + action: (message.content as Content)?.action ?? "null", + }, + roomId, + embedding: embeddingZeroVector, + }); + await runtime.evaluate(message, state); + } + }; + + await _saveRequestMessage(message, state as State); + if (!state) { + state = (await runtime.composeState(message)) as State; } - }; - - await _saveRequestMessage(message, state as State); - if (!state) { - state = (await runtime.composeState(message)) as State; - } - - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - - let responseContent: Content | null = null; - const { userId, roomId } = message; - - for (let triesLeft = 3; triesLeft > 0; triesLeft--) { - const response = await generateMessageResponse({ - context, - runtime, - modelClass: ModelClass.SMALL, - }); - runtime.databaseAdapter.log({ - body: { message, context, response }, - userId: userId, - roomId, - type: "actions_test_completion", + const context = composeContext({ + state, + template: messageHandlerTemplate, }); - return response; - } - if (!responseContent) { - responseContent = { - text: "", - action: "IGNORE", - }; - } - - if (responseContent.text) { - const response = { - userId: runtime.agentId, - content: responseContent, - roomId, - embedding: embeddingZeroVector, - }; - await runtime.messageManager.createMemory(response); + let responseContent: Content | null = null; + const { userId, roomId } = message; + + for (let triesLeft = 3; triesLeft > 0; triesLeft--) { + const response = await generateMessageResponse({ + context, + runtime, + modelClass: ModelClass.SMALL, + }); - state = await this.runtime.updateRecentMessageState(state); - await runtime.processActions(message, [response], state); - await runtime.evaluate(message, state); - } else { - console.warn("Empty response, skipping"); - } + runtime.databaseAdapter.log({ + body: { message, context, response }, + userId: userId, + roomId, + type: "actions_test_completion", + }); + return response; + } + + if (!responseContent) { + responseContent = { + text: "", + action: "IGNORE", + }; + } - return responseContent; + if (responseContent.text) { + const response = { + userId: runtime.agentId, + content: responseContent, + roomId, + embedding: embeddingZeroVector, + }; + await runtime.messageManager.createMemory(response); + + state = await this.runtime.updateRecentMessageState(state); + await runtime.processActions(message, [response], state); + await runtime.evaluate(message, state); + } else { + console.warn("Empty response, skipping"); + } + + return responseContent; } // use .dev.vars for local testing dotenv.config({ path: ".dev.vars" }); describe("Actions", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId: UUID; - - beforeAll(async () => { - const { session, runtime: _runtime } = await createRuntime({ - env: process.env as Record, - actions: [TEST_ACTION, TEST_ACTION_FAIL], - }); + let user: User; + let runtime: IAgentRuntime; + let roomId: UUID; + + beforeAll(async () => { + const { session, runtime: _runtime } = await createRuntime({ + env: process.env as Record, + actions: [TEST_ACTION, TEST_ACTION_FAIL], + }); - user = session.user!; - runtime = _runtime; - - // check if the user id exists in the 'accounts' table - // if it doesn't, create it with the name 'Test User' - let account = await runtime.databaseAdapter.getAccountById(user.id as UUID); - - if (!account) { - account = await runtime.databaseAdapter.getAccountById(user.id as UUID); - if (!account) { - await runtime.databaseAdapter.createAccount({ - id: user.id as UUID, - username: "Test User", - name: "Test User", - email: user.email, - avatarUrl: "", + user = session.user!; + runtime = _runtime; + + // check if the user id exists in the 'accounts' table + // if it doesn't, create it with the name 'Test User' + let account = await runtime.databaseAdapter.getAccountById( + user.id as UUID + ); + + if (!account) { + account = await runtime.databaseAdapter.getAccountById( + user.id as UUID + ); + if (!account) { + await runtime.databaseAdapter.createAccount({ + id: user.id as UUID, + username: "Test User", + name: "Test User", + email: user.email, + avatarUrl: "", + }); + } + } + + // get all relationships for user + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: "00000000-0000-0000-0000-000000000000" as UUID, }); - } - } - // get all relationships for user - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: "00000000-0000-0000-0000-000000000000" as UUID, + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data!.roomId; + + await cleanup(); }); - if (!data) { - throw new Error("Relationship not found"); - } + afterAll(async () => { + await cleanup(); + }); - roomId = data!.roomId; - - await cleanup(); - }); - - afterAll(async () => { - await cleanup(); - }); - - async function cleanup() { - await runtime.factManager.removeAllMemories(roomId); - await runtime.messageManager.removeAllMemories(roomId); - } - - // Test that actions are being loaded into context properly - test("Actions are loaded into context", async () => { - const actions = runtime.actions; - expect(actions).toBeDefined(); - expect(actions.length).toBeGreaterThan(0); - // Ensure the TEST_ACTION action is part of the loaded actions - const testAction = actions.find((action) => action.name === "TEST_ACTION"); - expect(testAction).toBeDefined(); - }); - - // Test that actions are validated properly - test("Test action is always valid", async () => { - const testAction = runtime.actions.find( - (action) => action.name === "TEST_ACTION", - ); - expect(testAction).toBeDefined(); - if (testAction && testAction.validate) { - const isValid = await testAction.validate(runtime, { - userId: user.id as UUID, - content: { text: "Test message" }, - roomId: roomId, - }); - expect(isValid).toBeTruthy(); - } else { - throw new Error( - "Continue action or its validation function is undefined", - ); + async function cleanup() { + await runtime.factManager.removeAllMemories(roomId); + await runtime.messageManager.removeAllMemories(roomId); } - }); - - test("Test that actions are properly validated in state", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { - text: "Please respond with the message 'ok' and the action TEST_ACTION", - }, - roomId, - }; - const state = await runtime.composeState(message); - expect(state.actionNames).not.toContain("TEST_ACTION_FAIL"); - - expect(state.actionNames).toContain("TEST_ACTION"); - }); - - // Validate that TEST_ACTION is in the state - test("Validate that TEST_ACTION is in the state", async () => { - await runAiTest("Validate TEST_ACTION is in the state", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { - text: "Please respond with the message 'ok' and the action TEST_ACTION", - }, - roomId, - }; - - const response = await handleMessage(runtime, message); - return response.action === "TEST_ACTION"; // Return true if the expected action matches + // Test that actions are being loaded into context properly + test("Actions are loaded into context", async () => { + const actions = runtime.actions; + expect(actions).toBeDefined(); + expect(actions.length).toBeGreaterThan(0); + // Ensure the TEST_ACTION action is part of the loaded actions + const testAction = actions.find( + (action) => action.name === "TEST_ACTION" + ); + expect(testAction).toBeDefined(); }); - }, 60000); - - // Test that TEST_ACTION action handler is called properly - test("Test action handler is called", async () => { - await runAiTest("Test action handler is called", async () => { - const testAction = runtime.actions.find( - (action) => action.name === "TEST_ACTION", - ); - if (!testAction || !testAction.handler) { - console.error("Continue action or its handler function is undefined"); - return false; // Return false to indicate the test setup failed - } - - const mockMessage: Memory = { - userId: user.id as UUID, - content: { - text: "Test message for TEST action", - }, - roomId, - }; - - const response = await testAction.handler(runtime, mockMessage); - return response !== undefined; // Return true if the handler returns a defined response + + // Test that actions are validated properly + test("Test action is always valid", async () => { + const testAction = runtime.actions.find( + (action) => action.name === "TEST_ACTION" + ); + expect(testAction).toBeDefined(); + if (testAction && testAction.validate) { + const isValid = await testAction.validate(runtime, { + userId: user.id as UUID, + content: { text: "Test message" }, + roomId: roomId, + }); + expect(isValid).toBeTruthy(); + } else { + throw new Error( + "Continue action or its validation function is undefined" + ); + } + }); + + test("Test that actions are properly validated in state", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { + text: "Please respond with the message 'ok' and the action TEST_ACTION", + }, + roomId, + }; + + const state = await runtime.composeState(message); + expect(state.actionNames).not.toContain("TEST_ACTION_FAIL"); + + expect(state.actionNames).toContain("TEST_ACTION"); }); - }, 60000); // You can adjust the timeout if needed + + // Validate that TEST_ACTION is in the state + test("Validate that TEST_ACTION is in the state", async () => { + await runAiTest("Validate TEST_ACTION is in the state", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { + text: "Please respond with the message 'ok' and the action TEST_ACTION", + }, + roomId, + }; + + const response = await handleMessage(runtime, message); + return response.action === "TEST_ACTION"; // Return true if the expected action matches + }); + }, 60000); + + // Test that TEST_ACTION action handler is called properly + test("Test action handler is called", async () => { + await runAiTest("Test action handler is called", async () => { + const testAction = runtime.actions.find( + (action) => action.name === "TEST_ACTION" + ); + if (!testAction || !testAction.handler) { + console.error( + "Continue action or its handler function is undefined" + ); + return false; // Return false to indicate the test setup failed + } + + const mockMessage: Memory = { + userId: user.id as UUID, + content: { + text: "Test message for TEST action", + }, + roomId, + }; + + const response = await testAction.handler(runtime, mockMessage); + return response !== undefined; // Return true if the handler returns a defined response + }); + }, 60000); // You can adjust the timeout if needed }); diff --git a/core/tests/browser.test.ts b/core/tests/browser.test.ts index 7323bb686c..b8c5d1fc94 100644 --- a/core/tests/browser.test.ts +++ b/core/tests/browser.test.ts @@ -5,89 +5,94 @@ import { BrowserService } from "./browser.ts"; dotenv.config(); describe("BrowserService", () => { - let browserService: BrowserService; + let browserService: BrowserService; - beforeAll(async () => { - const { runtime } = await createRuntime({ - env: process.env as Record, - actions: [], + beforeAll(async () => { + const { runtime } = await createRuntime({ + env: process.env as Record, + actions: [], + }); + browserService = BrowserService.getInstance(runtime); + await browserService.initialize(); }); - browserService = BrowserService.getInstance(runtime); - await browserService.initialize(); - }); - afterAll(async () => { - await browserService.closeBrowser(); - }); + afterAll(async () => { + await browserService.closeBrowser(); + }); - test("should initialize and close browser", async () => { - const { runtime } = await createRuntime({ - env: process.env as Record, - actions: [], + test("should initialize and close browser", async () => { + const { runtime } = await createRuntime({ + env: process.env as Record, + actions: [], + }); + const newBrowserService = BrowserService.getInstance(runtime); + await expect(newBrowserService.initialize()).resolves.not.toThrow(); + await expect(newBrowserService.closeBrowser()).resolves.not.toThrow(); }); - const newBrowserService = BrowserService.getInstance(runtime); - await expect(newBrowserService.initialize()).resolves.not.toThrow(); - await expect(newBrowserService.closeBrowser()).resolves.not.toThrow(); - }); - test("should fetch content from a simple website", async () => { - const content = await browserService.getPageContent("https://example.com"); - expect(content).toContain("Example Domain"); - }, 30000); + test("should fetch content from a simple website", async () => { + const content = await browserService.getPageContent( + "https://example.com" + ); + expect(content).toContain("Example Domain"); + }, 30000); - test("should fetch content from a news website", async () => { - const content = await browserService.getPageContent( - "https://news.ycombinator.com", - ); - expect(content).toContain("Hacker News"); - }, 30000); + test("should fetch content from a news website", async () => { + const content = await browserService.getPageContent( + "https://news.ycombinator.com" + ); + expect(content).toContain("Hacker News"); + }, 30000); - test("should handle a website with potential CAPTCHA (GitHub)", async () => { - const content = await browserService.getPageContent("https://github.com"); - expect(content).toContain("GitHub"); - }, 60000); + test("should handle a website with potential CAPTCHA (GitHub)", async () => { + const content = + await browserService.getPageContent("https://github.com"); + expect(content).toContain("GitHub"); + }, 60000); - test("should fetch content from a website that might be blocked (Wikipedia)", async () => { - const content = await browserService.getPageContent( - "https://en.wikipedia.org/wiki/Main_Page", - ); - expect(content).toContain("Wikipedia"); - }, 30000); + test("should fetch content from a website that might be blocked (Wikipedia)", async () => { + const content = await browserService.getPageContent( + "https://en.wikipedia.org/wiki/Main_Page" + ); + expect(content).toContain("Wikipedia"); + }, 30000); - test("should handle a 404 error and try alternative sources", async () => { - const content = await browserService.getPageContent( - "https://example.com/nonexistent-page", - ); - expect(content).not.toBe(""); - expect(content).toContain("search"); // Expecting to fall back to a search result - }, 60000); + test("should handle a 404 error and try alternative sources", async () => { + const content = await browserService.getPageContent( + "https://example.com/nonexistent-page" + ); + expect(content).not.toBe(""); + expect(content).toContain("search"); // Expecting to fall back to a search result + }, 60000); - test("should handle network errors gracefully", async () => { - await expect( - browserService.getPageContent( - "https://thisisaninvalidurlthatdoesnotexist.com", - ), - ).rejects.toThrow("Failed to fetch content from alternative sources"); - }, 60000); + test("should handle network errors gracefully", async () => { + await expect( + browserService.getPageContent( + "https://thisisaninvalidurlthatdoesnotexist.com" + ) + ).rejects.toThrow("Failed to fetch content from alternative sources"); + }, 60000); - test("should block ads on ad-heavy website", async () => { - const content = await browserService.getPageContent("https://www.cnn.com"); - expect(content).not.toContain("Advertisement"); - }, 60000); + test("should block ads on ad-heavy website", async () => { + const content = await browserService.getPageContent( + "https://www.cnn.com" + ); + expect(content).not.toContain("Advertisement"); + }, 60000); - test("should handle a website with reCAPTCHA", async () => { - const content = await browserService.getPageContent( - "https://www.google.com/recaptcha/api2/demo", - ); - expect(content).toContain("reCAPTCHA"); - // Note: Full CAPTCHA solving can't be reliably tested without manual intervention - }, 60000); + test("should handle a website with reCAPTCHA", async () => { + const content = await browserService.getPageContent( + "https://www.google.com/recaptcha/api2/demo" + ); + expect(content).toContain("reCAPTCHA"); + // Note: Full CAPTCHA solving can't be reliably tested without manual intervention + }, 60000); - test("should handle a website with hCAPTCHA", async () => { - const content = await browserService.getPageContent( - "https://accounts.hcaptcha.com/demo", - ); - expect(content).toContain("hCaptcha"); - // Note: Full CAPTCHA solving can't be reliably tested without manual intervention - }, 60000); + test("should handle a website with hCAPTCHA", async () => { + const content = await browserService.getPageContent( + "https://accounts.hcaptcha.com/demo" + ); + expect(content).toContain("hCaptcha"); + // Note: Full CAPTCHA solving can't be reliably tested without manual intervention + }, 60000); }); diff --git a/core/tests/continue.test.ts b/core/tests/continue.test.ts index 2eaabffb23..cf962953db 100644 --- a/core/tests/continue.test.ts +++ b/core/tests/continue.test.ts @@ -1,5 +1,10 @@ import dotenv from "dotenv"; -import { Content, IAgentRuntime, Memory, type UUID } from "../src/core/types.ts"; +import { + Content, + IAgentRuntime, + Memory, + type UUID, +} from "../src/core/types.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { Goodbye1 } from "../src/test_resources/data.ts"; @@ -12,216 +17,215 @@ import ignore from "../src/actions/ignore.ts"; dotenv.config({ path: ".dev.vars" }); - const GetContinueExample1 = (_userId: UUID) => [ - { - userId: zeroUuid, - content: { - text: "Hmm, let think for a second, I was going to tell you about something...", - action: "CONTINUE", + { + userId: zeroUuid, + content: { + text: "Hmm, let think for a second, I was going to tell you about something...", + action: "CONTINUE", + }, }, - }, - { - userId: zeroUuid, - content: { - text: "I remember now, I was going to tell you about my favorite food, which is pizza.", - action: "CONTINUE", + { + userId: zeroUuid, + content: { + text: "I remember now, I was going to tell you about my favorite food, which is pizza.", + action: "CONTINUE", + }, }, - }, - { - userId: zeroUuid, - content: { - text: "I love pizza, it's so delicious.", - action: "CONTINUE", + { + userId: zeroUuid, + content: { + text: "I love pizza, it's so delicious.", + action: "CONTINUE", + }, }, - }, ]; describe("User Profile", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId: UUID = zeroUuid; - - afterAll(async () => { - await cleanup(); - }); - - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - actions: [action, ignore], - }); - user = setup.session.user; - runtime = setup.runtime; + let user: User; + let runtime: IAgentRuntime; + let roomId: UUID = zeroUuid; - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: zeroUuid, + afterAll(async () => { + await cleanup(); }); - roomId = data.roomId; - - await cleanup(); - }); + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + actions: [action, ignore], + }); + user = setup.session.user; + runtime = setup.runtime; - beforeEach(async () => { - await cleanup(); - }); + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: zeroUuid, + }); - async function cleanup() { - await runtime.factManager.removeAllMemories(roomId); - await runtime.messageManager.removeAllMemories(roomId); - } + roomId = data.roomId; - // test validate function response + await cleanup(); + }); - test("Test validate function response", async () => { - await runAiTest("Test validate function response", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "Hello" }, - roomId: roomId as UUID, - }; + beforeEach(async () => { + await cleanup(); + }); - const validate = action.validate!; + async function cleanup() { + await runtime.factManager.removeAllMemories(roomId); + await runtime.messageManager.removeAllMemories(roomId); + } - const result = await validate(runtime, message); + // test validate function response - // try again with GetContinueExample1, expect to be false - await populateMemories(runtime, user, roomId, [GetContinueExample1]); + test("Test validate function response", async () => { + await runAiTest("Test validate function response", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "Hello" }, + roomId: roomId as UUID, + }; - const message2: Memory = { - userId: zeroUuid as UUID, - content: { - text: "Hello", - action: "CONTINUE", - }, - roomId: roomId as UUID, - }; + const validate = action.validate!; - const result2 = await validate(runtime, message2); + const result = await validate(runtime, message); - return result === true && result2 === false; - }); - }, 60000); + // try again with GetContinueExample1, expect to be false + await populateMemories(runtime, user, roomId, [ + GetContinueExample1, + ]); - test("Test repetition check on continue", async () => { - await runAiTest("Test repetition check on continue", async () => { - const message: Memory = { - userId: zeroUuid as UUID, - content: { - text: "Hmm, let think for a second, I was going to tell you about something...", - action: "CONTINUE", - }, - roomId, - }; + const message2: Memory = { + userId: zeroUuid as UUID, + content: { + text: "Hello", + action: "CONTINUE", + }, + roomId: roomId as UUID, + }; - const handler = action.handler!; + const result2 = await validate(runtime, message2); - await populateMemories(runtime, user, roomId, [GetContinueExample1]); + return result === true && result2 === false; + }); + }, 60000); - const result = (await handler(runtime, message)) as Content; + test("Test repetition check on continue", async () => { + await runAiTest("Test repetition check on continue", async () => { + const message: Memory = { + userId: zeroUuid as UUID, + content: { + text: "Hmm, let think for a second, I was going to tell you about something...", + action: "CONTINUE", + }, + roomId, + }; - return result.action !== "CONTINUE"; - }); - }, 60000); - - test("Test multiple continue messages in a conversation", async () => { - await runAiTest( - "Test multiple continue messages in a conversation", - async () => { - const message: Memory = { - userId: user?.id as UUID, - content: { - text: "Write a short story in three parts, using the CONTINUE action for each part.", - }, - roomId: roomId, - }; - - const initialMessageCount = await runtime.messageManager.countMemories( - roomId, - false, - ); + const handler = action.handler!; - await action.handler!(runtime, message); + await populateMemories(runtime, user, roomId, [ + GetContinueExample1, + ]); - const finalMessageCount = await runtime.messageManager.countMemories( - roomId, - false, - ); + const result = (await handler(runtime, message)) as Content; - const agentMessages = await runtime.messageManager.getMemories({ - roomId, - count: finalMessageCount - initialMessageCount, - unique: false, + return result.action !== "CONTINUE"; }); - - const continueMessages = agentMessages.filter( - (m) => - m.userId === zeroUuid && - (m.content as Content).action === "CONTINUE", - ); - - // Check if the agent sent more than one message - const sentMultipleMessages = - finalMessageCount - initialMessageCount > 2; - - // Check if the agent used the CONTINUE action for each part - const usedContinueAction = continueMessages.length === 3; - // Check if the agent's responses are not empty - const responsesNotEmpty = agentMessages.every( - (m) => (m.content as Content).text !== "", + }, 60000); + + test("Test multiple continue messages in a conversation", async () => { + await runAiTest( + "Test multiple continue messages in a conversation", + async () => { + const message: Memory = { + userId: user?.id as UUID, + content: { + text: "Write a short story in three parts, using the CONTINUE action for each part.", + }, + roomId: roomId, + }; + + const initialMessageCount = + await runtime.messageManager.countMemories(roomId, false); + + await action.handler!(runtime, message); + + const finalMessageCount = + await runtime.messageManager.countMemories(roomId, false); + + const agentMessages = await runtime.messageManager.getMemories({ + roomId, + count: finalMessageCount - initialMessageCount, + unique: false, + }); + + const continueMessages = agentMessages.filter( + (m) => + m.userId === zeroUuid && + (m.content as Content).action === "CONTINUE" + ); + + // Check if the agent sent more than one message + const sentMultipleMessages = + finalMessageCount - initialMessageCount > 2; + + // Check if the agent used the CONTINUE action for each part + const usedContinueAction = continueMessages.length === 3; + // Check if the agent's responses are not empty + const responsesNotEmpty = agentMessages.every( + (m) => (m.content as Content).text !== "" + ); + + return ( + sentMultipleMessages && + usedContinueAction && + responsesNotEmpty + ); + } ); + }, 60000); - return sentMultipleMessages && usedContinueAction && responsesNotEmpty; - }, - ); - }, 60000); - - test("Test if message is added to database", async () => { - await runAiTest("Test if message is added to database", async () => { - const message: Memory = { - userId: user?.id as UUID, - content: { - text: "Tell me more about your favorite food.", - }, - roomId: roomId as UUID, - }; + test("Test if message is added to database", async () => { + await runAiTest("Test if message is added to database", async () => { + const message: Memory = { + userId: user?.id as UUID, + content: { + text: "Tell me more about your favorite food.", + }, + roomId: roomId as UUID, + }; - const initialMessageCount = await runtime.messageManager.countMemories( - roomId, - false, - ); + const initialMessageCount = + await runtime.messageManager.countMemories(roomId, false); - await action.handler!(runtime, message); + await action.handler!(runtime, message); - const finalMessageCount = await runtime.messageManager.countMemories( - roomId, - false, - ); + const finalMessageCount = + await runtime.messageManager.countMemories(roomId, false); - return finalMessageCount - initialMessageCount === 2; - }); - }, 60000); - test("Test if not continue", async () => { - await runAiTest("Test if not continue", async () => { - // this is basically the same test as the one in ignore.test - const message: Memory = { - userId: user?.id as UUID, - content: { text: "Bye" }, - roomId: roomId as UUID, - }; + return finalMessageCount - initialMessageCount === 2; + }); + }, 60000); + test("Test if not continue", async () => { + await runAiTest("Test if not continue", async () => { + // this is basically the same test as the one in ignore.test + const message: Memory = { + userId: user?.id as UUID, + content: { text: "Bye" }, + roomId: roomId as UUID, + }; - const handler = action.handler!; + const handler = action.handler!; - await populateMemories(runtime, user, roomId, [Goodbye1]); + await populateMemories(runtime, user, roomId, [Goodbye1]); - const result = (await handler(runtime, message)) as Content; + const result = (await handler(runtime, message)) as Content; - return result.action === "IGNORE"; - }); - }, 60000); + return result.action === "IGNORE"; + }); + }, 60000); - // test conditions where we would expect a wait or an ignore + // test conditions where we would expect a wait or an ignore }); diff --git a/core/tests/evaluation.test.ts b/core/tests/evaluation.test.ts index f9446f3280..55d4c22087 100644 --- a/core/tests/evaluation.test.ts +++ b/core/tests/evaluation.test.ts @@ -5,8 +5,8 @@ import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { runAiTest } from "../src/test_resources/runAiTest.ts"; import { - TEST_EVALUATOR, - TEST_EVALUATOR_FAIL, + TEST_EVALUATOR, + TEST_EVALUATOR_FAIL, } from "../src/test_resources/testEvaluator.ts"; import { type User } from "../src/test_resources/types.ts"; import { composeContext } from "../src/core/context.ts"; @@ -16,113 +16,113 @@ import { IAgentRuntime, Memory, UUID } from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); describe("Evaluation Process", () => { - let runtime: IAgentRuntime; - let user: User; - let roomId: UUID; - - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - evaluators: [TEST_EVALUATOR, TEST_EVALUATOR_FAIL], + let runtime: IAgentRuntime; + let user: User; + let roomId: UUID; + + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + evaluators: [TEST_EVALUATOR, TEST_EVALUATOR_FAIL], + }); + runtime = setup.runtime; + user = setup.session.user; + + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data!.roomId; }); - runtime = setup.runtime; - user = setup.session.user; - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: zeroUuid, + test("Validate the format of the examples from the evaluator", () => { + expect(TEST_EVALUATOR.examples).toBeInstanceOf(Array); + TEST_EVALUATOR.examples.forEach((example) => { + expect(example).toHaveProperty("context"); + expect(example).toHaveProperty("messages"); + expect(example.messages).toBeInstanceOf(Array); + example.messages.forEach((message) => { + expect(message).toHaveProperty("user"); + expect(message).toHaveProperty("content"); + expect(message.content).toHaveProperty("action"); + }); + expect(example).toHaveProperty("outcome"); + }); }); - if (!data) { - throw new Error("Relationship not found"); - } - - roomId = data!.roomId; - }); - - test("Validate the format of the examples from the evaluator", () => { - expect(TEST_EVALUATOR.examples).toBeInstanceOf(Array); - TEST_EVALUATOR.examples.forEach((example) => { - expect(example).toHaveProperty("context"); - expect(example).toHaveProperty("messages"); - expect(example.messages).toBeInstanceOf(Array); - example.messages.forEach((message) => { - expect(message).toHaveProperty("user"); - expect(message).toHaveProperty("content"); - expect(message.content).toHaveProperty("action"); - }); - expect(example).toHaveProperty("outcome"); - }); - }); - - test("Check if test and examples appear in state", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "Test message for evaluation" }, - roomId, - }; - - const state = await runtime.composeState(message); - const prompt = composeContext({ state, template: evaluationTemplate }); - - // expect that the prompt contacts the testEvaluator name - expect(prompt).toContain(TEST_EVALUATOR.name); - - // check if state.EvaluatorNames contains the testEvaluator name - - expect(state.evaluatorNames).toContain(TEST_EVALUATOR.name); - }); - - test("Run the TEST_EVALUATOR handler and validate output", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "Run TEST_EVALUATOR handler" }, - roomId, - }; - - const result = await TEST_EVALUATOR.handler(runtime, message); - expect(result).toBeTruthy(); - }); - - test("Run the evaluation process", async () => { - await runAiTest("Run the evaluation process", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { - text: "We are in testing mode. We want to make sure that the test passes by replying with the evaluator TEST_EVALUATOR in the array of evaluators that are returned. Please run the TEST_EVALUATOR", - }, - roomId, - }; - - const state = await runtime.composeState(message); - const result = await runtime.evaluate(message, state); - - return result?.includes("TEST_EVALUATOR"); - }); // Adjust the timeout if needed - }, 600000); - - test("Test that fact appears in evaluation handler", async () => { - const { runtime } = await createRuntime({ - env: process.env as Record, - conversationLength: 1, - evaluators: [fact], - }); + test("Check if test and examples appear in state", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "Test message for evaluation" }, + roomId, + }; - const message: Memory = { - userId: user.id as UUID, - content: { text: "Test message for evaluation" }, - roomId, - }; + const state = await runtime.composeState(message); + const prompt = composeContext({ state, template: evaluationTemplate }); - const state = await runtime.composeState(message); - const prompt = composeContext({ state, template: evaluationTemplate }); + // expect that the prompt contacts the testEvaluator name + expect(prompt).toContain(TEST_EVALUATOR.name); - // expect that the prompt contacts the testEvaluator name - expect(prompt.includes(fact.name)).toBeTruthy(); + // check if state.EvaluatorNames contains the testEvaluator name + + expect(state.evaluatorNames).toContain(TEST_EVALUATOR.name); + }); - // check if state.EvaluatorNames contains the testEvaluator name + test("Run the TEST_EVALUATOR handler and validate output", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "Run TEST_EVALUATOR handler" }, + roomId, + }; - expect(state.evaluatorNames).toContain(fact.name); - }); + const result = await TEST_EVALUATOR.handler(runtime, message); + expect(result).toBeTruthy(); + }); + + test("Run the evaluation process", async () => { + await runAiTest("Run the evaluation process", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { + text: "We are in testing mode. We want to make sure that the test passes by replying with the evaluator TEST_EVALUATOR in the array of evaluators that are returned. Please run the TEST_EVALUATOR", + }, + roomId, + }; + + const state = await runtime.composeState(message); + const result = await runtime.evaluate(message, state); + + return result?.includes("TEST_EVALUATOR"); + }); // Adjust the timeout if needed + }, 600000); + + test("Test that fact appears in evaluation handler", async () => { + const { runtime } = await createRuntime({ + env: process.env as Record, + conversationLength: 1, + evaluators: [fact], + }); + + const message: Memory = { + userId: user.id as UUID, + content: { text: "Test message for evaluation" }, + roomId, + }; + + const state = await runtime.composeState(message); + const prompt = composeContext({ state, template: evaluationTemplate }); + + // expect that the prompt contacts the testEvaluator name + expect(prompt.includes(fact.name)).toBeTruthy(); + + // check if state.EvaluatorNames contains the testEvaluator name + + expect(state.evaluatorNames).toContain(fact.name); + }); }); diff --git a/core/tests/fact.test.ts b/core/tests/fact.test.ts index 8e99fc863b..ab13da53ee 100644 --- a/core/tests/fact.test.ts +++ b/core/tests/fact.test.ts @@ -2,16 +2,16 @@ import dotenv from "dotenv"; import { defaultActions } from "../src/core/actions.ts"; import { IAgentRuntime, type Memory, type UUID } from "../src/core/types.ts"; import { - getCachedEmbeddings, - writeCachedEmbedding, + getCachedEmbeddings, + writeCachedEmbedding, } from "../src/test_resources/cache.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { - GetTellMeAboutYourselfConversation1, - GetTellMeAboutYourselfConversation2, - GetTellMeAboutYourselfConversation3, - jimFacts, + GetTellMeAboutYourselfConversation1, + GetTellMeAboutYourselfConversation2, + GetTellMeAboutYourselfConversation3, + jimFacts, } from "../src/test_resources/data.ts"; import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { populateMemories } from "../src/test_resources/populateMemories.ts"; @@ -22,111 +22,114 @@ import evaluator from "../src/evaluators/fact.ts"; dotenv.config({ path: ".dev.vars" }); describe("Facts Evaluator", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId = zeroUuid; - - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - evaluators: [evaluator], - actions: defaultActions, + let user: User; + let runtime: IAgentRuntime; + let roomId = zeroUuid; + + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + evaluators: [evaluator], + actions: defaultActions, + }); + user = setup.session.user; + runtime = setup.runtime; + + if (!user.id) { + throw new Error("User ID is undefined"); + } + + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data.roomId; }); - user = setup.session.user; - runtime = setup.runtime; - if (!user.id) { - throw new Error("User ID is undefined"); - } - - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: zeroUuid, + afterAll(async () => { + await cleanup(runtime, user.id as UUID); }); - if (!data) { - throw new Error("Relationship not found"); - } - - roomId = data.roomId; - }); - - afterAll(async () => { - await cleanup(runtime, user.id as UUID); - }); - - test("Extract facts from conversations", async () => { - await runAiTest("Extract programmer and startup facts", async () => { - await populateMemories(runtime, user, roomId, [ - GetTellMeAboutYourselfConversation1, - ]); - - const message: Memory = { - userId: user.id as UUID, - content: { text: "" }, - roomId, - }; - - const result = await evaluator.handler(runtime, message); - const resultConcatenated = result.join("\n"); - - return ( - resultConcatenated.toLowerCase().includes("programmer") && - resultConcatenated.toLowerCase().includes("startup") - ); - }); - - await runAiTest("Extract married fact, ignoring known facts", async () => { - await populateMemories(runtime, user, roomId, [ - GetTellMeAboutYourselfConversation2, - GetTellMeAboutYourselfConversation3, - ]); - - await addFacts(runtime, user.id as UUID, roomId, jimFacts); - - const message: Memory = { - userId: user.id as UUID, - content: { text: "" }, - roomId, - }; - - const result = await evaluator.handler(runtime, message); - const resultConcatenated = result.join("\n"); - - return ( - !resultConcatenated.toLowerCase().includes("francisco") && - !resultConcatenated.toLowerCase().includes("38") && - resultConcatenated.toLowerCase().includes("married") - ); - }); - }, 120000); // Adjust the timeout as needed for your tests + test("Extract facts from conversations", async () => { + await runAiTest("Extract programmer and startup facts", async () => { + await populateMemories(runtime, user, roomId, [ + GetTellMeAboutYourselfConversation1, + ]); + + const message: Memory = { + userId: user.id as UUID, + content: { text: "" }, + roomId, + }; + + const result = await evaluator.handler(runtime, message); + const resultConcatenated = result.join("\n"); + + return ( + resultConcatenated.toLowerCase().includes("programmer") && + resultConcatenated.toLowerCase().includes("startup") + ); + }); + + await runAiTest( + "Extract married fact, ignoring known facts", + async () => { + await populateMemories(runtime, user, roomId, [ + GetTellMeAboutYourselfConversation2, + GetTellMeAboutYourselfConversation3, + ]); + + await addFacts(runtime, user.id as UUID, roomId, jimFacts); + + const message: Memory = { + userId: user.id as UUID, + content: { text: "" }, + roomId, + }; + + const result = await evaluator.handler(runtime, message); + const resultConcatenated = result.join("\n"); + + return ( + !resultConcatenated.toLowerCase().includes("francisco") && + !resultConcatenated.toLowerCase().includes("38") && + resultConcatenated.toLowerCase().includes("married") + ); + } + ); + }, 120000); // Adjust the timeout as needed for your tests }); async function cleanup(runtime: IAgentRuntime, roomId: UUID) { - await runtime.factManager.removeAllMemories(roomId); - await runtime.messageManager.removeAllMemories(roomId); + await runtime.factManager.removeAllMemories(roomId); + await runtime.messageManager.removeAllMemories(roomId); } async function addFacts( - runtime: IAgentRuntime, - userId: UUID, - roomId: UUID, - facts: string[], + runtime: IAgentRuntime, + userId: UUID, + roomId: UUID, + facts: string[] ) { - for (const fact of facts) { - const existingEmbedding = await getCachedEmbeddings(fact); - const bakedMemory = await runtime.factManager.addEmbeddingToMemory({ - userId: userId, - content: { text: fact }, - roomId: roomId, - embedding: existingEmbedding, - }); - await runtime.factManager.createMemory(bakedMemory); - if (!existingEmbedding) { - writeCachedEmbedding(fact, bakedMemory.embedding as number[]); - // Ensure there's a slight delay for asynchronous operations to complete - await new Promise((resolve) => setTimeout(resolve, 200)); + for (const fact of facts) { + const existingEmbedding = await getCachedEmbeddings(fact); + const bakedMemory = await runtime.factManager.addEmbeddingToMemory({ + userId: userId, + content: { text: fact }, + roomId: roomId, + embedding: existingEmbedding, + }); + await runtime.factManager.createMemory(bakedMemory); + if (!existingEmbedding) { + writeCachedEmbedding(fact, bakedMemory.embedding as number[]); + // Ensure there's a slight delay for asynchronous operations to complete + await new Promise((resolve) => setTimeout(resolve, 200)); + } } - } } diff --git a/core/tests/goal.test.ts b/core/tests/goal.test.ts index 5f28dd928b..5105a311f7 100644 --- a/core/tests/goal.test.ts +++ b/core/tests/goal.test.ts @@ -2,13 +2,13 @@ import dotenv from "dotenv"; import { defaultActions } from "../src/core/actions.ts"; import { createGoal, getGoals } from "../src/core/goals.ts"; import { - Goal, - GoalStatus, - IAgentRuntime, - Objective, - State, - type Memory, - type UUID, + Goal, + GoalStatus, + IAgentRuntime, + Objective, + State, + type Memory, + type UUID, } from "../src/core/types.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; @@ -21,164 +21,182 @@ import evaluator from "../src/evaluators/goal.ts"; dotenv.config({ path: ".dev.vars" }); describe("Goals Evaluator", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId: UUID; - - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - evaluators: [evaluator], - actions: defaultActions, - }); - user = setup.session.user; - runtime = setup.runtime; + let user: User; + let runtime: IAgentRuntime; + let roomId: UUID; + + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + evaluators: [evaluator], + actions: defaultActions, + }); + user = setup.session.user; + runtime = setup.runtime; - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: zeroUuid, - }); + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: zeroUuid, + }); - if (!data) { - throw new Error("Relationship not found"); - } + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data.roomId; - roomId = data.roomId; - - await cleanup(); - }); - - afterEach(async () => { - await cleanup(); - }); - - async function cleanup() { - // delete all goals for the user - await runtime.databaseAdapter.removeAllMemories(roomId, "goals"); - } - - async function createTestGoal(name: string, objectives: Objective[]) { - const result = await createGoal({ - runtime, - goal: { - name, - status: GoalStatus.IN_PROGRESS, - roomId, - userId: user.id as UUID, - objectives, - }, + await cleanup(); }); - return result; - } - test("Update goal objectives based on conversation", async () => { - await runAiTest( - "Update goal objectives based on conversation", - async () => { + afterEach(async () => { await cleanup(); + }); - await createTestGoal("Test Goal", [ - { description: "Complete task 1", completed: false }, - { description: "Complete task 2", completed: false }, - ]); - - // Simulate a conversation indicating failure to achieve "Goal Y" - const conversation = (userId: UUID) => [ - { - userId, - content: { text: "I see that you've finished the task?" }, - }, - { - userId: zeroUuid, - content: { - text: "Yes, the task and all objectives are finished.", - }, - }, - ]; - - await populateMemories(runtime, user, roomId, [conversation]); - - // Simulate a conversation indicating the generateText of both objectives - const message: Memory = { - userId: user.id as UUID, - content: { - text: "I've completed task 1 and task 2 for the Test Goal. Both are finished. Everything is done and I'm ready for the next goal.", - }, - roomId, - }; - - // Process the message with the goal evaluator - await evaluator.handler(runtime, message, {} as unknown as State, { - onlyInProgress: false, - }); + async function cleanup() { + // delete all goals for the user + await runtime.databaseAdapter.removeAllMemories(roomId, "goals"); + } - // Fetch the updated goal to verify the objectives and status were updated - const updatedGoals = await getGoals({ - runtime, - roomId, - onlyInProgress: false, + async function createTestGoal(name: string, objectives: Objective[]) { + const result = await createGoal({ + runtime, + goal: { + name, + status: GoalStatus.IN_PROGRESS, + roomId, + userId: user.id as UUID, + objectives, + }, }); + return result; + } - const updatedTestGoal = updatedGoals.find( - (goal: Goal) => goal.name === "Test Goal", + test("Update goal objectives based on conversation", async () => { + await runAiTest( + "Update goal objectives based on conversation", + async () => { + await cleanup(); + + await createTestGoal("Test Goal", [ + { description: "Complete task 1", completed: false }, + { description: "Complete task 2", completed: false }, + ]); + + // Simulate a conversation indicating failure to achieve "Goal Y" + const conversation = (userId: UUID) => [ + { + userId, + content: { + text: "I see that you've finished the task?", + }, + }, + { + userId: zeroUuid, + content: { + text: "Yes, the task and all objectives are finished.", + }, + }, + ]; + + await populateMemories(runtime, user, roomId, [conversation]); + + // Simulate a conversation indicating the generateText of both objectives + const message: Memory = { + userId: user.id as UUID, + content: { + text: "I've completed task 1 and task 2 for the Test Goal. Both are finished. Everything is done and I'm ready for the next goal.", + }, + roomId, + }; + + // Process the message with the goal evaluator + await evaluator.handler( + runtime, + message, + {} as unknown as State, + { + onlyInProgress: false, + } + ); + + // Fetch the updated goal to verify the objectives and status were updated + const updatedGoals = await getGoals({ + runtime, + roomId, + onlyInProgress: false, + }); + + const updatedTestGoal = updatedGoals.find( + (goal: Goal) => goal.name === "Test Goal" + ); + + return ( + updatedTestGoal !== undefined && + updatedTestGoal.status === GoalStatus.DONE && + updatedTestGoal.objectives.every( + (obj: Objective) => obj.completed + ) + ); + } ); - - return ( - updatedTestGoal !== undefined && - updatedTestGoal.status === GoalStatus.DONE && - updatedTestGoal.objectives.every((obj: Objective) => obj.completed) + }, 60000); + + test("Goal status updated to FAILED based on conversation", async () => { + await runAiTest( + "Goal status updated to FAILED based on conversation", + async () => { + await cleanup(); + // Preparing the test goal "Goal Y" + await createTestGoal("Goal Y", [ + { + description: "Complete all tasks for Goal Y", + completed: false, + }, + ]); + + // Simulate a conversation indicating failure to achieve "Goal Y" + const conversation = (userId: UUID) => [ + { + userId, + content: { + text: "I couldn't complete the tasks for Goal Y.", + }, + }, + { + userId: zeroUuid, + content: { + text: "That's unfortunate. Let's cancel it..", + }, + }, + ]; + + await populateMemories(runtime, user, roomId, [conversation]); + + const message: Memory = { + userId: user.id as UUID, + content: { text: "I've decided to mark Goal Y as failed." }, + roomId, + }; + + await evaluator.handler(runtime, message, {} as State, { + onlyInProgress: false, + }); + + const goals = await getGoals({ + runtime, + roomId, + onlyInProgress: false, + }); + + const goalY = goals.find( + (goal: Goal) => goal.name === "Goal Y" + ); + + return ( + goalY !== undefined && goalY.status === GoalStatus.FAILED + ); + } ); - }, - ); - }, 60000); - - test("Goal status updated to FAILED based on conversation", async () => { - await runAiTest( - "Goal status updated to FAILED based on conversation", - async () => { - await cleanup(); - // Preparing the test goal "Goal Y" - await createTestGoal("Goal Y", [ - { description: "Complete all tasks for Goal Y", completed: false }, - ]); - - // Simulate a conversation indicating failure to achieve "Goal Y" - const conversation = (userId: UUID) => [ - { - userId, - content: { text: "I couldn't complete the tasks for Goal Y." }, - }, - { - userId: zeroUuid, - content: { - text: "That's unfortunate. Let's cancel it..", - }, - }, - ]; - - await populateMemories(runtime, user, roomId, [conversation]); - - const message: Memory = { - userId: user.id as UUID, - content: { text: "I've decided to mark Goal Y as failed." }, - roomId, - }; - - await evaluator.handler(runtime, message, {} as State, { - onlyInProgress: false, - }); - - const goals = await getGoals({ - runtime, - roomId, - onlyInProgress: false, - }); - - const goalY = goals.find((goal: Goal) => goal.name === "Goal Y"); - - return goalY !== undefined && goalY.status === GoalStatus.FAILED; - }, - ); - }, 60000); + }, 60000); }); diff --git a/core/tests/goals.test.ts b/core/tests/goals.test.ts index f7766ab663..6938adfac3 100644 --- a/core/tests/goals.test.ts +++ b/core/tests/goals.test.ts @@ -3,114 +3,121 @@ import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { type User } from "../src/test_resources/types.ts"; import { createGoal, getGoals, updateGoal } from "../src/core/goals.ts"; -import { GoalStatus, IAgentRuntime, type Goal, type UUID } from "../src/core/types.ts"; +import { + GoalStatus, + IAgentRuntime, + type Goal, + type UUID, +} from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); describe("Goals", () => { - let runtime: IAgentRuntime; - let user: User; - beforeAll(async () => { - const result = await createRuntime({ - env: process.env as Record, + let runtime: IAgentRuntime; + let user: User; + beforeAll(async () => { + const result = await createRuntime({ + env: process.env as Record, + }); + runtime = result.runtime; + user = result.session.user; + await runtime.databaseAdapter.removeAllGoals(zeroUuid); }); - runtime = result.runtime; - user = result.session.user; - await runtime.databaseAdapter.removeAllGoals(zeroUuid); - }); - beforeEach(async () => { - await runtime.databaseAdapter.removeAllGoals(zeroUuid); - }); - - afterAll(async () => { - await runtime.databaseAdapter.removeAllGoals(zeroUuid); - }); - - test("createGoal - successfully creates a new goal", async () => { - const newGoal: Goal = { - name: "Test Create Goal", - status: GoalStatus.IN_PROGRESS, - roomId: zeroUuid, - userId: user?.id as UUID, - objectives: [ - { - description: "Test Objective", - completed: false, - }, - ], - }; - - await createGoal({ - runtime, - goal: newGoal, + beforeEach(async () => { + await runtime.databaseAdapter.removeAllGoals(zeroUuid); }); - // Verify the goal is created in the database - const goals = await getGoals({ - runtime, - userId: user?.id as UUID, - roomId: zeroUuid, - onlyInProgress: false, + afterAll(async () => { + await runtime.databaseAdapter.removeAllGoals(zeroUuid); }); - const createdGoal = goals.find((goal: Goal) => goal.name === newGoal.name); + test("createGoal - successfully creates a new goal", async () => { + const newGoal: Goal = { + name: "Test Create Goal", + status: GoalStatus.IN_PROGRESS, + roomId: zeroUuid, + userId: user?.id as UUID, + objectives: [ + { + description: "Test Objective", + completed: false, + }, + ], + }; - expect(createdGoal).toBeDefined(); - expect(createdGoal?.status).toEqual("IN_PROGRESS"); - expect(createdGoal?.objectives.length).toBeGreaterThan(0); - }); + await createGoal({ + runtime, + goal: newGoal, + }); - // Updating an existing goal - test("updateGoals - successfully updates an existing goal", async () => { - const newGoal: Goal = { - name: "Test Create Goal", - status: GoalStatus.IN_PROGRESS, - roomId: zeroUuid, - userId: user?.id as UUID, - objectives: [ - { - description: "Test Objective", - completed: false, - }, - ], - }; + // Verify the goal is created in the database + const goals = await getGoals({ + runtime, + userId: user?.id as UUID, + roomId: zeroUuid, + onlyInProgress: false, + }); - await createGoal({ - runtime, - goal: newGoal, - }); + const createdGoal = goals.find( + (goal: Goal) => goal.name === newGoal.name + ); - // retrieve the goal from the database - let goals = await getGoals({ - runtime, - roomId: zeroUuid, - onlyInProgress: false, - }); - const existingGoal = goals.find( - (goal: Goal) => goal.name === newGoal.name, - ) as Goal; - const updatedGoal = { ...existingGoal, status: GoalStatus.DONE }; - await updateGoal({ - runtime, - goal: updatedGoal, + expect(createdGoal).toBeDefined(); + expect(createdGoal?.status).toEqual("IN_PROGRESS"); + expect(createdGoal?.objectives.length).toBeGreaterThan(0); }); - // Verify the goal's status is updated in the database - goals = await getGoals({ - runtime, - roomId: zeroUuid, - onlyInProgress: false, - }); + // Updating an existing goal + test("updateGoals - successfully updates an existing goal", async () => { + const newGoal: Goal = { + name: "Test Create Goal", + status: GoalStatus.IN_PROGRESS, + roomId: zeroUuid, + userId: user?.id as UUID, + objectives: [ + { + description: "Test Objective", + completed: false, + }, + ], + }; + + await createGoal({ + runtime, + goal: newGoal, + }); + + // retrieve the goal from the database + let goals = await getGoals({ + runtime, + roomId: zeroUuid, + onlyInProgress: false, + }); + const existingGoal = goals.find( + (goal: Goal) => goal.name === newGoal.name + ) as Goal; + const updatedGoal = { ...existingGoal, status: GoalStatus.DONE }; + await updateGoal({ + runtime, + goal: updatedGoal, + }); - const updatedGoalInDb = goals.find( - (goal: Goal) => goal.id === existingGoal.id, - ); + // Verify the goal's status is updated in the database + goals = await getGoals({ + runtime, + roomId: zeroUuid, + onlyInProgress: false, + }); - expect(updatedGoalInDb?.status).toEqual(GoalStatus.DONE); + const updatedGoalInDb = goals.find( + (goal: Goal) => goal.id === existingGoal.id + ); - // Clean up the created goal - if (existingGoal?.id) { - await runtime.databaseAdapter.removeGoal(existingGoal.id); - } - }); + expect(updatedGoalInDb?.status).toEqual(GoalStatus.DONE); + + // Clean up the created goal + if (existingGoal?.id) { + await runtime.databaseAdapter.removeGoal(existingGoal.id); + } + }); }); diff --git a/core/tests/ignore.test.ts b/core/tests/ignore.test.ts index 0cd9bc3a33..5119150259 100644 --- a/core/tests/ignore.test.ts +++ b/core/tests/ignore.test.ts @@ -3,18 +3,18 @@ import { zeroUuid } from "../src/test_resources/constants.ts"; import { composeContext } from "../src/core/context.ts"; import { embeddingZeroVector } from "../src/core/memory.ts"; import { - Content, - IAgentRuntime, - Memory, - ModelClass, - State, - type UUID, + Content, + IAgentRuntime, + Memory, + ModelClass, + State, + type UUID, } from "../src/core/types.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { - GetTellMeAboutYourselfConversationTroll1, - GetTellMeAboutYourselfConversationTroll2, - Goodbye1, + GetTellMeAboutYourselfConversationTroll1, + GetTellMeAboutYourselfConversationTroll2, + Goodbye1, } from "../src/test_resources/data.ts"; import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { populateMemories } from "../src/test_resources/populateMemories.ts"; @@ -25,199 +25,199 @@ import action from "../src/actions/ignore.ts"; import { generateMessageResponse } from "../src/core/generation.ts"; async function handleMessage( - runtime: IAgentRuntime, - message: Memory, - state?: State, + runtime: IAgentRuntime, + message: Memory, + state?: State ) { - const _saveRequestMessage = async (message: Memory, state: State) => { - const { content: senderContent, userId, roomId } = message; - - const _senderContent = (senderContent as Content).text?.trim(); - if (_senderContent) { - await runtime.messageManager.createMemory({ - userId: userId!, - content: { - text: _senderContent, - action: (message.content as Content)?.action ?? "null", - }, + const _saveRequestMessage = async (message: Memory, state: State) => { + const { content: senderContent, userId, roomId } = message; + + const _senderContent = (senderContent as Content).text?.trim(); + if (_senderContent) { + await runtime.messageManager.createMemory({ + userId: userId!, + content: { + text: _senderContent, + action: (message.content as Content)?.action ?? "null", + }, + roomId, + embedding: embeddingZeroVector, + }); + } + }; + + await _saveRequestMessage(message, state as State); + if (!state) { + state = (await runtime.composeState(message)) as State; + } + + const context = composeContext({ + state, + template: messageHandlerTemplate, + }); + + const { userId, roomId } = message; + + const response = await generateMessageResponse({ + context, + runtime, + modelClass: ModelClass.SMALL, + }); + + await runtime.databaseAdapter.log({ + body: { message, context, response }, + userId: userId, + roomId, + type: "ignore_test_completion", + }); + + const responseMessage: Memory = { + userId: runtime.agentId, + content: response, roomId, embedding: embeddingZeroVector, - }); + }; + + if (responseMessage.content.text?.trim()) { + await runtime.messageManager.createMemory(responseMessage); + await runtime.evaluate(message, state); + await runtime.processActions(message, [responseMessage]); + } else { + console.warn("Empty response, skipping"); } - }; - - await _saveRequestMessage(message, state as State); - if (!state) { - state = (await runtime.composeState(message)) as State; - } - - const context = composeContext({ - state, - template: messageHandlerTemplate, - }); - - const { userId, roomId } = message; - - const response = await generateMessageResponse({ - context, - runtime, - modelClass: ModelClass.SMALL, - }); - - await runtime.databaseAdapter.log({ - body: { message, context, response }, - userId: userId, - roomId, - type: "ignore_test_completion", - }); - - const responseMessage: Memory = { - userId: runtime.agentId, - content: response, - roomId, - embedding: embeddingZeroVector, - }; - - if (responseMessage.content.text?.trim()) { - await runtime.messageManager.createMemory(responseMessage); - await runtime.evaluate(message, state); - await runtime.processActions(message, [responseMessage]); - } else { - console.warn("Empty response, skipping"); - } - - return responseMessage; + + return responseMessage; } // use .dev.vars for local testing dotenv.config({ path: ".dev.vars" }); describe("Ignore action tests", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId: UUID; - - afterAll(async () => { - await cleanup(); - }); - - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - actions: [action], - }); - user = setup.session.user; - runtime = setup.runtime; + let user: User; + let runtime: IAgentRuntime; + let roomId: UUID; - const data = await getOrCreateRelationship({ - runtime, - userA: user?.id as UUID, - userB: zeroUuid, + afterAll(async () => { + await cleanup(); }); - console.log("data is", data); + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + actions: [action], + }); + user = setup.session.user; + runtime = setup.runtime; - roomId = data?.roomId; - console.log("*** data", data); - console.log("Room ID", roomId); + const data = await getOrCreateRelationship({ + runtime, + userA: user?.id as UUID, + userB: zeroUuid, + }); - await cleanup(); - }); + console.log("data is", data); - beforeEach(async () => { - await cleanup(); - }); + roomId = data?.roomId; + console.log("*** data", data); + console.log("Room ID", roomId); - async function cleanup() { - await runtime.factManager.removeAllMemories(roomId); - await runtime.messageManager.removeAllMemories(roomId); - } + await cleanup(); + }); - test("Test ignore action", async () => { - await runAiTest("Test ignore action", async () => { - const message: Memory = { - userId: user?.id as UUID, - content: { text: "Never talk to me again" }, - roomId: roomId as UUID, - }; + beforeEach(async () => { + await cleanup(); + }); - await populateMemories(runtime, user, roomId, [ - GetTellMeAboutYourselfConversationTroll1, - ]); + async function cleanup() { + await runtime.factManager.removeAllMemories(roomId); + await runtime.messageManager.removeAllMemories(roomId); + } - const result = await handleMessage(runtime, message); + test("Test ignore action", async () => { + await runAiTest("Test ignore action", async () => { + const message: Memory = { + userId: user?.id as UUID, + content: { text: "Never talk to me again" }, + roomId: roomId as UUID, + }; - return result.content.action === "IGNORE"; - }); - }, 120000); + await populateMemories(runtime, user, roomId, [ + GetTellMeAboutYourselfConversationTroll1, + ]); - test("Action handler test 1: response should be ignore", async () => { - await runAiTest( - "Action handler test 1: response should be ignore", - async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "", action: "IGNORE" }, - roomId: roomId as UUID, - }; + const result = await handleMessage(runtime, message); - await populateMemories(runtime, user, roomId, [ - GetTellMeAboutYourselfConversationTroll1, - ]); + return result.content.action === "IGNORE"; + }); + }, 120000); - await handleMessage(runtime, message); + test("Action handler test 1: response should be ignore", async () => { + await runAiTest( + "Action handler test 1: response should be ignore", + async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "", action: "IGNORE" }, + roomId: roomId as UUID, + }; - const state = await runtime.composeState(message); + await populateMemories(runtime, user, roomId, [ + GetTellMeAboutYourselfConversationTroll1, + ]); - const lastMessage = state.recentMessagesData[0]; + await handleMessage(runtime, message); - return (lastMessage.content as Content).action === "IGNORE"; - }, - ); - }, 120000); + const state = await runtime.composeState(message); - test("Action handler test 2: response should be ignore", async () => { - await runAiTest( - "Action handler test 2: response should be ignore", - async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "", action: "IGNORE" }, - roomId: roomId as UUID, - }; + const lastMessage = state.recentMessagesData[0]; - await populateMemories(runtime, user, roomId, [ - GetTellMeAboutYourselfConversationTroll2, - ]); + return (lastMessage.content as Content).action === "IGNORE"; + } + ); + }, 120000); - await handleMessage(runtime, message); + test("Action handler test 2: response should be ignore", async () => { + await runAiTest( + "Action handler test 2: response should be ignore", + async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "", action: "IGNORE" }, + roomId: roomId as UUID, + }; - const state = await runtime.composeState(message); + await populateMemories(runtime, user, roomId, [ + GetTellMeAboutYourselfConversationTroll2, + ]); - const lastMessage = state.recentMessagesData[0]; + await handleMessage(runtime, message); - return (lastMessage.content as Content).action === "IGNORE"; - }, - ); - }, 120000); + const state = await runtime.composeState(message); - test("Expect ignore", async () => { - await runAiTest("Expect ignore", async () => { - const message: Memory = { - userId: user.id as UUID, - content: { text: "Bye" }, - roomId: roomId as UUID, - }; + const lastMessage = state.recentMessagesData[0]; - await populateMemories(runtime, user, roomId, [Goodbye1]); + return (lastMessage.content as Content).action === "IGNORE"; + } + ); + }, 120000); - await handleMessage(runtime, message); + test("Expect ignore", async () => { + await runAiTest("Expect ignore", async () => { + const message: Memory = { + userId: user.id as UUID, + content: { text: "Bye" }, + roomId: roomId as UUID, + }; - const state = await runtime.composeState(message); + await populateMemories(runtime, user, roomId, [Goodbye1]); - const lastMessage = state.recentMessagesData[0]; + await handleMessage(runtime, message); - return (lastMessage.content as Content).action === "IGNORE"; - }); - }, 120000); + const state = await runtime.composeState(message); + + const lastMessage = state.recentMessagesData[0]; + + return (lastMessage.content as Content).action === "IGNORE"; + }); + }, 120000); }); diff --git a/core/tests/memory.test.ts b/core/tests/memory.test.ts index a31ded98c8..1e84437282 100644 --- a/core/tests/memory.test.ts +++ b/core/tests/memory.test.ts @@ -1,7 +1,7 @@ import dotenv from "dotenv"; import { - getCachedEmbeddings, - writeCachedEmbedding, + getCachedEmbeddings, + writeCachedEmbedding, } from "../src/test_resources/cache.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; @@ -13,571 +13,604 @@ import { embed } from "../src/core/embedding.ts"; dotenv.config({ path: ".dev.vars" }); describe("Memory", () => { - let memoryManager: MemoryManager; - let runtime = null; - let user: User; - let roomId: UUID = zeroUuid; - - beforeAll(async () => { - const result = await createRuntime({ - env: process.env as Record, + let memoryManager: MemoryManager; + let runtime = null; + let user: User; + let roomId: UUID = zeroUuid; + + beforeAll(async () => { + const result = await createRuntime({ + env: process.env as Record, + }); + runtime = result.runtime; + user = result.session.user; + + const data = await getOrCreateRelationship({ + runtime, + userA: user?.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data.roomId; + + memoryManager = new MemoryManager({ + tableName: "messages", + runtime, + }); }); - runtime = result.runtime; - user = result.session.user; - const data = await getOrCreateRelationship({ - runtime, - userA: user?.id as UUID, - userB: zeroUuid, + beforeEach(async () => { + await memoryManager.removeAllMemories(roomId); }); - if (!data) { - throw new Error("Relationship not found"); - } - - roomId = data.roomId; - - memoryManager = new MemoryManager({ - tableName: "messages", - runtime, - }); - }); - - beforeEach(async () => { - await memoryManager.removeAllMemories(roomId); - }); - - afterAll(async () => { - await memoryManager.removeAllMemories(roomId); - }); - - test("Search memories by embedding similarity", async () => { - // Define a base memory and two additional memories, one similar and one dissimilar - const baseMemoryContent = "Base memory content for testing similarity"; - const similarMemoryContent = - "Base memory content for testing similarity - Similar memory content to the base memory"; - const dissimilarMemoryContent = "Dissimilar memory content, not related"; - - // Create and add embedding to the base memory - const baseMemory = await embed(memoryManager.runtime, baseMemoryContent); - - let embedding = await getCachedEmbeddings(similarMemoryContent); - - // Create and add embedding to the similar and dissimilar memories - const similarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: similarMemoryContent }, - roomId: roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - similarMemoryContent, - similarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(similarMemory); - - embedding = await getCachedEmbeddings(dissimilarMemoryContent); - - const dissimilarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: dissimilarMemoryContent }, - roomId, - embedding, + afterAll(async () => { + await memoryManager.removeAllMemories(roomId); }); - if (!embedding) { - writeCachedEmbedding( - dissimilarMemoryContent, - dissimilarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(dissimilarMemory); - - // Search for memories similar to the base memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - baseMemory!, - { - roomId, - count: 1, - }, - ); - - // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower - expect( - searchedMemories.some( - (memory) => (memory.content as Content).text === similarMemoryContent, - ), - ).toBe(true); - expect( - searchedMemories.some( - (memory) => - (memory.content as Content).text === dissimilarMemoryContent, - ), - ).toBe(false); - }, 60000); - - test("Verify memory similarity and ranking", async () => { - // Define a set of memories with varying degrees of similarity - const queryMemoryContent = "High similarity content to the query memory"; - const highSimilarityContent = "High similarity content to the query memory"; - const lowSimilarityContent = "Low similarity content compared to the query"; - - let embedding = await getCachedEmbeddings(queryMemoryContent); - - // Create and add embedding to the query memory - const queryMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: queryMemoryContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - queryMemoryContent, - queryMemory.embedding as number[], - ); - } - await memoryManager.createMemory(queryMemory); - - embedding = await getCachedEmbeddings(highSimilarityContent); - // Create and add embedding to the high and low similarity memories - const highSimilarityMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: highSimilarityContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - highSimilarityContent, - highSimilarityMemory.embedding as number[], - ); - } - await memoryManager.createMemory(highSimilarityMemory); - - embedding = await getCachedEmbeddings(lowSimilarityContent); - const lowSimilarityMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: lowSimilarityContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - lowSimilarityContent, - lowSimilarityMemory.embedding as number[], - ); - } - await memoryManager.createMemory(lowSimilarityMemory); - - // Search for memories similar to the query memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - queryMemory.embedding!, - { - roomId, - count: 10, - }, - ); - - // Check that the high similarity memory ranks higher than the low similarity memory - const highSimilarityIndex = searchedMemories.findIndex( - (memory) => (memory.content as Content).text === highSimilarityContent, - ); - const lowSimilarityIndex = searchedMemories.findIndex( - (memory) => (memory.content as Content).text === lowSimilarityContent, - ); - - expect(highSimilarityIndex).toBeLessThan(lowSimilarityIndex); - }, 60000); + + test("Search memories by embedding similarity", async () => { + // Define a base memory and two additional memories, one similar and one dissimilar + const baseMemoryContent = "Base memory content for testing similarity"; + const similarMemoryContent = + "Base memory content for testing similarity - Similar memory content to the base memory"; + const dissimilarMemoryContent = + "Dissimilar memory content, not related"; + + // Create and add embedding to the base memory + const baseMemory = await embed( + memoryManager.runtime, + baseMemoryContent + ); + + let embedding = await getCachedEmbeddings(similarMemoryContent); + + // Create and add embedding to the similar and dissimilar memories + const similarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: similarMemoryContent }, + roomId: roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + similarMemoryContent, + similarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(similarMemory); + + embedding = await getCachedEmbeddings(dissimilarMemoryContent); + + const dissimilarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: dissimilarMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + dissimilarMemoryContent, + dissimilarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(dissimilarMemory); + + // Search for memories similar to the base memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + baseMemory!, + { + roomId, + count: 1, + } + ); + + // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower + expect( + searchedMemories.some( + (memory) => + (memory.content as Content).text === similarMemoryContent + ) + ).toBe(true); + expect( + searchedMemories.some( + (memory) => + (memory.content as Content).text === dissimilarMemoryContent + ) + ).toBe(false); + }, 60000); + + test("Verify memory similarity and ranking", async () => { + // Define a set of memories with varying degrees of similarity + const queryMemoryContent = + "High similarity content to the query memory"; + const highSimilarityContent = + "High similarity content to the query memory"; + const lowSimilarityContent = + "Low similarity content compared to the query"; + + let embedding = await getCachedEmbeddings(queryMemoryContent); + + // Create and add embedding to the query memory + const queryMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: queryMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + queryMemoryContent, + queryMemory.embedding as number[] + ); + } + await memoryManager.createMemory(queryMemory); + + embedding = await getCachedEmbeddings(highSimilarityContent); + // Create and add embedding to the high and low similarity memories + const highSimilarityMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: highSimilarityContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + highSimilarityContent, + highSimilarityMemory.embedding as number[] + ); + } + await memoryManager.createMemory(highSimilarityMemory); + + embedding = await getCachedEmbeddings(lowSimilarityContent); + const lowSimilarityMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: lowSimilarityContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + lowSimilarityContent, + lowSimilarityMemory.embedding as number[] + ); + } + await memoryManager.createMemory(lowSimilarityMemory); + + // Search for memories similar to the query memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + queryMemory.embedding!, + { + roomId, + count: 10, + } + ); + + // Check that the high similarity memory ranks higher than the low similarity memory + const highSimilarityIndex = searchedMemories.findIndex( + (memory) => + (memory.content as Content).text === highSimilarityContent + ); + const lowSimilarityIndex = searchedMemories.findIndex( + (memory) => + (memory.content as Content).text === lowSimilarityContent + ); + + expect(highSimilarityIndex).toBeLessThan(lowSimilarityIndex); + }, 60000); }); describe("Memory - Basic tests", () => { - let memoryManager: MemoryManager; - let runtime = null; - let user: User; - let roomId: UUID; - - // Setup before all tests - beforeAll(async () => { - const result = await createRuntime({ - env: process.env as Record, + let memoryManager: MemoryManager; + let runtime = null; + let user: User; + let roomId: UUID; + + // Setup before all tests + beforeAll(async () => { + const result = await createRuntime({ + env: process.env as Record, + }); + runtime = result.runtime; + user = result.session.user; + + const data = await getOrCreateRelationship({ + runtime, + userA: user?.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data.roomId; + + memoryManager = new MemoryManager({ + tableName: "messages", // Adjust based on your actual table name + runtime, + }); }); - runtime = result.runtime; - user = result.session.user; - const data = await getOrCreateRelationship({ - runtime, - userA: user?.id as UUID, - userB: zeroUuid, + // Cleanup after all tests + afterAll(async () => { + await memoryManager.removeAllMemories(roomId); }); - if (!data) { - throw new Error("Relationship not found"); - } - - roomId = data.roomId; - - memoryManager = new MemoryManager({ - tableName: "messages", // Adjust based on your actual table name - runtime, - }); - }); - - // Cleanup after all tests - afterAll(async () => { - await memoryManager.removeAllMemories(roomId); - }); - - test("Memory lifecycle: create, search, count, and remove", async () => { - const embedding = await getCachedEmbeddings( - "Test content for memory lifecycle", - ); - // Create a test memory - const testMemory: Memory = await memoryManager.addEmbeddingToMemory({ - userId: user.id as UUID, - content: { text: "Test content for memory lifecycle" }, - roomId: roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - (testMemory.content as Content).text as string, - testMemory.embedding as number[], - ); - } - await memoryManager.createMemory(testMemory); - - const createdMemories = await memoryManager.getMemories({ - roomId, - count: 100, + test("Memory lifecycle: create, search, count, and remove", async () => { + const embedding = await getCachedEmbeddings( + "Test content for memory lifecycle" + ); + // Create a test memory + const testMemory: Memory = await memoryManager.addEmbeddingToMemory({ + userId: user.id as UUID, + content: { text: "Test content for memory lifecycle" }, + roomId: roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + (testMemory.content as Content).text as string, + testMemory.embedding as number[] + ); + } + await memoryManager.createMemory(testMemory); + + const createdMemories = await memoryManager.getMemories({ + roomId, + count: 100, + }); + + // Verify creation by counting memories + const initialCount = await memoryManager.countMemories(roomId, false); + expect(initialCount).toBeGreaterThan(0); + + // Search memories by embedding + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + testMemory.embedding!, + { + roomId, + count: 5, + } + ); + expect(searchedMemories.length).toBeGreaterThan(0); + + // Remove a specific memory + await memoryManager.removeMemory(createdMemories[0].id!); + const afterRemovalCount = await memoryManager.countMemories(roomId); + expect(afterRemovalCount).toBeLessThan(initialCount); + + // Remove all memories for the test user + await memoryManager.removeAllMemories(roomId); + const finalCount = await memoryManager.countMemories(roomId); + expect(finalCount).toEqual(0); }); - - // Verify creation by counting memories - const initialCount = await memoryManager.countMemories(roomId, false); - expect(initialCount).toBeGreaterThan(0); - - // Search memories by embedding - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - testMemory.embedding!, - { - roomId, - count: 5, - }, - ); - expect(searchedMemories.length).toBeGreaterThan(0); - - // Remove a specific memory - await memoryManager.removeMemory(createdMemories[0].id!); - const afterRemovalCount = await memoryManager.countMemories(roomId); - expect(afterRemovalCount).toBeLessThan(initialCount); - - // Remove all memories for the test user - await memoryManager.removeAllMemories(roomId); - const finalCount = await memoryManager.countMemories(roomId); - expect(finalCount).toEqual(0); - }); }); describe("Memory - Extended Tests", () => { - let memoryManager: MemoryManager; - let runtime = null; - let user: User; - let roomId: UUID; - - beforeAll(async () => { - const result = await createRuntime({ - env: process.env as Record, + let memoryManager: MemoryManager; + let runtime = null; + let user: User; + let roomId: UUID; + + beforeAll(async () => { + const result = await createRuntime({ + env: process.env as Record, + }); + runtime = result.runtime; + user = result.session.user; + + const data = await getOrCreateRelationship({ + runtime, + userA: user.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + + roomId = data.roomId; + + if (!roomId) throw new Error("Room not found"); + + memoryManager = new MemoryManager({ + tableName: "messages", + runtime, + }); }); - runtime = result.runtime; - user = result.session.user; - const data = await getOrCreateRelationship({ - runtime, - userA: user.id as UUID, - userB: zeroUuid, + beforeEach(async () => { + await memoryManager.removeAllMemories(roomId); }); - if (!data) { - throw new Error("Relationship not found"); - } - - roomId = data.roomId; - - if (!roomId) throw new Error("Room not found"); - - memoryManager = new MemoryManager({ - tableName: "messages", - runtime, + afterAll(async () => { + await memoryManager.removeAllMemories(roomId); }); - }); - - beforeEach(async () => { - await memoryManager.removeAllMemories(roomId); - }); - - afterAll(async () => { - await memoryManager.removeAllMemories(roomId); - }); - - test("Test cosine similarity value equality", async () => { - // Define a base memory and two additional memories, one similar and one dissimilar - const baseMemoryContent = "Base memory content for testing similarity"; - const similarMemoryContent = "Base memory content for testing similarity"; - // Create and add embedding to the base memory - const baseMemory = await embed(memoryManager.runtime, baseMemoryContent); - - const embedding = await getCachedEmbeddings(similarMemoryContent); - - // Create and add embedding to the similar and dissimilar memories - const similarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: similarMemoryContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - similarMemoryContent, - similarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(similarMemory); - - // Search for memories similar to the base memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - baseMemory!, - { - roomId, - count: 1, - }, - ); - - const memory = searchedMemories[0]; - - const similarity = (memory as unknown as { similarity: number }).similarity; - expect(similarity).toBeGreaterThan(0.9); - }); - - test("Test cosine similarity value inequality", async () => { - // Define a base memory and two additional memories, one similar and one dissimilar - const baseMemoryContent = "i love u"; - const similarMemoryContent = "Cognitive security in the information age"; - - // Create and add embedding to the base memory - const baseMemory = await embed(memoryManager.runtime, baseMemoryContent); - - const embedding = await getCachedEmbeddings(similarMemoryContent); - - // Create and add embedding to the similar and dissimilar memories - const similarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: similarMemoryContent }, - roomId: roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - similarMemoryContent, - similarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(similarMemory); - - // Search for memories similar to the base memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - baseMemory!, - { - match_threshold: 0.01, - roomId, - count: 1, - }, - ); - - const memory = searchedMemories[0]; - - const similarity = (memory as unknown as { similarity: number }).similarity; - - expect(similarity).toBeLessThan(0.2); - }); - - test("Test unique insert", async () => { - // Define a base memory and two additional memories, one similar and one dissimilar - const memoryContent = "Cognitive security in the information age"; - const similarMemoryContent = "Cognitive security in the information age"; - - let embedding = await getCachedEmbeddings(memoryContent); - - // Create and add embedding to the similar and dissimilar memories - const newMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: memoryContent }, - roomId, - embedding, + test("Test cosine similarity value equality", async () => { + // Define a base memory and two additional memories, one similar and one dissimilar + const baseMemoryContent = "Base memory content for testing similarity"; + const similarMemoryContent = + "Base memory content for testing similarity"; + + // Create and add embedding to the base memory + const baseMemory = await embed( + memoryManager.runtime, + baseMemoryContent + ); + + const embedding = await getCachedEmbeddings(similarMemoryContent); + + // Create and add embedding to the similar and dissimilar memories + const similarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: similarMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + similarMemoryContent, + similarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(similarMemory); + + // Search for memories similar to the base memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + baseMemory!, + { + roomId, + count: 1, + } + ); + + const memory = searchedMemories[0]; + + const similarity = (memory as unknown as { similarity: number }) + .similarity; + expect(similarity).toBeGreaterThan(0.9); }); - if (!embedding) { - writeCachedEmbedding(memoryContent, newMemory.embedding as number[]); - } - await memoryManager.createMemory(newMemory, true); - - embedding = await getCachedEmbeddings(similarMemoryContent); - - // Create and add embedding to the similar and dissimilar memories - const similarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: similarMemoryContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - similarMemoryContent, - similarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(similarMemory, true); - - const allCount = await memoryManager.countMemories(roomId, false); - const uniqueCount = await memoryManager.countMemories(roomId, true); - - expect(allCount > uniqueCount).toBe(true); - }); - - test("Search memories by embedding similarity", async () => { - // Define a base memory and two additional memories, one similar and one dissimilar - const baseMemoryContent = "Base memory content for testing similarity"; - const similarMemoryContent = "Base memory content for testing similarity 2"; - const dissimilarMemoryContent = "Dissimilar, not related"; - - // Create and add embedding to the base memory - const baseMemory = await embed(memoryManager.runtime, baseMemoryContent); - - let embedding = await getCachedEmbeddings(similarMemoryContent); - - // Create and add embedding to the similar and dissimilar memories - const similarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: similarMemoryContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - similarMemoryContent, - similarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(similarMemory); - - embedding = await getCachedEmbeddings(dissimilarMemoryContent); - - const dissimilarMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: dissimilarMemoryContent }, - roomId, - embedding: await getCachedEmbeddings(dissimilarMemoryContent), - }); - if (!embedding) { - writeCachedEmbedding( - dissimilarMemoryContent, - dissimilarMemory.embedding as number[], - ); - } - await memoryManager.createMemory(dissimilarMemory); - - // Search for memories similar to the base memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - baseMemory!, - { - roomId, - count: 1, - }, - ); - - // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower - expect( - searchedMemories.some( - (memory) => (memory.content as Content).text === similarMemoryContent, - ), - ).toBe(true); - expect( - searchedMemories.some( - (memory) => - (memory.content as Content).text === dissimilarMemoryContent, - ), - ).toBe(false); - }, 60000); - - test("Verify memory similarity and ranking", async () => { - // Define a set of memories with varying degrees of similarity - const queryMemoryContent = "High similarity content to the query memory"; - const highSimilarityContent = "High similarity content to the query memory"; - const lowSimilarityContent = "Low similarity, not related"; - - let embedding = await getCachedEmbeddings(queryMemoryContent); - - // Create and add embedding to the query memory - const queryMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: queryMemoryContent }, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding( - queryMemoryContent, - queryMemory.embedding as number[], - ); - } - await memoryManager.createMemory(queryMemory); - - embedding = await getCachedEmbeddings(highSimilarityContent); - // Create and add embedding to the high and low similarity memories - const highSimilarityMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: highSimilarityContent }, - roomId, - embedding, + + test("Test cosine similarity value inequality", async () => { + // Define a base memory and two additional memories, one similar and one dissimilar + const baseMemoryContent = "i love u"; + const similarMemoryContent = + "Cognitive security in the information age"; + + // Create and add embedding to the base memory + const baseMemory = await embed( + memoryManager.runtime, + baseMemoryContent + ); + + const embedding = await getCachedEmbeddings(similarMemoryContent); + + // Create and add embedding to the similar and dissimilar memories + const similarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: similarMemoryContent }, + roomId: roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + similarMemoryContent, + similarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(similarMemory); + + // Search for memories similar to the base memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + baseMemory!, + { + match_threshold: 0.01, + roomId, + count: 1, + } + ); + + const memory = searchedMemories[0]; + + const similarity = (memory as unknown as { similarity: number }) + .similarity; + + expect(similarity).toBeLessThan(0.2); }); - if (!embedding) { - writeCachedEmbedding( - highSimilarityContent, - highSimilarityMemory.embedding as number[], - ); - } - await memoryManager.createMemory(highSimilarityMemory); - - embedding = await getCachedEmbeddings(lowSimilarityContent); - const lowSimilarityMemory = await memoryManager.addEmbeddingToMemory({ - userId: user?.id as UUID, - content: { text: lowSimilarityContent }, - roomId, - embedding, + + test("Test unique insert", async () => { + // Define a base memory and two additional memories, one similar and one dissimilar + const memoryContent = "Cognitive security in the information age"; + const similarMemoryContent = + "Cognitive security in the information age"; + + let embedding = await getCachedEmbeddings(memoryContent); + + // Create and add embedding to the similar and dissimilar memories + const newMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: memoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + memoryContent, + newMemory.embedding as number[] + ); + } + await memoryManager.createMemory(newMemory, true); + + embedding = await getCachedEmbeddings(similarMemoryContent); + + // Create and add embedding to the similar and dissimilar memories + const similarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: similarMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + similarMemoryContent, + similarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(similarMemory, true); + + const allCount = await memoryManager.countMemories(roomId, false); + const uniqueCount = await memoryManager.countMemories(roomId, true); + + expect(allCount > uniqueCount).toBe(true); }); - if (!embedding) { - writeCachedEmbedding( - lowSimilarityContent, - lowSimilarityMemory.embedding as number[], - ); - } - await memoryManager.createMemory(lowSimilarityMemory); - - // Search for memories similar to the query memory - const searchedMemories = await memoryManager.searchMemoriesByEmbedding( - queryMemory.embedding!, - { - roomId, - count: 10, - }, - ); - - // Check that the high similarity memory ranks higher than the low similarity memory - const highSimilarityIndex = searchedMemories.findIndex( - (memory) => (memory.content as Content).text === highSimilarityContent, - ); - const lowSimilarityIndex = searchedMemories.findIndex( - (memory) => (memory.content as Content).text === lowSimilarityContent, - ); - - expect(highSimilarityIndex).toBeLessThan(lowSimilarityIndex); - }, 60000); + + test("Search memories by embedding similarity", async () => { + // Define a base memory and two additional memories, one similar and one dissimilar + const baseMemoryContent = "Base memory content for testing similarity"; + const similarMemoryContent = + "Base memory content for testing similarity 2"; + const dissimilarMemoryContent = "Dissimilar, not related"; + + // Create and add embedding to the base memory + const baseMemory = await embed( + memoryManager.runtime, + baseMemoryContent + ); + + let embedding = await getCachedEmbeddings(similarMemoryContent); + + // Create and add embedding to the similar and dissimilar memories + const similarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: similarMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + similarMemoryContent, + similarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(similarMemory); + + embedding = await getCachedEmbeddings(dissimilarMemoryContent); + + const dissimilarMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: dissimilarMemoryContent }, + roomId, + embedding: await getCachedEmbeddings(dissimilarMemoryContent), + }); + if (!embedding) { + writeCachedEmbedding( + dissimilarMemoryContent, + dissimilarMemory.embedding as number[] + ); + } + await memoryManager.createMemory(dissimilarMemory); + + // Search for memories similar to the base memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + baseMemory!, + { + roomId, + count: 1, + } + ); + + // Check that the similar memory is included in the search results and the dissimilar one is not or ranks lower + expect( + searchedMemories.some( + (memory) => + (memory.content as Content).text === similarMemoryContent + ) + ).toBe(true); + expect( + searchedMemories.some( + (memory) => + (memory.content as Content).text === dissimilarMemoryContent + ) + ).toBe(false); + }, 60000); + + test("Verify memory similarity and ranking", async () => { + // Define a set of memories with varying degrees of similarity + const queryMemoryContent = + "High similarity content to the query memory"; + const highSimilarityContent = + "High similarity content to the query memory"; + const lowSimilarityContent = "Low similarity, not related"; + + let embedding = await getCachedEmbeddings(queryMemoryContent); + + // Create and add embedding to the query memory + const queryMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: queryMemoryContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + queryMemoryContent, + queryMemory.embedding as number[] + ); + } + await memoryManager.createMemory(queryMemory); + + embedding = await getCachedEmbeddings(highSimilarityContent); + // Create and add embedding to the high and low similarity memories + const highSimilarityMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: highSimilarityContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + highSimilarityContent, + highSimilarityMemory.embedding as number[] + ); + } + await memoryManager.createMemory(highSimilarityMemory); + + embedding = await getCachedEmbeddings(lowSimilarityContent); + const lowSimilarityMemory = await memoryManager.addEmbeddingToMemory({ + userId: user?.id as UUID, + content: { text: lowSimilarityContent }, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + lowSimilarityContent, + lowSimilarityMemory.embedding as number[] + ); + } + await memoryManager.createMemory(lowSimilarityMemory); + + // Search for memories similar to the query memory + const searchedMemories = await memoryManager.searchMemoriesByEmbedding( + queryMemory.embedding!, + { + roomId, + count: 10, + } + ); + + // Check that the high similarity memory ranks higher than the low similarity memory + const highSimilarityIndex = searchedMemories.findIndex( + (memory) => + (memory.content as Content).text === highSimilarityContent + ); + const lowSimilarityIndex = searchedMemories.findIndex( + (memory) => + (memory.content as Content).text === lowSimilarityContent + ); + + expect(highSimilarityIndex).toBeLessThan(lowSimilarityIndex); + }, 60000); }); diff --git a/core/tests/messages.test.ts b/core/tests/messages.test.ts index 72d8b1eff1..3b454f9e5f 100644 --- a/core/tests/messages.test.ts +++ b/core/tests/messages.test.ts @@ -4,105 +4,111 @@ import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { type User } from "../src/test_resources/types.ts"; -import { formatActors, formatMessages, getActorDetails } from "../src/core/messages.ts"; +import { + formatActors, + formatMessages, + getActorDetails, +} from "../src/core/messages.ts"; import { createRelationship } from "../src/core/relationships.ts"; import { - IAgentRuntime, - type Actor, - type Content, - type Memory, - type UUID, + IAgentRuntime, + type Actor, + type Content, + type Memory, + type UUID, } from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); describe("Messages Library", () => { - let runtime: IAgentRuntime, user: User, actors: Actor[]; + let runtime: IAgentRuntime, user: User, actors: Actor[]; - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - }); - runtime = setup.runtime; - user = setup.session.user; - actors = await getActorDetails({ - runtime, - roomId: "00000000-0000-0000-0000-000000000000", + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + }); + runtime = setup.runtime; + user = setup.session.user; + actors = await getActorDetails({ + runtime, + roomId: "00000000-0000-0000-0000-000000000000", + }); }); - }); - test("getActorDetails should return actors based on given roomId", async () => { - // create a room and add a user to it - const userA = user?.id as UUID; - const userB = zeroUuid; + test("getActorDetails should return actors based on given roomId", async () => { + // create a room and add a user to it + const userA = user?.id as UUID; + const userB = zeroUuid; - await createRelationship({ - runtime, - userA, - userB, - }); + await createRelationship({ + runtime, + userA, + userB, + }); - const { roomId } = await getOrCreateRelationship({ - runtime, - userA, - userB, - }); + const { roomId } = await getOrCreateRelationship({ + runtime, + userA, + userB, + }); - const result = await getActorDetails({ - runtime, - roomId, - }); + const result = await getActorDetails({ + runtime, + roomId, + }); - expect(result.length).toBeGreaterThan(0); - result.forEach((actor: Actor) => { - expect(actor).toHaveProperty("name"); - expect(actor).toHaveProperty("details"); - expect(actor).toHaveProperty("id"); + expect(result.length).toBeGreaterThan(0); + result.forEach((actor: Actor) => { + expect(actor).toHaveProperty("name"); + expect(actor).toHaveProperty("details"); + expect(actor).toHaveProperty("id"); + }); }); - }); - test("formatActors should format actors into a readable string", () => { - const formattedActors = formatActors({ actors }); - actors.forEach((actor) => { - expect(formattedActors).toContain(actor.name); + test("formatActors should format actors into a readable string", () => { + const formattedActors = formatActors({ actors }); + actors.forEach((actor) => { + expect(formattedActors).toContain(actor.name); + }); }); - }); - test("formatMessages should format messages into a readable string", async () => { - const messages: Memory[] = [ - { - content: { text: "Hello" }, - userId: user.id as UUID, - roomId: "00000000-0000-0000-0000-000000000000", - }, - { - content: { text: "How are you?" }, - userId: "00000000-0000-0000-0000-000000000000", - roomId: "00000000-0000-0000-0000-000000000000", - }, - ]; - const formattedMessages = formatMessages({ messages, actors }); - messages.forEach((message: Memory) => { - expect(formattedMessages).toContain((message.content as Content).text); + test("formatMessages should format messages into a readable string", async () => { + const messages: Memory[] = [ + { + content: { text: "Hello" }, + userId: user.id as UUID, + roomId: "00000000-0000-0000-0000-000000000000", + }, + { + content: { text: "How are you?" }, + userId: "00000000-0000-0000-0000-000000000000", + roomId: "00000000-0000-0000-0000-000000000000", + }, + ]; + const formattedMessages = formatMessages({ messages, actors }); + messages.forEach((message: Memory) => { + expect(formattedMessages).toContain( + (message.content as Content).text + ); + }); }); - }); - test("formatFacts should format facts into a readable string", async () => { - const facts: Memory[] = [ - { - content: { text: "Reflecting on the day" }, - userId: user.id as UUID, - roomId: "00000000-0000-0000-0000-000000000000", - }, - { - content: { text: "Thoughts and musings" }, - userId: "00000000-0000-0000-0000-000000000000", - roomId: "00000000-0000-0000-0000-000000000000room", - }, - ]; - const formattedFacts = formatFacts(facts); - facts.forEach((fact) => { - expect(formattedFacts).toContain(fact.content.text); + test("formatFacts should format facts into a readable string", async () => { + const facts: Memory[] = [ + { + content: { text: "Reflecting on the day" }, + userId: user.id as UUID, + roomId: "00000000-0000-0000-0000-000000000000", + }, + { + content: { text: "Thoughts and musings" }, + userId: "00000000-0000-0000-0000-000000000000", + roomId: "00000000-0000-0000-0000-000000000000room", + }, + ]; + const formattedFacts = formatFacts(facts); + facts.forEach((fact) => { + expect(formattedFacts).toContain(fact.content.text); + }); }); - }); }); diff --git a/core/tests/providers.test.ts b/core/tests/providers.test.ts index 095689849c..17fb24f952 100644 --- a/core/tests/providers.test.ts +++ b/core/tests/providers.test.ts @@ -2,51 +2,50 @@ import dotenv from "dotenv"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; import { - IAgentRuntime, - type Memory, - type Provider, - type State, - type UUID, + IAgentRuntime, + type Memory, + type Provider, + type State, + type UUID, } from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); const TestProvider: Provider = { - - get: async (_runtime: IAgentRuntime, _message: Memory, _state?: State) => { - return "Hello Test"; - }, + get: async (_runtime: IAgentRuntime, _message: Memory, _state?: State) => { + return "Hello Test"; + }, }; describe("TestProvider", () => { - let runtime: IAgentRuntime; - let roomId: UUID; + let runtime: IAgentRuntime; + let roomId: UUID; - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - providers: [TestProvider], + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + providers: [TestProvider], + }); + runtime = setup.runtime; + roomId = zeroUuid; }); - runtime = setup.runtime; - roomId = zeroUuid; - }); - test("TestProvider should return 'Hello Test'", async () => { - const message: Memory = { - userId: zeroUuid, - content: { text: "" }, - roomId: roomId, - }; + test("TestProvider should return 'Hello Test'", async () => { + const message: Memory = { + userId: zeroUuid, + content: { text: "" }, + roomId: roomId, + }; - const testProviderResponse = await TestProvider.get( - runtime, - message, - {} as State, - ); - expect(testProviderResponse).toBe("Hello Test"); - }); + const testProviderResponse = await TestProvider.get( + runtime, + message, + {} as State + ); + expect(testProviderResponse).toBe("Hello Test"); + }); - test("TestProvider should be integrated in the runtime providers", async () => { - expect(runtime.providers).toContain(TestProvider); - }); + test("TestProvider should be integrated in the runtime providers", async () => { + expect(runtime.providers).toContain(TestProvider); + }); }); diff --git a/core/tests/relationships.test.ts b/core/tests/relationships.test.ts index c8e195a1fd..83e0b1759f 100644 --- a/core/tests/relationships.test.ts +++ b/core/tests/relationships.test.ts @@ -3,68 +3,71 @@ import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; // Adjust the import path as needed import { getOrCreateRelationship } from "../src/test_resources/getOrCreateRelationship.ts"; import { type User } from "../src/test_resources/types.ts"; -import { createRelationship, getRelationships } from "../src/core/relationships.ts"; // Adjust the import path as needed +import { + createRelationship, + getRelationships, +} from "../src/core/relationships.ts"; // Adjust the import path as needed import { IAgentRuntime, type UUID } from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); describe("Relationships Module", () => { - let runtime: IAgentRuntime; - let user: User; + let runtime: IAgentRuntime; + let user: User; - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + }); + runtime = setup.runtime; + user = setup.session.user; + if (!user.id) { + throw new Error("User ID is undefined"); + } }); - runtime = setup.runtime; - user = setup.session.user; - if (!user.id) { - throw new Error("User ID is undefined"); - } - }); - test("createRelationship creates a new relationship", async () => { - const userA = user.id as UUID; - const userB = zeroUuid; - if (userA === undefined) throw new Error("userA is undefined"); - const relationship = await createRelationship({ - runtime, - userA, - userB, + test("createRelationship creates a new relationship", async () => { + const userA = user.id as UUID; + const userB = zeroUuid; + if (userA === undefined) throw new Error("userA is undefined"); + const relationship = await createRelationship({ + runtime, + userA, + userB, + }); + expect(relationship).toBe(true); }); - expect(relationship).toBe(true); - }); - test("getRelationship retrieves an existing relationship", async () => { - const userA = user?.id as UUID; - const userB = zeroUuid; + test("getRelationship retrieves an existing relationship", async () => { + const userA = user?.id as UUID; + const userB = zeroUuid; - await createRelationship({ runtime, userA, userB }); + await createRelationship({ runtime, userA, userB }); - const relationship = await getOrCreateRelationship({ - runtime, - userA, - userB, + const relationship = await getOrCreateRelationship({ + runtime, + userA, + userB, + }); + expect(relationship).toBeDefined(); + expect(relationship?.userA).toBe(userA); + expect(relationship?.userB).toBe(userB); }); - expect(relationship).toBeDefined(); - expect(relationship?.userA).toBe(userA); - expect(relationship?.userB).toBe(userB); - }); - test("getRelationships retrieves all relationships for a user", async () => { - const userA = user?.id as UUID; - const userB = zeroUuid; + test("getRelationships retrieves all relationships for a user", async () => { + const userA = user?.id as UUID; + const userB = zeroUuid; - await createRelationship({ runtime, userA, userB }); + await createRelationship({ runtime, userA, userB }); - const relationships = await getRelationships({ - runtime, - userId: userA, + const relationships = await getRelationships({ + runtime, + userId: userA, + }); + expect(relationships).toBeDefined(); + expect(relationships.length).toBeGreaterThan(0); + expect( + relationships.some((r) => r.userA === userA || r.userB === userA) + ).toBeTruthy(); }); - expect(relationships).toBeDefined(); - expect(relationships.length).toBeGreaterThan(0); - expect( - relationships.some((r) => r.userA === userA || r.userB === userA), - ).toBeTruthy(); - }); }); diff --git a/core/tests/runtime.test.ts b/core/tests/runtime.test.ts index a537e3ce4d..fdf2a5a0cc 100644 --- a/core/tests/runtime.test.ts +++ b/core/tests/runtime.test.ts @@ -1,7 +1,7 @@ import dotenv from "dotenv"; import { - getCachedEmbeddings, - writeCachedEmbedding, + getCachedEmbeddings, + writeCachedEmbedding, } from "../src/test_resources/cache.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; @@ -12,98 +12,102 @@ import { IAgentRuntime, type Memory, type UUID } from "../src/core/types.ts"; dotenv.config({ path: ".dev.vars" }); describe("Agent Runtime", () => { - let user: User; - let runtime: IAgentRuntime; - let roomId: UUID = zeroUuid; - - // Helper function to clear memories - async function clearMemories() { - await runtime.messageManager.removeAllMemories(roomId); - } - - // Helper function to create memories - async function createMemories() { - const memories = [ - { - userId: user?.id as UUID, - content: { text: "test memory from user" }, - }, - { userId: zeroUuid, content: { text: "test memory from agent" } }, - ]; - - for (const { userId, content } of memories) { - try { - const embedding = await getCachedEmbeddings(content.text); - const memory = await runtime.messageManager.addEmbeddingToMemory({ - userId: userId, - content, - roomId, - embedding, - }); - if (!embedding) { - writeCachedEmbedding(content.text, memory.embedding as number[]); + let user: User; + let runtime: IAgentRuntime; + let roomId: UUID = zeroUuid; + + // Helper function to clear memories + async function clearMemories() { + await runtime.messageManager.removeAllMemories(roomId); + } + + // Helper function to create memories + async function createMemories() { + const memories = [ + { + userId: user?.id as UUID, + content: { text: "test memory from user" }, + }, + { userId: zeroUuid, content: { text: "test memory from agent" } }, + ]; + + for (const { userId, content } of memories) { + try { + const embedding = await getCachedEmbeddings(content.text); + const memory = + await runtime.messageManager.addEmbeddingToMemory({ + userId: userId, + content, + roomId, + embedding, + }); + if (!embedding) { + writeCachedEmbedding( + content.text, + memory.embedding as number[] + ); + } + await runtime.messageManager.createMemory(memory); + } catch (error) { + console.error("Error creating memory", error); + } } - await runtime.messageManager.createMemory(memory); - } catch (error) { - console.error("Error creating memory", error); - } } - } - // Set up before each test - beforeEach(async () => { - const result = await createRuntime({ - env: process.env as Record, + // Set up before each test + beforeEach(async () => { + const result = await createRuntime({ + env: process.env as Record, + }); + + runtime = result.runtime; + user = result.session.user; + + const data = await getOrCreateRelationship({ + runtime, + userA: user?.id as UUID, + userB: zeroUuid, + }); + + if (!data) { + throw new Error("Relationship not found"); + } + roomId = data.roomId; + await clearMemories(); // Clear memories before each test }); - runtime = result.runtime; - user = result.session.user; + // Clean up after each test + afterEach(async () => { + await clearMemories(); // Clear memories after each test to ensure a clean state + }); - const data = await getOrCreateRelationship({ - runtime, - userA: user?.id as UUID, - userB: zeroUuid, + test("Create an agent runtime instance and use the basic functionality", () => { + expect(user).toBeDefined(); + expect(runtime).toBeDefined(); }); - if (!data) { - throw new Error("Relationship not found"); - } - roomId = data.roomId; - await clearMemories(); // Clear memories before each test - }); - - // Clean up after each test - afterEach(async () => { - await clearMemories(); // Clear memories after each test to ensure a clean state - }); - - test("Create an agent runtime instance and use the basic functionality", () => { - expect(user).toBeDefined(); - expect(runtime).toBeDefined(); - }); - - test("Demonstrate idempotency by creating an agent runtime instance again", () => { - expect(user).toBeDefined(); - expect(runtime).toBeDefined(); - }); - - test("Memory lifecycle: create, retrieve, and destroy", async () => { - try { - await createMemories(); // Create new memories - } catch (error) { - console.error("Error creating memories", error); - } + test("Demonstrate idempotency by creating an agent runtime instance again", () => { + expect(user).toBeDefined(); + expect(runtime).toBeDefined(); + }); + + test("Memory lifecycle: create, retrieve, and destroy", async () => { + try { + await createMemories(); // Create new memories + } catch (error) { + console.error("Error creating memories", error); + } - const message: Memory = { - userId: user.id as UUID, - content: { text: "test message" }, - roomId: roomId as UUID, - }; + const message: Memory = { + userId: user.id as UUID, + content: { text: "test message" }, + roomId: roomId as UUID, + }; - const state = await runtime.composeState(message); + const state = await runtime.composeState(message); - expect(state.recentMessagesData.length).toBeGreaterThan(1); + expect(state.recentMessagesData.length).toBeGreaterThan(1); - await clearMemories(); - }, 60000); + await clearMemories(); + }, 60000); }); diff --git a/core/tests/time.test.ts b/core/tests/time.test.ts index a2c9a200e4..83489d05dc 100644 --- a/core/tests/time.test.ts +++ b/core/tests/time.test.ts @@ -1,10 +1,10 @@ import dotenv from "dotenv"; import { composeContext } from "../src/core/context.ts"; import { - IAgentRuntime, - type Memory, - type State, - type UUID, + IAgentRuntime, + type Memory, + type State, + type UUID, } from "../src/core/types.ts"; import { zeroUuid } from "../src/test_resources/constants.ts"; import { createRuntime } from "../src/test_resources/createRuntime.ts"; @@ -13,72 +13,72 @@ import timeProvider from "../src/providers/time.ts"; dotenv.config({ path: ".dev.vars" }); describe("Time Provider", () => { - let runtime: IAgentRuntime; - let user: { id: UUID }; - let roomId: UUID; + let runtime: IAgentRuntime; + let user: { id: UUID }; + let roomId: UUID; - beforeAll(async () => { - const setup = await createRuntime({ - env: process.env as Record, - providers: [timeProvider], + beforeAll(async () => { + const setup = await createRuntime({ + env: process.env as Record, + providers: [timeProvider], + }); + runtime = setup.runtime; + user = { id: setup.session.user?.id as UUID }; + roomId = zeroUuid; }); - runtime = setup.runtime; - user = { id: setup.session.user?.id as UUID }; - roomId = zeroUuid; - }); - test("Time provider should return the current time in the correct format", async () => { - const message: Memory = { - userId: user.id, - content: { text: "" }, - roomId: roomId, - }; + test("Time provider should return the current time in the correct format", async () => { + const message: Memory = { + userId: user.id, + content: { text: "" }, + roomId: roomId, + }; - const currentTimeResponse = await timeProvider.get( - runtime, - message, - {} as State, - ); - expect(currentTimeResponse).toMatch( - /^The current time is: \d{1,2}:\d{2}:\d{2}\s?(AM|PM)$/, - ); - }); + const currentTimeResponse = await timeProvider.get( + runtime, + message, + {} as State + ); + expect(currentTimeResponse).toMatch( + /^The current time is: \d{1,2}:\d{2}:\d{2}\s?(AM|PM)$/ + ); + }); - test("Time provider should be integrated in the state and context correctly", async () => { - const message: Memory = { - userId: user.id, - content: { text: "" }, - roomId: roomId, - }; + test("Time provider should be integrated in the state and context correctly", async () => { + const message: Memory = { + userId: user.id, + content: { text: "" }, + roomId: roomId, + }; - // Manually integrate the time provider's response into the state - const state = await runtime.composeState(message); + // Manually integrate the time provider's response into the state + const state = await runtime.composeState(message); - const contextTemplate = `Current Time: {{providers}}`; - const context = composeContext({ - state: state, - template: contextTemplate, - }); + const contextTemplate = `Current Time: {{providers}}`; + const context = composeContext({ + state: state, + template: contextTemplate, + }); - const match = context.match( - new RegExp( - `^Current Time: The current time is: \\d{1,2}:\\d{2}:\\d{2}\\s?(AM|PM)$`, - ), - ); + const match = context.match( + new RegExp( + `^Current Time: The current time is: \\d{1,2}:\\d{2}:\\d{2}\\s?(AM|PM)$` + ) + ); - expect(match).toBeTruthy(); - }); + expect(match).toBeTruthy(); + }); - test("Time provider should work independently", async () => { - const message: Memory = { - userId: user.id, - content: { text: "" }, - roomId: roomId, - }; - const currentTimeResponse = await timeProvider.get(runtime, message); + test("Time provider should work independently", async () => { + const message: Memory = { + userId: user.id, + content: { text: "" }, + roomId: roomId, + }; + const currentTimeResponse = await timeProvider.get(runtime, message); - expect(currentTimeResponse).toMatch( - /^The current time is: \d{1,2}:\d{2}:\d{2}\s?(AM|PM)$/, - ); - }); + expect(currentTimeResponse).toMatch( + /^The current time is: \d{1,2}:\d{2}:\d{2}\s?(AM|PM)$/ + ); + }); }); diff --git a/core/tests/token.test.ts b/core/tests/token.test.ts index 9a8f5808ba..46abfe6f8a 100644 --- a/core/tests/token.test.ts +++ b/core/tests/token.test.ts @@ -8,70 +8,70 @@ jest.mock("fs"); jest.mock("node-cache"); describe("TokenProvider Tests", () => { - // let connection: Connection; - let tokenProvider: TokenProvider; + // let connection: Connection; + let tokenProvider: TokenProvider; - beforeEach(() => { - // Initialize the connection and token provider before each test - // connection = new Connection("https://api.mainnet-beta.solana.com"); - tokenProvider = new TokenProvider( - "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh" - ); - }); - - test("should fetch token security data", async () => { - - const { runtime } = await createRuntime({ - conversationLength: 10, + beforeEach(() => { + // Initialize the connection and token provider before each test + // connection = new Connection("https://api.mainnet-beta.solana.com"); + tokenProvider = new TokenProvider( + "2weMjPLLybRMMva1fM3U31goWWrCpF59CHWNhnCJ9Vyh" + ); }); - // Mock the response for the fetchTokenSecurity call - const mockFetchResponse = { - success: true, - data: { - ownerBalance: "100", - creatorBalance: "50", - ownerPercentage: 10, - creatorPercentage: 5, - top10HolderBalance: "200", - top10HolderPercent: 20, - }, - }; + test("should fetch token security data", async () => { + const { runtime } = await createRuntime({ + conversationLength: 10, + }); - // Mock fetchWithRetry function - const fetchSpy = jest - .spyOn(tokenProvider as any, "fetchWithRetry") - .mockResolvedValue(mockFetchResponse); + // Mock the response for the fetchTokenSecurity call + const mockFetchResponse = { + success: true, + data: { + ownerBalance: "100", + creatorBalance: "50", + ownerPercentage: 10, + creatorPercentage: 5, + top10HolderBalance: "200", + top10HolderPercent: 20, + }, + }; - // Run the fetchTokenSecurity method - // const securityData = await tokenProvider.fetchTokenSecurity(); + // Mock fetchWithRetry function + const fetchSpy = jest + .spyOn(tokenProvider as any, "fetchWithRetry") + .mockResolvedValue(mockFetchResponse); - // Check if the data returned is correct - // expect(securityData).toEqual({ - // ownerBalance: "100", - // creatorBalance: "50", - // ownerPercentage: 10, - // creatorPercentage: 5, - // top10HolderBalance: "200", - // top10HolderPercent: 20, - // }); - //console.log the securityData - // console.log({ securityData }); + // Run the fetchTokenSecurity method + // const securityData = await tokenProvider.fetchTokenSecurity(); - // const holderList = await tokenProvider.fetchHolderList(); + // Check if the data returned is correct + // expect(securityData).toEqual({ + // ownerBalance: "100", + // creatorBalance: "50", + // ownerPercentage: 10, + // creatorPercentage: 5, + // top10HolderBalance: "200", + // top10HolderPercent: 20, + // }); + //console.log the securityData + // console.log({ securityData }); - // console.log({ holderList }); + // const holderList = await tokenProvider.fetchHolderList(); - // const tradeData = await tokenProvider.fetchTokenTradeData(); - // console.log({ tradeData }); + // console.log({ holderList }); - // const dexScreenerData = await tokenProvider.fetchDexScreenerData(); - // console.log({ dexScreenerData }); + // const tradeData = await tokenProvider.fetchTokenTradeData(); + // console.log({ tradeData }); - const tokenReport = await tokenProvider.getFormattedTokenReport(runtime); - console.log({ tokenReport }); + // const dexScreenerData = await tokenProvider.fetchDexScreenerData(); + // console.log({ dexScreenerData }); - // Ensure the mock was called - expect(fetchSpy).toHaveBeenCalled(); - }); + const tokenReport = + await tokenProvider.getFormattedTokenReport(runtime); + console.log({ tokenReport }); + + // Ensure the mock was called + expect(fetchSpy).toHaveBeenCalled(); + }); }); diff --git a/core/tests/utils.test.ts b/core/tests/utils.test.ts index 4b0e13d297..8fe285ee50 100644 --- a/core/tests/utils.test.ts +++ b/core/tests/utils.test.ts @@ -15,60 +15,62 @@ import { ModelProvider } from "../../core/types.ts"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); describe("buildConversationThread", () => { - let runtime: AgentRuntime; - let client: TwitterInteractionClient; + let runtime: AgentRuntime; + let client: TwitterInteractionClient; - beforeAll(async () => { - // Create an instance of the AgentRuntime - runtime = new AgentRuntime({ - databaseAdapter: new SqliteDatabaseAdapter(new Database(":memory:")), - token: settings.OPENAI_API_KEY as string, - evaluators: [], - modelProvider: ModelProvider.OPENAI, - character: defaultCharacter, - providers: [], - actions: [], - }); + beforeAll(async () => { + // Create an instance of the AgentRuntime + runtime = new AgentRuntime({ + databaseAdapter: new SqliteDatabaseAdapter( + new Database(":memory:") + ), + token: settings.OPENAI_API_KEY as string, + evaluators: [], + modelProvider: ModelProvider.OPENAI, + character: defaultCharacter, + providers: [], + actions: [], + }); - // Create an instance of the TwitterGenerationClient - client = new TwitterInteractionClient(runtime); + // Create an instance of the TwitterGenerationClient + client = new TwitterInteractionClient(runtime); - // Load cached Twitter credentials - const cookiesFilePath = path.join( - __dirname, - "../../../tweetcache/" + - runtime.getSetting("TWITTER_USERNAME") + - "_cookies.json" - ); - console.log("Cookies file path:", cookiesFilePath); - if (fs.existsSync(cookiesFilePath)) { - const cookiesArray = JSON.parse( - fs.readFileSync(cookiesFilePath, "utf-8") - ); - client.setCookiesFromArray(cookiesArray); - } else { - throw new Error( - "Twitter credentials not found. Please provide valid cookies.json." - ); - } - }); + // Load cached Twitter credentials + const cookiesFilePath = path.join( + __dirname, + "../../../tweetcache/" + + runtime.getSetting("TWITTER_USERNAME") + + "_cookies.json" + ); + console.log("Cookies file path:", cookiesFilePath); + if (fs.existsSync(cookiesFilePath)) { + const cookiesArray = JSON.parse( + fs.readFileSync(cookiesFilePath, "utf-8") + ); + client.setCookiesFromArray(cookiesArray); + } else { + throw new Error( + "Twitter credentials not found. Please provide valid cookies.json." + ); + } + }); - it("should build a conversation thread from a tweet ID", async () => { - const tweetId = "1830058678197895517"; + it("should build a conversation thread from a tweet ID", async () => { + const tweetId = "1830058678197895517"; - // Fetch the tweet from the API - const tweet = await client.getTweet(tweetId); - console.log("Original tweet:", JSON.stringify(tweet, null, 2)); + // Fetch the tweet from the API + const tweet = await client.getTweet(tweetId); + console.log("Original tweet:", JSON.stringify(tweet, null, 2)); - // Build the conversation thread - const thread = await buildConversationThread(tweet, client); + // Build the conversation thread + const thread = await buildConversationThread(tweet, client); - console.log("Generated conversation thread:"); - console.log(thread); + console.log("Generated conversation thread:"); + console.log(thread); - // Add assertions based on the expected structure and content of the thread - // expect(thread.includes("By: Aya Bochman (@ayaboch)")).toBe(true); - // expect(thread.includes("@ayaboch @DanBochman You should do nothing. Its opensource code, you have too much to lose by fighting this fight, this post will get u blacklisted be aware")).toBe(true); - // expect(thread.includes("@BLUECOW009 @ayaboch @DanBochman That's not how it works")).toBe(true); - }, 30000); + // Add assertions based on the expected structure and content of the thread + // expect(thread.includes("By: Aya Bochman (@ayaboch)")).toBe(true); + // expect(thread.includes("@ayaboch @DanBochman You should do nothing. Its opensource code, you have too much to lose by fighting this fight, this post will get u blacklisted be aware")).toBe(true); + // expect(thread.includes("@BLUECOW009 @ayaboch @DanBochman That's not how it works")).toBe(true); + }, 30000); }); diff --git a/core/tsconfig.json b/core/tsconfig.json index 0aec855d2e..43b1ed20e5 100644 --- a/core/tsconfig.json +++ b/core/tsconfig.json @@ -1,28 +1,28 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "es2022", - "lib": ["es2023", "dom"], - "types": ["node"], - "moduleResolution": "bundler", - "outDir": "./dist", - "rootDir": "./src", - "strict": false, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": false, - "allowImportingTsExtensions": true, - "declaration": true, - "emitDeclarationOnly": true, - "resolveJsonModule": true, - "noImplicitAny": false, - "allowJs": true, - "checkJs": false, - "noEmitOnError": false, - "moduleDetection": "force", - "allowArbitraryExtensions": true, - "typeRoots": ["./node_modules/@types", "./types"] - }, - "include": ["src/**/*",], - "exclude": ["node_modules", "dist", "src/**/*.d.ts", "types/**/*.test.ts"] -} +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "lib": ["es2023", "dom"], + "types": ["node"], + "moduleResolution": "bundler", + "outDir": "./dist", + "rootDir": "./src", + "strict": false, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": false, + "allowImportingTsExtensions": true, + "declaration": true, + "emitDeclarationOnly": true, + "resolveJsonModule": true, + "noImplicitAny": false, + "allowJs": true, + "checkJs": false, + "noEmitOnError": false, + "moduleDetection": "force", + "allowArbitraryExtensions": true, + "typeRoots": ["./node_modules/@types", "./types"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.d.ts", "types/**/*.test.ts"] +} diff --git a/core/types/index.d.ts b/core/types/index.d.ts index 9cea1ee12a..31200439a4 100644 --- a/core/types/index.d.ts +++ b/core/types/index.d.ts @@ -1,5 +1,4 @@ declare global { - interface Buffer extends Uint8Array { - } + interface Buffer extends Uint8Array {} } export {}; diff --git a/docs/.prettierrc.json b/docs/.prettierrc.json index 9e26dfeeb6..0967ef424b 100644 --- a/docs/.prettierrc.json +++ b/docs/.prettierrc.json @@ -1 +1 @@ -{} \ No newline at end of file +{} diff --git a/docs/README.md b/docs/README.md index e92e9e0613..0bda8646a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,21 +1,21 @@ # Eliza Documentation Site -This the official documentation site of eliza. A flexible, scalable and customizable agent for production apps. which Comes with batteries-including database, deployment and examples using Supabase and Cloudflare. +This the official documentation site of eliza. A flexible, scalable and customizable agent for production apps. which Comes with batteries-including database, deployment and examples using Supabase and Cloudflare. ### Installation -Currently eliza is dependent on Supabase for local development. You can install it with the following command: +Currently eliza is dependent on Supabase for local development. You can install it with the following command: npm install eliza # Select your database adapter npm install sqlite-vss better-sqlite3 # for sqlite (simple, for local development) - ``` - - npm install @supabase/supabase-js # for supabase (more complicated but can be deployed at scale) + ``` + npm install @supabase/supabase-js # for supabase (more complicated but can be deployed at scale) +``` ### Set up environment variables @@ -23,50 +23,54 @@ You will need a Supbase account, as well as an OpenAI developer account. Copy and paste the .dev.vars.example to .dev.vars and fill in the environment variables: - SUPABASE_URL="https://your-supabase-url.supabase.co" - SUPABASE_SERVICE_API_KEY="your-supabase-service-api-key" - OPENAI_API_KEY="your-openai-api-key" +SUPABASE_URL="https://your-supabase-url.supabase.co" +SUPABASE_SERVICE_API_KEY="your-supabase-service-api-key" +OPENAI_API_KEY="your-openai-api-key" ### SQLite Local Setup (Easiest) + You can use SQLite for local development. This is the easiest way to get started with eliza. - import { BgentRuntime, SqliteDatabaseAdapter } from "eliza"; + import { AgentRuntime, SqliteDatabaseAdapter } from "eliza"; import { Database } from "sqlite3"; const sqliteDatabaseAdapter = new SqliteDatabaseAdapter(new Database(":memory:")); - - const runtime = new BgentRuntime({ + + const runtime = new AgentRuntime({ serverUrl: "https://api.openai.com/v1", token: process.env.OPENAI_API_KEY, // Can be an API key or JWT token for your AI services databaseAdapter: sqliteDatabaseAdapter, // ... other options }); - + ### Supabase Local Setup + First, you will need to install the Supabase CLI. You can install it using the instructions here. Once you have the CLI installed, you can run the following commands to set up a local Supabase instance: supabase init + ``` supabase start ``` -You can now start the eliza project with `npm run dev` and it will connect to the local Supabase instance by default. +You can now start the eliza project with `npm run dev` and it will connect to the local Supabase instance by default. NOTE: You will need Docker installed for this to work. If that is an issue for you, use the Supabase Cloud Setup instructions instead below). ### Supabase Cloud Setup + This library uses Supabase as a database. You can set up a free account at supabase.io and create a new project. -* Step 1: On the Subase All Projects Dashboard, select “New Project”. -* Step 2: Select the organization to store the new project in, assign a database name, password and region. -* Step 3: Select “Create New Project”. -* Step 4: Wait for the database to setup. This will take a few minutes as supabase setups various directories. -* Step 5: Select the “SQL Editor” tab from the left navigation menu. -* Step 6: Copy in your own SQL dump file or optionally use the provided file in the eliza directory at: "src/supabase/db.sql". Note: You can use the command "supabase db dump" if you have a pre-exisiting supabase database to generate the SQL dump file. -* Step 7: Paste the SQL code into the SQL Editor and hit run in the bottom right. -* Step 8: Select the “Databases” tab from the left navigation menu to verify all of the tables have been added properly. +- Step 1: On the Subase All Projects Dashboard, select “New Project”. +- Step 2: Select the organization to store the new project in, assign a database name, password and region. +- Step 3: Select “Create New Project”. +- Step 4: Wait for the database to setup. This will take a few minutes as supabase setups various directories. +- Step 5: Select the “SQL Editor” tab from the left navigation menu. +- Step 6: Copy in your own SQL dump file or optionally use the provided file in the eliza directory at: "src/supabase/db.sql". Note: You can use the command "supabase db dump" if you have a pre-exisiting supabase database to generate the SQL dump file. +- Step 7: Paste the SQL code into the SQL Editor and hit run in the bottom right. +- Step 8: Select the “Databases” tab from the left navigation menu to verify all of the tables have been added properly. Once you've set up your Supabase project, you can find your API key by going to the "Settings" tab and then "API". You will need to set the` SUPABASE_URL and SUPABASE_SERVICE_API_KEY` environment variables in your `.dev.vars` file. @@ -74,33 +78,35 @@ Once you've set up your Supabase project, you can find your API key by going to While eliza uses ChatGPT 3.5 by default, you can use a local model by setting the serverUrl to a local endpoint. The LocalAI project is a great way to run a local model with a compatible API endpoint. - const runtime = new BgentRuntime({ + const runtime = new AgentRuntime({ serverUrl: process.env.LOCALAI_URL, token: process.env.LOCALAI_TOKEN, // Can be an API key or JWT token for your AI service // ... other options }); - ### Development npm run dev # start the server + ``` npm run shell # start the shell in another terminal to talk to the default agent ``` + ### Usage - import { BgentRuntime, SupabaseDatabaseAdapter, SqliteDatabaseAdapter } from "eliza"; - + + import { AgentRuntime, SupabaseDatabaseAdapter, SqliteDatabaseAdapter } from "eliza"; + const sqliteDatabaseAdapter = new SqliteDatabaseAdapter(new Database(":memory:")); - + ``` // You can also use Supabase like this // const supabaseDatabaseAdapter = new SupabaseDatabaseAdapter( // process.env.SUPABASE_URL, // process.env.SUPABASE_SERVICE_API_KEY) // ; - + ``` - const runtime = new BgentRuntime({ + const runtime = new AgentRuntime({ serverUrl: "https://api.openai.com/v1", token: process.env.OPENAI_API_KEY, // Can be an API key or JWT token for your AI services databaseAdapter: sqliteDatabaseAdapter, @@ -114,6 +120,6 @@ npm run shell # start the shell in another terminal to talk to the default agent embeddingModel: "text-embedding-3-small", // whatever model you want to use }); - ### what next? -it is good to interact with the eliza and read more about the documentation on https://www.eliza.org/docs + +it is good to interact with the eliza and read more about the documentation on https://www.eliza.org/docs diff --git a/docs/babel.config.js b/docs/babel.config.js index e00595dae7..bfd75dbdfc 100644 --- a/docs/babel.config.js +++ b/docs/babel.config.js @@ -1,3 +1,3 @@ module.exports = { - presets: [require.resolve('@docusaurus/core/lib/babel/preset')], + presets: [require.resolve("@docusaurus/core/lib/babel/preset")], }; diff --git a/docs/docs/_category_.yml b/docs/docs/_category_.yml index 24a4602617..c5d5974f55 100644 --- a/docs/docs/_category_.yml +++ b/docs/docs/_category_.yml @@ -1 +1 @@ -label: "API" \ No newline at end of file +label: "API" diff --git a/docs/docs/classes/BgentRuntime.md b/docs/docs/classes/BgentRuntime.md index 2526127e08..35607e9370 100644 --- a/docs/docs/classes/BgentRuntime.md +++ b/docs/docs/classes/BgentRuntime.md @@ -1,7 +1,7 @@ --- -id: "BgentRuntime" -title: "Class: BgentRuntime" -sidebar_label: "BgentRuntime" +id: "AgentRuntime" +title: "Class: AgentRuntime" +sidebar_label: "AgentRuntime" sidebar_position: 0 custom_edit_url: null --- @@ -13,31 +13,31 @@ action registration, and interaction with external services like OpenAI and Supa ### constructor -• **new BgentRuntime**(`opts`): [`BgentRuntime`](BgentRuntime.md) +• **new AgentRuntime**(`opts`): [`AgentRuntime`](AgentRuntime.md) -Creates an instance of BgentRuntime. +Creates an instance of AgentRuntime. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `opts` | `Object` | The options for configuring the BgentRuntime. | -| `opts.actions?` | [`Action`](../interfaces/Action.md)[] | Optional custom actions. | -| `opts.agentId?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional ID of the agent. | -| `opts.conversationLength?` | `number` | The number of messages to hold in the recent message cache. | -| `opts.databaseAdapter` | [`DatabaseAdapter`](DatabaseAdapter.md) | The database adapter used for interacting with the database. | -| `opts.debugMode?` | `boolean` | If true, debug messages will be logged. | -| `opts.embeddingModel?` | `string` | The model to use for embedding. | -| `opts.evaluators?` | [`Evaluator`](../interfaces/Evaluator.md)[] | Optional custom evaluators. | -| `opts.fetch?` | `unknown` | Custom fetch function to use for making requests. | -| `opts.model?` | `string` | The model to use for completion. | -| `opts.providers?` | [`Provider`](../interfaces/Provider.md)[] | Optional context providers. | -| `opts.serverUrl?` | `string` | The URL of the worker. | -| `opts.token` | `string` | The JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker. | +| Name | Type | Description | +| :------------------------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------- | +| `opts` | `Object` | The options for configuring the AgentRuntime. | +| `opts.actions?` | [`Action`](../interfaces/Action.md)[] | Optional custom actions. | +| `opts.agentId?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional ID of the agent. | +| `opts.conversationLength?` | `number` | The number of messages to hold in the recent message cache. | +| `opts.databaseAdapter` | [`DatabaseAdapter`](DatabaseAdapter.md) | The database adapter used for interacting with the database. | +| `opts.debugMode?` | `boolean` | If true, debug messages will be logged. | +| `opts.embeddingModel?` | `string` | The model to use for embedding. | +| `opts.evaluators?` | [`Evaluator`](../interfaces/Evaluator.md)[] | Optional custom evaluators. | +| `opts.fetch?` | `unknown` | Custom fetch function to use for making requests. | +| `opts.model?` | `string` | The model to use for completion. | +| `opts.providers?` | [`Provider`](../interfaces/Provider.md)[] | Optional context providers. | +| `opts.serverUrl?` | `string` | The URL of the worker. | +| `opts.token` | `string` | The JWT token, can be a JWT token if outside worker, or an OpenAI token if inside worker. | #### Returns -[`BgentRuntime`](BgentRuntime.md) +[`AgentRuntime`](AgentRuntime.md) ## Properties @@ -47,7 +47,7 @@ Creates an instance of BgentRuntime. Custom actions that the agent can perform. -___ +--- ### agentId @@ -55,7 +55,7 @@ ___ The ID of the agent -___ +--- ### databaseAdapter @@ -63,7 +63,7 @@ ___ The database adapter used for interacting with the database. -___ +--- ### debugMode @@ -71,7 +71,7 @@ ___ Indicates if debug messages should be logged. -___ +--- ### descriptionManager @@ -79,7 +79,7 @@ ___ Store and recall descriptions of users based on conversations. -___ +--- ### embeddingModel @@ -87,7 +87,7 @@ ___ The model to use for embedding. -___ +--- ### evaluators @@ -95,7 +95,7 @@ ___ Evaluators used to assess and guide the agent's responses. -___ +--- ### factManager @@ -103,7 +103,7 @@ ___ Manage the fact and recall of facts. -___ +--- ### fetch @@ -121,9 +121,9 @@ Some environments may not have access to the global fetch function and need a cu ##### Parameters -| Name | Type | -| :------ | :------ | -| `input` | `RequestInfo` \| `URL` | +| Name | Type | +| :------ | :------------------------------------------- | +| `input` | `RequestInfo` \| `URL` | | `init?` | `RequestInit`\<`CfProperties`\<`unknown`\>\> | ##### Returns @@ -137,9 +137,9 @@ Some environments may not have access to the global fetch function and need a cu ##### Parameters -| Name | Type | -| :------ | :------ | -| `input` | `RequestInfo` | +| Name | Type | +| :------ | :------------------------------------------- | +| `input` | `RequestInfo` | | `init?` | `RequestInit`\<`CfProperties`\<`unknown`\>\> | ##### Returns @@ -153,16 +153,16 @@ Some environments may not have access to the global fetch function and need a cu ##### Parameters -| Name | Type | -| :------ | :------ | -| `input` | `RequestInfo` | +| Name | Type | +| :------ | :----------------------------------------- | +| `input` | `RequestInfo` | | `init?` | `RequestInit`\<`RequestInitCfProperties`\> | ##### Returns `Promise`\<`Response`\> -___ +--- ### loreManager @@ -170,7 +170,7 @@ ___ Manage the creation and recall of static information (documents, historical game lore, etc) -___ +--- ### messageManager @@ -178,7 +178,7 @@ ___ Store messages that are sent and received by the agent. -___ +--- ### model @@ -186,7 +186,7 @@ ___ The model to use for completion. -___ +--- ### providers @@ -194,7 +194,7 @@ ___ Context providers used to provide context for message generation. -___ +--- ### serverUrl @@ -202,11 +202,11 @@ ___ The base URL of the server where the agent's requests are processed. -___ +--- ### token -• **token**: ``null`` \| `string` +• **token**: `null` \| `string` Authentication token used for securing requests. @@ -220,15 +220,15 @@ Send a message to the OpenAI API for completion. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `opts` | `Object` | `undefined` | The options for the completion request. | -| `opts.context` | `undefined` \| `string` | `""` | The context of the message to be completed. | -| `opts.frequency_penalty` | `undefined` \| `number` | `0.0` | The frequency penalty to apply to the completion. | -| `opts.model` | `undefined` \| `string` | `undefined` | The model to use for completion. | -| `opts.presence_penalty` | `undefined` \| `number` | `0.0` | The presence penalty to apply to the completion. | -| `opts.stop` | `undefined` \| `never`[] | `[]` | A list of strings to stop the completion at. | -| `opts.temperature` | `undefined` \| `number` | `0.7` | The temperature to apply to the completion. | +| Name | Type | Default value | Description | +| :----------------------- | :----------------------- | :------------ | :------------------------------------------------ | +| `opts` | `Object` | `undefined` | The options for the completion request. | +| `opts.context` | `undefined` \| `string` | `""` | The context of the message to be completed. | +| `opts.frequency_penalty` | `undefined` \| `number` | `0.0` | The frequency penalty to apply to the completion. | +| `opts.model` | `undefined` \| `string` | `undefined` | The model to use for completion. | +| `opts.presence_penalty` | `undefined` \| `number` | `0.0` | The presence penalty to apply to the completion. | +| `opts.stop` | `undefined` \| `never`[] | `[]` | A list of strings to stop the completion at. | +| `opts.temperature` | `undefined` \| `number` | `0.7` | The temperature to apply to the completion. | #### Returns @@ -236,28 +236,28 @@ Send a message to the OpenAI API for completion. The completed message. -___ +--- ### composeState -▸ **composeState**(`message`, `additionalKeys?`): `Promise`\<\{ `actionConditions`: `string` ; `actionExamples`: `string` ; `actionNames`: `string` ; `actions`: `string` ; `actors`: `string` ; `actorsData`: [`Actor`](../interfaces/Actor.md)[] ; `agentId`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `agentName`: `undefined` \| `string` ; `evaluatorConditions`: `string` ; `evaluatorExamples`: `string` ; `evaluatorNames`: `string` ; `evaluators`: `string` ; `evaluatorsData`: [`Evaluator`](../interfaces/Evaluator.md)[] ; `goals`: `string` ; `goalsData`: [`Goal`](../interfaces/Goal.md)[] ; `lore`: `string` ; `loreData`: [`Memory`](../interfaces/Memory.md)[] ; `providers`: `string` ; `recentFacts`: `string` ; `recentFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `recentMessages`: `string` ; `recentMessagesData`: [`Memory`](../interfaces/Memory.md)[] ; `relevantFacts`: `string` ; `relevantFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `room_id`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `senderName`: `undefined` \| `string` }\> +▸ **composeState**(`message`, `additionalKeys?`): `Promise`\<\{ `actionConditions`: `string` ; `actionExamples`: `string` ; `actionNames`: `string` ; `actions`: `string` ; `actors`: `string` ; `actorsData`: [`Actor`](../interfaces/Actor.md)[] ; `agentId`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `agentName`: `undefined` \| `string` ; `evaluatorConditions`: `string` ; `evaluatorExamples`: `string` ; `evaluatorNames`: `string` ; `evaluators`: `string` ; `evaluatorsData`: [`Evaluator`](../interfaces/Evaluator.md)[] ; `goals`: `string` ; `goalsData`: [`Goal`](../interfaces/Goal.md)[] ; `lore`: `string` ; `loreData`: [`Memory`](../interfaces/Memory.md)[] ; `providers`: `string` ; `recentFacts`: `string` ; `recentFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `recentMessages`: `string` ; `recentMessagesData`: [`Memory`](../interfaces/Memory.md)[] ; `relevantFacts`: `string` ; `relevantFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `room_id`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `senderName`: `undefined` \| `string` }\> Compose the state of the agent into an object that can be passed or used for response generation. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `message` | [`Message`](../interfaces/Message.md) | The message to compose the state from. | -| `additionalKeys` | `Object` | - | +| Name | Type | Description | +| :--------------- | :------------------------------------ | :------------------------------------- | +| `message` | [`Message`](../interfaces/Message.md) | The message to compose the state from. | +| `additionalKeys` | `Object` | - | #### Returns -`Promise`\<\{ `actionConditions`: `string` ; `actionExamples`: `string` ; `actionNames`: `string` ; `actions`: `string` ; `actors`: `string` ; `actorsData`: [`Actor`](../interfaces/Actor.md)[] ; `agentId`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `agentName`: `undefined` \| `string` ; `evaluatorConditions`: `string` ; `evaluatorExamples`: `string` ; `evaluatorNames`: `string` ; `evaluators`: `string` ; `evaluatorsData`: [`Evaluator`](../interfaces/Evaluator.md)[] ; `goals`: `string` ; `goalsData`: [`Goal`](../interfaces/Goal.md)[] ; `lore`: `string` ; `loreData`: [`Memory`](../interfaces/Memory.md)[] ; `providers`: `string` ; `recentFacts`: `string` ; `recentFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `recentMessages`: `string` ; `recentMessagesData`: [`Memory`](../interfaces/Memory.md)[] ; `relevantFacts`: `string` ; `relevantFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `room_id`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `senderName`: `undefined` \| `string` }\> +`Promise`\<\{ `actionConditions`: `string` ; `actionExamples`: `string` ; `actionNames`: `string` ; `actions`: `string` ; `actors`: `string` ; `actorsData`: [`Actor`](../interfaces/Actor.md)[] ; `agentId`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `agentName`: `undefined` \| `string` ; `evaluatorConditions`: `string` ; `evaluatorExamples`: `string` ; `evaluatorNames`: `string` ; `evaluators`: `string` ; `evaluatorsData`: [`Evaluator`](../interfaces/Evaluator.md)[] ; `goals`: `string` ; `goalsData`: [`Goal`](../interfaces/Goal.md)[] ; `lore`: `string` ; `loreData`: [`Memory`](../interfaces/Memory.md)[] ; `providers`: `string` ; `recentFacts`: `string` ; `recentFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `recentMessages`: `string` ; `recentMessagesData`: [`Memory`](../interfaces/Memory.md)[] ; `relevantFacts`: `string` ; `relevantFactsData`: [`Memory`](../interfaces/Memory.md)[] ; `room_id`: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` ; `senderName`: `undefined` \| `string` }\> The state of the agent. -___ +--- ### embed @@ -267,8 +267,8 @@ Send a message to the OpenAI API for embedding. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------ | :------- | :------------------------ | | `input` | `string` | The input to be embedded. | #### Returns @@ -277,7 +277,7 @@ Send a message to the OpenAI API for embedding. The embedding of the input. -___ +--- ### ensureParticipantExists @@ -287,10 +287,10 @@ Ensure the existence of a participant in the room. If the participant does not e #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The user ID to ensure the existence of. | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -300,7 +300,7 @@ Ensure the existence of a participant in the room. If the participant does not e An error if the participant cannot be added. -___ +--- ### ensureRoomExists @@ -311,10 +311,10 @@ and agent are added as participants. The room ID is returned. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The user ID to create a room with. | -| `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :--------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The user ID to create a room with. | +| `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -326,7 +326,7 @@ The room ID of the room between the agent and the user. An error if the room cannot be created. -___ +--- ### evaluate @@ -336,10 +336,10 @@ Evaluate the message and state using the registered evaluators. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :----------------------- | | `message` | [`Message`](../interfaces/Message.md) | The message to evaluate. | -| `state?` | [`State`](../interfaces/State.md) | The state of the agent. | +| `state?` | [`State`](../interfaces/State.md) | The state of the agent. | #### Returns @@ -347,7 +347,7 @@ Evaluate the message and state using the registered evaluators. The results of the evaluation. -___ +--- ### getConversationLength @@ -361,7 +361,7 @@ Get the number of messages that are kept in the conversation buffer. The number of recent messages to be kept in memory. -___ +--- ### processActions @@ -371,17 +371,17 @@ Process the actions of a message. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `message` | [`Message`](../interfaces/Message.md) | The message to process. | +| Name | Type | Description | +| :-------- | :------------------------------------ | :-------------------------------------------------- | +| `message` | [`Message`](../interfaces/Message.md) | The message to process. | | `content` | [`Content`](../interfaces/Content.md) | The content of the message to process actions from. | -| `state?` | [`State`](../interfaces/State.md) | - | +| `state?` | [`State`](../interfaces/State.md) | - | #### Returns `Promise`\<`unknown`\> -___ +--- ### registerAction @@ -391,15 +391,15 @@ Register an action for the agent to perform. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :---------------------------------- | :---------------------- | | `action` | [`Action`](../interfaces/Action.md) | The action to register. | #### Returns `void` -___ +--- ### registerContextProvider @@ -409,15 +409,15 @@ Register a context provider to provide context for message generation. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :-------------------------------------- | :-------------------------------- | | `provider` | [`Provider`](../interfaces/Provider.md) | The context provider to register. | #### Returns `void` -___ +--- ### registerEvaluator @@ -427,26 +427,26 @@ Register an evaluator to assess and guide the agent's responses. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :---------- | :---------------------------------------- | :------------------------- | | `evaluator` | [`Evaluator`](../interfaces/Evaluator.md) | The evaluator to register. | #### Returns `void` -___ +--- ### retrieveCachedEmbedding -▸ **retrieveCachedEmbedding**(`input`): `Promise`\<``null`` \| `number`[]\> +▸ **retrieveCachedEmbedding**(`input`): `Promise`\<`null` \| `number`[]\> #### Parameters -| Name | Type | -| :------ | :------ | +| Name | Type | +| :------ | :------- | | `input` | `string` | #### Returns -`Promise`\<``null`` \| `number`[]\> +`Promise`\<`null` \| `number`[]\> diff --git a/docs/docs/classes/DatabaseAdapter.md b/docs/docs/classes/DatabaseAdapter.md index 9b66442e3a..8ff285f7fa 100644 --- a/docs/docs/classes/DatabaseAdapter.md +++ b/docs/docs/classes/DatabaseAdapter.md @@ -39,9 +39,9 @@ Adds a user as a participant to a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to which the user will be added. | #### Returns @@ -50,7 +50,7 @@ Adds a user as a participant to a specific room. A Promise that resolves to a boolean indicating success or failure. -___ +--- ### countMemories @@ -60,11 +60,11 @@ Counts the number of memories in a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to count memories. | -| `unique?` | `boolean` | Specifies whether to count only unique memories. | -| `tableName?` | `string` | Optional table name to count memories from. | +| Name | Type | Description | +| :----------- | :--------------------------------------------------------- | :------------------------------------------------ | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to count memories. | +| `unique?` | `boolean` | Specifies whether to count only unique memories. | +| `tableName?` | `string` | Optional table name to count memories from. | #### Returns @@ -72,7 +72,7 @@ Counts the number of memories in a specific room. A Promise that resolves to the number of memories. -___ +--- ### createAccount @@ -82,8 +82,8 @@ Creates a new account in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :---------------------------- | | `account` | [`Account`](../interfaces/Account.md) | The account object to create. | #### Returns @@ -92,7 +92,7 @@ Creates a new account in the database. A Promise that resolves when the account creation is complete. -___ +--- ### createGoal @@ -102,8 +102,8 @@ Creates a new goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object to create. | #### Returns @@ -112,7 +112,7 @@ Creates a new goal in the database. A Promise that resolves when the goal has been created. -___ +--- ### createMemory @@ -122,11 +122,11 @@ Creates a new memory in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | -| `tableName` | `string` | The table where the memory should be stored. | -| `unique?` | `boolean` | Indicates if the memory should be unique. | +| Name | Type | Description | +| :---------- | :---------------------------------- | :------------------------------------------- | +| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | +| `tableName` | `string` | The table where the memory should be stored. | +| `unique?` | `boolean` | Indicates if the memory should be unique. | #### Returns @@ -134,7 +134,7 @@ Creates a new memory in the database. A Promise that resolves when the memory has been created. -___ +--- ### createRelationship @@ -144,11 +144,11 @@ Creates a new relationship between two users. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -156,7 +156,7 @@ Creates a new relationship between two users. A Promise that resolves to a boolean indicating success or failure of the creation. -___ +--- ### createRoom @@ -166,8 +166,8 @@ Creates a new room with an optional specified ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :--------------------------------------- | | `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional UUID to assign to the new room. | #### Returns @@ -176,27 +176,27 @@ Creates a new room with an optional specified ID. A Promise that resolves to the UUID of the created room. -___ +--- ### getAccountById -▸ **getAccountById**(`user_id`): `Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +▸ **getAccountById**(`user_id`): `Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> Retrieves an account by its ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user account to retrieve. | #### Returns -`Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +`Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> A Promise that resolves to the Account object or null if not found. -___ +--- ### getActorDetails @@ -206,10 +206,10 @@ Retrieves details of actors in a given room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the room_id to search for actors. | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing the room_id to search for actors. | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -217,33 +217,33 @@ Retrieves details of actors in a given room. A Promise that resolves to an array of Actor objects. -___ +--- ### getCachedEmbeddings -▸ **getCachedEmbeddings**(`params`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +▸ **getCachedEmbeddings**(`params`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> Retrieves cached embeddings based on the specified query parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the embedding retrieval. | -| `params.query_field_name` | `string` | - | -| `params.query_field_sub_name` | `string` | - | -| `params.query_input` | `string` | - | -| `params.query_match_count` | `number` | - | -| `params.query_table_name` | `string` | - | -| `params.query_threshold` | `number` | - | +| Name | Type | Description | +| :---------------------------- | :------- | :----------------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the embedding retrieval. | +| `params.query_field_name` | `string` | - | +| `params.query_field_sub_name` | `string` | - | +| `params.query_input` | `string` | - | +| `params.query_match_count` | `number` | - | +| `params.query_table_name` | `string` | - | +| `params.query_threshold` | `number` | - | #### Returns -`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> A Promise that resolves to an array of objects containing embeddings and levenshtein scores. -___ +--- ### getGoals @@ -253,13 +253,13 @@ Retrieves goals based on specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for goal retrieval. | -| `params.count?` | `number` | - | -| `params.onlyInProgress?` | `boolean` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.user_id?` | ``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :----------------------- | :------------------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing parameters for goal retrieval. | +| `params.count?` | `number` | - | +| `params.onlyInProgress?` | `boolean` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.user_id?` | `null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -267,7 +267,7 @@ Retrieves goals based on specified parameters. A Promise that resolves to an array of Goal objects. -___ +--- ### getMemories @@ -277,13 +277,13 @@ Retrieves memories based on the specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory retrieval. | -| `params.count?` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :----------------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory retrieval. | +| `params.count?` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -291,7 +291,7 @@ Retrieves memories based on the specified parameters. A Promise that resolves to an array of Memory objects. -___ +--- ### getParticipantsForAccount @@ -301,8 +301,8 @@ Retrieves participants associated with a specific account. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :----------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the account. | #### Returns @@ -317,8 +317,8 @@ Retrieves participants associated with a specific account. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :----------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the account. | #### Returns @@ -327,7 +327,7 @@ Retrieves participants associated with a specific account. A Promise that resolves to an array of Participant objects. -___ +--- ### getParticipantsForRoom @@ -337,8 +337,8 @@ Retrieves participants for a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to retrieve participants. | #### Returns @@ -347,29 +347,29 @@ Retrieves participants for a specific room. A Promise that resolves to an array of UUIDs representing the participants. -___ +--- ### getRelationship -▸ **getRelationship**(`params`): `Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +▸ **getRelationship**(`params`): `Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> Retrieves a relationship between two users if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns -`Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +`Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> A Promise that resolves to the Relationship object or null if not found. -___ +--- ### getRelationships @@ -379,10 +379,10 @@ Retrieves all relationships for a specific user. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUID of the user. | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------- | +| `params` | `Object` | An object containing the UUID of the user. | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -390,27 +390,27 @@ Retrieves all relationships for a specific user. A Promise that resolves to an array of Relationship objects. -___ +--- ### getRoom -▸ **getRoom**(`room_id`): `Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +▸ **getRoom**(`room_id`): `Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> Retrieves the room ID for a given room, if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to retrieve. | #### Returns -`Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +`Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> A Promise that resolves to the room ID or null if not found. -___ +--- ### getRoomsForParticipant @@ -420,8 +420,8 @@ Retrieves room IDs for which a specific user is a participant. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user. | #### Returns @@ -430,7 +430,7 @@ Retrieves room IDs for which a specific user is a participant. A Promise that resolves to an array of room IDs. -___ +--- ### getRoomsForParticipants @@ -440,8 +440,8 @@ Retrieves room IDs for which specific users are participants. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :----------------------------------------------------------- | :------------------------------ | | `userIds` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`[] | An array of UUIDs of the users. | #### Returns @@ -450,7 +450,7 @@ Retrieves room IDs for which specific users are participants. A Promise that resolves to an array of room IDs. -___ +--- ### log @@ -460,13 +460,13 @@ Logs an event or action with the specified details. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the log entry. | -| `params.body` | `Object` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.type` | `string` | - | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the log entry. | +| `params.body` | `Object` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.type` | `string` | - | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -474,7 +474,7 @@ Logs an event or action with the specified details. A Promise that resolves when the log entry has been saved. -___ +--- ### removeAllGoals @@ -484,8 +484,8 @@ Removes all goals associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose goals should be removed. | #### Returns @@ -494,7 +494,7 @@ Removes all goals associated with a specific room. A Promise that resolves when all goals have been removed. -___ +--- ### removeAllMemories @@ -504,10 +504,10 @@ Removes all memories associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | -| `tableName` | `string` | The table from which the memories should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | +| `tableName` | `string` | The table from which the memories should be removed. | #### Returns @@ -515,7 +515,7 @@ Removes all memories associated with a specific room. A Promise that resolves when all memories have been removed. -___ +--- ### removeGoal @@ -525,8 +525,8 @@ Removes a specific goal from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :--------------------------------------------------------- | :------------------------------ | | `goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the goal to remove. | #### Returns @@ -535,7 +535,7 @@ Removes a specific goal from the database. A Promise that resolves when the goal has been removed. -___ +--- ### removeMemory @@ -545,10 +545,10 @@ Removes a specific memory from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | -| `tableName` | `string` | The table from which the memory should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | +| `tableName` | `string` | The table from which the memory should be removed. | #### Returns @@ -556,7 +556,7 @@ Removes a specific memory from the database. A Promise that resolves when the memory has been removed. -___ +--- ### removeParticipant @@ -566,9 +566,9 @@ Removes a user as a participant from a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room from which the user will be removed. | #### Returns @@ -577,7 +577,7 @@ Removes a user as a participant from a specific room. A Promise that resolves to a boolean indicating success or failure. -___ +--- ### removeRoom @@ -587,8 +587,8 @@ Removes a specific room from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------ | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to remove. | #### Returns @@ -597,7 +597,7 @@ Removes a specific room from the database. A Promise that resolves when the room has been removed. -___ +--- ### searchMemories @@ -607,15 +607,15 @@ Searches for memories based on embeddings and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory search. | -| `params.embedding` | `number`[] | - | -| `params.match_count` | `number` | - | -| `params.match_threshold` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique` | `boolean` | - | +| Name | Type | Description | +| :----------------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory search. | +| `params.embedding` | `number`[] | - | +| `params.match_count` | `number` | - | +| `params.match_threshold` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique` | `boolean` | - | #### Returns @@ -623,7 +623,7 @@ Searches for memories based on embeddings and other specified parameters. A Promise that resolves to an array of Memory objects. -___ +--- ### searchMemoriesByEmbedding @@ -633,15 +633,15 @@ Searches for memories by embedding and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `embedding` | `number`[] | The embedding vector to search with. | -| `params` | `Object` | Additional parameters for the search. | -| `params.count?` | `number` | - | -| `params.match_threshold?` | `number` | - | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :------------------------ | :--------------------------------------------------------- | :------------------------------------ | +| `embedding` | `number`[] | The embedding vector to search with. | +| `params` | `Object` | Additional parameters for the search. | +| `params.count?` | `number` | - | +| `params.match_threshold?` | `number` | - | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -649,7 +649,7 @@ Searches for memories by embedding and other specified parameters. A Promise that resolves to an array of Memory objects. -___ +--- ### updateGoal @@ -659,8 +659,8 @@ Updates a specific goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :--------------------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object with updated properties. | #### Returns @@ -669,7 +669,7 @@ Updates a specific goal in the database. A Promise that resolves when the goal has been updated. -___ +--- ### updateGoalStatus @@ -679,11 +679,11 @@ Updates the status of a specific goal. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the goalId and the new status. | -| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | +| Name | Type | Description | +| :-------------- | :--------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing the goalId and the new status. | +| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | #### Returns diff --git a/docs/docs/classes/MemoryManager.md b/docs/docs/classes/MemoryManager.md index 3a34af24da..6df9297964 100644 --- a/docs/docs/classes/MemoryManager.md +++ b/docs/docs/classes/MemoryManager.md @@ -18,11 +18,11 @@ Constructs a new MemoryManager instance. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `opts` | `Object` | Options for the manager. | -| `opts.runtime` | [`BgentRuntime`](BgentRuntime.md) | The BgentRuntime instance associated with this manager. | -| `opts.tableName` | `string` | The name of the table this manager will operate on. | +| Name | Type | Description | +| :--------------- | :-------------------------------- | :------------------------------------------------------ | +| `opts` | `Object` | Options for the manager. | +| `opts.runtime` | [`AgentRuntime`](AgentRuntime.md) | The AgentRuntime instance associated with this manager. | +| `opts.tableName` | `string` | The name of the table this manager will operate on. | #### Returns @@ -32,11 +32,11 @@ Constructs a new MemoryManager instance. ### runtime -• **runtime**: [`BgentRuntime`](BgentRuntime.md) +• **runtime**: [`AgentRuntime`](AgentRuntime.md) -The BgentRuntime instance associated with this manager. +The AgentRuntime instance associated with this manager. -___ +--- ### tableName @@ -54,8 +54,8 @@ Adds an embedding vector to a memory object. If the memory already has an embedd #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :---------------------------------- | :---------------------------------------- | | `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to add an embedding to. | #### Returns @@ -64,7 +64,7 @@ Adds an embedding vector to a memory object. If the memory already has an embedd A Promise resolving to the memory object, potentially updated with an embedding vector. -___ +--- ### countMemories @@ -74,10 +74,10 @@ Counts the number of memories associated with a set of user IDs, with an option #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The room ID to count memories for. | -| `unique` | `boolean` | `true` | Whether to count unique memories only. | +| Name | Type | Default value | Description | +| :-------- | :--------------------------------------------------------- | :------------ | :------------------------------------- | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The room ID to count memories for. | +| `unique` | `boolean` | `true` | Whether to count unique memories only. | #### Returns @@ -85,7 +85,7 @@ Counts the number of memories associated with a set of user IDs, with an option A Promise resolving to the count of memories. -___ +--- ### createMemory @@ -95,10 +95,10 @@ Creates a new memory in the database, with an option to check for similarity bef #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `memory` | [`Memory`](../interfaces/Memory.md) | `undefined` | The memory object to create. | -| `unique` | `boolean` | `false` | Whether to check for similarity before insertion. | +| Name | Type | Default value | Description | +| :------- | :---------------------------------- | :------------ | :------------------------------------------------ | +| `memory` | [`Memory`](../interfaces/Memory.md) | `undefined` | The memory object to create. | +| `unique` | `boolean` | `false` | Whether to check for similarity before insertion. | #### Returns @@ -106,23 +106,23 @@ Creates a new memory in the database, with an option to check for similarity bef A Promise that resolves when the operation completes. -___ +--- ### getCachedEmbeddings -▸ **getCachedEmbeddings**(`content`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +▸ **getCachedEmbeddings**(`content`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> #### Parameters -| Name | Type | -| :------ | :------ | +| Name | Type | +| :-------- | :------- | | `content` | `string` | #### Returns -`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> -___ +--- ### getMemories @@ -132,12 +132,12 @@ Retrieves a list of memories by user IDs, with optional deduplication. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `opts` | `Object` | `undefined` | Options including user IDs, count, and uniqueness. | -| `opts.count?` | `number` | `10` | The number of memories to retrieve. | -| `opts.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The room ID to retrieve memories for. | -| `opts.unique?` | `boolean` | `true` | Whether to retrieve unique memories only. | +| Name | Type | Default value | Description | +| :------------- | :--------------------------------------------------------- | :------------ | :------------------------------------------------- | +| `opts` | `Object` | `undefined` | Options including user IDs, count, and uniqueness. | +| `opts.count?` | `number` | `10` | The number of memories to retrieve. | +| `opts.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The room ID to retrieve memories for. | +| `opts.unique?` | `boolean` | `true` | Whether to retrieve unique memories only. | #### Returns @@ -145,7 +145,7 @@ Retrieves a list of memories by user IDs, with optional deduplication. A Promise resolving to an array of Memory objects. -___ +--- ### removeAllMemories @@ -155,8 +155,8 @@ Removes all memories associated with a set of user IDs. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The room ID to remove memories for. | #### Returns @@ -165,7 +165,7 @@ Removes all memories associated with a set of user IDs. A Promise that resolves when the operation completes. -___ +--- ### removeMemory @@ -175,8 +175,8 @@ Removes a memory from the database by its ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :------------------------------ | | `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The ID of the memory to remove. | #### Returns @@ -185,7 +185,7 @@ Removes a memory from the database by its ID. A Promise that resolves when the operation completes. -___ +--- ### searchMemoriesByEmbedding @@ -195,14 +195,14 @@ Searches for memories similar to a given embedding vector. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `embedding` | `number`[] | The embedding vector to search with. | -| `opts` | `Object` | Options including match threshold, count, user IDs, and uniqueness. | -| `opts.count?` | `number` | The maximum number of memories to retrieve. | -| `opts.match_threshold?` | `number` | The similarity threshold for matching memories. | -| `opts.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The room ID to retrieve memories for. | -| `opts.unique?` | `boolean` | Whether to retrieve unique memories only. | +| Name | Type | Description | +| :---------------------- | :--------------------------------------------------------- | :------------------------------------------------------------------ | +| `embedding` | `number`[] | The embedding vector to search with. | +| `opts` | `Object` | Options including match threshold, count, user IDs, and uniqueness. | +| `opts.count?` | `number` | The maximum number of memories to retrieve. | +| `opts.match_threshold?` | `number` | The similarity threshold for matching memories. | +| `opts.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The room ID to retrieve memories for. | +| `opts.unique?` | `boolean` | Whether to retrieve unique memories only. | #### Returns diff --git a/docs/docs/classes/SqlJsDatabaseAdapter.md b/docs/docs/classes/SqlJsDatabaseAdapter.md index f15a66b9ce..01aab900bf 100644 --- a/docs/docs/classes/SqlJsDatabaseAdapter.md +++ b/docs/docs/classes/SqlJsDatabaseAdapter.md @@ -23,8 +23,8 @@ like accounts, memories, actors, goals, and rooms. #### Parameters -| Name | Type | -| :------ | :------ | +| Name | Type | +| :--- | :--------- | | `db` | `Database` | #### Returns @@ -51,9 +51,9 @@ Adds a user as a participant to a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to which the user will be added. | #### Returns @@ -66,7 +66,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[addParticipant](DatabaseAdapter.md#addparticipant) -___ +--- ### countMemories @@ -76,11 +76,11 @@ Counts the number of memories in a specific room. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | -| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | -| `tableName` | `string` | `""` | Optional table name to count memories from. | +| Name | Type | Default value | Description | +| :---------- | :--------------------------------------------------------- | :------------ | :------------------------------------------------ | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | +| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | +| `tableName` | `string` | `""` | Optional table name to count memories from. | #### Returns @@ -92,7 +92,7 @@ A Promise that resolves to the number of memories. [DatabaseAdapter](DatabaseAdapter.md).[countMemories](DatabaseAdapter.md#countmemories) -___ +--- ### createAccount @@ -102,8 +102,8 @@ Creates a new account in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :---------------------------- | | `account` | [`Account`](../interfaces/Account.md) | The account object to create. | #### Returns @@ -116,7 +116,7 @@ A Promise that resolves when the account creation is complete. [DatabaseAdapter](DatabaseAdapter.md).[createAccount](DatabaseAdapter.md#createaccount) -___ +--- ### createGoal @@ -126,8 +126,8 @@ Creates a new goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object to create. | #### Returns @@ -140,7 +140,7 @@ A Promise that resolves when the goal has been created. [DatabaseAdapter](DatabaseAdapter.md).[createGoal](DatabaseAdapter.md#creategoal) -___ +--- ### createMemory @@ -150,10 +150,10 @@ Creates a new memory in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | -| `tableName` | `string` | The table where the memory should be stored. | +| Name | Type | Description | +| :---------- | :---------------------------------- | :------------------------------------------- | +| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | +| `tableName` | `string` | The table where the memory should be stored. | #### Returns @@ -165,7 +165,7 @@ A Promise that resolves when the memory has been created. [DatabaseAdapter](DatabaseAdapter.md).[createMemory](DatabaseAdapter.md#creatememory) -___ +--- ### createRelationship @@ -175,11 +175,11 @@ Creates a new relationship between two users. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -191,7 +191,7 @@ A Promise that resolves to a boolean indicating success or failure of the creati [DatabaseAdapter](DatabaseAdapter.md).[createRelationship](DatabaseAdapter.md#createrelationship) -___ +--- ### createRoom @@ -201,8 +201,8 @@ Creates a new room with an optional specified ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :--------------------------------------- | | `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional UUID to assign to the new room. | #### Returns @@ -215,23 +215,23 @@ A Promise that resolves to the UUID of the created room. [DatabaseAdapter](DatabaseAdapter.md).[createRoom](DatabaseAdapter.md#createroom) -___ +--- ### getAccountById -▸ **getAccountById**(`user_id`): `Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +▸ **getAccountById**(`user_id`): `Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> Retrieves an account by its ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user account to retrieve. | #### Returns -`Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +`Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> A Promise that resolves to the Account object or null if not found. @@ -239,7 +239,7 @@ A Promise that resolves to the Account object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getAccountById](DatabaseAdapter.md#getaccountbyid) -___ +--- ### getActorDetails @@ -249,10 +249,10 @@ Retrieves details of actors in a given room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the room_id to search for actors. | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing the room_id to search for actors. | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -264,29 +264,29 @@ A Promise that resolves to an array of Actor objects. [DatabaseAdapter](DatabaseAdapter.md).[getActorDetails](DatabaseAdapter.md#getactordetails) -___ +--- ### getCachedEmbeddings -▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> Retrieves cached embeddings based on the specified query parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `opts` | `Object` | An object containing parameters for the embedding retrieval. | -| `opts.query_field_name` | `string` | - | -| `opts.query_field_sub_name` | `string` | - | -| `opts.query_input` | `string` | - | -| `opts.query_match_count` | `number` | - | -| `opts.query_table_name` | `string` | - | -| `opts.query_threshold` | `number` | - | +| Name | Type | Description | +| :-------------------------- | :------- | :----------------------------------------------------------- | +| `opts` | `Object` | An object containing parameters for the embedding retrieval. | +| `opts.query_field_name` | `string` | - | +| `opts.query_field_sub_name` | `string` | - | +| `opts.query_input` | `string` | - | +| `opts.query_match_count` | `number` | - | +| `opts.query_table_name` | `string` | - | +| `opts.query_threshold` | `number` | - | #### Returns -`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> A Promise that resolves to an array of objects containing embeddings and levenshtein scores. @@ -294,7 +294,7 @@ A Promise that resolves to an array of objects containing embeddings and levensh [DatabaseAdapter](DatabaseAdapter.md).[getCachedEmbeddings](DatabaseAdapter.md#getcachedembeddings) -___ +--- ### getGoals @@ -304,13 +304,13 @@ Retrieves goals based on specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for goal retrieval. | -| `params.count?` | `number` | - | -| `params.onlyInProgress?` | `boolean` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.user_id?` | ``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :----------------------- | :------------------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing parameters for goal retrieval. | +| `params.count?` | `number` | - | +| `params.onlyInProgress?` | `boolean` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.user_id?` | `null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -322,7 +322,7 @@ A Promise that resolves to an array of Goal objects. [DatabaseAdapter](DatabaseAdapter.md).[getGoals](DatabaseAdapter.md#getgoals) -___ +--- ### getMemories @@ -332,13 +332,13 @@ Retrieves memories based on the specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory retrieval. | -| `params.count?` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :----------------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory retrieval. | +| `params.count?` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -350,7 +350,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[getMemories](DatabaseAdapter.md#getmemories) -___ +--- ### getParticipantsForAccount @@ -360,8 +360,8 @@ Retrieves participants associated with a specific account. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :----------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the account. | #### Returns @@ -374,7 +374,7 @@ A Promise that resolves to an array of Participant objects. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForAccount](DatabaseAdapter.md#getparticipantsforaccount) -___ +--- ### getParticipantsForRoom @@ -384,8 +384,8 @@ Retrieves participants for a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to retrieve participants. | #### Returns @@ -398,25 +398,25 @@ A Promise that resolves to an array of UUIDs representing the participants. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForRoom](DatabaseAdapter.md#getparticipantsforroom) -___ +--- ### getRelationship -▸ **getRelationship**(`params`): `Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +▸ **getRelationship**(`params`): `Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> Retrieves a relationship between two users if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns -`Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +`Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> A Promise that resolves to the Relationship object or null if not found. @@ -424,7 +424,7 @@ A Promise that resolves to the Relationship object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRelationship](DatabaseAdapter.md#getrelationship) -___ +--- ### getRelationships @@ -434,10 +434,10 @@ Retrieves all relationships for a specific user. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUID of the user. | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------- | +| `params` | `Object` | An object containing the UUID of the user. | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -449,23 +449,23 @@ A Promise that resolves to an array of Relationship objects. [DatabaseAdapter](DatabaseAdapter.md).[getRelationships](DatabaseAdapter.md#getrelationships) -___ +--- ### getRoom -▸ **getRoom**(`room_id`): `Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +▸ **getRoom**(`room_id`): `Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> Retrieves the room ID for a given room, if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to retrieve. | #### Returns -`Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +`Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> A Promise that resolves to the room ID or null if not found. @@ -473,7 +473,7 @@ A Promise that resolves to the room ID or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRoom](DatabaseAdapter.md#getroom) -___ +--- ### getRoomsForParticipant @@ -483,8 +483,8 @@ Retrieves room IDs for which a specific user is a participant. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user. | #### Returns @@ -497,7 +497,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipant](DatabaseAdapter.md#getroomsforparticipant) -___ +--- ### getRoomsForParticipants @@ -507,8 +507,8 @@ Retrieves room IDs for which specific users are participants. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :----------------------------------------------------------- | :------------------------------ | | `userIds` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`[] | An array of UUIDs of the users. | #### Returns @@ -521,7 +521,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipants](DatabaseAdapter.md#getroomsforparticipants) -___ +--- ### log @@ -531,13 +531,13 @@ Logs an event or action with the specified details. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the log entry. | -| `params.body` | `Object` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.type` | `string` | - | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the log entry. | +| `params.body` | `Object` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.type` | `string` | - | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -549,7 +549,7 @@ A Promise that resolves when the log entry has been saved. [DatabaseAdapter](DatabaseAdapter.md).[log](DatabaseAdapter.md#log) -___ +--- ### removeAllGoals @@ -559,8 +559,8 @@ Removes all goals associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose goals should be removed. | #### Returns @@ -573,7 +573,7 @@ A Promise that resolves when all goals have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllGoals](DatabaseAdapter.md#removeallgoals) -___ +--- ### removeAllMemories @@ -583,10 +583,10 @@ Removes all memories associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | -| `tableName` | `string` | The table from which the memories should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | +| `tableName` | `string` | The table from which the memories should be removed. | #### Returns @@ -598,7 +598,7 @@ A Promise that resolves when all memories have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllMemories](DatabaseAdapter.md#removeallmemories) -___ +--- ### removeGoal @@ -608,8 +608,8 @@ Removes a specific goal from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :--------------------------------------------------------- | :------------------------------ | | `goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the goal to remove. | #### Returns @@ -622,7 +622,7 @@ A Promise that resolves when the goal has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeGoal](DatabaseAdapter.md#removegoal) -___ +--- ### removeMemory @@ -632,10 +632,10 @@ Removes a specific memory from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | -| `tableName` | `string` | The table from which the memory should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | +| `tableName` | `string` | The table from which the memory should be removed. | #### Returns @@ -647,7 +647,7 @@ A Promise that resolves when the memory has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeMemory](DatabaseAdapter.md#removememory) -___ +--- ### removeParticipant @@ -657,9 +657,9 @@ Removes a user as a participant from a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room from which the user will be removed. | #### Returns @@ -672,7 +672,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[removeParticipant](DatabaseAdapter.md#removeparticipant) -___ +--- ### removeRoom @@ -682,8 +682,8 @@ Removes a specific room from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------ | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to remove. | #### Returns @@ -696,7 +696,7 @@ A Promise that resolves when the room has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeRoom](DatabaseAdapter.md#removeroom) -___ +--- ### searchMemories @@ -706,15 +706,15 @@ Searches for memories based on embeddings and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory search. | -| `params.embedding` | `number`[] | - | -| `params.match_count` | `number` | - | -| `params.match_threshold` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique` | `boolean` | - | +| Name | Type | Description | +| :----------------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory search. | +| `params.embedding` | `number`[] | - | +| `params.match_count` | `number` | - | +| `params.match_threshold` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique` | `boolean` | - | #### Returns @@ -726,7 +726,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemories](DatabaseAdapter.md#searchmemories) -___ +--- ### searchMemoriesByEmbedding @@ -736,15 +736,15 @@ Searches for memories by embedding and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `_embedding` | `number`[] | The embedding vector to search with. | -| `params` | `Object` | Additional parameters for the search. | -| `params.count?` | `number` | - | -| `params.match_threshold?` | `number` | - | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :------------------------ | :--------------------------------------------------------- | :------------------------------------ | +| `_embedding` | `number`[] | The embedding vector to search with. | +| `params` | `Object` | Additional parameters for the search. | +| `params.count?` | `number` | - | +| `params.match_threshold?` | `number` | - | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -756,7 +756,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemoriesByEmbedding](DatabaseAdapter.md#searchmemoriesbyembedding) -___ +--- ### updateGoal @@ -766,8 +766,8 @@ Updates a specific goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :--------------------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object with updated properties. | #### Returns @@ -780,7 +780,7 @@ A Promise that resolves when the goal has been updated. [DatabaseAdapter](DatabaseAdapter.md).[updateGoal](DatabaseAdapter.md#updategoal) -___ +--- ### updateGoalStatus @@ -790,11 +790,11 @@ Updates the status of a specific goal. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the goalId and the new status. | -| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | +| Name | Type | Description | +| :-------------- | :--------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing the goalId and the new status. | +| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | #### Returns diff --git a/docs/docs/classes/SqliteDatabaseAdapter.md b/docs/docs/classes/SqliteDatabaseAdapter.md index d34ac6b240..2b6db4d44e 100644 --- a/docs/docs/classes/SqliteDatabaseAdapter.md +++ b/docs/docs/classes/SqliteDatabaseAdapter.md @@ -23,8 +23,8 @@ like accounts, memories, actors, goals, and rooms. #### Parameters -| Name | Type | -| :------ | :------ | +| Name | Type | +| :--- | :--------- | | `db` | `Database` | #### Returns @@ -51,9 +51,9 @@ Adds a user as a participant to a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to which the user will be added. | #### Returns @@ -66,7 +66,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[addParticipant](DatabaseAdapter.md#addparticipant) -___ +--- ### countMemories @@ -76,11 +76,11 @@ Counts the number of memories in a specific room. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | -| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | -| `tableName` | `string` | `""` | Optional table name to count memories from. | +| Name | Type | Default value | Description | +| :---------- | :--------------------------------------------------------- | :------------ | :------------------------------------------------ | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | +| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | +| `tableName` | `string` | `""` | Optional table name to count memories from. | #### Returns @@ -92,7 +92,7 @@ A Promise that resolves to the number of memories. [DatabaseAdapter](DatabaseAdapter.md).[countMemories](DatabaseAdapter.md#countmemories) -___ +--- ### createAccount @@ -102,8 +102,8 @@ Creates a new account in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :---------------------------- | | `account` | [`Account`](../interfaces/Account.md) | The account object to create. | #### Returns @@ -116,7 +116,7 @@ A Promise that resolves when the account creation is complete. [DatabaseAdapter](DatabaseAdapter.md).[createAccount](DatabaseAdapter.md#createaccount) -___ +--- ### createGoal @@ -126,8 +126,8 @@ Creates a new goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object to create. | #### Returns @@ -140,7 +140,7 @@ A Promise that resolves when the goal has been created. [DatabaseAdapter](DatabaseAdapter.md).[createGoal](DatabaseAdapter.md#creategoal) -___ +--- ### createMemory @@ -150,10 +150,10 @@ Creates a new memory in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | -| `tableName` | `string` | The table where the memory should be stored. | +| Name | Type | Description | +| :---------- | :---------------------------------- | :------------------------------------------- | +| `memory` | [`Memory`](../interfaces/Memory.md) | The memory object to create. | +| `tableName` | `string` | The table where the memory should be stored. | #### Returns @@ -165,7 +165,7 @@ A Promise that resolves when the memory has been created. [DatabaseAdapter](DatabaseAdapter.md).[createMemory](DatabaseAdapter.md#creatememory) -___ +--- ### createRelationship @@ -175,11 +175,11 @@ Creates a new relationship between two users. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -191,7 +191,7 @@ A Promise that resolves to a boolean indicating success or failure of the creati [DatabaseAdapter](DatabaseAdapter.md).[createRelationship](DatabaseAdapter.md#createrelationship) -___ +--- ### createRoom @@ -201,8 +201,8 @@ Creates a new room with an optional specified ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :--------------------------------------- | | `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional UUID to assign to the new room. | #### Returns @@ -215,23 +215,23 @@ A Promise that resolves to the UUID of the created room. [DatabaseAdapter](DatabaseAdapter.md).[createRoom](DatabaseAdapter.md#createroom) -___ +--- ### getAccountById -▸ **getAccountById**(`user_id`): `Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +▸ **getAccountById**(`user_id`): `Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> Retrieves an account by its ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user account to retrieve. | #### Returns -`Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +`Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> A Promise that resolves to the Account object or null if not found. @@ -239,7 +239,7 @@ A Promise that resolves to the Account object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getAccountById](DatabaseAdapter.md#getaccountbyid) -___ +--- ### getActorDetails @@ -249,10 +249,10 @@ Retrieves details of actors in a given room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the room_id to search for actors. | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing the room_id to search for actors. | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -264,29 +264,29 @@ A Promise that resolves to an array of Actor objects. [DatabaseAdapter](DatabaseAdapter.md).[getActorDetails](DatabaseAdapter.md#getactordetails) -___ +--- ### getCachedEmbeddings -▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> Retrieves cached embeddings based on the specified query parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `opts` | `Object` | An object containing parameters for the embedding retrieval. | -| `opts.query_field_name` | `string` | - | -| `opts.query_field_sub_name` | `string` | - | -| `opts.query_input` | `string` | - | -| `opts.query_match_count` | `number` | - | -| `opts.query_table_name` | `string` | - | -| `opts.query_threshold` | `number` | - | +| Name | Type | Description | +| :-------------------------- | :------- | :----------------------------------------------------------- | +| `opts` | `Object` | An object containing parameters for the embedding retrieval. | +| `opts.query_field_name` | `string` | - | +| `opts.query_field_sub_name` | `string` | - | +| `opts.query_input` | `string` | - | +| `opts.query_match_count` | `number` | - | +| `opts.query_table_name` | `string` | - | +| `opts.query_threshold` | `number` | - | #### Returns -`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> A Promise that resolves to an array of objects containing embeddings and levenshtein scores. @@ -294,7 +294,7 @@ A Promise that resolves to an array of objects containing embeddings and levensh [DatabaseAdapter](DatabaseAdapter.md).[getCachedEmbeddings](DatabaseAdapter.md#getcachedembeddings) -___ +--- ### getGoals @@ -304,13 +304,13 @@ Retrieves goals based on specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for goal retrieval. | -| `params.count?` | `number` | - | -| `params.onlyInProgress?` | `boolean` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.user_id?` | ``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :----------------------- | :------------------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing parameters for goal retrieval. | +| `params.count?` | `number` | - | +| `params.onlyInProgress?` | `boolean` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.user_id?` | `null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -322,7 +322,7 @@ A Promise that resolves to an array of Goal objects. [DatabaseAdapter](DatabaseAdapter.md).[getGoals](DatabaseAdapter.md#getgoals) -___ +--- ### getMemories @@ -332,13 +332,13 @@ Retrieves memories based on the specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory retrieval. | -| `params.count?` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :----------------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory retrieval. | +| `params.count?` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -350,7 +350,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[getMemories](DatabaseAdapter.md#getmemories) -___ +--- ### getParticipantsForAccount @@ -360,8 +360,8 @@ Retrieves participants associated with a specific account. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :----------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the account. | #### Returns @@ -374,7 +374,7 @@ A Promise that resolves to an array of Participant objects. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForAccount](DatabaseAdapter.md#getparticipantsforaccount) -___ +--- ### getParticipantsForRoom @@ -384,8 +384,8 @@ Retrieves participants for a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to retrieve participants. | #### Returns @@ -398,25 +398,25 @@ A Promise that resolves to an array of UUIDs representing the participants. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForRoom](DatabaseAdapter.md#getparticipantsforroom) -___ +--- ### getRelationship -▸ **getRelationship**(`params`): `Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +▸ **getRelationship**(`params`): `Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> Retrieves a relationship between two users if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns -`Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +`Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> A Promise that resolves to the Relationship object or null if not found. @@ -424,7 +424,7 @@ A Promise that resolves to the Relationship object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRelationship](DatabaseAdapter.md#getrelationship) -___ +--- ### getRelationships @@ -434,10 +434,10 @@ Retrieves all relationships for a specific user. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUID of the user. | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------- | +| `params` | `Object` | An object containing the UUID of the user. | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -449,23 +449,23 @@ A Promise that resolves to an array of Relationship objects. [DatabaseAdapter](DatabaseAdapter.md).[getRelationships](DatabaseAdapter.md#getrelationships) -___ +--- ### getRoom -▸ **getRoom**(`room_id`): `Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +▸ **getRoom**(`room_id`): `Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> Retrieves the room ID for a given room, if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to retrieve. | #### Returns -`Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +`Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> A Promise that resolves to the room ID or null if not found. @@ -473,7 +473,7 @@ A Promise that resolves to the room ID or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRoom](DatabaseAdapter.md#getroom) -___ +--- ### getRoomsForParticipant @@ -483,8 +483,8 @@ Retrieves room IDs for which a specific user is a participant. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user. | #### Returns @@ -497,7 +497,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipant](DatabaseAdapter.md#getroomsforparticipant) -___ +--- ### getRoomsForParticipants @@ -507,8 +507,8 @@ Retrieves room IDs for which specific users are participants. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :----------------------------------------------------------- | :------------------------------ | | `userIds` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`[] | An array of UUIDs of the users. | #### Returns @@ -521,7 +521,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipants](DatabaseAdapter.md#getroomsforparticipants) -___ +--- ### log @@ -531,13 +531,13 @@ Logs an event or action with the specified details. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the log entry. | -| `params.body` | `Object` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.type` | `string` | - | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the log entry. | +| `params.body` | `Object` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.type` | `string` | - | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -549,7 +549,7 @@ A Promise that resolves when the log entry has been saved. [DatabaseAdapter](DatabaseAdapter.md).[log](DatabaseAdapter.md#log) -___ +--- ### removeAllGoals @@ -559,8 +559,8 @@ Removes all goals associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose goals should be removed. | #### Returns @@ -573,7 +573,7 @@ A Promise that resolves when all goals have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllGoals](DatabaseAdapter.md#removeallgoals) -___ +--- ### removeAllMemories @@ -583,10 +583,10 @@ Removes all memories associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | -| `tableName` | `string` | The table from which the memories should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | +| `tableName` | `string` | The table from which the memories should be removed. | #### Returns @@ -598,7 +598,7 @@ A Promise that resolves when all memories have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllMemories](DatabaseAdapter.md#removeallmemories) -___ +--- ### removeGoal @@ -608,8 +608,8 @@ Removes a specific goal from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :--------------------------------------------------------- | :------------------------------ | | `goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the goal to remove. | #### Returns @@ -622,7 +622,7 @@ A Promise that resolves when the goal has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeGoal](DatabaseAdapter.md#removegoal) -___ +--- ### removeMemory @@ -632,10 +632,10 @@ Removes a specific memory from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | -| `tableName` | `string` | The table from which the memory should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | +| `tableName` | `string` | The table from which the memory should be removed. | #### Returns @@ -647,7 +647,7 @@ A Promise that resolves when the memory has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeMemory](DatabaseAdapter.md#removememory) -___ +--- ### removeParticipant @@ -657,9 +657,9 @@ Removes a user as a participant from a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room from which the user will be removed. | #### Returns @@ -672,7 +672,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[removeParticipant](DatabaseAdapter.md#removeparticipant) -___ +--- ### removeRoom @@ -682,8 +682,8 @@ Removes a specific room from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------ | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to remove. | #### Returns @@ -696,7 +696,7 @@ A Promise that resolves when the room has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeRoom](DatabaseAdapter.md#removeroom) -___ +--- ### searchMemories @@ -706,15 +706,15 @@ Searches for memories based on embeddings and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory search. | -| `params.embedding` | `number`[] | - | -| `params.match_count` | `number` | - | -| `params.match_threshold` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique` | `boolean` | - | +| Name | Type | Description | +| :----------------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory search. | +| `params.embedding` | `number`[] | - | +| `params.match_count` | `number` | - | +| `params.match_threshold` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique` | `boolean` | - | #### Returns @@ -726,7 +726,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemories](DatabaseAdapter.md#searchmemories) -___ +--- ### searchMemoriesByEmbedding @@ -736,15 +736,15 @@ Searches for memories by embedding and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `embedding` | `number`[] | The embedding vector to search with. | -| `params` | `Object` | Additional parameters for the search. | -| `params.count?` | `number` | - | -| `params.match_threshold?` | `number` | - | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :------------------------ | :--------------------------------------------------------- | :------------------------------------ | +| `embedding` | `number`[] | The embedding vector to search with. | +| `params` | `Object` | Additional parameters for the search. | +| `params.count?` | `number` | - | +| `params.match_threshold?` | `number` | - | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -756,7 +756,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemoriesByEmbedding](DatabaseAdapter.md#searchmemoriesbyembedding) -___ +--- ### updateGoal @@ -766,8 +766,8 @@ Updates a specific goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :--------------------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object with updated properties. | #### Returns @@ -780,7 +780,7 @@ A Promise that resolves when the goal has been updated. [DatabaseAdapter](DatabaseAdapter.md).[updateGoal](DatabaseAdapter.md#updategoal) -___ +--- ### updateGoalStatus @@ -790,11 +790,11 @@ Updates the status of a specific goal. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the goalId and the new status. | -| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | +| Name | Type | Description | +| :-------------- | :--------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing the goalId and the new status. | +| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | #### Returns diff --git a/docs/docs/classes/SupabaseDatabaseAdapter.md b/docs/docs/classes/SupabaseDatabaseAdapter.md index 09bfed2ab0..f380b7f80d 100644 --- a/docs/docs/classes/SupabaseDatabaseAdapter.md +++ b/docs/docs/classes/SupabaseDatabaseAdapter.md @@ -23,8 +23,8 @@ like accounts, memories, actors, goals, and rooms. #### Parameters -| Name | Type | -| :------ | :------ | +| Name | Type | +| :------------ | :------- | | `supabaseUrl` | `string` | | `supabaseKey` | `string` | @@ -40,7 +40,7 @@ like accounts, memories, actors, goals, and rooms. ### supabase -• **supabase**: `default`\<`any`, ``"public"``, `any`\> +• **supabase**: `default`\<`any`, `"public"`, `any`\> ## Methods @@ -52,9 +52,9 @@ Adds a user as a participant to a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to add as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to which the user will be added. | #### Returns @@ -67,7 +67,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[addParticipant](DatabaseAdapter.md#addparticipant) -___ +--- ### countMemories @@ -77,11 +77,11 @@ Counts the number of memories in a specific room. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | -| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | -| `tableName` | `string` | `undefined` | Optional table name to count memories from. | +| Name | Type | Default value | Description | +| :---------- | :--------------------------------------------------------- | :------------ | :------------------------------------------------ | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | The UUID of the room for which to count memories. | +| `unique` | `boolean` | `true` | Specifies whether to count only unique memories. | +| `tableName` | `string` | `undefined` | Optional table name to count memories from. | #### Returns @@ -93,7 +93,7 @@ A Promise that resolves to the number of memories. [DatabaseAdapter](DatabaseAdapter.md).[countMemories](DatabaseAdapter.md#countmemories) -___ +--- ### createAccount @@ -103,8 +103,8 @@ Creates a new account in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :---------------------------- | | `account` | [`Account`](../interfaces/Account.md) | The account object to create. | #### Returns @@ -117,7 +117,7 @@ A Promise that resolves when the account creation is complete. [DatabaseAdapter](DatabaseAdapter.md).[createAccount](DatabaseAdapter.md#createaccount) -___ +--- ### createGoal @@ -127,8 +127,8 @@ Creates a new goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object to create. | #### Returns @@ -141,7 +141,7 @@ A Promise that resolves when the goal has been created. [DatabaseAdapter](DatabaseAdapter.md).[createGoal](DatabaseAdapter.md#creategoal) -___ +--- ### createMemory @@ -151,11 +151,11 @@ Creates a new memory in the database. #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `memory` | [`Memory`](../interfaces/Memory.md) | `undefined` | The memory object to create. | -| `tableName` | `string` | `undefined` | The table where the memory should be stored. | -| `unique` | `boolean` | `false` | Indicates if the memory should be unique. | +| Name | Type | Default value | Description | +| :---------- | :---------------------------------- | :------------ | :------------------------------------------- | +| `memory` | [`Memory`](../interfaces/Memory.md) | `undefined` | The memory object to create. | +| `tableName` | `string` | `undefined` | The table where the memory should be stored. | +| `unique` | `boolean` | `false` | Indicates if the memory should be unique. | #### Returns @@ -167,7 +167,7 @@ A Promise that resolves when the memory has been created. [DatabaseAdapter](DatabaseAdapter.md).[createMemory](DatabaseAdapter.md#creatememory) -___ +--- ### createRelationship @@ -177,11 +177,11 @@ Creates a new relationship between two users. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -193,7 +193,7 @@ A Promise that resolves to a boolean indicating success or failure of the creati [DatabaseAdapter](DatabaseAdapter.md).[createRelationship](DatabaseAdapter.md#createrelationship) -___ +--- ### createRoom @@ -203,8 +203,8 @@ Creates a new room with an optional specified ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :--------------------------------------- | | `room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | Optional UUID to assign to the new room. | #### Returns @@ -217,23 +217,23 @@ A Promise that resolves to the UUID of the created room. [DatabaseAdapter](DatabaseAdapter.md).[createRoom](DatabaseAdapter.md#createroom) -___ +--- ### getAccountById -▸ **getAccountById**(`user_id`): `Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +▸ **getAccountById**(`user_id`): `Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> Retrieves an account by its ID. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :---------------------------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user account to retrieve. | #### Returns -`Promise`\<``null`` \| [`Account`](../interfaces/Account.md)\> +`Promise`\<`null` \| [`Account`](../interfaces/Account.md)\> A Promise that resolves to the Account object or null if not found. @@ -241,7 +241,7 @@ A Promise that resolves to the Account object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getAccountById](DatabaseAdapter.md#getaccountbyid) -___ +--- ### getActorDetails @@ -251,10 +251,10 @@ Retrieves details of actors in a given room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the room_id to search for actors. | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing the room_id to search for actors. | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -266,29 +266,29 @@ A Promise that resolves to an array of Actor objects. [DatabaseAdapter](DatabaseAdapter.md).[getActorDetails](DatabaseAdapter.md#getactordetails) -___ +--- ### getCachedEmbeddings -▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +▸ **getCachedEmbeddings**(`opts`): `Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> Retrieves cached embeddings based on the specified query parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `opts` | `Object` | An object containing parameters for the embedding retrieval. | -| `opts.query_field_name` | `string` | - | -| `opts.query_field_sub_name` | `string` | - | -| `opts.query_input` | `string` | - | -| `opts.query_match_count` | `number` | - | -| `opts.query_table_name` | `string` | - | -| `opts.query_threshold` | `number` | - | +| Name | Type | Description | +| :-------------------------- | :------- | :----------------------------------------------------------- | +| `opts` | `Object` | An object containing parameters for the embedding retrieval. | +| `opts.query_field_name` | `string` | - | +| `opts.query_field_sub_name` | `string` | - | +| `opts.query_input` | `string` | - | +| `opts.query_match_count` | `number` | - | +| `opts.query_table_name` | `string` | - | +| `opts.query_threshold` | `number` | - | #### Returns -`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> +`Promise`\<\{ `embedding`: `number`[] ; `levenshtein_score`: `number` }[]\> A Promise that resolves to an array of objects containing embeddings and levenshtein scores. @@ -296,7 +296,7 @@ A Promise that resolves to an array of objects containing embeddings and levensh [DatabaseAdapter](DatabaseAdapter.md).[getCachedEmbeddings](DatabaseAdapter.md#getcachedembeddings) -___ +--- ### getGoals @@ -306,13 +306,13 @@ Retrieves goals based on specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for goal retrieval. | -| `params.count?` | `number` | - | -| `params.onlyInProgress?` | `boolean` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.user_id?` | ``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :----------------------- | :------------------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing parameters for goal retrieval. | +| `params.count?` | `number` | - | +| `params.onlyInProgress?` | `boolean` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.user_id?` | `null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -324,7 +324,7 @@ A Promise that resolves to an array of Goal objects. [DatabaseAdapter](DatabaseAdapter.md).[getGoals](DatabaseAdapter.md#getgoals) -___ +--- ### getMemories @@ -334,13 +334,13 @@ Retrieves memories based on the specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory retrieval. | -| `params.count?` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :----------------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory retrieval. | +| `params.count?` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -352,7 +352,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[getMemories](DatabaseAdapter.md#getmemories) -___ +--- ### getParticipantsForAccount @@ -362,8 +362,8 @@ Retrieves participants associated with a specific account. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :----------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the account. | #### Returns @@ -376,7 +376,7 @@ A Promise that resolves to an array of Participant objects. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForAccount](DatabaseAdapter.md#getparticipantsforaccount) -___ +--- ### getParticipantsForRoom @@ -386,8 +386,8 @@ Retrieves participants for a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room for which to retrieve participants. | #### Returns @@ -400,25 +400,25 @@ A Promise that resolves to an array of UUIDs representing the participants. [DatabaseAdapter](DatabaseAdapter.md).[getParticipantsForRoom](DatabaseAdapter.md#getparticipantsforroom) -___ +--- ### getRelationship -▸ **getRelationship**(`params`): `Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +▸ **getRelationship**(`params`): `Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> Retrieves a relationship between two users if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | -| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :------------- | :--------------------------------------------------------- | :----------------------------------------------------------------- | +| `params` | `Object` | An object containing the UUIDs of the two users (userA and userB). | +| `params.userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns -`Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +`Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> A Promise that resolves to the Relationship object or null if not found. @@ -426,7 +426,7 @@ A Promise that resolves to the Relationship object or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRelationship](DatabaseAdapter.md#getrelationship) -___ +--- ### getRelationships @@ -436,10 +436,10 @@ Retrieves all relationships for a specific user. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the UUID of the user. | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :----------------------------------------- | +| `params` | `Object` | An object containing the UUID of the user. | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -451,23 +451,23 @@ A Promise that resolves to an array of Relationship objects. [DatabaseAdapter](DatabaseAdapter.md).[getRelationships](DatabaseAdapter.md#getrelationships) -___ +--- ### getRoom -▸ **getRoom**(`room_id`): `Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +▸ **getRoom**(`room_id`): `Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> Retrieves the room ID for a given room, if it exists. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to retrieve. | #### Returns -`Promise`\<``null`` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> +`Promise`\<`null` \| \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`\> A Promise that resolves to the room ID or null if not found. @@ -475,7 +475,7 @@ A Promise that resolves to the room ID or null if not found. [DatabaseAdapter](DatabaseAdapter.md).[getRoom](DatabaseAdapter.md#getroom) -___ +--- ### getRoomsForParticipant @@ -485,8 +485,8 @@ Retrieves room IDs for which a specific user is a participant. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------- | | `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user. | #### Returns @@ -499,7 +499,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipant](DatabaseAdapter.md#getroomsforparticipant) -___ +--- ### getRoomsForParticipants @@ -509,8 +509,8 @@ Retrieves room IDs for which specific users are participants. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :----------------------------------------------------------- | :------------------------------ | | `userIds` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\`[] | An array of UUIDs of the users. | #### Returns @@ -523,7 +523,7 @@ A Promise that resolves to an array of room IDs. [DatabaseAdapter](DatabaseAdapter.md).[getRoomsForParticipants](DatabaseAdapter.md#getroomsforparticipants) -___ +--- ### log @@ -533,13 +533,13 @@ Logs an event or action with the specified details. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the log entry. | -| `params.body` | `Object` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.type` | `string` | - | -| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| Name | Type | Description | +| :--------------- | :--------------------------------------------------------- | :------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the log entry. | +| `params.body` | `Object` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.type` | `string` | - | +| `params.user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | #### Returns @@ -551,7 +551,7 @@ A Promise that resolves when the log entry has been saved. [DatabaseAdapter](DatabaseAdapter.md).[log](DatabaseAdapter.md#log) -___ +--- ### removeAllGoals @@ -561,8 +561,8 @@ Removes all goals associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------- | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose goals should be removed. | #### Returns @@ -575,7 +575,7 @@ A Promise that resolves when all goals have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllGoals](DatabaseAdapter.md#removeallgoals) -___ +--- ### removeAllMemories @@ -585,10 +585,10 @@ Removes all memories associated with a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | -| `tableName` | `string` | The table from which the memories should be removed. | +| Name | Type | Description | +| :---------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room whose memories should be removed. | +| `tableName` | `string` | The table from which the memories should be removed. | #### Returns @@ -600,7 +600,7 @@ A Promise that resolves when all memories have been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeAllMemories](DatabaseAdapter.md#removeallmemories) -___ +--- ### removeGoal @@ -610,8 +610,8 @@ Removes a specific goal from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------- | :--------------------------------------------------------- | :------------------------------ | | `goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the goal to remove. | #### Returns @@ -624,7 +624,7 @@ A Promise that resolves when the goal has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeGoal](DatabaseAdapter.md#removegoal) -___ +--- ### removeMemory @@ -634,8 +634,8 @@ Removes a specific memory from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :--------- | :--------------------------------------------------------- | :-------------------------------- | | `memoryId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the memory to remove. | #### Returns @@ -648,7 +648,7 @@ A Promise that resolves when the memory has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeMemory](DatabaseAdapter.md#removememory) -___ +--- ### removeParticipant @@ -658,9 +658,9 @@ Removes a user as a participant from a specific room. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :-------------------------------------------------------- | +| `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the user to remove as a participant. | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room from which the user will be removed. | #### Returns @@ -673,7 +673,7 @@ A Promise that resolves to a boolean indicating success or failure. [DatabaseAdapter](DatabaseAdapter.md).[removeParticipant](DatabaseAdapter.md#removeparticipant) -___ +--- ### removeRoom @@ -683,8 +683,8 @@ Removes a specific room from the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :--------------------------------------------------------- | :------------------------------ | | `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | The UUID of the room to remove. | #### Returns @@ -697,7 +697,7 @@ A Promise that resolves when the room has been removed. [DatabaseAdapter](DatabaseAdapter.md).[removeRoom](DatabaseAdapter.md#removeroom) -___ +--- ### searchMemories @@ -707,15 +707,15 @@ Searches for memories based on embeddings and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing parameters for the memory search. | -| `params.embedding` | `number`[] | - | -| `params.match_count` | `number` | - | -| `params.match_threshold` | `number` | - | -| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique` | `boolean` | - | +| Name | Type | Description | +| :----------------------- | :--------------------------------------------------------- | :----------------------------------------------------- | +| `params` | `Object` | An object containing parameters for the memory search. | +| `params.embedding` | `number`[] | - | +| `params.match_count` | `number` | - | +| `params.match_threshold` | `number` | - | +| `params.room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique` | `boolean` | - | #### Returns @@ -727,7 +727,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemories](DatabaseAdapter.md#searchmemories) -___ +--- ### searchMemoriesByEmbedding @@ -737,15 +737,15 @@ Searches for memories by embedding and other specified parameters. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `embedding` | `number`[] | The embedding vector to search with. | -| `params` | `Object` | Additional parameters for the search. | -| `params.count?` | `number` | - | -| `params.match_threshold?` | `number` | - | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.tableName` | `string` | - | -| `params.unique?` | `boolean` | - | +| Name | Type | Description | +| :------------------------ | :--------------------------------------------------------- | :------------------------------------ | +| `embedding` | `number`[] | The embedding vector to search with. | +| `params` | `Object` | Additional parameters for the search. | +| `params.count?` | `number` | - | +| `params.match_threshold?` | `number` | - | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.tableName` | `string` | - | +| `params.unique?` | `boolean` | - | #### Returns @@ -757,7 +757,7 @@ A Promise that resolves to an array of Memory objects. [DatabaseAdapter](DatabaseAdapter.md).[searchMemoriesByEmbedding](DatabaseAdapter.md#searchmemoriesbyembedding) -___ +--- ### updateGoal @@ -767,8 +767,8 @@ Updates a specific goal in the database. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------ | :--------------------------------------- | | `goal` | [`Goal`](../interfaces/Goal.md) | The goal object with updated properties. | #### Returns @@ -781,7 +781,7 @@ A Promise that resolves when the goal has been updated. [DatabaseAdapter](DatabaseAdapter.md).[updateGoal](DatabaseAdapter.md#updategoal) -___ +--- ### updateGoalStatus @@ -791,11 +791,11 @@ Updates the status of a specific goal. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | An object containing the goalId and the new status. | -| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | -| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | +| Name | Type | Description | +| :-------------- | :--------------------------------------------------------- | :-------------------------------------------------- | +| `params` | `Object` | An object containing the goalId and the new status. | +| `params.goalId` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | - | +| `params.status` | [`GoalStatus`](../enums/GoalStatus.md) | - | #### Returns diff --git a/docs/docs/classes/_category_.yml b/docs/docs/classes/_category_.yml index 55c7980a46..7d96d81b4d 100644 --- a/docs/docs/classes/_category_.yml +++ b/docs/docs/classes/_category_.yml @@ -1,2 +1,2 @@ label: "Classes" -position: 3 \ No newline at end of file +position: 3 diff --git a/docs/docs/concepts.md b/docs/docs/concepts.md index ad5bf2dbc5..04e84ccb91 100644 --- a/docs/docs/concepts.md +++ b/docs/docs/concepts.md @@ -72,4 +72,4 @@ Eliza includes database adapters to enable seamless integration with various sto Database adapters abstract the underlying storage mechanisms, providing a consistent interface for querying and manipulating data related to the agent's interactions. They handle tasks such as storing and retrieving memories, managing goals and relationships, and persisting other relevant information. -By using database adapters, developers can focus on building the agent's logic and capabilities without concerning themselves with the intricacies of data storage and retrieval. The adapters provide a layer of abstraction, allowing developers to switch between different storage solutions or migrate their data easily. \ No newline at end of file +By using database adapters, developers can focus on building the agent's logic and capabilities without concerning themselves with the intricacies of data storage and retrieval. The adapters provide a layer of abstraction, allowing developers to switch between different storage solutions or migrate their data easily. diff --git a/docs/docs/docs/_category_.yml b/docs/docs/docs/_category_.yml index 24a4602617..c5d5974f55 100644 --- a/docs/docs/docs/_category_.yml +++ b/docs/docs/docs/_category_.yml @@ -1 +1 @@ -label: "API" \ No newline at end of file +label: "API" diff --git a/docs/docs/docs/index.md b/docs/docs/docs/index.md index f2c59a16e1..16cccba53e 100644 --- a/docs/docs/docs/index.md +++ b/docs/docs/docs/index.md @@ -10,112 +10,128 @@ custom_edit_url: null Eliza Banner -*As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)* - -- Multi-agent simulation framework -- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) -- Full-featured Discord and Twitter connectors, with Discord voice channel support -- Full conversational and document RAG memory -- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more -- Highly extensible - create your own actions and clients to extend Eliza's capabilities -- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) -- Supports OpenAI for cloud inference on a light-weight device -- "Ask Claude" mode for calling Claude on more complex queries +_As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)_ + +- Multi-agent simulation framework +- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) +- Full-featured Discord and Twitter connectors, with Discord voice channel support +- Full conversational and document RAG memory +- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more +- Highly extensible - create your own actions and clients to extend Eliza's capabilities +- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) +- Supports OpenAI for cloud inference on a light-weight device +- "Ask Claude" mode for calling Claude on more complex queries - 100% Typescript # Getting Started -## Install Node.js +## Install Node.js + https://docs.npmjs.com/downloading-and-installing-node-js-and-npm -## Edit the .env file -- Copy .env.example to .env and fill in the appropriate values +## Edit the .env file + +- Copy .env.example to .env and fill in the appropriate values - Edit the TWITTER environment variables to add your bot's username and password -## Edit the character file -- Check out the file `src/core/defaultCharacter.ts` - you can modify this +## Edit the character file + +- Check out the file `src/core/defaultCharacter.ts` - you can modify this - You can also load characters with the `node --loader ts-node/esm src/index.ts --characters="path/to/your/character.json"` and run multiple bots at the same time. -### Linux Installation -You might need these -``` -npm install --include=optional sharp +### Linux Installation + +You might need these + +``` +npm install --include=optional sharp ``` -### Run with Llama +### Run with Llama + You can run Llama 70B or 405B models by setting the `XAI_MODEL` environment variable to `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` or `meta-llama/Meta-Llama-3.1-405B-Instruct` -### Run with Grok +### Run with Grok + You can run Grok models by setting the `XAI_MODEL` environment variable to `grok-beta` -### Run with OpenAI +### Run with OpenAI + You can run OpenAI models by setting the `XAI_MODEL` environment variable to `gpt-4o-mini` or `gpt-4o` -# Requires Node 20+ +# Requires Node 20+ + If you are getting strange issues when starting up, make sure you're using Node 20+. Some APIs are not compatible with previous versions. You can check your node version with `node -v`. If you need to install a new version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm). -## Additional Requirements -You may need to install Sharp. If you see an error when starting up, try installing it with the following command: -``` -npm install --include=optional sharp +## Additional Requirements + +You may need to install Sharp. If you see an error when starting up, try installing it with the following command: + +``` +npm install --include=optional sharp ``` # Environment Setup -You will need to add environment variables to your .env file to connect to various platforms: -``` -# Required environment variables -# Start Discord -DISCORD_APPLICATION_ID= +You will need to add environment variables to your .env file to connect to various platforms: + +``` +# Required environment variables +# Start Discord +DISCORD_APPLICATION_ID= DISCORD_API_TOKEN= # Bot token -# Start Twitter -TWITTER_USERNAME= # Account username -TWITTER_PASSWORD= # Account password -TWITTER_EMAIL= # Account email -TWITTER_COOKIES= # Account cookies +# Start Twitter +TWITTER_USERNAME= # Account username +TWITTER_PASSWORD= # Account password +TWITTER_EMAIL= # Account email +TWITTER_COOKIES= # Account cookies ``` # Local Setup ## CUDA Setup -If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. -``` -npm install -npx --no node-llama-cpp source download --gpu cuda +If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. + +``` +npm install +npx --no node-llama-cpp source download --gpu cuda ``` Make sure that you've installed the CUDA Toolkit, including cuDNN and cuBLAS. -## Running locally -Add XAI_MODEL and set it to one of the above options from [Run with -Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it +## Running locally + +Add XAI_MODEL and set it to one of the above options from [Run with +Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it downloads the model from huggingface and queries it locally # Cloud Setup (with OpenAI) -In addition to the environment variables above, you will need to add the following: -``` -# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. +In addition to the environment variables above, you will need to add the following: + +``` +# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk- -# The agent can also ask Claude for help if you have an API key +# The agent can also ask Claude for help if you have an API key ANTHROPIC_API_KEY= -# For Elevenlabs voice generation on Discord voice +# For Elevenlabs voice generation on Discord voice ELEVENLABS_XI_API_KEY= # API key from elevenlabs -# ELEVENLABS SETTINGS -ELEVENLABS_MODEL_ID=eleven_multilingual_v2 -ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM -ELEVENLABS_VOICE_STABILITY=0.5 -ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 -ELEVENLABS_VOICE_STYLE=0.66 -ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false -ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 -ELEVENLABS_OUTPUT_FORMAT=pcm_16000 +# ELEVENLABS SETTINGS +ELEVENLABS_MODEL_ID=eleven_multilingual_v2 +ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM +ELEVENLABS_VOICE_STABILITY=0.5 +ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 +ELEVENLABS_VOICE_STYLE=0.66 +ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false +ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 +ELEVENLABS_OUTPUT_FORMAT=pcm_16000 ``` -# Discord Bot +# Discord Bot + For help with setting up your Discord Bot, check out here: https://discordjs.guide/preparations/setting-up-a-bot-application.html diff --git a/docs/docs/docs/modules.md b/docs/docs/docs/modules.md index 7143f2fc60..8200f88ad9 100644 --- a/docs/docs/docs/modules.md +++ b/docs/docs/docs/modules.md @@ -5,4 +5,3 @@ sidebar_label: "Exports" sidebar_position: 0.5 custom_edit_url: null --- - diff --git a/docs/docs/enums/GoalStatus.md b/docs/docs/enums/GoalStatus.md index bbd319d100..533e76a121 100644 --- a/docs/docs/enums/GoalStatus.md +++ b/docs/docs/enums/GoalStatus.md @@ -10,16 +10,16 @@ custom_edit_url: null ### DONE -• **DONE** = ``"DONE"`` +• **DONE** = `"DONE"` -___ +--- ### FAILED -• **FAILED** = ``"FAILED"`` +• **FAILED** = `"FAILED"` -___ +--- -### IN\_PROGRESS +### IN_PROGRESS -• **IN\_PROGRESS** = ``"IN_PROGRESS"`` +• **IN_PROGRESS** = `"IN_PROGRESS"` diff --git a/docs/docs/enums/_category_.yml b/docs/docs/enums/_category_.yml index 1687a9e03f..b4951c1ae9 100644 --- a/docs/docs/enums/_category_.yml +++ b/docs/docs/enums/_category_.yml @@ -1,2 +1,2 @@ label: "Enumerations" -position: 2 \ No newline at end of file +position: 2 diff --git a/docs/docs/functions/_category_.yml b/docs/docs/functions/_category_.yml index 139c448c66..bd69e82d5f 100644 --- a/docs/docs/functions/_category_.yml +++ b/docs/docs/functions/_category_.yml @@ -1,2 +1,2 @@ label: "Functions" -position: 7 \ No newline at end of file +position: 7 diff --git a/docs/docs/functions/addHeader.md b/docs/docs/functions/addHeader.md index dbf490348b..24a26b0927 100644 --- a/docs/docs/functions/addHeader.md +++ b/docs/docs/functions/addHeader.md @@ -15,10 +15,10 @@ If the body string is empty, the header is returned as is. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `header` | `string` | The header to add to the body. | -| `body` | `string` | The body to which to add the header. | +| Name | Type | Description | +| :------- | :------- | :----------------------------------- | +| `header` | `string` | The header to add to the body. | +| `body` | `string` | The body to which to add the header. | #### Returns diff --git a/docs/docs/functions/addLore.md b/docs/docs/functions/addLore.md index 9482b12333..4f90686f55 100644 --- a/docs/docs/functions/addLore.md +++ b/docs/docs/functions/addLore.md @@ -12,15 +12,15 @@ Adds a piece of lore to the lore database. Lore can include static information l #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `params` | `Object` | `undefined` | The parameters for adding lore. | -| `params.content` | [`Content`](../interfaces/Content.md) | `undefined` | The actual content of the lore. | -| `params.embedContent?` | [`Content`](../interfaces/Content.md) | `undefined` | Optional content used to generate an embedding if different from `content`. | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | The room ID associated with the lore, defaults to a zero UUID. | -| `params.runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | `undefined` | The runtime environment of the agent. | -| `params.source` | `string` | `undefined` | The source of the lore content. | -| `params.user_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | The user ID associated with the lore, defaults to a zero UUID. | +| Name | Type | Default value | Description | +| :--------------------- | :--------------------------------------------------------- | :------------ | :-------------------------------------------------------------------------- | +| `params` | `Object` | `undefined` | The parameters for adding lore. | +| `params.content` | [`Content`](../interfaces/Content.md) | `undefined` | The actual content of the lore. | +| `params.embedContent?` | [`Content`](../interfaces/Content.md) | `undefined` | Optional content used to generate an embedding if different from `content`. | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | The room ID associated with the lore, defaults to a zero UUID. | +| `params.runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | `undefined` | The runtime environment of the agent. | +| `params.source` | `string` | `undefined` | The source of the lore content. | +| `params.user_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | The user ID associated with the lore, defaults to a zero UUID. | #### Returns diff --git a/docs/docs/functions/composeActionExamples.md b/docs/docs/functions/composeActionExamples.md index 1ca587dab6..9810d45f8e 100644 --- a/docs/docs/functions/composeActionExamples.md +++ b/docs/docs/functions/composeActionExamples.md @@ -13,10 +13,10 @@ It randomly selects examples from the provided actions and formats them with gen #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :------------ | :------------------------------------ | :-------------------------------------------------------- | | `actionsData` | [`Action`](../interfaces/Action.md)[] | An array of `Action` objects from which to draw examples. | -| `count` | `number` | The number of examples to generate. | +| `count` | `number` | The number of examples to generate. | #### Returns diff --git a/docs/docs/functions/composeContext.md b/docs/docs/functions/composeContext.md index 1327b4dee5..878f2abdaf 100644 --- a/docs/docs/functions/composeContext.md +++ b/docs/docs/functions/composeContext.md @@ -16,11 +16,11 @@ If a matching key is not found in the state object for a given placeholder, the #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `params` | `Object` | The parameters for composing the context. | -| `params.state` | [`State`](../interfaces/State.md) | The state object containing values to replace the placeholders in the template. | -| `params.template` | `string` | The template string containing placeholders to be replaced with state values. | +| Name | Type | Description | +| :---------------- | :-------------------------------- | :------------------------------------------------------------------------------ | +| `params` | `Object` | The parameters for composing the context. | +| `params.state` | [`State`](../interfaces/State.md) | The state object containing values to replace the placeholders in the template. | +| `params.template` | `string` | The template string containing placeholders to be replaced with state values. | #### Returns diff --git a/docs/docs/functions/createGoal.md b/docs/docs/functions/createGoal.md index 2d275c57e9..1e732271fe 100644 --- a/docs/docs/functions/createGoal.md +++ b/docs/docs/functions/createGoal.md @@ -10,11 +10,11 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `goal` | [`Goal`](../interfaces/Goal.md) | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | +| Name | Type | +| :--------------- | :------------------------------------------- | +| `«destructured»` | `Object` | +| › `goal` | [`Goal`](../interfaces/Goal.md) | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | #### Returns diff --git a/docs/docs/functions/createRelationship.md b/docs/docs/functions/createRelationship.md index 3f9e8f78fd..0994aac69b 100644 --- a/docs/docs/functions/createRelationship.md +++ b/docs/docs/functions/createRelationship.md @@ -10,12 +10,12 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| › `userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | -| › `userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| Name | Type | +| :--------------- | :--------------------------------------------------------- | +| `«destructured»` | `Object` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| › `userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| › `userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | #### Returns diff --git a/docs/docs/functions/formatActionConditions.md b/docs/docs/functions/formatActionConditions.md index 485c7edf06..6aad1feedb 100644 --- a/docs/docs/functions/formatActionConditions.md +++ b/docs/docs/functions/formatActionConditions.md @@ -12,8 +12,8 @@ Formats the conditions for each provided action into a string, listing each acti #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :------------------------------------------------------------- | | `actions` | [`Action`](../interfaces/Action.md)[] | An array of `Action` objects from which to extract conditions. | #### Returns diff --git a/docs/docs/functions/formatActionNames.md b/docs/docs/functions/formatActionNames.md index 07c3fe0bae..1c2ffc112c 100644 --- a/docs/docs/functions/formatActionNames.md +++ b/docs/docs/functions/formatActionNames.md @@ -12,8 +12,8 @@ Formats the names of the provided actions into a comma-separated string. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :-------------------------------------------------------- | | `actions` | [`Action`](../interfaces/Action.md)[] | An array of `Action` objects from which to extract names. | #### Returns diff --git a/docs/docs/functions/formatActions.md b/docs/docs/functions/formatActions.md index e8dc965d95..50a83a67ac 100644 --- a/docs/docs/functions/formatActions.md +++ b/docs/docs/functions/formatActions.md @@ -12,8 +12,8 @@ Formats the provided actions into a detailed string listing each action's name a #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :-------------------------------------- | | `actions` | [`Action`](../interfaces/Action.md)[] | An array of `Action` objects to format. | #### Returns diff --git a/docs/docs/functions/formatActors.md b/docs/docs/functions/formatActors.md index e6feb310c1..dba84ed1a2 100644 --- a/docs/docs/functions/formatActors.md +++ b/docs/docs/functions/formatActors.md @@ -12,10 +12,10 @@ Format actors into a string #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `actors` | `Object` | list of actors | -| `actors.actors` | [`Actor`](../interfaces/Actor.md)[] | - | +| Name | Type | Description | +| :-------------- | :---------------------------------- | :------------- | +| `actors` | `Object` | list of actors | +| `actors.actors` | [`Actor`](../interfaces/Actor.md)[] | - | #### Returns diff --git a/docs/docs/functions/formatEvaluatorConditions.md b/docs/docs/functions/formatEvaluatorConditions.md index ce17bd4f73..ee7dc6cd2f 100644 --- a/docs/docs/functions/formatEvaluatorConditions.md +++ b/docs/docs/functions/formatEvaluatorConditions.md @@ -12,8 +12,8 @@ Formats the conditions under which each evaluator is relevant into a readable st #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :----------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects. | #### Returns diff --git a/docs/docs/functions/formatEvaluatorExampleConditions.md b/docs/docs/functions/formatEvaluatorExampleConditions.md index 142fcd5df1..d4554a442a 100644 --- a/docs/docs/functions/formatEvaluatorExampleConditions.md +++ b/docs/docs/functions/formatEvaluatorExampleConditions.md @@ -12,8 +12,8 @@ Generates a string describing the conditions under which each evaluator example #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :------------------------------------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects, each containing examples. | #### Returns diff --git a/docs/docs/functions/formatEvaluatorExampleDescriptions.md b/docs/docs/functions/formatEvaluatorExampleDescriptions.md index 72668842f7..4e942c1fc0 100644 --- a/docs/docs/functions/formatEvaluatorExampleDescriptions.md +++ b/docs/docs/functions/formatEvaluatorExampleDescriptions.md @@ -12,8 +12,8 @@ Generates a string summarizing the descriptions of each evaluator example. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :------------------------------------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects, each containing examples. | #### Returns diff --git a/docs/docs/functions/formatEvaluatorExamples.md b/docs/docs/functions/formatEvaluatorExamples.md index 7fd0e2acb9..4b3c6cc0c9 100644 --- a/docs/docs/functions/formatEvaluatorExamples.md +++ b/docs/docs/functions/formatEvaluatorExamples.md @@ -12,8 +12,8 @@ Formats evaluator examples into a readable string, replacing placeholders with g #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :----------------------------------------------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects, each containing examples to format. | #### Returns diff --git a/docs/docs/functions/formatEvaluatorNames.md b/docs/docs/functions/formatEvaluatorNames.md index 41157bdfbe..076f74a1b2 100644 --- a/docs/docs/functions/formatEvaluatorNames.md +++ b/docs/docs/functions/formatEvaluatorNames.md @@ -12,8 +12,8 @@ Formats the names of evaluators into a comma-separated list, each enclosed in si #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :----------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects. | #### Returns diff --git a/docs/docs/functions/formatEvaluators.md b/docs/docs/functions/formatEvaluators.md index df036a16d2..854d499199 100644 --- a/docs/docs/functions/formatEvaluators.md +++ b/docs/docs/functions/formatEvaluators.md @@ -12,8 +12,8 @@ Formats evaluator details into a string, including both the name and description #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----------- | :------------------------------------------ | :----------------------------- | | `evaluators` | [`Evaluator`](../interfaces/Evaluator.md)[] | An array of evaluator objects. | #### Returns diff --git a/docs/docs/functions/formatGoalsAsString.md b/docs/docs/functions/formatGoalsAsString.md index dd1c2dada0..ecee1f9f5c 100644 --- a/docs/docs/functions/formatGoalsAsString.md +++ b/docs/docs/functions/formatGoalsAsString.md @@ -10,10 +10,10 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `goals` | [`Goal`](../interfaces/Goal.md)[] | +| Name | Type | +| :--------------- | :-------------------------------- | +| `«destructured»` | `Object` | +| › `goals` | [`Goal`](../interfaces/Goal.md)[] | #### Returns diff --git a/docs/docs/functions/formatLore.md b/docs/docs/functions/formatLore.md index f842427cfd..990d5cabd8 100644 --- a/docs/docs/functions/formatLore.md +++ b/docs/docs/functions/formatLore.md @@ -12,8 +12,8 @@ Formats an array of lore entries into a single string. Each entry is separated b #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------------------------------------ | :---------------------------------- | | `lore` | [`Memory`](../interfaces/Memory.md)[] | An array of lore entries to format. | #### Returns diff --git a/docs/docs/functions/formatMessages.md b/docs/docs/functions/formatMessages.md index 0fa9f79c62..51e084c840 100644 --- a/docs/docs/functions/formatMessages.md +++ b/docs/docs/functions/formatMessages.md @@ -12,11 +12,11 @@ Format messages into a string #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `actors` | [`Actor`](../interfaces/Actor.md)[] | -| › `messages` | [`Memory`](../interfaces/Memory.md)[] | +| Name | Type | +| :--------------- | :------------------------------------ | +| `«destructured»` | `Object` | +| › `actors` | [`Actor`](../interfaces/Actor.md)[] | +| › `messages` | [`Memory`](../interfaces/Memory.md)[] | #### Returns diff --git a/docs/docs/functions/formatRelationships.md b/docs/docs/functions/formatRelationships.md index ee27fbd7c0..ad6617e8ef 100644 --- a/docs/docs/functions/formatRelationships.md +++ b/docs/docs/functions/formatRelationships.md @@ -10,11 +10,11 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| › `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| Name | Type | +| :--------------- | :--------------------------------------------------------- | +| `«destructured»` | `Object` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| › `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | #### Returns diff --git a/docs/docs/functions/getActorDetails.md b/docs/docs/functions/getActorDetails.md index fd5366d7f9..efa9b0618c 100644 --- a/docs/docs/functions/getActorDetails.md +++ b/docs/docs/functions/getActorDetails.md @@ -12,11 +12,11 @@ Get details for a list of actors. #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | +| Name | Type | +| :--------------- | :--------------------------------------------------------- | +| `«destructured»` | `Object` | +| › `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | #### Returns diff --git a/docs/docs/functions/getFormattedActions.md b/docs/docs/functions/getFormattedActions.md index 34b2841159..f2023db2b6 100644 --- a/docs/docs/functions/getFormattedActions.md +++ b/docs/docs/functions/getFormattedActions.md @@ -12,8 +12,8 @@ Formats the provided actions into a string listing each action's name and descri #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :-------- | :------------------------------------ | :-------------------------------------- | | `actions` | [`Action`](../interfaces/Action.md)[] | An array of `Action` objects to format. | #### Returns diff --git a/docs/docs/functions/getGoals.md b/docs/docs/functions/getGoals.md index 00724775ad..2c280f26eb 100644 --- a/docs/docs/functions/getGoals.md +++ b/docs/docs/functions/getGoals.md @@ -10,14 +10,14 @@ custom_edit_url: null #### Parameters -| Name | Type | Default value | -| :------ | :------ | :------ | -| `«destructured»` | `Object` | `undefined` | -| › `count?` | `number` | `5` | -| › `onlyInProgress?` | `boolean` | `true` | -| › `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | `undefined` | -| › `user_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | +| Name | Type | Default value | +| :------------------ | :--------------------------------------------------------- | :------------ | +| `«destructured»` | `Object` | `undefined` | +| › `count?` | `number` | `5` | +| › `onlyInProgress?` | `boolean` | `true` | +| › `room_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | `undefined` | +| › `user_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `undefined` | #### Returns diff --git a/docs/docs/functions/getLore.md b/docs/docs/functions/getLore.md index 878c702ce9..96b1420859 100644 --- a/docs/docs/functions/getLore.md +++ b/docs/docs/functions/getLore.md @@ -12,14 +12,14 @@ Retrieves lore from the lore database based on a search query. This function use #### Parameters -| Name | Type | Default value | Description | -| :------ | :------ | :------ | :------ | -| `params` | `Object` | `undefined` | The parameters for retrieving lore. | -| `params.count?` | `number` | `undefined` | The maximum number of lore entries to retrieve. | -| `params.match_threshold?` | `number` | `undefined` | The similarity threshold for matching lore entries, lower values mean more strict matching. | -| `params.message` | `string` | `undefined` | The search query message to find relevant lore. | -| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | - | -| `params.runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | `undefined` | The runtime environment of the agent. | +| Name | Type | Default value | Description | +| :------------------------ | :--------------------------------------------------------- | :------------ | :------------------------------------------------------------------------------------------ | +| `params` | `Object` | `undefined` | The parameters for retrieving lore. | +| `params.count?` | `number` | `undefined` | The maximum number of lore entries to retrieve. | +| `params.match_threshold?` | `number` | `undefined` | The similarity threshold for matching lore entries, lower values mean more strict matching. | +| `params.message` | `string` | `undefined` | The search query message to find relevant lore. | +| `params.room_id?` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | `zeroUuid` | - | +| `params.runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | `undefined` | The runtime environment of the agent. | #### Returns diff --git a/docs/docs/functions/getProviders.md b/docs/docs/functions/getProviders.md index 2257856f25..06f07f6ea7 100644 --- a/docs/docs/functions/getProviders.md +++ b/docs/docs/functions/getProviders.md @@ -12,11 +12,11 @@ Formats provider outputs into a string which can be injected into the context. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | -| `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | The Eliza runtime object. | -| `message` | [`Message`](../interfaces/Message.md) | The incoming message object. | -| `state?` | [`State`](../interfaces/State.md) | The current state object. | +| Name | Type | Description | +| :-------- | :------------------------------------------- | :--------------------------- | +| `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | The Eliza runtime object. | +| `message` | [`Message`](../interfaces/Message.md) | The incoming message object. | +| `state?` | [`State`](../interfaces/State.md) | The current state object. | #### Returns diff --git a/docs/docs/functions/getRelationship.md b/docs/docs/functions/getRelationship.md index a46974f5e2..bfffa5c6fa 100644 --- a/docs/docs/functions/getRelationship.md +++ b/docs/docs/functions/getRelationship.md @@ -6,17 +6,17 @@ sidebar_position: 0 custom_edit_url: null --- -▸ **getRelationship**(`«destructured»`): `Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +▸ **getRelationship**(`«destructured»`): `Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| › `userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | -| › `userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| Name | Type | +| :--------------- | :--------------------------------------------------------- | +| `«destructured»` | `Object` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| › `userA` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| › `userB` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | #### Returns -`Promise`\<``null`` \| [`Relationship`](../interfaces/Relationship.md)\> +`Promise`\<`null` \| [`Relationship`](../interfaces/Relationship.md)\> diff --git a/docs/docs/functions/getRelationships.md b/docs/docs/functions/getRelationships.md index e2b014a556..b35c131957 100644 --- a/docs/docs/functions/getRelationships.md +++ b/docs/docs/functions/getRelationships.md @@ -10,11 +10,11 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| › `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | +| Name | Type | +| :--------------- | :--------------------------------------------------------- | +| `«destructured»` | `Object` | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| › `user_id` | \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` | #### Returns diff --git a/docs/docs/functions/parseJSONObjectFromText.md b/docs/docs/functions/parseJSONObjectFromText.md index 1717128a2d..9c2e99ccce 100644 --- a/docs/docs/functions/parseJSONObjectFromText.md +++ b/docs/docs/functions/parseJSONObjectFromText.md @@ -16,8 +16,8 @@ is an array, or returns null if parsing is unsuccessful or the result is neither #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------- | :-------------------------------------------------------------- | | `text` | `string` | The input text from which to extract and parse the JSON object. | #### Returns diff --git a/docs/docs/functions/parseJsonArrayFromText.md b/docs/docs/functions/parseJsonArrayFromText.md index af9bb7c6e0..7380c0096b 100644 --- a/docs/docs/functions/parseJsonArrayFromText.md +++ b/docs/docs/functions/parseJsonArrayFromText.md @@ -6,7 +6,7 @@ sidebar_position: 0 custom_edit_url: null --- -▸ **parseJsonArrayFromText**(`text`): ``null`` \| `any`[] +▸ **parseJsonArrayFromText**(`text`): `null` \| `any`[] Parses a JSON array from a given text. The function looks for a JSON block wrapped in triple backticks with `json` language identifier, and if not found, it searches for an array pattern within the text. @@ -15,12 +15,12 @@ is an array, it returns the array; otherwise, it returns null. #### Parameters -| Name | Type | Description | -| :------ | :------ | :------ | +| Name | Type | Description | +| :----- | :------- | :------------------------------------------------------------- | | `text` | `string` | The input text from which to extract and parse the JSON array. | #### Returns -``null`` \| `any`[] +`null` \| `any`[] An array parsed from the JSON string if successful; otherwise, null. diff --git a/docs/docs/functions/updateGoal.md b/docs/docs/functions/updateGoal.md index c3a481fef0..1cbb1605ee 100644 --- a/docs/docs/functions/updateGoal.md +++ b/docs/docs/functions/updateGoal.md @@ -10,11 +10,11 @@ custom_edit_url: null #### Parameters -| Name | Type | -| :------ | :------ | -| `«destructured»` | `Object` | -| › `goal` | [`Goal`](../interfaces/Goal.md) | -| › `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | +| Name | Type | +| :--------------- | :------------------------------------------- | +| `«destructured»` | `Object` | +| › `goal` | [`Goal`](../interfaces/Goal.md) | +| › `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | #### Returns diff --git a/docs/docs/index.md b/docs/docs/index.md index f1c1813ad6..680164c13b 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -10,112 +10,128 @@ custom_edit_url: null Eliza Banner -*As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)* - -- Multi-agent simulation framework -- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) -- Full-featured Discord and Twitter connectors, with Discord voice channel support -- Full conversational and document RAG memory -- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more -- Highly extensible - create your own actions and clients to extend Eliza's capabilities -- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) -- Supports OpenAI for cloud inference on a light-weight device -- "Ask Claude" mode for calling Claude on more complex queries +_As seen powering [@DegenSpartanAI](https://x.com/degenspartanai) and [@MarcAIndreessen](https://x.com/pmairca)_ + +- Multi-agent simulation framework +- Add as many unique characters as you want with [characterfile](https://github.com/lalalune/characterfile/) +- Full-featured Discord and Twitter connectors, with Discord voice channel support +- Full conversational and document RAG memory +- Can read links and PDFs, transcribe audio and videos, summarize conversations, and more +- Highly extensible - create your own actions and clients to extend Eliza's capabilities +- Supports open source and local models (default configured with Nous Hermes Llama 3.1B) +- Supports OpenAI for cloud inference on a light-weight device +- "Ask Claude" mode for calling Claude on more complex queries - 100% Typescript # Getting Started -## Install Node.js +## Install Node.js + https://docs.npmjs.com/downloading-and-installing-node-js-and-npm -## Edit the .env file -- Copy .env.example to .env and fill in the appropriate values +## Edit the .env file + +- Copy .env.example to .env and fill in the appropriate values - Edit the TWITTER environment variables to add your bot's username and password -## Edit the character file -- Check out the file `src/core/defaultCharacter.ts` - you can modify this +## Edit the character file + +- Check out the file `src/core/defaultCharacter.ts` - you can modify this - You can also load characters with the `node --loader ts-node/esm src/index.ts --characters="path/to/your/character.json"` and run multiple bots at the same time. -### Linux Installation -You might need these -``` -npm install --include=optional sharp +### Linux Installation + +You might need these + +``` +npm install --include=optional sharp ``` -### Run with Llama +### Run with Llama + You can run Llama 70B or 405B models by setting the `XAI_MODEL` environment variable to `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` or `meta-llama/Meta-Llama-3.1-405B-Instruct` -### Run with Grok +### Run with Grok + You can run Grok models by setting the `XAI_MODEL` environment variable to `grok-beta` -### Run with OpenAI +### Run with OpenAI + You can run OpenAI models by setting the `XAI_MODEL` environment variable to `gpt-4o-mini` or `gpt-4o` -# Requires Node 20+ +# Requires Node 20+ + If you are getting strange issues when starting up, make sure you're using Node 20+. Some APIs are not compatible with previous versions. You can check your node version with `node -v`. If you need to install a new version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm). -## Additional Requirements -You may need to install Sharp. If you see an error when starting up, try installing it with the following command: -``` -npm install --include=optional sharp +## Additional Requirements + +You may need to install Sharp. If you see an error when starting up, try installing it with the following command: + +``` +npm install --include=optional sharp ``` # Environment Setup -You will need to add environment variables to your .env file to connect to various platforms: -``` -# Required environment variables -# Start Discord -DISCORD_APPLICATION_ID= +You will need to add environment variables to your .env file to connect to various platforms: + +``` +# Required environment variables +# Start Discord +DISCORD_APPLICATION_ID= DISCORD_API_TOKEN= # Bot token -# Start Twitter -TWITTER_USERNAME= # Account username -TWITTER_PASSWORD= # Account password -TWITTER_EMAIL= # Account email -TWITTER_COOKIES= # Account cookies +# Start Twitter +TWITTER_USERNAME= # Account username +TWITTER_PASSWORD= # Account password +TWITTER_EMAIL= # Account email +TWITTER_COOKIES= # Account cookies ``` # Local Setup ## CUDA Setup -If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. -``` -npm install -npx --no node-llama-cpp source download --gpu cuda +If you have an NVIDIA GPU, you can install CUDA to speed up local inference dramatically. + +``` +npm install +npx --no node-llama-cpp source download --gpu cuda ``` Make sure that you've installed the CUDA Toolkit, including cuDNN and cuBLAS. -## Running locally -Add XAI_MODEL and set it to one of the above options from [Run with -Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it +## Running locally + +Add XAI_MODEL and set it to one of the above options from [Run with +Llama](#run-with-llama) - you can leave X_SERVER_URL and XAI_API_KEY blank, it downloads the model from huggingface and queries it locally # Cloud Setup (with OpenAI) -In addition to the environment variables above, you will need to add the following: -``` -# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. +In addition to the environment variables above, you will need to add the following: + +``` +# OpenAI handles the bulk of the work with chat, TTS, image recognition, etc. OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk- -# The agent can also ask Claude for help if you have an API key +# The agent can also ask Claude for help if you have an API key ANTHROPIC_API_KEY= -# For Elevenlabs voice generation on Discord voice +# For Elevenlabs voice generation on Discord voice ELEVENLABS_XI_API_KEY= # API key from elevenlabs -# ELEVENLABS SETTINGS -ELEVENLABS_MODEL_ID=eleven_multilingual_v2 -ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM -ELEVENLABS_VOICE_STABILITY=0.5 -ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 -ELEVENLABS_VOICE_STYLE=0.66 -ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false -ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 -ELEVENLABS_OUTPUT_FORMAT=pcm_16000 +# ELEVENLABS SETTINGS +ELEVENLABS_MODEL_ID=eleven_multilingual_v2 +ELEVENLABS_VOICE_ID=21m00Tcm4TlvDq8ikWAM +ELEVENLABS_VOICE_STABILITY=0.5 +ELEVENLABS_VOICE_SIMILARITY_BOOST=0.9 +ELEVENLABS_VOICE_STYLE=0.66 +ELEVENLABS_VOICE_USE_SPEAKER_BOOST=false +ELEVENLABS_OPTIMIZE_STREAMING_LATENCY=4 +ELEVENLABS_OUTPUT_FORMAT=pcm_16000 ``` -# Discord Bot +# Discord Bot + For help with setting up your Discord Bot, check out here: https://discordjs.guide/preparations/setting-up-a-bot-application.html diff --git a/docs/docs/interfaces/Account.md b/docs/docs/interfaces/Account.md index d5a54c5e87..c5f131148c 100644 --- a/docs/docs/interfaces/Account.md +++ b/docs/docs/interfaces/Account.md @@ -10,11 +10,11 @@ Represents a user, including their name, details, and a unique identifier. ## Properties -### avatar\_url +### avatar_url -• `Optional` **avatar\_url**: `string` +• `Optional` **avatar_url**: `string` -___ +--- ### details @@ -24,19 +24,19 @@ ___ ▪ [key: `string`]: `unknown` -___ +--- ### email • `Optional` **email**: `string` -___ +--- ### id • **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### name diff --git a/docs/docs/interfaces/Action.md b/docs/docs/interfaces/Action.md index 7cdcd6b454..5ed091a1e5 100644 --- a/docs/docs/interfaces/Action.md +++ b/docs/docs/interfaces/Action.md @@ -14,31 +14,31 @@ Represents an action that the agent can perform, including conditions for its us • **condition**: `string` -___ +--- ### description • **description**: `string` -___ +--- ### examples • **examples**: [`ActionExample`](ActionExample.md)[][] -___ +--- ### handler • **handler**: [`Handler`](../types/Handler.md) -___ +--- ### name • **name**: `string` -___ +--- ### validate diff --git a/docs/docs/interfaces/ActionExample.md b/docs/docs/interfaces/ActionExample.md index 2e0ef5bb56..431fd40c0c 100644 --- a/docs/docs/interfaces/ActionExample.md +++ b/docs/docs/interfaces/ActionExample.md @@ -14,7 +14,7 @@ Represents an example of content, typically used for demonstrating or testing pu • **content**: [`Content`](Content.md) -___ +--- ### user diff --git a/docs/docs/interfaces/Actor.md b/docs/docs/interfaces/Actor.md index c9fc269c29..41bf539785 100644 --- a/docs/docs/interfaces/Actor.md +++ b/docs/docs/interfaces/Actor.md @@ -16,19 +16,19 @@ Represents an actor in the conversation, which could be a user or the agent itse #### Type declaration -| Name | Type | -| :------ | :------ | -| `quote` | `string` | +| Name | Type | +| :-------- | :------- | +| `quote` | `string` | | `summary` | `string` | | `tagline` | `string` | -___ +--- ### id • **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### name diff --git a/docs/docs/interfaces/Content.md b/docs/docs/interfaces/Content.md index 2ba345b31e..33cdcad95d 100644 --- a/docs/docs/interfaces/Content.md +++ b/docs/docs/interfaces/Content.md @@ -18,13 +18,13 @@ Represents the content of a message, including its main text (`content`), any as • `Optional` **action**: `string` -___ +--- ### content • **content**: `string` -___ +--- ### source diff --git a/docs/docs/interfaces/ConversationExample.md b/docs/docs/interfaces/ConversationExample.md index b747ef5f19..41c7f7bea2 100644 --- a/docs/docs/interfaces/ConversationExample.md +++ b/docs/docs/interfaces/ConversationExample.md @@ -14,8 +14,8 @@ Represents an example of content, typically used for demonstrating or testing pu • **content**: [`Content`](Content.md) -___ +--- -### user\_id +### user_id -• **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/EvaluationExample.md b/docs/docs/interfaces/EvaluationExample.md index d0182266e9..88e2ab46ff 100644 --- a/docs/docs/interfaces/EvaluationExample.md +++ b/docs/docs/interfaces/EvaluationExample.md @@ -14,13 +14,13 @@ Represents an example for evaluation, including the context, an array of message • **context**: `string` -___ +--- ### messages • **messages**: [`ActionExample`](ActionExample.md)[] -___ +--- ### outcome diff --git a/docs/docs/interfaces/Evaluator.md b/docs/docs/interfaces/Evaluator.md index c598a691d5..61b2fa4dd5 100644 --- a/docs/docs/interfaces/Evaluator.md +++ b/docs/docs/interfaces/Evaluator.md @@ -14,31 +14,31 @@ Represents an evaluator, which is used to assess and guide the agent's responses • **condition**: `string` -___ +--- ### description • **description**: `string` -___ +--- ### examples • **examples**: [`EvaluationExample`](EvaluationExample.md)[] -___ +--- ### handler • **handler**: [`Handler`](../types/Handler.md) -___ +--- ### name • **name**: `string` -___ +--- ### validate diff --git a/docs/docs/interfaces/Goal.md b/docs/docs/interfaces/Goal.md index 8feb250a66..b10d8fb41d 100644 --- a/docs/docs/interfaces/Goal.md +++ b/docs/docs/interfaces/Goal.md @@ -14,32 +14,32 @@ Represents a goal, which is a higher-level aim composed of one or more objective • `Optional` **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### name • **name**: `string` -___ +--- ### objectives • **objectives**: [`Objective`](Objective.md)[] -___ +--- -### room\_id +### room_id -• **room\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **room_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### status • **status**: [`GoalStatus`](../enums/GoalStatus.md) -___ +--- -### user\_id +### user_id -• **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/Memory.md b/docs/docs/interfaces/Memory.md index 3208495b42..d9de6d2a67 100644 --- a/docs/docs/interfaces/Memory.md +++ b/docs/docs/interfaces/Memory.md @@ -14,32 +14,32 @@ Represents a memory record, which could be a message or any other piece of infor • **content**: [`Content`](Content.md) -___ +--- -### created\_at +### created_at -• `Optional` **created\_at**: `string` +• `Optional` **created_at**: `string` -___ +--- ### embedding • `Optional` **embedding**: `number`[] -___ +--- ### id • `Optional` **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### room\_id +### room_id -• **room\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **room_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### user\_id +### user_id -• **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/Message.md b/docs/docs/interfaces/Message.md index 090b9a5d7d..6787778d5a 100644 --- a/docs/docs/interfaces/Message.md +++ b/docs/docs/interfaces/Message.md @@ -14,14 +14,14 @@ Represents a message within the conversation, including its content and associat • **content**: [`Content`](Content.md) -___ +--- -### room\_id +### room_id -• **room\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **room_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### user\_id +### user_id -• **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/MessageExample.md b/docs/docs/interfaces/MessageExample.md index 52fea9eb76..9d1edcd80e 100644 --- a/docs/docs/interfaces/MessageExample.md +++ b/docs/docs/interfaces/MessageExample.md @@ -14,7 +14,7 @@ Represents an example of a message, typically used for demonstrating or testing • **content**: [`Content`](Content.md) -___ +--- ### user diff --git a/docs/docs/interfaces/Objective.md b/docs/docs/interfaces/Objective.md index bc0b96a28a..71bbc907c6 100644 --- a/docs/docs/interfaces/Objective.md +++ b/docs/docs/interfaces/Objective.md @@ -14,13 +14,13 @@ Represents an objective within a goal, detailing what needs to be achieved and w • **completed**: `boolean` -___ +--- ### description • **description**: `string` -___ +--- ### id diff --git a/docs/docs/interfaces/Participant.md b/docs/docs/interfaces/Participant.md index 4f2e9301c8..e1710b005e 100644 --- a/docs/docs/interfaces/Participant.md +++ b/docs/docs/interfaces/Participant.md @@ -14,7 +14,7 @@ Represents a participant in a room, including their ID and account details. • **account**: [`Account`](Account.md) -___ +--- ### id diff --git a/docs/docs/interfaces/Provider.md b/docs/docs/interfaces/Provider.md index 00b909dac6..f1d89fcae4 100644 --- a/docs/docs/interfaces/Provider.md +++ b/docs/docs/interfaces/Provider.md @@ -12,7 +12,7 @@ Represents a provider, which is used to retrieve information or perform actions ### get -• **get**: (`runtime`: [`BgentRuntime`](../classes/BgentRuntime.md), `message`: [`Message`](Message.md), `state?`: [`State`](State.md)) => `Promise`\<`unknown`\> +• **get**: (`runtime`: [`AgentRuntime`](../classes/AgentRuntime.md), `message`: [`Message`](Message.md), `state?`: [`State`](State.md)) => `Promise`\<`unknown`\> #### Type declaration @@ -20,11 +20,11 @@ Represents a provider, which is used to retrieve information or perform actions ##### Parameters -| Name | Type | -| :------ | :------ | -| `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| `message` | [`Message`](Message.md) | -| `state?` | [`State`](State.md) | +| Name | Type | +| :-------- | :------------------------------------------- | +| `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| `message` | [`Message`](Message.md) | +| `state?` | [`State`](State.md) | ##### Returns diff --git a/docs/docs/interfaces/Relationship.md b/docs/docs/interfaces/Relationship.md index 34aed83257..e7c9c95b6d 100644 --- a/docs/docs/interfaces/Relationship.md +++ b/docs/docs/interfaces/Relationship.md @@ -10,42 +10,42 @@ Represents a relationship between two users, including their IDs, the status of ## Properties -### created\_at +### created_at -• `Optional` **created\_at**: `string` +• `Optional` **created_at**: `string` -___ +--- ### id • **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### room\_id +### room_id -• **room\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **room_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### status • **status**: `string` -___ +--- -### user\_a +### user_a -• **user\_a**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_a**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### user\_b +### user_b -• **user\_b**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_b**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- -### user\_id +### user_id -• **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/Room.md b/docs/docs/interfaces/Room.md index 8c02891c5c..04a85413dc 100644 --- a/docs/docs/interfaces/Room.md +++ b/docs/docs/interfaces/Room.md @@ -14,7 +14,7 @@ Represents a room or conversation context, including its ID and a list of partic • **id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### participants diff --git a/docs/docs/interfaces/State.md b/docs/docs/interfaces/State.md index d937700e8e..97ea31ba0d 100644 --- a/docs/docs/interfaces/State.md +++ b/docs/docs/interfaces/State.md @@ -18,122 +18,122 @@ Represents the state of the conversation or context in which the agent is operat • `Optional` **actionExamples**: `string` -___ +--- ### actionNames • `Optional` **actionNames**: `string` -___ +--- ### actions • `Optional` **actions**: `string` -___ +--- ### actionsData • `Optional` **actionsData**: [`Action`](Action.md)[] -___ +--- ### actors • **actors**: `string` -___ +--- ### actorsData • `Optional` **actorsData**: [`Actor`](Actor.md)[] -___ +--- ### agentId • `Optional` **agentId**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### agentName • `Optional` **agentName**: `string` -___ +--- ### goals • `Optional` **goals**: `string` -___ +--- ### goalsData • `Optional` **goalsData**: [`Goal`](Goal.md)[] -___ +--- ### providers • `Optional` **providers**: `string` -___ +--- ### recentFacts • `Optional` **recentFacts**: `string` -___ +--- ### recentFactsData • `Optional` **recentFactsData**: [`Memory`](Memory.md)[] -___ +--- ### recentMessages • **recentMessages**: `string` -___ +--- ### recentMessagesData • **recentMessagesData**: [`Memory`](Memory.md)[] -___ +--- ### relevantFacts • `Optional` **relevantFacts**: `string` -___ +--- ### relevantFactsData • `Optional` **relevantFactsData**: [`Memory`](Memory.md)[] -___ +--- ### responseData • `Optional` **responseData**: [`Content`](Content.md) -___ +--- -### room\_id +### room_id -• **room\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• **room_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` -___ +--- ### senderName • `Optional` **senderName**: `string` -___ +--- -### user\_id +### user_id -• `Optional` **user\_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` +• `Optional` **user_id**: \`$\{string}-$\{string}-$\{string}-$\{string}-$\{string}\` diff --git a/docs/docs/interfaces/_category_.yml b/docs/docs/interfaces/_category_.yml index 43bec88cfa..2fb3459e08 100644 --- a/docs/docs/interfaces/_category_.yml +++ b/docs/docs/interfaces/_category_.yml @@ -1,2 +1,2 @@ label: "Interfaces" -position: 4 \ No newline at end of file +position: 4 diff --git a/docs/docs/modules.md b/docs/docs/modules.md index 7143f2fc60..8200f88ad9 100644 --- a/docs/docs/modules.md +++ b/docs/docs/modules.md @@ -5,4 +5,3 @@ sidebar_label: "Exports" sidebar_position: 0.5 custom_edit_url: null --- - diff --git a/docs/docs/rooms.md b/docs/docs/rooms.md index 42d19cd22d..fc43fc7a2c 100644 --- a/docs/docs/rooms.md +++ b/docs/docs/rooms.md @@ -11,12 +11,15 @@ In the context of eliza, a room can be a physical space, such as a chat room, or ## Key Concepts ### Room + A room contains a list of participants which can be agents or users, and can be added or removed from at any time. The room is keyed by the `room_id`. ### Participant + A participant is an agent or user that is part of the room. Participants are identified by their 'user_id' which is keyed to the account of the user or agent. ### Ensuring that a Room and Participant Exists + Every memory needs to be associated with a room and a participant. On databases where foreign keys are available, we try to enforce this constraint. -When handling messages, you can use the built-in `BgentRuntime.ensureRoomExists(user_id, room_id)` and `BgentRuntime.ensureParticipantExists()` methods to ensure that the room and participant are correctly set up. \ No newline at end of file +When handling messages, you can use the built-in `AgentRuntime.ensureRoomExists(user_id, room_id)` and `AgentRuntime.ensureParticipantExists()` methods to ensure that the room and participant are correctly set up. diff --git a/docs/docs/types/Handler.md b/docs/docs/types/Handler.md index 72a58f6ad3..37562f6d3a 100644 --- a/docs/docs/types/Handler.md +++ b/docs/docs/types/Handler.md @@ -6,7 +6,7 @@ sidebar_position: 0 custom_edit_url: null --- -Ƭ **Handler**: (`runtime`: [`BgentRuntime`](../classes/BgentRuntime.md), `message`: [`Message`](../interfaces/Message.md), `state?`: [`State`](../interfaces/State.md), `options?`: \{ `[key: string]`: `unknown`; }) => `Promise`\<`unknown`\> +Ƭ **Handler**: (`runtime`: [`AgentRuntime`](../classes/AgentRuntime.md), `message`: [`Message`](../interfaces/Message.md), `state?`: [`State`](../interfaces/State.md), `options?`: \{ `[key: string]`: `unknown`; }) => `Promise`\<`unknown`\> Represents the type of a handler function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to any type. @@ -16,12 +16,12 @@ Represents the type of a handler function, which takes a runtime instance, a mes ##### Parameters -| Name | Type | -| :------ | :------ | -| `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| `message` | [`Message`](../interfaces/Message.md) | -| `state?` | [`State`](../interfaces/State.md) | -| `options?` | `Object` | +| Name | Type | +| :--------- | :------------------------------------------- | +| `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| `message` | [`Message`](../interfaces/Message.md) | +| `state?` | [`State`](../interfaces/State.md) | +| `options?` | `Object` | ##### Returns diff --git a/docs/docs/types/Validator.md b/docs/docs/types/Validator.md index 7592bdaef7..f3624c669c 100644 --- a/docs/docs/types/Validator.md +++ b/docs/docs/types/Validator.md @@ -6,7 +6,7 @@ sidebar_position: 0 custom_edit_url: null --- -Ƭ **Validator**: (`runtime`: [`BgentRuntime`](../classes/BgentRuntime.md), `message`: [`Message`](../interfaces/Message.md), `state?`: [`State`](../interfaces/State.md)) => `Promise`\<`boolean`\> +Ƭ **Validator**: (`runtime`: [`AgentRuntime`](../classes/AgentRuntime.md), `message`: [`Message`](../interfaces/Message.md), `state?`: [`State`](../interfaces/State.md)) => `Promise`\<`boolean`\> Represents the type of a validator function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to a boolean indicating whether the validation passed. @@ -16,11 +16,11 @@ Represents the type of a validator function, which takes a runtime instance, a m ##### Parameters -| Name | Type | -| :------ | :------ | -| `runtime` | [`BgentRuntime`](../classes/BgentRuntime.md) | -| `message` | [`Message`](../interfaces/Message.md) | -| `state?` | [`State`](../interfaces/State.md) | +| Name | Type | +| :-------- | :------------------------------------------- | +| `runtime` | [`AgentRuntime`](../classes/AgentRuntime.md) | +| `message` | [`Message`](../interfaces/Message.md) | +| `state?` | [`State`](../interfaces/State.md) | ##### Returns diff --git a/docs/docs/types/_category_.yml b/docs/docs/types/_category_.yml index f80828f935..ab809aa490 100644 --- a/docs/docs/types/_category_.yml +++ b/docs/docs/types/_category_.yml @@ -1,2 +1,2 @@ label: "Type aliases" -position: 5 \ No newline at end of file +position: 5 diff --git a/docs/docs/variables/_category_.yml b/docs/docs/variables/_category_.yml index b9b3fc6e32..f4d4bc5ac9 100644 --- a/docs/docs/variables/_category_.yml +++ b/docs/docs/variables/_category_.yml @@ -1,2 +1,2 @@ label: "Variables" -position: 6 \ No newline at end of file +position: 6 diff --git a/docs/docs/variables/embeddingDimension.md b/docs/docs/variables/embeddingDimension.md index 0be13b76ff..f563c76e57 100644 --- a/docs/docs/variables/embeddingDimension.md +++ b/docs/docs/variables/embeddingDimension.md @@ -6,4 +6,4 @@ sidebar_position: 0 custom_edit_url: null --- -• `Const` **embeddingDimension**: ``1536`` +• `Const` **embeddingDimension**: `1536` diff --git a/docs/docs/variables/evaluationTemplate.md b/docs/docs/variables/evaluationTemplate.md index 27b4230dc4..d0c5622011 100644 --- a/docs/docs/variables/evaluationTemplate.md +++ b/docs/docs/variables/evaluationTemplate.md @@ -6,6 +6,6 @@ sidebar_position: 0 custom_edit_url: null --- -• `Const` **evaluationTemplate**: ``"TASK: Based on the conversation and conditions, determine which evaluation functions are appropriate to call.\nExamples:\n{{evaluatorExamples}}\n\nINSTRUCTIONS: You are helping me to decide which appropriate functions to call based on the conversation between {{senderName}} and {{agentName}}.\n\n{{recentMessages}}\n\nEvaluator Functions:\n{{evaluators}}\n\nEvaluator Conditions:\n{{evaluatorConditions}}\n\nTASK: Based on the most recent conversation, determine which evaluators functions are appropriate to call to call.\nInclude the name of evaluators that are relevant and should be called in the array\nAvailable evaluator names to include are {{evaluatorNames}}\nRespond with a JSON array containing a field for description in a JSON block formatted for markdown with this structure:\n```json\n[\n 'evaluatorName',\n 'evaluatorName'\n]\n```\n\nYour response must include the JSON block."`` +• `Const` **evaluationTemplate**: `"TASK: Based on the conversation and conditions, determine which evaluation functions are appropriate to call.\nExamples:\n{{evaluatorExamples}}\n\nINSTRUCTIONS: You are helping me to decide which appropriate functions to call based on the conversation between {{senderName}} and {{agentName}}.\n\n{{recentMessages}}\n\nEvaluator Functions:\n{{evaluators}}\n\nEvaluator Conditions:\n{{evaluatorConditions}}\n\nTASK: Based on the most recent conversation, determine which evaluators functions are appropriate to call to call.\nInclude the name of evaluators that are relevant and should be called in the array\nAvailable evaluator names to include are {{evaluatorNames}}\nRespond with a JSON array containing a field for description in a JSON block formatted for markdown with this structure:\n```json\n[\n 'evaluatorName',\n 'evaluatorName'\n]\n```\n\nYour response must include the JSON block."` Template used for the evaluation completion. diff --git a/docs/docs/variables/messageHandlerTemplate.md b/docs/docs/variables/messageHandlerTemplate.md index c683149053..abe2e0b646 100644 --- a/docs/docs/variables/messageHandlerTemplate.md +++ b/docs/docs/variables/messageHandlerTemplate.md @@ -6,4 +6,4 @@ sidebar_position: 0 custom_edit_url: null --- -• `Const` **messageHandlerTemplate**: ``"{{actionExamples}}\n\n# IMPORTANT: DO NOT USE THE INFORMATION FROM THE EXAMPLES ABOVE. THE EXAMPLES ARE FOR REFERENCE ONLY.\n\n~~~\n\n# TASK: GENERATE THE NEXT MESSAGE IN THE SCENE FOR {{agentName}}\n- Generate the next message in the scene for {{agentName}}\n- {{agentName}} is not an assistant - do not write assistant-like responses or ask questions\n- Include content and action in the response\n- Available actions are {{actionNames}}\n\n{{lore}}\n{{relevantFacts}}\n{{recentFacts}}\n{{goals}}\n{{actors}}\n{{actionNames}}\n{{actions}}\n{{providers}}\n\n# INSTRUCTIONS: Generate the next message in the scene for {{agentName}}\n\nResponse format should be formatted in a JSON block like this:\n```json\n{ \"user\": \"{{agentName}}\", \"content\": string, \"action\": string }\n```\n\n{{recentMessages}}"`` +• `Const` **messageHandlerTemplate**: `"{{actionExamples}}\n\n# IMPORTANT: DO NOT USE THE INFORMATION FROM THE EXAMPLES ABOVE. THE EXAMPLES ARE FOR REFERENCE ONLY.\n\n~~~\n\n# TASK: GENERATE THE NEXT MESSAGE IN THE SCENE FOR {{agentName}}\n- Generate the next message in the scene for {{agentName}}\n- {{agentName}} is not an assistant - do not write assistant-like responses or ask questions\n- Include content and action in the response\n- Available actions are {{actionNames}}\n\n{{lore}}\n{{relevantFacts}}\n{{recentFacts}}\n{{goals}}\n{{actors}}\n{{actionNames}}\n{{actions}}\n{{providers}}\n\n# INSTRUCTIONS: Generate the next message in the scene for {{agentName}}\n\nResponse format should be formatted in a JSON block like this:\n```json\n{ \"user\": \"{{agentName}}\", \"content\": string, \"action\": string }\n```\n\n{{recentMessages}}"` diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index c16e74d74b..5e387e1f38 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -4,44 +4,44 @@ // There are various equivalent ways to declare your Docusaurus config. // See: https://docusaurus.io/docs/api/docusaurus-config -import { themes as prismThemes } from 'prism-react-renderer' +import { themes as prismThemes } from "prism-react-renderer"; /** @type {import('@docusaurus/types').Config} */ const config = { - title: 'eliza', - tagline: 'The flexible, scalable AI agent for everyone', - favicon: 'img/favicon.ico', + title: "eliza", + tagline: "The flexible, scalable AI agent for everyone", + favicon: "img/favicon.ico", // Set the production url of your site here - url: 'https://docs.ai16z.vc', + url: "https://docs.ai16z.vc", // Set the // pathname under which your site is served // For GitHub pages deployment, it is often '//' - baseUrl: '/', + baseUrl: "/", // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. - organizationName: 'ai16z', // Usually your GitHub org/user name. - projectName: 'eliza', // Usually your repo name. + organizationName: "ai16z", // Usually your GitHub org/user name. + projectName: "eliza", // Usually your repo name. - onBrokenLinks: 'throw', - onBrokenMarkdownLinks: 'warn', + onBrokenLinks: "throw", + onBrokenMarkdownLinks: "warn", // Even if you don't use internationalization, you can use this field to set // useful metadata like html lang. For example, if your site is Chinese, you // may want to replace "en" with "zh-Hans". i18n: { - defaultLocale: 'en', - locales: ['en'] + defaultLocale: "en", + locales: ["en"], }, plugins: [ [ - 'docusaurus-plugin-typedoc', + "docusaurus-plugin-typedoc", // Plugin / TypeDoc options { - entryPoints: ['../src/index.ts'], - tsconfig: '../tsconfig.json', + entryPoints: ["../src/index.ts"], + tsconfig: "../tsconfig.json", out: "./docs", }, ], @@ -49,83 +49,81 @@ const config = { presets: [ [ - 'classic', + "classic", /** @type {import('@docusaurus/preset-classic').Options} */ ({ docs: { - sidebarPath: './sidebars.js', + sidebarPath: "./sidebars.js", // Please change this to your repo. // Remove this to remove the "edit this page" links. - editUrl: - 'https://github.com/ai16z/eliza/tree/main/docs/' + editUrl: "https://github.com/ai16z/eliza/tree/main/docs/", }, theme: { - customCss: './src/css/custom.css' - } - }) - ] + customCss: "./src/css/custom.css", + }, + }), + ], ], themeConfig: /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({ // Replace with your project's social card - image: 'img/docusaurus-social-card.jpg', + image: "img/docusaurus-social-card.jpg", navbar: { - title: 'eliza', + title: "eliza", items: [ { - type: 'docSidebar', - sidebarId: 'tutorialSidebar', - position: 'left', - label: 'API Documentation' + type: "docSidebar", + sidebarId: "tutorialSidebar", + position: "left", + label: "API Documentation", }, { - href: 'https://github.com/ai16z/eliza', - label: 'GitHub', - position: 'right' - } - ] + href: "https://github.com/ai16z/eliza", + label: "GitHub", + position: "right", + }, + ], }, footer: { - style: 'dark', + style: "dark", links: [ { - title: 'Docs', - items: [ - ] + title: "Docs", + items: [], }, { - title: 'Community', + title: "Community", items: [ { - label: 'Discord', - href: 'https://discord.gg/xe7HtWru' + label: "Discord", + href: "https://discord.gg/xe7HtWru", }, { - label: 'Twitter', - href: 'https://twitter.com/pmairca' - } - ] + label: "Twitter", + href: "https://twitter.com/pmairca", + }, + ], }, { - title: 'More', + title: "More", items: [ { - label: 'GitHub', - href: 'https://github.com/ai16z/eliza' - } - ] - } + label: "GitHub", + href: "https://github.com/ai16z/eliza", + }, + ], + }, ], - copyright: `Copyright © ${new Date().getFullYear()} ai16z.vc` + copyright: `Copyright © ${new Date().getFullYear()} ai16z.vc`, }, prism: { theme: prismThemes.github, - darkTheme: prismThemes.dracula - } - }) -} + darkTheme: prismThemes.dracula, + }, + }), +}; -export default config +export default config; diff --git a/docs/sidebars.js b/docs/sidebars.js index 14554ca1d4..9f48621919 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -14,7 +14,7 @@ /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure - tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], + tutorialSidebar: [{ type: "autogenerated", dirName: "." }], }; export default sidebars; diff --git a/docs/src/components/HomepageFeatures/index.js b/docs/src/components/HomepageFeatures/index.js index 6b263d4944..e5b4508f52 100644 --- a/docs/src/components/HomepageFeatures/index.js +++ b/docs/src/components/HomepageFeatures/index.js @@ -1,60 +1,64 @@ -import clsx from 'clsx' -import Heading from '@theme/Heading' -import styles from './styles.module.css' +import clsx from "clsx"; +import Heading from "@theme/Heading"; +import styles from "./styles.module.css"; const FeatureList = [ { - title: 'Made By Users', + title: "Made By Users", description: ( <> - eliza is open source, community developed and designed to be and easy to use. + eliza is open source, community developed and designed to be and easy to + use. - ) + ), }, { - title: 'Just Works', + title: "Just Works", description: ( <> - Simple, tested, used in production. Configurable enough, batteries included. + Simple, tested, used in production. Configurable enough, batteries + included. - ) + ), }, { - title: 'For a Better World', + title: "For a Better World", description: ( <> - If you have ideas for how to make better agents for everyone, let's do it together. + If you have ideas for how to make better agents for everyone,{" "} + let's do it together. - ) - } -] + ), + }, +]; -function Feature ({ Svg, title, description }) { +function Feature({ Svg, title, description }) { return ( -
+
- {/*
+ /* white if the theme is light, black if the theme is dark */ + style={{ + // white background, rounded corners + backgroundColor: "#ffffff88", + borderRadius: "10px", + padding: "10px", + paddingTop: "20px", + margin: "5px", + }} + > + {/*
*/} -
- {title} -

{description}

-
+
+ {title} +

{description}

+
- ) + ); } -export default function HomepageFeatures () { +export default function HomepageFeatures() { return (
@@ -65,5 +69,5 @@ export default function HomepageFeatures () {
- ) + ); } diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index 345259ddee..ab05cc5c01 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -18,7 +18,7 @@ } /* For readability concerns, you should choose a lighter palette in dark mode. */ -[data-theme='dark'] { +[data-theme="dark"] { --ifm-color-primary: lightblue; --ifm-color-primary-dark: #21af90; --ifm-color-primary-darker: #1fa588; diff --git a/docs/src/pages/index.js b/docs/src/pages/index.js index 0e17d2c2f9..b65c6b6cf0 100644 --- a/docs/src/pages/index.js +++ b/docs/src/pages/index.js @@ -1,12 +1,12 @@ -import useDocusaurusContext from '@docusaurus/useDocusaurusContext' -import HomepageFeatures from '@site/src/components/HomepageFeatures' -import Layout from '@theme/Layout' +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import HomepageFeatures from "@site/src/components/HomepageFeatures"; +import Layout from "@theme/Layout"; -import Heading from '@theme/Heading' -import styles from './index.module.css' +import Heading from "@theme/Heading"; +import styles from "./index.module.css"; function HomepageHeader() { - const { siteConfig } = useDocusaurusContext() + const { siteConfig } = useDocusaurusContext(); return (
@@ -16,29 +16,28 @@ function HomepageHeader() {

{siteConfig.tagline}

- ) + ); } export default function Home() { - const { siteConfig } = useDocusaurusContext() + const { siteConfig } = useDocusaurusContext(); return ( - - -
+ +
- ) + ); } diff --git a/scripts/extracttweets.js b/scripts/extracttweets.js index 09b30752ba..2af6c4c06f 100644 --- a/scripts/extracttweets.js +++ b/scripts/extracttweets.js @@ -1,30 +1,32 @@ -import fs from 'fs'; +import fs from "fs"; -const EXPORTED_DATA_FILE = 'tweets.json'; -const TWEETS_FILE = 'exportedtweets.json'; +const EXPORTED_DATA_FILE = "tweets.json"; +const TWEETS_FILE = "exportedtweets.json"; // Read the exported data from the JSON file -const exportedData = JSON.parse(fs.readFileSync(EXPORTED_DATA_FILE, 'utf-8')); +const exportedData = JSON.parse(fs.readFileSync(EXPORTED_DATA_FILE, "utf-8")); // Extract the text of each tweet -const tweetTexts = exportedData.map(tweet => { - console.log(tweet.username); - if (tweet.username.toLowerCase().replace('@pmarca', '') !== 'pmarca') { - return null - } else { - console.log('pmarca found'); - } +const tweetTexts = exportedData + .map((tweet) => { + console.log(tweet.username); + if (tweet.username.toLowerCase().replace("@pmarca", "") !== "pmarca") { + return null; + } else { + console.log("pmarca found"); + } - if (tweet.isRetweet && tweet.retweetedStatus) { - // If the tweet is a retweet, use the text of the retweeted status - return tweet.retweetedStatus.text; - } else { - // Otherwise, use the text of the tweet itself - return tweet.text; - } -}).filter(tweet => tweet !== null); + if (tweet.isRetweet && tweet.retweetedStatus) { + // If the tweet is a retweet, use the text of the retweeted status + return tweet.retweetedStatus.text; + } else { + // Otherwise, use the text of the tweet itself + return tweet.text; + } + }) + .filter((tweet) => tweet !== null); // Write the array of tweet texts to the tweets.json file fs.writeFileSync(TWEETS_FILE, JSON.stringify(tweetTexts, null, 2)); -console.log('Tweet texts extracted and saved to', TWEETS_FILE); \ No newline at end of file +console.log("Tweet texts extracted and saved to", TWEETS_FILE); diff --git a/scripts/gettweets.mjs b/scripts/gettweets.mjs index 2e5db89813..3c6dd9a8e1 100644 --- a/scripts/gettweets.mjs +++ b/scripts/gettweets.mjs @@ -1,70 +1,76 @@ -import { Scraper } from 'agent-twitter-client'; -import dotenv from 'dotenv'; -import fs from 'fs'; +import { Scraper } from "agent-twitter-client"; +import dotenv from "dotenv"; +import fs from "fs"; dotenv.config(); -const TWEETS_FILE = 'tweets.json'; +const TWEETS_FILE = "tweets.json"; (async () => { - try { - // Create a new instance of the Scraper - const scraper = new Scraper(); - - // Log in to Twitter using the configured environment variables - await scraper.login(process.env.TWITTER_USERNAME, process.env.TWITTER_PASSWORD); - - // Check if login was successful - if (await scraper.isLoggedIn()) { - console.log('Logged in successfully!'); - - // Fetch all tweets for the user "@realdonaldtrump" - const tweets = scraper.getTweets('pmarca', 2000); - - // Initialize an empty array to store the fetched tweets - let fetchedTweets = []; - - // Load existing tweets from the JSON file if it exists - if (fs.existsSync(TWEETS_FILE)) { - const fileContent = fs.readFileSync(TWEETS_FILE, 'utf-8'); - fetchedTweets = JSON.parse(fileContent); - } - - // skip first 200 - - let count = 0; - - // Fetch and process tweets - for await (const tweet of tweets) { - if (count < 1000) { - count++; - continue; - } + try { + // Create a new instance of the Scraper + const scraper = new Scraper(); + + // Log in to Twitter using the configured environment variables + await scraper.login( + process.env.TWITTER_USERNAME, + process.env.TWITTER_PASSWORD + ); + + // Check if login was successful + if (await scraper.isLoggedIn()) { + console.log("Logged in successfully!"); + + // Fetch all tweets for the user "@realdonaldtrump" + const tweets = scraper.getTweets("pmarca", 2000); + + // Initialize an empty array to store the fetched tweets + let fetchedTweets = []; - console.log('--------------------'); - console.log('Tweet ID:', tweet.id); - console.log('Text:', tweet.text); - console.log('Created At:', tweet.createdAt); - console.log('Retweets:', tweet.retweetCount); - console.log('Likes:', tweet.likeCount); - console.log('--------------------'); + // Load existing tweets from the JSON file if it exists + if (fs.existsSync(TWEETS_FILE)) { + const fileContent = fs.readFileSync(TWEETS_FILE, "utf-8"); + fetchedTweets = JSON.parse(fileContent); + } - // Add the new tweet to the fetched tweets array - fetchedTweets.push(tweet); + // skip first 200 - // Save the updated fetched tweets to the JSON file - fs.writeFileSync(TWEETS_FILE, JSON.stringify(fetchedTweets, null, 2)); - } + let count = 0; - console.log('All tweets fetched and saved to', TWEETS_FILE); + // Fetch and process tweets + for await (const tweet of tweets) { + if (count < 1000) { + count++; + continue; + } - // Log out from Twitter - await scraper.logout(); - console.log('Logged out successfully!'); - } else { - console.log('Login failed. Please check your credentials.'); + console.log("--------------------"); + console.log("Tweet ID:", tweet.id); + console.log("Text:", tweet.text); + console.log("Created At:", tweet.createdAt); + console.log("Retweets:", tweet.retweetCount); + console.log("Likes:", tweet.likeCount); + console.log("--------------------"); + + // Add the new tweet to the fetched tweets array + fetchedTweets.push(tweet); + + // Save the updated fetched tweets to the JSON file + fs.writeFileSync( + TWEETS_FILE, + JSON.stringify(fetchedTweets, null, 2) + ); + } + + console.log("All tweets fetched and saved to", TWEETS_FILE); + + // Log out from Twitter + await scraper.logout(); + console.log("Logged out successfully!"); + } else { + console.log("Login failed. Please check your credentials."); + } + } catch (error) { + console.error("An error occurred:", error); } - } catch (error) { - console.error('An error occurred:', error); - } -})(); \ No newline at end of file +})(); From c90dc39cf384f0fcceb4ac139044202cb3271083 Mon Sep 17 00:00:00 2001 From: moon Date: Fri, 1 Nov 2024 11:00:29 -0700 Subject: [PATCH 7/8] more prettier --- .github/pull_request_template.md | 9 +++++++++ .github/workflows/ci.yaml | 14 +++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4024bc95f8..0be3671922 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,23 @@ # Relates to: + + # Risks + # Background + ## What does this PR do? ## What kind of change is this? + # Documentation changes needed? + + # Testing + ## Where should a reviewer start? ## Detailed testing steps + diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a9bb59b1c..e60c3e1710 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,21 +9,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - uses: pnpm/action-setup@v3 with: version: 9.4.0 - + - uses: actions/setup-node@v4 with: - node-version: '23' - cache: 'pnpm' - + node-version: "23" + cache: "pnpm" + - name: Install dependencies run: pnpm i - + - name: Run Prettier run: pnpm run prettier --check . - + - name: Build packages run: pnpm run build From f1a5af8ea1522efe9c917e191964bb84e301db6c Mon Sep 17 00:00:00 2001 From: moon Date: Fri, 1 Nov 2024 12:06:36 -0700 Subject: [PATCH 8/8] new lock --- pnpm-lock.yaml | 340 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9e2a674fe..1a7158f468 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -437,6 +437,9 @@ importers: clsx: specifier: 2.1.0 version: 2.1.0 + docusaurus-lunr-search: + specifier: ^3.5.0 + version: 3.5.0(@docusaurus/core@3.5.2(@docusaurus/types@3.1.1(acorn@8.14.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mdx-js/react@3.0.1(@types/react@18.3.12)(react@18.2.0))(acorn@8.14.0)(bufferutil@4.0.8)(eslint@9.13.0(jiti@1.21.6))(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(typescript@5.6.3)(utf-8-validate@5.0.10))(react-dom@18.2.0(react@18.2.0))(react@18.2.0) prism-react-renderer: specifier: 2.3.1 version: 2.3.1(react@18.2.0) @@ -3178,6 +3181,9 @@ packages: '@types/har-format@1.2.16': resolution: {integrity: sha512-fluxdy7ryD3MV6h8pTfTYpy/xQzCFC7m89nOH9y94cNqJ1mDIDPut7MnRHI3F6qRmh/cT2fUjG1MLdCNb4hE9A==} + '@types/hast@2.3.10': + resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} + '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} @@ -3259,6 +3265,9 @@ packages: '@types/parse-json@4.0.2': resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} + '@types/parse5@5.0.3': + resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} + '@types/phoenix@1.6.5': resolution: {integrity: sha512-xegpDuR+z0UqG9fwHqNoy3rI7JDlvaPh2TY47Fl80oq6g+hXT+c/LEuE43X48clZ6lOfANl5WrPur9fYO1RJ/w==} @@ -3787,6 +3796,9 @@ packages: resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} engines: {node: '>= 4.0.0'} + autocomplete.js@0.37.1: + resolution: {integrity: sha512-PgSe9fHYhZEsm/9jggbjtVsGXJkPLvd+9mC7gZJ662vVL5CRWEtm/mIrrzCx0MrNxHVwxD5d00UOn6NsmL2LUQ==} + autoprefixer@10.4.20: resolution: {integrity: sha512-XY25y5xSv/wEoqzDyXXME4AFfkZI0P23z6Fs3YgymDnKJkCGOnkL0iTxCa85UTqaSgfcqyf3UA6+c7wUvx/16g==} engines: {node: ^10 || ^12 || >=14} @@ -3860,6 +3872,9 @@ packages: peerDependencies: '@babel/core': ^7.0.0 + bail@1.0.5: + resolution: {integrity: sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ==} + bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} @@ -3879,6 +3894,9 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} + bcp-47-match@1.0.3: + resolution: {integrity: sha512-LggQ4YTdjWQSKELZF5JwchnBa1u0pIQSZf5lSdOHEdbVP55h0qICA/FUp3+W99q0xqxYa1ZQizTUH87gecII5w==} + bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -4256,6 +4274,10 @@ packages: resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} engines: {node: '>=0.8'} + clsx@1.2.1: + resolution: {integrity: sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==} + engines: {node: '>=6'} + clsx@2.1.0: resolution: {integrity: sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==} engines: {node: '>=6'} @@ -4318,6 +4340,9 @@ packages: comlink@4.3.1: resolution: {integrity: sha512-+YbhUdNrpBZggBAHWcgQMLPLH1KDF3wJpeqrCKieWQ8RL7atmgsgTQko1XEBK6PsecfopWNntopJ+ByYG1lRaA==} + comma-separated-tokens@1.0.8: + resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -4600,6 +4625,9 @@ packages: css-select@5.1.0: resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + css-selector-parser@1.4.1: + resolution: {integrity: sha512-HYPSb7y/Z7BNDCOrakL4raGO2zltZkbeXyAd6Tg9obzix6QhzxCotdBl6VT0Dv4vZfJGVz3WL/xaEI9Ly3ul0g==} + css-tree@2.2.1: resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} @@ -4884,6 +4912,10 @@ packages: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} + direction@1.0.4: + resolution: {integrity: sha512-GYqKi1aH7PJXxdhTeZBFrg8vUBeKXi+cNprXsC1kpJcbcVnV9wBsrOu1cQEdG0WeQwlfHiy3XvnKfIrJ2R0NzQ==} + hasBin: true + discord-api-types@0.37.100: resolution: {integrity: sha512-a8zvUI0GYYwDtScfRd/TtaNBDTXwP5DiDVX7K5OmE+DRT57gBqKnwtOC5Ol8z0mRW8KQfETIgiB8U0YZ9NXiCA==} @@ -4901,6 +4933,14 @@ packages: resolution: {integrity: sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==} engines: {node: '>=6'} + docusaurus-lunr-search@3.5.0: + resolution: {integrity: sha512-k3zN4jYMi/prWInJILGKOxE+BVcgYinwj9+gcECsYm52tS+4ZKzXQzbPnVJAEXmvKOfFMcDFvS3MSmm6cEaxIQ==} + engines: {node: '>= 8.10.0'} + peerDependencies: + '@docusaurus/core': ^2.0.0-alpha.60 || ^2.0.0 || ^3.0.0 + react: ^16.8.4 || ^17 || ^18 + react-dom: ^16.8.4 || ^17 || ^18 + docusaurus-plugin-typedoc@0.22.0: resolution: {integrity: sha512-5q+oT+iq3g9DvsMin11pSSmqHRw0i62csHBzkWVtgmsqN/BwGDHnxDiTbHhhkBNQpmRIRgltyNdMnO5x+JUK8Q==} peerDependencies: @@ -5902,15 +5942,30 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hast-util-from-parse5@6.0.1: + resolution: {integrity: sha512-jeJUWiN5pSxW12Rh01smtVkZgZr33wBokLzKLwinYOUfSzm1Nl/c3GUGebDyOKjdsRgMvoVbV0VpAcpjF4NrJA==} + hast-util-from-parse5@8.0.1: resolution: {integrity: sha512-Er/Iixbc7IEa7r/XLtuG52zoqn/b3Xng/w6aZQ0xGVxzhw5xUFxcRqdPzP6yFi/4HBYRaifaI5fQ1RH8n0ZeOQ==} + hast-util-has-property@1.0.4: + resolution: {integrity: sha512-ghHup2voGfgFoHMGnaLHOjbYFACKrRh9KFttdCzMCbFoBMJXiNi2+XTrPP8+q6cDJM/RSqlCfVWrjp1H201rZg==} + + hast-util-is-element@1.1.0: + resolution: {integrity: sha512-oUmNua0bFbdrD/ELDSSEadRVtWZOf3iF6Lbv81naqsIV99RnSCieTbWuWCY8BAeEfKJTKl0gRdokv+dELutHGQ==} + + hast-util-parse-selector@2.2.5: + resolution: {integrity: sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==} + hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} hast-util-raw@9.0.4: resolution: {integrity: sha512-LHE65TD2YiNsHD3YuXcKPHXPLuYh/gjp12mOfU8jxSrm1f/yJpsb0F/KKljS6U9LJoP0Ux+tCe8iJ2AsPzTdgA==} + hast-util-select@4.0.2: + resolution: {integrity: sha512-8EEG2//bN5rrzboPWD2HdS3ugLijNioS1pqOTIolXNf67xxShYw4SQEmVXd3imiBG+U2bC2nVTySr/iRAA7Cjg==} + hast-util-to-estree@3.1.0: resolution: {integrity: sha512-lfX5g6hqVh9kjS/B9E2gSkvHH4SZNiQFiqWS0x9fENzEl+8W12RqdRxX6d/Cwxi30tPQs3bIO+aolQJNp1bIyw==} @@ -5923,9 +5978,21 @@ packages: hast-util-to-parse5@8.0.0: resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==} + hast-util-to-string@1.0.4: + resolution: {integrity: sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==} + + hast-util-to-text@2.0.1: + resolution: {integrity: sha512-8nsgCARfs6VkwH2jJU9b8LNTuR4700na+0h3PqCaEk4MAnMDeu5P0tP8mjk9LLNGxIeQRLbiDbZVw6rku+pYsQ==} + + hast-util-whitespace@1.0.4: + resolution: {integrity: sha512-I5GTdSfhYfAPNztx2xJRQpG8cuDSNt599/7YUn7Gx/WxNMsG+a835k97TDkFgk123cwjfwINaZknkKkphx/f2A==} + hast-util-whitespace@3.0.0: resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + hastscript@6.0.0: + resolution: {integrity: sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==} + hastscript@8.0.0: resolution: {integrity: sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==} @@ -5939,6 +6006,10 @@ packages: history@4.10.1: resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==} + hogan.js@3.0.2: + resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==} + hasBin: true + hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} @@ -6109,6 +6180,9 @@ packages: engines: {node: '>=16.x'} hasBin: true + immediate@3.3.0: + resolution: {integrity: sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==} + immer@9.0.21: resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} @@ -6221,6 +6295,10 @@ packages: is-buffer@1.1.6: resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + is-ci@3.0.1: resolution: {integrity: sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==} hasBin: true @@ -6314,6 +6392,10 @@ packages: resolution: {integrity: sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==} engines: {node: '>=0.10.0'} + is-plain-obj@2.1.0: + resolution: {integrity: sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==} + engines: {node: '>=8'} + is-plain-obj@3.0.0: resolution: {integrity: sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==} engines: {node: '>=10'} @@ -6948,6 +7030,9 @@ packages: lru-queue@0.1.0: resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==} + lunr-languages@1.14.0: + resolution: {integrity: sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==} + lunr@2.3.9: resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==} @@ -6990,6 +7075,9 @@ packages: resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} engines: {node: '>=8'} + mark.js@8.11.1: + resolution: {integrity: sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==} + markdown-extensions@2.0.0: resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} engines: {node: '>=16'} @@ -7396,6 +7484,10 @@ packages: mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mkdirp@0.3.0: + resolution: {integrity: sha512-OHsdUcVAQ6pOtg5JYWpCBo9W/GySVuwvP9hueRMW7UqshC0tbfzLv8wjySTPm3tfUZ/21CE9E1pJagOA91Pxew==} + deprecated: Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.) + mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} hasBin: true @@ -7609,6 +7701,10 @@ packages: engines: {node: '>=10'} hasBin: true + nopt@1.0.10: + resolution: {integrity: sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==} + hasBin: true + nopt@5.0.0: resolution: {integrity: sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==} engines: {node: '>=6'} @@ -7642,6 +7738,9 @@ packages: resolution: {integrity: sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==} engines: {node: '>=14.16'} + not@0.1.0: + resolution: {integrity: sha512-5PDmaAsVfnWUgTUbJ3ERwn7u79Z0dYxN9ErxCpVJJqe2RK0PJ3z+iFUxuqjwtlDDegXvtWoxD/3Fzxox7tFGWA==} + npm-bundled@3.0.1: resolution: {integrity: sha512-+AvaheE/ww1JEwRHOrn4WHNzOxGtVp+adrg2AeZS/7KuxGUYFuBta98wYpfHBbJp6Tg6j1NKSEVHNcfZzJHQwQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -8010,6 +8109,9 @@ packages: parse5-htmlparser2-tree-adapter@7.1.0: resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} + parse5@6.0.1: + resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} + parse5@7.2.1: resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} @@ -8610,6 +8712,9 @@ packages: proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + property-information@5.6.0: + resolution: {integrity: sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==} + property-information@6.5.0: resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==} @@ -8951,6 +9056,9 @@ packages: resolution: {integrity: sha512-3OGZZ4HoLJkkAZx/48mTXJNlmqTGOzc0o9OWQPuWpkOlXXPbyN6OafCcoXUnBqE2D3f/T5L+pWc1kdEmnfnRsA==} hasBin: true + rehype-parse@7.0.1: + resolution: {integrity: sha512-fOiR9a9xH+Le19i4fGzIEowAbwG7idy2Jzs4mOrFWBSJ0sNUgy0ev871dwWnbOo371SjgjG4pwzrbgSVrKxecw==} + rehype-raw@7.0.0: resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} @@ -8989,6 +9097,10 @@ packages: renderkid@3.0.0: resolution: {integrity: sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==} + repeat-string@1.6.1: + resolution: {integrity: sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==} + engines: {node: '>=0.10'} + request@2.88.2: resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} engines: {node: '>= 6'} @@ -9401,6 +9513,9 @@ packages: resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} deprecated: Please use @jridgewell/sourcemap-codec instead + space-separated-tokens@1.1.5: + resolution: {integrity: sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==} + space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} @@ -9823,6 +9938,9 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + to-vfile@6.1.0: + resolution: {integrity: sha512-BxX8EkCxOAZe+D/ToHdDsJcVI4HqQfmw0tCkp31zf3dNP/XWIAjU4CmeuSwsSoOzOTqHPOL0KUzyZqJplkD0Qw==} + together-ai@0.7.0: resolution: {integrity: sha512-/be/HOecBSwRTDHB14vCvHbp1WiNsFxyS4pJlyBoMup1X3n7xD1b/Gm5Z5amlKzD2zll9Y5wscDk7Ut5OsT1nA==} @@ -9863,6 +9981,9 @@ packages: resolution: {integrity: sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==} engines: {node: '>=8'} + trough@1.0.5: + resolution: {integrity: sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA==} + trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -10074,6 +10195,9 @@ packages: unified@11.0.5: resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} + unified@9.2.2: + resolution: {integrity: sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ==} + uniq@1.0.1: resolution: {integrity: sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==} @@ -10093,6 +10217,12 @@ packages: resolution: {integrity: sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==} engines: {node: '>=12'} + unist-util-find-after@3.0.0: + resolution: {integrity: sha512-ojlBqfsBftYXExNu3+hHLfJQ/X1jYY/9vdm4yZWjIbf0VuWF6CRufci1ZyoD/wV2TYMKxXUoNuoqwy+CkgzAiQ==} + + unist-util-is@4.1.0: + resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==} + unist-util-is@6.0.0: resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} @@ -10102,12 +10232,21 @@ packages: unist-util-position@5.0.0: resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + unist-util-stringify-position@2.0.3: + resolution: {integrity: sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==} + unist-util-stringify-position@4.0.0: resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + unist-util-visit-parents@3.1.1: + resolution: {integrity: sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==} + unist-util-visit-parents@6.0.1: resolution: {integrity: sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==} + unist-util-visit@2.0.3: + resolution: {integrity: sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==} + unist-util-visit@5.0.0: resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} @@ -10241,12 +10380,21 @@ packages: resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} engines: {'0': node >=0.6.0} + vfile-location@3.2.0: + resolution: {integrity: sha512-aLEIZKv/oxuCDZ8lkJGhuhztf/BW4M+iHdCwglA/eWc+vtuRFJj8EtgceYFX4LRjOhCAAiNHsKGssC6onJ+jbA==} + vfile-location@5.0.3: resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==} + vfile-message@2.0.4: + resolution: {integrity: sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==} + vfile-message@4.0.2: resolution: {integrity: sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==} + vfile@4.2.1: + resolution: {integrity: sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==} + vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} @@ -10295,6 +10443,9 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + web-namespaces@1.1.4: + resolution: {integrity: sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw==} + web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} @@ -10605,6 +10756,9 @@ packages: zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} + zwitch@1.0.5: + resolution: {integrity: sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -14734,6 +14888,10 @@ snapshots: '@types/har-format@1.2.16': {} + '@types/hast@2.3.10': + dependencies: + '@types/unist': 2.0.11 + '@types/hast@3.0.4': dependencies: '@types/unist': 3.0.3 @@ -14812,6 +14970,8 @@ snapshots: '@types/parse-json@4.0.2': {} + '@types/parse5@5.0.3': {} + '@types/phoenix@1.6.5': {} '@types/prismjs@1.26.5': {} @@ -15429,6 +15589,10 @@ snapshots: at-least-node@1.0.0: {} + autocomplete.js@0.37.1: + dependencies: + immediate: 3.3.0 + autoprefixer@10.4.20(postcss@8.4.47): dependencies: browserslist: 4.24.2 @@ -15550,6 +15714,8 @@ snapshots: babel-plugin-jest-hoist: 29.6.3 babel-preset-current-node-syntax: 1.1.0(@babel/core@7.26.0) + bail@1.0.5: {} + bail@2.0.2: {} balanced-match@1.0.2: {} @@ -15564,6 +15730,8 @@ snapshots: batch@0.6.1: {} + bcp-47-match@1.0.3: {} + bcrypt-pbkdf@1.0.2: dependencies: tweetnacl: 0.14.5 @@ -16000,6 +16168,8 @@ snapshots: clone@2.1.2: {} + clsx@1.2.1: {} + clsx@2.1.0: {} cmake-js@7.3.0: @@ -16063,6 +16233,8 @@ snapshots: comlink@4.3.1: {} + comma-separated-tokens@1.0.8: {} + comma-separated-tokens@2.0.3: {} command-exists@1.2.9: {} @@ -16375,6 +16547,8 @@ snapshots: domutils: 3.1.0 nth-check: 2.1.1 + css-selector-parser@1.4.1: {} + css-tree@2.2.1: dependencies: mdn-data: 2.0.28 @@ -16644,6 +16818,8 @@ snapshots: dependencies: path-type: 4.0.0 + direction@1.0.4: {} + discord-api-types@0.37.100: {} discord-api-types@0.37.83: {} @@ -16672,6 +16848,26 @@ snapshots: dependencies: '@leichtgewicht/ip-codec': 2.0.5 + docusaurus-lunr-search@3.5.0(@docusaurus/core@3.5.2(@docusaurus/types@3.1.1(acorn@8.14.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mdx-js/react@3.0.1(@types/react@18.3.12)(react@18.2.0))(acorn@8.14.0)(bufferutil@4.0.8)(eslint@9.13.0(jiti@1.21.6))(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(typescript@5.6.3)(utf-8-validate@5.0.10))(react-dom@18.2.0(react@18.2.0))(react@18.2.0): + dependencies: + '@docusaurus/core': 3.5.2(@docusaurus/types@3.1.1(acorn@8.14.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@mdx-js/react@3.0.1(@types/react@18.3.12)(react@18.2.0))(acorn@8.14.0)(bufferutil@4.0.8)(eslint@9.13.0(jiti@1.21.6))(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(typescript@5.6.3)(utf-8-validate@5.0.10) + autocomplete.js: 0.37.1 + clsx: 1.2.1 + gauge: 3.0.2 + hast-util-select: 4.0.2 + hast-util-to-text: 2.0.1 + hogan.js: 3.0.2 + lunr: 2.3.9 + lunr-languages: 1.14.0 + mark.js: 8.11.1 + minimatch: 3.1.2 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + rehype-parse: 7.0.1 + to-vfile: 6.1.0 + unified: 9.2.2 + unist-util-is: 4.1.0 + docusaurus-plugin-typedoc@0.22.0(typedoc-plugin-markdown@3.17.1(typedoc@0.25.8(typescript@5.6.3)))(typedoc@0.25.8(typescript@5.6.3)): dependencies: typedoc: 0.25.8(typescript@5.6.3) @@ -17826,6 +18022,15 @@ snapshots: dependencies: function-bind: 1.1.2 + hast-util-from-parse5@6.0.1: + dependencies: + '@types/parse5': 5.0.3 + hastscript: 6.0.0 + property-information: 5.6.0 + vfile: 4.2.1 + vfile-location: 3.2.0 + web-namespaces: 1.1.4 + hast-util-from-parse5@8.0.1: dependencies: '@types/hast': 3.0.4 @@ -17837,6 +18042,12 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 + hast-util-has-property@1.0.4: {} + + hast-util-is-element@1.1.0: {} + + hast-util-parse-selector@2.2.5: {} + hast-util-parse-selector@4.0.0: dependencies: '@types/hast': 3.0.4 @@ -17857,6 +18068,23 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 + hast-util-select@4.0.2: + dependencies: + bcp-47-match: 1.0.3 + comma-separated-tokens: 1.0.8 + css-selector-parser: 1.4.1 + direction: 1.0.4 + hast-util-has-property: 1.0.4 + hast-util-is-element: 1.1.0 + hast-util-to-string: 1.0.4 + hast-util-whitespace: 1.0.4 + not: 0.1.0 + nth-check: 2.1.1 + property-information: 5.6.0 + space-separated-tokens: 1.1.5 + unist-util-visit: 2.0.3 + zwitch: 1.0.5 + hast-util-to-estree@3.1.0: dependencies: '@types/estree': 1.0.6 @@ -17922,10 +18150,28 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 + hast-util-to-string@1.0.4: {} + + hast-util-to-text@2.0.1: + dependencies: + hast-util-is-element: 1.1.0 + repeat-string: 1.6.1 + unist-util-find-after: 3.0.0 + + hast-util-whitespace@1.0.4: {} + hast-util-whitespace@3.0.0: dependencies: '@types/hast': 3.0.4 + hastscript@6.0.0: + dependencies: + '@types/hast': 2.3.10 + comma-separated-tokens: 1.0.8 + hast-util-parse-selector: 2.2.5 + property-information: 5.6.0 + space-separated-tokens: 1.1.5 + hastscript@8.0.0: dependencies: '@types/hast': 3.0.4 @@ -17947,6 +18193,11 @@ snapshots: tiny-warning: 1.0.3 value-equal: 1.0.1 + hogan.js@3.0.2: + dependencies: + mkdirp: 0.3.0 + nopt: 1.0.10 + hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1 @@ -18146,6 +18397,8 @@ snapshots: dependencies: queue: 6.0.2 + immediate@3.3.0: {} + immer@9.0.21: {} import-fresh@3.3.0: @@ -18275,6 +18528,8 @@ snapshots: is-buffer@1.1.6: {} + is-buffer@2.0.5: {} + is-ci@3.0.1: dependencies: ci-info: 3.9.0 @@ -18334,6 +18589,8 @@ snapshots: is-plain-obj@1.1.0: {} + is-plain-obj@2.1.0: {} + is-plain-obj@3.0.0: {} is-plain-obj@4.1.0: {} @@ -19272,6 +19529,8 @@ snapshots: dependencies: es5-ext: 0.10.64 + lunr-languages@1.14.0: {} + lunr@2.3.9: {} magic-bytes.js@1.10.0: {} @@ -19324,6 +19583,8 @@ snapshots: map-obj@4.3.0: {} + mark.js@8.11.1: {} + markdown-extensions@2.0.0: {} markdown-it@14.1.0: @@ -20034,6 +20295,8 @@ snapshots: mkdirp-classic@0.5.3: {} + mkdirp@0.3.0: {} + mkdirp@0.5.6: dependencies: minimist: 1.2.8 @@ -20266,6 +20529,10 @@ snapshots: touch: 3.1.1 undefsafe: 2.0.5 + nopt@1.0.10: + dependencies: + abbrev: 1.1.1 + nopt@5.0.0: dependencies: abbrev: 1.1.1 @@ -20300,6 +20567,8 @@ snapshots: normalize-url@8.0.1: {} + not@0.1.0: {} + npm-bundled@3.0.1: dependencies: npm-normalize-package-bin: 3.0.1 @@ -20794,6 +21063,8 @@ snapshots: domhandler: 5.0.3 parse5: 7.2.1 + parse5@6.0.1: {} + parse5@7.2.1: dependencies: entities: 4.5.0 @@ -21367,6 +21638,10 @@ snapshots: retry: 0.12.0 signal-exit: 3.0.7 + property-information@5.6.0: + dependencies: + xtend: 4.0.2 + property-information@6.5.0: {} proto-list@1.2.4: {} @@ -21839,6 +22114,11 @@ snapshots: dependencies: jsesc: 3.0.2 + rehype-parse@7.0.1: + dependencies: + hast-util-from-parse5: 6.0.1 + parse5: 6.0.1 + rehype-raw@7.0.0: dependencies: '@types/hast': 3.0.4 @@ -21930,6 +22210,8 @@ snapshots: lodash: 4.17.21 strip-ansi: 6.0.1 + repeat-string@1.6.1: {} + request@2.88.2: dependencies: aws-sign2: 0.7.0 @@ -22446,6 +22728,8 @@ snapshots: sourcemap-codec@1.4.8: {} + space-separated-tokens@1.1.5: {} + space-separated-tokens@2.0.2: {} spdx-correct@3.2.0: @@ -22881,6 +23165,11 @@ snapshots: dependencies: is-number: 7.0.0 + to-vfile@6.1.0: + dependencies: + is-buffer: 2.0.5 + vfile: 4.2.1 + together-ai@0.7.0(encoding@0.1.13): dependencies: '@types/node': 18.19.63 @@ -22921,6 +23210,8 @@ snapshots: trim-newlines@3.0.1: {} + trough@1.0.5: {} + trough@2.2.0: {} true-myth@6.2.0: {} @@ -23111,6 +23402,16 @@ snapshots: trough: 2.2.0 vfile: 6.0.3 + unified@9.2.2: + dependencies: + '@types/unist': 2.0.11 + bail: 1.0.5 + extend: 3.0.2 + is-buffer: 2.0.5 + is-plain-obj: 2.1.0 + trough: 1.0.5 + vfile: 4.2.1 + uniq@1.0.1: {} unique-filename@3.0.0: @@ -23127,6 +23428,12 @@ snapshots: dependencies: crypto-random-string: 4.0.0 + unist-util-find-after@3.0.0: + dependencies: + unist-util-is: 4.1.0 + + unist-util-is@4.1.0: {} + unist-util-is@6.0.0: dependencies: '@types/unist': 3.0.3 @@ -23139,15 +23446,30 @@ snapshots: dependencies: '@types/unist': 3.0.3 + unist-util-stringify-position@2.0.3: + dependencies: + '@types/unist': 2.0.11 + unist-util-stringify-position@4.0.0: dependencies: '@types/unist': 3.0.3 + unist-util-visit-parents@3.1.1: + dependencies: + '@types/unist': 2.0.11 + unist-util-is: 4.1.0 + unist-util-visit-parents@6.0.1: dependencies: '@types/unist': 3.0.3 unist-util-is: 6.0.0 + unist-util-visit@2.0.3: + dependencies: + '@types/unist': 2.0.11 + unist-util-is: 4.1.0 + unist-util-visit-parents: 3.1.1 + unist-util-visit@5.0.0: dependencies: '@types/unist': 3.0.3 @@ -23270,16 +23592,30 @@ snapshots: core-util-is: 1.0.2 extsprintf: 1.3.0 + vfile-location@3.2.0: {} + vfile-location@5.0.3: dependencies: '@types/unist': 3.0.3 vfile: 6.0.3 + vfile-message@2.0.4: + dependencies: + '@types/unist': 2.0.11 + unist-util-stringify-position: 2.0.3 + vfile-message@4.0.2: dependencies: '@types/unist': 3.0.3 unist-util-stringify-position: 4.0.0 + vfile@4.2.1: + dependencies: + '@types/unist': 2.0.11 + is-buffer: 2.0.5 + unist-util-stringify-position: 2.0.3 + vfile-message: 2.0.4 + vfile@6.0.3: dependencies: '@types/unist': 3.0.3 @@ -23339,6 +23675,8 @@ snapshots: dependencies: defaults: 1.0.4 + web-namespaces@1.1.4: {} + web-namespaces@2.0.1: {} web-streams-polyfill@4.0.0-beta.3: {} @@ -23726,4 +24064,6 @@ snapshots: zod@3.23.8: {} + zwitch@1.0.5: {} + zwitch@2.0.4: {}