Skip to content

Commit

Permalink
Load wrapper clients
Browse files Browse the repository at this point in the history
Testing with:

```yaml
name: gpt-4o
pipeline:
 tts: voice-it-riccardo_fasol-x-low
 transcription: whisper-base-q5_1
 llm: llama-3.2-1b-instruct:q4_k_m
```

Signed-off-by: Ettore Di Giacinto <[email protected]>
  • Loading branch information
mudler committed Nov 8, 2024
1 parent 4034562 commit 11eee84
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 7 deletions.
2 changes: 1 addition & 1 deletion core/config/backend_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ type BackendConfig struct {
type Pipeline struct {
TTS string `yaml:"tts"`
LLM string `yaml:"llm"`
Transcription string `yaml:"sst"`
Transcription string `yaml:"transcription"`
}

type File struct {
Expand Down
41 changes: 35 additions & 6 deletions core/http/endpoints/openai/realtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import (
"github.com/gofiber/websocket/v2"
"github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
grpc "github.com/mudler/LocalAI/pkg/grpc"
model "github.com/mudler/LocalAI/pkg/model"

"github.com/rs/zerolog/log"
)

Expand Down Expand Up @@ -111,13 +113,17 @@ type Model interface {
}

type wrappedModel struct {
TTS *config.BackendConfig
SST *config.BackendConfig
LLM *config.BackendConfig
TTSConfig *config.BackendConfig
TranscriptionConfig *config.BackendConfig
LLMConfig *config.BackendConfig
TTSClient grpc.Backend
TranscriptionClient grpc.Backend
LLMClient grpc.Backend
}

// returns and loads either a wrapped model or a model that support audio-to-audio
func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, modelName string) (Model, error) {

cfg, err := cl.LoadBackendConfigFileByName(modelName, ml.ModelPath)
if err != nil {
return nil, fmt.Errorf("failed to load backend config: %w", err)
Expand All @@ -134,6 +140,8 @@ func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *
return ml.BackendLoader(opts...)
}

log.Debug().Msg("Loading a wrapped model")

// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
cfgLLM, err := cl.LoadBackendConfigFileByName(cfg.Pipeline.LLM, ml.ModelPath)
if err != nil {
Expand Down Expand Up @@ -165,10 +173,31 @@ func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *
return nil, fmt.Errorf("failed to validate config: %w", err)
}

opts := backend.ModelOptions(*cfgTTS, appConfig)
ttsClient, err := ml.BackendLoader(opts...)
if err != nil {
return nil, fmt.Errorf("failed to load tts model: %w", err)
}

opts = backend.ModelOptions(*cfgSST, appConfig)
transcriptionClient, err := ml.BackendLoader(opts...)
if err != nil {
return nil, fmt.Errorf("failed to load SST model: %w", err)
}

opts = backend.ModelOptions(*cfgLLM, appConfig)
llmClient, err := ml.BackendLoader(opts...)
if err != nil {
return nil, fmt.Errorf("failed to load LLM model: %w", err)
}

return &wrappedModel{
TTS: cfgTTS,
SST: cfgSST,
LLM: cfgLLM,
TTSConfig: cfgTTS,
TranscriptionConfig: cfgSST,
LLMConfig: cfgLLM,
TTSClient: ttsClient,
TranscriptionClient: transcriptionClient,
LLMClient: llmClient,
}, nil
}

Expand Down

0 comments on commit 11eee84

Please sign in to comment.