fix(initializer): correctly reap dangling processes (mudler#3717)

Signed-off-by: Ettore Di Giacinto <[email protected]>

pkg/model/initializers.go

@@ -376,7 +376,9 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
 
 		if !ready {
 			log.Debug().Msgf("GRPC Service NOT ready")
-			ml.deleteProcess(o.model)
+			if process := client.Process(); process != nil {
+				process.Stop()
+			}
 			return nil, fmt.Errorf("grpc service not ready")
 		}
 
@@ -388,11 +390,15 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
 
 		res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options)
 		if err != nil {
-			ml.deleteProcess(o.model)
+			if process := client.Process(); process != nil {
+				process.Stop()
+			}
 			return nil, fmt.Errorf("could not load model: %w", err)
 		}
 		if !res.Success {
-			ml.deleteProcess(o.model)
+			if process := client.Process(); process != nil {
+				process.Stop()
+			}
 			return nil, fmt.Errorf("could not load model (no success): %s", res.Message)
 		}
 

pkg/model/loader.go

@@ -128,7 +128,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
 	defer ml.mu.Unlock()
 	model, err := loader(modelID, modelName, modelFile)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to load model with internal loader: %s", err)
 	}
 
 	if model == nil {

pkg/model/process.go

@@ -18,14 +18,18 @@ import (
 func (ml *ModelLoader) deleteProcess(s string) error {
 	defer delete(ml.models, s)
 
+	log.Debug().Msgf("Deleting process %s", s)
+
 	m, exists := ml.models[s]
 	if !exists {
+		log.Error().Msgf("Model does not exist %s", s)
 		// Nothing to do
 		return nil
 	}
 
 	process := m.Process()
 	if process == nil {
+		log.Error().Msgf("No process for %s", s)
 		// Nothing to do as there is no process
 		return nil
 	}