minor

configuration/backend_rules_onlytwo.json

@@ -0,0 +1,31 @@
+[
+    {
+        "name": "OnlyTwoLT2",
+        "desc": "Test Rule: Only allow 2 backends to be loaded at once, unload the least recently used if a 3rd request comes in",
+        "salience": 10,
+        "when": [
+            "Result.Action == ActionDefs.Blank",
+            "LoadedModelCount <= 1"
+        ],
+        "then": [
+            "Result.Action = ActionDefs.Continue",
+            "Result.ModelName = RequestedModelName"
+        ]
+    },
+    {
+        "name": "OnlyTwoGT2",
+        "desc": "Test Rule: Only allow 2 backends to be loaded at once, unload the least recently used if a 3rd request comes in",
+        "salience": 10,
+        "when": [
+            "Result.Action == ActionDefs.Blank",
+            "LoadedModelCount >= 2"
+        ],
+        "then": [
+            "Log(\"Too many backends in use, unloading least recently used...\")",
+            "LRU = LoadedModels[0].ModelName",
+            "ModelLoader.ShutdownModel(LRU)",
+            "Result.Action = ActionDefs.Continue",
+            "Result.ModelName = RequestedModelName"
+        ]
+    }
+]

core/services/backend_rules.go

@@ -92,15 +92,17 @@ func (rbbs RuleBasedBackendService) RuleBasedLoad(modelName string, alreadyLoade
 	ruleBasedLoadDataCtx := ast.NewDataContext()
 
 	ruleBasedLoadDataCtx.Add("ModelLoader", rbbs.modelLoader)
-	ruleBasedLoadDataCtx.Add("LoadedModelCount", rbbs.modelLoader.LoadedModelCount()) // Still relevant after second line???
+	ruleBasedLoadDataCtx.Add("LoadedModelCount", rbbs.modelLoader.LoadedModelCount())
 	ruleBasedLoadDataCtx.Add("LoadedModels", rbbs.modelLoader.SortedLoadedModelMetadata())
 
 	ruleBasedLoadDataCtx.Add("ActionDefs", ruleBasedBackendResultActionDefinitions)
 
 	ruleBasedLoadDataCtx.Add("RequestedModelName", modelName)
 	ruleBasedLoadDataCtx.Add("Source", source)
 	ruleBasedLoadDataCtx.Add("Request", optionalRequest)
+
 	ruleBasedLoadDataCtx.Add("BackendConfig", bc)
+	ruleBasedLoadDataCtx.Add("AppConfig", rbbs.appConfig)
 
 	ruleBasedLoadDataCtx.Add("Result", result)
 
@@ -114,5 +116,4 @@ func (rbbs RuleBasedBackendService) RuleBasedLoad(modelName string, alreadyLoade
 		return nil, err
 	}
 	return &result, nil
-
 }