From e81db116fd77ad62cb24d41fcf586307be744c6d Mon Sep 17 00:00:00 2001 From: Ishaan Sehgal Date: Tue, 24 Oct 2023 18:37:04 -0700 Subject: [PATCH 1/2] fix: filepath for inference file (#104) Small fix this filepath was standardized in the image and now needs to be reflected in KAITO --- pkg/inference/preset-inference-types.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/inference/preset-inference-types.go b/pkg/inference/preset-inference-types.go index 4685359f7..66db6684d 100644 --- a/pkg/inference/preset-inference-types.go +++ b/pkg/inference/preset-inference-types.go @@ -39,9 +39,7 @@ var ( presetFalcon40bImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon40BModel) presetFalcon40bInstructImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon40BInstructModel) - baseCommandPresetLlama2AChat = fmt.Sprintf("cd /workspace/llama/%s && torchrun", kaitov1alpha1.PresetLlama2AChat) - baseCommandPresetLlama2BChat = fmt.Sprintf("cd /workspace/llama/%s && torchrun", kaitov1alpha1.PresetLlama2BChat) - baseCommandPresetLlama2CChat = fmt.Sprintf("cd /workspace/llama/%s && torchrun", kaitov1alpha1.PresetLlama2CChat) + baseCommandPresetLlama = "cd /workspace/llama/llama-2 && torchrun" // llamaTextInferenceFile = "inference-api.py" TODO: To support Text Generation Llama Models llamaChatInferenceFile = "inference-api.py" llamaRunParams = map[string]string{ @@ -105,7 +103,7 @@ var ( ModelRunParams: llamaRunParams, InferenceFile: llamaChatInferenceFile, DeploymentTimeout: time.Duration(10) * time.Minute, - BaseCommand: baseCommandPresetLlama2AChat, + BaseCommand: baseCommandPresetLlama, WorldSize: 1, DefaultVolumeMountPath: "/dev/shm", }, @@ -119,7 +117,7 @@ var ( ModelRunParams: llamaRunParams, InferenceFile: llamaChatInferenceFile, DeploymentTimeout: time.Duration(20) * time.Minute, - BaseCommand: baseCommandPresetLlama2BChat, + BaseCommand: baseCommandPresetLlama, WorldSize: 2, DefaultVolumeMountPath: "/dev/shm", }, @@ -133,7 +131,7 @@ var ( ModelRunParams: llamaRunParams, InferenceFile: llamaChatInferenceFile, DeploymentTimeout: time.Duration(30) * time.Minute, - BaseCommand: baseCommandPresetLlama2CChat, + BaseCommand: baseCommandPresetLlama, WorldSize: 8, DefaultVolumeMountPath: "/dev/shm", }, From ffd96e6d712bdd21781dabae3d3b1099663c764d Mon Sep 17 00:00:00 2001 From: Ishaan Sehgal Date: Wed, 25 Oct 2023 11:28:33 -0700 Subject: [PATCH 2/2] fix: tag name (#105) --- pkg/inference/preset-inference-types.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/inference/preset-inference-types.go b/pkg/inference/preset-inference-types.go index 66db6684d..e2816a231 100644 --- a/pkg/inference/preset-inference-types.go +++ b/pkg/inference/preset-inference-types.go @@ -29,15 +29,15 @@ const ( var ( registryName = os.Getenv("PRESET_REGISTRY_NAME") - presetLlama2AChatImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetLlama2AChat) - presetLlama2BChatImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetLlama2BChat) - presetLlama2CChatImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetLlama2CChat) + presetLlama2AChatImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetLlama2AChat) + presetLlama2BChatImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetLlama2BChat) + presetLlama2CChatImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetLlama2CChat) - presetFalcon7bImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon7BModel) - presetFalcon7bInstructImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon7BInstructModel) + presetFalcon7bImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetFalcon7BModel) + presetFalcon7bInstructImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetFalcon7BInstructModel) - presetFalcon40bImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon40BModel) - presetFalcon40bInstructImage = registryName + fmt.Sprintf("/%s:latest", kaitov1alpha1.PresetFalcon40BInstructModel) + presetFalcon40bImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetFalcon40BModel) + presetFalcon40bInstructImage = registryName + fmt.Sprintf("/%s:0.0.1", kaitov1alpha1.PresetFalcon40BInstructModel) baseCommandPresetLlama = "cd /workspace/llama/llama-2 && torchrun" // llamaTextInferenceFile = "inference-api.py" TODO: To support Text Generation Llama Models