Replies: 1 comment 6 replies
-
|
Works fine on my end: $ ▶ curl -s --request POST --url http://127.0.0.1:7020/v1/chat/completions --header "Content-Type: application/json" --data '{"messages": [ { "role": "system", "content": "You are a helpful assistant." }, { "role": "user", "content": "Hello, how are you today?" } ], "n_predict": 64}' | jq
{
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "I'm doing well, thank you for asking! I'm a large language model, so I don't have emotions like humans do, but I'm always ready to help and assist with any questions or tasks you may have. How about you? How's your day going so far?",
"role": "assistant"
}
}
],
"created": 1727176285,
"model": "gpt-3.5-turbo-0613",
"object": "chat.completion",
"usage": {
"completion_tokens": 58,
"prompt_tokens": 28,
"total_tokens": 86
},
"id": "chatcmpl-vB0PElZDtrMn2PXwBDhF7tkNl54IFFQl"
}Can you show the output of the command above? |
Beta Was this translation helpful? Give feedback.
6 replies
-
|
When I omit |
Beta Was this translation helpful? Give feedback.
-
|
The Can you post the full log of: ./llama-cli -m ~/Meta-Llama-3.1-8B-Instruct-Q6_K_L.gguf -p "I believe the meaning of life is" -ngl 99 -c 1024 -s 1 -n 64 |
Beta Was this translation helpful? Give feedback.
-
|
The cli completion works as expected: Funnily when I hit the completion endpoint when I started the llama cpp server from my python application then a sensible completion does get generated but in the wrong place (the completion is in model_extra - it should be in choices[0].text):
I'll refactor my code so that llama cpp doesn't start as a subprocess but rather as a seperate thread see if that solves it 😅 |
Beta Was this translation helpful? Give feedback.
-
|
I cannot figure out whats wrong. I'm using llama.cpp built from source: FROM nvidia/cuda:12.6.1-devel-ubuntu24.04 AS base
...
#############################
##### LLAMA.CPP BUILDER #####
#############################
FROM base AS llama-cpp-builder
# Install basic build utilities
RUN apt-get update && apt-get install --no-install-recommends -y \
build-essential \
cmake \
git
# Build llama.cpp from source
RUN git clone https://github.com/ggerganov/llama.cpp /llama.cpp
WORKDIR /llama.cpp
RUN make GGML_CUDA=1 -j4
...
#############################
### DEVELOPMENT CONTAINER ###
#############################
FROM python-base AS dev
...
COPY --from=llama-cpp-builder /llama.cpp /llama.cpp
...then the following python script gives a completion that does not meet OpenAI API spec (the completion should be in import asyncio
from pprint import pprint
import signal
import subprocess
import threading
from typing import List
import time
import httpx
import openai
from huggingface_hub import hf_hub_download # type: ignore[import-untyped]
class InferenceThread(threading.Thread):
cmd: List[str]
process: subprocess.Popen | None
stop_event: threading.Event
def __init__(self, *, id: str, cmd: List[str]) -> None:
super().__init__()
self.cmd = cmd
self.name = id
self.process = None
self.stop_event = threading.Event()
def run(self) -> None:
self.process = subprocess.Popen(
self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
while True:
retcode = self.process.poll()
if retcode is not None:
print(f"Unexpected exit of inference process: {retcode}")
break
if self.stop_event.is_set():
break
time.sleep(1)
print(f"{self.name}: Sending SIGINT")
self.process.send_signal(signal.SIGINT)
def stop(self) -> None:
self.stop_event.set()
async def check_health(port: int, api_key: str):
async with httpx.AsyncClient() as client:
try:
response = await client.get(
f"http://localhost:{port}/health",
headers={"Authorization": f"Bearer {api_key})"},
)
return response.status_code == 200
except Exception:
return False
async def main() -> None:
id = "Meta-Llama-3.1-8B-Instruct-GGUF-Q6_K_L"
huggingface_repo = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
huggingface_file = "Meta-Llama-3.1-8B-Instruct-Q6_K_L.gguf"
context_size = 256
chat_template = "llama3"
port = 7020
api_key = "llama-cpp"
model_gguf = hf_hub_download(repo_id=huggingface_repo, filename=huggingface_file)
args = "--host 0.0.0.0"
args += f" --port {port}"
args += f" --api-key {api_key}"
args += f" --alias {id}"
args += f" --model {model_gguf}"
args += f" --chat-template {chat_template}"
args += f" --ctx-size {context_size}"
args += " --gpu-layers 33"
args += " --n-predict 256"
args = args.split(" ")
cmd = ["/llama.cpp/llama-server", *args]
thread = InferenceThread(id=id, cmd=cmd)
thread.start()
while not await check_health(port, api_key):
await asyncio.sleep(3)
openai_client = openai.AsyncOpenAI(
base_url=f"http://localhost:{port}/v1",
api_key=api_key,
)
async for openai_model in openai_client.models.list():
assert id == openai_model.id
prompt = "I believe the meaning of life is"
completion_start = time.time()
completion = await openai_client.completions.create(
model=id,
prompt=prompt,
timeout=10,
)
completion_end = time.time()
chat_start = time.time()
chat_completion = await openai_client.chat.completions.create(
model=id,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello, how are you today?"},
],
timeout=10,
)
chat_end = time.time()
thread.stop()
thread.join()
print(f"Inference cmd:\n{" ".join(cmd)}")
print("\n")
print("OpenAI completion:")
print(f"{completion_end - completion_start} seconds")
pprint(completion.model_dump())
# This fails but shouldn't
# assert len(completion.choices[0].text) > 0
print("\n")
print("OpenAI chat completion:")
print(f"{chat_end - chat_start} seconds")
pprint(chat_completion.model_dump())
print("\n\n")
# This is ok but nonsence content
assert chat_completion.choices[0].message.content is not None
if __name__ == "__main__":
asyncio.run(main())the above script prints the following output: |
Beta Was this translation helpful? Give feedback.
-
|
Gibberish output may indicate that your model file is corrupted. You should try re-download the file, or try another model file |
Beta Was this translation helpful? Give feedback.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
I'm using the following gguf model: https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q6_K_L.gguf
I'm starting the llama cpp server by running the following command:
When I now try to do a completion with the openai client it times out:
And here's the log from the llama-cpp server corresponding to the completion request which just times out:
It doesn't return anything. I've tried setting
--n-predict -2according to the following [issue](# #3969 (comment)) but that makes the model only produce a single token:{ "choices": [ { "finish_reason": "stop", "index": 0, "message": { "content": "G", "role": "assistant" } } ], "created": 1727164154, "model": "Meta-Llama-3.1-8B-Instruct-GGUF-Q6_K_L", "object": "chat.completion", "usage": { "completion_tokens": 1, "prompt_tokens": 28, "total_tokens": 29 }, "id": "chatcmpl-3PaWsHJQpjhQN0Ekzv8MqAmat4YcQYLf", "__verbose": { "content": "G", "id_slot": 0, "stop": true, "model": "Meta-Llama-3.1-8B-Instruct-GGUF-Q6_K_L", "tokens_predicted": 1, "tokens_evaluated": 28, "generation_settings": { "n_ctx": 1024, "n_predict": -2, "model": "Meta-Llama-3.1-8B-Instruct-GGUF-Q6_K_L", "seed": 4294967295, "seed_cur": 1116617295, "temperature": 0.800000011920929, "dynatemp_range": 0, "dynatemp_exponent": 1, "top_k": 40, "top_p": 0.949999988079071, "min_p": 0.05000000074505806, "tfs_z": 1, "typical_p": 1, "repeat_last_n": 64, "repeat_penalty": 1, "presence_penalty": 0, "frequency_penalty": 0, "mirostat": 0, "mirostat_tau": 5, "mirostat_eta": 0.10000000149011612, "penalize_nl": false, "stop": [], "max_tokens": -1, "n_keep": 0, "n_discard": 0, "ignore_eos": false, "stream": false, "n_probs": 0, "min_keep": 0, "grammar": "", "samplers": [ "top_k", "tfs_z", "typ_p", "top_p", "min_p", "temperature" ] }, "prompt": "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello, how are you today?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", "truncated": false, "stopped_eos": false, "stopped_word": false, "stopped_limit": true, "stopping_word": "", "tokens_cached": 28, "timings": { "prompt_n": 28, "prompt_ms": 126.814, "prompt_per_token_ms": 4.529071428571428, "prompt_per_second": 220.79581118803918, "predicted_n": 1, "predicted_ms": 0.041, "predicted_per_token_ms": 0.041, "predicted_per_second": 24390.243902439022 }, "index": 0, "oaicompat_token_ctr": 1 } }I have the same issue with Llama 3 the OpenAI API server just does not work...
Beta Was this translation helpful? Give feedback.
All reactions