Run the Server

./scripts/start_llama_server.sh    # http://localhost:8080

.\scripts\start_llama_server.ps1   # http://localhost:8080

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "bonsai",
    "messages": [{"role": "user", "content": "What is the capital of France?"}],
    "temperature": 0.5,
    "top_p": 0.85
  }'

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed")

response = client.chat.completions.create(
    model="bonsai",
    messages=[{"role": "user", "content": "What is the capital of France?"}],
    temperature=0.5,
    top_p=0.85,
)
print(response.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "http://localhost:8080/v1",
  apiKey: "not-needed",
});

const response = await client.chat.completions.create({
  model: "bonsai",
  messages: [{ role: "user", content: "What is the capital of France?" }],
  temperature: 0.5,
  top_p: 0.85,
});
console.log(response.choices[0].message.content);

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed")

stream = client.chat.completions.create(
    model="bonsai",
    messages=[{"role": "user", "content": "Write a haiku about memory bandwidth."}],
    stream=True,
)
for chunk in stream:
    print(chunk.choices[0].delta.content or "", end="", flush=True)