Lokale LLM-Fallstudien

ollama pull qwen3:30b-a3b-q5_K_M

export ANTHROPIC_BASE_URL=https://api.tokios.com
export ANTHROPIC_API_KEY=sk-tok-YOUR_KEY
claude --model mac-coder

vllm serve Qwen/Qwen2.5-72B-Instruct-AWQ \
  --host 127.0.0.1 \
  --port 8000 \
  --quantization awq \
  --gpu-memory-utilization 0.90 \
  --max-model-len 8192 \
  --max-num-seqs 8

{
  "Upstream": {
    "BaseUrl": "http://127.0.0.1:8000/v1"
  }
}

vllm serve Qwen/Qwen3-30B-A3B-FP8 \
  --host 127.0.0.1 \
  --port 8000 \
  --gpu-memory-utilization 0.85 \
  --max-model-len 16384

{
  "Routes": [
    { "Deployment": "daily",      "Upstream": "vllm-main" },
    { "Deployment": "reasoning",  "Upstream": "vllm-reason" },
    { "Deployment": "classifier", "Upstream": "vllm-fast" }
  ],
  "Upstreams": {
    "vllm-main":   { "BaseUrl": "http://127.0.0.1:8000/v1" },
    "vllm-reason": { "BaseUrl": "http://127.0.0.1:8001/v1" },
    "vllm-fast":   { "BaseUrl": "http://127.0.0.1:8002/v1" }
  }
}

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 127.0.0.1 \
  --port 8000 \
  --gpu-memory-utilization 0.90 \
  --max-model-len 8192

vllm serve microsoft/phi-4 \
  --host 127.0.0.1 \
  --port 8001 \
  --gpu-memory-utilization 0.85 \
  --max-model-len 4096

{
  "Routes": [
    { "Deployment": "analyzer",   "Upstream": "analysis" },
    { "Deployment": "classifier", "Upstream": "classify" }
  ],
  "Upstreams": {
    "analysis": { "BaseUrl": "http://127.0.0.1:8000/v1" },
    "classify": { "BaseUrl": "http://127.0.0.1:8001/v1" }
  }
}

# From HuggingFace — 4 quantization levels of the same model
huggingface-cli download Qwen/Qwen2.5-7B-Instruct-GGUF \
  qwen2.5-7b-instruct-q4_k_m.gguf \
  qwen2.5-7b-instruct-q5_k_m.gguf \
  qwen2.5-7b-instruct-q8_0.gguf \
  --local-dir ./models

./llama-server \
  -m ./models/qwen2.5-7b-instruct-q5_k_m.gguf \
  --host 127.0.0.1 \
  --port 8080 \
  -ngl 99 \
  -c 4096

# Point evaluation harness at Tokios endpoint
export OPENAI_API_BASE=https://api.tokios.com/v1
export OPENAI_API_KEY=sk-tok-YOUR_KEY

# Run eval suite
python -m lm_eval --model local-completions \
  --model_args model=bench-q5km,base_url=$OPENAI_API_BASE \
  --tasks humaneval \
  --batch_size 1