vLLM을 로컬에서 실행한 뒤 Tokios를 통해 어디서든 접속할 수 있습니다

pip install vllm

vllm serve mistralai/Mistral-7B-Instruct-v0.2 \
  --host 127.0.0.1 \
  --port 8000

curl http://127.0.0.1:8000/v1/models

# 2 GPUs, tensor parallel
vllm serve meta-llama/Llama-3-70B-Instruct \
  --host 127.0.0.1 \
  --port 8000 \
  --tensor-parallel-size 2 \
  --gpu-memory-utilization 0.90

# docker-compose.yml
services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    ports:
      - "127.0.0.1:8000:8000"
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
    command: >
      --model mistralai/Mistral-7B-Instruct-v0.2
      --host 0.0.0.0
      --port 8000
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

docker compose up

curl http://127.0.0.1:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mistralai/Mistral-7B-Instruct-v0.2",
    "messages": [{"role": "user", "content": "Write hello world in Python"}],
    "max_tokens": 100
  }'

tokios-connector TKS-XXXX-XXXX

{
  "Upstream": {
    "BaseUrl": "http://127.0.0.1:8000/v1"
  }
}

Your GPU machine                  api.tokios.com               Remote client
──────────────                    ──────────────               ─────────────
vLLM ─┐                                                        sk-tok-… key
  (localhost:8000)                                               │
       ├─ connector ──► outbound TLS tunnel ──► gateway ──────► HTTPS
       │  (no inbound     (initiates             (auth +       request
       │   port open)      connection)            routing)
       └─ nothing listens
          on any port