- 1× or 2× NVIDIA RTX 3090 (24 GB each)
- Linux (Ubuntu 22.04+ recommended)
- Docker + NVIDIA Container Toolkit
- NVIDIA driver 580.x+
- ~30 GB free disk per model

git clone https://github.com/noonghunna/club-3090.git
cd club-3090

undefined

undefined

bash scripts/launch.sh

undefined

undefined

---

undefined

---

vllm/default          Single-card, chat-optimized (recommended first start)
vllm/dual             Dual-card, 262K ctx, vision, tools, MTP n=3
vllm/long-vision      Dual-card, long-context + vision workloads
vllm/turbo            Dual-card, 4 concurrent streams @ 262K
llamacpp/default      Single-card, full 262K, no prefill cliffs
llamacpp/65k          Single-card, 65K ctx (faster, more VRAM headroom)
llamacpp/dual         Dual-card llama.cpp recipe

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8020/v1",
    api_key="ignored",  # local server, no auth needed
)

undefined

stream = client.chat.completions.create(
    model="qwen3.6-27b-autoround",
    messages=[{"role": "user", "content": "Write a Python quicksort."}],
    max_tokens=1024,
    stream=True,
)
for chunk in stream:
    delta = chunk.choices[0].delta.content
    if delta:
        print(delta, end="", flush=True)
print()

import requests, json

payload = {
    "model": "qwen3.6-27b-autoround",
    "messages": [
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is the capital of France?"},
    ],
    "max_tokens": 200,
    "temperature": 0.7,
}

resp = requests.post(
    "http://localhost:8020/v1/chat/completions",
    headers={"Content-Type": "application/json"},
    json=payload,
    timeout=120,
)
resp.raise_for_status()
print(resp.json()["choices"][0]["message"]["content"])

tools = [
    {
        "type": "function",
        "function": {
            "name": "search_web",
            "description": "Search the web for recent information",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {"type": "string", "description": "Search query"},
                },
                "required": ["query"],
            },
        },
    }
]

response = client.chat.completions.create(
    model="qwen3.6-27b-autoround",
    messages=[{"role": "user", "content": "What's the latest news on CUDA 13?"}],
    tools=tools,
    tool_choice="auto",
    max_tokens=512,
)

msg = response.choices[0].message
if msg.tool_calls:
    for call in msg.tool_calls:
        print(f"Tool: {call.function.name}")
        print(f"Args: {call.function.arguments}")

undefined

undefined

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "http://localhost:8020/v1",
  apiKey: "ignored",
});

async function chat(prompt: string): Promise<string> {
  const response = await client.chat.completions.create({
    model: "qwen3.6-27b-autoround",
    messages: [{ role: "user", content: prompt }],
    max_tokens: 512,
  });
  return response.choices[0].message.content ?? "";
}

// Streaming in Node
async function streamChat(prompt: string): Promise<void> {
  const stream = await client.chat.completions.create({
    model: "qwen3.6-27b-autoround",
    messages: [{ role: "user", content: prompt }],
    max_tokens: 1024,
    stream: true,
  });
  for await (const chunk of stream) {
    process.stdout.write(chunk.choices[0]?.delta?.content ?? "");
  }
  console.log();
}

curl -sf http://localhost:8020/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "qwen3.6-27b-autoround",
    "messages": [{"role": "user", "content": "Capital of France?"}],
    "max_tokens": 200
  }' | jq '.choices[0].message.content'

curl -sf http://localhost:8020/v1/models | jq '.data[].id'

undefined

For dual-card, `tensor-parallel-size 2` and `NVIDIA_VISIBLE_DEVICES=0,1` are set, and `max-model-len` extends to 262144.

---

API Base URL: http://localhost:8020/v1
API Key:      (leave blank or type anything)
Model:        qwen3.6-27b-autoround

{
  "openai.baseURL": "http://localhost:8020/v1",
  "openai.apiKey": "local",
  "openai.model": "qwen3.6-27b-autoround"
}

undefined

---

club-3090/
├── scripts/               Shared model-aware scripts (setup, launch, bench, health)
├── models/
│   └── qwen3.6-27b/
│       ├── vllm/
│       │   ├── compose/   Docker Compose files (all variants)
│       │   └── patches/   tolist_cudagraph, Marlin pad, Genesis pointer
│       ├── llama-cpp/
│       │   └── recipes/   Single-card 65K / 262K-max / dual recipes
│       └── sglang/        Blocked — watch list only
└── docs/
    ├── SINGLE_CARD.md     1× 3090 workload → config guide
    ├── DUAL_CARD.md       2× 3090 workload → config guide
    ├── HARDWARE.md        PCIe vs NVLink, power draw, card compatibility
    ├── GLOSSARY.md        TPS / KV / MTP / TP / prefill cliff definitions
    ├── CLIFFS.md          Prefill cliff root causes and fix landscape
    ├── COMPARISONS.md     Self-host vs cloud cost crossover analysis
    ├── UPSTREAM.md        Tracked upstream issues and PRs
    └── engines/           Per-engine deep dives (vLLM / llama.cpp / SGLang)

undefined

undefined

undefined

undefined

bash scripts/switch.sh llamacpp/default

For vLLM workarounds, see `models/qwen3.6-27b/INTERNALS.md` and `docs/CLIFFS.md`.

bash scripts/health.sh

undefined

bash scripts/switch.sh llamacpp/default

undefined

undefined

undefined

docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Ports}}"

undefined

---