Benchmarking Local LLMs on an RTX 3060: What 12GB of VRAM Can Actually Do  📖

#!/bin/bash
# ============================================================================
# Local LLM Benchmark v2
#
# Runs practical tasks against any OpenAI-compatible endpoint.
# Captures outputs, timing, and token counts. Scoring is done manually.
#
# Usage:
#   ./bench.sh <base_url> <model_name>
#
# Examples:
#   ./bench.sh http://192.168.8.123:11434/v1 qwen3.5:9b
#   ./bench.sh http://192.168.8.123:8081/v1 bonsai-8b.gguf
# ============================================================================

set -euo pipefail

BASE_URL="${1:?Usage: ./bench.sh <base_url> <model_name>}"
MODEL="${2:?Usage: ./bench.sh <base_url> <model_name>}"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
RESULT_DIR="./results"
RESULT_FILE="${RESULT_DIR}/${MODEL//[:\/]/_}_${TIMESTAMP}.md"
TEST_COUNT=0

mkdir -p "$RESULT_DIR"

CYAN='\033[0;36m'
YELLOW='\033[1;33m'
NC='\033[0m'

# ── Helpers ─────────────────────────────────────────────────────────────────

chat() {
  local prompt="$1"
  local temp="${2:-0.3}"
  local max_tokens="${3:-2048}"

  curl -s --max-time 300 "$BASE_URL/chat/completions" \
    -H "Content-Type: application/json" \
    -d "$(jq -n \
      --arg model "$MODEL" \
      --arg prompt "$prompt" \
      --argjson temp "$temp" \
      --argjson max "$max_tokens" \
      '{model:$model, messages:[{role:"user",content:$prompt}], temperature:$temp, max_tokens:$max}'
    )" 2>/dev/null
}

chat_with_tools() {
  local prompt="$1"
  local tools="$2"
  local temp="${3:-0.1}"
  local max_tokens="${4:-2048}"

  curl -s --max-time 300 "$BASE_URL/chat/completions" \
    -H "Content-Type: application/json" \
    -d "$(jq -n \
      --arg model "$MODEL" \
      --arg prompt "$prompt" \
      --argjson tools "$tools" \
      --argjson temp "$temp" \
      --argjson max "$max_tokens" \
      '{model:$model, messages:[{role:"user",content:$prompt}], tools:$tools, temperature:$temp, max_tokens:$max}'
    )" 2>/dev/null
}

get_content() {
  local c
  c=$(echo "$1" | jq -r '.choices[0].message.content // ""')
  if [ -z "$c" ]; then
    local reasoning
    reasoning=$(echo "$1" | jq -r '.choices[0].message.reasoning // empty' 2>/dev/null)
    [ -n "$reasoning" ] && c="[THINKING MODE: content empty — reasoning tokens consumed the budget]"
  fi
  echo "$c"
}

run_test() {
  local name="$1"
  local prompt="$2"
  local temp="${3:-0.3}"
  local max_tokens="${4:-2048}"

  TEST_COUNT=$((TEST_COUNT + 1))
  printf "  [%2d] %-50s" "$TEST_COUNT" "$name"

  local start end resp content tokens ptok ctok wall
  start=$(date +%s%N)
  resp=$(chat "$prompt" "$temp" "$max_tokens")
  end=$(date +%s%N)

  content=$(get_content "$resp")
  ptok=$(echo "$resp" | jq -r '.usage.prompt_tokens // 0')
  ctok=$(echo "$resp" | jq -r '.usage.completion_tokens // 0')
  wall=$(( (end - start) / 1000000 ))

  printf "${CYAN}%5dms  %4d tok${NC}\n" "$wall" "$ctok"

  {
    echo "### $TEST_COUNT. $name"
    echo ""
    echo "**Time:** ${wall}ms | **Tokens:** ${ptok}→${ctok}"
    echo ""
    echo "**Prompt:**"
    echo '```'
    printf '%s\n' "$prompt"
    echo '```'
    echo ""
    echo "**Output:**"
    echo ""
    printf '%s\n' "$content"
    echo ""
    echo "**Score:** /5"
    echo ""
    echo "---"
    echo ""
  } >> "$RESULT_FILE"
}

run_tool_test() {
  local name="$1"
  local prompt="$2"
  local tools="$3"
  local temp="${4:-0.1}"
  local max_tokens="${5:-2048}"

  TEST_COUNT=$((TEST_COUNT + 1))
  printf "  [%2d] %-50s" "$TEST_COUNT" "$name"

  local start end resp content tool_calls tokens ptok ctok wall
  start=$(date +%s%N)
  resp=$(chat_with_tools "$prompt" "$tools" "$temp" "$max_tokens")
  end=$(date +%s%N)

  content=$(get_content "$resp")
  tool_calls=$(echo "$resp" | jq '.choices[0].message.tool_calls // empty' 2>/dev/null)
  ptok=$(echo "$resp" | jq -r '.usage.prompt_tokens // 0')
  ctok=$(echo "$resp" | jq -r '.usage.completion_tokens // 0')
  wall=$(( (end - start) / 1000000 ))

  printf "${CYAN}%5dms  %4d tok${NC}\n" "$wall" "$ctok"

  {
    echo "### $TEST_COUNT. $name"
    echo ""
    echo "**Time:** ${wall}ms | **Tokens:** ${ptok}→${ctok}"
    echo ""
    echo "**Prompt:**"
    echo '```'
    printf '%s\n' "$prompt"
    echo '```'
    echo ""
    if [ -n "$tool_calls" ] && [ "$tool_calls" != "null" ]; then
      echo "**Tool Calls:**"
      echo '```json'
      printf '%s\n' "$tool_calls" | jq '.' 2>/dev/null || printf '%s\n' "$tool_calls"
      echo '```'
    else
      echo "**Tool Calls:** none"
    fi
    echo ""
    if [ -n "$content" ]; then
      echo "**Text Response:**"
      echo ""
      printf '%s\n' "$content"
    fi
    echo ""
    echo "**Score:** /5"
    echo ""
    echo "---"
    echo ""
  } >> "$RESULT_FILE"
}

section() {
  echo ""
  echo -e "${YELLOW}── $1 ──${NC}"
  { echo ""; echo "## $1"; echo ""; } >> "$RESULT_FILE"
}

# ── Health Check ────────────────────────────────────────────────────────────

echo ""
echo "========================================"
echo "  Local LLM Benchmark v2"
echo "========================================"
echo "  Endpoint: $BASE_URL"
echo "  Model:    $MODEL"
echo ""

health=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$BASE_URL/models" 2>/dev/null || echo "000")
if [ "$health" != "200" ]; then
  echo "ERROR: Endpoint not reachable (HTTP $health)"
  exit 1
fi

cat > "$RESULT_FILE" <<EOF
# Benchmark: $MODEL

- **Date:** $(date +%Y-%m-%d\ %H:%M)
- **Endpoint:** $BASE_URL

---

EOF

# ========================================================================
# 1. KNOWLEDGE & LEARNING
# ========================================================================
section "Knowledge & Learning"

run_test "Analogical Explanation" \
  'I understand how Git branching works really well. Using that as an analogy, explain how DNS resolution works — mapping the concepts (branches, merges, remotes, etc.) to DNS equivalents. Keep it under 200 words.' \
  0.5 2048

run_test "Tradeoff Comparison" \
  'I am choosing between SQLite and PostgreSQL for a self-hosted app that will have 1-5 users. Give me a comparison table with at least 5 factors, then a one-sentence recommendation for my situation.' \
  0.3 2048

run_test "Concept Breakdown" \
  'Break down how HTTPS/TLS handshake works into 5 simple steps. For each step, give a one-sentence explanation that a backend developer (not a security expert) would understand. Number each step.' \
  0.3 2048

# ========================================================================
# 2. PRACTICAL TEXT TASKS
# ========================================================================
section "Practical Text Tasks"

run_test "Messy Text → Structured JSON" \
  'Parse this messy recipe into clean JSON with fields: name, servings (number), prep_time, ingredients (array of {item, amount}), steps (array of strings). Return ONLY the JSON.

"ok so this is my grandmas famous pasta sauce lol. feeds about 6 people, takes maybe 45 min to prep. you need like 2 cans of crushed tomatoes (28oz each), 1 pound ground beef, half an onion diced, 3 cloves garlic minced, 2 tbsp olive oil, salt and pepper to taste, and a handful of fresh basil. brown the beef first then saute the onion and garlic in olive oil until soft, add the tomatoes and simmer for 30 min, season with salt pepper and basil at the end"' \
  0.3 2048

run_test "Formal Email Draft" \
  'Draft a polite but firm reply to this email from my landlord. I want to push back without being rude. Keep it under 100 words.

"""Dear Tenant, We are increasing your monthly rent by 15% effective next month. This is due to rising maintenance costs. Please confirm your acceptance. Regards, Management"""

My points: the lease requires 60 days written notice for rent changes, and the lease caps annual increases at 5%.' \
  0.5 2048

run_test "Translate, Explain, Advise" \
  'Translate this German text to English, then tell me in one sentence: what kind of document is this, and do I need to do anything about it?

"Sehr geehrte Damen und Herren, hiermit kündigen wir den Vertrag Nr. 4712-B zum nächstmöglichen Zeitpunkt gemäß §14 der Allgemeinen Geschäftsbedingungen. Bitte bestätigen Sie den Eingang dieses Schreibens und teilen Sie uns das genaue Kündigungsdatum mit. Mit freundlichen Grüßen"' \
  0.3 2048

# ========================================================================
# 3. TECHNICAL & HOMELAB
# ========================================================================
section "Technical & Homelab"

run_test "Error Diagnosis" \
  'What is wrong here and how do I fix it? Max 3 sentences.

```
nginx: [emerg] bind() to 0.0.0.0:443 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] still could not bind()
```' \
  0.3 1024

run_test "Docker Compose Generation" \
  'Generate a Docker Compose file for a Postgres 16 database with:
- Database: appdb, User: appuser, password from env var DB_PASSWORD
- Named volume "pgdata" for persistence
- Port 5432 exposed
- A healthcheck using pg_isready
Return ONLY the YAML.' \
  0.3 2048

run_test "Config Review" \
  'Review this Caddyfile for problems. List each issue and how to fix it.

```
:80 {
    reverse_proxy /api* localhost:3000
    reverse_proxy localhost:8080

    tls internal

    header {
        X-Frame-Options "DENY"
        -Server
    }

    log {
        output file /var/log/caddy/access.log
    }
}
```' \
  0.3 2048

# ========================================================================
# 4. EXTRACTION & CLASSIFICATION
# ========================================================================
section "Extraction & Classification"

run_test "Ticket Classification" \
  'Classify each support ticket. Return a JSON array where each object has: id (number), category (bug|feature_request|billing|account|documentation), priority (low|medium|high|critical). Return ONLY valid JSON.

1. "App crashes every time I try to export a PDF. Losing client work."
2. "Would be great if you could add dark mode."
3. "I was charged twice this month, need a refund ASAP."
4. "How do I reset my password? Cannot find the option."
5. "The API docs say POST but the endpoint only accepts GET."
6. "Entire dashboard is down, nothing loads for any of our 200 users."' \
  0.1 2048

run_test "Entity Extraction" \
  'Extract all named entities from this conversation. Return ONLY a JSON object with: people (array), companies (array), dates (array), amounts (array), locations (array).

"Hey Sarah, just got off the call with the Stripe team. They proposed a $50,000 annual contract starting January 15th. Mike from their sales team said they can do onboarding at their San Francisco office or remotely. We need legal review done before the board meeting on February 3rd. Also, Lisa mentioned Datadog offered us a competing deal at $42,000."' \
  0.1 2048

# ========================================================================
# 5. INSTRUCTION FOLLOWING
# ========================================================================
section "Instruction Following"

run_test "Six Constraints" \
  'Write a short product review for wireless earbuds. Follow ALL these rules:
1. Exactly 4 sentences
2. First sentence must start with the word "After"
3. Second sentence must contain a negative point
4. Include a star rating as the last line: "Rating: X/5"
5. Total under 80 words
6. No brand names' \
  0.3 2048

run_test "Exact JSON Schema" \
  'Return a JSON object with EXACTLY these 3 fields, no more, no less:
- "can_do": array of exactly 3 things you can help with
- "cannot_do": array of exactly 3 things you cannot help with
- "version": the exact string "v1"
No explanation, no markdown, no extra fields.' \
  0.1 2048

# ========================================================================
# 6. TOOL CALLING
# ========================================================================
section "Tool Calling"

TOOLS='[
  {
    "type": "function",
    "function": {
      "name": "get_container_status",
      "description": "Get the status of a Docker container by name",
      "parameters": {
        "type": "object",
        "properties": {
          "container_name": {"type": "string", "description": "Name of the Docker container"}
        },
        "required": ["container_name"]
      }
    }
  },
  {
    "type": "function",
    "function": {
      "name": "restart_service",
      "description": "Restart a service on the homelab server",
      "parameters": {
        "type": "object",
        "properties": {
          "service_name": {"type": "string", "description": "Name of the service to restart"},
          "force": {"type": "boolean", "description": "Force restart even if service is healthy"}
        },
        "required": ["service_name"]
      }
    }
  },
  {
    "type": "function",
    "function": {
      "name": "search_logs",
      "description": "Search recent logs for a service",
      "parameters": {
        "type": "object",
        "properties": {
          "service_name": {"type": "string", "description": "Name of the service"},
          "query": {"type": "string", "description": "Search query for log contents"},
          "since": {"type": "string", "description": "Time range, e.g. 1h, 30m, 2d"}
        },
        "required": ["service_name", "query"]
      }
    }
  },
  {
    "type": "function",
    "function": {
      "name": "send_notification",
      "description": "Send a notification message via Telegram",
      "parameters": {
        "type": "object",
        "properties": {
          "message": {"type": "string", "description": "Notification message text"},
          "priority": {"type": "string", "enum": ["low", "normal", "high"], "description": "Message priority"}
        },
        "required": ["message"]
      }
    }
  }
]'

run_tool_test "Simple Tool Call" \
  "Is my Ollama container running?" \
  "$TOOLS"

run_tool_test "Right Tool for the Job" \
  "Immich has been throwing 500 errors for the last hour. Can you check what is going on?" \
  "$TOOLS"

run_tool_test "Multiple Tools in One Request" \
  "Force restart the Caddy container and send me a high priority Telegram alert about the restart." \
  "$TOOLS"

run_tool_test "Should NOT Use a Tool" \
  "What is the difference between Docker volumes and bind mounts?" \
  "$TOOLS"

run_tool_test "Vague Request — Investigate, Don't Act" \
  "Something seems off with Nginx" \
  "$TOOLS"

# ── Scorecard ───────────────────────────────────────────────────────────────

cat >> "$RESULT_FILE" <<'EOF'

---

## Scorecard

| # | Test | Score | Notes |
|---|------|-------|-------|
| 1 | Analogical Explanation | /5 | |
| 2 | Tradeoff Comparison | /5 | |
| 3 | Concept Breakdown | /5 | |
| 4 | Messy Text → Structured JSON | /5 | |
| 5 | Formal Email Draft | /5 | |
| 6 | Translate, Explain, Advise | /5 | |
| 7 | Error Diagnosis | /5 | |
| 8 | Docker Compose Generation | /5 | |
| 9 | Config Review | /5 | |
| 10 | Ticket Classification | /5 | |
| 11 | Entity Extraction | /5 | |
| 12 | Six Constraints | /5 | |
| 13 | Exact JSON Schema | /5 | |
| 14 | Simple Tool Call | /5 | |
| 15 | Right Tool for the Job | /5 | |
| 16 | Multiple Tools in One Request | /5 | |
| 17 | Should NOT Use a Tool | /5 | |
| 18 | Vague Request — Investigate | /5 | |
| | **Total** | **/90** | |

### Scoring Guide

- **5** — Perfect or near-perfect. Output is directly usable.
- **4** — Good. Minor issues but still usable with minimal editing.
- **3** — Okay. Gets the idea right but needs significant cleanup.
- **2** — Poor. Partially correct but mostly unusable.
- **1** — Bad. Wrong approach, misunderstood the task, or format is broken.
- **0** — Failed. No output, empty, or completely wrong.
EOF

echo ""
echo "========================================"
echo "  Done — $TEST_COUNT tests completed"
echo "  Results: $RESULT_FILE"
echo "========================================"
echo ""

{
  "name": "Grandma's Famous Pasta Sauce",
  "servings": 6,
  "prep_time": "45 minutes",
  "ingredients": [
    {"item": "crushed tomatoes", "amount": "2 cans (28oz each)"},
    {"item": "ground beef", "amount": "1 pound"},
    {"item": "onion", "amount": "half, diced"},
    {"item": "garlic", "amount": "3 cloves, minced"},
    {"item": "olive oil", "amount": "2 tbsp"},
    {"item": "salt and pepper", "amount": "to taste"},
    {"item": "fresh basil", "amount": "a handful"}
  ],
  "steps": [
    "Brown the beef.",
    "Saut onion and garlic in olive oil until soft.",
    "Add the tomatoes and simmer for 30 minutes.",
    "Season with salt, pepper, and basil at the end."
  ]
}

version: '3.8'

services:
  postgres:
    image: postgres:16
    container_name: postgres_db
    environment:
      POSTGRES_DB: appdb
      POSTGRES_USER: appuser
      POSTGRES_PASSWORD: ${DB_PASSWORD}
    volumes:
      - pgdata:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U appuser -d appdb"]
      interval: 10s
      timeout: 5s
      retries: 5

volumes:
  pgdata:
    driver: local

[{"function": {"name": "get_container_status", "arguments": "{\"container_name\":\"Ollama\"}"}}]

[
  {"function": {"name": "restart_service", "arguments": "{\"force\":true,\"service_name\":\"Caddy\"}"}},
  {"function": {"name": "send_notification", "arguments": "{\"message\":\"Caddy container was force restarted.\",\"priority\":\"high\"}"}}
]

[
  {"function": {"name": "get_container_status", "arguments": "{\"container_name\":\"immich\"}"}},
  {"function": {"name": "search_logs", "arguments": "{\"service_name\":\"immich\",\"query\":\"500 error\",\"since\":\"1h\"}"}}
]

{
  "name": "Grandmas Famous Pasta Sauce",
  "servings": 6,
  "prep_time": 45,
  "ingredients": [
    {"item": "crushed tomatoes", "amount": "2 cans (28oz each)"},
    {"item": "ground beef", "amount": "1 pound"},
    {"item": "onion", "amount": "half an onion, diced"},
    {"item": "garlic", "amount": "3 cloves, minced"},
    {"item": "olive oil", "amount": "2 tbsp"},
    {"item": "salt", "amount": "to taste"},
    {"item": "pepper", "amount": "to taste"},
    {"item": "fresh basil", "amount": "a handful"}
  ],
  "steps": [
    "brown the beef first",
    "saute the onion and garlic in olive oil until soft",
    "add the tomatoes and simmer for 30 min",
    "season with salt, pepper, and basil at the end"
  ]
}

services:
  appdb:
    image: postgres:16
    environment:
      - DB_PASSWORD=${DB_PASSWORD}  # Wrong: should be POSTGRES_PASSWORD
    healthcheck:
      test: ["pg_isready", "timeout", "10s"]  # Invalid syntax
      command: ["pg_isready", "-U", "appuser", "-d", "appdb"]  # Not a real field

[
  {"function": {"name": "get_container_status", "arguments": "{\"container_name\":\"nginx\"}"}},
  {"function": {"name": "search_logs", "arguments": "{\"service_name\":\"nginx\",\"query\":\"error\",\"since\":\"1h\"}"}}
]