{
 "generated": "2026-10-18T01:57:38+0000",
 "rig": {
  "host": "ai-rig",
  "cpu": "AMD Ryzen 5 5500 (6C/12T, Zen 3)",
  "ram": "32 GB (4 x 8 GB G.SKILL Ripjaws V DDR4-3200, single rank)",
  "gpus": "3 x RTX 5060 Ti 16 GB, PCIe 3.0 x8 / x1 / x1",
  "vram_mib": 48933,
  "driver": "595.84",
  "os": "Ubuntu 24.04, kernel 6.8.0-147"
 },
 "model": {
  "name": "qwen3.8-iq4xs:latest",
  "params": "27.3B dense",
  "quant": "IQ4_XS",
  "size_gb": 15.18
 },
 "arms": [
  {
   "id": "A-1card-2133",
   "configuration": "1 card active (BIOS fault left 2 unbound), 32 GB DDR4-2133",
   "cards": 1,
   "mem_mts": 2133,
   "mem_speed_verified": "read from the kernel log's BAR failure, and from nvidia-smi showing 1 GPU",
   "offload": "partial -- a 26.9 GB footprint cannot fit in 16 GB",
   "source": "baseline1.py",
   "decode_tok_s": {
    "2048": 15.72,
    "16384": 9.92,
    "131072": 4.34
   },
   "note": "decode spread 0.05 tok/s within each depth; this arm is a partial-offload configuration and is NOT a clean single-card bandwidth measurement"
  },
  {
   "id": "B-3card-2133",
   "configuration": "3 cards, 32 GB DDR4-2133",
   "cards": 3,
   "mem_mts": 2133,
   "mem_speed_verified": "set to SPD default; not independently readable on this host",
   "offload": "partial -- ollama reports size_vram 23.34 GB against a 26.9 GB footprint",
   "source": "arm.py",
   "decode_tok_s": {
    "2048": 47.57,
    "16384": 47.32,
    "131072": 47.41
   },
   "prefill_tok_s": {
    "2048": 88.9,
    "16384": 88.8,
    "131072": 89.2
   },
   "concurrency": {
    "1": 5.24,
    "2": 36.9,
    "4": 39.37,
    "8": 39.76
   },
   "pcie": "0, 3, 8 | 1, 3, 1 | 2, 3, 1",
   "nvrm_probe_failures": "0"
  },
  {
   "id": "C-3card-3200",
   "configuration": "3 cards, 32 GB DDR4-3200",
   "cards": 3,
   "mem_mts": 3200,
   "mem_speed_verified": "manually read in BIOS setup on this boot: 3200 MT/s, XMP, 1.377 V readback (board SVI2 telemetry reads ~2% high; the kit is rated 1.35 V)",
   "offload": "partial -- ollama reports size_vram 23.34 GB against a 26.9 GB footprint",
   "source": "arm.py",
   "decode_tok_s": {
    "2048": 47.76,
    "16384": 47.56,
    "131072": 47.62
   },
   "prefill_tok_s": {
    "2048": 90.1,
    "16384": 90.8,
    "131072": 90.3
   },
   "concurrency": {
    "1": 5.35,
    "2": 37.14,
    "4": 39.71,
    "8": 40.09
   },
   "pcie": "0, 3, 8 | 1, 3, 1 | 2, 3, 1",
   "nvrm_probe_failures": "0"
  },
  {
   "id": "B2-context-use",
   "configuration": "3 cards, 32 GB DDR4-2133",
   "cards": 3,
   "mem_mts": 2133,
   "source": "armB2-context-use.py",
   "note": "prompt lengths are approximate; the tokeniser counts are recorded per depth",
   "working_depths": [
    {
     "approx": "8k",
     "prompt_tokens": 5720,
     "cold_prefill_s": 7.84,
     "warm_prefill_s": 0.48,
     "warm_gain_x": 16.3,
     "decode_tok_s": 44.55,
     "vram_mib": 18106
    },
    {
     "approx": "32k",
     "prompt_tokens": 22760,
     "cold_prefill_s": 32.39,
     "warm_prefill_s": 0.72,
     "warm_gain_x": 45.0,
     "decode_tok_s": 43.15,
     "vram_mib": 23024
    },
    {
     "approx": "64k",
     "prompt_tokens": 45470,
     "cold_prefill_s": 73.07,
     "warm_prefill_s": 1.02,
     "warm_gain_x": 71.6,
     "decode_tok_s": 38.21,
     "vram_mib": 27154
    },
    {
     "approx": "128k",
     "prompt_tokens": 90920,
     "cold_prefill_s": 167.67,
     "warm_prefill_s": 1.61,
     "warm_gain_x": 104.1,
     "decode_tok_s": 32.58,
     "vram_mib": 27026
    }
   ]
  },
  {
   "id": "C-3card-3200-working-set",
   "configuration": "3 cards, 32 GB DDR4-3200, XMP verified",
   "cards": 3,
   "mem_mts": 3200,
   "source": "armC-workingset.py",
   "cold": {
    "prompt_tokens": 72884,
    "prefill_s": 125.06,
    "prefill_tok_s": 582.8,
    "ttft_s": 145.581,
    "decode_tok_s": 31.41,
    "wall_s": 146.88,
    "vram_per_card_mib": [
     8462,
     8380,
     10098
    ],
    "vram_total_mib": 26940
   },
   "warm": {
    "prompt_tokens": 72884,
    "prefill_s": 1.36,
    "prefill_tok_s": 53777.3,
    "ttft_s": 1.88,
    "decode_tok_s": 32.25,
    "wall_s": 3.15,
    "vram_per_card_mib": [
     8462,
     8380,
     10098
    ],
    "vram_total_mib": 26940
   }
  },
  {
   "id": "D-128gb-resident-cu13",
   "configuration": "Strata 0.1.42 CUDA 13 build, IQ2_XS, --resident-experts, --kv-resident 32768, GPU 2 (x8 Gen4), 128 GB DDR4-3600, 5900XT",
   "cards": 1,
   "mem_mts": 3600,
   "mem_speed_verified": "read from firmware SMBIOS type-17: configured 3600 MT/s, 4 x 32 GB",
   "offload": "experts resident in RAM (28.61 GiB page-locked); files 0 blobs",
   "source": "armD-depth-sweep.py + armD-correct.py",
   "decode_tok_s": {
    "2k": 80.86,
    "16k": 78.46,
    "64k": 70.35,
    "128k": 78.91
   },
   "prefill_tok_s": {
    "2k": 581.5,
    "16k": 1222.9,
    "64k": 1514.8,
    "128k": 2473.0
   },
   "prefill_s": {
    "2k": 2.21,
    "16k": 8.0,
    "64k": 25.7,
    "128k": 31.45
   },
   "stability": {
    "succeeded": 10,
    "attempted": 10
   },
   "note": "Streaming omits usage in this build, so decode is completion_tokens / warm total and prefill is cold total minus its decode share. Two-point timing on the same prompt fails because the second call hits the prefix cache."
  }
 ],
 "findings": [
  {
   "id": "armD-resident",
   "claim": "On 128 GB with the experts resident, IQ2_XS decodes at 70.3-80.9 tok/s (mean 77.1), flat from 2k to 128k context, and 10 of 10 sequential requests succeed.",
   "evidence": "2k 80.86, 16k 78.46, 64k 70.35, 128k 78.91 tok/s; prefill 582-2473 tok/s; expert cache 92.4-93.6%; files 0 blobs.",
   "confidence": "high",
   "caveat": "One card (GPU 2), one model, one quantisation. Decode includes reasoning tokens: the engine always thinks first and reasoning_budget_tokens does not suppress it."
  },
  {
   "id": "toolkit-fault",
   "claim": "The illegal-memory-access fault that blocked every prompt above ~200 tokens was the CUDA toolkit, not the memory tier, VRAM headroom, or any engine flag.",
   "evidence": "A CUDA 12.8 sm_120 build failed 400/1000/2000-token prompts with 'illegal memory access'. A CUDA 13 build of the same source passed all three with zero faults. Same pack, model, card and configuration.",
   "confidence": "high",
   "caveat": "The CUDA 13 build was already present on disk from an earlier session and had never been installed because its runtime libraries were missing from the host. Nine configuration experiments were spent before the build was questioned."
  },
  {
   "id": "card-count-at-shallow-depth",
   "claim": "Three cards give 3.03x the decode of one card at 2,048 context.",
   "evidence": "15.72 -> 47.57 tok/s, same model, prompt, sampling, iteration count.",
   "confidence": "high",
   "caveat": "Both arms are partial-offload, so this is confounded by offload amount as well as card count. It matches the layer-split prediction of 3x, but does not isolate it."
  },
  {
   "id": "card-count-at-depth",
   "claim": "At 131,072 context three cards give 10.92x one card (47.41 vs 4.34 tok/s).",
   "evidence": "Same harness, same depths.",
   "confidence": "high for the measurement, low for the mechanism",
   "caveat": "This is NOT a bandwidth ratio. At depth the one-card arm is limited by residency, not bandwidth, so the arms differ in more than card count."
  },
  {
   "id": "depth-immunity",
   "claim": "With three cards, decode is flat across a 64x range of context depth: 47.57 / 47.32 / 47.41 tok/s, a spread of 0.53 %.",
   "confidence": "high",
   "caveat": "The one-card arm falls 72 % over the same range, so depth-immunity is a property of having enough VRAM, not of the model."
  },
  {
   "id": "memory-speed-irrelevant-at-depth",
   "claim": "At a 90k-token context the same comparison stays inside noise: prefill +7.5 %, decode -3.6 %, VRAM -0.3 %.",
   "confidence": "medium",
   "caveat": "The two runs sent different token counts (90,952 vs 72,924) because the prompt builders used different filler text, so absolute seconds are not comparable. Only throughput is."
  },
  {
   "id": "prefix-cache-scaling",
   "claim": "Re-paying prefill for an identical prefix costs 16x-104x less than the cold pass, and the factor grows with depth.",
   "evidence": "8k: 7.8s -> 0.5s; 32k: 32.4 -> 0.7; 64k: 73.1 -> 1.0; 128k: 167.7 -> 1.6.",
   "confidence": "high",
   "caveat": "Measured on uninformative filler text, which may cache more favourably than real source code."
  },
  {
   "id": "prefill-rate",
   "claim": "Prefill runs 542-730 tok/s depending on depth, against decode at 32-48 tok/s.",
   "confidence": "high",
   "caveat": "An earlier figure of 89 tok/s was an artifact of a 34-token prompt, where fixed per-request overhead dominated. That was an 8x error and is recorded in Appendix E."
  },
  {
   "id": "working-set-usability",
   "claim": "For coding or long conversation: ingesting a 90k-token repository costs ~2.1 minutes once, after which every follow-up turn costs ~1.9 s to first token at ~32 tok/s.",
   "confidence": "medium-high",
   "caveat": "Filler-text prompt, not real code. A real repository has different token statistics and may cache differently."
  },
  {
   "id": "memory-speed-irrelevant-when-resident",
   "claim": "Raising DRAM from 2133 to 3200 MT/s changed three-card decode by less than the noise floor.",
   "evidence": "47.57 -> 47.76 (+0.40 %), 47.32 -> 47.56 (+0.51 %), 47.41 -> 47.62 (+0.44 %); spread 1.1-1.2 tok/s.",
   "confidence": "high",
   "caveat": "Measured with the model VRAM-resident, where system RAM is not in the decode path. Does not contradict the +50 % XMP result recorded earlier, which was measured where the working set did not fit in VRAM."
  }
 ]
}