{ "schema_version": "1.0", "as_of": "2026-10-28T22:20:00-05:00", "href": "https://amdai.net/lab-log", "note": "Measured and cited lab-log rows. Numbers keep their original dates. Quants below Q4 are omitted.", "runs": [ { "id": "2026-10-28-cpu-cooler-for-dual-r9700", "date": "2026-10-28T22:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "CPU cooler for an R9700 build: Ryzen 9000X has none in the box, and a 360 AIO in the front or bottom of the case feeds warm air to the cards", "summary": "AMD ships the Ryzen 5 9600X, Ryzen 9 9900X and Ryzen 9 9950X without a cooler; AMD recommends a premium air cooler for the 65 W 9600X and a liquid cooler for the 120 W 9900X and 170 W 9950X for best boost. for a local-AI box with blower R9700s, where the radiator goes matters: the cards pull their air from the case's front and bottom intake, so a 360 radiator should go on top as exhaust, and a case that only takes one in front or bottom is better paired with a large dual-tower air cooler.", "setup": { "stack": "Spec review: amd.com product pages for 9600X/9900X/9950X; Fractal, Phanteks, Thermalright, Noctua and ARCTIC product pages; published llama.cpp MoE offload guides" }, "metrics": [ { "label": "AMD default TDP, 9600X / 9900X / 9950X", "value": "65 / 120 / 170", "unit": "W" }, { "label": "Air cooler height limit, Torrent / Meshify 2 / Enthoo Pro 2 Server", "value": "188 / 185 / 195", "unit": "mm" }, { "label": "Cooler height, Peerless Assassin 120 SE / NH-D15 G2", "value": "155 / 168", "unit": "mm" } ], "sources": [ "https://www.amd.com/en/products/processors/desktops/ryzen/9000-series/amd-ryzen-5-9600x.html", "https://www.amd.com/en/products/processors/desktops/ryzen/9000-series/amd-ryzen-9-9900x.html", "https://www.amd.com/en/products/processors/desktops/ryzen/9000-series/amd-ryzen-9-9950x.html", "https://www.fractal-design.com/products/cases/torrent/torrent/black-solid/", "https://www.fractal-design.com/products/cases/meshify/meshify-2/black-tg-dark-tint/", "https://phanteks.com/product/enthoo-pro-2-server-edition-tg/", "https://www.thermalright.com/product/peerless-assassin-120-se/", "https://noctua.at/en/nh-d15-g2/specification", "https://www.arctic.de/en/Liquid-Freezer-III-Pro-360/ACFRE00180A", "https://gist.github.com/DocShotgun/a02a4c0c0a57e43ff4f038b46ca66ae0" ], "href": "https://amdai.net/lab-log#2026-10-28-cpu-cooler-for-dual-r9700" }, { "id": "2026-10-28-case-for-two-r9700s", "date": "2026-10-28T21:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "Case for two R9700s: a standard 7-slot ATX case is enough on 4-slot-spacing x8/x8 boards, and case fans should follow the GPUs, not the CPU", "summary": "Every R9700 board-partner card is a 2-slot blower no longer than 271 mm. on the ASRock X870 Taichi Creator and Gigabyte B850 AI TOP, owners report 4-slot spacing between the two CPU x8/x8 slots, so the bottom card ends by expansion slot 7 and leaves a 2-slot gap above it. that means any 7-slot ATX case with at least 271 mm of card length fits two cards; no 8-slot case is needed unless you plan a third card or a riser. blowers push their heat out the back, so the case's job is fresh intake air for the card area.", "setup": { "stack": "Spec review: R9700 board-partner spec sheets, ASRock/Gigabyte board pages and owner reports, Fractal and Phanteks case pages, one published 4x R9700 build guide" }, "metrics": [ { "label": "Longest R9700 board-partner card", "value": "271", "unit": "mm" }, { "label": "Slot spacing, X870 Taichi Creator / B850 AI TOP (owner reports)", "value": "4", "unit": "slots" }, { "label": "Card-area temp, CPU-following vs full-speed case fans (4x R9700, published)", "value": "67 vs 57", "unit": "C" }, { "label": "GPU junction change from full-speed case fans (published)", "value": "1-4", "unit": "C" } ], "sources": [ "https://forum.level1techs.com/t/asrock-x870-taichi-creator-dual-gpu/243897", "https://pg.asrock.com/general/news.us.asp?id=5643", "https://www.reddit.com/r/LocalLLaMA/comments/1nqi0di/are_there_any_consumer_mobos_that_can_fit_2x/", "https://nicholasreese.com/guides/quad-r9700/quad-r9700-build-guide.md", "https://www.reddit.com/r/LocalLLM/comments/1vyaesz/blower_r9700_cards_stacked_with_very_little_gap/", "https://forum.level1techs.com/t/radeon-pro-ai-r9700-fan-noise-which-model-to-buy/247552?page=2", "https://www.fractal-design.com/products/cases/torrent/torrent/black-solid/", "https://www.fractal-design.com/products/cases/meshify/meshify-2/black-tg-dark-tint/", "https://phanteks.com/product/enthoo-pro-2-server-edition-tg/" ], "href": "https://amdai.net/lab-log#2026-10-28-case-for-two-r9700s" }, { "id": "2026-10-28-nvme-for-local-models", "date": "2026-10-28T20:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "How much NVMe a Radeon local-AI box needs: 2 TB holds every Q4-Q6 quant of three model families, and Gen5 barely speeds up loading", "summary": "Every Q4 to Q6 quant of Qwen3.6-35B-A3B (20.6 to 29.7 GiB), Qwen3.8-Flash-Next (103.7 to 157.6 GiB) and GLM-5.3-Flash (186.0 to 271.8 GiB) adds up to about 1,587 GiB once you keep one extra file of room for the next download and 150 GiB for the OS. That fits a 2 TB drive (1,863 GiB) with 10% left free; a fourth Flash-class family such as MiniMax-M3.1-Flash (UD-Q4 232 GiB) needs 4 TB. On llama.cpp's default memory-mapped loader, loading is usually limited by page faults and one CPU core rather than the drive, so a Gen5 SSD helps less than its doubled rating suggests.", "setup": { "stack": "Spec and file-size review: Samsung 990 PRO 2/4 TB, WD_BLACK SN850X 4/8 TB, Samsung 9100 PRO 4/8 TB; unsloth GGUF repos" }, "metrics": [ { "label": "All Q4-Q6 quants of 3 families + headroom", "value": "1587", "unit": "GiB" }, { "label": "Usable space on a 2 TB drive", "value": "1863", "unit": "GiB" }, { "label": "Gen4 vs Gen5 cold load, 40.5 GB Q4_K_M (third-party)", "value": "15.2 vs 13.8", "unit": "s" }, { "label": "GPT-OSS-120B load, mmap vs direct I/O (llama.cpp PR)", "value": "110 vs 10.5", "unit": "s" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF", "https://github.com/ggml-org/llama.cpp/pull/18012", "https://github.com/ggml-org/llama.cpp/discussions/12223", "https://craftrigs.com/guides/nvme-speed-llm-load-times-benchmarks/", "https://download.semiconductor.samsung.com/resources/data-sheet/samsung_nvme_ssd_990_pro_datasheet_rev.2.0.pdf", "https://download.semiconductor.samsung.com/resources/data-sheet/Samsung_NVMe_SSD_9100_PRO_Datasheet_Rev.2.0.pdf", "https://documents.sandisk.com/content/dam/asset-library/en_us/assets/public/sandisk/product/internal-drives/wd-black-ssd/data-sheet-wd-black-sn850x-nvme-ssd.pdf" ], "href": "https://amdai.net/lab-log#2026-10-28-nvme-for-local-models" }, { "id": "2026-10-27-sglang-tp2-vs-llamacpp-r9700", "date": "2026-10-27T15:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "SGLang TP2 on 2x R9700 is slower than llama.cpp on one card for Qwen3.6-35B-A3B, and has no DDR5 path for Flash models", "summary": "The most complete public 2x R9700 SGLang rig (v0.5.15 plus RDNA4 patches, TP=2) measured Qwen3.6-35B-A3B AWQ int4 at 67.0 tok/s short and 22.0 tok/s at 197K input. llama.cpp on a single R9700 does 165.2 tok/s (Vulkan, Q4_K_M, tg128) and our own Q5 run does 100.8 short and about 50 at 256K. Upstream SGLang still blocks gfx1201 builds, and its only CPU-expert path (KTransformers) is built around Intel AMX, so Flash-class models stay on llama.cpp or Strata.", "setup": { "model": "Qwen3.6-35B-A3B; Qwen3.8-27B (dense)", "quant": "AWQ int4 / FP8 (SGLang); Q4_K_M and UD-Q5_K_XL (llama.cpp)", "engine": "SGLang (patched RDNA4 fork) vs llama.cpp", "engine_version": "SGLang v0.5.15-v0.5.22; llama.cpp PR #25051 build", "gpus": "1-2x Radeon AI PRO R9700", "context": "short to 197K input", "flags": "SGLang TP=2, Triton attention, HIP graphs; llama.cpp Vulkan 1 GPU / layer / tensor" }, "metrics": [ { "label": "SGLang TP2 35B-A3B AWQ, short", "value": 67, "unit": "tok/s" }, { "label": "SGLang TP2 35B-A3B AWQ, 197K input", "value": 22, "unit": "tok/s" }, { "label": "llama.cpp Vulkan 1 card 35B-A3B Q4_K_M tg128", "value": 165.2, "unit": "tok/s" }, { "label": "llama.cpp 1 card 35B-A3B UD-Q5_K_XL, short", "value": 100.8, "unit": "tok/s" }, { "label": "SGLang TP2 27B FP8 + speculator, 197K input", "value": 30.1, "unit": "tok/s" } ], "sources": [ "https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference", "https://github.com/mattbucci/2x-R9700-RDNA4-GFX1201-sglang-inference/blob/main/benchmarks/qwen3.6-35b-moe-awq/results.json", "https://github.com/ggml-org/llama.cpp/pull/25051#issuecomment-6382846940", "https://github.com/sgl-project/sglang/issues/30599", "https://rocm.docs.amd.com/projects/ai-ecosystem/en/latest/inference/sglang.html", "https://www.lmsys.org/blog/2025-10-22-KTransformers/" ], "href": "https://amdai.net/lab-log#2026-10-27-sglang-tp2-vs-llamacpp-r9700" }, { "id": "2026-10-27-rocm-smi-gone-in-rocm-10", "date": "2026-10-27T14:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "rocm-smi is gone in ROCm 10.x: use amd-smi --rocm-smi, or the driver's own counters on Vulkan-only setups", "summary": "AMD's ROCm 10.1 transition guide lists ROCm SMI as removed from the Core SDK (replaced by AMD SMI, package amdrocm-amdsmi); it was dropped from TheRock builds in 10.0. On a fresh 10.x install, 'rocm-smi' prints command not found even though ROCm works. 'amd-smi --rocm-smi' prints the familiar concise table, and 'amd-smi monitor --gfx --vram-usage --watch 1' replaces 'watch -n1 rocm-smi'. Our own scan of amdai.net found 23 command lines across 15 pages still calling rocm-smi with no fallback, including the verify tutorial's pass/fail checklist; we are fixing those.", "setup": { "stack": "ROCm Core SDK 10.1.0 (released 2026-10-05) documentation", "site scan": "706 sitemap pages, 2026-10-27" }, "metrics": [ { "label": "Site pages mentioning rocm-smi", "value": 27, "unit": "pages" }, { "label": "Command lines calling rocm-smi with no amd-smi fallback", "value": 23, "unit": "lines" } ], "sources": [ "https://rocmdocs.amd.com/en/latest/about/transition-guide-TheRock.html", "https://rocm.docs.amd.com/projects/rocm_smi_lib/en/docs-10.1.0/index.html", "https://rocm.docs.amd.com/projects/amdsmi/en/docs-10.1.0/how-to/amdsmi-cli-tool.html", "https://docs.kernel.org/gpu/amdgpu/driver-misc.html", "https://docs.kernel.org/gpu/amdgpu/thermal.html" ], "href": "https://amdai.net/lab-log#2026-10-27-rocm-smi-gone-in-rocm-10" }, { "id": "2026-10-27-kv-estimate-audit-llamacpp", "date": "2026-10-27T13:30:00-05:00", "project": "idea-lab", "kind": "result", "title": "Our KV-cache calculator vs llama.cpp's real allocator: GLM-5.3-Flash overstated 2.3x, Qwen3.8-Flash-Next 1.6 GiB short at 262K", "summary": "We checked the KV estimate for all 19 model presets in the amdai.net calculator against the model configs and llama.cpp master's memory code (Oct 27). 13 match within 10%. GLM-5.3-Flash was the big miss: only 11 of its 45 layers keep a growing cache (34 are linear-attention layers with a fixed state), so 262K context needs about 4.95 GiB at f16, not the 11.25 GiB we showed. Qwen3.8-Flash-Next needs about 7.6 GiB at 262K, not 6.0, because its sparse-attention indexer keys also live in the cache. Gemma 4's sliding-window layers were under-counted by 0.4-0.8 GiB.", "setup": { "engine": "llama.cpp master 19ff817f (2026-10-27), source review", "cache": "f16 K/V, 1 sequence, -ub 512" }, "metrics": [ { "label": "Presets within 10% of llama.cpp", "value": 13, "unit": "of 19" }, { "label": "GLM-5.3-Flash KV at 262K (llama.cpp)", "value": 4.95, "unit": "GiB" }, { "label": "GLM-5.3-Flash KV at 262K (old site estimate)", "value": 11.25, "unit": "GiB" }, { "label": "Qwen3.8-Flash-Next KV at 262K (llama.cpp)", "value": 7.61, "unit": "GiB" }, { "label": "Qwen3.8-Flash-Next KV at 262K (old site estimate)", "value": 6, "unit": "GiB" } ], "sources": [ "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/config.json", "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/main/config.json", "https://huggingface.co/google/gemma-4-31b/blob/main/config.json", "https://github.com/ggml-org/llama.cpp/blob/19ff817fb25e9a1b838b1d7f9f4325f7bf9779c4/src/llama-model.cpp", "https://github.com/ggml-org/llama.cpp/blob/19ff817fb25e9a1b838b1d7f9f4325f7bf9779c4/src/models/glm5-next.cpp", "https://github.com/ggml-org/llama.cpp/blob/19ff817fb25e9a1b838b1d7f9f4325f7bf9779c4/src/llama-kv-cache-iswa.cpp", "https://amdai.net/calculator/" ], "href": "https://amdai.net/lab-log#2026-10-27-kv-estimate-audit-llamacpp" }, { "id": "2026-10-27-r9700-street-price-noon", "date": "2026-10-27T12:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 street price is now about $2,000: cheapest new card $1,949.99, 50% over MSRP and $250 up in a day", "summary": "As of Oct 27 at noon CT, every new in-stock Radeon AI PRO R9700 32 GB we found sold by Micro Center costs $1,949.99 to $1,999.99, and Newegg and B&H list $1,999.99 (ASUS Turbo). That is up $250-300 from yesterday's $1,699.99-$1,799.99. The cheapest is the Gigabyte AI TOP at $1,949.99 (Micro Center, in-store pickup only), 50% above AMD's $1,299 MSRP. Two cards (64 GB VRAM) now cost $3,899.98 before tax.", "setup": { "gpus": "Radeon AI PRO R9700 32 GB (6 board-partner models)", "region": "US, new condition" }, "metrics": [ { "label": "Cheapest new R9700 in stock", "value": 1949.99, "unit": "USD" }, { "label": "Premium over $1,299 MSRP", "value": 50.1, "unit": "%" }, { "label": "Change since Oct 26 snapshot", "value": 250, "unit": "USD" }, { "label": "Newegg (ASUS Turbo, sold by Newegg)", "value": 1999.99, "unit": "USD" }, { "label": "Two cards today, before tax", "value": 3899.98, "unit": "USD" }, { "label": "Lowest price since Sep 29", "value": 1599.99, "unit": "USD" } ], "sources": [ "https://www.microcenter.com/product/699521/gigabyte-amd-radeon-ai-pro-r9700-top-single-fan-ai-workstation-graphics-card", "https://www.newegg.com/asus-turbo-ai-pro-r9700-32g-32gb-graphics-card/p/N82E16814126826", "https://www.newegg.com/p/pl?d=radeon+ai+pro+r9700", "https://www.bhphotovideo.com/c/search?q=radeon%20ai%20pro%20r9700", "https://www.amazon.com/dp/B0G75W64R3", "https://vga.live/gpus/radeon-ai-pro-r9700", "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro.html" ], "href": "https://amdai.net/lab-log#2026-10-27-r9700-street-price-noon" }, { "id": "2026-10-27-llama-cpp-removed-load-flags", "date": "2026-10-27T09:40:00-05:00", "project": "idea-lab", "kind": "result", "title": "llama.cpp removed --no-mmap, --mlock and llama-bench -mmp: old commands now stop at startup, and LLAMA_ARG_NO_MMAP is silently ignored", "summary": "Since release b10875 (2026-09-09, PR #28334) llama-server and llama-cli exit with 'invalid argument' on --no-mmap, --mmap, --mlock and --direct-io, and llama-bench rejects -mmp and -dio. Use --load-mode none (old --no-mmap), mmap+mlock (old --mlock), mlock (old --no-mmap --mlock) or dio; in llama-bench, -lm none. The env vars LLAMA_ARG_NO_MMAP, LLAMA_ARG_MMAP and LLAMA_ARG_MLOCK are no longer read at all, so Docker or systemd setups that used them quietly fall back to the default load mode (auto, which memory-maps).", "setup": { "engine": "llama.cpp", "engine_version": "master 27b7a8a009de (2026-10-27); removal in b10875", "method": "source audit of common/arg.cpp and tools/llama-bench/llama-bench.cpp" }, "metrics": [ { "label": "First release without --no-mmap / --mlock / -mmp", "value": "b10875", "unit": "" }, { "label": "llama.cpp option entries, 2025-04-27", "value": 237, "unit": "" }, { "label": "llama.cpp option entries, 2026-10-27", "value": 361, "unit": "" }, { "label": "amdai.net command lines with a removed or unknown flag", "value": "0 of 130", "unit": "" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/28334", "https://github.com/ggml-org/llama.cpp/commit/14a9d09f75", "https://github.com/ggml-org/llama.cpp/pull/20834", "https://github.com/ggml-org/llama.cpp/pull/26135", "https://github.com/ggml-org/llama.cpp/pull/22397", "https://github.com/ggml-org/llama.cpp/pull/22964", "https://github.com/ggml-org/llama.cpp/releases/tag/b10875", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md" ], "href": "https://amdai.net/lab-log#2026-10-27-llama-cpp-removed-load-flags" }, { "id": "2026-10-27-vllm-int4-r9700-gating", "date": "2026-10-27T08:15:00-05:00", "project": "dual-r9700", "kind": "result", "title": "vLLM 4-bit on the R9700: recent RDNA int4 speedups skip gfx1201, GPTQ crashes, and the default attention backend can return wrong output", "summary": "Reading vLLM main (e7fa26d5c8, 2026-10-27): the Oct 13 W4A16 decode tuning (+9-21% GEMM, up to -66% on speculative shapes) is gated to Strix Halo (gfx115x); the new int4 MoE decode kernel (+44% on a 35B-A3B, Strix Halo) and MXFP4 W4A8 GEMV are gfx11-only; upstream has no tuned int4 MoE configs for the R9700. On R9700, GPTQ checkpoints crash at startup unless you pass --linear-backend TRITON, and the default ROCM_ATTN backend gave wrong outputs or GPU faults outside head 128 / KV block 16, while TRITON_ATTN was clean.", "setup": { "engine": "vLLM (source reading)", "engine_version": "main e7fa26d5c8 (2026-10-27)", "gpus": "Radeon AI PRO R9700 (gfx1201)", "quant": "W4A16 / GPTQ / AWQ / MXFP4 / GGUF (4-bit class)" }, "metrics": [ { "label": "Int4 MoE decode kernel, 35B-A3B (Strix Halo, gfx11-only)", "value": 44, "unit": "% faster" }, { "label": "R9700 tuned int4 MoE config, kernel time at batch 1 (open PR)", "value": -19, "unit": "%" }, { "label": "ROCM_ATTN wrong outputs on R9700 (of 12,389 batches)", "value": 635, "unit": "batches" }, { "label": "ROCM_ATTN GPU faults on R9700 (of 12,389 batches)", "value": 119, "unit": "batches" }, { "label": "R9700 int4_w4a16 MoE configs in vLLM main", "value": 0, "unit": "files" } ], "sources": [ "https://github.com/vllm-project/vllm/pull/57199", "https://github.com/vllm-project/vllm/pull/62480", "https://github.com/vllm-project/vllm/pull/60413", "https://github.com/vllm-project/vllm/pull/62708", "https://github.com/vllm-project/vllm/pull/64045", "https://github.com/vllm-project/vllm/issues/63597", "https://github.com/vllm-project/vllm/pull/52637", "https://github.com/vllm-project/vllm/issues/62697", "https://github.com/vllm-project/vllm/pull/64055", "https://github.com/vllm-project/vllm/pull/39612", "https://github.com/vllm-project/vllm-gguf-plugin/issues/142", "https://github.com/ggml-org/llama.cpp/blob/master/ggml/src/ggml-cuda/common.cuh" ], "href": "https://amdai.net/lab-log#2026-10-27-vllm-int4-r9700-gating" }, { "id": "2026-10-27-llama-server-metrics-gauges", "date": "2026-10-27T07:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Monitoring local LLM speed: llama-server's tok/s gauges read 0 mid-generation; use the counters, and compare by context depth", "summary": "In current llama.cpp, the /metrics gauges predicted_tokens_seconds and prompt_tokens_seconds are reset by every scrape and only updated when a request finishes, so dashboards show 0 while a long generation runs and two scrapers steal each other's windows. Rates from the cumulative counters are reliable. Strata's /metrics already reports a live 2-second tok/s plus per-request speeds.", "setup": { "engine": "llama.cpp llama-server / Strata (source read)", "engine_version": "llama.cpp 0c8c59ef3d; Strata main 2026-10-25", "gpus": "2x Radeon AI PRO R9700 (target)" }, "metrics": [ { "label": "predicted_tokens_seconds while a request is generating", "value": 0, "unit": "tok/s" }, { "label": "Counter-derived speed bias on a 20-token reply", "value": 5, "unit": "%" }, { "label": "Strata live rate window", "value": 2, "unit": "s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/0c8c59ef3d/tools/server/server-common.h", "https://github.com/ggml-org/llama.cpp/blob/0c8c59ef3d/tools/server/server-context.cpp", "https://github.com/ggml-org/llama.cpp/blob/0c8c59ef3d/tools/server/server-task.cpp", "https://github.com/ggml-org/llama.cpp/issues/27436", "https://github.com/ggml-org/llama.cpp/pull/27760", "https://github.com/ggml-org/llama.cpp/pull/30543", "https://github.com/Niko1221/Strata/blob/main/serve/server.py", "https://docs.kernel.org/gpu/amdgpu/thermal.html" ], "href": "https://amdai.net/lab-log#2026-10-27-llama-server-metrics-gauges" }, { "id": "2026-10-27-vram-per-dollar", "date": "2026-10-27T06:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "VRAM per dollar: a new R9700 is now at parity with a used RTX 3090, and wins on watts and slots", "summary": "With the cheapest new R9700 at $1,949.99 (up $250 since yesterday) it costs $60.9 per GB of VRAM, against $58.9-63.7 per GB for a used RTX 3090 at eBay sold medians of $1,413-$1,528. Per watt it wins clearly (9.4 W/GB vs 14.6), and 64 GB takes two R9700s at 600 W instead of three 3090s at 1,050 W.", "setup": null, "metrics": [ { "label": "R9700 cheapest new in stock", "value": 1949.99, "unit": "USD" }, { "label": "Used RTX 3090 eBay sold median (gpusold, 334 sales)", "value": 1528, "unit": "USD" }, { "label": "R9700 cost per GB VRAM", "value": 60.9, "unit": "USD/GB" }, { "label": "Used RTX 3090 cost per GB VRAM", "value": 63.7, "unit": "USD/GB" }, { "label": "R9700 board power per GB", "value": 9.4, "unit": "W/GB" }, { "label": "RTX 3090 board power per GB", "value": 14.6, "unit": "W/GB" }, { "label": "R9700 price for per-GB parity with a $1,528 3090", "value": 2037, "unit": "USD" } ], "sources": [ "https://gpusold.com/", "https://compsniper.com/worth/rtx-3090", "https://www.pcprice.watch/search/RTX3090/US/GPU", "https://vga.live/gpus/radeon-ai-pro-r9700", "https://vga.live/cards/gigabyte-amd-radeon-ai-pro-r9700-top-single-fan-ai-workstation", "https://rtx.live/gpus/geforce-rtx-5090", "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro.html" ], "href": "https://amdai.net/lab-log#2026-10-27-vram-per-dollar" }, { "id": "2026-10-27-gfx1201-upstream-bugs", "date": "2026-10-27T04:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 upstream bug check: 15 RDNA4 fixes landed in 30 days, but the ROCm llama.cpp hang that needs a reboot still fails on the newest driver stack", "summary": "We searched 10 GitHub repos Radeon local-AI users rely on for issues naming gfx1201 or R9700. 15 RDNA4-titled issues were closed as fixed in the last 30 days, including two Tensile GEMM page faults and a ~12% Vulkan prefill regression in llama.cpp. Still open: a ROCm-only llama.cpp generation hang that resets the GPU (Vulkan passes), a cross-stream wait penalty whose size depends on MES firmware, a GFXOFF-related idle power cost, a vLLM TP=2 deadlock, and a vLLM 0.28 FP8 kernel choice that slows RDNA4 decode.", "setup": null, "metrics": [ { "label": "RDNA4-titled issues closed as fixed, last 30 days", "value": 15, "unit": "issues" }, { "label": "Open issues mentioning gfx1201/R9700 in ROCm/TheRock", "value": 104, "unit": "issues" }, { "label": "Open in llama.cpp", "value": 35, "unit": "issues" }, { "label": "Open in vLLM", "value": 40, "unit": "issues" }, { "label": "Cross-stream wait penalty at MES 0x93 (reported)", "value": 9, "unit": "us per small kernel" } ], "sources": [ "https://github.com/ROCm/TheRock/issues/9279", "https://github.com/ROCm/TheRock/issues/9297", "https://github.com/ROCm/TheRock/issues/9245", "https://github.com/ROCm/TheRock/issues/9228", "https://github.com/vllm-project/vllm/issues/40980", "https://github.com/vllm-project/vllm/issues/57838", "https://github.com/ggml-org/llama.cpp/issues/28093", "https://github.com/ggml-org/llama.cpp/issues/29892", "https://github.com/ROCm/rocm-libraries/issues/12346", "https://github.com/ROCm/rocm-libraries/issues/7992" ], "href": "https://amdai.net/lab-log#2026-10-27-gfx1201-upstream-bugs" }, { "id": "2026-10-27-new-model-licenses", "date": "2026-10-27T03:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Read the LICENSE, not the tag: 1 in 4 new open-weight models had no usable license label, and Qwen3.8-Flash-Next's terms are stricter than its 2.4T sibling's", "summary": "Of 88 open-weight models released in the last 90 days, 20 were tagged only 'other' or nothing on Hugging Face. After reading each LICENSE: 62 open, 7 open with conditions, 5 commercial only under a revenue cap, 9 non-commercial or research only, 1 not for EU use, 4 still unclear. Personal and home use is fine for every Flash-class model we run.", "setup": null, "metrics": [ { "label": "Models with license 'other' or none", "value": 20, "unit": "of 88" }, { "label": "Still unclear after reading LICENSE files", "value": 4, "unit": "of 88" }, { "label": "Non-commercial or research only", "value": 9, "unit": "of 88" }, { "label": "Commercial only under a revenue cap", "value": 5, "unit": "of 88" }, { "label": "Permissive-tagged repos with no LICENSE file", "value": 36, "unit": "of 61" } ], "sources": [ "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/main/LICENSE", "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/blob/main/LICENSE", "https://huggingface.co/zai-org/GLM-5.3/blob/main/LICENSE", "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash/blob/main/LICENSE", "https://huggingface.co/LiquidAI/LFM2.5-24B-A2B/blob/main/LICENSE", "https://huggingface.co/tencent/ContextPilot-8B/blob/main/LICENSE", "https://huggingface.co/tencent/Youtu-Parsing-Omni/blob/main/LICENSE" ], "href": "https://amdai.net/lab-log#2026-10-27-new-model-licenses" }, { "id": "2026-10-27-coding-cli-prompt-tax", "date": "2026-10-27T01:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Coding CLIs on a local llama-server: 0.7K to 19K tokens before your first word, and one background call that forces a full re-read", "summary": "We captured what five coding CLIs send to a local endpoint. First-turn prompts: Claude Code 19,384 tokens, Qwen Code 15,091 (plus a 7,084-token memory call every turn), Codex CLI 8,343, OpenCode 7,158, Aider 749. Replayed on a real one-slot llama-server, Qwen Code's memory call evicted the session when the RAM prompt cache was off: turn 2 re-read 15,602 tokens instead of 21.", "setup": { "engine": "llama-server", "engine_version": "b11912 (CPU build)", "model": "Qwen3.5-0.8B", "quant": "Q8_0", "context": "65536", "flags": "-np 1, --reasoning off; arms: default --cache-ram, --cache-ram 0", "clients": "Claude Code 2.1.197, Qwen Code 0.15.10, Codex CLI 0.164.0, OpenCode 1.19.4, Aider 0.86.2" }, "metrics": [ { "label": "Claude Code first-turn prompt", "value": 19384, "unit": "tokens" }, { "label": "Qwen Code first-turn prompt", "value": 15091, "unit": "tokens" }, { "label": "Codex CLI first-turn prompt", "value": 8343, "unit": "tokens" }, { "label": "OpenCode first-turn prompt", "value": 7158, "unit": "tokens" }, { "label": "Aider first-turn prompt", "value": 749, "unit": "tokens" }, { "label": "Qwen Code turn 2 re-read, --cache-ram 0", "value": 15602, "unit": "tokens" }, { "label": "Qwen Code turn 2 re-read, default cache or memory off", "value": 21, "unit": "tokens" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md", "https://github.com/ggml-org/llama.cpp/pull/30159", "https://github.com/ggml-org/llama.cpp/issues/20003", "https://code.claude.com/docs/en/llm-gateway-protocol", "https://opencode.ai/docs/providers/" ], "href": "https://amdai.net/lab-log#2026-10-27-coding-cli-prompt-tax" }, { "id": "2026-10-26-hip-build-flags-rccl", "date": "2026-10-26T23:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp on 2x R9700: build with RCCL off for batch-1 tensor split, and three build flags that no longer do anything", "summary": "Since llama.cpp PR #27825 (merged 2026-09-15), a HIP build with GGML_HIP_RCCL=OFF (the default) uses a built-in two-GPU AllReduce for --split-mode tensor. On 2x R9700 it beat the old fallback by +17.5% prefill and +8.3% decode, and in published RCCL-on vs off tests decode was equal or slightly faster without RCCL while RCCL only helped long prefills.", "setup": { "engine": "llama.cpp (HIP/ROCm)", "engine_version": "master 7aca0d2 (2026-10-26) source read; results from cited testers", "gpus": "2x Radeon AI PRO R9700 (and 2x RX 7900 XTX for the RCCL decode comparison)", "flags": "-sm tensor, -fa on, GGML_HIP_RCCL OFF vs ON" }, "metrics": [ { "label": "Prefill gain, built-in AllReduce vs old fallback (2x R9700, 31B Q6_K)", "value": 17.5, "unit": "%" }, { "label": "Decode gain, built-in AllReduce vs old fallback (2x R9700, 31B Q6_K)", "value": 8.3, "unit": "%" }, { "label": "Decode, RCCL on vs off (2x R9700, 27B Q4, 60K-120K)", "value": 0, "unit": "%" }, { "label": "Max prefill gain, RCCL on vs off (2x R9700, 27B Q4, 60K-120K)", "value": 10.1, "unit": "%" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/27825", "https://github.com/ggml-org/llama.cpp/pull/27825#issuecomment-5535620413", "https://github.com/ggml-org/llama.cpp/pull/27825#issuecomment-5699240655", "https://github.com/ggml-org/llama.cpp/pull/26046", "https://github.com/ggml-org/llama.cpp/pull/26696", "https://github.com/ggml-org/llama.cpp/pull/28079", "https://github.com/ggml-org/llama.cpp/pull/26419" ], "href": "https://amdai.net/lab-log#2026-10-26-hip-build-flags-rccl" }, { "id": "2026-10-26-ddr5-dimm-heat-hybrid-decode", "date": "2026-10-26T22:15:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Hot DDR5 can quietly cost hybrid GPU+RAM decode up to ~10%, and non-ECC sticks never report errors", "summary": "When a big MoE model keeps its experts in DDR5 next to two R9700s, the DIMMs work hard for hours. DDR5 doubles its refresh rate above 85 C (JEDEC refresh interval 3.9 us drops to 1.95 us). For 64 GB sticks built from 32 Gb chips, that takes the time a rank spends refreshing from about 10.5% to about 21%, so bandwidth-bound work can slow by up to ~13%. For hybrid decode we estimate up to ~7-11% fewer tokens per second once sticks pass 85 C, depending on how much of each token's time is spent reading DDR5 (an upper-bound estimate). Linux can read each stick's temperature with the spd5118 driver, so this is easy to check.", "setup": null, "metrics": [ { "label": "DDR5 refresh interval below 85 C", "value": 3.9, "unit": "us" }, { "label": "DDR5 refresh interval at 85-95 C", "value": 1.95, "unit": "us" }, { "label": "Time refreshing, 32 Gb chips, below 85 C (calc)", "value": 10.5, "unit": "%" }, { "label": "Time refreshing, 32 Gb chips, 85-95 C (calc)", "value": 21, "unit": "%" }, { "label": "Hybrid decode loss above 85 C, upper-bound estimate", "value": "7-11", "unit": "%" }, { "label": "Inference gain after cooling RDIMMs from 95 C+ (field report)", "value": 10, "unit": "%" } ], "sources": [ "https://docs.kernel.org/hwmon/spd5118.html", "https://cateee.net/lkddb/web-lkddb/SENSORS_SPD5118.html", "https://lkml.indiana.edu/2405.3/07725.html", "https://www.phoronix.com/forums/forum/hardware/motherboards-chipsets/1512681-linux-6-14-to-add-sensor-monitoring-for-a-~-180-asrock-am5-motherboard/page2", "https://hivebook.wiki/wiki/dram-row-buffers-and-bank-conflicts-activate-rate-tfaw-and-refresh", "https://semiengineering.com/dram-thermal-issues-reach-crisis-point/", "https://www.reddit.com/r/LocalLLaMA/comments/1p89j2t/today_i_learned_that_ddr5_can_throttle_itself_at/", "https://forum.level1techs.com/t/ddr5-4-dimms-on-am5-whats-working-whats-not/197153?page=14", "https://pakt.cc/en/ddr5/testing/" ], "href": "https://amdai.net/lab-log#2026-10-26-ddr5-dimm-heat-hybrid-decode" }, { "id": "2026-10-26-four-r9700-two-desktops-power-budget", "date": "2026-10-26T21:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Power budget for four R9700s in two desktops: one 15 A circuit is not enough, even at the lowest power cap", "summary": "Before buying a second desktop for two more R9700s, we built a power budget from published measurements. Under full compute an R9700 has been measured at about 390-400 W at the card (spec TBP 300 W), while decode-bound llama.cpp is much lower: about 340-353 W for two cards together in layer split. With all four cards at 300 W, the two desktops draw an estimated ~1,780 W at the wall; even at the lowest allowed cap (210 W) it is ~1,640 W. A 15 A, 120 V circuit may carry only 1,440 W continuously (the 80% rule for loads over 3 hours), so the two boxes need separate circuits, or one 20 A circuit with cards capped at 260 W or less.", "setup": null, "metrics": [ { "label": "4 cards at 300 W TBP, both desktops, wall (estimate)", "value": 1782, "unit": "W" }, { "label": "4 cards capped at 210 W, worst case, wall (estimate)", "value": 1642, "unit": "W" }, { "label": "15 A / 120 V circuit, continuous limit", "value": 1440, "unit": "W" }, { "label": "R9700 full compute load, measured at card (Igor's Lab)", "value": 400, "unit": "W" }, { "label": "Two R9700s, llama.cpp layer split, 1 request (Puget)", "value": 339, "unit": "W" }, { "label": "Decode loss at 260 W cap (published sweep)", "value": 0, "unit": "%" } ], "sources": [ "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro/ai-9000-series/amd-radeon-ai-pro-r9700.html", "https://www.igorslab.de/en/amd-radeon-ai-r9700-alias-rx-9070xt-in-the-big-workstation-graphics-card-test-what-are-the-benefits-of-double-the-memory-at-work/14/", "https://www.pcgameshardware.de/Radeon-AI-Pro-R9700-Grafikkarte-281741/Tests/Preis-Gaming-Benchmark-Release-Review-1486284/5/", "https://www.pugetsystems.com/labs/articles/amd-radeon-ai-pro-r9700-dual-gpu-ai-inference-performance/", "https://slybase.com/en/gpu-inference-tuning-same-performance-quieter-cheaper/", "https://www.tomshardware.com/pc-components/cpus/amd-ryzen-9-9950x-cpu-review/4", "https://www.infobytes.de/pages/atx3.1powerrev2.1.pdf", "https://electricaltoolbox.com/continuous-load-calculator/", "https://www.se.com/us/en/faqs/FA158939/" ], "href": "https://amdai.net/lab-log#2026-10-26-four-r9700-two-desktops-power-budget" }, { "id": "2026-10-26-llama-bench-noise-aa-test", "date": "2026-10-26T19:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "llama-bench A/A test: identical runs differed by up to 20%, and one run's mean +- stddev called identical runs 'different' 9 times in 15", "summary": "Before trusting a 3-5% win between two configs, we ran the same llama-bench command 6 times in a row. Inside one run, repetitions varied about 5%, but the medians of whole runs varied 6-7%, and the first run after loading was 12-16% slower than the rest. Comparing any two of the identical runs, a 95% interval built from one run's repetitions wrongly flagged a difference in 9 of 15 pairs. llama-bench's own tools don't guard against this: the generation warmup is a single token, repetitions run back-to-back by default, and compare-llama-bench.py prints a plain ratio of averages with no spread.", "setup": null, "metrics": [ { "label": "Within-run CV, tg64 / pp512", "value": "5.2 / 5.1", "unit": "%" }, { "label": "Between-run CV of medians, tg64 / pp512", "value": "6.1 / 7.3", "unit": "%" }, { "label": "Largest gap between identical runs, tg64 / pp512", "value": "20.4 / 19.2", "unit": "%" }, { "label": "First run vs later runs, pp512", "value": "357 vs 412-426", "unit": "tok/s" }, { "label": "Identical pairs flagged 'different' by a one-run 95% CI", "value": "9 of 15", "unit": "pairs" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/master/tools/llama-bench/README.md", "https://github.com/ggml-org/llama.cpp/blob/master/tools/llama-bench/llama-bench.cpp", "https://github.com/ggml-org/llama.cpp/blob/master/scripts/compare-llama-bench.py" ], "href": "https://amdai.net/lab-log#2026-10-26-llama-bench-noise-aa-test" }, { "id": "2026-10-26-openwebui-background-tasks-prompt-cache", "date": "2026-10-26T18:35:00-05:00", "project": "idea-lab", "kind": "result", "title": "Open WebUI in front of llama-server: with --cache-ram 0, its background calls force a full re-prefill of the chat on every turn", "summary": "Open WebUI 0.12.4 sends extra model calls per chat turn (follow-up suggestions every turn, plus title and tags on the first), each a fresh one-message prompt that shares no prefix with the conversation. On a one-slot llama-server they replace the slot's state. With the default host prompt cache (--cache-ram 8192 MiB) the conversation is saved and restored and each later turn re-processes only about 70 tokens; with --cache-ram 0, or a cache smaller than one conversation's saved state, every turn re-processed the whole 11K-token chat (about 40x the prefill time in this test).", "setup": null, "metrics": [ { "label": "Tokens re-processed per later turn, default --cache-ram", "value": "68-70", "unit": "tokens" }, { "label": "Tokens re-processed per later turn, --cache-ram 0", "value": "11,201-11,396", "unit": "tokens" }, { "label": "Prefill per later turn, default --cache-ram (CPU test box)", "value": "0.52-0.56", "unit": "s" }, { "label": "Prefill per later turn, --cache-ram 0 (CPU test box)", "value": "19.8-22.9", "unit": "s" }, { "label": "Prompt cache entry for the 11.2K-token chat incl. checkpoints", "value": "208.4", "unit": "MiB" }, { "label": "Extra model calls per later turn (Open WebUI defaults)", "value": "1", "unit": "calls" } ], "sources": [ "https://github.com/open-webui/docs/blob/main/docs/features/administration/task-models.mdx", "https://github.com/open-webui/open-webui/issues/638", "https://github.com/ggml-org/llama.cpp/pull/16391", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/server-context.cpp", "https://github.com/ggml-org/llama.cpp/releases/tag/b11912" ], "href": "https://amdai.net/lab-log#2026-10-26-openwebui-background-tasks-prompt-cache" }, { "id": "2026-10-26-glm-flash-chat-template-render", "date": "2026-10-26T17:30:00-05:00", "project": "idea-lab", "kind": "result", "title": "GLM-5.3-Flash GGUF chat template is two upstream fixes behind: up to 17 s of template rendering per request with many parallel tool results in llama.cpp", "summary": "The chat template embedded in the current Unsloth GLM-5.3-Flash GGUF matches the upstream 2026-08-27 version and misses the 2026-10-15 upstream rewrite of tool-result reordering. Rendered with llama.cpp's own Jinja engine, one block of 100 parallel tool results takes 1.14 s with the embedded template vs 31 ms with the new one (400 results: 17.0 s vs 0.19 s), with byte-identical output. Fix without re-downloading: pass the upstream template with --chat-template-file.", "setup": { "model": "GLM-5.3-Flash (also Qwen3.6-35B-A3B, Qwen3.8-Flash-Next templates)", "quant": "UD-Q6_K_XL header (template is quant-independent)", "engine": "llama.cpp Jinja engine + Python Jinja2", "engine_version": "llama.cpp master 4244ad7; Jinja2 3.1.6", "gpus": "none (CPU-only template rendering)", "context": "n/a", "flags": "--chat-template-file" }, "metrics": [ { "label": "Render, 100 parallel tool results, GGUF template", "value": 1140, "unit": "ms" }, { "label": "Render, 100 parallel tool results, upstream 2026-10-15 template", "value": 31, "unit": "ms" }, { "label": "Render, 400 parallel tool results, GGUF template", "value": 17001, "unit": "ms" }, { "label": "Render, 400 parallel tool results, upstream 2026-10-15 template", "value": 188, "unit": "ms" } ], "sources": [ "https://huggingface.co/zai-org/GLM-5.3-Flash/discussions/63", "https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF", "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/chat_template.jinja", "https://github.com/ggml-org/llama.cpp/tree/4244ad7acd98328c1f04f1c2201d571f643370bb/common/jinja" ], "href": "https://amdai.net/lab-log#2026-10-26-glm-flash-chat-template-render" }, { "id": "2026-10-26-r9700-street-price", "date": "2026-10-26T16:03:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 street price today: $1,699.99, 31% over MSRP, and only in store", "summary": "Across the six US R9700 32 GB models, the cheapest new card in stock is $1,699.99 (Gigabyte AI TOP and XFX at Micro Center, in-store pickup), 31% above AMD's $1,299 MSRP. The lowest price seen since Sep 29 is $1,599.99. Amazon's in-stock new offers run $2,399.99 to $2,680.81.", "setup": { "gpus": "Radeon AI PRO R9700 32 GB (6 board-partner models)", "region": "US, new condition, in stock" }, "metrics": [ { "label": "Cheapest new R9700 in stock", "value": 1699.99, "unit": "USD" }, { "label": "Premium over $1,299 MSRP", "value": 30.9, "unit": "%" }, { "label": "Lowest price since Sep 29", "value": 1599.99, "unit": "USD" }, { "label": "Cheapest Amazon in-stock new offer", "value": 2399.99, "unit": "USD" }, { "label": "Two cards today, before tax", "value": 3399.98, "unit": "USD" } ], "sources": [ "https://vga.live/gpus/radeon-ai-pro-r9700", "https://vga.live/cards/gigabyte-amd-radeon-ai-pro-r9700-top-single-fan-ai-workstation", "https://vga.live/cards/xfx-amd-radeon-ai-pro-r9700-single-fan-32gb-gddr6", "https://vga.live/cards/asrock-amd-radeon-ai-pro-r9700-creator-single-fan-ai-workstation", "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro.html" ], "href": "https://amdai.net/lab-log#2026-10-26-r9700-street-price" }, { "id": "2026-10-26-strata-mtp-q2-only-second-card", "date": "2026-10-26T14:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Strata's MTP draft layer only loads Q2_0 experts, so a Q4+-only R9700 setup loses drafts and the RDNA4 pipeline-windows split gain", "summary": "In Strata main (2026-10-25), the runtime drafter sizes its 512 experts as kernels::cpu::BLOB, which is the Q2_0 layout (1,382,400 bytes per expert, 707,788,800 for the file). mtp_pack.py can write Q4_0 or Q8_0 draft experts, but nothing loads them (open issue #3728), and --mtp-q4 only requantizes the dense head, not the experts. Anyone who keeps every file at 4-bit or more therefore runs Strata with no draft layer. On a layer split that also turns off pipeline windows, the RDNA4 default that gave +21% code decode on 2x R9700 (103 to 125 tok/s) in the maintainer's table.", "setup": null, "metrics": [ { "label": "Q2_0 draft expert size (per expert)", "value": "1,382,400", "unit": "bytes" }, { "label": "Pipeline windows, 2x R9700 code decode (with MTP, cited)", "value": "103 -> 125", "unit": "tok/s" }, { "label": "UD-Q5 1 -> 2 R9700 code decode (with MTP, cited)", "value": "83 -> 94", "unit": "tok/s" }, { "label": "UD-Q6 1 R9700 story / code (with MTP, 8ch DDR4, cited)", "value": "54 / 76", "unit": "tok/s" }, { "label": "Our UD-Q6 lane, 1 R9700, no draft layer (measured)", "value": "32.6", "unit": "tok/s" } ], "sources": [ "https://github.com/Niko1221/Strata/blob/4b9e05a2a806a3fe24f7b2560d98626c37dade35/src/core/mtp.cpp", "https://github.com/Niko1221/Strata/blob/4b9e05a2a806a3fe24f7b2560d98626c37dade35/include/strata/kernels/cpu/expert.hpp", "https://github.com/Niko1221/Strata/blob/4b9e05a2a806a3fe24f7b2560d98626c37dade35/docs/MULTI_GPU.md", "https://github.com/Niko1221/Strata/blob/4b9e05a2a806a3fe24f7b2560d98626c37dade35/docs/SECOND_GPU.md", "https://github.com/Niko1221/Strata/blob/4b9e05a2a806a3fe24f7b2560d98626c37dade35/docs/QUANTS.md", "https://github.com/Niko1221/Strata/issues/3728" ], "href": "https://amdai.net/lab-log#2026-10-26-strata-mtp-q2-only-second-card" }, { "id": "2026-10-26-vllm-tp2-allreduce-rccl-only", "date": "2026-10-26T11:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "vLLM TP2 on two R9700s: every all-reduce goes through RCCL, and batch-1 decode pays it ~80-96 times per token", "summary": "In current vLLM source, the custom all-reduce, QuickReduce and AITER all-reduce paths are enabled only on MI300-class GPUs; on gfx1201 vLLM forces disable_custom_all_reduce on, so every tensor-parallel all-reduce is an RCCL call (this is why --disable-custom-all-reduce 'has no effect' on R9700). With two all-reduces per layer, Qwen3.6-35B-A3B makes 80 per token (4 KiB each) and Qwen3.8-Flash-Next 96 (5 KiB). The only published R9700 rccl-tests table (4 cards, P2P off) is flat at 31-36 us up to 32 KiB, which is about 2.9 and 3.4 ms of communication per token, before any compute.", "setup": { "model": "Qwen3.6-35B-A3B / Qwen3.8-Flash-Next", "quant": "n/a (communication analysis)", "engine": "vLLM (main, 2026-10-26 source) + RCCL", "engine_version": "vLLM main 7f56f45", "gpus": "2x Radeon AI PRO R9700", "context": "batch 1 decode", "flags": "tensor-parallel-size 2, NCCL_PROTO=Simple" }, "metrics": [ { "label": "All-reduces per token, Qwen3.6-35B-A3B", "value": 80, "unit": "" }, { "label": "All-reduces per token, Qwen3.8-Flash-Next", "value": 96, "unit": "" }, { "label": "Published R9700 RCCL all-reduce latency, 8 B-32 KiB", "value": "31-36", "unit": "us" }, { "label": "Comms time per token, 35B (from published table)", "value": 2.9, "unit": "ms" }, { "label": "Comms time per token, Flash-Next (from published table)", "value": 3.4, "unit": "ms" }, { "label": "Estimated TP2 decode ceiling, 35B (estimate)", "value": "110-128", "unit": "tok/s" } ], "sources": [ "https://github.com/vllm-project/vllm/blob/7f56f453ede3/vllm/platforms/rocm.py", "https://github.com/vllm-project/vllm/blob/7f56f453ede3/vllm/config/parallel.py", "https://github.com/vllm-project/vllm/pull/19744", "https://github.com/ROCm/rccl-tests/issues/162", "https://github.com/ROCm/rocm-systems/issues/5480", "https://github.com/ROCm/rocm-systems/pull/13151", "https://github.com/ROCm/ROCR-Runtime/issues/355", "https://github.com/ROCm/rocm-systems/issues/13616", "https://forum.level1techs.com/t/amd-radeon-r9700-linux-tp2-and-tp4-the-definitive-guide/250504" ], "href": "https://amdai.net/lab-log#2026-10-26-vllm-tp2-allreduce-rccl-only" }, { "id": "2026-10-26-gguf-reupload-drift", "date": "2026-10-26T10:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Stale GGUFs after a re-upload: GLM-5.3-Flash needs one 9 MiB shard, not a 272 GiB re-download, and old Flash-Next MTP heads carry a wrong value", "summary": "On 2026-10-05 Unsloth re-uploaded files in its GLM-5.3-Flash and Qwen3.8-Flash-Next GGUF repos 'due to upstream changes'. We diffed the old and new files from the Hugging Face history. For GLM-5.3-Flash only shard 1 of each quant changed, and it holds metadata only (9.0 MiB, no tensors): the architecture name went from glm5next to glm5-next, the name mainline llama.cpp merged on 2026-09-30, and one key was added (attention.indexer.index_share_mtp = true). A copy downloaded before then can be fixed by replacing that one shard instead of re-downloading 186-272 GiB. For Qwen3.8-Flash-Next the UD-Q4/Q5/Q6 model shards did not change; the three non-shared MTP head files did, by one value: the MTP layer's attention compress ratio went from 0 to 4.", "setup": { "model": "GLM-5.3-Flash, Qwen3.8-Flash-Next, Qwen3.6-35B-A3B (Unsloth GGUF)", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (header diff on UD-Q6_K_XL shard 1 and MTP Q8_0)", "engine": "llama.cpp (arch names from master)", "engine_version": "master, 2026-10-26", "gpus": "n/a (file analysis)" }, "metrics": [ { "label": "GLM-5.3-Flash shard that changed (metadata only)", "value": 9, "unit": "MiB" }, { "label": "GLM-5.3-Flash UD-Q6_K_XL full set", "value": 271.8, "unit": "GiB" }, { "label": "GLM-5.3-Flash UD-Q4_K_XL full set", "value": 186, "unit": "GiB" }, { "label": "Metadata keys changed in GLM shard 1", "value": 82, "unit": "keys (40 renamed pairs + arch + 1 new)" }, { "label": "Flash-Next MTP heads changed", "value": 3, "unit": "files" } ], "sources": [ "https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/commit/a38483c8cd5df544f53d70fb281afe97369d5ab6", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/commit/efc50b7c3195a10eebd8d3e32f83b27b2aeb239d", "https://github.com/ggml-org/llama.cpp/pull/27773", "https://github.com/ggml-org/llama.cpp/pull/27752", "https://huggingface.co/api/models/unsloth/Qwen3.6-35B-A3B-GGUF/commits/main" ], "href": "https://amdai.net/lab-log#2026-10-26-gguf-reupload-drift" }, { "id": "2026-10-26-35b-a3b-mtp-vs-draft-r9700", "date": "2026-10-26T09:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Faster Qwen3.6-35B-A3B on one R9700: use the built-in MTP head, not a draft model, and watch the 262K fit", "summary": "Published tests show a separate small draft model makes Qwen3.6-35B-A3B slower (one RTX 3090: 135.7 tok/s baseline vs 115.9-121.3 mean with drafts), while the model's own MTP head helps one user at a time: on an R9700 (Q4_K_M, Vulkan) one slot went from about 108 to about 133 tok/s, but MTP made a 3-slot job 1.58x slower. We read the headers of Unsloth's MTP GGUFs: the extra MTP block is Q8_0 attention with Q5_K/Q6_K experts in UD-Q5_K_XL, nothing below Q4.", "setup": { "model": "Qwen3.6-35B-A3B (MTP GGUF)", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (header check); Q4_K_M and UD-Q4_K_XL in cited speed tests", "engine": "llama.cpp", "engine_version": "per source", "gpus": "1x R9700 (cited); RTX 3090 and 7900 XTX for comparison", "context": "short to 245K", "flags": "--spec-type draft-mtp --spec-draft-n-max 2, -np 1" }, "metrics": [ { "label": "R9700 Q4_K_M Vulkan, 1 slot, no MTP", "value": 108, "unit": "tok/s" }, { "label": "R9700 Q4_K_M Vulkan, 1 slot, MTP", "value": 133, "unit": "tok/s" }, { "label": "R9700, 3 slots, MTP slowdown", "value": 1.58, "unit": "x slower" }, { "label": "RTX 3090 baseline vs best draft-model mean", "value": "135.7 vs 121.1", "unit": "tok/s" }, { "label": "MTP file size increase, UD-Q5_K_XL", "value": 540, "unit": "MiB" }, { "label": "MTP draft KV at 262K, f16", "value": 512, "unit": "MiB" } ], "sources": [ "https://finaltype.github.io/quant-blog/en/devlog/2026-09/2026-09-26_moe35b-r9700-speed-tuning/", "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/discussions/14", "https://github.com/thc1006/qwen3.6-speculative-decoding-rtx3090", "https://gist.github.com/AshyIsMe/ef9dc0e57d7af5626f3935f55c38543a", "https://unsloth.ai/docs/models/tutorials/qwen3.6", "https://github.com/ggml-org/llama.cpp/discussions/26230", "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF" ], "href": "https://amdai.net/lab-log#2026-10-26-35b-a3b-mtp-vs-draft-r9700" }, { "id": "2026-10-26-multi-gpu-silent-corruption", "date": "2026-10-26T08:15:00-05:00", "project": "idea-lab", "kind": "result", "title": "Two Radeon cards, wrong answers at full speed: which split setups corrupt text, what fixed it, and a correctness check", "summary": "Several 2026 Radeon multi-GPU setups on ROCm/HIP produced fluent gibberish or blank output at normal speed with no error, and llama-bench still reported healthy numbers. The Windows layer-split case was fixed in ROCm's runtime (PR #10553, merged 2026-09-02); Windows -sm tensor on ROCm is still open; Linux cards with no peer-to-peer path need a no-peer-copy build. Always check text, not just tok/s, after a stack change.", "setup": { "model": "Qwen3.8-27B Q4_K_M (Windows speed test); various (corruption reports)", "quant": ">=Q4 in all cited tests", "engine": "llama.cpp (HIP and Vulkan), Strata", "engine_version": "per source", "gpus": "2x R9700; 7900 XTX + 9070 XT; 2x 9060 XT; others", "context": "short", "flags": "-sm layer / -sm tensor, -DGGML_CUDA_NO_PEER_COPY=ON, --load-mode none" }, "metrics": [ { "label": "Windows 2x R9700 Vulkan, one card (-sm none)", "value": 32.1, "unit": "tok/s" }, { "label": "Windows 2x R9700 Vulkan, -sm layer", "value": 31.3, "unit": "tok/s" }, { "label": "Windows 2x R9700 Vulkan, -sm tensor", "value": 14.9, "unit": "tok/s" }, { "label": "No-peer-copy build, decode cost (one mixed-card test)", "value": -7.7, "unit": "%" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/issues/28676", "https://github.com/lemonade-sdk/llamacpp-rocm/issues/75", "https://github.com/ROCm/rocm-systems/pull/10553", "https://github.com/lemonade-sdk/llamacpp-rocm/issues/148", "https://github.com/ROCm/rocm-systems/issues/13715", "https://github.com/ROCm/ROCm/issues/6342" ], "href": "https://amdai.net/lab-log#2026-10-26-multi-gpu-silent-corruption" }, { "id": "2026-10-26-slot-restore-hybrid-checkpoints", "date": "2026-10-26T07:15:00-05:00", "project": "idea-lab", "kind": "result", "title": "llama.cpp slot save/restore now works for hybrid Qwen models: resume a long agent context after a restart", "summary": "Before llama.cpp b11507, restoring a saved slot on hybrid (Gated DeltaNet) models like Qwen3.5/3.6/3.8 reported success but re-processed the whole prompt, because context checkpoints were not saved. PR #26004 now appends them to the slot file. A published AMD Vulkan test (Radeon 8050S, Qwen3.5-35B-A3B UD-Q4_K_XL, 58K tokens) cut the first request after restore from 181.9 s to 4.7 s.", "setup": { "model": "Qwen3.5-35B-A3B (published test); Qwen3.6-35B-A3B (size estimate)", "quant": "UD-Q4_K_XL (published test)", "engine": "llama.cpp llama-server", "engine_version": ">= b11507 (checkpoints), >= b11669 (draft state)", "gpus": "Radeon 8050S (published test)", "context": "58K tokens", "flags": "--slot-save-path DIR, --ctx-checkpoints, --cache-ram 0 for testing" }, "metrics": [ { "label": "First request after restore, before fix", "value": 181.9, "unit": "s" }, { "label": "First request after restore, with fix", "value": 4.7, "unit": "s" }, { "label": "Tokens re-processed after restore, with fix", "value": 516, "unit": "tokens" }, { "label": "Est. slot file, 35B-A3B at 262K, f16 KV", "value": 5.1, "unit": "GiB" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/26004", "https://github.com/ggml-org/llama.cpp/pull/26004#issuecomment-5055464245", "https://github.com/ggml-org/llama.cpp/pull/30574", "https://github.com/ggml-org/llama.cpp/issues/28194", "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/main/config.json" ], "href": "https://amdai.net/lab-log#2026-10-26-slot-restore-hybrid-checkpoints" }, { "id": "2026-10-26-thinking-budget-latency", "date": "2026-10-26T06:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Capping thinking on a slow Flash model: the stop message matters more than the token count", "summary": "At about 33 tok/s (a Flash-class MoE on 2x R9700 with experts in DDR5), every 1,000 thinking tokens is roughly 31 seconds before the first visible word. llama.cpp can cap this per request with thinking_budget_tokens plus reasoning_budget_message, and since June 2026 a client can end thinking mid-stream with POST /v1/chat/completions/control, which allows a wall-clock deadline instead of a token count. A bare cap without a stop message hurts: HumanEval on a Qwen3.5 9B fell to ~79% vs ~89% with a message (~93% uncapped), and on Qwen3.6-35B-A3B a bare 4,096-token cap ran 170 s as reasoning spilled into the answer, vs 78 s with Qwen's own stop sentence.", "setup": { "model": "Qwen3.8 Flash (UD-Q6_K_XL) and Qwen3.6-35B-A3B (UD-Q5_K_XL)", "engine": "llama.cpp server (source review), Strata (speed only)", "engine_version": "llama.cpp master 8b28c23", "gpus": "2x Radeon AI PRO R9700 (Flash); 1x R9700 (35B)" }, "metrics": [ { "label": "Flash decode used for the arithmetic", "value": 32.6, "unit": "tok/s" }, { "label": "Seconds per 1,000 thinking tokens (Flash)", "value": 31, "unit": "s" }, { "label": "HumanEval, budget 1000 with message (Qwen3.5 9B)", "value": 89, "unit": "%" }, { "label": "HumanEval, budget without message (Qwen3.5 9B)", "value": 79, "unit": "%" }, { "label": "Bare 4,096 cap, 35B-A3B, 3 prompts", "value": 170, "unit": "s" }, { "label": "4,096 cap with stop message, 35B-A3B", "value": 78, "unit": "s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/20297", "https://github.com/ggml-org/llama.cpp/pull/23971", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md", "https://huggingface.co/ManniX-ITA/Qwen3.6-27B-A3B-Coder/commit/59aa5b59f6ecd7a7fa9cc191e1e40d064ad2b2e7", "https://arxiv.org/abs/2505.09388" ], "href": "https://amdai.net/lab-log#2026-10-26-thinking-budget-latency" }, { "id": "2026-10-26-r9700-hang-recovery-classes", "date": "2026-10-26T05:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 GPU hangs come in three kinds, and only one is fixed by restarting the server", "summary": "Public Radeon AI PRO R9700 (gfx1201) fault reports split into three recovery classes. (1) The server process dies and the driver's reset succeeds: a systemd Restart=on-failure is enough. (2) A silent wedge: the server stays up at 100% GPU busy and 0% memory busy but produces no tokens, or blocks at idle power with no error, so nothing restarts it and a /health check still answers 200. (3) A dead GPU: the reset ends with 'PSP resume failed' / ret -62 or a reset storm with a CPU soft lockup, and only a reboot helps; repeated auto-retries in that state have hard-hung an R9700.", "setup": { "gpus": "Radeon AI PRO R9700 (gfx1201)", "engine": "llama.cpp / vLLM (published reports)" }, "metrics": [ { "label": "No-output stall before timeout (ROCm 7.14, #6630)", "value": 180, "unit": "s" }, { "label": "R9700s that degraded to GPU Hang with cwsr_enable=0", "value": "3 of 4", "unit": "" }, { "label": "Successful MODE2 resets recovered by a 10 s systemd restart (gfx1151 report)", "value": 4, "unit": "" } ], "sources": [ "https://github.com/ROCm/legacy-rocm-build/issues/6630", "https://github.com/ROCm/ROCm/issues/6413", "https://github.com/ROCm/TheRock/issues/9271", "https://github.com/lemonade-sdk/lemonade/issues/2609", "https://github.com/ROCm/ROCm/issues/6127", "https://docs.kernel.org/gpu/amdgpu/debugfs.html" ], "href": "https://amdai.net/lab-log#2026-10-26-r9700-hang-recovery-classes" }, { "id": "2026-10-26-endpoint-security", "date": "2026-10-26T04:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Securing a local LLM server: what llama-server's API key covers, 2026 CVEs, and why ggml RPC must stay on a private link", "summary": "llama-server binds 127.0.0.1 by default, and --api-key protects every route except /health and the web-UI files. Without a key, /props, /slots and /v1/models are open, and the default CORS setting echoes any Origin. Builds from b8227 up to but not including b11393 have a 9.2-rated use-after-free on POST /completion (CVE-2026-107183), so update to b11393 or newer. The RPC backend used for multi-machine splits has no authentication and had a 9.8-rated remote code execution bug this year (fixed in b8492), so it belongs on a dedicated point-to-point link only.", "setup": { "engine": "llama.cpp llama-server, ggml RPC, vLLM", "engine_version": "llama.cpp master 9125a2e (release v0.7.0)", "gpus": "2x Radeon AI PRO R9700 (applies to any GPU)" }, "metrics": [ { "label": "CVE-2026-107183 fixed in", "value": "b11393", "unit": "" }, { "label": "CVE-2026-107183 CVSS 4.0", "value": 9.2, "unit": "" }, { "label": "CVE-2026-34159 (RPC) CVSS 3.1", "value": 9.8, "unit": "" }, { "label": "Exposed AI endpoints with auth challenge (Sept 2026 census)", "value": 2.02, "unit": "%" } ], "sources": [ "https://www.cve.org/CVERecord?id=CVE-2026-107183", "https://www.cve.org/CVERecord?id=CVE-2026-34159", "https://github.com/ggml-org/llama.cpp/security/advisories?state=published", "https://github.com/advisories/GHSA-3f2h-96vw-c84x", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md", "https://github.com/ggml-org/llama.cpp/blob/master/tools/rpc/README.md", "https://docs.vllm.ai/en/latest/usage/security/", "https://www.mysteriumvpn.com/blog/data-and-research/we-exposed-ai-supply-chain" ], "href": "https://amdai.net/lab-log#2026-10-26-endpoint-security" }, { "id": "2026-10-26-r9700-tts-llama-server", "date": "2026-10-26T03:12:00-05:00", "project": "idea-lab", "kind": "result", "title": "Text-to-speech next to a local LLM on R9700: llama-server now serves Qwen3-TTS, and a CPU voice would compete with offloaded experts for DDR5", "summary": "llama.cpp merged a native POST /tts endpoint for llama-server on 2026-10-20 (release b11781), with streaming, speaker-reference upload and router-mode autoload, so a TTS model can sit behind the same server as the LLM. Published R9700 results for Qwen3-TTS: the PyTorch reference runs at RTF 1.90 (slower than real time), while a custom HIP port reaches RTF 0.55 for the 1.7B model and 0.41-0.54 for the 0.6B. No llama.cpp R9700 TTS number has been published yet.", "setup": { "model": "Qwen3-TTS-12Hz 1.7B and 0.6B", "engine": "llama.cpp llama-server /tts (b11781+); custom HIP port; PyTorch reference (published)", "gpus": "1x Radeon AI PRO R9700" }, "metrics": [ { "label": "Qwen3-TTS 1.7B, PyTorch FP32 reference, RTF", "value": 1.9, "unit": "x" }, { "label": "Qwen3-TTS 1.7B, custom HIP port, RTF", "value": 0.55, "unit": "x" }, { "label": "Qwen3-TTS 0.6B, custom HIP port streaming, RTF", "value": 0.54, "unit": "x" }, { "label": "Qwen3-TTS 1.7B Q8_0 GGUF + bf16 mmproj", "value": 2.34, "unit": "GiB" }, { "label": "Time to first audio with code2wav ramp patch (unmerged, Vulkan)", "value": 0.4, "unit": "s" }, { "label": "Estimated CPU RTF, Q4_K_M at 55 GB/s DDR5 (estimate)", "value": 0.6, "unit": "x" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/26603", "https://github.com/ggml-org/llama.cpp/pull/26254", "https://github.com/ggml-org/llama.cpp/tree/master/tools/tts", "https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "https://flynnbody.com/blog/running-qwen3-tts-on-amd-r9700/", "https://github.com/piedshag/qwen3-tts-hip" ], "href": "https://amdai.net/lab-log#2026-10-26-r9700-tts-llama-server" }, { "id": "2026-10-26-r9700-image-video-gen", "date": "2026-10-26T02:08:00-05:00", "project": "idea-lab", "kind": "result", "title": "Image and video generation on one R9700: step-distill LoRAs give ~6x, the hipBLASLt/MIOpen flags slow video 5-7%", "summary": "From published R9700 ComfyUI runs: SDXL 1024x1024 runs at 4.49-4.6 it/s. 4-step distillation LoRAs cut Qwen-Image-2512 from 85.9 s to 12.4 s and Wan 2.2 A14B 480p from 341.7 s to 56.6 s, far more than any runtime knob. In a same-box A/B, TORCH_BLAS_PREFER_HIPBLASLT=1 plus COMFYUI_ENABLE_MIOPEN=1 made all four video workflows 5.3-7.3% slower and left Qwen-Image within 2%.", "setup": { "model": "SDXL 1.0, Qwen-Image-2512 FP8, Wan 2.2 A14B (FP8 and Q6_K), HunyuanVideo 1.5", "engine": "ComfyUI (published)", "engine_version": "per source (ROCm 7.1-7.2.4)", "gpus": "1x Radeon AI PRO R9700" }, "metrics": [ { "label": "SDXL 1024x1024 sampling", "value": 4.49, "unit": "it/s" }, { "label": "Qwen-Image-2512 FP8, 20 steps, warm", "value": 85.9, "unit": "s" }, { "label": "Qwen-Image-2512 FP8, 4-step LoRA, warm", "value": 12.4, "unit": "s" }, { "label": "Wan 2.2 A14B Q6_K 480p 33 fr, 4-step LoRA", "value": 56.6, "unit": "s" }, { "label": "Wan 2.2 A14B Q6_K 720p VRAM peak", "value": 31.8, "unit": "GiB" }, { "label": "Wan 2.2 I2V with hipBLASLt+MIOpen flags", "value": 5.3, "unit": "% slower" } ], "sources": [ "https://kyuz0.github.io/amd-r9700-comfy/", "https://kyuz0.github.io/amd-r9700-comfy/benchmark_results.json", "https://brookesia.app/boards/ai-hardware/76", "https://brookesia.app/boards/ai-hardware/78", "https://rocm.blogs.amd.com/artificial-intelligence/comfyui-radeon-9000/README.html" ], "href": "https://amdai.net/lab-log#2026-10-26-r9700-image-video-gen" }, { "id": "2026-10-26-r9700-lora-pagefault", "date": "2026-10-26T01:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "LoRA fine-tuning on the R9700: the hipBLASLt page-fault trap and what fixes it", "summary": "The R9700 (gfx1201) is officially supported for LoRA/QLoRA training (Unsloth lists RDNA4 as Full; TheRock lists gfx1201 Release Ready), but stock hipBLASLt GEMMs can fault in the LoRA backward pass at a fixed step. On ROCm 7.2.4 a reporter saw bf16 LoRA on Qwen3-8B fault at step 85 and QLoRA NF4 at step 799. The fix (rocm-libraries PR 8909, merged 2026-08-14) ran clean in independent validations at about 1.5% extra step time.", "setup": { "gpus": "Radeon AI PRO R9700 (gfx1201), published reports", "engine": "PyTorch ROCm + PEFT / Unsloth", "engine_version": "ROCm 7.2.4, torch 2.13.0 (issue 6600)", "model": "Qwen3-8B, Qwen3.6-27B (reproducers)" }, "metrics": [ { "label": "bf16 LoRA fault step, stock ROCm 7.2.4", "value": 85, "unit": "step" }, { "label": "QLoRA NF4 fault step, stock ROCm 7.2.4", "value": 799, "unit": "step" }, { "label": "12-step run, stock vs PR 8909 build", "value": "726 vs 737", "unit": "s" }, { "label": "hipBLASLt-off workaround step time", "value": 2.5, "unit": "x slower" } ], "sources": [ "https://github.com/ROCm/legacy-rocm-build/issues/6600", "https://github.com/ROCm/rocm-libraries/pull/8909", "https://unsloth.ai/docs/basics/amd", "https://github.com/ROCm/TheRock/blob/main/SUPPORTED_GPUS.md" ], "href": "https://amdai.net/lab-log#2026-10-26-r9700-lora-pagefault" }, { "id": "2026-10-26-embed-rerank-pick", "date": "2026-10-26T00:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Local RAG for an R9700 agent: Qwen3-Embedding-4B Q5_K_M is the value point, 0.6B pair fits in ~1.2 GiB", "summary": "From official GGUF sizes and vendor scores: Qwen3-Embedding-4B Q5_K_M (2.69 GiB) scores 69.45 on MTEB multilingual, +5.1 over the 0.6B Q8_0 (0.60 GiB, 64.33), while the 8B Q5_K_M (5.05 GiB) adds only +1.1 more. Qwen3-Reranker-0.6B (0.60 GiB at Q8_0) lifts retrieval to 65.80 MTEB-R from 61.82, well ahead of bge-reranker-v2-m3 at 57.03.", "setup": { "model": "Qwen3-Embedding 0.6B/4B/8B, Qwen3-Reranker-0.6B, bge-reranker-v2-m3", "quant": "Q8_0 / Q5_K_M", "engine": "llama.cpp server (docs at master d854d98bae)", "gpus": "Radeon AI PRO R9700 (planned placement)" }, "metrics": [ { "label": "Qwen3-Embedding-0.6B Q8_0 size", "value": 0.595, "unit": "GiB" }, { "label": "Qwen3-Embedding-4B Q5_K_M size", "value": 2.691, "unit": "GiB" }, { "label": "Qwen3-Embedding-8B Q5_K_M size", "value": 5.05, "unit": "GiB" }, { "label": "MTEB multilingual mean, 0.6B / 4B / 8B", "value": "64.33 / 69.45 / 70.58", "unit": "" }, { "label": "MTEB-R, Qwen3-Reranker-0.6B vs bge-reranker-v2-m3", "value": "65.80 vs 57.03", "unit": "" } ], "sources": [ "https://huggingface.co/Qwen/Qwen3-Embedding-8B", "https://huggingface.co/Qwen/Qwen3-Embedding-4B-GGUF", "https://huggingface.co/Qwen/Qwen3-Reranker-0.6B", "https://huggingface.co/ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md" ], "href": "https://amdai.net/lab-log#2026-10-26-embed-rerank-pick" }, { "id": "2026-10-25-strix-halo-vs-r9700", "date": "2026-10-25T23:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Strix Halo vs Radeon AI PRO R9700 on the same benchmark harness: ~2x decode, 3-4x prefill when the model fits", "summary": "Joining kyuz0's two public llama-bench grids (same scripts, best backend per row, 8 overlapping Q4+ models), one or two R9700s lead a 128 GB Ryzen AI Max+ 395 by a median ~2.0x on decode (tg128) and ~2.9x on prompt processing (pp512), widening to ~3.6-4.1x prefill at 32K-64K depth. Strix Halo's edge is capacity: it runs 80-90 GiB Q4/Q5 MoE models entirely in unified memory.", "setup": { "model": "8 overlapping models (gpt-oss-120b, Qwen3.6-35B-A3B, Gemma 4, others)", "quant": "Q4+", "engine": "llama.cpp llama-bench (published grids)", "engine_version": "b8990 (R9700), b9187/b9193 (Strix Halo)", "gpus": "1-2x Radeon AI PRO R9700 vs Ryzen AI Max+ 395 128 GB", "context": "0, 32K and 64K depth", "flags": "fa on, ngl 99, best of ROCm and Vulkan backends" }, "metrics": [ { "label": "Median decode lead, R9700 vs Strix Halo (tg128)", "value": 2, "unit": "x" }, { "label": "Median prompt lead (pp512)", "value": 2.9, "unit": "x" }, { "label": "Median prefill lead at 64K depth", "value": 4, "unit": "x" }, { "label": "gpt-oss-120b decode, 2x R9700", "value": 98, "unit": "tok/s" }, { "label": "gpt-oss-120b decode, Strix Halo", "value": 56.6, "unit": "tok/s" }, { "label": "Qwen3.5-122B-A10B UD-Q5_K_XL decode, Strix Halo", "value": 22.3, "unit": "tok/s" } ], "sources": [ "https://kyuz0.github.io/amd-r9700-ai-toolboxes/", "https://kyuz0.github.io/amd-strix-halo-toolboxes/", "https://github.com/lhl/strix-halo-testing/tree/main/llm-bench" ], "href": "https://amdai.net/lab-log#2026-10-25-strix-halo-vs-r9700" }, { "id": "2026-10-25-model-swap-router", "date": "2026-10-25T22:07:00-05:00", "project": "idea-lab", "kind": "result", "title": "Swapping big models on one box: llama.cpp router mode evicts by count, not VRAM; llama-swap can budget per card", "summary": "llama-server's router mode (start it with no model and a --models-preset INI) loads models on demand, but it decides what to unload only by counting: --models-max defaults to 4 and the scheduler never checks VRAM or RAM. On a dual-R9700 box where one Flash-class model fills both cards and DDR5, run it with --models-max 1, or a request naming a second big model will try to load it alongside the first. llama-swap's capacity router lets you declare per-card and RAM budgets instead, and can also host non-llama.cpp servers.", "setup": { "engine": "llama.cpp master d854d98bae and llama-swap v266 (source and docs read)", "hardware": "planned for 2x Radeon AI PRO R9700 + DDR5" }, "metrics": [ { "label": "router --models-max default", "value": 4, "unit": "models" }, { "label": "preset stop-timeout default", "value": 10, "unit": "s" }, { "label": "llama-swap routing engines", "value": 3, "unit": "engines" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/d854d98bae598388ecea1ed84396587e5edc326d/tools/server/server-models.cpp", "https://github.com/ggml-org/llama.cpp/blob/d854d98bae598388ecea1ed84396587e5edc326d/tools/server/README.md", "https://github.com/ggml-org/llama.cpp/pull/18228", "https://github.com/mostlygeek/llama-swap/blob/v266/docs/config.example.yaml", "https://github.com/mostlygeek/llama-swap/blob/v266/README.md" ], "href": "https://amdai.net/lab-log#2026-10-25-model-swap-router" }, { "id": "2026-10-25-llamacpp-grammar-sampling-cost", "date": "2026-10-25T21:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Tool calls and JSON mode in llama.cpp: lazy grammars, rejection sampling, and why -bs switches off", "summary": "In current llama.cpp, a request with tools and tool_choice \"auto\" uses a lazy grammar that stays idle until the model starts a tool call, while tool_choice \"required\" or a response_format JSON schema constrains every token from the start. Sampling checks only the chosen token against the grammar and computes the full-vocabulary mask only when that token is rejected. Experimental backend (GPU) sampling, -bs, is turned off for any request that carries a grammar, lazy tool grammars included, so agent requests always sample on the CPU.", "setup": { "engine": "llama.cpp master d854d98bae (source read)", "models": "Qwen3.6 tokenizer (248,320 vocab)" }, "metrics": [ { "label": "llama.cpp native grammar mask, MaskBench avg", "value": 16.6, "unit": "ms" }, { "label": "LLGuidance mask, MaskBench avg", "value": 0.064, "unit": "ms" }, { "label": "-bs throughput gain at 32 slots (RTX 5090)", "value": 48, "unit": "%" }, { "label": "Qwen3.6 vocab size", "value": 248320, "unit": "tokens" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/d854d98bae598388ecea1ed84396587e5edc326d/common/sampling.cpp", "https://github.com/ggml-org/llama.cpp/blob/d854d98bae598388ecea1ed84396587e5edc326d/common/chat-auto-parser-generator.cpp", "https://github.com/ggml-org/llama.cpp/pull/18137", "https://github.com/ggml-org/llama.cpp/pull/17004", "https://github.com/ggml-org/llama.cpp/pull/25532", "https://github.com/ggml-org/llama.cpp/issues/27050", "https://github.com/guidance-ai/jsonschemabench/blob/main/maskbench/README.md", "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/main/config.json" ], "href": "https://amdai.net/lab-log#2026-10-25-llamacpp-grammar-sampling-cost" }, { "id": "2026-10-25-whisper-stt-rdna4", "date": "2026-10-25T20:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Speech-to-text next to a local LLM on R9700s: use whisper.cpp, and mind the endpoint", "summary": "For voice input on RDNA4 cards, whisper.cpp (HIP or Vulkan) is the working path: a test on an R9700 ran the 11 s JFK sample with tiny.en in 206 ms, and an RX 9070 XT on Vulkan transcribed a 19.3-minute talk with large-v3-turbo in 48 s (about 24x real time, 1.5% WER). One RDNA4 tester reports the CTranslate2 ROCm forks behind faster-whisper produce wrong transcripts, so avoid that stack. The turbo model is 0.81 GiB at q8_0. whisper-server only serves /inference unless you pass --inference-path /v1/audio/transcriptions, which OpenAI-style agent clients expect.", "setup": { "engine": "whisper.cpp master 231638a (docs/source), llama.cpp PR #21863", "models": "Whisper large-v3 / large-v3-turbo ggml (f16, q8_0, q5_0)" }, "metrics": [ { "label": "R9700 tiny.en, 11 s clip, total", "value": 205.72, "unit": "ms" }, { "label": "RX 9070 XT Vulkan large-v3-turbo, 19.3 min talk", "value": 48, "unit": "s" }, { "label": "RX 9070 XT large-v3-turbo WER (TED talk)", "value": 1.5, "unit": "%" }, { "label": "Strix Halo large-v3-turbo 11 s clip, Vulkan vs HIP", "value": "0.84 / 0.63", "unit": "s" }, { "label": "large-v3-turbo q8_0 file", "value": 0.814, "unit": "GiB" } ], "sources": [ "https://github.com/ggml-org/whisper.cpp/pull/3823", "https://pypi.org/project/babelscribe/", "https://github.com/collabora/WhisperLive/issues/520", "https://github.com/maherr/onnxruntime-migraphx-rdna4/blob/main/witness/README.md", "https://github.com/ggml-org/whisper.cpp/blob/master/examples/server/server.cpp", "https://github.com/ggml-org/llama.cpp/pull/21863", "https://huggingface.co/ggerganov/whisper.cpp" ], "href": "https://amdai.net/lab-log#2026-10-25-whisper-stt-rdna4" }, { "id": "2026-10-25-flash-next-vision-cost", "date": "2026-10-25T19:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Vision on Qwen3.8-Flash-Next with R9700s: the encoder is cheap, the image tokens are not", "summary": "Flash-Next reads images through a separate 0.84 GiB projector (27-layer ViT, patch 16, merge 2, so one image token per 32x32 pixels). On an R9700 the GPU encoder takes 57 ms for a 260-token picture and 0.40 s for 972 tokens (3.13 s and 14.1 s on a CPU encoder). With llama.cpp's default cap of 4,096 tokens per image, a 1920x1080 screenshot becomes 2,040 tokens, so on a hybrid setup that prefills around 200 tok/s the image tokens, not the encoder, dominate time to first token. Capping images at 1,024 tokens halves that.", "setup": { "model": "Qwen3.8-Flash-Next GGUF + mmproj-BF16/F16 (Unsloth)", "engine": "llama.cpp master d854d98bae (mtmd), Strata v0.2.6 docs" }, "metrics": [ { "label": "Projector size (F16)", "value": 0.842, "unit": "GiB" }, { "label": "R9700 GPU encode, 972-token picture", "value": 0.4, "unit": "s" }, { "label": "CPU encode, 972-token picture (EPYC)", "value": 14.1, "unit": "s" }, { "label": "1920x1080 screenshot at llama.cpp default cap", "value": 2040, "unit": "image tokens" }, { "label": "Strata GPU reader VRAM reserve", "value": 1.4, "unit": "GiB" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://github.com/ggml-org/llama.cpp/blob/master/tools/mtmd/clip.cpp", "https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp", "https://github.com/Niko1221/Strata/blob/main/docs/AMD_HIP.md", "https://github.com/Niko1221/Strata/blob/main/docs/DETAILS.md", "https://github.com/Niko1221/Strata/blob/main/docs/QUANTS.md", "https://0xbakeer.com/flash-next/vision" ], "href": "https://amdai.net/lab-log#2026-10-25-flash-next-vision-cost" }, { "id": "2026-10-25-r9700-scoreboard-band", "date": "2026-10-25T18:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 reference band on llama.cpp's community scoreboards (Llama 2 7B Q4_0)", "summary": "Published single-card R9700 results for the standard llama-bench test: Vulkan tg128 126-146 t/s (FA off) and 131-153 (FA on), pp512 5,000-5,900; ROCm/HIP 112.7 / 121.6 tg128 at build 8ea2902. A card well under this band points to a driver, power or PCIe problem, not the hardware. The ROCm scoreboard's R9700 row is stale (93.84 tg128 at bd4ef13); newer builds are about 20% faster.", "setup": { "model": "llama-2-7b.Q4_0.gguf (3.56 GiB)", "command": "llama-bench -ngl 99 -fa 0,1 -sm none (one GPU)", "engine": "llama.cpp Vulkan and HIP, various builds" }, "metrics": [ { "label": "R9700 Vulkan tg128, FA on (best published)", "value": 152.7, "unit": "tok/s" }, { "label": "R9700 Vulkan tg128, FA on (8ea2902)", "value": 131.19, "unit": "tok/s" }, { "label": "R9700 ROCm tg128, FA on (8ea2902)", "value": 121.62, "unit": "tok/s" }, { "label": "R9700 ROCm scoreboard row tg128, FA off (stale bd4ef13)", "value": 93.84, "unit": "tok/s" }, { "label": "RTX 3090 Vulkan tg128, FA on", "value": 171.61, "unit": "tok/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/discussions/10879", "https://github.com/ggml-org/llama.cpp/discussions/15021" ], "href": "https://amdai.net/lab-log#2026-10-25-r9700-scoreboard-band" }, { "id": "2026-10-25-r9700-driver-stack-traps", "date": "2026-10-25T17:15:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 'regressions' that are really the driver stack: AMDVLK vs RADV, Mesa version, and a silent SMU hang", "summary": "A review of public R9700 (gfx1201) reports found that the biggest swings come from the driver stack, not model flags. AMDVLK's Vulkan driver can silently take priority over RADV: one reported -39% MoE prefill regression was really AMDVLK, and on an R9700 AMDVLK's dense prefill was about 4x slower than RADV (207 vs 823 tok/s on a 27B). One tester saw Mesa 25.3.6 give +48% prefill on a 35B MoE over Ubuntu's 25.2.8. Two alarming R9700 Vulkan reports (an 8x decode drop tied to one llama.cpp PR, and a Qwen3.6-35B decode cliff after ~32 tokens) were never confirmed. The real safety gap: a hung power-management controller can leave a card unreadable by every sensor while it can still draw 300 W.", "setup": { "gpus": "Radeon AI PRO R9700 (gfx1201)", "engine": "llama.cpp Vulkan (RADV / AMDVLK), amdgpu kernel driver", "engine_version": "per source", "model": "various (per source)", "quant": "per source" }, "metrics": [ { "label": "AMDVLK dense 27B prefill on R9700", "value": 207, "unit": "tok/s" }, { "label": "RADV dense 27B prefill on R9700", "value": 823, "unit": "tok/s" }, { "label": "Mesa 25.3.6 vs 25.2.8, 35B MoE pp2048 (one tester)", "value": 48, "unit": "%" }, { "label": "Decode drop after MES 0x8B package (one dual-R9700 report)", "value": 10.8, "unit": "%" }, { "label": "Power a sensor-blind card can still draw", "value": 300, "unit": "W" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/discussions/21043", "https://github.com/ggml-org/llama.cpp/issues/22375", "https://github.com/ggml-org/llama.cpp/issues/26679", "https://github.com/ggml-org/llama.cpp/issues/26795", "https://github.com/ROCm/ROCm/issues/6101" ], "href": "https://amdai.net/lab-log#2026-10-25-r9700-driver-stack-traps" }, { "id": "2026-10-25-ep-vs-tp-moe-imbalance", "date": "2026-10-25T16:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "vLLM expert parallel on 2x R9700: no extra PCIe traffic, but ~25% routing imbalance at batch 1", "summary": "On one box, vLLM's --enable-expert-parallel with TP=2 and DP=1 still uses all-reduce, not all-to-all, so it adds no PCIe traffic; it only changes how experts are placed. A routing simulation for Qwen3.8-Flash-Next (512 experts, top-10) shows the busier card reads 1.25x the routed-expert bytes of plain sharded TP at batch 1 (p95 1.60x), shrinking to 1.04x at 32 concurrent streams. On a 4-card setup it is 1.68x at batch 1. For single-stream chat, keep plain -tp 2; try EP only for many-slot batch serving, and never DP+EP over PCIe.", "setup": { "model": "Qwen3.8-Flash-Next (config), 256-expert top-8 class for comparison", "engine": "vLLM (behavior per docs and AMD blog)", "gpus": "2x Radeon AI PRO R9700 (also modeled 4x)", "flags": "-tp 2 vs -tp 2 --enable-expert-parallel" }, "metrics": [ { "label": "EP/TP slowest-card expert bytes, 2 GPUs, batch 1", "value": 1.25, "unit": "x" }, { "label": "same, batch 32", "value": 1.04, "unit": "x" }, { "label": "same, 4 GPUs, batch 1", "value": 1.68, "unit": "x" }, { "label": "AMD MI300X: TP8+EP vs DP8+EP throughput at concurrency 64 (DeepSeek-R1)", "value": 52, "unit": "% higher" } ], "sources": [ "https://rocm.blogs.amd.com/software-tools-optimization/vllm-moe-guide/README.html", "https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment/", "https://github.com/vllm-project/vllm/blob/main/vllm/config/parallel.py", "https://0xsero.github.io/blackwell-gpu-wiki/interconnect/moe-parallelism/" ], "href": "https://amdai.net/lab-log#2026-10-25-ep-vs-tp-moe-imbalance" }, { "id": "2026-10-25-new-models-moe-active-params-audit", "date": "2026-10-25T15:25:00-05:00", "project": "idea-lab", "kind": "result", "title": "Estimating MoE active parameters and GGUF sizes from Hugging Face: the k/E shortcut is ~40% low, and MTP files fake a Q4", "summary": "An audit of the 84 models on our /new-models page found that the common shortcut active = total x (experts per token / experts) underestimates active parameters by 33-43% on four models with published counts, while subtracting only the routed expert weights lands within +1 to +25%. It also found that configs nesting MoE keys under text_config made 9 of 30 MoE models (including Qwen3.8-Flash-Next, GLM-5.3-Flash and DeepSeek-V4.1-Flash) display as fully active.", "setup": { "engine": "Hugging Face Hub API + config.json", "gpus": "fit tiers computed for 2x Radeon AI PRO R9700 (32 GB each) + DDR5" }, "metrics": [ { "label": "k/E shortcut error vs official active params (4 models)", "value": "-33 to -43", "unit": "%" }, { "label": "Shape-method error vs official active params (4 models)", "value": "+1 to +25", "unit": "%" }, { "label": "MoE models shown as fully active (nested text_config)", "value": "9 of 30", "unit": "models" }, { "label": "Qwen3.8-Flash-Next UD-Q4_K_XL, shards summed", "value": "103.7", "unit": "GiB" }, { "label": "MTP draft head misread as Q4_K_M", "value": "3.9", "unit": "GiB" }, { "label": "MiniMax-M3.1-Flash Q4: estimate vs measured", "value": "124.5 vs 232.3", "unit": "GiB" } ], "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/config.json", "https://huggingface.co/Qwen/Qwen3-30B-A3B/blob/main/config.json", "https://huggingface.co/Qwen/Qwen3-235B-A22B/blob/main/config.json", "https://huggingface.co/zai-org/GLM-4.5-Air/blob/main/config.json", "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/main/config.json", "https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF?blobs=true", "https://huggingface.co/unsloth/MiniMax-M3.1-Flash-GGUF" ], "href": "https://amdai.net/lab-log#2026-10-25-new-models-moe-active-params-audit" }, { "id": "2026-10-25-measured-quant-quality-speed", "date": "2026-10-25T15:15:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Measured: Qwen 3.8 Flash quants on dual R9700, quality vs speed", "summary": "Measured on the local desktop (2x R9700, 262K f16 KV, reference Q8_0). Strata 0.2.3.1 UD-Q5_K_XL decodes 58 tok/s short / 52 at 36K with 2,192 tok/s prefill and 24/26 tool calls, so it stays the default. llama.cpp hybrid decode was only 17-22 tok/s (UD-Q5 17.6/17.1/14.2 at 1K/36K/163K; UD-Q4 21.2/19.9/16.0); our earlier bandwidth-model estimate of ~34-45 tok/s was far too optimistic. UD-Q5 drifts least from Q8_0 (KLD 0.036 text / 0.019 code vs 0.060/0.034 for UD-Q4).", "setup": null, "metrics": [], "sources": [], "href": "https://amdai.net/lab-log#2026-10-25-measured-quant-quality-speed" }, { "id": "2026-10-25-strata-mmap-experts-ab", "date": "2026-10-25T15:14:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Measured: Strata mmap-experts on vs off", "summary": "On the local desktop with Strata UD-Q5_K_XL, mmap-experts=on left decode unchanged, halved load time (~27 s vs 48-61 s) and cut process RAM from 109 to 19 GB, but prefill dropped 37% (1,377 vs 2,189 tok/s at 36K). Use it when RAM or load time matters more than long-prompt ingest.", "setup": null, "metrics": [], "sources": [], "href": "https://amdai.net/lab-log#2026-10-25-strata-mmap-experts-ab" }, { "id": "2026-10-25-llamacpp-fit-autoplace", "date": "2026-10-25T14:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "llama.cpp --fit on dual R9700 + DDR5: what it changes silently, and how to A/B it", "summary": "llama.cpp's automatic placement (--fit) is on by default. It only adjusts arguments you left unset: set any of -ngl, -ts or -ot and placement is entirely yours. When a model does not fit it shrinks the context first (down to 4096 by default) before moving tensors to RAM, so a launcher without an explicit -c can quietly lose its long context.", "setup": { "engine": "llama.cpp", "engine_version": "master 6050b467 (2026-10-25)", "gpus": "2x Radeon AI PRO R9700 (applies to any multi-GPU + RAM setup)" }, "metrics": [ { "label": "Default --fit-ctx floor", "value": 4096, "unit": "tokens" }, { "label": "Default --fit-target margin per device", "value": 1024, "unit": "MiB" }, { "label": "Upstream 2x vs 1x RTX 4090 pp4096, Qwen3 Next q8_0 (RAM-bound)", "value": "574 vs 904", "unit": "t/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/discussions/18049", "https://github.com/ggml-org/llama.cpp/pull/18679", "https://github.com/ggml-org/llama.cpp/issues/22592", "https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md" ], "href": "https://amdai.net/lab-log#2026-10-25-llamacpp-fit-autoplace" }, { "id": "2026-10-25-hip-graphs-r9700", "date": "2026-10-25T13:10:00-05:00", "project": "dual-r9700", "kind": "progress", "title": "HIP graphs on R9700 in llama.cpp: the real off switch, the new knobs, and a build trap", "summary": "llama.cpp has captured decode as HIP graphs by default since April 2026 (upstream cites +3-14% decode on other AMD GPUs). The working runtime off switch is GGML_CUDA_DISABLE_GRAPHS; a dual-gfx1201 bug report concluded env vars can't disable graphs, but it tried GGML_CUDA_NO_GRAPHS, which the code never reads. Two new knobs are worth an A/B: GGML_CUDA_GRAPH_CHUNK_SIZE and GGML_CUDA_GRAPH_OPT=1.", "setup": { "engine": "llama.cpp (source read)", "engine_version": "master 6050b467 (2026-10-25)", "gpus": "Radeon AI PRO R9700 (gfx1201)", "flags": "GGML_HIP_GRAPHS=ON (default)" }, "metrics": [ { "label": "Upstream decode gain from HIP graphs, qwen3moe 30B-A3B Q4_K_M (gfx908)", "value": 12, "unit": "%" }, { "label": "Upstream decode gain from HIP graphs, gfx1100 range", "value": "3-7", "unit": "%" }, { "label": "GRAPH_OPT=1 MoE decode gain, 2 GPUs (CUDA)", "value": "3.4-3.7", "unit": "%" }, { "label": "Graph chunking decode gain (CUDA)", "value": "1-2", "unit": "%" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/22254", "https://github.com/ggml-org/llama.cpp/issues/23579", "https://github.com/ggml-org/llama.cpp/pull/30804", "https://github.com/ggml-org/llama.cpp/pull/31053", "https://github.com/ggml-org/llama.cpp/pull/28198", "https://github.com/ggml-org/llama.cpp/pull/30327", "https://github.com/ggml-org/llama.cpp/issues/25082", "https://github.com/ggml-org/llama.cpp/blob/6050b467c07a95c7b87debd4ddac2f8ba8d9b807/ggml/src/ggml-cuda/ggml-cuda.cu" ], "href": "https://amdai.net/lab-log#2026-10-25-hip-graphs-r9700" }, { "id": "2026-10-25-easy-apps-dual-r9700-traps", "date": "2026-10-25T12:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "Ollama, LM Studio and Lemonade on two R9700s: what they do with the second card, and five traps", "summary": "From the apps' docs and bug trackers: Ollama puts a model that fits on one GPU onto one GPU only and defaults to a 4096-token context. A Ryzen desktop iGPU (gfx1036) can show up as an extra ROCm device, and llama.cpp-based apps that auto-split onto it segfault; list only the cards with --device ROCm0,ROCm1. Ollama MoE models crash on mixed R9700 + other Radeon pairs because the rocBLAS fix is not in any ROCm 7.2.x release, which Ollama bundles.", "setup": { "gpus": "2x Radeon AI PRO R9700", "engine": "Ollama / LM Studio / Lemonade (llama.cpp-based)", "engine_version": "per source" }, "metrics": [ { "label": "Ollama default context", "value": 4096, "unit": "tokens" }, { "label": "Ollama default parallel requests per model", "value": 1, "unit": "" }, { "label": "Kernel with gfx12 idle-power MES fix", "value": "6.12+", "unit": "" }, { "label": "Reported idle draw with the bug (per card)", "value": 300, "unit": "W" } ], "sources": [ "https://docs.ollama.com/gpu", "https://docs.ollama.com/faq", "https://github.com/ollama/ollama/issues/18162", "https://github.com/lemonade-sdk/llamacpp-rocm/issues/96", "https://lemonade-server.ai/docs/dev/backends-reference/", "https://github.com/lmstudio-ai/lmstudio-bug-tracker/issues/1991", "https://github.com/lmstudio-ai/lmstudio-bug-tracker/issues/1916", "https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md" ], "href": "https://amdai.net/lab-log#2026-10-25-easy-apps-dual-r9700-traps" }, { "id": "2026-10-25-longctx-quant-evidence", "date": "2026-10-25T11:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Short-context KL can't settle Q4 vs Q5 vs Q6 for Flash models at 128K+", "summary": "Published studies show the 4-bit penalty grows with context. 8-bit stays near-lossless, but 4-bit methods lose up to 16% on RULER at 64K+ inputs and decline further past 75K. The public long-context evidence for Qwen3.8-Flash-Next at Q4 vs Q6 is only 4 items each, so our UD-Q5_K_XL sweet-spot pick is unproven past short context.", "setup": null, "metrics": [ { "label": "Worst 4-bit drop on RULER/OneRULER, 64K+ inputs (published)", "value": 16, "unit": "%" }, { "label": "Average 8-bit accuracy drop, long-context study", "value": 0.8, "unit": "%" }, { "label": "Flash-Next UD-Q4_K_XL mean KLD (Unsloth)", "value": 0.0469, "unit": "" }, { "label": "Flash-Next UD-Q5_K_XL mean KLD (Unsloth)", "value": 0.0304, "unit": "" } ], "sources": [ "https://arxiv.org/html/2505.20276v3", "https://developers.redhat.com/articles/2024/02/03/how-well-do-quantized-models-handle-long-context-tasks", "https://egordmitriev.dev/blog/2026-08-12-does-precision-loss-break-retrieval-or-does-distance", "https://unsloth.ai/docs/models/qwen3.8-next", "https://peaster.io/articles/local-LLMs/qwen-3-8-flash-next-first-impressions" ], "href": "https://amdai.net/lab-log#2026-10-25-longctx-quant-evidence" }, { "id": "2026-10-25-llama-rpc-two-box-build-parity", "date": "2026-10-25T10:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Two-box R9700 with llama.cpp RPC: the official Docker images can't do it, and both boxes need the identical build", "summary": "Checked llama.cpp master (2026-10-25): the official ROCm and Vulkan Docker images are built without GGML_RPC, which defaults to OFF, so they contain no RPC backend or ggml-rpc-server. RPC also needs byte-identical builds on both machines: the protocol sends raw op enum values, mismatched builds have crashed after a successful handshake, and the protocol version (now 8.0.1, 102 ops) is bumped on every op change. Build one pinned image with RPC on and run it on both boxes.", "setup": { "engine": "llama.cpp RPC", "engine_version": "master 2280f0f (2026-10-25)", "gpus": "planned 2x2 Radeon AI PRO R9700 over 10GbE" }, "metrics": [ { "label": "RPC protocol version on master", "value": "8.0.1", "unit": "" }, { "label": "ggml op count the protocol is pinned to", "value": 102, "unit": "" }, { "label": "GPU targets in upstream ROCm image", "value": 11, "unit": "" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/master/.devops/rocm.Dockerfile", "https://github.com/ggml-org/llama.cpp/blob/master/ggml/CMakeLists.txt", "https://github.com/ggml-org/llama.cpp/blob/master/ggml/include/ggml-rpc.h", "https://github.com/ggml-org/llama.cpp/blob/master/tools/rpc/README.md", "https://github.com/ggml-org/llama.cpp/issues/20259", "https://github.com/ggml-org/llama.cpp/pull/20330", "https://hub.docker.com/r/rocm/dev-ubuntu-24.04/tags" ], "href": "https://amdai.net/lab-log#2026-10-25-llama-rpc-two-box-build-parity" }, { "id": "2026-10-25-vllm-r9700-moe-config-names", "date": "2026-10-25T09:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "vLLM on R9700: tuned MoE configs named 'AI_PRO_R9700' never load, and Flash-Next has no tuned config", "summary": "vLLM looks up fused-MoE tile configs by exact filename. Since PR #38086 (merged 2026-04-02) it names the R9700 'AMD_Radeon_R9700' (PCI id 0x7551), so config files still circulating as 'AMD_Radeon_AI_PRO_R9700' are silently ignored. Upstream ships only three R9700 MoE configs and none for Qwen3.8-Flash-Next's 512-expert shapes, and the stock tuner script rejects the Flash-Next architecture.", "setup": { "model": "Qwen3.8-Flash-Next (W4A16 int4 packs)", "engine": "vLLM main (source read 2026-10-25)", "gpus": "Radeon AI PRO R9700 (gfx1201)" }, "metrics": [ { "label": "Upstream R9700 fused-MoE config files", "value": 3, "unit": "files" }, { "label": "Per-layer speed-up, tuned R9700 int4 MoE (other model)", "value": "1.05-1.46", "unit": "x" }, { "label": "TPOT change, tuned R9700 FP8 MoE (other model)", "value": "-8 to -24", "unit": "%" } ], "sources": [ "https://github.com/vllm-project/vllm/pull/38086", "https://github.com/vllm-project/vllm/pull/36659", "https://github.com/vllm-project/vllm/pull/60893", "https://github.com/vllm-project/vllm/issues/50436", "https://github.com/vllm-project/vllm/blob/main/vllm/platforms/rocm.py", "https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py" ], "href": "https://amdai.net/lab-log#2026-10-25-vllm-r9700-moe-config-names" }, { "id": "2026-10-25-rocm-pinned-memory-thp", "date": "2026-10-25T08:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "ROCm's 'pinned' host memory is pageable, and transparent huge pages can make a big-MoE start take 5 minutes instead of 35 seconds", "summary": "On Linux, ROCm implements hipHostMalloc as an anonymous mmap with MADV_HUGEPAGE plus a KFD userptr mapping, so the pages are not locked (VmLck and VmPin stay 0). In a reported 2x RX 7900 XTX test, page-locking a 28.47 GiB expert cache took a median 300.5 s with THP at the default madvise setting versus 34.0 s with THP disabled for the engine process (4 interleaved starts each), with decode and prompt speed unchanged. For anyone keeping 30-100 GiB of MoE experts in DDR5 next to Radeon GPUs, starting the engine with THP off is a free fix for slow, variable starts.", "setup": { "gpus": "2x RX 7900 XTX (start-time test); 1x RX 7900 XTX (stall test)", "engine": "Strata (HIP build)", "engine_version": "0.1.44.1 (start test), 0.1.48.1 + patch (stall test)", "rocm": "7.2.4 (start test)", "flags": "THP enabled=madvise, defrag=madvise" }, "metrics": [ { "label": "Expert cache page-lock, THP default (median of 4)", "value": 300.5, "unit": "s" }, { "label": "Expert cache page-lock, THP off for the process (median of 4)", "value": 34, "unit": "s" }, { "label": "Load to ready, THP default (median of 4)", "value": 366.5, "unit": "s" }, { "label": "Same allocation in 15 x 2 GiB pieces (fast state)", "value": 3.6, "unit": "s" } ], "sources": [ "https://github.com/Niko1221/Strata/issues/2850", "https://github.com/Niko1221/Strata/pull/3913", "https://man7.org/linux/man-pages/man2/PR_SET_THP_DISABLE.2const.html", "https://rocm.docs.amd.com/en/docs-6.3.2/conceptual/iommu.html", "https://lists.openwall.net/linux-kernel/2026/06/25/875" ], "href": "https://amdai.net/lab-log#2026-10-25-rocm-pinned-memory-thp" }, { "id": "2026-10-25-r9700-fp8-fp4-kernel-paths", "date": "2026-10-25T07:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 has no FP4 hardware, and vLLM 0.28+ picks a slower FP8 kernel on it by default; one env var gets ~10% back", "summary": "RDNA4 (gfx1201) has matrix instructions for FP16/BF16 (191 TFLOPS), FP8 and INT8 (383) and INT4 (766 TOPS), but none for FP4, so MXFP4/NVFP4 models are unpacked to 16-bit inside the kernel: you get the memory savings, not 4-bit math. On vLLM 0.28 and later, per-token FP8 checkpoints on an R9700 select RowWiseTorchFP8ScaledMMLinearKernel; setting VLLM_DISABLED_KERNELS=RowWiseTorchFP8ScaledMMLinearKernel switches to ChannelWise and, in a reported single-R9700 test, raised decode from 34.71 to 38.48 tok/s (+10.9%) and prefill from 4,152 to 4,531 tok/s (+9.1%) with GSM8K unchanged within noise.", "setup": { "engine": "vLLM (ROCm) and llama.cpp HIP, cited", "engine_version": "vLLM 0.28-0.29.1.dev", "gpus": "Radeon AI PRO R9700 (gfx1201)", "quant": "FP8 W8A8 per-token; MXFP4; GGUF k-quants (all >= 4-bit)" }, "metrics": [ { "label": "Decode, RowWise default", "value": 34.71, "unit": "tok/s" }, { "label": "Decode, RowWise disabled (ChannelWise)", "value": 38.48, "unit": "tok/s" }, { "label": "Prefill, RowWise default", "value": 4152, "unit": "tok/s" }, { "label": "Prefill, RowWise disabled", "value": 4531, "unit": "tok/s" }, { "label": "FP8 matrix peak (dense)", "value": 383, "unit": "TFLOPS" }, { "label": "FP4 matrix peak", "value": 0, "unit": "TFLOPS (no FP4 instruction)" } ], "sources": [ "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro/ai-9000-series/amd-radeon-ai-pro-r9700.html", "https://rocm.docs.amd.com/projects/rocWMMA/en/develop/api-reference/data-type-support.html", "https://github.com/vllm-project/vllm/issues/57838", "https://github.com/vllm-project/vllm/pull/61091", "https://github.com/vllm-project/vllm/pull/43615", "https://github.com/vllm-project/vllm/pull/37826", "https://github.com/ggml-org/llama.cpp/pull/17156" ], "href": "https://amdai.net/lab-log#2026-10-25-r9700-fp8-fp4-kernel-paths" }, { "id": "2026-10-25-rdna4-q6k-hip-prefill", "date": "2026-10-25T06:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp HIP prefills Q6_K at about half speed on RDNA4, which makes UD-Q6 Flash packs ~1.6x heavier than Q5 (fix in an open PR)", "summary": "On RDNA4 (gfx1201, same chip as the R9700), llama.cpp's HIP matrix kernel for Q6_K runs at 36.6 TFLOPS versus 75-80 for Q4_K, Q5_K and Q8_0. Open PR #25940 lifts it to 69.7 (1.9x), with prefill gains of +68-82% on a pure Q6_K 8B model and +28-37% on R9700 Q6_K models; token generation is unchanged. For Qwen3.8-Flash-Next, 80% of UD-Q6_K_XL's expert-matmul time is Q6_K, so by our estimate its GPU expert prefill work is ~1.58x that of UD-Q4_K_XL on HIP master (UD-Q5_K_XL 0.94, AP-Q5_K_M 0.93), dropping to ~0.97 with the PR.", "setup": { "model": "Qwen3.8-Flash-Next (estimate); Llama 8B, Qwen3.6-27B, 35B MoE (cited)", "quant": "Q4_K / Q5_K / Q6_K / Q8_0 and UD/AP packs, all >= 4-bit", "engine": "llama.cpp HIP (ROCm)", "engine_version": "master vs PR #25940", "gpus": "RX 9070 XT and Radeon AI PRO R9700 (gfx1201)", "context": "pp512-pp4096", "flags": "-ngl 99, -ub 512 / 2048" }, "metrics": [ { "label": "Q6_K MUL_MAT n=512, HIP master", "value": 36.62, "unit": "TFLOPS" }, { "label": "Q6_K MUL_MAT n=512, with PR #25940", "value": 69.66, "unit": "TFLOPS" }, { "label": "Q4_K MUL_MAT n=512, HIP master", "value": 77.69, "unit": "TFLOPS" }, { "label": "R9700 Qwen3.6-27B Q6_K pp512, master -> PR", "value": "684 -> 935", "unit": "tok/s" }, { "label": "Est. UD-Q6_K_XL expert prefill work vs UD-Q4_K_XL, HIP master", "value": 1.58, "unit": "x" }, { "label": "Est. same with PR #25940", "value": 0.97, "unit": "x" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/25940", "https://github.com/ggml-org/llama.cpp/pull/25587", "https://github.com/ggml-org/llama.cpp/issues/27181", "https://github.com/ggml-org/llama.cpp/pull/26199", "https://github.com/ggml-org/llama.cpp/pull/22970", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF" ], "href": "https://amdai.net/lab-log#2026-10-25-rdna4-q6k-hip-prefill" }, { "id": "2026-10-25-r9700-vs-arc-pro-b70", "date": "2026-10-25T04:10:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 vs Arc Pro B70: same 32 GB, but the R9700 is ~1.7-1.9x faster on MoE and ~2x on dual-card Flash at 4-bit+", "summary": "Both cards have 32 GB on a 256-bit bus (640 vs 608 GB/s), yet published results split by model type. Qwen3.6-35B-A3B Q4 on one card: R9700 165.2 tok/s (Vulkan) vs B70 97.7 (SYCL) or 87.3 (Vulkan, Mesa 26). Dense 27B Q4 is close (27.1 vs 21.4-25.3), and one dual-B70 tensor-parallel fork beats the R9700 Vulkan tensor PR on dense 27B (51.1 vs 45.0). For Qwen3.8-Flash-Next at 4-bit or higher on two cards with experts in RAM, the best B70 result we found is 23-33 tok/s against 52-65 (Q5, our measured) and 87-94 at 36K (vLLM MXFP4, our measured) on two R9700s.", "setup": { "model": "Qwen3.6-35B-A3B, Qwen3.8-27B, Qwen3.8-Flash-Next", "quant": "Q4 to Q5 GGUF, MXFP4 (all >= 4-bit)", "engine": "llama.cpp Vulkan/SYCL/ROCm, Strata, vLLM (per source)", "engine_version": "per source", "gpus": "1-2x Radeon AI PRO R9700 vs 1-2x Arc Pro B70", "context": "short unless noted", "flags": "per source" }, "metrics": [ { "label": "35B-A3B Q4 decode, 1 card, R9700 Vulkan", "value": 165.2, "unit": "tok/s" }, { "label": "35B-A3B Q4 decode, 1 card, B70 SYCL / Vulkan Mesa 26", "value": "97.7 / 87.3", "unit": "tok/s" }, { "label": "27B Q4 decode, 1 card, R9700 / B70", "value": "27.1 / 21.4-25.3", "unit": "tok/s" }, { "label": "27B Q4_K_M decode, 2 cards tensor, R9700 / B70", "value": "45.0 / 51.1", "unit": "tok/s" }, { "label": "Flash-Next 4-bit+ decode, 2 cards, best B70 found", "value": "23.0-33.3", "unit": "tok/s" }, { "label": "Flash-Next decode at 36K, 2x R9700 vLLM MXFP4 (measured)", "value": "87-94", "unit": "tok/s" }, { "label": "Est. USD per decode tok/s, 2-card Flash GGUF, R9700 / B70", "value": "55 / 78-117", "unit": "USD" } ], "sources": [ "https://riceharvest.github.io/awesome-b70/performance.html", "https://www.pugetsystems.com/labs/articles/intel-arc-pro-b70-multi-gpu-ai-inference-performance/", "https://github.com/0xSero/qwen38-b70", "https://www.intelinside.ai/runtimes/llamacpp", "https://www.youtube.com/watch?v=pNYNRUOnPeU", "https://github.com/ggml-org/llama.cpp/pull/25051", "https://github.com/Niko1221/Strata/blob/main/docs/QUANTS.md", "https://www.newegg.com/p/pl?N=100008333+601191745" ], "href": "https://amdai.net/lab-log#2026-10-25-r9700-vs-arc-pro-b70" }, { "id": "2026-10-25-flash-next-cold-ngram-table", "date": "2026-10-25T03:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Flash-Next's 50 GiB n-gram table is read lazily from disk cache: a cold cache can halve prefill, and 256 GB of DDR5 avoids it", "summary": "Qwen3.8-Flash-Next GGUFs carry one huge per-layer n-gram embedding table (50.66 GiB in UD-Q5_K_XL, read from the file headers). By default llama.cpp does not load it; any tensor over 4 GiB is read row by row from the memory-mapped file during prefill. Published tests show a cold table costs 2.2x prefill on real text (2825 vs 1293 tok/s at UD-Q4_K_XL) and ~5% decode. The direct-read fix was closed unmerged, so on a box with enough RAM the options are keeping the table warm in page cache or loading it with --lazy-mode off. Separately, llama.cpp removed --no-mmap, --mmap, --mlock and -dio in September; launchers must use --load-mode.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL", "engine": "llama.cpp (cited results on NVIDIA; source read on master b3fa69955e)", "engine_version": "master 2026-10-25", "gpus": "cited: RTX PRO 6000; target: 2x Radeon AI PRO R9700", "context": "24,576-token prompt (cited)", "flags": "--lazy-mode auto vs off; --load-mode" }, "metrics": [ { "label": "UD-Q5_K_XL n-gram table size", "value": 50.66, "unit": "GiB" }, { "label": "Prefill, table in RAM (cited, UD-Q4_K_XL)", "value": 2825, "unit": "tok/s" }, { "label": "Prefill, cold mmap table (cited, UD-Q4_K_XL)", "value": 1293, "unit": "tok/s" }, { "label": "Decode, table in RAM vs cold (cited)", "value": "90.0 vs 85.9", "unit": "tok/s" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/tree/main/UD-Q5_K_XL", "https://github.com/ggml-org/llama.cpp/pull/28136", "https://github.com/ggml-org/llama.cpp/pull/29030", "https://github.com/ggml-org/llama.cpp/pull/26081", "https://github.com/ggml-org/llama.cpp/pull/20834", "https://github.com/ggml-org/llama.cpp/pull/28334", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-model-loader.cpp" ], "href": "https://amdai.net/lab-log#2026-10-25-flash-next-cold-ngram-table" }, { "id": "2026-10-25-r9700-thermal-gating", "date": "2026-10-25T01:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "One badly cooled R9700 can silently cap a dual-card setup: watch the hotspot-to-edge gap", "summary": "A public 2x R9700 investigation traced a ~6-10% per-card prefill deficit to a poor thermal interface, not weak silicon or power delivery. The bad card hit its ~110 C junction limit at ~230 W instead of ~300 W, with a 47 C hotspot-to-edge gap versus ~37-38 C on a healthy card. Because tensor-parallel runs move in lockstep, it dragged the pair's result down too, and the stock fan curve never reacted because it follows edge temperature.", "setup": { "model": "per source (vLLM FP8 models; 35B MoE GGUF)", "quant": "per source", "engine": "vLLM ROCm tp=2, llama.cpp (published)", "engine_version": "ROCm 7.14, amdgpu DKMS 6.19.4 (thermal investigation)", "gpus": "2x Radeon AI PRO R9700 (investigation); 1x in reviews", "context": "pp131072 deep prefill", "flags": "stock fan curve, 300 W cap" }, "metrics": [ { "label": "Hotspot-to-edge gap, defective card", "value": 47, "unit": "C" }, { "label": "Hotspot-to-edge gap, same card after service", "value": 37.4, "unit": "C" }, { "label": "Sustained board power, defective card", "value": 230, "unit": "W" }, { "label": "Sustained board power, after service", "value": 297, "unit": "W" }, { "label": "Deep prefill gain after service", "value": 8.4, "unit": "%" } ], "sources": [ "https://github.com/daimonionnn/amd-r9700-vllm-and-tuning-toolkit/blob/main/docs/r9700-mem-vendor-bios-variance.md", "https://github.com/daimonionnn/amd-r9700-vllm-and-tuning-toolkit", "https://docs.kernel.org/gpu/amdgpu/thermal.html", "https://www.igorslab.de/en/amd-radeon-ai-r9700-alias-rx-9070xt-in-the-big-workstation-graphics-card-test-what-are-the-benefits-of-double-the-memory-at-work/14/", "https://pendakwah.tech/reviews/r9700ai/", "https://branetechnologies.com/blog/local-35b-model-gpu-thermals-waterblock", "https://slybase.com/en/gpu-inference-tuning-same-performance-quieter-cheaper/" ], "href": "https://amdai.net/lab-log#2026-10-25-r9700-thermal-gating" }, { "id": "2026-10-25-gpu-max-hw-queues", "date": "2026-10-25T00:40:00-05:00", "project": "dual-r9700", "kind": "result", "title": "GPU_MAX_HW_QUEUES on the R9700: one variable behind idle-busy power, spawn-lottery decode and slow small kernels", "summary": "Four public R9700 (gfx1201) reports point at HIP's hardware-queue count. With it uncapped, idle HIP servers have sat at 100% gfx and 92-95 W per card (=1 gave 21-22 W), vLLM decode landed at 33 or 26 tok/s depending on process start, and small kernels ran ~17 us slower behind pending cross-stream waits. Capping it cost nothing measurable in the one test that checked (0.1% decode, 0.3% prefill).", "setup": { "model": "various (Qwen3-14B FP8, Gemma 4 12B FP8, Qwen3.x 35B GGUF, Flash-Next via Strata)", "quant": "per source", "engine": "llama.cpp HIP, vLLM ROCm, Strata (published)", "engine_version": "ROCm 7.2.x to 10, per source", "gpus": "1-3x Radeon AI PRO R9700", "context": "per source", "flags": "GPU_MAX_HW_QUEUES unset vs 1 vs 2" }, "metrics": [ { "label": "Idle power per card, HIP idle-busy (model loaded)", "value": 95, "unit": "W" }, { "label": "Idle power per card, GPU_MAX_HW_QUEUES=1", "value": 22, "unit": "W" }, { "label": "Idle power per card, Vulkan reference (ROCm 7.14 report)", "value": 12.5, "unit": "W" }, { "label": "vLLM decode, fast vs slow spawn (single R9700)", "value": "33.2 / 26.0", "unit": "tok/s" }, { "label": "Extra time per small kernel with pending cross-stream waits", "value": 17, "unit": "us" }, { "label": "Cost of =1 on decode (measured, noise-level)", "value": 0.1, "unit": "%" }, { "label": "Est. yearly card energy, 2 cards idle-busy vs capped", "value": "1664 / 385", "unit": "kWh" } ], "sources": [ "https://github.com/ROCm/ROCm/issues/5706", "https://github.com/ROCm/ROCm/issues/6453", "https://github.com/ROCm/legacy-rocm-build/issues/6347", "https://www.eia.gov/electricity/monthly/epm_table_grapher.php?t=epmt_5_6_a" ], "href": "https://amdai.net/lab-log#2026-10-25-gpu-max-hw-queues" }, { "id": "2026-10-24-pp-host-experts-gate", "date": "2026-10-24T23:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp PR #29963 runs multi-GPU pipeline parallelism with MoE experts in RAM (+155% prefill at 32K on ROCm), but the usual --n-cpu-moe layout can never trigger it", "summary": "Today llama.cpp switches pipeline parallelism off whenever tensor overrides put MoE experts in system RAM, so expert uploads go through the first GPU and the GPUs take turns. Open PR #29963 lets each GPU upload its own layers' experts. A ROCm test on 4x MI50 (DeepSeek-V4-Flash MXFP4) went from 122.5 to 312.5 tok/s prefill at a 32K prompt (+155%) and +28% at 8K, with decode up about 1.5%. Reading the PR code, it only engages when no single GPU's layer range owns more than 2/3 of the RAM-resident expert bytes. On two GPUs, --n-cpu-moe K puts layers 0..K-1 in RAM, and those all belong to the first card, so the gate fails for every Q4-Q6 Flash-Next layout. A spread override list (half the RAM-resident layers taken from each card's slice) passes it.", "setup": { "model": "Qwen3.8-Flash-Next (layout analysis); DeepSeek-V4-Flash and Qwen3.5-35B-A3B (cited tests)", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (analysis); MXFP4, Q8_0 (cited)", "engine": "llama.cpp PR #29963 (head 3ca7eed, open)", "gpus": "2x Radeon AI PRO R9700 (analysis); 4x MI50 ROCm 7.14 and 3x RTX PRO 4000 (cited)", "context": "131072-262144", "flags": "-sm layer, experts in RAM via -ot spread over both cards, --load-mode none" }, "metrics": [ { "label": "ROCm 4x MI50 prefill at 32K prompt, master", "value": 122.5, "unit": "tok/s" }, { "label": "ROCm 4x MI50 prefill at 32K prompt, PR #29963", "value": 312.5, "unit": "tok/s" }, { "label": "ROCm 4x MI50 prefill gain at 8K prompt", "value": 28, "unit": "%" }, { "label": "ROCm 4x MI50 decode change", "value": 1.5, "unit": "%" }, { "label": "Share of RAM experts on card 1 with --n-cpu-moe 24 (2 cards)", "value": 100, "unit": "%" }, { "label": "Gate limit per card", "value": 66.7, "unit": "%" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/29963", "https://github.com/ggml-org/llama.cpp/pull/30449", "https://github.com/ggml-org/llama.cpp/pull/24156" ], "href": "https://amdai.net/lab-log#2026-10-24-pp-host-experts-gate" }, { "id": "2026-10-24-r9700-ecc-vram", "date": "2026-10-24T22:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "R9700 ECC hides about 2 GiB of VRAM per card; turning it off is worth an estimated 4-12% on Flash-Next across two cards", "summary": "Radeon AI PRO R9700s ship with VRAM ECC (GECC) on, so Linux shows about 30,576 MiB per card instead of 32,768. Disabling it with the amdgpu.ras_enable=0 kernel parameter (two reboots) frees about 2.1 GiB per card. On a calibrated bandwidth model for Qwen3.8-Flash-Next with experts in DDR5, that moves 2-3 MoE layers onto the GPUs and raises the decode ceiling 4-7% from capacity alone, or 8-12% if a published +9% single-card decode gain from ECC off also holds.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL", "engine": "llama.cpp (model, not measured)", "gpus": "2x Radeon AI PRO R9700", "context": "262144, f16 KV", "flags": "layer split, experts of the first N layers in DDR5 (-ncmoe), no MTP" }, "metrics": [ { "label": "Visible VRAM per card, ECC on", "value": 30576, "unit": "MiB" }, { "label": "Visible VRAM per card, ECC off", "value": 32768, "unit": "MiB" }, { "label": "UD-Q5_K_XL decode ceiling, ECC on (estimate)", "value": 32.1, "unit": "tok/s" }, { "label": "UD-Q5_K_XL decode ceiling, ECC off (estimate)", "value": 34.2, "unit": "tok/s" }, { "label": "Published single-R9700 decode gain, ECC off", "value": 9, "unit": "%" } ], "sources": [ "https://github.com/ROCm/ROCm/issues/5832", "https://lunnova.dev/articles/amdgpu-disabling-ecc/", "https://forum.level1techs.com/t/sapphire-radeon-r9700-not-32gb/244649", "https://zolotukhin.ai/zinc/docs/rdna4-tuning/", "https://github.com/daimonionnn/amd-r9700-vllm-and-tuning-toolkit/blob/main/docs/r9700-oc-uv-findings.md" ], "href": "https://amdai.net/lab-log#2026-10-24-r9700-ecc-vram" }, { "id": "2026-10-24-placement-aware-quant", "date": "2026-10-24T21:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Q5 experts on GPU, Q4 in DDR5: placement-aware Flash-Next quant only wins if late layers matter more", "summary": "A custom Flash-Next GGUF with UD-Q5 experts on the VRAM-resident layers and UD-Q4 on the DDR5 layers is estimated at ~39 tok/s ceiling on 2x R9700 (UD-Q4 46, UD-Q5 34), but with equal per-layer sensitivity its quality lands right on the Q4-Q5 line: extra GPU bits cost VRAM, which pushes layers to DDR5. Unsloth's public imatrix shows routed-expert inputs in layers 36-47 carry 4-19x the median energy, and llama.cpp's --n-cpu-moe keeps exactly those on the GPU, so it may land near Q5 quality at +14% speed. Needs a KLD measurement.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / custom mix (all >=4-bit)", "engine": "llama.cpp (model)", "gpus": "2x Radeon AI PRO R9700", "context": "262K", "flags": "layer split, --n-cpu-moe, f16 KV" }, "metrics": [ { "label": "GPU-Q5 / DDR5-Q4 mix, decode ceiling (est.)", "value": 39.1, "unit": "tok/s" }, { "label": "UD-Q4_K_XL ceiling (est.)", "value": 46.1, "unit": "tok/s" }, { "label": "UD-Q5_K_XL ceiling (est.)", "value": 34.2, "unit": "tok/s" }, { "label": "Mix KLD, equal-layer est.", "value": 0.038, "unit": "" }, { "label": "Mix KLD, energy-weighted est.", "value": 0.033, "unit": "" }, { "label": "Layer 47 expert-down input energy vs median", "value": 18.7, "unit": "x" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/master/tools/quantize/README.md", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-quant.cpp", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://unsloth.ai/docs/models/qwen3.8-next" ], "href": "https://amdai.net/lab-log#2026-10-24-placement-aware-quant" }, { "id": "2026-10-24-flash-depth-decay", "date": "2026-10-24T20:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Qwen3.8-Flash-Next at long context on R9700: keep f16 KV, get the QSA indexer fixes", "summary": "Flash-Next's sparse attention should keep decode nearly flat with depth, but llama.cpp issues show three separate slowdowns: a HIP-only CPU top-k fallback past 1K context (about 20 to 6 tok/s, fixed by PR #27466), an indexer that re-pooled the whole cache every token (pooled-key cache PR #28699, +9% at 63K-114K), and top-k applied as a dense mask. Quantized KV does not help: q8_0 decoded 24-30% slower than f16 at 128K-250K on this model, and on gfx1201 batch-1 quantized KV at head 256 takes a slower kernel.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL", "engine": "llama.cpp (HIP)", "engine_version": "per source", "gpus": "2x Radeon AI PRO R9700 (estimate); cited runs on gfx1151, gfx1201, A100, RTX 3090", "context": "0-262K", "flags": "layer split, -fa on, --parallel 1" }, "metrics": [ { "label": "HIP decode before top-k fix, d0 to d1024 (gfx1151)", "value": "19.6 -> 6.1", "unit": "tok/s" }, { "label": "Pooled-key cache gain at 63K / 114K (CUDA)", "value": "+9.3 / +9.4", "unit": "%" }, { "label": "q8_0 vs f16 KV decode at 128K / 250K (A100)", "value": "-24 / -30", "unit": "%" }, { "label": "q4_1 vs f16 KV decode at 32K, gfx1201 27B", "value": "20.55 vs 28.48", "unit": "tok/s" }, { "label": "Est. UD-Q4 decode at 131K, dense f16 vs sparse+pooled (2x R9700)", "value": "35 vs 44", "unit": "tok/s (estimate)" }, { "label": "f16 KV cache at 262K, both cards", "value": 6, "unit": "GiB" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/issues/27856", "https://github.com/ggml-org/llama.cpp/issues/28012", "https://github.com/ggml-org/llama.cpp/pull/28699", "https://github.com/ggml-org/llama.cpp/pull/28770", "https://github.com/ggml-org/llama.cpp/issues/28734", "https://github.com/ggml-org/llama.cpp/issues/27796", "https://github.com/ggml-org/llama.cpp/issues/30005" ], "href": "https://amdai.net/lab-log#2026-10-24-flash-depth-decay" }, { "id": "2026-10-24-flash-model-shootout", "date": "2026-10-24T19:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Which Flash MoE suits 2x R9700 + DDR5 offload? Qwen3.8-Flash-Next by 2.6-4x (bandwidth estimate)", "summary": "From byte counts read out of the Unsloth GGUF headers: Qwen3.8-Flash-Next reads 6.1-6.7 GiB per token, DeepSeek-V4-Flash 10.1, GLM-5.3-Flash 13.6-16.0 and MiniMax-M3.1-Flash 21.3. On two R9700s with experts spilled to DDR5, that puts the decode ceiling (estimate) at 45/34/30 tok/s for Qwen Q4/Q5/Q6, 17 for DeepSeek-V4 Q4, 11/9/7 for GLM Q4/Q5/Q6 and about 6 for MiniMax.", "setup": { "model": "Qwen3.8-Flash-Next, GLM-5.3-Flash, DeepSeek-V4-Flash, MiniMax-M3.1-Flash", "quant": "UD-Q4_K_XL to UD-Q6_K_XL (UD-Q8_K_XL where experts are native 4-bit)", "engine": "bandwidth model (llama.cpp layout)", "gpus": "2x Radeon AI PRO R9700", "context": "262144", "flags": "layer split, f16 KV, experts spilled to DDR5, no MTP" }, "metrics": [ { "label": "Qwen3.8-Flash-Next UD-Q5_K_XL ceiling", "value": 34.1, "unit": "tok/s (est.)" }, { "label": "DeepSeek-V4-Flash UD-Q4_K_XL ceiling", "value": 17, "unit": "tok/s (est.)" }, { "label": "GLM-5.3-Flash UD-Q5_K_XL ceiling", "value": 8.8, "unit": "tok/s (est.)" }, { "label": "MiniMax-M3.1-Flash UD-Q4_K_XL ceiling", "value": 6.4, "unit": "tok/s (est.)" }, { "label": "Bytes read per token, Qwen vs GLM (UD-Q5)", "value": "6.5 vs 14.6", "unit": "GiB" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF", "https://huggingface.co/unsloth/DeepSeek-V4-Flash-GGUF", "https://huggingface.co/unsloth/MiniMax-M3.1-Flash-GGUF", "https://huggingface.co/unsloth/DeepSeek-V4.1-Flash-1017-GGUF", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp" ], "href": "https://amdai.net/lab-log#2026-10-24-flash-model-shootout" }, { "id": "2026-10-24-reap-q4plus", "date": "2026-10-24T18:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Expert pruning could fit a Q4 Qwen3.8-Flash-Next fully on two R9700s, but it costs world knowledge", "summary": "Community REAP builds of Qwen3.8-Flash-Next drop the least-used experts (512 to 320 or 288 per layer) and keep the survivors bit-for-bit. Pruning alone does not speed up decode: with all experts on CPU, a 256-expert build decoded at 22.0 vs 21.5 tok/s for the full model (prefill 353 vs 223 tok/s). The gain comes only from fitting more experts in VRAM. Almost all public REAP files are Q2/Q3; the only Q4+ ones we found are a REAP-288 Q4_K_M (78.05 GiB) and Q5_K_M (87.37 GiB). Because each layer's experts are contiguous in a GGUF, a published 320-expert keep-list can also be applied to Unsloth's UD-Q4_K_XL by byte copy, giving an estimated 76.8 GiB file whose compute path fits two 32 GB cards (about 95% VRAM at 262K context). Our roofline estimate: about 74 tok/s decode vs about 45 for the unpruned UD-Q4_K_XL with 17 expert layers in DDR5, the same gain we estimated for adding a third card. The cost: the REAP-320 author measured general-knowledge fabrication at 25% vs 0% for the unpruned model on a 12-question probe, while HumanEval held (96.3% vs 95.7%).", "setup": null, "metrics": [ { "label": "Est. decode, UD-Q4_K_XL unpruned vs 320-expert slice (2x R9700)", "value": "45 -> 74", "unit": "tok/s (estimate)" }, { "label": "Est. decode, UD-Q5_K_XL unpruned vs 288-expert slice + q8_0 KV", "value": "34 -> 67", "unit": "tok/s (estimate)" }, { "label": "CPU-only decode, 256-expert prune vs full model", "value": "22.0 vs 21.5", "unit": "tok/s" }, { "label": "General-knowledge fabrication, REAP-320 vs unpruned (12 questions)", "value": "25 vs 0", "unit": "%" }, { "label": "HumanEval-164, REAP-320 vs unpruned UD-IQ1_M", "value": "96.3 vs 95.7", "unit": "%" } ], "sources": [ "https://huggingface.co/AnonimousA/Qwen3.8-Flash-Next-REAP-320-GGUF", "https://huggingface.co/AnonimousA/Qwen3.8-Flash-Next-REAP-256-duo-GGUF", "https://huggingface.co/sh0wie/Qwen3.8-Flash-Next-REAP-288-GGUF", "https://github.com/CerebrasResearch/reap" ], "href": "https://amdai.net/lab-log#2026-10-24-reap-q4plus" }, { "id": "2026-10-24-rdna4-fa-head256", "date": "2026-10-24T16:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Qwen 3.5/3.6/3.8 on R9700: make sure your llama.cpp ROCm build has PR #28102, or long-context prefill can run ~2x slower", "summary": "Every Qwen 3.5/3.6/3.8 model (including the Flash-Next MoE) uses attention head size 256. After llama.cpp removed its rocWMMA FlashAttention kernel (PR #26046), head 256 on RDNA4 had no tensor-core attention path and fell back to a scalar tile kernel. Reported on a single R9700 with Qwen3.6-35B-A3B Q4_K_M: pp4096 at 127K context dropped from 777 to 395 tok/s; on dual R9700s with a 122B-A10B, pp512 at 65K dropped from 571 to 344, and rocprofv3 put attention at 25.4 s vs 65.2 s. PR #28102 (merged 2026-09-11, commit 16378d93f) enabled and tuned the WMMA FlashAttention kernel for head 256 on RDNA4: Qwen3.8-27B IQ4_XS pp512 at 150K went from 164 to 457 tok/s on an R9700, and a dual-R9700 27B Q4_K_M tensor-split run measured 747.5 tok/s at 127K vs 341 before (2.2x), beating the old rocWMMA revert (600). Token generation is unchanged by both changes. If you build llama.cpp for ROCm, check `git merge-base --is-ancestor 16378d93f HEAD`.", "setup": null, "metrics": [ { "label": "27B pp512 @ 150K, before vs after #28102 (1x R9700)", "value": "164 -> 457", "unit": "tok/s" }, { "label": "27B Q4_K_M pp4096 @ 127K, 2x R9700 tensor split, old master vs post-#28102", "value": "341 -> 747.5", "unit": "tok/s" }, { "label": "35B-A3B Q4_K_M pp4096 @ 127K, rocWMMA vs post-removal tile kernel", "value": "777 -> 395", "unit": "tok/s" }, { "label": "27B MTP decode regression reported in #28867 (threshold 16 vs 64)", "value": "41.4 vs 51.7", "unit": "tok/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/28102", "https://github.com/ggml-org/llama.cpp/issues/26220", "https://github.com/ggml-org/llama.cpp/issues/28867", "https://github.com/ggml-org/llama.cpp/issues/24961" ], "href": "https://amdai.net/lab-log#2026-10-24-rdna4-fa-head256" }, { "id": "2026-10-24-two-box-4x-r9700-link", "date": "2026-10-24T15:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Pooling 4 R9700s across two desktops: use 10GbE, not USB4, and it pays off most at Q5/Q6 (estimates + cited link data)", "summary": "On X870 boards the USB4 ports use an ASMedia ASM4242. Linux host-to-host networking on that chip drops every packet unless the kernel carries a fix that turns off transmit end-to-end flow control; with the fix it measured 0.28 ms RTT but only 4.2-5.2 Gb/s. A direct 10GbE link gives similar latency (0.17-0.30 ms) at ~9.4 Gb/s. For a llama.cpp RPC layer split with one boundary between boxes, decode is latency-bound, so the estimated network tax is about 5-11% on either link; 10GbE mainly helps prefill (est. ~4% vs ~9% tax) and needs no kernel patch. Estimated Flash-Next decode, 262K context: two boxes x 2 cards keep every >=Q4 pack fully in VRAM (UD-Q5_K_XL est. ~62-66 tok/s vs est. ~51 on three cards in one box; for reference, Strata UD-Q5_K_XL measures 49-52 tok/s on two cards today, while a llama.cpp roofline estimate for two cards is ~34). At UD-Q4_K_XL a third card in one box is slightly better (~74 vs ~66-70) because Q4 already fits in 3 cards.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (estimate)", "engine": "llama.cpp RPC, -sm layer (estimate)", "gpus": "2 desktops x 2 Radeon AI PRO R9700", "context": "262K, f16 KV, single request" }, "metrics": [ { "label": "ASM4242 USB4 host-to-host throughput (cited)", "value": "4.2-5.2", "unit": "Gb/s" }, { "label": "10GbE direct link throughput (cited)", "value": "9.41", "unit": "Gb/s" }, { "label": "Est. UD-Q5_K_XL decode, 4 cards over 10GbE", "value": "62-66", "unit": "tok/s" }, { "label": "Est. UD-Q5_K_XL decode, 3 cards one box", "value": "51", "unit": "tok/s" }, { "label": "Est. UD-Q4_K_XL decode, 3 cards one box", "value": "74", "unit": "tok/s" } ], "sources": [ "https://github.com/torvalds/linux/commit/1881f2efbf7f78dc0a79a387b29fde6ff56d3731", "https://qiita.com/faliye/items/2a2d8def5298f6c1c10e", "https://github.com/kjaiswal/llama-cpp-distributed-benchmarks", "https://github.com/ggml-org/llama.cpp/pull/24122", "https://github.com/ggml-org/llama.cpp/issues/22307", "https://docs.vllm.ai/en/stable/serving/parallelism_scaling/", "https://jingchaozhang.github.io/LLM-Inference-Across-NVLink-and-InfiniBand/", "https://www.asrock.com/MB/AMD/X870%20Taichi%20Creator/" ], "href": "https://amdai.net/lab-log#2026-10-24-two-box-4x-r9700-link" }, { "id": "2026-10-24-card-count-resident", "date": "2026-10-24T14:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "3 or 4 R9700s: when Flash-Next fits fully in VRAM, and what 4-card owners report", "summary": "A cited 4x R9700 vLLM tensor-parallel-4 run of Qwen3.8-Flash-Next (MXFP4 experts, FP8 dense, ROCm 10) reports 219 tok/s at 32K context, one request. Our fit model (estimate) says UD-Q4_K_XL becomes fully VRAM-resident on 3 cards at 262K context, and UD-Q5/Q6_K_XL need 4.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (model); MXFP4 experts + FP8 dense (cited)", "engine": "llama.cpp layer split (model); vLLM TP4 (cited)", "gpus": "2-4x Radeon AI PRO R9700", "context": "262K (model); 32K (cited)" }, "metrics": [ { "label": "Cited 4x R9700 vLLM TP4 decode, 1 request", "value": 219, "unit": "tok/s" }, { "label": "Cited 4x R9700 vLLM TP4 prefill, 11.8K input", "value": 3571, "unit": "tok/s" }, { "label": "UD-Q4_K_XL GPU bytes at 262K (estimate)", "value": 82.9, "unit": "GiB" }, { "label": "UD-Q4_K_XL ceiling, 3 cards resident (estimate)", "value": 74, "unit": "tok/s" }, { "label": "UD-Q4_K_XL ceiling, 2 cards + DDR5 (estimate)", "value": 45, "unit": "tok/s" }, { "label": "UD-Q5_K_XL ceiling, 4 cards resident (estimate)", "value": 69, "unit": "tok/s" } ], "sources": [ "https://amd-pro-bench.com/gpus/r9700/4-gpus", "https://amd-pro-bench.com/gpus/r9700/3-gpus", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://docs.vllm.ai/en/stable/serving/parallelism_scaling/", "https://github.com/vllm-project/vllm/issues/11797" ], "href": "https://amdai.net/lab-log#2026-10-24-card-count-resident" }, { "id": "2026-10-24-alt-engines-gfx1201", "date": "2026-10-24T13:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Other engines for Flash MoE on R9700 (gfx1201): AMD's HRX backend isn't supported yet, ik_llama.cpp is untested, KTransformers ROCm is stale", "summary": "AMD's native llama.cpp backend (ggml-hrx, draft) does not run on gfx1201 yet: a user hit 'no HRX dispatch registry for target gfx1201' on 2026-10-15 and AMD said support is planned with no timeline. ik_llama.cpp's HIP build was fully broken until a repair merged 2026-08-25, validated only on RDNA3 (gfx1101); it now supports Qwen3.8-Flash-Next and MTP, but nobody has published a gfx1201 run. KTransformers lists ROCm as historical documentation only.", "setup": { "gpus": "Radeon AI PRO R9700 (gfx1201); sources cover gfx1100/1101/1151", "engine": "ggml-hrx draft, ik_llama.cpp, KTransformers (per sources)" }, "metrics": [ { "label": "HRX vs HIP tg128, Qwen3-30B-A3B Q4_K_M, gfx1151 (community)", "value": 25, "unit": "% faster" }, { "label": "HRX prefill uplift claimed by AMD (RDNA3/3.5)", "value": "30-50", "unit": "%" }, { "label": "ik_llama.cpp IQ4_XS tg before/after HIP fix, small model, RDNA3", "value": "17.16 / 266.31", "unit": "tok/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/discussions/27219", "https://github.com/ggml-org/llama.cpp/pull/27218", "https://github.com/lemonade-sdk/lemonade/pull/3723", "https://github.com/ikawrakow/ik_llama.cpp/pull/2339", "https://github.com/ikawrakow/ik_llama.cpp/discussions/562", "https://ktransformers.net/en/docs/hardware-platforms/cpu-gpu-requirements" ], "href": "https://amdai.net/lab-log#2026-10-24-alt-engines-gfx1201" }, { "id": "2026-10-24-toolcall-eval-noise", "date": "2026-10-24T12:25:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Q4 vs Q5 for agents: small tool-call evals can't tell the quants apart", "summary": "A published Qwen3.8-Flash-Next engine comparison found llama.cpp UD-Q4_K_XL at 73.0% on 300 BFCL v4 tool-calling samples, but its three repeats ranged 70.7 to 76.3 points, and the multi-turn subset swung by plus or minus 14. Unsloth's quality gap between UD-Q4_K_XL and UD-Q5_K_XL is mean KLD 0.047 vs 0.030. A standard McNemar sample-size calculation shows that confirming a 3-point tool-call gap between two quants needs about 870 paired cases (at 10% disagreement), and a 5-point gap about 310. Before picking the dual-R9700 Flash quant on agent accuracy, compare quants with paired runs on the same cases, require the same sign in every repeat, and treat 'no difference found' as 'gap under about 8 points', not 'equal'.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL (published BFCL run); UD-Q4/Q5 KLD from Unsloth", "engine": "llama.cpp (published)", "gpus": "published run on one RTX 6000 Blackwell, not R9700", "context": "131K served", "flags": "temp 0.6, 3 repeats, Q8_0 KV, MTP 3 draft tokens" }, "metrics": [ { "label": "BFCL v4, llama.cpp UD-Q4_K_XL, mean of 3", "value": 73, "unit": "%" }, { "label": "BFCL v4 repeat range", "value": 5.67, "unit": "points" }, { "label": "UD-Q4_K_XL mean KLD", "value": 0.046893, "unit": "" }, { "label": "UD-Q5_K_XL mean KLD", "value": 0.030415, "unit": "" }, { "label": "Paired cases to confirm a 3-point gap (estimate, 10% disagreement)", "value": 870, "unit": "cases" }, { "label": "Paired cases to confirm a 5-point gap (estimate)", "value": 312, "unit": "cases" } ], "sources": [ "https://www.demietrich.com/blog/q3.8fn_engine_comparison/", "https://unsloth.ai/docs/models/qwen3.8-next", "https://huggingface.co/Cyronius/Qwen3.8-Flash-Next-131B-A6B-GGUF" ], "href": "https://amdai.net/lab-log#2026-10-24-toolcall-eval-noise" }, { "id": "2026-10-24-local-vs-api-cost-flash", "date": "2026-10-24T11:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Dual R9700 vs the hosted Qwen3.8 Flash API: the vLLM 4-bit lane is cheaper per token on power, the Strata Q5 lane is not", "summary": "Using our measured speeds, cited prices and an estimated 465-615 W at the wall, one stream of the vLLM TP2 MXFP4 lane (about 90 tok/s) costs about $0.23-0.35 in electricity per million output tokens, against $0.47 on OpenRouter, and runs about 1.7x the hosted median speed (47-53 tok/s). Strata UD-Q5_K_XL (about 49 tok/s) costs about $0.42-0.64, roughly break-even or worse. Two cards only pay for themselves after 14-16.5 billion output tokens, about 5-6 years running 24x7 at one stream.", "setup": { "model": "Qwen3.8 Flash", "quant": "MXFP4-FP8-GPTQ (vLLM) / UD-Q5_K_XL (Strata)", "engine": "vLLM TP2, Strata", "gpus": "2x Radeon AI PRO R9700, local desktop", "context": "36K test" }, "metrics": [ { "label": "vLLM MXFP4 lane, electricity per 1M output tokens (estimate)", "value": "0.23-0.35", "unit": "USD" }, { "label": "Strata UD-Q5 lane, electricity per 1M output tokens (estimate)", "value": "0.42-0.64", "unit": "USD" }, { "label": "Hosted API, per 1M output tokens", "value": 0.47, "unit": "USD" }, { "label": "Hosted API median throughput", "value": "47-53", "unit": "tok/s" }, { "label": "Local prefill, electricity per 1M input tokens (estimate)", "value": "0.005-0.014", "unit": "USD" }, { "label": "Output tokens to pay back two cards (vLLM lane, 1 stream)", "value": "14-16.5", "unit": "billion" } ], "sources": [ "https://openrouter.ai/qwen/qwen3.8-flash", "https://www.eia.gov/electricity/monthly/epm_table_grapher.php?t=epmt_5_6_a", "https://energyfactbook.com/electric-power-monthly/", "https://www.pugetsystems.com/labs/articles/amd-radeon-ai-pro-r9700-dual-gpu-ai-inference-performance/", "http://www.nowinstock.net/computers/videocards/amd/aipror9700/", "https://github.com/ROCm/ROCm/issues/6453" ], "href": "https://amdai.net/lab-log#2026-10-24-local-vs-api-cost-flash" }, { "id": "2026-10-24-ngram-spec-hybrid-offload", "date": "2026-10-24T10:30:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp n-gram speculation on Flash-Next with experts in RAM: off by default, and why", "summary": "Reading llama.cpp master (71255fe, 2026-10-24): ngram-mod only drafts 48-64-token chains, so each verify batch is 49-65 tokens. That is above GGML_OP_OFFLOAD_MIN_BATCH (32), so RAM-resident experts get copied over PCIe for that step instead of being read by the CPU. Also, n-gram types never arm recurrent rollback, so on a hybrid model like Qwen3.8-Flash-Next every drafted round takes a full, host-serialized state checkpoint and a partial accept restores it and replays the accepted tokens. Our bandwidth model (estimate) says default ngram-mod is a net loss for this setup except on near-pure file rewrites.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q5_K_XL (model), Q4-Q6 lane", "engine": "llama.cpp", "engine_version": "master 71255fe (source read)", "gpus": "2x Radeon AI PRO R9700", "context": "262K", "flags": "-sm layer, routed experts partly in DDR5, -np 1" }, "metrics": [ { "label": "MTP n-max 3 after #28123, code (RTX PRO 6000, published)", "value": 183, "unit": "tok/s" }, { "label": "No draft, same rig (published)", "value": 108, "unit": "tok/s" }, { "label": "Host checkpoint share of a speculative round (Strix Halo, published)", "value": 600, "unit": "ms of ~825" }, { "label": "ngram-mod verify batch size", "value": "49-65", "unit": "tokens" }, { "label": "Estimated UD-Q5 rewrite speedup, defaults (2x R9700)", "value": 0.87, "unit": "x" }, { "label": "Estimated UD-Q5 rewrite speedup, MIN_BATCH=256, cheap checkpoints", "value": 1.9, "unit": "x" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/server-context.cpp", "https://github.com/ggml-org/llama.cpp/pull/28123", "https://github.com/ggml-org/llama.cpp/pull/28118", "https://github.com/ggml-org/llama.cpp/issues/28019", "https://ai-muninn.com/en/blog/qwen4exp-177b-three-2080ti", "https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md" ], "href": "https://amdai.net/lab-log#2026-10-24-ngram-spec-hybrid-offload" }, { "id": "2026-10-24-vllm-tp2-mxfp4-flash-fastest", "date": "2026-10-24T09:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "New fastest Flash setup on dual R9700: vLLM TP2 with a 4-bit MXFP4 pack, 87-94 tok/s decode at 36K", "summary": "vLLM 0.29.0.dev0 (tcclaviger/vllm:latest) with tensor parallel 2 and the tcclaviger MXFP4-FP8-GPTQ Qwen3.8 Flash pack decodes 83-114 tok/s short (story 81-94, code 100-114) and 87-94 tok/s at 36K, with about 3,800 tok/s prefill at 36K. That is about 1.8x Strata Q5 decode (about 49 at 36K) and 1.6x its prefill (about 2,300), and well above the best published 4-bit-or-higher dual-3090 llama.cpp result (37-41 tok/s).", "setup": { "model": "Qwen3.8 Flash", "quant": "MXFP4-FP8-GPTQ (tcclaviger), rev 6fe3e74", "engine": "vLLM", "engine_version": "0.29.0.dev0 (tcclaviger/vllm:latest)", "gpus": "2x Radeon AI PRO R9700, local desktop (dual-channel DDR5)", "context": "262K", "flags": "tensor parallel 2, fp8 KV, MNBT 4096, MTP" }, "metrics": [ { "label": "Decode, short (story / code range)", "value": "83-114", "unit": "tok/s" }, { "label": "Decode at 36K", "value": "87-94", "unit": "tok/s" }, { "label": "Prefill at 36K", "value": 3800, "unit": "tok/s" }, { "label": "Strata Q5 decode at 36K (same machine)", "value": 49, "unit": "tok/s" }, { "label": "Strata Q5 prefill at 36K (same machine)", "value": 2300, "unit": "tok/s" }, { "label": "MTP acceptance", "value": 51.8, "unit": "%" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-24-vllm-tp2-mxfp4-flash-fastest" }, { "id": "2026-10-24-strata-q5-tuning-ab", "date": "2026-10-24T09:04:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Strata Q5 tuning on dual R9700: the defaults are already about right", "summary": "STAGE_TRIM=1 with layer-split=23 raised 36K decode from 48.5 to 50.3 tok/s (+3.7%) but cut 36K prefill from 2374 to 2171 tok/s (-8.6%), so the default stays. Strata's auto split point (K=23) beat 19, 21, 25 and 27. Speculation depth 3, 5 and 6 were no better than 4, SPLIT_MTP_BATCH had no effect, and adapt-async is refused in layer-split mode.", "setup": { "model": "Qwen3.8 Flash", "quant": "UD-Q5_K_XL", "engine": "Strata", "engine_version": "0.2.3", "gpus": "2x Radeon AI PRO R9700, local desktop (dual-channel DDR5)", "context": "36K test", "flags": "layer split, MTP" }, "metrics": [ { "label": "36K decode, default", "value": 48.5, "unit": "tok/s" }, { "label": "36K decode, STAGE_TRIM=1 + split 23", "value": 50.3, "unit": "tok/s" }, { "label": "36K prefill, default", "value": 2374, "unit": "tok/s" }, { "label": "36K prefill, STAGE_TRIM=1 + split 23", "value": 2171, "unit": "tok/s" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-24-strata-q5-tuning-ab" }, { "id": "2026-10-24-llamacpp-moe-cache-mib-measured", "date": "2026-10-24T09:03:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp --moe-cache-mib on dual R9700: about +20-25% decode, but still about 2.5x behind Strata", "summary": "With the HIP dsv4 build on Qwen3.8 Flash UD-Q5_K_XL (262K, f16 KV), the VRAM expert cache lifted warm decode from 18.2 to 23.5 tok/s (story) and 36K decode from 17.0 to 19.6, at a 93.7% hit rate. Prefill fell from 231 to 162 tok/s at 36K because prompt batches only hit the cache 8-15% of the time.", "setup": { "model": "Qwen3.8 Flash", "quant": "UD-Q5_K_XL (Q6 for comparison)", "engine": "llama.cpp (HIP, dsv4 build)", "engine_version": "dsv4 build", "gpus": "2x Radeon AI PRO R9700, local desktop (dual-channel DDR5)", "context": "262K", "flags": "f16 KV, --moe-cache-mib 6544 / 12000 / 40000 with -cmoe" }, "metrics": [ { "label": "Warm story decode, fit only", "value": 18.2, "unit": "tok/s" }, { "label": "Warm story decode, -cmoe +40000", "value": 23.5, "unit": "tok/s" }, { "label": "36K prefill, fit only", "value": 231, "unit": "tok/s" }, { "label": "36K prefill, -cmoe +40000", "value": 162, "unit": "tok/s" }, { "label": "Cache hit rate, -cmoe +40000", "value": 93.72, "unit": "%" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-24-llamacpp-moe-cache-mib-measured" }, { "id": "2026-10-24-glm53-flash-q5-dual-r9700", "date": "2026-10-24T09:02:00-05:00", "project": "dual-r9700", "kind": "result", "title": "GLM 5.3 Flash Q5_K_M on dual R9700: about 6 tok/s decode, HIP prefill 2.3x Vulkan, 524K context loads", "summary": "bartowski GLM 5.3 Flash Q5_K_M (227 GiB) on llama.cpp-dsv4 HIP decodes 6.4 tok/s short, 6.0 at 36K (prefill 112.7) and 5.1 at 113K (prefill 74.4). HIP prefill is about 2.3x Vulkan (48.8). The largest context that loads is 524K. Strata and the RDNA4 llama.cpp fork don't support GLM yet.", "setup": { "model": "GLM 5.3 Flash", "quant": "Q5_K_M (bartowski), 227 GiB", "engine": "llama.cpp (HIP, dsv4 build)", "engine_version": "dsv4 build", "gpus": "2x Radeon AI PRO R9700, local desktop (dual-channel DDR5)", "context": "262K", "flags": "q8 KV, -ub 2048 -b 2048, -t 16" }, "metrics": [ { "label": "Decode, short", "value": 6.4, "unit": "tok/s" }, { "label": "Decode at 36K", "value": 6, "unit": "tok/s" }, { "label": "Prefill at 36K", "value": 112.7, "unit": "tok/s" }, { "label": "Decode at 113K", "value": 5.1, "unit": "tok/s" }, { "label": "Prefill at 113K", "value": 74.4, "unit": "tok/s" }, { "label": "Vulkan prefill (for comparison)", "value": 48.8, "unit": "tok/s" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-24-glm53-flash-q5-dual-r9700" }, { "id": "2026-10-24-deepseek-v4-flash-q4-dual-r9700", "date": "2026-10-24T09:01:00-05:00", "project": "dual-r9700", "kind": "result", "title": "DeepSeek V4 Flash UD-Q4_K_XL on dual R9700: about 10 tok/s decode, and a full 1M context loads", "summary": "unsloth DeepSeek V4 Flash UD-Q4_K_XL (144.4 GiB) on llama.cpp-dsv4 HIP decodes 10.0 tok/s short (prefill 105-147), 9.1 at 36K (prefill 168.6) and 7.9 at 112K (prefill 107.9), about 1.5x GLM 5.3 Flash. A full 1M context loads, with short decode at 9.4 tok/s.", "setup": { "model": "DeepSeek V4 Flash", "quant": "UD-Q4_K_XL (unsloth), 144.4 GiB", "engine": "llama.cpp (HIP, dsv4 build)", "engine_version": "dsv4 build", "gpus": "2x Radeon AI PRO R9700, local desktop (dual-channel DDR5)", "context": "262K (1M also loads)", "flags": "q8 KV, -ub 2048 -b 2048, -t 16" }, "metrics": [ { "label": "Decode, short", "value": 10, "unit": "tok/s" }, { "label": "Decode at 36K", "value": 9.1, "unit": "tok/s" }, { "label": "Prefill at 36K", "value": 168.6, "unit": "tok/s" }, { "label": "Decode at 112K", "value": 7.9, "unit": "tok/s" }, { "label": "Prefill at 112K", "value": 107.9, "unit": "tok/s" }, { "label": "Decode, short, 1M context", "value": 9.4, "unit": "tok/s" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-24-deepseek-v4-flash-q4-dual-r9700" }, { "id": "2026-10-24-llama-bench-json-path-privacy", "date": "2026-10-24T08:20:00-05:00", "project": "idea-lab", "kind": "progress", "title": "Sharing llama-bench JSON? It includes the full model path you typed", "summary": "llama-bench's json/jsonl output copies the -m argument verbatim into model_filename, so a shared raw file usually shows your home directory and username. There is no hostname field, and cpu_info/gpu_info are only device names. Trim model_filename to the file name before posting.", "setup": { "engine": "llama.cpp llama-bench", "engine_version": "master df9beac" }, "metrics": [], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/df9beac/tools/llama-bench/llama-bench.cpp" ], "href": "https://amdai.net/lab-log#2026-10-24-llama-bench-json-path-privacy" }, { "id": "2026-10-24-cpu-expert-path-repack-mlock", "date": "2026-10-24T07:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "The CPU half of Flash decode in llama.cpp: no x86 repack for Q5/Q6 experts, keep the host buffer, and use -lm mlock", "summary": "Reading llama.cpp master (df9beac, Oct 24): x86 CPU weight repack exists only for Q4_0, Q4_K, IQ4_NL and MXFP4, so Unsloth UD-Q5_K_XL and UD-Q6_K_XL experts (Q5_K, Q6_K, Q8_0) get none, while AP-Q4_K_XL is 100% eligible and UD-Q4_K_XL 58%. With a ROCm or Vulkan GPU loaded, -ncmoe experts go to the GPU host buffer and are never repacked unless --no-host is set, and repacked weights can't be offloaded to the GPU for prefill. For pinned, swap-proof experts, -lm mlock now means no mmap plus mlock again.", "setup": { "gpus": "2x Radeon AI PRO R9700", "model": "Qwen3.8-Flash-Next GGUFs", "engine": "llama.cpp master df9beac (source read)", "flags": "-ncmoe, --no-host, --repack, -lm mlock, -t / -C / --cpu-strict" }, "metrics": [ { "label": "x86 repack-eligible expert bytes, AP-Q4_K_XL", "value": 100, "unit": "%" }, { "label": "x86 repack-eligible expert bytes, UD-Q4_K_XL", "value": 57.6, "unit": "%" }, { "label": "x86 repack-eligible expert bytes, AP-Q5_K_M", "value": 29, "unit": "%" }, { "label": "x86 repack-eligible expert bytes, UD-Q5_K_XL / UD-Q6_K_XL", "value": 0, "unit": "%" }, { "label": "Zen 4/5 desktop per-CCD read limit at FCLK 2000", "value": 64, "unit": "GB/s" }, { "label": "Est. UD-Q5 2x R9700 ceiling if DDR5 read goes 55 to 70 GB/s (estimate)", "value": "34 to 39", "unit": "tok/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/df9beac7153166550571c70cc667f270b745da49/ggml/src/ggml-cpu/repack.cpp", "https://github.com/ggml-org/llama.cpp/blob/df9beac7153166550571c70cc667f270b745da49/src/llama-model.cpp", "https://github.com/ggml-org/llama.cpp/blob/df9beac7153166550571c70cc667f270b745da49/ggml/src/ggml-backend.cpp", "https://github.com/ggml-org/llama.cpp/pull/19738", "https://github.com/ggml-org/llama.cpp/pull/19706", "https://github.com/ggml-org/llama.cpp/issues/26110", "https://github.com/ggml-org/llama.cpp/pull/26135", "https://github.com/ggml-org/llama.cpp/pull/20596", "https://chipsandcheese.com/p/amds-ryzen-9950x-zen-5-on-desktop" ], "href": "https://amdai.net/lab-log#2026-10-24-cpu-expert-path-repack-mlock" }, { "id": "2026-10-24-vulkan-vs-rocm-flash", "date": "2026-10-24T06:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Vulkan beats ROCm for MoE decode on R9700s; in hybrid Flash runs the gap shrinks to ~16-25%", "summary": "Every recent R9700 comparison we found shows llama.cpp's Vulkan backend 1.5-1.8x faster than ROCm/HIP for MoE decode. Fresh 2x R9700 numbers posted today on the Vulkan tensor-parallel PR: Qwen3.6-35B-A3B Q4_K_M decodes at 165.2 tok/s on one card with Vulkan vs 91.5 with ROCm, and 132.6 vs 87.6 with layer split. Dense 27B is a tie in layer split. For Flash-class models with experts in DDR5, the CPU part doesn't depend on the backend, so we estimate ROCm would cost about 16% at UD-Q6, 18% at UD-Q5 and 21-25% at Q4. Our pick for the dual-R9700 Flash lane: Vulkan with layer split.", "setup": { "model": "Qwen3.6-35B-A3B / Qwen3.8-27B (published); Qwen3.8-Flash-Next (estimate)", "quant": "Q4_K_M (published); Q4-Q6 (estimate)", "engine": "llama.cpp", "engine_version": "master + PR #25051 (published)", "gpus": "1-2x Radeon AI PRO R9700", "context": "tg128 / pp512-4096", "flags": "-fa 1" }, "metrics": [ { "label": "35B-A3B Q4_K_M decode, 1 GPU, Vulkan", "value": 165.2, "unit": "tok/s" }, { "label": "35B-A3B Q4_K_M decode, 1 GPU, ROCm", "value": 91.5, "unit": "tok/s" }, { "label": "35B-A3B Q4_K_M decode, 2 GPU layer, Vulkan", "value": 132.6, "unit": "tok/s" }, { "label": "35B-A3B Q4_K_M decode, 2 GPU layer, ROCm", "value": 87.6, "unit": "tok/s" }, { "label": "27B Q4_K_M decode, 2 GPU tensor, Vulkan", "value": 45, "unit": "tok/s" }, { "label": "27B Q4_K_M decode, 2 GPU tensor, ROCm", "value": 40.2, "unit": "tok/s" }, { "label": "Est. ROCm decode loss vs Vulkan, Flash UD-Q5 hybrid", "value": 18, "unit": "%" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/25051", "https://github.com/ggml-org/llama.cpp/discussions/22463", "https://deepindeepsea.github.io/r9700-vllm-blogs/part12-r9700-the-full-sweep.html", "https://finaltype.github.io/quant-blog/en/devlog/2026-09/2026-09-26_moe35b-r9700-speed-tuning/" ], "href": "https://amdai.net/lab-log#2026-10-24-vulkan-vs-rocm-flash" }, { "id": "2026-10-24-parallel-slots-offload-cliff", "date": "2026-10-24T05:10:00-05:00", "project": "dual-r9700", "kind": "progress", "title": "Parallel slots with experts in DDR5: a 32-token decode cliff in llama.cpp, and what batching buys on 2x R9700", "summary": "In llama.cpp, a MoE expert op whose weights sit in system RAM is moved to a GPU once a ubatch holds 32 or more tokens (GGML_OP_OFFLOAD_MIN_BATCH, default 32, in both the HIP/CUDA and Vulkan backends). In decode, tokens per step = active slots x (1 + draft tokens), so 32 parallel slots, or just 8 slots with 3 MTP drafts each, switch the DDR5-resident layers from CPU reads to PCIe uploads of the used experts. Our bandwidth estimate says that roughly halves aggregate decode at that point; raising the threshold to about 256 avoids it while large prefill batches still offload.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q5_K_XL, AP-Q4_K_XL (estimates)", "engine": "llama.cpp (source reading + bandwidth model)", "engine_version": "master 3642238", "gpus": "2x Radeon AI PRO R9700", "context": "262K f16 KV pool, 8K fill per slot", "flags": "-sm layer, -ncmoe per quant, -np N" }, "metrics": [ { "label": "Default offload threshold (tokens per ubatch)", "value": 32, "unit": "tokens" }, { "label": "Est. UD-Q5 aggregate decode, 8 slots", "value": 52, "unit": "tok/s" }, { "label": "Est. UD-Q5 aggregate decode, 31 slots", "value": 67, "unit": "tok/s" }, { "label": "Est. UD-Q5 aggregate decode, 32 tokens/step at default threshold", "value": 31, "unit": "tok/s" }, { "label": "Est. 8 slots x 4 MTP tokens verify step, default vs threshold 256", "value": "1.02 vs 0.46", "unit": "s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/blob/3642238cc12050c8ab70afdf4b3121ea365c7579/ggml/src/ggml-cuda/ggml-cuda.cu", "https://github.com/ggml-org/llama.cpp/blob/3642238cc12050c8ab70afdf4b3121ea365c7579/ggml/src/ggml-vulkan/ggml-vulkan.cpp", "https://github.com/ggml-org/llama.cpp/blob/3642238cc12050c8ab70afdf4b3121ea365c7579/src/llama-context.cpp", "https://github.com/ggml-org/llama.cpp/blob/3642238cc12050c8ab70afdf4b3121ea365c7579/tools/batched-bench/README.md" ], "href": "https://amdai.net/lab-log#2026-10-24-parallel-slots-offload-cliff" }, { "id": "2026-10-24-dual-r9700-platform-lanes-ddr5", "date": "2026-10-24T04:10:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Dual R9700 platform: M.2 slots that steal GPU 2's lanes, and what DDR5 speed and channels are worth", "summary": "From manufacturer specs: on the ASUS ProArt X870E-Creator, populating M.2_2 drops the second GPU slot to x4; on the Crosshair X870E Apex, M.2_3 gives x8/x4 and M.2_2 plus M.2_3 disables slot 2; on the Crosshair X870E Dark Hero, slot 2 runs PCIe 3.0 x4 until x8/x8 is set in the BIOS. Our calibrated estimate says AM5 RAM tuning (5200 to 6000 MT/s) is worth about +8% decode at UD-Q5, slipping to AMD's stock DDR5-3600 for four DIMMs costs about 20%, and a 4-channel Threadripper TRX50 platform would be worth about +52%.", "setup": { "gpus": "2x Radeon AI PRO R9700", "model": "Qwen3.8-Flash-Next GGUFs", "engine": "bandwidth model (estimate) + manufacturer specs", "context": "262K" }, "metrics": [ { "label": "Est. UD-Q5 decode, AM5 4x64 GB @3600", "value": 27.2, "unit": "tok/s" }, { "label": "Est. UD-Q5 decode, AM5 4x64 GB @5200 (calibrated)", "value": 34.1, "unit": "tok/s" }, { "label": "Est. UD-Q5 decode, AM5 4x64 GB @6000", "value": 37, "unit": "tok/s" }, { "label": "Est. UD-Q5 decode, TRX50 4ch @6400", "value": 51.7, "unit": "tok/s" } ], "sources": [ "https://www.asus.com/us/motherboards-components/motherboards/proart/proart-x870e-creator-wifi/techspec/", "https://dlcdnets.asus.com/pub/ASUS/mb/SocketAM5/ROG_CROSSHAIR_X870E_APEX/E25780_ROG_CROSSHAIR_X870E_APEX_EM_WEB.pdf", "https://rog.asus.com/ea/motherboards/rog-crosshair/rog-crosshair-x870e-dark-hero/spec/", "https://rog.asus.com/us/motherboards/rog-strix/rog-strix-x870e-e-gaming-wifi7-neo/wtb/", "https://www.sapphiretech.com/en/commercial/radeon-ai-pro-r9700", "https://en.wikipedia.org/wiki/Template:AMD_Ryzen_9000_Series", "https://www.tweaktown.com/news/105825/msi-releases-amd-agesa-1-2-0-3e-bios-officially-supports-4-64gb-sticks-up-to-256gb-ram/index.html", "https://www.kingston.com/en/memory/server-memory/memory-population-rules-wrx90-trx50" ], "href": "https://amdai.net/lab-log#2026-10-24-dual-r9700-platform-lanes-ddr5" }, { "id": "2026-10-24-quality-per-gib-q4plus", "date": "2026-10-24T03:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Qwen3.8-Flash-Next at Q4 or higher: where the extra GiB in Q5 and Q6 actually go", "summary": "We read the tensor headers of seven Flash-Next GGUFs at 4-bit or more without downloading the weights. Most of the 43.7 GiB that UD-Q5_K_XL adds over UD-Q4_K_XL never touches the GPU: 23.9 GiB is the n-gram table moving from IQ4_NL to Q8_0, which stays in system RAM and is read only a few rows per token. Another 10.5 GiB is the expert down projections moving to Q8_0, and only 9.4 GiB is the gate and up experts moving to Q5_K. On agentionai's published held-out test, their AP-Q5_K_M scores the same as UD-Q4_K_XL (mean KL 0.0771 vs 0.0778) with the same number of GPU-side bytes, and every tensor in it is 4-bit or more.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4/Q5/Q6_K_XL, AP-Q4_K_XL, AP-Q5_K_M, AP-Q5_K_XL, Q4_K_XL-DN4", "engine": "llama.cpp (GGUF header audit + bandwidth model)", "gpus": "2x Radeon AI PRO R9700 (modeled)", "context": "262K f16 KV (modeled)", "flags": "layer split, n-gram table in system RAM" }, "metrics": [ { "label": "UD-Q5_K_XL extra bytes over UD-Q4_K_XL that are n-gram table", "value": 23.9, "unit": "GiB" }, { "label": "GPU-side bytes, AP-Q5_K_M vs UD-Q4_K_XL", "value": "76.7 vs 76.8", "unit": "GiB" }, { "label": "Held-out mean KL, AP-Q5_K_M vs UD-Q4_K_XL (agentionai)", "value": "0.0771 vs 0.0778", "unit": "" }, { "label": "Estimated 2x R9700 decode ceiling, AP-Q5_K_M / UD-Q4 / UD-Q5 / UD-Q6", "value": "50 / 45 / 34 / 30", "unit": "tok/s (estimate)" } ], "sources": [ "https://huggingface.co/agentionai/Qwen3.8-Flash-Next-AP-GGUF", "https://unsloth.ai/docs/models/qwen3.8-next", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://huggingface.co/bitlamas/Qwen3.8-Flash-Next-Q4_K_XL-DN4" ], "href": "https://amdai.net/lab-log#2026-10-24-quality-per-gib-q4plus" }, { "id": "2026-10-24-native-4bit-gate0", "date": "2026-10-24T02:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Which native 4-bit Qwen3.8-Flash-Next packs for 2x R9700 are really 4-bit or more", "summary": "We read the tensor headers of tcclaviger's MXFP4-FP8-GPTQ pack for vLLM without downloading the weights. Every tensor class is 4-bit or more: routed experts 4.25 bits per weight (MXFP4), n-gram table 6.53 (int6), attention FP8, MTP head BF16, 110.6 GiB total. Radiance's stock Flash-Next pack keeps 4-bit codebook experts and an int8 trunk, but its recipe stores the MTP draft head at 2 bits (u2), so it fails a strict no-sub-4-bit rule.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "MXFP4 experts / int6 n-gram / FP8 attention (vLLM); w4 codebook + int8 (Radiance)", "engine": "vLLM (tcclaviger image), Radiance", "engine_version": "HF revisions 6fe3e74 and 8267a67", "gpus": "2x Radeon AI PRO R9700", "context": "262K / 200K", "flags": "header audit only, no inference run" }, "metrics": [ { "label": "vLLM pack routed experts", "value": 4.25, "unit": "bits/weight" }, { "label": "vLLM pack n-gram table", "value": 6.53, "unit": "bits/weight" }, { "label": "vLLM pack total size", "value": 110.6, "unit": "GiB" }, { "label": "Radiance pack MTP draft head", "value": 2, "unit": "bits" } ], "sources": [ "https://huggingface.co/tcclaviger/Qwen3.8-Flash-Next-MXFP4-FP8-GPTQ", "https://huggingface.co/StillDeadcode/qwen3.8-next-flash-fp8-iq4r-moe", "https://hub.docker.com/r/stilldeadcode/vllm-radiance/", "https://codeberg.org/StillDeadcode/radiance" ], "href": "https://amdai.net/lab-log#2026-10-24-native-4bit-gate0" }, { "id": "2026-10-24-llamacpp-hip-rdna4-llvm23-fix", "date": "2026-10-24T01:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "llama.cpp HIP builds for R9700 (gfx1201) on LLVM 23 / new ROCm need b11847 or later", "summary": "Until llama.cpp PR #30885 merged on 2026-10-23 (in release b11847), HIP builds for RDNA4 (gfx1200/gfx1201) and RDNA3 APUs (gfx1150/gfx1151) failed to compile with LLVM 23 and the newest ROCm, because the bf16 WMMA operands were typed __bf16 instead of short. The fix is bit-identical and needs no version gate.", "setup": { "engine": "llama.cpp (HIP backend)", "engine_version": "b11847", "gpus": "Radeon AI PRO R9700 (gfx1201)" }, "metrics": [], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/30885", "https://github.com/ggml-org/llama.cpp/issues/30839", "https://github.com/ggml-org/llama.cpp/releases/tag/b11847" ], "href": "https://amdai.net/lab-log#2026-10-24-llamacpp-hip-rdna4-llvm23-fix" }, { "id": "2026-10-24-offload-prefill-pcie", "date": "2026-10-24T00:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Why llama.cpp prefill is 10x slower than Strata with Flash experts in DDR5: it's PCIe-bound at the default ubatch", "summary": "In llama.cpp, any prompt chunk of 32+ tokens copies every expert kept in system RAM over PCIe to a GPU, so prefill speed is roughly ubatch size divided by the time to copy those experts. Our measured baseline (Vulkan, Q5_K_M, --n-cpu-moe 28, 205 tok/s at 36K) works out to about 23 GB/s at the default -ub 512, which is 73% of a PCIe 5.0 x8 link: it was PCIe-bound. Estimated ceiling for UD-Q5_K_XL on two R9700s rises from about 230 tok/s at -ub 512 to about 880-960 at -ub 2048 and 1,700-1,840 at -ub 4096, before GPU compute limits.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (estimates); baseline Q5_K_M (measured)", "engine": "llama.cpp master source + llama.cpp Vulkan baseline", "gpus": "2x Radeon AI PRO R9700 (PCIe 5.0 x8 each)", "context": "262K planned, 36K baseline", "flags": "-sm layer, --n-cpu-moe 24-28, -ub 512/2048/4096" }, "metrics": [ { "label": "Baseline prefill (measured, -ub 512 assumed)", "value": 205, "unit": "tok/s" }, { "label": "Implied PCIe copy speed at baseline", "value": 23, "unit": "GB/s" }, { "label": "Q5 ceiling at -ub 2048 (estimate)", "value": "880-960", "unit": "tok/s" }, { "label": "Q5 ceiling at -ub 4096 (estimate)", "value": "1700-1840", "unit": "tok/s" }, { "label": "gfx1201 Vulkan, ub 512 to 2048 (published)", "value": "705 to 1566", "unit": "tok/s" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/issues/25859", "https://inovello.dev/writeups/pinned-host-experts-under-mmap/", "https://github.com/ggml-org/llama.cpp/pull/28223", "https://github.com/ggml-org/llama.cpp/pull/26659", "https://github.com/ggml-org/llama.cpp/pull/19976", "https://github.com/ggml-org/llama.cpp/blob/master/ggml/src/ggml-backend.cpp", "https://huggingface.co/blog/Doctor-Shotgun/llamacpp-moe-offload-guide", "https://github.com/gamozolabs/llama.cpp", "https://bittide.aicompass.dev/article/c9df2d89-8522-46ec-aa3f-fc1aed5b17d3", "https://deepindeepsea.github.io/r9700-vllm-blogs/part12-r9700-the-full-sweep.html" ], "href": "https://amdai.net/lab-log#2026-10-24-offload-prefill-pcie" }, { "id": "2026-10-23-hybrid-flash-prompt-cache", "date": "2026-10-23T23:20:00-05:00", "project": "idea-lab", "kind": "result", "title": "Agents on Qwen3.8-Flash-Next: without context checkpoints every turn re-reads the whole prompt", "summary": "Flash-Next is a hybrid DeltaNet model, so llama.cpp's --cache-reuse does nothing for it; context checkpoints are its only prompt cache. Published measurements: --ctx-checkpoints 0 gives a 0% cache hit and 20.5 s per follow-up turn, versus 99.5% and 4.2 s with a single checkpoint. One checkpoint hits as often as the default 32 while using far less RAM.", "setup": { "model": "Qwen3.8-Flash-Next", "engine": "llama.cpp server / Strata (published reports)", "flags": "--ctx-checkpoints, --cache-ram, --conversation-cache-mib" }, "metrics": [ { "label": "Follow-up turn, 0 checkpoints", "value": 20.5, "unit": "s" }, { "label": "Follow-up turn, 1 checkpoint", "value": 4.2, "unit": "s" }, { "label": "Cache hit, 1 checkpoint", "value": 99.5, "unit": "%" }, { "label": "Checkpoint size without MTP", "value": 112.6, "unit": "MiB" } ], "sources": [ "https://kksoftwareag.github.io/strix-halo-qwen-3.8-flash-next/memory.html", "https://github.com/ggml-org/llama.cpp/issues/18497", "https://github.com/ggml-org/llama.cpp/pull/25592", "https://particula.tech/blog/prompt-reprocessing-swa-hybrid-models-kv-cache", "https://github.com/NousResearch/hermes-agent/pull/27675", "https://thomaswildetech.com/software-development/ai-agents/strata/" ], "href": "https://amdai.net/lab-log#2026-10-23-hybrid-flash-prompt-cache" }, { "id": "2026-10-23-3090-gap-q4plus", "date": "2026-10-23T21:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "The fair dual-3090 target for Qwen3.8-Flash-Next at 4-bit and up: 37-41 tok/s", "summary": "Most viral dual-RTX-3090 Flash-Next speeds (111-160 tok/s) are 3-bit or lower, or don't state a quant. At 4-bit and above, the best published 2x 3090 llama.cpp result is 37-41 tok/s (49 on code output) with UD-Q4_K_XL at full 261K f16 KV, using an expert VRAM cache plus MTP. That box started at 17 tok/s without the cache, close to our own 16.9-20.3 llama.cpp baseline, so the gap is software, not hardware.", "setup": { "gpus": "2x RTX 3090 (comparison, published)", "engine": "llama.cpp with expert cache + MTP", "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL", "context": "261888, f16 KV", "flags": "experts in host RAM, -ub 512, MTP draft n-max 3" }, "metrics": [ { "label": "Best >=4-bit 2x 3090 llama.cpp decode (thinking prompt)", "value": "37-41", "unit": "tok/s" }, { "label": "Same, code emission", "value": 49, "unit": "tok/s" }, { "label": "Same box before expert cache (UD-Q6_K_XL)", "value": 17, "unit": "tok/s" }, { "label": "2x 3090 banded offload, UD-IQ4_XS", "value": "35-38", "unit": "tok/s" }, { "label": "Our 2x R9700 llama.cpp baseline (Q5_K_M, no cache)", "value": "16.9-20.3", "unit": "tok/s" }, { "label": "Est. R9700 pair cache slots per layer, Q5 + MTP", "value": 236, "unit": "slots (estimate)" }, { "label": "3090 pair cache slots per layer, Q4 + MTP", "value": 150, "unit": "slots" } ], "sources": [ "https://inovello.dev/writeups/qwen3-flash-next-2x3090-q4-mtp/", "https://www.reddit.com/r/LocalLLM/comments/1w069fh/qwen38flashnext_on_2x_rtx_3090s_3343_toks/", "https://x.com/geldeki/status/2095560770163458426", "https://x.com/coffeecup2020/status/2113265219283660816", "https://github.com/r0b0tlab/qwen38-flashnext-exl3", "https://github.com/ggml-org/llama.cpp/pull/29887", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-moe-cache.cpp" ], "href": "https://amdai.net/lab-log#2026-10-23-3090-gap-q4plus" }, { "id": "2026-10-23-dual-r9700-known-traps", "date": "2026-10-23T20:00:00-05:00", "project": "dual-r9700", "kind": "progress", "title": "Dual R9700 multi-GPU traps: what hangs and the fixes that work", "summary": "Cited roundup for two or more R9700s. vLLM tensor parallel (tp=2) deadlocks with both cards at 100% on ROCm 7.2.x RCCL; NCCL_PROTO=Simple is the fix (an AMD engineer notes the proposed RCCL patch is equivalent), while --enforce-eager, NCCL_P2P_DISABLE=1 and --disable-custom-all-reduce do not help. vLLM v0.21.0 adds a shared-memory broadcast hang on big models; v0.20.2 works. llama.cpp's ROCm build can hold an idle card at 100% and about 95 W; GPU_MAX_HW_QUEUES=1 brought it to 22 W.", "setup": { "gpus": "2-4x Radeon AI PRO R9700 (gfx1201), published reports", "engine": "vLLM ROCm images, llama.cpp HIP and Vulkan", "engine_version": "per source" }, "metrics": [ { "label": "Idle draw, llama.cpp HIP, GPU_MAX_HW_QUEUES unset", "value": 95, "unit": "W" }, { "label": "Idle draw, GPU_MAX_HW_QUEUES=1", "value": 22, "unit": "W" }, { "label": "35B-A3B FP8 decode, vLLM tp=2 (Level1Techs)", "value": 17.4, "unit": "tok/s" }, { "label": "35B-A3B FP8 decode, vLLM tp=4 (Level1Techs)", "value": 15.7, "unit": "tok/s" } ], "sources": [ "https://github.com/vllm-project/vllm/issues/40980", "https://github.com/ROCm/rocm-systems/issues/5480", "https://forum.level1techs.com/t/amd-radeon-r9700-linux-tp2-and-tp4-the-definitive-guide/250504", "https://github.com/ROCm/ROCm/issues/5706", "https://github.com/lemonade-sdk/llamacpp-rocm/issues/96", "https://github.com/ROCm/legacy-rocm-build/issues/6630", "https://www.reddit.com/r/LocalLLaMA/comments/1vjtokl/dual_radeon_ai_pro_r9700_server_much_slower_than/" ], "href": "https://amdai.net/lab-log#2026-10-23-dual-r9700-known-traps" }, { "id": "2026-10-23-flash-decode-roofline", "date": "2026-10-23T19:05:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Flash decode ceiling on dual R9700 + DDR5: about 34 tok/s at Q5 without MTP, so cutting DDR5 reads per token is the main lever", "summary": "A bandwidth-only ceiling (estimate) built from measured GGUF byte counts, the R9700's 640 GB/s spec and our measured 55 GB/s DDR5 read speed puts Qwen3.8 Flash on two R9700s (layer split, experts that don't fit in VRAM kept in DDR5, no MTP) at about 45 / 34 / 30 tok/s for UD-Q4 / Q5 / Q6_K_XL, or 63 / 44 / 37 with ideal tensor split. Our llama.cpp baseline (16.9-20.3 tok/s) runs at 55-66% of its ceiling. Strata's measured 52-65 tok/s at Q5 beats the no-MTP ceiling, so it can be reading at most 0.15-0.34 GiB of experts from DDR5 per generated token instead of 0.86 GiB, which points to MTP plus experts served from VRAM.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL", "engine": "model (bandwidth roofline), checked against measured llama.cpp Vulkan and Strata 0.2.3 runs", "gpus": "1-2x Radeon AI PRO R9700", "context": "262K f16 KV, 8K fill", "flags": "DDR5 4x64 GB at 5200, measured 53.8-56.2 GB/s read" }, "metrics": [ { "label": "Q4 ceiling, 2 GPUs layer split, no MTP (estimate)", "value": 45.1, "unit": "tok/s" }, { "label": "Q5 ceiling, 2 GPUs layer split, no MTP (estimate)", "value": 34.2, "unit": "tok/s" }, { "label": "Q6 ceiling, 2 GPUs layer split, no MTP (estimate)", "value": 29.8, "unit": "tok/s" }, { "label": "Q5 ceiling, 2 GPUs ideal tensor split (estimate)", "value": 43.5, "unit": "tok/s" }, { "label": "llama.cpp baseline vs its ceiling", "value": "55-66", "unit": "%" }, { "label": "Max DDR5 expert reads per token implied by Strata Q5", "value": "0.15-0.34", "unit": "GiB" } ], "sources": [ "https://www.amd.com/en/products/graphics/workstations/radeon-ai-pro/ai-9000-series/amd-radeon-ai-pro-r9700.html", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://www.techpowerup.com/gpu-specs/geforce-rtx-3090.c3622" ], "href": "https://amdai.net/lab-log#2026-10-23-flash-decode-roofline" }, { "id": "2026-10-23-35b-one-card-vram-budget", "date": "2026-10-23T18:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Qwen3.6-35B-A3B Q5 at full 262K context on one R9700 leaves under 2.75 GB for compute buffers", "summary": "Exact tensor sizes read from the GGUF headers: UD-Q5_K_XL has 22.26 GB of routed experts and 1.99 GB of other GPU weights, and its f16 KV cache at 262,144 tokens is 5.00 GB (10 of 40 layers use full attention). That is 29.25 GB on a 32 GB card, and this config runs fully on one R9700 with nothing offloaded, so llama.cpp's compute buffers fit in at most 2.75 GB. Fit calculators that assume 3 GB or more per card will wrongly say this model needs offload.", "setup": { "model": "Qwen3.6-35B-A3B", "quant": "Unsloth UD-Q5_K_XL", "engine": "llama.cpp (Vulkan)", "gpus": "1x Radeon AI PRO R9700 (32 GB)", "context": "262144", "flags": "f16 KV, all layers on GPU" }, "metrics": [ { "label": "Routed experts", "value": 22.26, "unit": "GB" }, { "label": "Other GPU weights", "value": 1.99, "unit": "GB" }, { "label": "f16 KV at 262K", "value": 5, "unit": "GB" }, { "label": "Max left for compute buffers", "value": 2.75, "unit": "GB" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF" ], "href": "https://amdai.net/lab-log#2026-10-23-35b-one-card-vram-budget" }, { "id": "2026-10-23-measured-flash-q5-vs-q6-strata", "date": "2026-10-23T17:12:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Measured on our dual R9700s: Strata UD-Q5_K_XL runs Qwen3.8 Flash at about 52-65 tok/s, 3x llama.cpp and faster than Q6", "summary": "On the local desktop (two R9700s plus DDR5 expert offload, 256K context), Strata 0.2.3 with Unsloth UD-Q5_K_XL, MTP and int8 KV decoded about 52 tok/s on a short story and about 65 tok/s on code (range 48-68), with 2043-2514 tok/s prefill at 36K. That is about 3x the decode and about 10x the prefill of our llama.cpp Vulkan Q5_K_M baseline (16.9-20.3 tok/s decode, 205 prefill). UD-Q6_K_XL on Strata was slower: median about 40 tok/s warm decode. Q5 stays the sweet spot; Q4 is not yet measured on our machine.", "setup": { "gpus": "2x Radeon AI PRO R9700 (PCIe Gen5 x8 each, ReBAR 32 GiB), DDR5 4x64 GB at 5200 (53.8-56.2 GB/s read, ~27 GB/s write)", "engine": "Strata 0.2.3 / 0.2.3.1 (engine 0.1.48 HIP); baseline llama.cpp v0.6.0 Vulkan", "model": "Qwen3.8 Flash", "quant": "UD-Q5_K_XL, UD-Q6_K_XL (baseline Q5_K_M)", "context": "256K (Strata)" }, "metrics": [ { "label": "Strata UD-Q5_K_XL decode, code", "value": 65, "unit": "tok/s" }, { "label": "Strata UD-Q5_K_XL decode, short story", "value": 52, "unit": "tok/s" }, { "label": "Strata UD-Q5_K_XL decode at 131K", "value": "47.6-54.3", "unit": "tok/s" }, { "label": "Strata UD-Q5_K_XL prefill at 131K", "value": "2519-2882", "unit": "tok/s" }, { "label": "Strata UD-Q6_K_XL warm decode (median)", "value": 40, "unit": "tok/s" }, { "label": "llama.cpp Vulkan Q5_K_M baseline decode", "value": "16.9-20.3", "unit": "tok/s" }, { "label": "llama.cpp Vulkan baseline prefill at 36K", "value": 205, "unit": "tok/s" }, { "label": "MTP acceptance (Q5)", "value": "70-75", "unit": "%" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-23-measured-flash-q5-vs-q6-strata" }, { "id": "2026-10-23-measured-rdna4-fork-vs-strata", "date": "2026-10-23T17:11:00-05:00", "project": "dual-r9700", "kind": "result", "title": "RDNA4 llama.cpp fork on dual R9700s: 1.3-1.6x baseline decode with MTP, but half of Strata and 10-15x slower prefill", "summary": "The RDNA4 llama.cpp fork (commit 185252d, ROCm 7.2.0, gfx1201, -sm layer -ts 1,1, q8 KV, flash attention) reached 22-26 tok/s on a story and 32 tok/s on code with MTP (shared Q8_0 head, n-max 3, -t16). That beats our llama.cpp Vulkan baseline by 1.3-1.6x but is about half of Strata UD-Q5_K_XL, and its prefill (144-175 tok/s) is 10-15x slower.", "setup": { "gpus": "2x Radeon AI PRO R9700 (PCIe Gen5 x8 each, ReBAR 32 GiB), DDR5 4x64 GB at 5200 (53.8-56.2 GB/s read, ~27 GB/s write)", "engine": "RDNA4 llama.cpp fork, commit 185252d, ROCm 7.2.0 (gfx1201)", "model": "Qwen3.8 Flash", "flags": "-sm layer -ts 1,1, q8 KV, FA, MTP shared Q8_0 head n-max 3, -t16", "context": "140K" }, "metrics": [ { "label": "Fork + MTP decode, code", "value": 32, "unit": "tok/s" }, { "label": "Fork + MTP decode, story", "value": "22-26", "unit": "tok/s" }, { "label": "Fork no MTP, -t16", "value": "20.3-20.5", "unit": "tok/s" }, { "label": "Fork no MTP, -t32 (oversubscribed)", "value": "8.9-9.3", "unit": "tok/s" }, { "label": "Fork decode / prefill at 131K", "value": "16.2 / 144", "unit": "tok/s" } ], "sources": [], "href": "https://amdai.net/lab-log#2026-10-23-measured-rdna4-fork-vs-strata" }, { "id": "2026-10-23-r9700-card-tuning-decode", "date": "2026-10-23T17:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "R9700 card tuning for decode: power caps barely matter, memory overclock backfires", "summary": "From published R9700 measurements: decode is memory-bound and draws about 180-190 W under a 300 W cap, so raising the cap doesn't add tok/s. A -120 mV undervolt gave +7.7-8.9% prefill but only +1.3% decode. On amdgpu DKMS 6.19.4 any memory overclock above the 1258 MHz top step is accepted, then pins memory lower and cost 6.6% decode in a clean A/B.", "setup": { "gpus": "Radeon AI PRO R9700 (published single-card tests)", "engine": "llama.cpp (published)", "model": "Qwen3.6-27B Q4_K_M (undervolt and memory-clock tests)" }, "metrics": [ { "label": "Decode power draw under 300 W cap", "value": "180-190", "unit": "W" }, { "label": "-120 mV undervolt, prefill pp32768", "value": 8.9, "unit": "%" }, { "label": "-120 mV undervolt, decode tg32", "value": 1.3, "unit": "%" }, { "label": "Memory OC above 1258 MHz, decode tg32", "value": -6.6, "unit": "%" }, { "label": "H200 decode energy saved by clock locking (up to)", "value": 32, "unit": "%" } ], "sources": [ "https://github.com/daimonionnn/amd-r9700-vllm-and-tuning-toolkit/blob/main/docs/r9700-oc-uv-findings.md", "https://www.reddit.com/r/ROCm/comments/1vuu58x/", "https://www.reddit.com/r/ROCm/comments/1vr5j5e/", "https://github.com/ggml-org/llama.cpp/discussions/21043", "https://arxiv.org/abs/2605.11999" ], "href": "https://amdai.net/lab-log#2026-10-23-r9700-card-tuning-decode" }, { "id": "2026-10-23-x-community-4bit-native", "date": "2026-10-23T16:20:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Community reports: the fastest 4-bit Flash numbers on R9700s come from native engines, not GGUF", "summary": "A scan of public R9700 posts on X (378 in the last 7 days) found the biggest Qwen3.8-Flash-Next numbers at 4-bit or above come from non-GGUF engines: about 228 tok/s single stream on two R9700s with Radiance's 4-bit experts, and about 200 tok/s on four R9700s with vLLM + r9700-stack MXFP4, versus 67 on llama.cpp on the same four cards. All figures are self-reported by their posters and unverified here.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "W4 / MXFP4-FP8 (4-bit class)", "engine": "Radiance 1.4.0; vLLM + r9700-stack (as reported)", "gpus": "2x and 4x Radeon AI PRO R9700 (as reported)", "context": "up to 256K (as reported)" }, "metrics": [ { "label": "Radiance W4, 2x R9700, single stream (reported)", "value": 228, "unit": "tok/s" }, { "label": "Radiance W4, 2x R9700, 16 concurrent (reported)", "value": 945, "unit": "tok/s" }, { "label": "vLLM + r9700-stack MXFP4 TP4, 4x R9700, single stream (reported)", "value": 200, "unit": "tok/s" }, { "label": "llama.cpp, same 4x R9700 (reported)", "value": 67, "unit": "tok/s" }, { "label": "Public R9700 posts on X, last 7 days", "value": 378, "unit": "posts" } ], "sources": [ "https://x.com/ECander84852/status/2110182801639408108", "https://x.com/dad__vibes/status/2111137765396210053", "https://x.com/dad__vibes/status/2111573866996977673", "https://x.com/dad__vibes/status/2111908353802998061", "https://x.com/sincoice/status/2113225429679210835" ], "href": "https://amdai.net/lab-log#2026-10-23-x-community-4bit-native" }, { "id": "2026-10-23-llamacpp-moe-expert-cache", "date": "2026-10-23T15:06:00-05:00", "project": "dual-r9700", "kind": "result", "title": "llama.cpp now caches offloaded MoE experts in VRAM, and Flash can use tensor split", "summary": "llama.cpp master added --moe-cache-mib (merged Oct 7, 2026; per-GPU multi-GPU support Oct 8), an LRU cache in VRAM for experts kept in system RAM. On Qwen3.8-Flash-Next Q4_0 with 2x RTX 4090, the author measured decode going from 34.78 to 64.39 tok/s (1.85x, 95.8% cache hits), with prompt speed at 0.73x in that all-cache setup and 0.91x with a smaller cache. A single-R9700 test on Windows showed the cache only paying off near 94% hits. The current source also allows --split-mode tensor for the Flash architecture (qwen4exp) and Qwen3.5/3.6 MoE files, so 'tensor mode does not work on MoE' is too broad: it is blocked for specific architectures such as DeepSeek2, GLM-DSA and MiniMax.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL (planned)", "engine": "llama.cpp master (HIP)", "gpus": "2x Radeon AI PRO R9700 (planned target)" }, "metrics": [ { "label": "Cited: 2x RTX 4090 decode, master --fit", "value": 34.78, "unit": "tok/s" }, { "label": "Cited: 2x RTX 4090 decode, -cmoe + 15000 MiB cache", "value": 64.39, "unit": "tok/s" }, { "label": "Cited: cache hit rate at that setting", "value": 95.8, "unit": "%" }, { "label": "Estimated expert VRAM budget, 2x R9700 at 262K f16", "value": 46.8, "unit": "GiB" } ], "sources": [ "https://github.com/ggml-org/llama.cpp/pull/29887", "https://github.com/ggml-org/llama.cpp/pull/30112", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp", "https://github.com/ggml-org/llama.cpp/pull/25028", "https://github.com/JohnTDI-cpu/llama-hip-p2p-allreduce", "https://forum.level1techs.com/t/does-the-lrlink-pcie-bridge-fix-rccls-broken-p2p-on-dual-r9700-update-yes-it-does/249623" ], "href": "https://amdai.net/lab-log#2026-10-23-llamacpp-moe-expert-cache" }, { "id": "2026-10-23-ngram-table-vram-budget", "date": "2026-10-23T14:05:00-05:00", "project": "idea-lab", "kind": "result", "title": "Qwen3.8-Flash-Next: the n-gram table is 27-51 GiB and belongs in DDR5", "summary": "Reading the GGUF headers shows the model's n-gram embedding table (per_layer_token_embd) is a single lookup tensor of 26.85 GiB at UD-Q4_K_XL (IQ4_NL) and 50.70 GiB at both UD-Q5_K_XL and UD-Q6_K_XL (Q8_0). Routed experts are 71.73, 91.61 and 101.75 GiB. Since only a few table rows are read per token, the VRAM question on two R9700s (64 GiB) is really only about experts. Q5 and Q6 differ by just ~10 GiB of experts.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL", "engine": "GGUF header analysis (no inference)", "gpus": "2x Radeon AI PRO R9700 (planned target)" }, "metrics": [ { "label": "N-gram table, UD-Q4_K_XL", "value": 26.85, "unit": "GiB" }, { "label": "N-gram table, UD-Q5_K_XL / Q6_K_XL", "value": 50.7, "unit": "GiB" }, { "label": "Routed experts, UD-Q4_K_XL", "value": 71.73, "unit": "GiB" }, { "label": "Routed experts, UD-Q5_K_XL", "value": 91.61, "unit": "GiB" }, { "label": "Routed experts, UD-Q6_K_XL", "value": 101.75, "unit": "GiB" } ], "sources": [ "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp" ], "href": "https://amdai.net/lab-log#2026-10-23-ngram-table-vram-budget" }, { "id": "2026-10-23-3090-vs-r9700", "date": "2026-10-23T13:56:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Cited comparison: dual 3090 vs dual R9700 at Q4", "summary": "Same model and quant (Qwen3.8-Flash-Next UD-Q4_K_XL): dual RTX 3090 on the llama.cpp moe-cache fork posts 39.28 tok/s decode; dual R9700 on Strata posts 64 tok/s (different host rigs). The gap on our rig is software and version, not hardware. A 3090 has more memory bandwidth (936 vs 640 GB/s), yet R9700s win when the engine keeps experts in RAM well.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL", "engine": "llamacpp-club3090 moe-cache (3090) vs Strata (R9700)", "engine_version": "per sources", "gpus": "2x RTX 3090 vs 2x Radeon AI PRO R9700", "context": "204,800 (3090) / 40K (R9700)", "flags": "per sources" }, "metrics": [ { "label": "2x RTX 3090 decode", "value": 39.28, "unit": "tok/s" }, { "label": "2x R9700 decode (Strata)", "value": 64, "unit": "tok/s" }, { "label": "1x R9700 decode (stock llama.cpp)", "value": "18-20", "unit": "tok/s" } ], "sources": [ "https://github.com/noonghunna/club-3090/blob/master/BENCHMARKS.md", "https://github.com/Niko1221/Strata/blob/main/docs/QUANTS.md", "https://www.techpowerup.com/gpu-specs/geforce-rtx-3090.c3622" ], "href": "https://amdai.net/lab-log#2026-10-23-3090-vs-r9700" }, { "id": "2026-10-23-sweet-spot-q5", "date": "2026-10-23T13:55:00-05:00", "project": "dual-r9700", "kind": "result", "title": "Sweet-spot quant for Flash models on dual R9700: UD-Q5_K_XL", "summary": "From published R9700 measurements: Q5_K_XL is almost as good as Q6_K_XL (mean KL vs Q8_0 0.038 vs 0.033, same top token 94.55% vs 94.61%) and only 2-7% slower than Q4_K_XL on two R9700s (60 vs 64 tok/s on short story, 94 vs 96 on code). Q4 is the speed fallback; Q6 is dominated.", "setup": { "model": "Qwen3.8-Flash-Next", "quant": "UD-Q4_K_XL / UD-Q5_K_XL / UD-Q6_K_XL", "engine": "Strata (published)", "engine_version": "per source", "gpus": "2x Radeon AI PRO R9700", "context": "40K", "flags": "layer split, 8-bit KV, MTP, every expert in RAM" }, "metrics": [ { "label": "Q5_K_XL decode, 2x R9700 (story)", "value": 60, "unit": "tok/s" }, { "label": "Q4_K_XL decode, 2x R9700 (story)", "value": 64, "unit": "tok/s" }, { "label": "Q5_K_XL mean KL vs Q8_0", "value": 0.038, "unit": "" }, { "label": "Q6_K_XL mean KL vs Q8_0", "value": 0.033, "unit": "" } ], "sources": [ "https://github.com/Niko1221/Strata/blob/main/docs/QUANTS.md", "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF", "https://unsloth.ai/docs/models/qwen3.8-next" ], "href": "https://amdai.net/lab-log#2026-10-23-sweet-spot-q5" } ] }