diff --git a/.github/ci/benchmark_config.json b/.github/ci/benchmark_config.json index 43f8895d..fbbe7b94 100644 --- a/.github/ci/benchmark_config.json +++ b/.github/ci/benchmark_config.json @@ -107,12 +107,16 @@ }, { "address": "0x02000000", - "paths": ["yolo/weights_region.bin"], + "paths": [ + "yolo/weights_region.bin" + ], "required": true }, { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -122,7 +126,11 @@ "image_count": 5, "min_image_count": 5, "reference_contract": ".github/ci/reference/yolo.json", - "source_shape": [480, 640, 3] + "source_shape": [ + 480, + 640, + 3 + ] }, "benchmark_cases": [ { @@ -130,7 +138,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_room_000139_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -146,7 +156,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_cat_524280_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -162,7 +174,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_giraffes_296969_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -178,7 +192,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_elephants_445248_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -194,7 +210,9 @@ "file_loads": [ { "address": "0x04A00000", - "paths": ["yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin"], + "paths": [ + "yolo/coco_baseball_043816_raw_480x640x3_uint8_rgb.bin" + ], "required": true } ], @@ -234,17 +252,24 @@ "file_loads": [ { "address": "0x0", - "paths": ["zero2m.bin", "common/zero2m.bin"], + "paths": [ + "zero2m.bin", + "common/zero2m.bin" + ], "required": true }, { "address": "0x2000", - "paths": ["dncnn/dncnn20l64_input.bin"], + "paths": [ + "dncnn/dncnn20l64_input.bin" + ], "required": true }, { "address": "0x14000", - "paths": ["dncnn/dncnn20l64_weights.bin"], + "paths": [ + "dncnn/dncnn20l64_weights.bin" + ], "required": true } ], @@ -253,7 +278,10 @@ "accuracy": { "kind": "uint8_npy", "offset": "0x10000", - "shape": [64, 64], + "shape": [ + 64, + 64 + ], "max_abs": 2, "reference_path": "ported_models/dncnn/refs/dncnn20l64_reference.npy", "comment": "Gates the 64x64 denoised output @0x10000 against the PyTorch/deepinv oracle (refs/dncnn20l64_reference.npy, produced by scripts/gen_dncnn_oracle.py running deepinv.models.DnCNN on the pinned weights). Board-verified: the int8 kernel matches the FP32 oracle at max_abs=1 (3/3 board runs); gate max_abs<=2 is a 1-unit margin." @@ -315,6 +343,18 @@ }, "smolvlm_500m": { "config": "ported_models/llama_cpp_et/benchmarks/smolvlm_500m.json" + }, + "smollm3_3b": { + "config": "ported_models/llama_cpp_et/benchmarks/smollm3_3b.json" + }, + "nemotron_mini_4b": { + "config": "ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json" + }, + "apertus_8b": { + "config": "ported_models/llama_cpp_et/benchmarks/apertus_8b.json" + }, + "glm4_9b": { + "config": "ported_models/llama_cpp_et/benchmarks/glm4_9b.json" } } } diff --git a/ported_models/apertus_8b/docs/RECIPE.md b/ported_models/apertus_8b/docs/RECIPE.md new file mode 100644 index 00000000..07af4a3f --- /dev/null +++ b/ported_models/apertus_8b/docs/RECIPE.md @@ -0,0 +1,51 @@ +# Apertus-8B-Instruct Porting Recipe + +## Overview + +Adds `swiss-ai/Apertus-8B-Instruct-2509` (8B-parameter causal LM, the Swiss +AI Initiative's fully-open, transparency-focused model) to the +`llama.cpp-et` framework. Confirmed via local GGUF metadata inspection: +`general.architecture = apertus` -- a genuinely distinct execution family, +not registered by any existing board identity. + +## Model Reference + +- **Source**: `swiss-ai/Apertus-8B-Instruct-2509` (Hugging Face), revision + `b946d40447b2b597999b9c86d44bee0b452c919f` +- **License**: Apache-2.0 +- **GGUF source**: `DevQuasar/swiss-ai.Apertus-8B-Instruct-2509-GGUF`, file + `swiss-ai.Apertus-8B-Instruct-2509.Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=823a2ce1b89f8e31c197167e144fb4e5220f0adb44132b8e54ccdb2dd5bedae7` + (verified locally against the downloaded file) +- **Architecture**: `arch = apertus` per GGUF metadata, 324 tensors. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly: + +``` +Final estimate: PPL = 7.8747 +/- 1.46638 +``` + +## Why this port likely needs no new ET-SoC1 kernel work + +Apertus uses a standard pre-norm transformer with RMSNorm, RoPE, GQA, and a +gated FFN (xIELU activation -- a parameterized variant of an existing +elementwise unary op, not a new op type) -- every primitive is already +proven on the ET backend by other decoders in this suite. ET-SoC1 board +execution itself is not something this session can produce. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/apertus_8b.json`, + and `.github/ci/benchmark_config.json` (port 18137) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/apertus_8b.json` is the model-ports + track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. diff --git a/ported_models/apertus_8b/oracle/perplexity_oracle.json b/ported_models/apertus_8b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..18f22f38 --- /dev/null +++ b/ported_models/apertus_8b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "DevQuasar/swiss-ai.Apertus-8B-Instruct-2509-GGUF", + "revision": "main", + "filename": "swiss-ai.Apertus-8B-Instruct-2509.Q8_0.gguf", + "sha256": "823a2ce1b89f8e31c197167e144fb4e5220f0adb44132b8e54ccdb2dd5bedae7" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model swiss-ai.Apertus-8B-Instruct-2509-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 7.8747, + "final_ppl_stderr": 1.46638, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [6.30, 9.45] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/glm4_9b/docs/RECIPE.md b/ported_models/glm4_9b/docs/RECIPE.md new file mode 100644 index 00000000..584aaa13 --- /dev/null +++ b/ported_models/glm4_9b/docs/RECIPE.md @@ -0,0 +1,54 @@ +# GLM-4-9B-0414 Porting Recipe + +## Overview + +Adds `zai-org/GLM-4-9B-0414` (9B-parameter causal LM, Zhipu AI's GLM-4 +series) to the `llama.cpp-et` framework. Confirmed via local GGUF metadata +inspection: `general.architecture = glm4` -- a genuinely distinct execution +family, not registered by any existing board identity (`chatglm`/`glm4` are +separate archs in llama.cpp; this is `glm4` specifically). + +## Model Reference + +- **Source**: `zai-org/GLM-4-9B-0414` (Hugging Face; this model was + previously published under `THUDM/GLM-4-9B-0414`, which now redirects to + `zai-org`), revision `645b8482494e31b6b752272bf7f7f273ef0f3caf` +- **License**: MIT +- **GGUF source**: `bartowski/THUDM_GLM-4-9B-0414-GGUF`, file + `THUDM_GLM-4-9B-0414-Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=7b4ea2795934ca05dc409251dddd289160a194a4d920b575d1de4516808bb50d` + (verified locally against the downloaded file) +- **Architecture**: `arch = glm4` per GGUF metadata, 523 tensors. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly: + +``` +Final estimate: PPL = 12.2894 +/- 2.42663 +``` + +## Why this port likely needs no new ET-SoC1 kernel work + +`LLM_ARCH_GLM4`'s graph builder uses RMSNorm (with an extra post-attention/ +post-FFN norm pair GLM4 adds -- still `GGML_OP_NORM`, no new op type), RoPE, +GQA, and a gated FFN -- all already proven on the ET backend. ET-SoC1 board +execution itself is not something this session can produce. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/glm4_9b.json`, + and `.github/ci/benchmark_config.json` (port 18138) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/glm4_9b.json` is the model-ports + track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. +- GGUF quantization used (`bartowski/THUDM_GLM-4-9B-0414-GGUF`) predates + the org rename and was built from the `THUDM` namespace; the sha256 above + is the artifact actually verified, independent of the naming change. diff --git a/ported_models/glm4_9b/oracle/perplexity_oracle.json b/ported_models/glm4_9b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..7a95794c --- /dev/null +++ b/ported_models/glm4_9b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "bartowski/THUDM_GLM-4-9B-0414-GGUF", + "revision": "main", + "filename": "THUDM_GLM-4-9B-0414-Q8_0.gguf", + "sha256": "7b4ea2795934ca05dc409251dddd289160a194a4d920b575d1de4516808bb50d" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model GLM-4-9B-0414-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 12.2894, + "final_ppl_stderr": 2.42663, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [9.83, 14.75] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/llama_cpp_et/artifacts.json b/ported_models/llama_cpp_et/artifacts.json index 9c419ae1..68907ee7 100644 --- a/ported_models/llama_cpp_et/artifacts.json +++ b/ported_models/llama_cpp_et/artifacts.json @@ -485,6 +485,74 @@ "sha256": "d1eb8b6b23979205fdf63703ed10f788131a3f812c7b1f72e0119d5d81295150", "size_bytes": 108783360, "note": "SmolVLM 500M vision projector (SigLIP ~93M + MLP). Q8_0 quantized. Must be loaded alongside smolvlm_500m_q8_gguf." + }, + "smollm3_3b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "SmolLM3-3B-Q8_0", + "filename": "SmolLM3-3B-Q8_0.gguf", + "env": "SMOLLM3_3B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "ggml-org/SmolLM3-3B-GGUF", + "revision": "main", + "filename": "SmolLM3-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/resolve/main/SmolLM3-Q8_0.gguf" + }, + "sha256": "8aa8cc74656137174a1988d993b00828e65a86fd68773412b632a75aa1373248", + "local_cache": "local-artifacts/models/SmolLM3-3B-Q8_0.gguf", + "board_path": "/data/models/SmolLM3-3B-Q8_0.gguf" + }, + "nemotron_mini_4b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "Nemotron-Mini-4B-Instruct-Q8_0", + "filename": "Nemotron-Mini-4B-Instruct-Q8_0.gguf", + "env": "NEMOTRON_MINI_4B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "bartowski/Nemotron-Mini-4B-Instruct-GGUF", + "revision": "main", + "filename": "Nemotron-Mini-4B-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/bartowski/Nemotron-Mini-4B-Instruct-GGUF/resolve/main/Nemotron-Mini-4B-Instruct-Q8_0.gguf" + }, + "sha256": "8ed6d148ded733d4401495e44dc834ffa79e8cfee6f44d8fb60808dddd78b8eb", + "local_cache": "local-artifacts/models/Nemotron-Mini-4B-Instruct-Q8_0.gguf", + "board_path": "/data/models/Nemotron-Mini-4B-Instruct-Q8_0.gguf" + }, + "apertus_8b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "swiss-ai.Apertus-8B-Instruct-2509-Q8_0", + "filename": "swiss-ai.Apertus-8B-Instruct-2509-Q8_0.gguf", + "env": "APERTUS_8B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "DevQuasar/swiss-ai.Apertus-8B-Instruct-2509-GGUF", + "revision": "main", + "filename": "swiss-ai.Apertus-8B-Instruct-2509.Q8_0.gguf", + "url": "https://huggingface.co/DevQuasar/swiss-ai.Apertus-8B-Instruct-2509-GGUF/resolve/main/swiss-ai.Apertus-8B-Instruct-2509.Q8_0.gguf" + }, + "sha256": "823a2ce1b89f8e31c197167e144fb4e5220f0adb44132b8e54ccdb2dd5bedae7", + "local_cache": "local-artifacts/models/swiss-ai.Apertus-8B-Instruct-2509-Q8_0.gguf", + "board_path": "/data/models/swiss-ai.Apertus-8B-Instruct-2509-Q8_0.gguf" + }, + "glm4_9b_q8_gguf": { + "kind": "model", + "framework": "llama.cpp-et", + "variant": "GLM-4-9B-0414-Q8_0", + "filename": "GLM-4-9B-0414-Q8_0.gguf", + "env": "GLM4_9B_MODEL_PATH", + "source": { + "type": "huggingface", + "repo": "bartowski/THUDM_GLM-4-9B-0414-GGUF", + "revision": "main", + "filename": "THUDM_GLM-4-9B-0414-Q8_0.gguf", + "url": "https://huggingface.co/bartowski/THUDM_GLM-4-9B-0414-GGUF/resolve/main/THUDM_GLM-4-9B-0414-Q8_0.gguf" + }, + "sha256": "7b4ea2795934ca05dc409251dddd289160a194a4d920b575d1de4516808bb50d", + "local_cache": "local-artifacts/models/GLM-4-9B-0414-Q8_0.gguf", + "board_path": "/data/models/GLM-4-9B-0414-Q8_0.gguf" } } } diff --git a/ported_models/llama_cpp_et/benchmarks/apertus_8b.json b/ported_models/llama_cpp_et/benchmarks/apertus_8b.json new file mode 100644 index 00000000..56b4921e --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/apertus_8b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "swiss-ai.Apertus-8B-Instruct-2509-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "apertus_8b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18137, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/llama_cpp_et/benchmarks/glm4_9b.json b/ported_models/llama_cpp_et/benchmarks/glm4_9b.json new file mode 100644 index 00000000..1b670c98 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/glm4_9b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "GLM-4-9B-0414-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "glm4_9b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18138, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json b/ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json new file mode 100644 index 00000000..39fc59a2 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "Nemotron-Mini-4B-Instruct-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "nemotron_mini_4b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18136, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/llama_cpp_et/benchmarks/smollm3_3b.json b/ported_models/llama_cpp_et/benchmarks/smollm3_3b.json new file mode 100644 index 00000000..5261c1e3 --- /dev/null +++ b/ported_models/llama_cpp_et/benchmarks/smollm3_3b.json @@ -0,0 +1,52 @@ +{ + "runner": "llama_server", + "board": true, + "framework": { + "name": "llama.cpp-et", + "runner": "llama_server", + "source_artifact": "llama_cpp_source" + }, + "artifacts_file": "../artifacts.json", + "canonical_variant": "SmolLM3-3B-Q8_0", + "score": { + "metric": "tokens_per_second", + "label": "Decode tokens/s", + "higher_is_better": true + }, + "llama_server": { + "source_artifact": "llama_cpp_source", + "model_artifact": "smollm3_3b_q8_gguf", + "server_artifact": "llama_server", + "workdir_artifact": "llama_cpp_build", + "host": "127.0.0.1", + "port": 18135, + "device": "ET", + "gpu_layers": 99, + "ctx_size": 2048, + "batch_size": 256, + "ubatch_size": 128, + "parallel": 1, + "cache_ram_mib": 0, + "ready_timeout_s": 300, + "request_timeout_s": 420, + "flash_attn": false, + "api": "completion", + "prompt": "Repeat this token sequence without commentary: OK OK OK OK OK OK OK OK OK OK", + "max_tokens": 96, + "temperature": 0, + "ignore_eos": true, + "min_completion_tokens": 32, + "perplexity": { + "enabled": true, + "perplexity_artifact": "llama_perplexity", + "corpus_artifact": "wikitext2_raw_test", + "ctx_size": 128, + "batch_size": 128, + "ubatch_size": 128, + "timeout_s": 420, + "min_ppl": 1.0, + "max_ppl": 1000.0, + "chunks": 4 + } + } +} diff --git a/ported_models/nemotron_mini_4b/docs/RECIPE.md b/ported_models/nemotron_mini_4b/docs/RECIPE.md new file mode 100644 index 00000000..ea0bd4f5 --- /dev/null +++ b/ported_models/nemotron_mini_4b/docs/RECIPE.md @@ -0,0 +1,51 @@ +# Nemotron-Mini-4B-Instruct Porting Recipe + +## Overview + +Adds `nvidia/Nemotron-Mini-4B-Instruct` (4B-parameter causal LM, NVIDIA's +distilled/pruned Minitron family) to the `llama.cpp-et` framework. Confirmed +via local GGUF metadata inspection: `general.architecture = nemotron` -- +a genuinely distinct execution family, not registered by any existing board +identity. + +## Model Reference + +- **Source**: `nvidia/Nemotron-Mini-4B-Instruct` (Hugging Face), revision + `791833e92ebddb0bc2c1007f6d2b6764f886a2ae` +- **License**: NVIDIA Community Model License +- **GGUF source**: `bartowski/Nemotron-Mini-4B-Instruct-GGUF`, file + `Nemotron-Mini-4B-Instruct-Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=8ed6d148ded733d4401495e44dc834ffa79e8cfee6f44d8fb60808dddd78b8eb` + (verified locally against the downloaded file) +- **Architecture**: `arch = nemotron` per GGUF metadata, 324 tensors. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly: + +``` +Final estimate: PPL = 10.5717 +/- 2.16348 +``` + +## Why this port likely needs no new ET-SoC1 kernel work + +`LLM_ARCH_NEMOTRON`'s graph builder is a standard pre-norm transformer +(squared-ReLU FFN activation instead of SiLU/GELU is its main quirk, but +that's an existing `GGML_UNARY_OP`, not a new op type) using RMSNorm, +standard MHA/GQA, and RoPE -- all already proven on the ET backend. ET-SoC1 +board execution itself is not something this session can produce. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json`, + and `.github/ci/benchmark_config.json` (port 18136) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/nemotron_mini_4b.json` is the + model-ports track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. diff --git a/ported_models/nemotron_mini_4b/oracle/perplexity_oracle.json b/ported_models/nemotron_mini_4b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..44be19e8 --- /dev/null +++ b/ported_models/nemotron_mini_4b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "bartowski/Nemotron-Mini-4B-Instruct-GGUF", + "revision": "main", + "filename": "Nemotron-Mini-4B-Instruct-Q8_0.gguf", + "sha256": "8ed6d148ded733d4401495e44dc834ffa79e8cfee6f44d8fb60808dddd78b8eb" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model Nemotron-Mini-4B-Instruct-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 10.5717, + "final_ppl_stderr": 2.16348, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [8.46, 12.69] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/smollm3_3b/docs/RECIPE.md b/ported_models/smollm3_3b/docs/RECIPE.md new file mode 100644 index 00000000..b5534b9b --- /dev/null +++ b/ported_models/smollm3_3b/docs/RECIPE.md @@ -0,0 +1,55 @@ +# SmolLM3-3B Porting Recipe + +## Overview + +Adds `HuggingFaceTB/SmolLM3-3B` (3B-parameter causal LM, Hugging Face's +third-generation Smol series) to the `llama.cpp-et` framework. Confirmed via +local GGUF metadata inspection: `general.architecture = smollm3` -- distinct +from `smollm2` (already a seed identity on this board's identity registry, +ineligible for credit as a size/checkpoint variant), llama.cpp registers +SmolLM3 as its own execution family (adds NoPE layers -- a subset of layers +skip RoPE entirely -- and native long-context/reasoning-mode support absent +from SmolLM2). + +## Model Reference + +- **Source**: `HuggingFaceTB/SmolLM3-3B` (Hugging Face), revision + `a07cc9a04f16550a088caea529712d1d335b0ac1` +- **License**: Apache-2.0 +- **GGUF source**: `ggml-org/SmolLM3-3B-GGUF` (official llama.cpp org + quantization), file `SmolLM3-Q8_0.gguf` +- **Quantization**: Q8_0, + `sha256=8aa8cc74656137174a1988d993b00828e65a86fd68773412b632a75aa1373248` + (verified locally against the downloaded file) +- **Architecture**: `arch = smollm3` per GGUF metadata, 326 tensors. + +## Verification performed this round + +Host reference: built a plain CPU-only (`GGML_ET=OFF`) configuration of the +same vendored `llama.cpp-et` source and ran `llama-perplexity` against the +board-pinned WikiText-2 corpus (`wikitext2_raw_test`, +`sha256=173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08`), +context 128 / batch 128 / ubatch 128 / 4 chunks. The model loads and runs +cleanly: + +``` +Final estimate: PPL = 12.3653 +/- 2.40724 +``` + +## Why this port likely needs no new ET-SoC1 kernel work + +SmolLM3's distinguishing feature (NoPE on a subset of layers) means those +layers simply skip the `GGML_OP_ROPE` call rather than needing a new op -- +the graph builder conditionally omits an already-proven op per layer. Every +other op (RMSNorm, GQA attention, SwiGLU FFN) is identical to every other +Smol-family/Llama-family decoder already proven on this backend. Not +confirmed live this round. + +## Open items for maintainer review + +- Registered in `artifacts.json`, `ported_models/llama_cpp_et/benchmarks/smollm3_3b.json`, + and `.github/ci/benchmark_config.json` (port 18135) -- board-testable now, + independent of the model-ports track claim below. +- `ported_models/submissions/model_ports/smollm3_3b.json` is the model-ports + track claim, pending identity approval. +- No changes to any protected file or the vendored submodule. diff --git a/ported_models/smollm3_3b/oracle/perplexity_oracle.json b/ported_models/smollm3_3b/oracle/perplexity_oracle.json new file mode 100644 index 00000000..1971fe24 --- /dev/null +++ b/ported_models/smollm3_3b/oracle/perplexity_oracle.json @@ -0,0 +1,25 @@ +{ + "oracle_type": "perplexity_threshold", + "model_artifact": { + "repo": "ggml-org/SmolLM3-3B-GGUF", + "revision": "main", + "filename": "SmolLM3-Q8_0.gguf", + "sha256": "8aa8cc74656137174a1988d993b00828e65a86fd68773412b632a75aa1373248" + }, + "corpus": { + "artifact": "wikitext2_raw_test", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + }, + "command": "llama-perplexity --model SmolLM3-3B-Q8_0.gguf -f wiki.test.raw -c 128 -b 128 -ub 128 --chunks 4", + "reference_run": { + "final_ppl": 12.3653, + "final_ppl_stderr": 2.40724, + "measured_on": "CPU (ggml-cpu backend, GGML_ET=OFF build)", + "measured_date": "2026-07-25" + }, + "comparison_threshold": { + "metric": "final_ppl", + "max_relative_deviation": 0.2, + "note": "Matches this repo's own leaderboard-gate policy (PPL must stay within 20% of best-seen value). A full-offload ET-SoC1 re-run against this exact command/corpus/artifact should land at final_ppl within [9.89, 14.84] to be considered consistent with this reference run. No ET-SoC1 hardware was available to this session to perform that re-run directly." + } +} diff --git a/ported_models/submissions/model_ports/apertus_8b.json b/ported_models/submissions/model_ports/apertus_8b.json new file mode 100644 index 00000000..20293f5b --- /dev/null +++ b/ported_models/submissions/model_ports/apertus_8b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "apertus_8b", + "identity_id": "apertus", + "source": { + "repo": "swiss-ai/Apertus-8B-Instruct-2509", + "revision": "b946d40447b2b597999b9c86d44bee0b452c919f", + "license": "apache-2.0" + }, + "implementation_paths": [ + "ported_models/apertus_8b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/apertus_8b.json", + "recipe": "ported_models/apertus_8b/docs/RECIPE.md" +} diff --git a/ported_models/submissions/model_ports/glm4_9b.json b/ported_models/submissions/model_ports/glm4_9b.json new file mode 100644 index 00000000..a80af7e8 --- /dev/null +++ b/ported_models/submissions/model_ports/glm4_9b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "glm4_9b", + "identity_id": "glm4", + "source": { + "repo": "zai-org/GLM-4-9B-0414", + "revision": "645b8482494e31b6b752272bf7f7f273ef0f3caf", + "license": "mit" + }, + "implementation_paths": [ + "ported_models/glm4_9b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/glm4_9b.json", + "recipe": "ported_models/glm4_9b/docs/RECIPE.md" +} diff --git a/ported_models/submissions/model_ports/nemotron_mini_4b.json b/ported_models/submissions/model_ports/nemotron_mini_4b.json new file mode 100644 index 00000000..fd90b5a4 --- /dev/null +++ b/ported_models/submissions/model_ports/nemotron_mini_4b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "nemotron_mini_4b", + "identity_id": "nemotron", + "source": { + "repo": "nvidia/Nemotron-Mini-4B-Instruct", + "revision": "791833e92ebddb0bc2c1007f6d2b6764f886a2ae", + "license": "nvidia-community-model-license" + }, + "implementation_paths": [ + "ported_models/nemotron_mini_4b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/nemotron_mini_4b.json", + "recipe": "ported_models/nemotron_mini_4b/docs/RECIPE.md" +} diff --git a/ported_models/submissions/model_ports/smollm3_3b.json b/ported_models/submissions/model_ports/smollm3_3b.json new file mode 100644 index 00000000..f031555c --- /dev/null +++ b/ported_models/submissions/model_ports/smollm3_3b.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "track": "most_models_ported", + "benchmark_model": "smollm3_3b", + "identity_id": "smollm3", + "source": { + "repo": "HuggingFaceTB/SmolLM3-3B", + "revision": "a07cc9a04f16550a088caea529712d1d335b0ac1", + "license": "apache-2.0" + }, + "implementation_paths": [ + "ported_models/smollm3_3b" + ], + "benchmark_config": "ported_models/llama_cpp_et/benchmarks/smollm3_3b.json", + "recipe": "ported_models/smollm3_3b/docs/RECIPE.md" +}