Transaction-Science/open-standards
Watch
1
0
Fork
You've already forked open-standards
 0
Code Issues Pull requests Projects Releases 17 Packages Wiki Activity Actions 138

jouleclaw-loader-gguf: 161 clippy warnings -> 0, and -D warnings now gates it #414

Merged
dcharlot merged 5 commits from fix/loader-gguf-lints into main 2026-10-22 04:30:57 -04:00
Conversation 0 Commits 5 Files changed 44 +550 -512
44 changed files with 550 additions and 512 deletions
Download patch file Download diff file Expand all files Collapse all files

11
.github/workflows/standards-build.yml vendored
View file

@ -594,4 +594,15 @@ jobs:
# behind `cfg(feature = "fabric")` was deleted on exactly that
# evidence and broke the fabric-meter build in this very job.
lint jouleclaw/jouleclaw-rs -p jouleclaw-omni --features metal
# And the crate omni loads its weights through, which was linted
# by nothing at all: 161 warnings when first measured, now 0.
#
# Default features only. Under `rayon fabric-meter` this crate is
# still clean, but clippy also reports jouleclaw-backend-apple
# through the path dependency (`--no-deps` does not exclude
# workspace members), and that crate has 43 warnings of its own --
# mostly `unused_unsafe` and `unused_imports` that are unused in
# THAT configuration and needed in others, so they want cfg work
# rather than deletion.
lint jouleclaw/jouleclaw-rs -p jouleclaw-loader-gguf
lint sandbox/sandbox-rs -p sandbox-meter -p sandbox-proof

5
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/benches/matmul_q4_k.rs
View file

@ -5,6 +5,11 @@
//! models (Q4_K_M). These numbers + the wave-24 Q8_0 numbers
//! tell the full quantized-matmul story on Apple Silicon.
// Benches and integration tests are their own crates, so the decision
// recorded in src/lib.rs does not reach them: these loops fill arrays by
// index, which is what the index is for.
#![allow(clippy::needless_range_loop)]
use criterion::{
black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput,
};

7
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/benches/matmul_q56_k.rs
View file

@ -1,5 +1,10 @@
//! Criterion bench for Q5_K + Q6_K matmul kernels (wave 28).
// Benches and integration tests are their own crates, so the decision
// recorded in src/lib.rs does not reach them: these loops fill arrays by
// index, which is what the index is for.
#![allow(clippy::needless_range_loop)]
use criterion::{
black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput,
};
@ -61,7 +66,7 @@ fn bench_q5_k(c: &mut Criterion) {
g.finish();
let mut probe_out = vec![0f32; 1];
if matmul_q5_k_amx_tiled(
&vec![0f32; 256], &vec![0u8; Q5_K_BLOCK_BYTES],
&vec![0f32; 256], &[0u8; Q5_K_BLOCK_BYTES],
&[], 1, 1, 256, &mut probe_out,
).is_err() { return; }
let mut g = c.benchmark_group("matmul_q5_k_amx_tiled");

9
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/benches/matmul_q8_0.rs
View file

@ -6,6 +6,11 @@
//! wave-12 prefetch wins show up as steady-state throughput
//! that's stable as `n` grows past the L2 cache.
// Benches and integration tests are their own crates, so the decision
// recorded in src/lib.rs does not reach them: these loops fill arrays by
// index, which is what the index is for.
#![allow(clippy::needless_range_loop)]
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use jouleclaw_loader_gguf::matmul_q8_0::{
matmul_q8_0, matmul_q8_0_scalar, Q8_0_BLOCK_BYTES, Q8_0_ELEMS,
@ -273,7 +278,7 @@ fn bench_matmul_q8_0_threaded(c: &mut Criterion) {
// Skip cleanly when rayon feature is off.
let mut probe_out = vec![0f32; 1];
if matmul_q8_0_threaded(
&vec![0f32; 32], &vec![0u8; Q8_0_BLOCK_BYTES], &[],
&[0f32; 32], &[0u8; Q8_0_BLOCK_BYTES], &[],
1, 1, 32, &mut probe_out,
)
.is_err()
@ -323,7 +328,7 @@ fn bench_matmul_q8_0_amx_tiled_threaded(c: &mut Criterion) {
use jouleclaw_loader_gguf::matmul_q8_0::matmul_q8_0_amx_tiled_threaded;
let mut probe_out = vec![0f32; 1];
if matmul_q8_0_amx_tiled_threaded(
&vec![0f32; 32], &vec![0u8; Q8_0_BLOCK_BYTES], &[],
&[0f32; 32], &[0u8; Q8_0_BLOCK_BYTES], &[],
1, 1, 32, &mut probe_out,
)
.is_err()

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/fabric_calibrate.rs
View file

@ -162,7 +162,7 @@ fn main() {
// Synthetic BF16 weight + activation.
let mut w = vec![0u8; 2 * n * k];
for (i, c) in w.chunks_exact_mut(2).enumerate() {
for (i, c) in w.as_chunks_mut::<2>().0.iter_mut().enumerate() {
let v = (((i * 40503) % 2000) as f32 / 1000.0 - 1.0) * 0.05;
c.copy_from_slice(
&jouleclaw_loader_gguf::matmul_bf16::f32_to_bf16_rne(v).to_le_bytes(),

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/load_model.rs
View file

@ -187,7 +187,7 @@ fn compute_metas(model: &GgufModel) -> Vec<StoreTensorMeta> {
fn print_top_n_by_size(metas: &[StoreTensorMeta], n: usize) {
println!("=== top tensors by byte span ===");
let mut sorted: Vec<&StoreTensorMeta> = metas.iter().collect();
sorted.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes));
sorted.sort_by_key(|a| std::cmp::Reverse(a.size_bytes));
for m in sorted.iter().take(n) {
println!(
" {:<60} {:>12} bytes ({})",
@ -229,7 +229,7 @@ fn run_tier_store(
store.register_tensors("m1", metas);
// Pin the top-K by byte span Hot.
let mut sorted: Vec<&StoreTensorMeta> = metas.iter().collect();
sorted.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes));
sorted.sort_by_key(|a| std::cmp::Reverse(a.size_bytes));
let t0 = Instant::now();
for (i, m) in sorted.iter().take(pin_top).enumerate() {
store.pin_hot("m1", m, i as u64 + 1)?;

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/load_openvino.rs
View file

@ -55,7 +55,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Top 10 tensors.
let mut sorted: Vec<&_> = m.tensors.iter().collect();
sorted.sort_by(|a, b| b.bin_size.cmp(&a.bin_size));
sorted.sort_by_key(|a| std::cmp::Reverse(a.bin_size));
println!();
println!("=== top 10 tensors by byte size ===");
for t in sorted.iter().take(10) {

6
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/load_safetensors.rs
View file

@ -5,9 +5,7 @@
//! cargo run --release --example load_safetensors -- path/to/model.safetensors
//! ```
use jouleclaw_loader_gguf::safetensors_reader::{
read_safetensors_file, SafetensorsDtype,
};
use jouleclaw_loader_gguf::safetensors_reader::read_safetensors_file;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::Instant;
@ -53,7 +51,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Top tensors by size.
let mut sorted: Vec<&_> = m.tensors.iter().collect();
sorted.sort_by(|a, b| b.data_len.cmp(&a.data_len));
sorted.sort_by_key(|a| std::cmp::Reverse(a.data_len));
println!();
println!("=== top 10 tensors by byte size ===");
for t in sorted.iter().take(10) {

5
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_bt_bench.rs
View file

@ -6,7 +6,7 @@ fn main() {
use std::time::Instant;
fn f32_to_bf16(v: f32) -> u16 {
let b = v.to_bits();
(((b + (((b >> 16) & 1) + 0x7fff)) >> 16)) as u16
((b + (((b >> 16) & 1) + 0x7fff)) >> 16) as u16
}
// (M tokens, N out, K in)
let shapes = [
@ -38,7 +38,8 @@ fn main() {
let mut neon = vec![0f32; m * n];
let t1 = Instant::now();
for _ in 0..iters {
jouleclaw_loader_gguf::matmul_bf16::matmul_bf16_threaded(&a, &w, &[], m, n, k, &mut neon);
jouleclaw_loader_gguf::matmul_bf16::matmul_bf16_threaded(&a, &w, &[], m, n, k, &mut neon)
.expect("threaded bf16 matmul");
}
let neon_ms = t1.elapsed().as_secs_f64() * 1e3 / iters as f64;

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_energy.rs
View file

@ -27,7 +27,7 @@ fn main() {
MemberRuns, NotMeasured,
};
use std::time::{Duration, Instant};
fn f32_to_bf16(v: f32) -> u16 { let b = v.to_bits(); (((b + (((b >> 16) & 1) + 0x7fff)) >> 16)) as u16 }
fn f32_to_bf16(v: f32) -> u16 { let b = v.to_bits(); ((b + (((b >> 16) & 1) + 0x7fff)) >> 16) as u16 }
/// Write nothing: say why, with the IOReport status, on stderr and exit non-zero.
fn refuse(path: &str, why: impl std::fmt::Display) -> ! {
eprintln!("{}", refusal(path, why, jouleclaw_energy::ioreport_status()));

14
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/attention.rs
View file

@ -272,13 +272,19 @@ pub fn attention_forward_f32_flash(
/// touched.
pub const FLASH_KV_CHUNK: usize = 256;
// Checked at compile time rather than in a test: a test only holds if someone
// runs it, and these are properties of a constant. Not required by the
// algorithm, but desirable for SIMD alignment when a NEON variant lands.
const _: () = assert!(FLASH_KV_CHUNK.is_power_of_two());
const _: () = assert!(FLASH_KV_CHUNK >= 64);
fn as_u8(s: &[f32]) -> &[u8] {
// SAFETY: f32 is plain-old-data; reinterpret to bytes
// for the prefetch byte-offset API. Lifetime bound to s.
unsafe {
std::slice::from_raw_parts(
s.as_ptr() as *const u8,
s.len() * std::mem::size_of::<f32>(),
std::mem::size_of_val(s),
)
}
}
@ -479,9 +485,7 @@ mod tests {
#[test]
fn flash_kv_chunk_constant_is_power_of_two() {
// Not required by the algorithm, but desirable for SIMD
// alignment when a future NEON variant lands.
assert!(FLASH_KV_CHUNK.is_power_of_two());
assert!(FLASH_KV_CHUNK >= 64);
// Both invariants are now checked at compile time, below; this test
// is kept only so the intent is findable by name.
}
}

10
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/bpe_tokenizer.rs
View file

@ -337,12 +337,11 @@ impl BpeTokenizer {
.map(|b| self.b2c[b as usize])
.collect();
// ignore_merges: whole pretoken already a vocab entry.
if self.ignore_merges {
if let Some(&id) = self.vocab.get(&mapped) {
if self.ignore_merges
&& let Some(&id) = self.vocab.get(&mapped) {
out.push(id);
return;
}
}
// BPE merge loop.
let mut parts: Vec<String> = mapped.chars().map(String::from).collect();
loop {
@ -350,11 +349,10 @@ impl BpeTokenizer {
for i in 0..parts.len().saturating_sub(1) {
if let Some(&rank) = self.merges.get(
&(parts[i].clone(), parts[i + 1].clone()),
) {
if best.is_none() || rank < best.unwrap().0 {
)
&& (best.is_none() || rank < best.unwrap().0) {
best = Some((rank, i));
}
}
}
let Some((_, i)) = best else { break };
let merged = format!("{}{}", parts[i], parts[i + 1]);

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/cohere2_moe.rs
View file

@ -216,7 +216,7 @@ impl Cohere2MoeModel {
}
let bytes = shards[m.shard].tensor_bytes(t);
Ok(bytes
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect())
};
@ -293,7 +293,7 @@ impl Cohere2MoeModel {
let k = t.shape[1] as usize;
let bytes = self.shards[self.embeddings.shard].tensor_bytes(t);
let row = &bytes[(token as usize) * k * 2..((token as usize) + 1) * k * 2];
for (i, c) in row.chunks_exact(2).enumerate() {
for (i, c) in row.as_chunks::<2>().0.iter().enumerate() {
out[i] = bf16_to_f32(u16::from_le_bytes([c[0], c[1]]));
}
}

44
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/dequant.rs
View file

@ -905,7 +905,7 @@ pub fn mxfp4_e8m0_to_scale(byte: u8) -> f32 {
return 0.0;
}
let exp = byte as i32 - 127;
if exp >= -126 && exp <= 127 {
if (-126..=127).contains(&exp) {
let bits = ((exp + 127) as u32) << 23;
f32::from_bits(bits)
} else if exp < -126 {
@ -1110,11 +1110,11 @@ mod iq4_nl_tests {
let exp = ((bits >> 23) & 0xFF) as i32 - 127 + 15;
let mant = bits & 0x007F_FFFF;
if exp <= 0 {
(sign << 15) as u16
sign << 15
} else if exp >= 31 {
((sign << 15) | (0x1F << 10)) as u16
(sign << 15) | (0x1F << 10)
} else {
((sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)) as u16
(sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)
}
}
@ -1191,11 +1191,11 @@ mod q40_q41_q50_q51_q81_tests {
let exp = ((bits >> 23) & 0xFF) as i32 - 127 + 15;
let mant = bits & 0x007F_FFFF;
if exp <= 0 {
(sign << 15) as u16
sign << 15
} else if exp >= 31 {
((sign << 15) | (0x1F << 10)) as u16
(sign << 15) | (0x1F << 10)
} else {
((sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)) as u16
(sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)
}
}
@ -1513,9 +1513,9 @@ mod stq1_0_tests {
fn f16_le(v: f32) -> [u8; 2] {
let bits: u16 = match v {
x if x == 1.0 => 0x3C00,
x if x == 0.5 => 0x3800,
x if x == 2.0 => 0x4000,
1.0 => 0x3C00,
0.5 => 0x3800,
2.0 => 0x4000,
_ => unreachable!(),
};
bits.to_le_bytes()
@ -1654,9 +1654,9 @@ mod q1_0_tests {
fn f16_le(v: f32) -> [u8; 2] {
let bits: u16 = match v {
x if x == 1.0 => 0x3C00,
x if x == 0.5 => 0x3800,
x if x == 2.0 => 0x4000,
1.0 => 0x3C00,
0.5 => 0x3800,
2.0 => 0x4000,
_ => unreachable!(),
};
bits.to_le_bytes()
@ -1669,7 +1669,7 @@ mod q1_0_tests {
let mut buf = Vec::new();
buf.extend_from_slice(&f16_le(0.5));
buf.push(0xA5);
buf.extend(std::iter::repeat(0u8).take(15)); // rest = bit 0 → -0.5
buf.extend(std::iter::repeat_n(0u8, 15)); // rest = bit 0 → -0.5
let v = dequantize_q1_0(&buf, Q1_0_BLOCK_ELEMS).unwrap();
assert_eq!(&v[0..8], &[0.5, -0.5, 0.5, -0.5, -0.5, 0.5, -0.5, 0.5]);
assert!(v[8..].iter().all(|&x| x == -0.5));
@ -1681,10 +1681,10 @@ mod q1_0_tests {
let mut buf = Vec::new();
// block0: d=1.0, all bits 1 → all +1.0
buf.extend_from_slice(&f16_le(1.0));
buf.extend(std::iter::repeat(0xFFu8).take(16));
buf.extend(std::iter::repeat_n(0xFFu8, 16));
// block1: d=2.0, all bits 0 → all -2.0
buf.extend_from_slice(&f16_le(2.0));
buf.extend(std::iter::repeat(0x00u8).take(16));
buf.extend(std::iter::repeat_n(0x00u8, 16));
let v = dequantize_q1_0(&buf, 2 * Q1_0_BLOCK_ELEMS).unwrap();
assert!(v[..128].iter().all(|&x| x == 1.0));
assert!(v[128..].iter().all(|&x| x == -2.0));
@ -1765,9 +1765,9 @@ mod q2_0_tests {
// minimal round-trippable encodings for the small exact values
// used in tests (1.0, 0.5, 2.0) — exercises f16_from_le too.
let bits: u16 = match v {
x if x == 1.0 => 0x3C00,
x if x == 0.5 => 0x3800,
x if x == 2.0 => 0x4000,
1.0 => 0x3C00,
0.5 => 0x3800,
2.0 => 0x4000,
_ => unreachable!("test uses only 1.0/0.5/2.0"),
};
bits.to_le_bytes()
@ -1782,7 +1782,7 @@ mod q2_0_tests {
let mut buf = Vec::new();
buf.extend_from_slice(&f16_le(0.5));
buf.push(0xE4);
buf.extend(std::iter::repeat(0u8).take(31)); // rest of qs (code 0 → -1)
buf.extend(std::iter::repeat_n(0u8, 31)); // rest of qs (code 0 → -1)
let v = dequantize_q2_0(&buf, Q2_0_BLOCK_ELEMS).unwrap();
assert_eq!(&v[0..4], &[-0.5, 0.0, 0.5, 1.0]);
// every remaining element is code 0 → (0-1)*0.5 = -0.5
@ -1796,10 +1796,10 @@ mod q2_0_tests {
let mut buf = Vec::new();
// block 0: d=1.0, all code 2 (+1) → all +1.0
buf.extend_from_slice(&f16_le(1.0));
buf.extend(std::iter::repeat(0xAAu8).take(32)); // 0b10_10_10_10 → all q=2
buf.extend(std::iter::repeat_n(0xAAu8, 32)); // 0b10_10_10_10 → all q=2
// block 1: d=2.0, all code 0 (-1) → all -2.0
buf.extend_from_slice(&f16_le(2.0));
buf.extend(std::iter::repeat(0x00u8).take(32));
buf.extend(std::iter::repeat_n(0x00u8, 32));
let v = dequantize_q2_0(&buf, 2 * Q2_0_BLOCK_ELEMS).unwrap();
assert!(v[..128].iter().all(|&x| x == 1.0));
assert!(v[128..].iter().all(|&x| x == -2.0));

6
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/devstral_resident.rs
View file

@ -79,15 +79,15 @@ fn read_f32_tensor(model: &GgufModel, name: &str) -> Result<Vec<f32>, LoadError>
fn dequant_to_f32(bytes: &[u8], dtype: GgmlType, n: usize) -> Vec<f32> {
match dtype {
GgmlType::F32 => bytes[..n * 4]
.chunks_exact(4)
.as_chunks::<4>().0.iter()
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect(),
GgmlType::F16 => bytes[..n * 2]
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| f16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect(),
GgmlType::BF16 => bytes[..n * 2]
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| f32::from_bits((u16::from_le_bytes([c[0], c[1]]) as u32) << 16))
.collect(),
GgmlType::Q4_K => crate::dequant::dequantize_q4_k(bytes, n).unwrap_or_else(|_| vec![0.0; n]),

5
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/fabric_dispatch.rs
View file

@ -1145,8 +1145,7 @@ mod enabled {
// measured µJ/GFLOP prices it. Skipping it silently
// under-credits the GPU and inflates the CPU's share.
let hd = kv.hd();
if hd > 0 {
let n_q = weights[0].1 / hd;
if let Some(n_q) = weights[0].1.checked_div(hd) {
let seq = kv.len();
ledger_record("local/metal-gpu", n_q, seq, hd);
ledger_record("local/metal-gpu", n_q, seq, hd);
@ -1203,7 +1202,7 @@ mod enabled {
// The kernel needs N%32 and K%32; M is padded up to a
// multiple of 64 (real prompt lengths aren't %64). Tiny
// problems aren't worth the GPU round trip.
if !eligible || n % 32 != 0 || k % 32 != 0 || m * n < 64 * 128 {
if !eligible || !n.is_multiple_of(32) || !k.is_multiple_of(32) || m * n < 64 * 128 {
return false;
}
debug_assert_eq!(a.len(), m * k);

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/falcon_h1.rs
View file

@ -182,7 +182,7 @@ impl FalconH1Model {
return Err(FalconH1Error::WrongDtype(name.to_string(), t.dtype));
}
let bytes = st.tensor_bytes(t);
Ok(bytes.chunks_exact(2)
Ok(bytes.as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect())
};
@ -274,7 +274,7 @@ impl FalconH1Model {
let bytes = self.st.tensor_bytes(t);
let row = &bytes[(token as usize) * k * 2..((token as usize) + 1) * k * 2];
let mult = self.config.embedding_multiplier;
for (i, c) in row.chunks_exact(2).enumerate() {
for (i, c) in row.as_chunks::<2>().0.iter().enumerate() {
out[i] = bf16_to_f32(u16::from_le_bytes([c[0], c[1]])) * mult;
}
}

5
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/gemma.rs
View file

@ -438,11 +438,10 @@ mod tests {
match remap_gemma_tensor_name(name) {
Some(g) => {
text_mapped += 1;
if let Some(rest) = g.strip_prefix("blk.") {
if let Some(d) = rest.find('.') {
if let Some(rest) = g.strip_prefix("blk.")
&& let Some(d) = rest.find('.') {
seen_layers.insert(rest[..d].parse::<usize>().unwrap());
}
}
if g == "per_layer_token_embd.weight" {
have_ple = true;
}

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/gemma4.rs
View file

@ -148,7 +148,7 @@ pub struct ForwardOut {
pub(crate) fn gelu_tanh(x: f32) -> f32 {
// gelu_pytorch_tanh
const C: f32 = 0.797_884_6; // sqrt(2/pi)
0.5 * x * (1.0 + ((C * (x + 0.044715 * x * x * x))).tanh())
0.5 * x * (1.0 + (C * (x + 0.044715 * x * x * x)).tanh())
}
/// `y[o] = Σ_i x[i] * W[o,i]` for the decode hot path (seq=1).

10
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/granite_h.rs
View file

@ -238,7 +238,7 @@ impl GraniteHModel {
if t.dtype != SafetensorsDtype::BF16 {
return Err(GraniteHError::WrongDtype(name.to_string(), t.dtype));
}
Ok(st.tensor_bytes(t).chunks_exact(2)
Ok(st.tensor_bytes(t).as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect())
};
@ -399,7 +399,7 @@ impl GraniteHModel {
let k = t.shape[1] as usize;
let bytes = self.tensor_bytes(self.embeddings);
let row = &bytes[(token as usize) * k * 2..((token as usize) + 1) * k * 2];
for (i, c) in row.chunks_exact(2).enumerate() {
for (i, c) in row.as_chunks::<2>().0.iter().enumerate() {
out[i] = bf16_to_f32(u16::from_le_bytes([c[0], c[1]]));
}
}
@ -1036,7 +1036,7 @@ pub(crate) mod tests {
let vocab = cfg.vocab_size as u64;
let mut t: Vec<(String, Vec<u64>, Vec<f32>)> = Vec::new();
let mut seed = 1usize;
let mut push = |name: String, shape: Vec<u64>, t: &mut Vec<_>, seed: &mut usize| {
let push = |name: String, shape: Vec<u64>, t: &mut Vec<_>, seed: &mut usize| {
let n: usize = shape.iter().product::<u64>() as usize;
t.push((name, shape, synth(n, *seed)));
*seed += 1;
@ -1202,7 +1202,7 @@ pub(crate) mod tests {
build_tiny_checkpoint(&path, &tiny_config());
let st = read_safetensors_file(&path).unwrap();
for info in &st.tensors {
let vals: Vec<f32> = st.tensor_bytes(info).chunks_exact(2)
let vals: Vec<f32> = st.tensor_bytes(info).as_chunks::<2>().0.iter()
.map(|c| crate::matmul_bf16::bf16_to_f32(
u16::from_le_bytes([c[0], c[1]])))
.collect();
@ -1262,7 +1262,7 @@ pub(crate) mod tests {
build_tiny_checkpoint(&p2, &tiny_config());
let st = read_safetensors_file(&p2).unwrap();
for info in &st.tensors {
let vals: Vec<f32> = st.tensor_bytes(info).chunks_exact(2)
let vals: Vec<f32> = st.tensor_bytes(info).as_chunks::<2>().0.iter()
.map(|c| crate::matmul_bf16::bf16_to_f32(
u16::from_le_bytes([c[0], c[1]])))
.collect();

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/hybrid_lm.rs
View file

@ -114,12 +114,12 @@ impl GenerateParams {
pub fn generate<M: HybridLm>(model: &M, prompt: &[u32], params: &GenerateParams) -> Vec<u32> {
assert!(!prompt.is_empty(), "empty prompt");
let mut state = model.new_state();
let mut sampling = params.sampling.clone();
let sampling = params.sampling.clone();
let mut history: Vec<u32> = prompt.to_vec();
let mut out = Vec::new();
let mut logits = model.forward_prompt(prompt, &mut state);
for _ in 0..params.max_new_tokens {
let token = sample_logits_with_history(&logits, &mut sampling, &history);
let token = sample_logits_with_history(&logits, &sampling, &history);
if params.stop_tokens.contains(&token) {
break;
}

518
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/kv_cache_inplace.rs
View file

@ -746,265 +746,7 @@ unsafe fn dequantize_row_neon(q: &[i8], scale: f32, out: &mut [f32]) {
}
}
#[cfg(test)]
mod kv_quant_tests {
use super::*;
#[test]
fn quant_dequant_roundtrip_is_bounded() {
// Synthetic K-like values in [-2, 2] over 4 rows of d_head=8.
let n_rows = 4;
let d_head = 8;
let mut fp32_bytes = Vec::with_capacity(n_rows * d_head * 4);
for r in 0..n_rows {
for i in 0..d_head {
let v = ((r * d_head + i) as f32) * 0.13 - 1.5;
fp32_bytes.extend_from_slice(&v.to_le_bytes());
}
}
let (q, scales) = quantize_per_row(&fp32_bytes, n_rows, d_head);
let back = dequantize_per_row(&q, &scales, n_rows, d_head);
// Error per element <= scale/2 (theoretical bound on symmetric
// round-to-nearest quant).
let mut max_abs_err = 0.0_f32;
for r in 0..n_rows {
for i in 0..d_head {
let off = (r * d_head + i) * 4;
let orig = f32::from_le_bytes([
fp32_bytes[off], fp32_bytes[off + 1],
fp32_bytes[off + 2], fp32_bytes[off + 3],
]);
let deq = f32::from_le_bytes([
back[off], back[off + 1], back[off + 2], back[off + 3],
]);
let err = (orig - deq).abs();
if err > max_abs_err { max_abs_err = err; }
assert!(err <= scales[r] / 2.0 + 1e-7,
"row {} col {}: err {} > bound {}", r, i, err, scales[r] / 2.0);
}
}
eprintln!("quant roundtrip max_abs_err = {max_abs_err:.6}");
}
#[test]
fn empty_cache_take_replace_roundtrip_int8_path() {
// Zero cache initially. take_buffers should yield zero fp32
// tensors; replace_buffers with non-zero values should
// quantize+store; next take_buffers should dequant to ~the
// same values (within bound).
let mut cache = InPlaceKvCache::new_with_quant(1, 2, 4, 4, KvQuant::Int8);
let (k0, v0) = cache.take_buffers(0);
// All zero on a fresh cache.
for &b in k0.storage.bytes.iter() { assert_eq!(b, 0); }
for &b in v0.storage.bytes.iter() { assert_eq!(b, 0); }
// Build a non-trivial fp32 K to put back.
let elems = 2 * 4 * 4;
let mut k_bytes = Vec::with_capacity(elems * 4);
for i in 0..elems {
let v = (i as f32) * 0.05 - 0.5;
k_bytes.extend_from_slice(&v.to_le_bytes());
}
let k_new = Tensor {
meta: TensorMeta::new(Dtype::F32, &[2, 4, 4]),
storage: std::sync::Arc::new(TensorStorage { bytes: k_bytes.clone(), mapped: None }),
};
cache.replace_buffers(0, k_new, v0);
// Take again, dequant, compare.
let (k_back, _) = cache.take_buffers(0);
let mut max_err = 0.0_f32;
for i in 0..elems {
let off = i * 4;
let orig = f32::from_le_bytes([
k_bytes[off], k_bytes[off + 1], k_bytes[off + 2], k_bytes[off + 3],
]);
let deq = f32::from_le_bytes([
k_back.storage.bytes[off], k_back.storage.bytes[off + 1],
k_back.storage.bytes[off + 2], k_back.storage.bytes[off + 3],
]);
let e = (orig - deq).abs();
if e > max_err { max_err = e; }
}
eprintln!("int8 cache roundtrip max_err = {max_err:.6}");
// Per-row max_abs/127 bound; rows here have max_abs <= 1, so
// err per element <= 1/254 ≈ 0.004.
assert!(max_err < 0.01, "max_err too high: {max_err}");
}
#[test]
fn cache_bytes_reports_4x_savings() {
let fp32 = InPlaceKvCache::new(24, 8, 128, 64);
let int8 = InPlaceKvCache::new_with_quant(24, 8, 128, 64, KvQuant::Int8);
let ratio = fp32.cache_bytes() as f64 / int8.cache_bytes() as f64;
eprintln!(
"cache_bytes: fp32={} int8={} ratio={:.2}x",
fp32.cache_bytes(), int8.cache_bytes(), ratio,
);
// ~4× — int8 trades 4× data for tiny scale overhead.
assert!(ratio > 3.5 && ratio < 4.5, "ratio {ratio} not ~4×");
}
/// Dequantize is pure integer-to-float multiply — NEON and scalar
/// must agree bit-for-bit (same f32 multiply, just lane-batched).
#[cfg(target_arch = "aarch64")]
#[test]
fn dequantize_row_neon_matches_scalar_bitwise() {
let d_head = 64;
let scale = 0.0123_f32;
// Cover the full int8 range a few times, plus tail elements
// that don't fit a 4-wide chunk.
let q: Vec<i8> = (0..d_head).map(|i| ((i as i32 * 7 - 3) & 0xFF) as i8).collect();
let mut out_neon = vec![0.0_f32; d_head];
let mut out_scalar = vec![0.0_f32; d_head];
unsafe { super::dequantize_row_neon(&q, scale, &mut out_neon); }
super::dequantize_row_scalar(&q, scale, &mut out_scalar);
for (i, (&n, &s)) in out_neon.iter().zip(&out_scalar).enumerate() {
assert_eq!(n.to_bits(), s.to_bits(),
"row[{i}]: NEON {n} bits={:08x} vs scalar {s} bits={:08x}",
n.to_bits(), s.to_bits());
}
}
/// Standalone kernel bench: NEON vs scalar quant + dequant, no
/// allocator / cache / executor in the loop. Confirms the inner
/// SIMD kernel does the work it should — if this ratio is <1 (NEON
/// faster) but the full Bonsai bench is barely affected, the
/// bottleneck is somewhere else (likely allocation churn).
///
/// Run: cargo test --release -p jouleclaw-loader-gguf
/// bench_quant_kernel -- --ignored --nocapture
#[cfg(target_arch = "aarch64")]
#[test]
#[ignore]
fn bench_quant_kernel() {
use std::time::Instant;
// Match Bonsai-1.7B: d_head=64, n_heads_kv * max_seq = 8 * 128 = 1024 rows.
let d_head = 64;
let n_rows = 1024;
let row_count = n_rows;
// Pre-generate the input.
let mut rows = Vec::with_capacity(row_count * d_head);
for r in 0..row_count {
for i in 0..d_head {
let v = ((r * d_head + i) as f32) * 0.001 - 0.5;
rows.push(v);
}
}
let mut q_scalar = vec![0_i8; row_count * d_head];
let mut q_neon = vec![0_i8; row_count * d_head];
// Warmup.
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
let _ = super::quantize_row_scalar(row, &mut q_scalar[r * d_head..(r + 1) * d_head]);
}
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
unsafe {
let _ = super::quantize_row_neon(row, &mut q_neon[r * d_head..(r + 1) * d_head]);
}
}
let iters = 100;
// Scalar quantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
let _ = super::quantize_row_scalar(row, &mut q_scalar[r * d_head..(r + 1) * d_head]);
}
}
let t_scalar_q = t.elapsed();
// NEON quantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
unsafe {
let _ = super::quantize_row_neon(row, &mut q_neon[r * d_head..(r + 1) * d_head]);
}
}
}
let t_neon_q = t.elapsed();
// Scalar dequantize.
let scale = 0.01;
let mut out_scalar = vec![0.0_f32; row_count * d_head];
let mut out_neon = vec![0.0_f32; row_count * d_head];
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
super::dequantize_row_scalar(
&q_scalar[r * d_head..(r + 1) * d_head], scale,
&mut out_scalar[r * d_head..(r + 1) * d_head]);
}
}
let t_scalar_d = t.elapsed();
// NEON dequantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
unsafe {
super::dequantize_row_neon(
&q_scalar[r * d_head..(r + 1) * d_head], scale,
&mut out_neon[r * d_head..(r + 1) * d_head]);
}
}
}
let t_neon_d = t.elapsed();
let q_ratio = t_neon_q.as_secs_f64() / t_scalar_q.as_secs_f64();
let d_ratio = t_neon_d.as_secs_f64() / t_scalar_d.as_secs_f64();
eprintln!(
"[d_head={d_head}, n_rows={n_rows}, iters={iters}] \
quant: scalar={:.3}ms neon={:.3}ms neon/scalar={:.2}x \
dequant: scalar={:.3}ms neon={:.3}ms neon/scalar={:.2}x",
t_scalar_q.as_secs_f64() * 1000.0,
t_neon_q.as_secs_f64() * 1000.0,
q_ratio,
t_scalar_d.as_secs_f64() * 1000.0,
t_neon_d.as_secs_f64() * 1000.0,
d_ratio,
);
}
/// Quantize differs in rounding semantics — `vrndnq_f32` is round-
/// to-nearest-ties-to-even, `f32::round` is round-to-nearest-ties-
/// away-from-zero. They agree on every non-tie value; ties can
/// differ by ±1 in the i8 output. For typical fp values this
/// happens essentially never; we test that NEON is within ±1 of
/// scalar everywhere, and bit-exact on non-tie cases.
#[cfg(target_arch = "aarch64")]
#[test]
fn quantize_row_neon_within_one_ulp_of_scalar() {
let d_head = 64;
// Synthetic fp32 row with values that span [-2, 2], a mix of
// tie and non-tie quantize points.
let row: Vec<f32> = (0..d_head)
.map(|i| (i as f32) * 0.061 - 1.95)
.collect();
let mut q_neon = vec![0_i8; d_head];
let mut q_scalar = vec![0_i8; d_head];
let s_neon = unsafe { super::quantize_row_neon(&row, &mut q_neon) };
let s_scalar = super::quantize_row_scalar(&row, &mut q_scalar);
assert!((s_neon - s_scalar).abs() < 1e-6, "scales differ");
let mut max_diff: i32 = 0;
for (i, (&n, &s)) in q_neon.iter().zip(&q_scalar).enumerate() {
let d = (n as i32 - s as i32).abs();
if d > max_diff { max_diff = d; }
assert!(d <= 1,
"row[{i}]: NEON {n} vs scalar {s} differs by {d} (>1 ulp)");
}
eprintln!("quantize_row_neon vs scalar max diff = {max_diff} ULP");
}
}
/// In-place decode-step graph. Inputs are token IDs + per-layer K/V buffers
/// (always present, always full-sized); outputs are per-layer updated K/V
@ -1497,3 +1239,263 @@ pub fn build_head_only_graph(
logits_output_name: "logits".into(),
})
}
#[cfg(test)]
mod kv_quant_tests {
use super::*;
#[test]
fn quant_dequant_roundtrip_is_bounded() {
// Synthetic K-like values in [-2, 2] over 4 rows of d_head=8.
let n_rows = 4;
let d_head = 8;
let mut fp32_bytes = Vec::with_capacity(n_rows * d_head * 4);
for r in 0..n_rows {
for i in 0..d_head {
let v = ((r * d_head + i) as f32) * 0.13 - 1.5;
fp32_bytes.extend_from_slice(&v.to_le_bytes());
}
}
let (q, scales) = quantize_per_row(&fp32_bytes, n_rows, d_head);
let back = dequantize_per_row(&q, &scales, n_rows, d_head);
// Error per element <= scale/2 (theoretical bound on symmetric
// round-to-nearest quant).
let mut max_abs_err = 0.0_f32;
for r in 0..n_rows {
for i in 0..d_head {
let off = (r * d_head + i) * 4;
let orig = f32::from_le_bytes([
fp32_bytes[off], fp32_bytes[off + 1],
fp32_bytes[off + 2], fp32_bytes[off + 3],
]);
let deq = f32::from_le_bytes([
back[off], back[off + 1], back[off + 2], back[off + 3],
]);
let err = (orig - deq).abs();
if err > max_abs_err { max_abs_err = err; }
assert!(err <= scales[r] / 2.0 + 1e-7,
"row {} col {}: err {} > bound {}", r, i, err, scales[r] / 2.0);
}
}
eprintln!("quant roundtrip max_abs_err = {max_abs_err:.6}");
}
#[test]
fn empty_cache_take_replace_roundtrip_int8_path() {
// Zero cache initially. take_buffers should yield zero fp32
// tensors; replace_buffers with non-zero values should
// quantize+store; next take_buffers should dequant to ~the
// same values (within bound).
let mut cache = InPlaceKvCache::new_with_quant(1, 2, 4, 4, KvQuant::Int8);
let (k0, v0) = cache.take_buffers(0);
// All zero on a fresh cache.
for &b in k0.storage.bytes.iter() { assert_eq!(b, 0); }
for &b in v0.storage.bytes.iter() { assert_eq!(b, 0); }
// Build a non-trivial fp32 K to put back.
let elems = 2 * 4 * 4;
let mut k_bytes = Vec::with_capacity(elems * 4);
for i in 0..elems {
let v = (i as f32) * 0.05 - 0.5;
k_bytes.extend_from_slice(&v.to_le_bytes());
}
let k_new = Tensor {
meta: TensorMeta::new(Dtype::F32, &[2, 4, 4]),
storage: std::sync::Arc::new(TensorStorage { bytes: k_bytes.clone(), mapped: None }),
};
cache.replace_buffers(0, k_new, v0);
// Take again, dequant, compare.
let (k_back, _) = cache.take_buffers(0);
let mut max_err = 0.0_f32;
for i in 0..elems {
let off = i * 4;
let orig = f32::from_le_bytes([
k_bytes[off], k_bytes[off + 1], k_bytes[off + 2], k_bytes[off + 3],
]);
let deq = f32::from_le_bytes([
k_back.storage.bytes[off], k_back.storage.bytes[off + 1],
k_back.storage.bytes[off + 2], k_back.storage.bytes[off + 3],
]);
let e = (orig - deq).abs();
if e > max_err { max_err = e; }
}
eprintln!("int8 cache roundtrip max_err = {max_err:.6}");
// Per-row max_abs/127 bound; rows here have max_abs <= 1, so
// err per element <= 1/254 ≈ 0.004.
assert!(max_err < 0.01, "max_err too high: {max_err}");
}
#[test]
fn cache_bytes_reports_4x_savings() {
let fp32 = InPlaceKvCache::new(24, 8, 128, 64);
let int8 = InPlaceKvCache::new_with_quant(24, 8, 128, 64, KvQuant::Int8);
let ratio = fp32.cache_bytes() as f64 / int8.cache_bytes() as f64;
eprintln!(
"cache_bytes: fp32={} int8={} ratio={:.2}x",
fp32.cache_bytes(), int8.cache_bytes(), ratio,
);
// ~4× — int8 trades 4× data for tiny scale overhead.
assert!(ratio > 3.5 && ratio < 4.5, "ratio {ratio} not ~4×");
}
/// Dequantize is pure integer-to-float multiply — NEON and scalar
/// must agree bit-for-bit (same f32 multiply, just lane-batched).
#[cfg(target_arch = "aarch64")]
#[test]
fn dequantize_row_neon_matches_scalar_bitwise() {
let d_head = 64;
let scale = 0.0123_f32;
// Cover the full int8 range a few times, plus tail elements
// that don't fit a 4-wide chunk.
let q: Vec<i8> = (0..d_head).map(|i| ((i as i32 * 7 - 3) & 0xFF) as i8).collect();
let mut out_neon = vec![0.0_f32; d_head];
let mut out_scalar = vec![0.0_f32; d_head];
unsafe { super::dequantize_row_neon(&q, scale, &mut out_neon); }
super::dequantize_row_scalar(&q, scale, &mut out_scalar);
for (i, (&n, &s)) in out_neon.iter().zip(&out_scalar).enumerate() {
assert_eq!(n.to_bits(), s.to_bits(),
"row[{i}]: NEON {n} bits={:08x} vs scalar {s} bits={:08x}",
n.to_bits(), s.to_bits());
}
}
/// Standalone kernel bench: NEON vs scalar quant + dequant, no
/// allocator / cache / executor in the loop. Confirms the inner
/// SIMD kernel does the work it should — if this ratio is <1 (NEON
/// faster) but the full Bonsai bench is barely affected, the
/// bottleneck is somewhere else (likely allocation churn).
///
/// Run: cargo test --release -p jouleclaw-loader-gguf
/// bench_quant_kernel -- --ignored --nocapture
#[cfg(target_arch = "aarch64")]
#[test]
#[ignore]
fn bench_quant_kernel() {
use std::time::Instant;
// Match Bonsai-1.7B: d_head=64, n_heads_kv * max_seq = 8 * 128 = 1024 rows.
let d_head = 64;
let n_rows = 1024;
let row_count = n_rows;
// Pre-generate the input.
let mut rows = Vec::with_capacity(row_count * d_head);
for r in 0..row_count {
for i in 0..d_head {
let v = ((r * d_head + i) as f32) * 0.001 - 0.5;
rows.push(v);
}
}
let mut q_scalar = vec![0_i8; row_count * d_head];
let mut q_neon = vec![0_i8; row_count * d_head];
// Warmup.
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
let _ = super::quantize_row_scalar(row, &mut q_scalar[r * d_head..(r + 1) * d_head]);
}
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
unsafe {
let _ = super::quantize_row_neon(row, &mut q_neon[r * d_head..(r + 1) * d_head]);
}
}
let iters = 100;
// Scalar quantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
let _ = super::quantize_row_scalar(row, &mut q_scalar[r * d_head..(r + 1) * d_head]);
}
}
let t_scalar_q = t.elapsed();
// NEON quantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
let row = &rows[r * d_head..(r + 1) * d_head];
unsafe {
let _ = super::quantize_row_neon(row, &mut q_neon[r * d_head..(r + 1) * d_head]);
}
}
}
let t_neon_q = t.elapsed();
// Scalar dequantize.
let scale = 0.01;
let mut out_scalar = vec![0.0_f32; row_count * d_head];
let mut out_neon = vec![0.0_f32; row_count * d_head];
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
super::dequantize_row_scalar(
&q_scalar[r * d_head..(r + 1) * d_head], scale,
&mut out_scalar[r * d_head..(r + 1) * d_head]);
}
}
let t_scalar_d = t.elapsed();
// NEON dequantize.
let t = Instant::now();
for _ in 0..iters {
for r in 0..row_count {
unsafe {
super::dequantize_row_neon(
&q_scalar[r * d_head..(r + 1) * d_head], scale,
&mut out_neon[r * d_head..(r + 1) * d_head]);
}
}
}
let t_neon_d = t.elapsed();
let q_ratio = t_neon_q.as_secs_f64() / t_scalar_q.as_secs_f64();
let d_ratio = t_neon_d.as_secs_f64() / t_scalar_d.as_secs_f64();
eprintln!(
"[d_head={d_head}, n_rows={n_rows}, iters={iters}] \
quant: scalar={:.3}ms neon={:.3}ms neon/scalar={:.2}x \
dequant: scalar={:.3}ms neon={:.3}ms neon/scalar={:.2}x",
t_scalar_q.as_secs_f64() * 1000.0,
t_neon_q.as_secs_f64() * 1000.0,
q_ratio,
t_scalar_d.as_secs_f64() * 1000.0,
t_neon_d.as_secs_f64() * 1000.0,
d_ratio,
);
}
/// Quantize differs in rounding semantics — `vrndnq_f32` is round-
/// to-nearest-ties-to-even, `f32::round` is round-to-nearest-ties-
/// away-from-zero. They agree on every non-tie value; ties can
/// differ by ±1 in the i8 output. For typical fp values this
/// happens essentially never; we test that NEON is within ±1 of
/// scalar everywhere, and bit-exact on non-tie cases.
#[cfg(target_arch = "aarch64")]
#[test]
fn quantize_row_neon_within_one_ulp_of_scalar() {
let d_head = 64;
// Synthetic fp32 row with values that span [-2, 2], a mix of
// tie and non-tie quantize points.
let row: Vec<f32> = (0..d_head)
.map(|i| (i as f32) * 0.061 - 1.95)
.collect();
let mut q_neon = vec![0_i8; d_head];
let mut q_scalar = vec![0_i8; d_head];
let s_neon = unsafe { super::quantize_row_neon(&row, &mut q_neon) };
let s_scalar = super::quantize_row_scalar(&row, &mut q_scalar);
assert!((s_neon - s_scalar).abs() < 1e-6, "scales differ");
let mut max_diff: i32 = 0;
for (i, (&n, &s)) in q_neon.iter().zip(&q_scalar).enumerate() {
let d = (n as i32 - s as i32).abs();
if d > max_diff { max_diff = d; }
assert!(d <= 1,
"row[{i}]: NEON {n} vs scalar {s} differs by {d} (>1 ulp)");
}
eprintln!("quantize_row_neon vs scalar max diff = {max_diff} ULP");
}
}

31
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/lib.rs
View file

@ -16,6 +16,24 @@
// handle, the path a store was opened from), and a few helpers are written
// ahead of the path that will call them.
#![allow(dead_code)]
//
// ── pedantic/style lints this crate has decided against ─────────────────────
//
// The same three decisions jouleclaw-omni's lib.rs records, for the same
// reasons: this is quantisation and tokeniser code with the same shapes.
//
// Quantised kernels walk several arrays at one index, often at an offset from
// the loop variable, which iterator form cannot express without obscuring
// which index is read.
#![allow(clippy::needless_range_loop)]
// A dequantiser's signature is its block format: pointer, length, scale,
// output, element count. Grouping them would add single-use structs per
// format.
#![allow(clippy::too_many_arguments)]
// Doc-comment continuation indentation and list nesting, on comments that
// already read correctly.
#![allow(clippy::doc_lazy_continuation)]
#![allow(clippy::doc_overindented_list_items)]
//! # jouleclaw-loader-gguf
//!
@ -667,6 +685,7 @@ pub fn read_gguf_file_with_policy<P: AsRef<Path>>(
/// policy mismatch costs at most one extra page-fault round
/// vs the no-prefetch path.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Default)]
pub enum PrefetchPolicy {
/// No prefetch — pages fault in synchronously on first
/// touch. Identical to pre-wave-2 behaviour. Pick this
@ -679,6 +698,7 @@ pub enum PrefetchPolicy {
/// embeddings, Sequential for matmul weights, WillNeed
/// for norms). Default. Best cold-cache TTFT for
/// full-model inference.
#[default]
All,
/// Hint the top `N` tensors by byte span as `WillNeed`,
/// leaving the long tail to fault in lazily. Per-tensor
@ -695,11 +715,6 @@ pub enum PrefetchPolicy {
PerTensorOnly,
}
impl Default for PrefetchPolicy {
fn default() -> Self {
Self::All
}
}
/// Parse a GGUF model whose tensor data is already mmapped. Header
/// is parsed via a `Cursor` over the mmap; the tensor data section
@ -995,7 +1010,7 @@ fn apply_prefetch_policy(
ranked[w].2 = next_off.saturating_sub(ranked[w].1);
}
// Sort by computed byte span descending; take top N.
ranked.sort_by(|a, b| b.2.cmp(&a.2));
ranked.sort_by_key(|a| std::cmp::Reverse(a.2));
for &(_, off, span) in ranked.iter().take(n) {
let start = data_offset + off as usize;
let end = (start + span as usize).min(bytes.len());
@ -1323,8 +1338,8 @@ pub fn qmatmul_expert(
/// non-quantised tensors must be dequantised to f32 instead.
pub fn qmatmul_ok(dtype: GgmlType, in_d: usize) -> bool {
match dtype {
GgmlType::Q8_0 => in_d % 32 == 0,
GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K => in_d % 256 == 0,
GgmlType::Q8_0 => in_d.is_multiple_of(32),
GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K => in_d.is_multiple_of(256),
_ => false,
}
}

9
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/llama.rs
View file

@ -544,21 +544,18 @@ fn build_block(
// intermediates instead of three separate per-projection command
// buffers (each a commit+wait). Value-preserving; decode only
// (seq_len == 1). Prefill / other quant keep the unfused path.
if seq_len == 1 && std::env::var("NO_MLP_BLOCK").is_err() {
if let (
if seq_len == 1 && std::env::var("NO_MLP_BLOCK").is_err()
&& let (
&Weight::Q4K { node: gn, out: gf, k: gh },
&Weight::Q4K { node: un, out: uf, k: uh },
&Weight::Q6K { node: dn, out: dout, k: dk },
) = (&w_gate, &w_up, &w_down)
{
if uh == gh && uf == gf && dk == gf && dout == gh
&& uh == gh && uf == gf && dk == gf && dout == gh
&& gh % 256 == 0 && gf % 256 == 0
{
let ffn_out = g.mlp_block_q4k_q6k(xn2, gn, un, dn, gh, gf);
return Ok(g.add(after_attn, ffn_out));
}
}
}
let gate = wmm(g, xn2, &w_gate);
let gate = g.activation(gate, ActivationKind::SiLU);

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/matmul_bf16.rs
View file

@ -420,7 +420,7 @@ mod tests {
m: usize, n: usize, k: usize,
) -> Vec<f32> {
let w_f32: Vec<f32> = w_bf16
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect();
let mut out = vec![0f32; m * n];

10
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/matmul_f.rs
View file

@ -236,7 +236,7 @@ fn bytemuck_f32_as_u8(s: &[f32]) -> &[u8] {
unsafe {
std::slice::from_raw_parts(
s.as_ptr() as *const u8,
s.len() * std::mem::size_of::<f32>(),
std::mem::size_of_val(s),
)
}
}
@ -328,7 +328,7 @@ mod tests {
matmul_f16_scalar(&a, &w_f16, &bias, m, n, k, &mut got);
// Reference via the f16-decoded-as-f32 weight.
let w_f32_via_f16: Vec<f32> = w_f16
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| f16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect();
let expected = naive_matmul_f32(&a, &w_f32_via_f16, &bias, m, n, k);
@ -346,11 +346,11 @@ mod tests {
let exp = ((bits >> 23) & 0xFF) as i32 - 127 + 15;
let mant = bits & 0x007F_FFFF;
if exp <= 0 {
(sign << 15) as u16
sign << 15
} else if exp >= 31 {
((sign << 15) | (0x1F << 10)) as u16
(sign << 15) | (0x1F << 10)
} else {
((sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)) as u16
(sign << 15) | ((exp as u16) << 10) | ((mant >> 13) as u16)
}
}
}

14
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/matmul_q4_k.rs
View file

@ -287,17 +287,15 @@ unsafe fn sub_block_neon_sums(
// half (low or high) gives 32 i8 values.
let qs_a = vld1q_u8(qs_ptr);
let qs_b = vld1q_u8(qs_ptr.add(16));
let nibbles_a;
let nibbles_b;
if high_nibble {
let (nibbles_a, nibbles_b) = if high_nibble {
// Take high 4 bits.
nibbles_a = vshrq_n_u8(qs_a, 4);
nibbles_b = vshrq_n_u8(qs_b, 4);
(vshrq_n_u8(qs_a, 4), vshrq_n_u8(qs_b, 4))
} else {
let mask = vdupq_n_u8(0x0F);
nibbles_a = vandq_u8(qs_a, mask);
nibbles_b = vandq_u8(qs_b, mask);
}
(vandq_u8(qs_a, mask), vandq_u8(qs_b, mask))
};
// Widen u8 → u16 (zero-extend).
let u16_a_lo = vmovl_u8(vget_low_u8(nibbles_a));
let u16_a_hi = vmovl_u8(vget_high_u8(nibbles_a));

14
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/matmul_q5_k.rs
View file

@ -227,16 +227,14 @@ unsafe fn q5_k_sub_block_neon_sums(
let high_half = sub % 2 == 1;
let qs_a = vld1q_u8(qs_ptr);
let qs_b = vld1q_u8(qs_ptr.add(16));
let nib_a;
let nib_b;
if high_half {
nib_a = vshrq_n_u8(qs_a, 4);
nib_b = vshrq_n_u8(qs_b, 4);
let (nib_a, nib_b) = if high_half {
(vshrq_n_u8(qs_a, 4), vshrq_n_u8(qs_b, 4))
} else {
let mask = vdupq_n_u8(0x0F);
nib_a = vandq_u8(qs_a, mask);
nib_b = vandq_u8(qs_b, mask);
}
(vandq_u8(qs_a, mask), vandq_u8(qs_b, mask))
};
// Extract the sub-th bit from each qh byte; shift it
// into position 4 so we can OR into the nibble.
// For output element k = sub*32 + l (l in 0..32): high

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/mellum.rs
View file

@ -286,7 +286,7 @@ impl MellumModel {
}
let bytes = shards[m.shard].tensor_bytes(t);
Ok(bytes
.chunks_exact(2)
.as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect())
};
@ -381,7 +381,7 @@ impl MellumModel {
let k = t.shape[1] as usize;
let bytes = self.shards[self.embeddings.shard].tensor_bytes(t);
let row = &bytes[(token as usize) * k * 2..((token as usize) + 1) * k * 2];
for (i, c) in row.chunks_exact(2).enumerate() {
for (i, c) in row.as_chunks::<2>().0.iter().enumerate() {
out[i] = bf16_to_f32(u16::from_le_bytes([c[0], c[1]]));
}
}

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/mlx.rs
View file

@ -57,8 +57,8 @@ mod tests {
#[test]
fn mlx_re_exports_compile() {
// Smoke test: the re-export names all resolve.
let _: Option<MlxDtype> = MlxDtype::from_str("F8_E8M0");
let _: Option<MlxDtype> = MlxDtype::from_str("C64");
let _: Option<MlxDtype> = MlxDtype::parse("F8_E8M0");
let _: Option<MlxDtype> = MlxDtype::parse("C64");
// MlxError / MlxModel / etc. exist as the same types
// — proven by the existing safetensors_reader tests.
assert_eq!(MlxDtype::F8_E8M0.element_size(), 1);

5
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/moe.rs
View file

@ -245,15 +245,14 @@ pub fn advise_stacked_experts(
if active.contains(&e) {
continue;
}
if let Some((start, end)) = tensor.expert_byte_range(e) {
if end <= data_len {
if let Some((start, end)) = tensor.expert_byte_range(e)
&& end <= data_len {
let _ = advise(
&data[start as usize..end as usize],
AdviceKind::DontNeed,
);
count += 1;
}
}
}
Ok(count)
}

6
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/nemotron_h.rs
View file

@ -194,7 +194,7 @@ impl NemotronHModel {
return Err(NemotronHError::WrongDtype(name.to_string(), t.dtype));
}
let bytes = st.tensor_bytes(t);
Ok(bytes.chunks_exact(2)
Ok(bytes.as_chunks::<2>().0.iter()
.map(|c| bf16_to_f32(u16::from_le_bytes([c[0], c[1]])))
.collect())
};
@ -288,7 +288,7 @@ impl NemotronHModel {
let k = t.shape[1] as usize;
let bytes = self.st.tensor_bytes(t);
let row = &bytes[(token as usize) * k * 2..((token as usize) + 1) * k * 2];
for (i, c) in row.chunks_exact(2).enumerate() {
for (i, c) in row.as_chunks::<2>().0.iter().enumerate() {
out[i] = bf16_to_f32(u16::from_le_bytes([c[0], c[1]]));
}
}
@ -822,7 +822,7 @@ pub(crate) mod tests {
let vocab = cfg.vocab_size;
let mut t: Vec<(String, Vec<u64>, Vec<f32>)> = Vec::new();
let mut seed = 1;
let mut push = |name: String, shape: Vec<u64>, t: &mut Vec<_>, seed: &mut usize| {
let push = |name: String, shape: Vec<u64>, t: &mut Vec<_>, seed: &mut usize| {
let n: usize = shape.iter().product::<u64>() as usize;
t.push((name, shape, synth(n, *seed)));
*seed += 1;

10
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/onnx.rs
View file

@ -376,7 +376,7 @@ fn parse_tensor(
(4, 2) => {
let (s, e) = p.read_length_delimited_range()?;
let chunk = &mmap[s..e];
for c in chunk.chunks_exact(4) {
for c in chunk.as_chunks::<4>().0 {
typed.float_data.push(f32::from_le_bytes([c[0], c[1], c[2], c[3]]));
}
}
@ -408,10 +408,10 @@ fn parse_tensor(
(10, 2) => {
let (s, e) = p.read_length_delimited_range()?;
let chunk = &mmap[s..e];
for c in chunk.chunks_exact(8) {
typed.double_data.push(f64::from_le_bytes(
c.try_into().unwrap(),
));
// `as_chunks` already yields `&[u8; 8]`, so the `try_into`
// the old `chunks_exact(8)` needed is gone.
for c in chunk.as_chunks::<8>().0 {
typed.double_data.push(f64::from_le_bytes(*c));
}
}
// TensorProto.uint64_data = 11 (uint64 repeated, packed varint)

23
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/openvino.rs
View file

@ -89,7 +89,7 @@ pub enum OvDtype {
impl OvDtype {
/// Parse OpenVINO IR's `element_type` attribute value.
/// Case-insensitive on the OpenVINO side.
pub fn from_str(s: &str) -> Self {
pub fn parse(s: &str) -> Self {
match s.to_ascii_lowercase().as_str() {
"f32" => Self::F32, "f16" => Self::F16, "bf16" => Self::BF16,
"f64" => Self::F64,
@ -324,7 +324,7 @@ pub fn parse_openvino_xml(
let val = attr.unescape_value().unwrap_or_default().to_string();
match attr.key.as_ref() {
b"element_type" => {
cur_dtype = OvDtype::from_str(&val);
cur_dtype = OvDtype::parse(&val);
}
b"shape" => {
cur_shape.clear();
@ -360,8 +360,8 @@ pub fn parse_openvino_xml(
_ => {}
}
}
Event::End(e) => {
if e.name().as_ref() == b"layer" && in_layer_const {
Event::End(e)
if e.name().as_ref() == b"layer" && in_layer_const => {
tensors.push(OvTensorInfo {
id: cur_id,
name: std::mem::take(&mut cur_name),
@ -372,7 +372,6 @@ pub fn parse_openvino_xml(
});
in_layer_const = false;
}
}
_ => {}
}
buf.clear();
@ -438,14 +437,14 @@ mod tests {
#[test]
fn dtype_parse_round_trip() {
assert_eq!(OvDtype::from_str("f32"), OvDtype::F32);
assert_eq!(OvDtype::from_str("F32"), OvDtype::F32); // case-insensitive
assert_eq!(OvDtype::from_str("bf16"), OvDtype::BF16);
assert_eq!(OvDtype::from_str("nf4"), OvDtype::NF4);
assert_eq!(OvDtype::from_str("u1"), OvDtype::U1);
assert_eq!(OvDtype::from_str("f8e4m3"), OvDtype::F8E4M3);
assert_eq!(OvDtype::parse("f32"), OvDtype::F32);
assert_eq!(OvDtype::parse("F32"), OvDtype::F32); // case-insensitive
assert_eq!(OvDtype::parse("bf16"), OvDtype::BF16);
assert_eq!(OvDtype::parse("nf4"), OvDtype::NF4);
assert_eq!(OvDtype::parse("u1"), OvDtype::U1);
assert_eq!(OvDtype::parse("f8e4m3"), OvDtype::F8E4M3);
assert!(matches!(
OvDtype::from_str("hypothetical_2030_dtype"),
OvDtype::parse("hypothetical_2030_dtype"),
OvDtype::Unknown(_)
));
}

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/qwen35moe.rs
View file

@ -105,7 +105,7 @@ impl QwenConfig {
})
}
fn is_recurrent(&self, l: usize) -> bool {
(l + 1) % self.full_attn_interval != 0
!(l + 1).is_multiple_of(self.full_attn_interval)
}
}

12
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/safetensors_reader.rs
View file

@ -92,7 +92,7 @@ impl SafetensorsDtype {
}
/// Parse from the JSON dtype string. Returns `None` for
/// unknown strings.
pub fn from_str(s: &str) -> Option<Self> {
pub fn parse(s: &str) -> Option<Self> {
match s {
"F32" => Some(Self::F32),
"F16" => Some(Self::F16),
@ -284,7 +284,7 @@ pub fn parse_safetensors_mmap(
name: key.clone(),
reason: "missing dtype".into(),
})?;
let dtype = SafetensorsDtype::from_str(dtype_str).ok_or_else(|| {
let dtype = SafetensorsDtype::parse(dtype_str).ok_or_else(|| {
SafetensorsError::UnknownDtype(dtype_str.to_string())
})?;
let shape_arr = t_obj
@ -505,7 +505,7 @@ mod tests {
// MLX-specific additions (wave 35):
"F8_E8M0", "C64",
] {
assert!(SafetensorsDtype::from_str(s).is_some(), "missing {s}");
assert!(SafetensorsDtype::parse(s).is_some(), "missing {s}");
}
}
@ -533,7 +533,7 @@ mod tests {
#[test]
fn dtype_rejects_unknown() {
assert!(SafetensorsDtype::from_str("WAT").is_none());
assert!(SafetensorsDtype::parse("WAT").is_none());
}
#[test]
@ -581,7 +581,7 @@ mod tests {
#[test]
fn read_rejects_file_too_small() {
let path = tempfile("tiny");
std::fs::write(&path, &[0u8, 1, 2]).unwrap();
std::fs::write(&path, [0u8, 1, 2]).unwrap();
let r = read_safetensors_file(&path);
assert!(matches!(r, Err(SafetensorsError::FileTooSmall(3))));
let _ = std::fs::remove_file(path);
@ -591,7 +591,7 @@ mod tests {
fn read_rejects_oversized_header() {
let path = tempfile("bad-header");
// header_len = u64::MAX
std::fs::write(&path, &(u64::MAX).to_le_bytes()).unwrap();
std::fs::write(&path, (u64::MAX).to_le_bytes()).unwrap();
let r = read_safetensors_file(&path);
assert!(matches!(r, Err(SafetensorsError::HeaderTooLarge { .. })));
let _ = std::fs::remove_file(path);

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/ssm.rs
View file

@ -366,7 +366,7 @@ pub fn gated_rmsnorm_inplace(
assert_eq!(y.len(), z.len());
assert_eq!(y.len(), weight.len());
let n = y.len();
assert!(group_size > 0 && n % group_size == 0,
assert!(group_size > 0 && n.is_multiple_of(group_size),
"len {n} not divisible by group_size {group_size}");
for i in 0..n {
y[i] *= silu(z[i]);

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/synthetic.rs
View file

@ -120,7 +120,7 @@ pub fn synthesize_llama_gguf(config: &SyntheticConfig) -> Vec<u8> {
let dff = config.feed_forward_length;
let v = config.vocab_size;
// Under GQA: K and V have fewer heads.
let d_head = if config.head_count > 0 { d / config.head_count } else { d };
let d_head = d.checked_div(config.head_count).unwrap_or(d);
let d_kv = config.head_count_kv * d_head;
let mut tensors: Vec<(String, Vec<u64>, GgmlType, Vec<u8>)> = Vec::new();

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/tflite.rs
View file

@ -554,7 +554,7 @@ mod tests {
// Helper: push a value, pad-aligned to its width.
fn pad_to(buf: &mut Vec<u8>, align: usize) {
while buf.len() % align != 0 {
while !buf.len().is_multiple_of(align) {
buf.push(0);
}
}

4
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/src/tier_store.rs
View file

@ -573,8 +573,8 @@ impl TierStore {
/// the lifetime of the borrow; promotion / demotion calls
/// don't invalidate the underlying mmap, only the LRU
/// metadata.
pub fn bytes<'a, R>(
&'a self,
pub fn bytes<R>(
&self,
model_id: &str,
meta: &StoreTensorMeta,
f: impl FnOnce(&[u8]) -> R,

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/tests/bpe.rs
View file

@ -191,7 +191,7 @@ fn bpe_pre_tokenizes_on_whitespace() {
tokens.push((format!("{}c", space_marker), 0.0)); // " c"
tokens.push((space_marker.clone(), 0.0)); // " " alone
let owned_vocab: Vec<(String, f32)> = tokens.iter().cloned().collect();
let owned_vocab: Vec<(String, f32)> = tokens.to_vec();
let pairs: Vec<(&str, f32)> = owned_vocab.iter()
.map(|(s, sc)| (s.as_str(), *sc)).collect();

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/tests/bpe_regex.rs
View file

@ -121,7 +121,7 @@ fn regex_bpe_round_trips() {
tokens.push((space_mark.clone(), 0.0));
// Add " w", " h" forms for "hello world" round-trip
tokens.push((format!("{}w", space_mark), 0.0));
let owned = tokens.iter().cloned().collect::<Vec<_>>();
let owned = tokens.to_vec();
let pairs: Vec<(&str, f32)> = owned.iter()
.map(|(s, sc)| (s.as_str(), *sc)).collect();
let v = vocab_with_merges(pairs, vec![(&space_mark, "w")], None, None, None);

7
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/tests/dequant.rs
View file

@ -2,6 +2,11 @@
//! hand-constructed blocks where the expected output is computable from
//! the formula.
// Benches and integration tests are their own crates, so the decision
// recorded in src/lib.rs does not reach them: these loops fill arrays by
// index, which is what the index is for.
#![allow(clippy::needless_range_loop)]
use jouleclaw_loader_gguf::dequant::{dequantize_q4_k, dequantize_q5_k, dequantize_q8_0};
/// Encode an f32 as IEEE 754 binary16 little-endian. Just inverse of the
@ -54,7 +59,7 @@ fn q8_0_unit_scale_returns_quants_as_floats() {
block[2 + i] = q as u8;
}
let out = dequantize_q8_0(&block, 32).unwrap();
let expected: Vec<f32> = (0..32).map(|i| (i as i32 - 5) as f32).collect();
let expected: Vec<f32> = (0..32).map(|i| (i - 5) as f32).collect();
assert_eq!(out, expected);
}

2
jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/tests/tokenizer.rs
View file

@ -187,7 +187,7 @@ fn encode_falls_back_to_bytes_for_unknown_chars() {
for b in 0..256u32 {
tokens.push((format!("<0x{:02X}>", b), -100.0));
}
let owned: Vec<(String, f32)> = tokens.iter().cloned().collect();
let owned: Vec<(String, f32)> = tokens.to_vec();
let pairs: Vec<(&str, f32)> = owned.iter()
.map(|(s, sc)| (s.as_str(), *sc)).collect();