//! Wave 60 bring-up: Cohere2-MoE (North Mini Code 30B-A3B) //! end-to-end on the real checkpoint — the substrate's //! fourth topology, first attention-only one. //! //! ```bash //! cargo run --release --features rayon --example north_decode -- \ //! /path/to/snapshot-dir [n_tokens] [prompt text] //! ``` use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; use jouleclaw_loader_gguf::cohere2_moe::{Cohere2MoeConfig, Cohere2MoeModel}; use jouleclaw_loader_gguf::safetensors_reader::read_safetensors_file; use std::path::PathBuf; use std::time::Instant; fn main() -> Result<(), Box> { let args: Vec = std::env::args().collect(); if args.len() < 2 { eprintln!("usage: north_decode [n_tokens] [prompt]"); std::process::exit(1); } let dir = PathBuf::from(&args[1]); let n_tokens: usize = args.get(2).map(|s| s.parse()).transpose()?.unwrap_or(8); let prompt_text = args.get(3).cloned() .unwrap_or_else(|| "The capital of France is".to_string()); let cfg_bytes = std::fs::read(dir.join("config.json"))?; let config: Cohere2MoeConfig = serde_json::from_slice(&cfg_bytes)?; let n_full = config.layer_types.iter().filter(|t| *t == "full_attention").count(); println!( "config: {} layers ({} full + {} sliding, window {}), {} experts top-{}, hidden {}, vocab {}", config.num_hidden_layers, n_full, config.num_hidden_layers - n_full, config.sliding_window, config.num_experts, config.num_experts_per_tok, config.hidden_size, config.vocab_size, ); let t0 = Instant::now(); let mut shard_paths: Vec = std::fs::read_dir(&dir)? .filter_map(|e| e.ok().map(|e| e.path())) .filter(|p| { p.extension().is_some_and(|e| e == "safetensors") && p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("model")) }) .collect(); shard_paths.sort(); let shards = shard_paths.iter() .map(|p| read_safetensors_file(p).map_err(|e| format!("{p:?}: {e:?}"))) .collect::, _>>()?; let n_tensors: usize = shards.iter().map(|s| s.tensors.len()).sum(); println!("checkpoint mapped in {:.2}s ({} shards, {} tensors)", t0.elapsed().as_secs_f64(), shards.len(), n_tensors); let model = Cohere2MoeModel::bind_sharded(config, shards)?; let tokenizer = BpeTokenizer::from_file(dir.join("tokenizer.json"))?; println!("tokenizer: {} vocab", tokenizer.vocab_size()); let bos = 2u32; // bos_token_id from config.json let mut prompt = vec![bos]; prompt.extend(tokenizer.encode(&prompt_text)); println!("prompt {prompt_text:?} → {prompt:?}"); let mut state = model.new_state(); let t_pre = Instant::now(); let mut logits_next = model.forward_prompt(&prompt, &mut state); let dt_pre = t_pre.elapsed(); println!( "prefill: {} tokens in {:.0} ms ({:.1} tok/s)", prompt.len(), dt_pre.as_secs_f64() * 1e3, prompt.len() as f64 / dt_pre.as_secs_f64(), ); let mut generated: Vec = Vec::new(); println!("greedy decode:"); for step in 0..n_tokens { let t = Instant::now(); let logits = if step == 0 { std::mem::take(&mut logits_next) } else { model.forward_token(*generated.last().unwrap(), &mut state) }; let dt = t.elapsed(); let finite = logits.iter().all(|v| v.is_finite()); let (argmax, max) = logits.iter().enumerate() .fold((0usize, f32::NEG_INFINITY), |(bi, bv), (i, &v)| { if v > bv { (i, v) } else { (bi, bv) } }); println!(" step {step:>2}: out={argmax:>7} max={max:>8.3} finite={finite} {:.0} ms", dt.as_secs_f64() * 1e3); if !finite { eprintln!("FAILED: non-finite logits"); std::process::exit(1); } generated.push(argmax as u32); } println!("\ngenerated text: {:?}", tokenizer.decode(&generated)); #[cfg(feature = "fabric")] { let entries = jouleclaw_loader_gguf::fabric_dispatch::ledger_snapshot(); if !entries.is_empty() { let total_gf: f64 = entries.iter().map(|e| e.gflops).sum(); println!("\nfabric attribution (whole run):"); for e in &entries { let uj = e .projected_uj .map(|u| format!("{:.1} J projected", u / 1e6)) .unwrap_or_else(|| "no calibration".into()); println!( " {:>18}: {:>6} ops, {:>9.1} GFLOP ({:>4.1}%), {}", e.member_id, e.ops, e.gflops, 100.0 * e.gflops / total_gf, uj, ); } } } println!("PASS"); Ok(()) }