open-standards/joule-code/joule-code-rs/crates/joule-code-reviewer/src/llm.rs

//! A **model-backed semantic reviewer**: the real judgment of whether code
//! means what the clause asks, by a model, behind a clean seam.
//!
//! The structural reviewer checks what a machine can know for certain. The
//! semantic question — *does this op actually implement the clause?* — needs a
//! model. This module is that reviewer: it builds a review prompt from the
//! clause and the op's diff, runs a model, parses the verdict, and signs
//! two-stage reviews with its own key. It is a distinct principal like any
//! other; the gate counts its reviews identically.
//!
//! The model is a seam, not a dependency. [`Generate`] is "prompt in, text
//! out" — a deployment plugs in whatever it runs (agent-zed already drives a
//! GGUF model through `jouleclaw-loader-gguf`; wiring its `l3_generate` into
//! `Generate` is the whole binding). Keeping the model behind the trait lets
//! the *real* logic here — prompt construction, robust parsing, the
//! fail-safe — be deterministic and fully tested, while the heavyweight,
//! non-deterministic inference stays where the model already lives.
//!
//! ## Fail-safe by construction
//!
//! A model's output is untrusted text. This reviewer **never approves on
//! ambiguity**: if the model's verdict cannot be parsed, or it does not
//! clearly say approve, the op is *sent back* with the raw output as the
//! finding. The only path to an approval is an unambiguous `APPROVE` line.
//! A confused model cannot wave code through — the same no-silent-lane stance
//! the rest of the system takes.

use ed25519_dalek::SigningKey;
use joule_code_cargo::SignedOp;
use joule_code_review::{Review, ReviewStage, ReviewVerdict, SignedReview, ALG_ED25519};
use joule_code_spec::SignedSpec;

use crate::cascade::{ReviewSkill, SemanticOutcome, SemanticReviewer};
use crate::ReviewerError;

/// The model seam: prompt in, completion out. A deployment supplies its model
/// (e.g. agent-zed's loaded GGUF via `jouleclaw-loader-gguf`).
pub trait Generate {
    /// Complete `prompt`. An `Err` is a model/runtime failure (which the
    /// reviewer turns into a fail-safe rejection, never an approval).
    fn generate(&self, prompt: &str) -> Result<String, String>;
}

/// A [`Generate`] that runs an external model **as a subprocess**: the prompt
/// is written to the command's stdin, the completion read from its stdout.
///
/// This is the dependency-free model binding. Point it at any "prompt in,
/// completion out" CLI — a `jouleclaw` inference binary, a `llama.cpp` runner,
/// a shell wrapper around a hosted endpoint — and the reviewer is genuinely
/// model-backed without linking (and recompiling against) a heavy, fast-moving
/// inference crate. The model is a separate process, which also keeps the
/// reviewer a separate principal by construction.
pub struct CommandGenerate {
    program: String,
    args: Vec<String>,
}

impl CommandGenerate {
    /// A generator that runs `program args…`, feeding the prompt on stdin.
    pub fn new(program: impl Into<String>, args: Vec<String>) -> Self {
        Self { program: program.into(), args }
    }
}

impl Generate for CommandGenerate {
    fn generate(&self, prompt: &str) -> Result<String, String> {
        use std::io::Write;
        use std::process::{Command, Stdio};
        let mut child = Command::new(&self.program)
            .args(&self.args)
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::null())
            .spawn()
            .map_err(|e| format!("spawn {}: {e}", self.program))?;
        child
            .stdin
            .take()
            .ok_or("no stdin")?
            .write_all(prompt.as_bytes())
            .map_err(|e| format!("write prompt: {e}"))?;
        let out = child.wait_with_output().map_err(|e| format!("wait: {e}"))?;
        if !out.status.success() {
            return Err(format!("model command exited {}", out.status));
        }
        Ok(String::from_utf8_lossy(&out.stdout).into_owned())
    }
}

/// A semantic reviewer backed by a [`Generate`] model. A distinct principal:
/// it signs with its own key and refuses to review its own ops.
pub struct LlmSemanticReviewer<G: Generate> {
    key: SigningKey,
    model: G,
}

impl<G: Generate> LlmSemanticReviewer<G> {
    /// A model-backed reviewer with the given signing identity and model.
    pub fn new(key: SigningKey, model: G) -> Self {
        Self { key, model }
    }

    /// This reviewer's principal.
    pub fn principal(&self) -> [u8; 32] {
        self.key.verifying_key().to_bytes()
    }

    /// The review prompt: the clause(s) under review and the op's diff, with a
    /// strict output contract the parser keys on.
    pub fn prompt(&self, op: &SignedOp, spec: &SignedSpec, claimed: &[String]) -> String {
        let mut clauses = String::new();
        for id in claimed {
            if let Some(c) = spec.spec.clauses.iter().find(|c| &c.id == id) {
                clauses.push_str(&format!("- {} [{:?}]: {}\n", c.id, c.kind, c.text));
            }
        }
        if clauses.is_empty() {
            clauses.push_str("(the op cites no spec clause)\n");
        }
        let mut diff = String::new();
        for edit in &op.op.edits {
            diff.push_str(&format!("--- {}\n{}\n", edit.path, String::from_utf8_lossy(&edit.replacement)));
        }
        format!(
            "You are an independent code reviewer. Decide whether the change below \
             correctly implements the cited requirement(s) and nothing out of scope.\n\n\
             Intent: {intent}\n\nRequirement(s):\n{clauses}\nChange:\n{diff}\n\
             Answer in exactly this format:\n\
             VERDICT: APPROVE or REQUEST_CHANGES\n\
             FINDINGS: one short reason per line (only if REQUEST_CHANGES)\n\
             INDUCE: a single literal code substring that generalizes the defect, \
             or NONE (only if REQUEST_CHANGES)\n",
            intent = op.op.intent,
            clauses = clauses,
            diff = diff,
        )
    }

    fn approve(&self, op_id: &joule_code_schema::ContentId, spec_id: &joule_code_schema::ContentId) -> Result<Vec<SignedReview>, ReviewerError> {
        let mk = |stage, spec_id: Option<joule_code_schema::ContentId>| -> Result<SignedReview, ReviewerError> {
            Ok(Review {
                alg: ALG_ED25519,
                op_id: op_id.clone(),
                stage,
                spec_id,
                verdict: ReviewVerdict::Approve,
                findings: vec![],
                round: 1,
                reviewer: self.principal(),
            }
            .sign(&self.key)?)
        };
        Ok(vec![
            mk(ReviewStage::SpecCompliance, Some(spec_id.clone()))?,
            mk(ReviewStage::Quality, None)?,
        ])
    }

    fn reject(&self, op_id: &joule_code_schema::ContentId, findings: Vec<String>) -> Result<SignedReview, ReviewerError> {
        Ok(Review {
            alg: ALG_ED25519,
            op_id: op_id.clone(),
            stage: ReviewStage::Quality,
            spec_id: None,
            verdict: ReviewVerdict::RequestChanges,
            findings,
            round: 1,
            reviewer: self.principal(),
        }
        .sign(&self.key)?)
    }
}

impl<G: Generate> SemanticReviewer for LlmSemanticReviewer<G> {
    fn review(
        &self,
        op: &SignedOp,
        spec: &SignedSpec,
        claimed: &[String],
    ) -> Result<SemanticOutcome, ReviewerError> {
        if op.op.author == self.principal() {
            return Err(ReviewerError::SelfReview);
        }
        let op_id = op.op.op_id()?;
        let spec_id = spec.verify()?;
        let prompt = self.prompt(op, spec, claimed);

        // A model/runtime failure is a fail-safe rejection, never an approval.
        let output = match self.model.generate(&prompt) {
            Ok(t) => t,
            Err(e) => {
                return Ok(SemanticOutcome {
                    reviews: vec![self.reject(&op_id, vec![format!("model unavailable: {e}")])?],
                    induced: vec![],
                });
            }
        };

        let parsed = ParsedVerdict::parse(&output);
        match parsed.verdict {
            Some(true) => Ok(SemanticOutcome { reviews: self.approve(&op_id, &spec_id)?, induced: vec![] }),
            Some(false) => {
                let findings = if parsed.findings.is_empty() {
                    vec!["the model requested changes (no reason given)".into()]
                } else {
                    parsed.findings
                };
                let induced = parsed
                    .induce
                    .map(|marker| {
                        vec![ReviewSkill {
                            marker,
                            stage: ReviewStage::Quality,
                            finding: findings.first().cloned().unwrap_or_else(|| "learned defect".into()),
                        }]
                    })
                    .unwrap_or_default();
                Ok(SemanticOutcome { reviews: vec![self.reject(&op_id, findings)?], induced })
            }
            // unparseable / ambiguous → fail safe to a rejection
            None => Ok(SemanticOutcome {
                reviews: vec![self.reject(
                    &op_id,
                    vec![format!("model verdict was not parseable; raw: {}", truncate(&output, 200))],
                )?],
                induced: vec![],
            }),
        }
    }
}

/// The parsed model verdict.
struct ParsedVerdict {
    /// `Some(true)` = APPROVE, `Some(false)` = REQUEST_CHANGES, `None` = neither found.
    verdict: Option<bool>,
    findings: Vec<String>,
    induce: Option<String>,
}

impl ParsedVerdict {
    fn parse(output: &str) -> Self {
        let mut verdict = None;
        let mut findings = Vec::new();
        let mut induce = None;
        let mut in_findings = false;
        for raw in output.lines() {
            let line = raw.trim();
            let upper = line.to_ascii_uppercase();
            if let Some(rest) = upper.strip_prefix("VERDICT:") {
                let r = rest.trim();
                if r.starts_with("APPROVE") {
                    verdict = Some(true);
                } else if r.starts_with("REQUEST_CHANGES") || r.starts_with("REQUEST CHANGES") {
                    verdict = Some(false);
                }
                in_findings = false;
            } else if let Some(rest) = upper.strip_prefix("INDUCE:") {
                // take the original-case remainder after the colon
                let val = line[line.find(':').map(|i| i + 1).unwrap_or(0)..].trim();
                if !val.is_empty() && !rest.trim().starts_with("NONE") {
                    induce = Some(val.to_string());
                }
                in_findings = false;
            } else if upper.starts_with("FINDINGS:") {
                let val = line[line.find(':').map(|i| i + 1).unwrap_or(0)..].trim();
                if !val.is_empty() {
                    findings.push(val.to_string());
                }
                in_findings = true;
            } else if in_findings && !line.is_empty() {
                findings.push(line.trim_start_matches(['-', '*', ' ']).to_string());
            }
        }
        ParsedVerdict { verdict, findings, induce }
    }
}

fn truncate(s: &str, n: usize) -> String {
    if s.len() <= n {
        s.to_string()
    } else {
        format!("{}…", &s[..n])
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::cascade::{ReviewCascade, ReviewTier};
    use joule_code_cargo::{Anchor, CodingOp, Edit};
    use joule_code_review::approval;
    use joule_code_schema::{ContentId, Tier};
    use joule_code_spec::{Clause, ClauseKind, Spec, ALG_ED25519 as SALG};
    use crate::Reviewer;

    fn sk(s: u8) -> SigningKey {
        SigningKey::from_bytes(&[s; 32])
    }

    fn spec(author: &SigningKey) -> SignedSpec {
        Spec {
            alg: SALG,
            intent: "x".into(),
            clauses: vec![Clause { id: "C1".into(), text: "persist".into(), kind: ClauseKind::Requirement }],
            parent: None,
            author: author.verifying_key().to_bytes(),
        }
        .sign(author)
        .unwrap()
    }

    fn op(author: &SigningKey, body: &str) -> SignedOp {
        CodingOp {
            alg: 0x01,
            prior: ContentId::of_bytes(body.as_bytes()),
            edits: vec![Edit { path: "src/s.rs".into(), anchor: Anchor::ByteRange { start: 0, end: 0 }, replacement: body.as_bytes().to_vec() }],
            author: author.verifying_key().to_bytes(),
            intent: "C1: persist".into(),
            tier: Tier::L3,
        }
        .sign(author)
        .unwrap()
    }

    /// A fake model returning canned output — the deterministic stand-in for a
    /// real GGUF decode.
    struct CannedModel(&'static str);
    impl Generate for CannedModel {
        fn generate(&self, _prompt: &str) -> Result<String, String> {
            Ok(self.0.to_string())
        }
    }
    struct DeadModel;
    impl Generate for DeadModel {
        fn generate(&self, _prompt: &str) -> Result<String, String> {
            Err("backend not loaded".into())
        }
    }

    #[test]
    fn a_clear_approve_signs_a_two_stage_approval() {
        let author = sk(1);
        let r = LlmSemanticReviewer::new(sk(2), CannedModel("VERDICT: APPROVE\n"));
        let o = op(&author, "fn save() { write_to_disk(); }");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(approval(&o, &out.reviews).is_approved());
        assert!(out.induced.is_empty());
    }

    #[test]
    fn a_request_changes_with_an_induce_marker_rejects_and_teaches() {
        let author = sk(1);
        let r = LlmSemanticReviewer::new(
            sk(2),
            CannedModel("VERDICT: REQUEST_CHANGES\nFINDINGS: uses eval on untrusted input\nINDUCE: eval(\n"),
        );
        let o = op(&author, "fn run() { eval(x); }");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(!approval(&o, &out.reviews).is_approved());
        assert_eq!(out.induced.len(), 1);
        assert_eq!(out.induced[0].marker, "eval(");
    }

    #[test]
    fn an_unparseable_verdict_fails_safe_to_a_rejection_never_an_approval() {
        let author = sk(1);
        let r = LlmSemanticReviewer::new(sk(2), CannedModel("I think this looks fine, ship it!"));
        let o = op(&author, "fn save() { write_to_disk(); }");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(!approval(&o, &out.reviews).is_approved()); // ambiguity never approves
        assert!(out.induced.is_empty());
    }

    #[test]
    fn a_dead_model_fails_safe_to_a_rejection() {
        let author = sk(1);
        let r = LlmSemanticReviewer::new(sk(2), DeadModel);
        let o = op(&author, "fn f() {}");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(!approval(&o, &out.reviews).is_approved());
        assert!(out.reviews[0].review.findings[0].contains("model unavailable"));
    }

    #[test]
    fn the_subprocess_model_seam_runs_a_real_command() {
        // a real external "model": printf a canned verdict, ignoring the prompt.
        let author = sk(1);
        let r = LlmSemanticReviewer::new(
            sk(2),
            CommandGenerate::new("sh", vec!["-c".into(), "cat >/dev/null; printf 'VERDICT: APPROVE\\n'".into()]),
        );
        let o = op(&author, "fn save() { write_to_disk(); }");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(approval(&o, &out.reviews).is_approved());
    }

    #[test]
    fn a_failing_subprocess_model_fails_safe() {
        let author = sk(1);
        let r = LlmSemanticReviewer::new(
            sk(2),
            CommandGenerate::new("sh", vec!["-c".into(), "exit 1".into()]),
        );
        let o = op(&author, "fn f() { write_to_disk(); }");
        let out = r.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert!(!approval(&o, &out.reviews).is_approved()); // nonzero exit → rejection
    }

    #[test]
    fn it_drops_into_the_cascade_as_the_semantic_tier() {
        // the model reviewer is just a SemanticReviewer — the cascade escalates
        // a structurally-clean op to it, and learns from its rejection.
        let author = sk(1);
        let mut cascade = ReviewCascade::new(Reviewer::new(sk(2))).with_semantic(Box::new(
            LlmSemanticReviewer::new(sk(3), CannedModel("VERDICT: REQUEST_CHANGES\nFINDINGS: eval is unsafe\nINDUCE: eval(\n")),
        ));
        let o = op(&author, "fn run() { eval(x); }");
        let out = cascade.review(&o, &spec(&author), &["C1".into()]).unwrap();
        assert_eq!(out.tier, ReviewTier::Semantic);
        assert_eq!(out.induced.len(), 1);
        for s in out.induced {
            cascade.learn(s);
        }
        // recurrence caught at the learned tier, model never consulted
        let again = op(&author, "fn other() { eval(y); }");
        assert_eq!(cascade.review(&again, &spec(&author), &["C1".into()]).unwrap().tier, ReviewTier::Learned);
    }
}