open-standards/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/tests/end_to_end.rs

//! End-to-end acceptance test for the Edge-First v6 vertical slice.
//!
//! Wires every crate the port produced into a single flow:
//!
//!   user query
//!     → fixture QueryAnalysis (intent extraction stub)
//!     → jouleclaw-plan CSP planner       (selects Wikidata as the store)
//!     → jouleclaw-execute orchestrator   (live Wikidata SPARQL retrieval)
//!     → jouleclaw-compose TemplateComposer (draft segments per item)
//!     → jouleclaw-diagnose atomizer       (sentence-level claims)
//!     → jouleclaw-diagnose verify         (DeBERTa-v3 entailment vs items)
//!     → jouleclaw-compose verified composer (Answer with full Provenance)
//!
//! Acceptance criteria (matches spec §13):
//!   - Structurally-valid Answer object emitted (not Refusal).
//!   - Verdict ∈ {Verified, Degraded} (no critical violations).
//!   - At least one cited Wikidata source.
//!   - DeBERTa was actually consulted (entailments_consulted nonempty).
//!   - All required invariants in `invariants_verified`.
//!
//! Marked `#[ignore]` because it hits both Wikidata (network) and
//! DeBERTa (~22s/inference). Opt in with:
//!
//!     cargo test --release -p jouleclaw-edge-cli -- --ignored end_to_end

use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;

use chrono::Utc;

use jouleclaw_compose::{compose_verified_answer, ComposeInputs, DraftComposer, TemplateComposer};
use jouleclaw_deberta::NliEngine;
use jouleclaw_diagnose::{atomize_sentences, verify, DebertaEntailer, VerifyInputs};
use jouleclaw_execute::orchestrator::{execute as orch_execute, OrchestratorConfig};
use jouleclaw_execute::retriever::RetrieverRegistry;
use jouleclaw_execute::retrievers::wikidata::WikidataRetriever;
use jouleclaw_plan::{
    plan, FixtureUnderstanding, QueryAnalysis, QueryUnderstanding, RawSubQuery, SelfModel,
    StakesSignal, StoreCatalog,
};
use jouleclaw_schema::*;

fn workspace_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .and_then(|p| p.parent())
        .map(|p| p.to_path_buf())
        .expect("workspace root")
}

fn model_dir() -> Option<PathBuf> {
    let p = workspace_root().join("models/deberta-v3-large-mnli");
    if p.join("model.safetensors").exists() {
        Some(p)
    } else {
        None
    }
}

/// Fixture analysis for the test query — what a real
/// QueryUnderstanding impl would produce. Hand-crafted so the
/// acceptance test doesn't depend on running an LLM to extract
/// intent. The `decomposition_text` is the bare relational phrase
/// the retriever consumes (drops the "What is the …?" framing
/// since the planner's property-path matcher already strips that).
fn fixture_analysis(query_text: &str, decomposition_text: &str) -> QueryAnalysis {
    QueryAnalysis {
        original_query: OriginalQuery {
            text: Some(query_text.into()),
            image_ref: None,
            audio_ref: None,
            video_ref: None,
            language_detected: "en".into(),
            timestamp: Utc::now(),
        },
        intent: Intent::Lookup,
        modalities_in: vec![Modality::Text],
        modalities_out: vec![Modality::Text],
        entities_extracted: vec![],
        relations_extracted: vec![],
        temporal_anchors: vec![],
        geographic_anchors: vec![],
        domain_tags: vec!["geography".into()],
        freshness_signal: false,
        stakes_signal: StakesSignal::Low,
        raw_decomposition: vec![RawSubQuery {
            sub_id: "q0".into(),
            text: decomposition_text.into(),
            required_modalities: vec![Modality::Text],
            depends_on: vec![],
            priority: 1.0,
            preferred_store: None,
        }],
        confidence: 0.95,
    }
}

fn fixture_analysis_for(query_text: &str) -> QueryAnalysis {
    fixture_analysis(query_text, "capital of France")
}

fn wikidata_retriever_capability() -> RetrieverCapability {
    RetrieverCapability {
        retriever_id: "wikidata".into(),
        status: CapabilityStatus::Healthy,
        domains_covered: vec!["geography".into()],
        modalities_supported: vec![Modality::Text, Modality::Structured],
        typical_latency_ms: 1000,
        p99_latency_ms: 5000,
        success_rate_recent: 0.97,
        last_failure_at: None,
        known_limitations: vec![],
        populates_valid_time: true,
        populates_transaction_time: true,
        populates_granularity: false,
        populates_scope: true,
        populates_certainty: false,
        populates_provenance: true,
        authority_tier: 1,
    }
}

fn wikidata_catalog_and_raps() -> (StoreCatalog, BTreeMap<String, ReactiveActionPackage>) {
    let mut catalog = StoreCatalog::new();
    catalog.add(jouleclaw_plan::RetrieverProfile {
        retriever_id: "wikidata".into(),
        default_rap_id: "wikidata_sparql_v1".into(),
        retrieval_method: RetrievalMethod::Sparql,
        estimated_latency_ms: 2000,
        estimated_cost_usd: 0.0,
        estimated_joules: 5.0,
    });
    let rap = ReactiveActionPackage {
        schema_version: "2.0".into(),
        rap_id: "wikidata_sparql_v1".into(),
        description: "Wikidata SPARQL: EntitySearch primary + property-path fallback".into(),
        applies_to_methods: vec![RetrievalMethod::Sparql],
        steps: vec![
            RapStep {
                step_id: "primary".into(),
                description: "Entity-search SPARQL, filtered for meta-entities".into(),
                method: "wikidata_primary".into(),
                condition: RapStepCondition::Always,
                parameters: Default::default(),
                timeout_ms: 8000,
                cost_estimate_usd: 0.0,
            },
            RapStep {
                step_id: "property_path".into(),
                description: "Property-path SPARQL for relational queries".into(),
                method: "wikidata_property_path".into(),
                condition: RapStepCondition::OnEmpty,
                parameters: Default::default(),
                timeout_ms: 8000,
                cost_estimate_usd: 0.0,
            },
        ],
        max_total_attempts: 2,
        metadata: Default::default(),
    };
    catalog.add_rap(rap.clone());
    let mut raps = BTreeMap::new();
    raps.insert("wikidata_sparql_v1".into(), rap);
    (catalog, raps)
}

fn axes_for_template_claim() -> KnowledgeAxes {
    KnowledgeAxes {
        schema_version: "5.0".into(),
        valid_time_start: None,
        valid_time_end: None,
        transaction_time: None,
        reference_time: Utc::now(),
        temporal_stability: TemporalStabilityClass::Slow,
        granularity: GranularityClass::Medium,
        granularity_notes: None,
        scope: ScopeClass::Particular,
        scope_domain: Some("Wikidata".into()),
        certainty: 0.95,
        certainty_basis: "wikidata_structured_kb".into(),
        source_uri: None,
        source_authority_tier: 1,
        extraction_method: Some("sparql_entity_search".into()),
        citation_chain: vec![],
        metadata: Default::default(),
    }
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore]
async fn capital_of_france_produces_verified_answer_with_provenance() {
    // Bail cleanly if the model isn't downloaded.
    let Some(deberta_dir) = model_dir() else {
        panic!("DeBERTa model not present at models/deberta-v3-large-mnli");
    };

    let started = Instant::now();
    let query_text = "What is the capital of France?";
    eprintln!("[query] {query_text}");

    // 1. Plan pillar.
    let understanding = FixtureUnderstanding::new();
    understanding.insert(query_text, fixture_analysis_for(query_text));
    let analysis = understanding
        .analyze(&OriginalQuery {
            text: Some(query_text.into()),
            image_ref: None,
            audio_ref: None,
            video_ref: None,
            language_detected: "en".into(),
            timestamp: Utc::now(),
        })
        .expect("analyze");

    let self_model = SelfModel::new();
    self_model.register_retriever(wikidata_retriever_capability());
    let capabilities = self_model.snapshot();
    let (catalog, raps) = wikidata_catalog_and_raps();

    let plan = plan(
        &analysis,
        &capabilities,
        &catalog,
        Constraints::default(),
        Budget::default(),
    )
    .expect("plan");
    eprintln!(
        "[plan] {} sub-queries; invariants_satisfied={}",
        plan.decomposition.len(),
        plan.invariants_satisfied.all_satisfied()
    );

    // 2. Execute pillar — real Wikidata SPARQL.
    let wikidata = Arc::new(WikidataRetriever::new().expect("wikidata client"));
    let mut registry = RetrieverRegistry::new();
    registry.insert(wikidata);

    let exec_result = orch_execute(
        &plan,
        &raps,
        &registry,
        &self_model,
        &OrchestratorConfig::default(),
    )
    .await
    .expect("execute");
    eprintln!(
        "[execute] retrieved {} items in {:?}; errors={}",
        exec_result.items.len(),
        exec_result.elapsed,
        exec_result.subquery_errors.len()
    );
    assert!(!exec_result.items.is_empty(), "no items retrieved");

    // 3. Compose draft.
    let composer = TemplateComposer::new();
    let draft = composer.draft(&plan, &exec_result.items, &[]).expect("draft");
    eprintln!("[draft] {} segments", draft.len());
    for s in &draft {
        eprintln!("  - {}: {:?}", s.segment_id, &s.text);
    }

    // 4. Diagnose pillar — atomize + DeBERTa-backed verify.
    let segments_for_atomizer: Vec<(String, String)> = draft
        .iter()
        .map(|s| (s.segment_id.clone(), s.text.clone()))
        .collect();
    let claims =
        atomize_sentences(&segments_for_atomizer, &axes_for_template_claim()).expect("atomize");
    eprintln!("[atomize] {} claims", claims.len());

    let engine = NliEngine::from_dir(&deberta_dir).expect("load DeBERTa");
    let entailer = DebertaEntailer::new(engine);
    let report = verify(
        &VerifyInputs::new(&plan, &exec_result.items, &[], &claims),
        &entailer,
    )
    .expect("verify");

    eprintln!(
        "[diagnose] verdict={:?} violations={} entailments={} joules={:.1}",
        report.verdict,
        report.violations.len(),
        report.entailments_consulted.len(),
        report.joules_spent
    );

    // 5. Compose verified Answer.
    let elapsed_ms = started.elapsed().as_millis() as u64;
    let joules_total = exec_result.estimated_cost_usd * 0.0  // execute energy is illustrative
        + report.joules_spent;
    let inputs = ComposeInputs {
        plan: &plan,
        draft_segments: &draft,
        claims: &claims,
        items: &exec_result.items,
        report: &report,
        joules_spent_total: joules_total,
        latency_ms: elapsed_ms,
    };
    let result = compose_verified_answer(&inputs).expect("compose");

    // Acceptance criteria.
    let ans = result.as_answer().expect("expected an Answer, got Refusal");
    eprintln!(
        "[answer] status={:?} segments={} caveats={} latency={}ms joules={}",
        ans.status,
        ans.segments.len(),
        ans.caveats.len(),
        ans.latency_ms,
        ans.joules_spent_total
    );
    eprintln!(
        "[invariants_verified] {}",
        ans.invariants_verified.join(", ")
    );
    for s in &ans.segments {
        eprintln!("  segment {}: {:?}", s.segment_id, &s.text);
    }

    // 1. Structurally valid Answer (not Refusal).
    assert!(result.is_answer(), "expected Answer, got Refusal");
    assert!(matches!(
        ans.status,
        AnswerStatus::Verified | AnswerStatus::Degraded
    ));

    // 2. Provenance carries the retrieved items.
    assert!(!ans.provenance.items.is_empty());
    assert!(!ans.provenance.cache_key.is_empty());

    // 3. At least one cited source.
    assert!(
        ans.segments.iter().any(|s| !s.cited_item_ids.is_empty()),
        "no segment has cited items"
    );

    // 4. DeBERTa was actually consulted.
    assert!(
        !ans.provenance.entailments.is_empty(),
        "no entailments in provenance — DeBERTa wasn't consulted"
    );
    let model_id = &ans.provenance.entailments[0].model_id;
    assert!(
        model_id.contains("deberta"),
        "entailment model_id should mention 'deberta', got {model_id:?}"
    );

    // 5. Core invariants are in the verified list.
    for required in [
        "I1", // every claim has provenance
        "I4", // authority tier respected
        "I6", // conflicts surfaced
        "I9", // reroute bounded
        "I10", // structured refusal
        "I13", // epistemic mode declared
    ] {
        assert!(
            ans.invariants_verified.iter().any(|s| s == required),
            "missing invariant {required} in {:?}",
            ans.invariants_verified
        );
    }

    eprintln!(
        "\n[result] Answer for {query_text:?}:\n{}",
        ans.segments
            .iter()
            .map(|s| s.text.as_str())
            .collect::<Vec<_>>()
            .join("\n  ")
    );
}

/// Multi-query acceptance suite: load DeBERTa + Wikidata once,
/// loop over a small set of relational queries, and verify each
/// produces a Verified answer that mentions an expected token in
/// the final text.
///
/// Run with:
///   cargo test --release -p jouleclaw-edge-cli -- --ignored \
///       acceptance_suite_covers_capital_currency_language_borders
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore]
async fn acceptance_suite_covers_capital_currency_language_borders() {
    let Some(deberta_dir) = model_dir() else {
        panic!("DeBERTa model not present at models/deberta-v3-large-mnli");
    };

    // Each case: (user query, the sub-query text the planner emits,
    // a substring we expect to appear somewhere in the final answer).
    let cases: Vec<(&str, &str, &str)> = vec![
        (
            "What is the capital of France?",
            "capital of France",
            "Paris",
        ),
        (
            "What is the currency of Japan?",
            "currency of Japan",
            "yen",
        ),
        (
            "What is the official language of Brazil?",
            "official language of Brazil",
            "Portuguese",
        ),
    ];

    // One DeBERTa load for the whole suite — saves ~15s per
    // additional case versus loading per-test.
    let engine = NliEngine::from_dir(&deberta_dir).expect("load DeBERTa");
    let entailer = Arc::new(DebertaEntailer::new(engine));

    let wikidata = Arc::new(WikidataRetriever::new().expect("wikidata client"));
    let mut registry = RetrieverRegistry::new();
    registry.insert(wikidata);

    let (catalog, raps) = wikidata_catalog_and_raps();

    let mut results: Vec<(String, String, AnswerStatus)> = Vec::new();
    for (query_text, decomposition_text, expected_substring) in &cases {
        eprintln!("\n========= {query_text} =========");

        let understanding = FixtureUnderstanding::new();
        understanding.insert(
            query_text,
            fixture_analysis(query_text, decomposition_text),
        );
        let analysis = understanding
            .analyze(&OriginalQuery {
                text: Some((*query_text).into()),
                image_ref: None,
                audio_ref: None,
                video_ref: None,
                language_detected: "en".into(),
                timestamp: Utc::now(),
            })
            .expect("analyze");

        let self_model = SelfModel::new();
        self_model.register_retriever(wikidata_retriever_capability());
        let capabilities = self_model.snapshot();

        let started = Instant::now();
        let plan = plan(
            &analysis,
            &capabilities,
            &catalog,
            Constraints::default(),
            Budget::default(),
        )
        .expect("plan");

        let exec_result = orch_execute(
            &plan,
            &raps,
            &registry,
            &self_model,
            &OrchestratorConfig::default(),
        )
        .await
        .expect("execute");
        eprintln!(
            "  retrieved {} item(s) in {:?}",
            exec_result.items.len(),
            exec_result.elapsed
        );
        if exec_result.items.is_empty() {
            results.push((
                (*query_text).into(),
                "<no retrieval>".into(),
                AnswerStatus::Degraded,
            ));
            continue;
        }

        let composer = TemplateComposer::new();
        let draft = composer
            .draft(&plan, &exec_result.items, &[])
            .expect("draft");
        eprintln!("  draft: {:?}", draft.first().map(|s| &s.text));

        let segments_for_atomizer: Vec<(String, String)> = draft
            .iter()
            .map(|s| (s.segment_id.clone(), s.text.clone()))
            .collect();
        let claims = atomize_sentences(&segments_for_atomizer, &axes_for_template_claim())
            .expect("atomize");

        let report = verify(
            &VerifyInputs::new(&plan, &exec_result.items, &[], &claims),
            entailer.as_ref(),
        )
        .expect("verify");
        eprintln!(
            "  diagnose: verdict={:?} violations={} entailments={}",
            report.verdict,
            report.violations.len(),
            report.entailments_consulted.len()
        );

        let inputs = ComposeInputs {
            plan: &plan,
            draft_segments: &draft,
            claims: &claims,
            items: &exec_result.items,
            report: &report,
            joules_spent_total: report.joules_spent,
            latency_ms: started.elapsed().as_millis() as u64,
        };
        let composed = compose_verified_answer(&inputs).expect("compose");
        let ans = composed.as_answer().expect("expected Answer, got Refusal");

        let composite_text: String = ans
            .segments
            .iter()
            .map(|s| s.text.clone())
            .collect::<Vec<_>>()
            .join(" ");
        eprintln!("  answer: {composite_text:?}");

        results.push(((*query_text).into(), composite_text.clone(), ans.status));

        // Per-case acceptance: Verified or Degraded (not Refusal),
        // and the expected substring appears in the output.
        assert!(
            matches!(ans.status, AnswerStatus::Verified | AnswerStatus::Degraded),
            "[{query_text}] expected an Answer with non-failure status, got {:?}",
            ans.status
        );
        assert!(
            composite_text.to_lowercase().contains(&expected_substring.to_lowercase()),
            "[{query_text}] expected output to contain {expected_substring:?}, got {composite_text:?}"
        );
    }

    eprintln!("\n========= suite summary =========");
    let verified_count = results
        .iter()
        .filter(|(_, _, s)| matches!(s, AnswerStatus::Verified))
        .count();
    let degraded_count = results
        .iter()
        .filter(|(_, _, s)| matches!(s, AnswerStatus::Degraded))
        .count();
    for (q, a, s) in &results {
        eprintln!("  [{s:?}] {q}\n        → {a}");
    }
    eprintln!(
        "verified: {verified_count}/{}, degraded: {degraded_count}/{}",
        results.len(),
        results.len()
    );

    // The interesting invariant isn't "every query is Verified" —
    // some queries (e.g. "currency of Japan") legitimately yield
    // Degraded because Wikidata's EntitySearch returns multiple
    // entities with the same label (modern Japan + Empire of Japan
    // + Tokugawa shogunate) and the historical entities carry
    // different currencies. DeBERTa correctly scores those as
    // contradicting and the diagnose pillar surfaces a
    // `[contested]` marker per invariant I6.
    //
    // That's the spec working as designed. The acceptance bar is:
    //   - every case produced an Answer (not Refusal)
    //   - every case's final text contains the expected substring
    //   - at least one Verified case demonstrates the clean path
    assert!(
        verified_count >= 1,
        "expected at least one Verified case to demonstrate the clean RAP→DeBERTa path"
    );
}

/// Reroute loop integration test (spec §8.3).
///
/// "population of Tokyo" doesn't match any property_path pattern
/// and has no Wikidata entity literally named that, so pass 1
/// returns zero items → coverage violation → verdict
/// `ReRouteWithRefinement`. The pipeline's `apply_refinement`
/// drops the leading "population of " prefix, leaving "Tokyo".
/// Pass 2 retrieves real entities and the verifier produces a
/// usable answer.
///
/// Verified bar:
///   - reroute_passes == 1 (one refinement happened)
///   - result is an Answer (not a Refusal)
///   - the final answer text mentions Tokyo
///   - latency is bounded (didn't loop forever)
///
/// Uses `--no-verify` semantics by skipping the DeBERTa engine
/// load — the test only cares about the loop's structural
/// behavior, not the entailment numbers.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore]
async fn reroute_loop_recovers_from_no_match_via_refinement() {
    use jouleclaw_edge_cli::cli::Options;
    use jouleclaw_edge_cli::pipeline;
    use std::path::PathBuf;

    let opts = Options {
        query: "what is the population of Tokyo?".to_string(),
        json: false,
        no_verify: true, // skip DeBERTa load — we only test the loop
        model_dir: PathBuf::from("./models/deberta-v3-large-mnli"),
        cache_dir: PathBuf::from("/tmp/joule-test-cache-reroute"),
        no_cache: true, // tests of the loop don't want cache interference
        verbose: false,
        wikidata_endpoint: None,
        wikipedia_endpoint: None,
    };

    let log: Box<dyn Fn(&str) + Send + Sync> = Box::new(|s: &str| eprintln!("{s}"));

    let output = pipeline::run(&opts, log.as_ref())
        .await
        .expect("pipeline ran");

    eprintln!(
        "[reroute] passes={} verdict={:?} items={} segments={}",
        output.reroute_passes,
        output.report.verdict,
        output.items.len(),
        match &output.result {
            jouleclaw_compose::AnswerOrRefusal::Answer(a) => a.segments.len(),
            jouleclaw_compose::AnswerOrRefusal::Refusal(_) => 0,
        },
    );

    assert_eq!(
        output.reroute_passes, 1,
        "expected exactly one reroute pass for 'population of Tokyo'"
    );
    assert!(
        matches!(output.result, jouleclaw_compose::AnswerOrRefusal::Answer(_)),
        "expected an Answer (not Refusal) after refinement"
    );
    let ans = match &output.result {
        jouleclaw_compose::AnswerOrRefusal::Answer(a) => a,
        _ => unreachable!(),
    };
    let composite = ans
        .segments
        .iter()
        .map(|s| s.text.as_str())
        .collect::<Vec<_>>()
        .join(" ");
    assert!(
        composite.to_lowercase().contains("tokyo"),
        "expected 'Tokyo' in {composite:?}"
    );
    assert!(
        output.stages.total_ms < 30_000,
        "reroute loop should complete within 30s, got {} ms",
        output.stages.total_ms
    );
}

/// When no refinement helps (the prefix isn't in our registry and
/// Wikidata has nothing for the literal phrase), the loop should
/// not spin — it should detect "no actionable refinement", break,
/// re-verify at the budget-exhausted level, and emit a structured
/// Refusal.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore]
async fn reroute_loop_refuses_when_no_refinement_possible() {
    use jouleclaw_edge_cli::cli::Options;
    use jouleclaw_edge_cli::pipeline;
    use std::path::PathBuf;

    let opts = Options {
        query: "what is the elevation of Mont Blanc?".to_string(),
        json: false,
        no_verify: true,
        model_dir: PathBuf::from("./models/deberta-v3-large-mnli"),
        cache_dir: PathBuf::from("/tmp/joule-test-cache-no-refinement"),
        no_cache: true,
        verbose: false,
        wikidata_endpoint: None,
        wikipedia_endpoint: None,
    };

    let log: Box<dyn Fn(&str) + Send + Sync> = Box::new(|s: &str| eprintln!("{s}"));
    let output = pipeline::run(&opts, log.as_ref())
        .await
        .expect("pipeline ran");

    eprintln!(
        "[no-refinement] passes={} verdict={:?}",
        output.reroute_passes, output.report.verdict,
    );

    // Refinement registry doesn't know "elevation of " → loop
    // settles on the first pass without retrying.
    assert_eq!(output.reroute_passes, 0);
    assert!(
        matches!(output.result, jouleclaw_compose::AnswerOrRefusal::Refusal(_)),
        "expected a Refusal when refinement can't help"
    );
    let r = match &output.result {
        jouleclaw_compose::AnswerOrRefusal::Refusal(r) => r,
        _ => unreachable!(),
    };
    assert!(
        r.reason_code.contains("unsatisfiable"),
        "refusal reason_code should mention 'unsatisfiable', got {:?}",
        r.reason_code
    );
}