//! The app's own words go through its catalogue (src/i18n.rs), and every //! catalogue says what the English says with the same placeholders. //! //! 1. No English sentence or label is written into the app's code outside the //! catalogue: every string literal in `src/` (tests aside) that reads as //! words a person sees fails, unless it is technical — a log line, an //! error for developers, a key, a class — or vocabulary (the words on //! boards, translated by translate.rs). Mark those with a comment on the //! line or the line before: `// i18n: not UI ()`, or around a block, //! `// i18n-off: ` … `// i18n-on`. //! 2. Every message id the code uses is an English message, and every //! English message is used (ids are literals, so this can be checked). //! 3. Each catalogue parses, has only English ids, and each message has the //! English one's placeholders — no more, no fewer (`$speaker`, the //! person's "Words about myself", may be added). use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use fluent_syntax::ast; fn root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } fn rust_files(dir: &Path, out: &mut Vec) { for entry in std::fs::read_dir(dir).unwrap().flatten() { let p = entry.path(); if p.is_dir() { rust_files(&p, out); } else if p.extension().is_some_and(|x| x == "rs") { out.push(p); } } } /// A string literal in Rust source. struct Lit { start: usize, value: String, } /// The string literals of `src`, with comments and `#[cfg(test)]` modules /// left out. fn literals(src: &str) -> Vec { let b = src.as_bytes(); let n = b.len(); let mut i = 0; let mut out = Vec::new(); // Brace depth, and the depth at which a test module began. let mut depth = 0usize; let mut skip_from: Option = None; let mut test_pending = false; while i < n { let c = b[i]; if src[i..].starts_with("//") { i = src[i..].find('\n').map_or(n, |j| i + j); continue; } if src[i..].starts_with("/*") { let mut d = 1; i += 2; while i < n && d > 0 { if src[i..].starts_with("/*") { d += 1; i += 2; } else if src[i..].starts_with("*/") { d -= 1; i += 2; } else { i += 1; } } continue; } if src[i..].starts_with("#[cfg(test)]") { test_pending = true; i += 12; continue; } let prev_ident = i > 0 && (b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_'); // Raw strings: r"…", r#"…"#, br"…". if !prev_ident && (c == b'r' || (c == b'b' && i + 1 < n && b[i + 1] == b'r')) { let mut j = i + if c == b'b' { 2 } else { 1 }; let mut hashes = 0; while j < n && b[j] == b'#' { hashes += 1; j += 1; } if j < n && b[j] == b'"' && (c == b'b' || b[i] == b'r') && (j > i + 1 || c == b'r') { let close: String = std::iter::once('"').chain(std::iter::repeat_n('#', hashes)).collect(); let body_start = j + 1; let end = src[body_start..].find(&close).map_or(n, |k| body_start + k); if skip_from.is_none() { out.push(Lit { start: i, value: src[body_start..end].to_string() }); } i = end + close.len(); continue; } } if c == b'"' || (c == b'b' && !prev_ident && i + 1 < n && b[i + 1] == b'"') { let start = i; i += if c == b'b' { 2 } else { 1 }; let mut value = String::new(); while i < n && b[i] != b'"' { if b[i] == b'\\' { let rest = &src[i + 1..]; if rest.starts_with('\n') || rest.starts_with("\r\n") { // A line continuation: the next line's indent goes. i += 1; while i < n && (b[i] as char).is_ascii_whitespace() { i += 1; } continue; } if let Some(r) = rest.strip_prefix("u{") { let close = r.find('}').unwrap_or(0); if let Some(ch) = u32::from_str_radix(&r[..close], 16).ok().and_then(char::from_u32) { value.push(ch); } i += 1 + 2 + close + 1; continue; } value.push(match rest.as_bytes().first() { Some(b'n') => '\n', Some(b't') => '\t', Some(&x) => x as char, None => ' ', }); i += 2; continue; } let ch = src[i..].chars().next().unwrap(); value.push(ch); i += ch.len_utf8(); } i += 1; if skip_from.is_none() { out.push(Lit { start, value }); } continue; } if c == b'\'' { // A char literal ('a', '\n', '\u{2192}'), or a lifetime. let rest = &src[i + 1..]; if let Some(r) = rest.strip_prefix('\\') { let close = r.find('\'').unwrap_or(0); i += 2 + close + 1; continue; } let mut chars = rest.chars(); if let (Some(ch), Some('\'')) = (chars.next(), chars.next()) { i += 1 + ch.len_utf8() + 1; continue; } i += 1; continue; } if c == b'{' { depth += 1; if test_pending { // `#[cfg(test)] mod name {`: skip to its closing brace. let before = &src[..i]; let line_start = before.rfind("#[cfg(test)]").unwrap_or(0); if before[line_start..].contains("mod ") && skip_from.is_none() { skip_from = Some(depth); } test_pending = false; } } else if c == b'}' { if skip_from == Some(depth) { skip_from = None; } depth = depth.saturating_sub(1); } else if c == b';' { test_pending = false; } i += 1; } out } /// Whether `s` reads as words a person sees (in English). fn reads_as_words(s: &str) -> bool { // Placeholders (`{name}`, `{:?}`) are not words. let mut without = String::new(); let mut depth = 0; for c in s.chars() { match c { '{' => depth += 1, '}' if depth > 0 => depth -= 1, _ if depth == 0 => without.push(c), _ => {} } } let t = without.trim(); // CSS declarations (`padding: 6px; border: 2px solid #555;`). if t.matches(';').count() >= 2 && t.split(';').filter(|d| !d.trim().is_empty()).all(|d| { d.split_once(':').is_some_and(|(p, _)| p.trim().chars().all(|c| c.is_ascii_lowercase() || c == '-')) }) { return false; } if !t.as_bytes().windows(2).any(|w| w[0].is_ascii_alphabetic() && w[1].is_ascii_alphabetic()) { return false; } let technical_start = ["http:", "https:", "/", "./", "#", "data:", "aac", "--", "{}", "${", "<"]; if technical_start.iter().any(|p| t.starts_with(p)) { return false; } // CSS classes, ids, keys, paths, MIME types. if t.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "_-:./ =,;()%#*[]\"'>+~".contains(c)) { let words = t.split_whitespace().filter(|w| w.chars().all(|c| c.is_ascii_lowercase() || c == '\'')).count(); // Three plain lowercase words is a sentence ("tap to choose"). return words >= 3 && !t.contains(':') && !t.contains(';') && !t.contains('-'); } // CamelCase and paths (`SwitchScan`, `aac_core::Board`). if !t.contains(' ') && (t.contains("::") || t.chars().skip(1).any(|c| c.is_ascii_uppercase())) { return false; } // Leading emoji, arrows and marks before the words. let words_part = t.trim_start_matches(|c: char| !c.is_alphanumeric()); let first = words_part.chars().next().unwrap_or(' '); let words: Vec<&str> = words_part .split(|c: char| c.is_whitespace()) .filter(|w| w.chars().filter(|c| c.is_alphabetic()).count() >= 1) .collect(); if words.len() >= 2 { return first.is_uppercase() || t.contains(['.', '?', '!', ':', ',', '\u{2019}', '\u{2026}', '\u{201c}']) || words.len() >= 3; } // One word: a capitalised label ("Dark", "Settings"). first.is_uppercase() && words_part.chars().skip(1).take_while(|c| c.is_alphabetic()).all(|c| c.is_lowercase()) && words_part.chars().filter(|c| c.is_alphabetic()).count() >= 2 } /// Calls whose arguments are technical: logs, developer errors, DOM and /// JS plumbing. A literal inside one (its parenthesis still open) is not UI. const TECHNICAL_CALLS: &[&str] = &[ "console::", "warn_1(", "log_1(", "error_1(", ".expect(", "panic!(", "unreachable!(", "assert!(", "assert_eq!(", "assert_ne!(", "debug_assert", "todo!(", "tracing::", "i18n::log(", "query_selector", "get_attribute(", "set_attribute(", "remove_attribute(", "has_attribute(", "Reflect::get(", "Reflect::set(", "include_str!(", "env!(", "#[serde(", "#[error(", "add_event_listener", "remove_event_listener", "dyn_into", "closest(", "matches(", "set_property(", "get_property_value(", "class_list", "JsValue::from_str(\"aac", ]; /// Attributes whose values are technical (a literal right after one). const TECHNICAL_ATTRIBUTES: &[&str] = &[ "class", "style", "id", "key", "href", "src", "type", "rel", "role", "inputmode", "autocomplete", "name", "for", "accept", "capture", "lang", "dir", "target", "download", "viewBox", "d", "fill", "stroke", "xmlns", "content", "charset", "media", "sizes", "value", "min", "max", "step", "pattern", "aria-hidden", "aria-live", "aria-controls", "aria-labelledby", "aria-describedby", "aria-haspopup", "aria-expanded", "aria-pressed", "aria-checked", "aria-modal", "aria-atomic", "aria-current", "aria-selected", "aria-disabled", "aria-orientation", "tabindex", ]; /// Whether the literal starting after `lead` (the text before it on its /// line; `prev`, the line before) is technical: an argument of one of /// [`TECHNICAL_CALLS`] still open, or the value of a technical attribute /// (or `data-*`), in the same element (no `>` between). fn technical(lead: &str, prev: &str) -> bool { let open_call = |text: &str| { TECHNICAL_CALLS.iter().any(|c| { text.rfind(c).is_some_and(|i| { let after = &text[i..]; after.matches('(').count() > after.matches(')').count() }) }) }; if open_call(lead) || (lead.trim().is_empty() || lead.trim_end().ends_with('(') || lead.trim_end().ends_with(',')) && open_call(&format!("{prev} {lead}")) { return true; } // The attribute this literal is the value of: `name=` right before it // (allowing `name={`, `name=format!(`, `name=move || if … {`), with no // tag end between. let tail = &lead[lead.rfind('>').map_or(0, |i| i + 1)..]; let Some(eq) = tail.rfind('=') else { return false }; if tail[eq..].contains("==") || tail[..eq].ends_with(['=', '!', '<', '>']) { return false; } let name: String = tail[..eq].chars().rev().take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == ':' || *c == '_').collect::>().into_iter().rev().collect(); let name = name.rsplit(':').next().unwrap_or(&name); name.starts_with("data-") || TECHNICAL_ATTRIBUTES.contains(&name) } /// `AAC_UI_STRINGS_ONLY=components/lock.rs,voices.rs`: check only these /// files (paths under src/), and only their catalogues for unused messages. fn only() -> Option> { std::env::var("AAC_UI_STRINGS_ONLY").ok().map(|v| v.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()) } /// The English catalogue of a source file: `components/panels/access_settings.rs` → `access-settings.ftl`. fn catalogue_of(rel: &str) -> String { let mut parts = rel.rsplit('/'); let mut stem = parts.next().unwrap_or(rel).trim_end_matches(".rs"); // A module's mod.rs: its directory's name (`components/panels/mod.rs` → `panels.ftl`). if stem == "mod" { stem = parts.next().unwrap_or(stem); } format!("{}.ftl", stem.replace('_', "-")) } /// Files that are vocabulary or data, not the app's own words: the words on /// the app's boards (translated by translate.rs), the help messages, print /// texts and nav labels already in their own reviewed catalogues. const VOCABULARY_FILES: &[&str] = &["defaults.rs", "i18n.rs", "translate.rs"]; #[test] fn no_english_words_are_written_into_the_app_outside_the_catalogue() { let src = root().join("src"); let mut files = Vec::new(); rust_files(&src, &mut files); files.sort(); let mut found = Vec::new(); for path in &files { let rel = path.strip_prefix(&src).unwrap().to_string_lossy().replace('\\', "/"); if VOCABULARY_FILES.contains(&rel.as_str()) || only().is_some_and(|o| !o.contains(&rel)) { continue; } let text = std::fs::read_to_string(path).unwrap(); let lines: Vec<&str> = text.lines().collect(); // Line starts, for line numbers. let starts: Vec = std::iter::once(0).chain(text.match_indices('\n').map(|(i, _)| i + 1)).collect(); // Lines inside `// i18n-off` … `// i18n-on`. let mut off = vec![false; lines.len() + 1]; let mut in_off = false; for (k, line) in lines.iter().enumerate() { if line.contains("// i18n-off") { in_off = true; } off[k] = in_off; if line.contains("// i18n-on") { in_off = false; } } for lit in literals(&text) { if !reads_as_words(&lit.value) { continue; } let line = starts.partition_point(|&s| s <= lit.start) - 1; if off[line] { continue; } let this = lines.get(line).copied().unwrap_or(""); let before = if line > 0 { lines[line - 1] } else { "" }; if this.contains("i18n: not UI") || before.trim_start().starts_with("// i18n: not UI") { continue; } let col = lit.start - starts[line]; let lead = &text[starts[line]..starts[line] + col]; // A literal continuing a call begun on an earlier line: look at // that line too (`console::warn_1(\n "…"`). if technical(lead, before) { continue; } found.push(format!("src/{rel}:{}: {:?}", line + 1, lit.value)); } } assert!( found.is_empty(), "{} English string(s) in the app's code are not in the catalogue (i18n/en/*.ftl, src/i18n.rs) — \ or, if technical or vocabulary, mark them `// i18n: not UI ()`:\n{}", found.len(), found.join("\n") ); } // ─── The catalogues ────────────────────────────────────────────────────────── /// A message's placeholders (`$name`), in its value and every variant. fn variables(pattern: &ast::Pattern<&str>, out: &mut BTreeSet) { for el in &pattern.elements { if let ast::PatternElement::Placeable { expression } = el { expression_vars(expression, out); } } } fn expression_vars(e: &ast::Expression<&str>, out: &mut BTreeSet) { match e { ast::Expression::Select { selector, variants } => { inline_vars(selector, out); for v in variants { variables(&v.value, out); } } ast::Expression::Inline(i) => inline_vars(i, out), } } fn inline_vars(i: &ast::InlineExpression<&str>, out: &mut BTreeSet) { match i { ast::InlineExpression::VariableReference { id } => { out.insert(id.name.to_string()); } ast::InlineExpression::Placeable { expression } => expression_vars(expression, out), ast::InlineExpression::FunctionReference { arguments, .. } => { for a in &arguments.positional { inline_vars(a, out); } for a in &arguments.named { inline_vars(&a.value, out); } } _ => {} } } /// The messages of a catalogue source: id → placeholders. Fails on a parse /// error, a term, an attribute, a message with no value, or an id twice. fn messages(name: &str, source: &str, seen: &mut BTreeMap) -> BTreeMap> { let resource = match fluent_syntax::parser::parse(source) { Ok(r) => r, Err((_, errors)) => panic!("{name} does not parse: {errors:?}"), }; let mut out = BTreeMap::new(); for entry in resource.body { match entry { ast::Entry::Message(m) => { assert!(m.attributes.is_empty(), "{name}: {} has attributes (one message per text, please)", m.id.name); let value = m.value.as_ref().unwrap_or_else(|| panic!("{name}: {} has no text", m.id.name)); if let Some(other) = seen.insert(m.id.name.to_string(), name.to_string()) { panic!("{} is in both {other} and {name}", m.id.name); } let mut vars = BTreeSet::new(); variables(value, &mut vars); out.insert(m.id.name.to_string(), vars); } ast::Entry::Term(t) => panic!("{name}: terms (-{}) are not used; write the words in each message", t.id.name), _ => {} } } out } fn english() -> BTreeMap> { english_by_file().into_values().flatten().collect() } fn english_by_file() -> BTreeMap>> { let dir = root().join("i18n/en"); let mut files: Vec<_> = std::fs::read_dir(&dir).unwrap().flatten().map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "ftl")).collect(); files.sort(); let mut seen = BTreeMap::new(); let mut all = BTreeMap::new(); for f in files { let file = f.file_name().unwrap().to_string_lossy().to_string(); let m = messages(&format!("en/{file}"), &std::fs::read_to_string(&f).unwrap(), &mut seen); all.insert(file, m); } all } fn translations() -> Vec<(String, String)> { let mut out: Vec<_> = std::fs::read_dir(root().join("i18n")) .unwrap() .flatten() .map(|e| e.path()) .filter(|p| p.is_file() && p.extension().is_some_and(|x| x == "ftl")) .map(|p| (p.file_stem().unwrap().to_string_lossy().to_string(), std::fs::read_to_string(&p).unwrap())) .collect(); out.sort(); out } #[test] fn every_catalogue_has_the_english_placeholders_and_only_english_ids() { let en = english(); assert!(!en.is_empty()); for (code, source) in translations() { assert!(aac_core::language_by_code(&code).is_some(), "i18n/{code}.ftl: {code} is not one of the app's languages"); let mut seen = BTreeMap::new(); let messages = messages(&format!("{code}.ftl"), &source, &mut seen); let mut wrong = Vec::new(); for (id, vars) in &messages { let Some(english) = en.get(id) else { wrong.push(format!("{id}: not an English message")); continue; }; let mut vars = vars.clone(); vars.remove("speaker"); if &vars != english { wrong.push(format!("{id}: placeholders {vars:?}, the English has {english:?}")); } } assert!(wrong.is_empty(), "i18n/{code}.ftl:\n{}", wrong.join("\n")); } } /// The message ids the code names: the first argument of the i18n /// functions, which must be a literal. fn used_ids() -> BTreeMap { let src = root().join("src"); let mut files = Vec::new(); rust_files(&src, &mut files); let mut out = BTreeMap::new(); let calls = ["t(", "ts(", "t_with(", "ts_with(", "tr(", "tr_args(", "lang_of(", "ui_text(", "said(", "said_args("]; for path in files { let text = std::fs::read_to_string(&path).unwrap(); let rel = path.strip_prefix(&src).unwrap().to_string_lossy().to_string(); for call in calls { for (i, _) in text.match_indices(call) { if i > 0 { let p = text.as_bytes()[i - 1]; if p.is_ascii_alphanumeric() || p == b'_' || p == b'.' { continue; } } let rest = text[i + call.len()..].trim_start(); if let Some(r) = rest.strip_prefix('"') { let id = &r[..r.find('"').unwrap_or(0)]; // Every id is `-`: a hyphen tells it from // another function's argument (`tr("hi")` in a local helper). if id.contains('-') && id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { out.entry(id.to_string()).or_insert_with(|| rel.clone()); } } } } } out } #[test] fn every_message_the_app_names_is_english_and_every_english_message_is_used() { let en = english(); let used = used_ids(); let missing: Vec<_> = used.iter().filter(|(id, _)| !en.contains_key(*id)).map(|(id, f)| format!("{id} (src/{f})")).collect(); assert!(missing.is_empty(), "message ids not in i18n/en/*.ftl:\n{}", missing.join("\n")); let mine: Option> = only().map(|o| o.iter().map(|r| catalogue_of(r)).collect()); let unused: Vec<_> = english_by_file() .into_iter() .filter(|(file, _)| mine.as_ref().is_none_or(|m| m.contains(file))) .flat_map(|(_, ids)| ids.into_keys()) .filter(|id| !used.contains_key(id)) // The page-set catalogue's texts, named by a hash of their English // and looked up by it (aac-web pagesets.rs checks them both ways). .filter(|id| !id.starts_with("pageset-catalogue-")) // The printed book's texts, looked up by print_texts.rs by their key // (checked against print_texts.json there). .filter(|id| !id.starts_with("print-text-")) .collect(); assert!(unused.is_empty(), "English messages the app never uses (remove them, or use them):\n{}", unused.join("\n")); }