diff --git a/Cargo.lock b/Cargo.lock index 84097f1d8..c7c2bac27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3025,10 +3025,12 @@ dependencies = [ "camino", "clap", "mas-i18n", + "minijinja", "serde_json", "tera", "tracing", "tracing-subscriber", + "walkdir", ] [[package]] @@ -3436,6 +3438,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minijinja" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80084fa3099f58b7afab51e5f92e24c2c2c68dcad26e96ad104bd6011570461d" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" diff --git a/crates/i18n-scan/Cargo.toml b/crates/i18n-scan/Cargo.toml index f710f7118..45e0c6d40 100644 --- a/crates/i18n-scan/Cargo.toml +++ b/crates/i18n-scan/Cargo.toml @@ -10,9 +10,11 @@ repository.workspace = true [dependencies] camino.workspace = true clap.workspace = true -tera.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true +minijinja = { version = "1.0.8", features = ["unstable_machinery"] } serde_json.workspace = true +tera.workspace = true +tracing-subscriber.workspace = true +tracing.workspace = true +walkdir = "2.4.0" mas-i18n = { path = "../i18n" } \ No newline at end of file diff --git a/crates/i18n-scan/src/key.rs b/crates/i18n-scan/src/key.rs new file mode 100644 index 000000000..5e7b85312 --- /dev/null +++ b/crates/i18n-scan/src/key.rs @@ -0,0 +1,56 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use mas_i18n::{translations::TranslationTree, Message}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyKind { + Message, + Plural, +} + +#[derive(Debug, Clone)] +pub struct Key { + kind: KeyKind, + key: String, +} + +impl Key { + pub fn new(kind: KeyKind, key: String) -> Self { + Self { kind, key } + } + + pub fn default_value(&self) -> String { + match self.kind { + KeyKind::Message => self.key.clone(), + KeyKind::Plural => format!("%(count)d {}", self.key), + } + } +} + +pub fn add_missing(translation_tree: &mut TranslationTree, keys: &[Key]) { + for translatable in keys { + let message = Message::from_literal(translatable.default_value()); + let key = translatable + .key + .split('.') + .chain(if translatable.kind == KeyKind::Plural { + Some("other") + } else { + None + }); + + translation_tree.set_if_not_defined(key, message); + } +} diff --git a/crates/i18n-scan/src/main.rs b/crates/i18n-scan/src/main.rs index 21a7ee4b6..5eb0809ab 100644 --- a/crates/i18n-scan/src/main.rs +++ b/crates/i18n-scan/src/main.rs @@ -20,10 +20,13 @@ use std::fs::File; use ::tera::Tera; use camino::Utf8PathBuf; use clap::Parser; +use key::add_missing; use mas_i18n::translations::TranslationTree; -use crate::tera::{add_missing, find_keys}; +use crate::tera::find_keys; +mod key; +mod minijinja; mod tera; /// Scan a directory of templates for usage of the translation function and @@ -36,6 +39,10 @@ struct Options { /// Path of the existing translation file existing: Option, + /// Whether to use minijinja instead of tera + #[clap(long)] + minijinja: bool, + /// The name of the translation function #[clap(long, default_value = "t")] function: String, @@ -45,11 +52,6 @@ fn main() { tracing_subscriber::fmt::init(); let options = Options::parse(); - let glob = format!("{base}/**/*.{{html,txt,subject}}", base = options.templates); - tracing::debug!("Scanning templates in {}", glob); - let tera = Tera::new(&glob).expect("Failed to load templates"); - - let keys = find_keys(&tera, &options.function).unwrap(); let mut tree = if let Some(path) = options.existing { let mut file = File::open(path).expect("Failed to open existing translation file"); @@ -58,6 +60,35 @@ fn main() { TranslationTree::default() }; + let keys = if options.minijinja { + let mut keys = Vec::new(); + for entry in walkdir::WalkDir::new(&options.templates) { + let entry = entry.unwrap(); + let filename = entry.file_name().to_str().expect("Invalid filename"); + if entry.file_type().is_file() + && (filename.ends_with(".html") + || filename.ends_with(".txt") + || filename.ends_with(".subject")) + { + let content = std::fs::read_to_string(entry.path()).unwrap(); + match minijinja::parse(&content, filename) { + Ok(ast) => { + keys.extend(minijinja::find_in_stmt(&ast).unwrap()); + } + Err(err) => { + tracing::error!("Failed to parse {}: {}", entry.path().display(), err); + } + } + } + } + keys + } else { + let glob = format!("{base}/**/*.{{html,txt,subject}}", base = options.templates); + tracing::debug!("Scanning templates in {}", glob); + let tera = Tera::new(&glob).expect("Failed to load templates"); + + find_keys(&tera, &options.function).unwrap() + }; add_missing(&mut tree, &keys); serde_json::to_writer_pretty(std::io::stdout(), &tree) diff --git a/crates/i18n-scan/src/minijinja.rs b/crates/i18n-scan/src/minijinja.rs new file mode 100644 index 000000000..b6d95ac6c --- /dev/null +++ b/crates/i18n-scan/src/minijinja.rs @@ -0,0 +1,244 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub use minijinja::machinery::parse; +use minijinja::{ + machinery::ast::{Call, Const, Expr, Stmt}, + ErrorKind, +}; + +use crate::key::{Key, KeyKind}; + +pub fn find_in_stmt<'a>(stmt: &'a Stmt<'a>) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + match stmt { + Stmt::Template(template) => keys.extend(find_in_stmts(&template.children)?), + Stmt::EmitExpr(emit_expr) => keys.extend(find_in_expr(&emit_expr.expr)?), + Stmt::EmitRaw(_raw) => {} + Stmt::ForLoop(for_loop) => { + keys.extend(find_in_expr(&for_loop.iter)?); + keys.extend(find_in_optional_expr(&for_loop.filter_expr)?); + keys.extend(find_in_expr(&for_loop.target)?); + keys.extend(find_in_stmts(&for_loop.body)?); + keys.extend(find_in_stmts(&for_loop.else_body)?); + } + Stmt::IfCond(if_cond) => { + keys.extend(find_in_expr(&if_cond.expr)?); + keys.extend(find_in_stmts(&if_cond.true_body)?); + keys.extend(find_in_stmts(&if_cond.false_body)?); + } + Stmt::WithBlock(with_block) => { + keys.extend(find_in_stmts(&with_block.body)?); + for (left, right) in &with_block.assignments { + keys.extend(find_in_expr(left)?); + keys.extend(find_in_expr(right)?); + } + } + Stmt::Set(set) => { + keys.extend(find_in_expr(&set.target)?); + keys.extend(find_in_expr(&set.expr)?); + } + Stmt::SetBlock(set_block) => { + keys.extend(find_in_expr(&set_block.target)?); + keys.extend(find_in_stmts(&set_block.body)?); + if let Some(expr) = &set_block.filter { + keys.extend(find_in_expr(expr)?); + } + } + Stmt::AutoEscape(auto_escape) => { + keys.extend(find_in_expr(&auto_escape.enabled)?); + keys.extend(find_in_stmts(&auto_escape.body)?); + } + Stmt::FilterBlock(filter_block) => { + keys.extend(find_in_expr(&filter_block.filter)?); + keys.extend(find_in_stmts(&filter_block.body)?); + } + Stmt::Block(block) => { + keys.extend(find_in_stmts(&block.body)?); + } + Stmt::Import(import) => { + keys.extend(find_in_expr(&import.name)?); + keys.extend(find_in_expr(&import.expr)?); + } + Stmt::FromImport(from_import) => { + keys.extend(find_in_expr(&from_import.expr)?); + for (name, alias) in &from_import.names { + keys.extend(find_in_expr(name)?); + keys.extend(find_in_optional_expr(alias)?); + } + } + Stmt::Extends(extends) => { + keys.extend(find_in_expr(&extends.name)?); + } + Stmt::Include(include) => { + keys.extend(find_in_expr(&include.name)?); + } + Stmt::Macro(macro_) => { + keys.extend(find_in_stmts(¯o_.body)?); + keys.extend(find_in_exprs(¯o_.args)?); + keys.extend(find_in_exprs(¯o_.defaults)?); + } + Stmt::CallBlock(call_block) => { + keys.extend(find_in_call(&call_block.call)?); + // TODO: call_block.macro_decl + } + Stmt::Do(do_) => { + keys.extend(find_in_call(&do_.call)?); + } + } + + Ok(keys) +} + +fn as_const<'a>(expr: &'a Expr<'a>) -> Option<&'a Const> { + match expr { + Expr::Const(const_) => Some(const_), + _ => None, + } +} + +fn find_in_call<'a>(call: &'a Call<'a>) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + if let Expr::Var(var_) = &call.expr { + // TODO: pass the function name + if var_.id == "t" { + // TODO: don't unwrap + let key = call + .args + .get(0) + .and_then(as_const) + .and_then(|const_| const_.value.as_str()) + .ok_or(minijinja::Error::new( + ErrorKind::UndefinedError, + "t() first argument must be a string literal", + ))?; + + let has_count = call.args.iter().any(|arg| { + if let Expr::Kwargs(kwargs) = arg { + kwargs.pairs.iter().any(|(key, _value)| *key == "count") + } else { + false + } + }); + + // TODO: detect plurals + keys.push(Key::new( + if has_count { + KeyKind::Plural + } else { + KeyKind::Message + }, + key.to_owned(), + )); + } + } + + keys.extend(find_in_expr(&call.expr)?); + for arg in &call.args { + keys.extend(find_in_expr(arg)?); + } + + Ok(keys) +} + +fn find_in_stmts<'a>(stmts: &'a [Stmt<'a>]) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + for stmt in stmts { + keys.extend(find_in_stmt(stmt)?); + } + + Ok(keys) +} + +fn find_in_expr<'a>(expr: &'a Expr<'a>) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + match expr { + Expr::Var(_var) => {} + Expr::Const(_const) => {} + Expr::Slice(slice) => { + keys.extend(find_in_expr(&slice.expr)?); + keys.extend(find_in_optional_expr(&slice.start)?); + keys.extend(find_in_optional_expr(&slice.stop)?); + keys.extend(find_in_optional_expr(&slice.step)?); + } + Expr::UnaryOp(unary_op) => { + keys.extend(find_in_expr(&unary_op.expr)?); + } + Expr::BinOp(bin_op) => { + keys.extend(find_in_expr(&bin_op.left)?); + keys.extend(find_in_expr(&bin_op.right)?); + } + Expr::IfExpr(if_expr) => { + keys.extend(find_in_expr(&if_expr.test_expr)?); + keys.extend(find_in_expr(&if_expr.true_expr)?); + keys.extend(find_in_optional_expr(&if_expr.false_expr)?); + } + Expr::Filter(filter) => { + keys.extend(find_in_optional_expr(&filter.expr)?); + keys.extend(find_in_exprs(&filter.args)?); + } + Expr::Test(test) => { + keys.extend(find_in_expr(&test.expr)?); + keys.extend(find_in_exprs(&test.args)?); + } + Expr::GetAttr(get_attr) => { + keys.extend(find_in_expr(&get_attr.expr)?); + } + Expr::GetItem(get_item) => { + keys.extend(find_in_expr(&get_item.expr)?); + keys.extend(find_in_expr(&get_item.subscript_expr)?); + } + Expr::Call(call) => { + keys.extend(find_in_call(call)?); + } + Expr::List(list) => { + keys.extend(find_in_exprs(&list.items)?); + } + Expr::Map(map) => { + keys.extend(find_in_exprs(&map.keys)?); + keys.extend(find_in_exprs(&map.values)?); + } + Expr::Kwargs(kwargs) => { + for (_key, value) in &kwargs.pairs { + keys.extend(find_in_expr(value)?); + } + } + } + + Ok(keys) +} + +fn find_in_exprs<'a>(exprs: &'a [Expr<'a>]) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + for expr in exprs { + keys.extend(find_in_expr(expr)?); + } + + Ok(keys) +} + +fn find_in_optional_expr<'a>(expr: &'a Option>) -> Result, minijinja::Error> { + let mut keys = Vec::new(); + + if let Some(expr) = expr { + keys.extend(find_in_expr(expr)?); + } + + Ok(keys) +} diff --git a/crates/i18n-scan/src/tera.rs b/crates/i18n-scan/src/tera.rs index 812c2c04a..9d78f0ed8 100644 --- a/crates/i18n-scan/src/tera.rs +++ b/crates/i18n-scan/src/tera.rs @@ -12,47 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use mas_i18n::{translations::TranslationTree, Message}; use tera::{ ast::{Block, Expr, ExprVal, FunctionCall, MacroDefinition, Node}, Error, Template, Tera, }; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum KeyKind { - Message, - Plural, -} - -pub struct Key { - kind: KeyKind, - key: String, -} - -impl Key { - fn default_value(&self) -> String { - match self.kind { - KeyKind::Message => self.key.clone(), - KeyKind::Plural => format!("%(count)d {}", self.key), - } - } -} - -pub fn add_missing(translation_tree: &mut TranslationTree, keys: &[Key]) { - for translatable in keys { - let message = Message::from_literal(translatable.default_value()); - let key = translatable - .key - .split('.') - .chain(if translatable.kind == KeyKind::Plural { - Some("other") - } else { - None - }); - - translation_tree.set_if_not_defined(key, message); - } -} +use crate::key::{Key, KeyKind}; /// Find all translatable strings in a Tera instance. /// @@ -309,7 +274,7 @@ fn find_in_function_call( KeyKind::Message }; - keys.push(Key { kind, key }); + keys.push(Key::new(kind, key)) } Ok(keys) @@ -320,6 +285,7 @@ mod tests { use tera::Tera; use super::*; + use crate::key::add_missing; #[test] fn test_find_keys() {