sailfish-compiler-0.9.0/Cargo.toml0000644000000032030000000000100124500ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "sailfish-compiler" version = "0.9.0" authors = ["Ryohei Machida "] description = "Simple, small, and extremely fast template engine for Rust" homepage = "https://github.com/rust-sailfish/sailfish" readme = "README.md" keywords = [ "markup", "template", "html", ] categories = ["template-engine"] license = "MIT" repository = "https://github.com/rust-sailfish/sailfish" [lib] name = "sailfish_compiler" doctest = false [dependencies.filetime] version = "0.2.21" [dependencies.home] version = "0.5.4" [dependencies.memchr] version = "2.5.0" [dependencies.proc-macro2] version = ">=1.0.11, <1.1.0" features = ["span-locations"] default-features = false [dependencies.quote] version = "1.0.26" default-features = false [dependencies.serde] version = "1.0" features = ["derive"] optional = true [dependencies.syn] version = "2.0" features = [ "parsing", "full", "visit-mut", "printing", ] default-features = false [dependencies.toml] version = "0.8.2" optional = true [dev-dependencies.pretty_assertions] version = "1.3.0" [features] config = [ "serde", "toml", ] default = ["config"] procmacro = [] sailfish-compiler-0.9.0/Cargo.toml.orig000075500000000000000000000020751046102023000161420ustar 00000000000000[package] name = "sailfish-compiler" version = "0.9.0" authors = ["Ryohei Machida "] description = "Simple, small, and extremely fast template engine for Rust" homepage = "https://github.com/rust-sailfish/sailfish" repository = "https://github.com/rust-sailfish/sailfish" readme = "../README.md" keywords = ["markup", "template", "html"] categories = ["template-engine"] license = "MIT" workspace = ".." edition = "2018" [lib] name = "sailfish_compiler" doctest = false [features] default = ["config"] procmacro = [] config = ["serde", "toml"] [dependencies] memchr = "2.5.0" quote = { version = "1.0.26", default-features = false } serde = { version = "1.0", features = ["derive"], optional = true } toml = { version = "0.8.2", optional = true } home = "0.5.4" filetime = "0.2.21" [dependencies.syn] version = "2.0" default-features = false features = ["parsing", "full", "visit-mut", "printing"] [dependencies.proc-macro2] version = ">=1.0.11, <1.1.0" default-features = false features = ["span-locations"] [dev-dependencies] pretty_assertions = "1.3.0" sailfish-compiler-0.9.0/README.md000075500000000000000000000071771046102023000145420ustar 00000000000000
![SailFish](./resources/logo.png) Simple, small, and extremely fast template engine for Rust ![Tests](https://github.com/rust-sailfish/sailfish/workflows/Tests/badge.svg)![Version](https://img.shields.io/crates/v/sailfish)![dependency status](https://deps.rs/repo/github/rust-sailfish/sailfish/status.svg)![Rust 1.60](https://img.shields.io/badge/rust-1.60+-lightgray.svg)![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) [User Guide](https://rust-sailfish.github.io/sailfish/) | [API Docs](https://docs.rs/sailfish) | [Examples](./examples)
## ✨ Features - Simple and intuitive syntax inspired by [EJS](https://ejs.co/) - Include another template file inside template - Built-in filters - Minimal dependencies (<15 crates in total) - Extremely fast (See [benchmarks](https://github.com/djc/template-benchmarks-rs)) - Better error message - Syntax highlighting support ([vscode](./syntax/vscode), [vim](./syntax/vim)) - Works on Rust 1.60 or later ## 🐟 Example Dependencies: ```toml [dependencies] sailfish = "0.9.0" ``` You can choose to use `TemplateSimple` to access fields directly: > Template file (templates/hello.stpl): > > ```erb > > > <% for msg in messages { %> >
<%= msg %>
> <% } %> > > > ``` > > Code: > > ```rust > use sailfish::TemplateSimple; > > #[derive(TemplateSimple)] > #[template(path = "hello.stpl")] > struct HelloTemplate { > messages: Vec > } > > fn main() { > let ctx = HelloTemplate { > messages: vec![String::from("foo"), String::from("bar")], > }; > println!("{}", ctx.render_once().unwrap()); > } > ``` Or use the more powerful `Template/TemplateMut/TemplateOnce`: > Template file (templates/hello.stpl): > > ```erb > > > <% for msg in &self.messages { %> >
<%= msg %>
> <% } %> >
<%= self.say_hello() %>
> > > ``` > > Code: > > ```rust > use sailfish::Template; > > #[derive(Template)] > #[template(path = "hello.stpl")] > struct HelloTemplate { > messages: Vec > } > > impl HelloTemplate { > fn say_hello(&self) -> String { > String::from("Hello!") > } > } > > fn main() { > let ctx = HelloTemplate { > messages: vec![String::from("foo"), String::from("bar")], > }; > println!("{}", ctx.render().unwrap()); > } > ``` You can find more examples in [examples](./examples) directory. ## 🐾 Roadmap - `Template` trait ([RFC](https://github.com/rust-sailfish/sailfish/issues/3)) - Template inheritance (block, partials, etc.) ## 👤 Author 🇯🇵 **Ryohei Machida** * GitHub: [@Kogia-sima](https://github.com/Kogia-sima) ## 🤝 Contributing Contributions, issues and feature requests are welcome! Since sailfish is an immature library, there are many [planned features](https://github.com/rust-sailfish/sailfish/labels/Type%3A%20RFC) that is on a stage of RFC. Please leave a comment if you have an idea about its design! Also I welcome any pull requests to improve sailfish! Find issues with [Status: PR Welcome](https://github.com/rust-sailfish/sailfish/issues?q=is%3Aissue+is%3Aopen+label%3A%22Status%3A+PR+Welcome%22) label, and [let's create a new pull request](https://github.com/rust-sailfish/sailfish/pulls)! ## Show your support Give a ⭐️ if this project helped you! ## 📝 License Copyright © 2020 [Ryohei Machida](https://github.com/Kogia-sima). This project is [MIT](https://github.com/rust-sailfish/sailfish/blob/master/LICENSE) licensed. --- *This README was generated with ❤️ by [readme-md-generator](https://github.com/kefranabg/readme-md-generator)*sailfish-compiler-0.9.0/build.rs000064400000000000000000000000151046102023000147050ustar 00000000000000fn main() {} sailfish-compiler-0.9.0/src/analyzer.rs000064400000000000000000000010701046102023000162240ustar 00000000000000use crate::error::*; use syn::visit_mut::VisitMut; use syn::Block; #[derive(Clone)] pub struct Analyzer {} impl Analyzer { pub fn new() -> Self { Self {} } #[inline] pub fn analyze(&self, ast: &mut Block) -> Result<(), Error> { let mut child = AnalyzerImpl { error: None }; child.visit_block_mut(ast); if let Some(e) = child.error { Err(e) } else { Ok(()) } } } struct AnalyzerImpl { error: Option, } impl VisitMut for AnalyzerImpl { // write code here } sailfish-compiler-0.9.0/src/compiler.rs000064400000000000000000000107561046102023000162240ustar 00000000000000use quote::ToTokens; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::Arc; use syn::Block; use crate::analyzer::Analyzer; use crate::config::Config; use crate::error::*; use crate::optimizer::Optimizer; use crate::parser::Parser; use crate::resolver::Resolver; use crate::translator::{TranslatedSource, Translator}; use crate::util::{read_to_string, rustfmt_block}; #[derive(Default)] pub struct Compiler { config: Config, } pub struct CompilationReport { pub deps: Vec, } impl Compiler { pub fn new() -> Self { Self::default() } pub fn with_config(config: Config) -> Self { Self { config } } fn translate_file_contents(&self, input: &Path) -> Result { let parser = Parser::new().delimiter(self.config.delimiter); let translator = Translator::new().escape(self.config.escape); let content = read_to_string(input) .chain_err(|| format!("Failed to open template file: {:?}", input))?; let stream = parser.parse(&content); translator.translate(stream) } pub fn resolve_file( &self, input: &Path, ) -> Result<(TranslatedSource, CompilationReport), Error> { let include_handler = Arc::new(|child_file: &Path| -> Result<_, Error> { Ok(self.translate_file_contents(child_file)?.ast) }); let resolver = Resolver::new().include_handler(include_handler); let mut tsource = self.translate_file_contents(input)?; let mut report = CompilationReport { deps: Vec::new() }; let r = resolver.resolve(input, &mut tsource.ast)?; report.deps = r.deps; Ok((tsource, report)) } pub fn compile_file( &self, input: &Path, tsource: TranslatedSource, output: &Path, ) -> Result<(), Error> { let analyzer = Analyzer::new(); let optimizer = Optimizer::new().rm_whitespace(self.config.rm_whitespace); let compile_file = |mut tsource: TranslatedSource, output: &Path| -> Result<(), Error> { analyzer.analyze(&mut tsource.ast)?; optimizer.optimize(&mut tsource.ast); if let Some(parent) = output.parent() { fs::create_dir_all(parent) .chain_err(|| format!("Failed to save artifacts in {:?}", parent))?; } let string = tsource.ast.into_token_stream().to_string(); let mut f = fs::File::create(output) .chain_err(|| format!("Failed to create artifact: {:?}", output))?; writeln!(f, "// Template compiled from: {}", input.display()) .chain_err(|| format!("Failed to write artifact into {:?}", output))?; writeln!(f, "{}", rustfmt_block(&string).unwrap_or(string)) .chain_err(|| format!("Failed to write artifact into {:?}", output))?; drop(f); Ok(()) }; compile_file(tsource, output) .chain_err(|| "Failed to compile template.") .map_err(|mut e| { e.source = fs::read_to_string(input).ok(); e.source_file = Some(input.to_owned()); e }) } pub fn compile_str(&self, input: &str) -> Result { let dummy_path = Path::new(env!("CARGO_MANIFEST_DIR")); let include_handler = Arc::new(|_: &Path| -> Result { Err(make_error!( ErrorKind::AnalyzeError( "include! macro is not allowed in inline template".to_owned() ), source = input.to_owned() )) }); let parser = Parser::new().delimiter(self.config.delimiter); let translator = Translator::new().escape(self.config.escape); let resolver = Resolver::new().include_handler(include_handler); let optimizer = Optimizer::new().rm_whitespace(self.config.rm_whitespace); let compile = || -> Result { let stream = parser.parse(input); let mut tsource = translator.translate(stream)?; resolver.resolve(dummy_path, &mut tsource.ast)?; optimizer.optimize(&mut tsource.ast); Ok(tsource.ast.into_token_stream().to_string()) }; compile() .chain_err(|| "Failed to compile template.") .map_err(|mut e| { e.source = Some(input.to_owned()); e }) } } sailfish-compiler-0.9.0/src/config.rs000064400000000000000000000163431046102023000156550ustar 00000000000000use std::path::{Path, PathBuf}; #[derive(Clone, Debug, Hash)] pub struct Config { pub delimiter: char, pub escape: bool, pub rm_whitespace: bool, pub template_dirs: Vec, #[doc(hidden)] pub cache_dir: PathBuf, #[doc(hidden)] pub _non_exhaustive: (), } impl Default for Config { fn default() -> Self { Self { template_dirs: Vec::new(), delimiter: '%', escape: true, cache_dir: Path::new(env!("OUT_DIR")).join("cache"), rm_whitespace: false, _non_exhaustive: (), } } } #[cfg(feature = "config")] mod imp { use serde::Deserialize; use std::fs; use super::*; use crate::error::*; impl Config { pub fn search_file_and_read(base: &Path) -> Result { // search config file let mut path = PathBuf::new(); let mut config = Config::default(); for component in base.iter() { path.push(component); path.push("sailfish.toml"); if path.is_file() { let config_file = ConfigFile::read_from_file(&path).map_err(|mut e| { e.source_file = Some(path.to_owned()); e })?; if let Some(template_dirs) = config_file.template_dirs { for template_dir in template_dirs.into_iter().rev() { let expanded = expand_env_vars(template_dir).map_err(|mut e| { e.source_file = Some(path.to_owned()); e })?; let template_dir = PathBuf::from(expanded); if template_dir.is_absolute() { config.template_dirs.push(template_dir); } else { config .template_dirs .push(path.parent().unwrap().join(template_dir)); } } } if let Some(delimiter) = config_file.delimiter { config.delimiter = delimiter; } if let Some(escape) = config_file.escape { config.escape = escape; } if let Some(optimizations) = config_file.optimizations { if let Some(rm_whitespace) = optimizations.rm_whitespace { config.rm_whitespace = rm_whitespace; } } } path.pop(); } Ok(config) } } #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] struct Optimizations { rm_whitespace: Option, } #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] struct ConfigFile { template_dirs: Option>, delimiter: Option, escape: Option, optimizations: Option, } impl ConfigFile { fn read_from_file(path: &Path) -> Result { let content = fs::read_to_string(path) .chain_err(|| format!("Failed to read configuration file {:?}", path))?; Self::from_string(&content) } fn from_string(content: &str) -> Result { toml::from_str::(content).map_err(|e| error(e.to_string())) } } fn expand_env_vars>(input: S) -> Result { use std::env; let input = input.as_ref(); let len = input.len(); let mut iter = input.chars().enumerate(); let mut result = String::new(); let mut found = false; let mut env_var = String::new(); while let Some((i, c)) = iter.next() { match c { '$' if !found => { if let Some((_, cc)) = iter.next() { if cc == '{' { found = true; } else { // We didn't find a trailing { after the $ // so we push the chars read onto the result result.push(c); result.push(cc); } } } '}' if found => { let val = env::var(&env_var).map_err(|e| match e { env::VarError::NotPresent => { error(format!("Environment variable ({}) not set", env_var)) } env::VarError::NotUnicode(_) => error(format!( "Environment variable ({}) contents not valid unicode", env_var )), })?; result.push_str(&val); env_var.clear(); found = false; } _ => { if found { env_var.push(c); // Check if we're at the end with an unclosed environment variable: // ${MYVAR instead of ${MYVAR} // If so, push it back onto the string as some systems allows the $ { characters in paths. if i == len - 1 { result.push_str("${"); result.push_str(&env_var); } } else { result.push(c); } } } } Ok(result) } fn error>(msg: T) -> Error { make_error!(ErrorKind::ConfigError(msg.into())) } #[cfg(test)] mod tests { use crate::config::imp::expand_env_vars; use std::env; #[test] fn expands_env_vars() { env::set_var("TESTVAR", "/a/path"); let input = "/path/to/${TESTVAR}Templates"; let output = expand_env_vars(input).unwrap(); assert_eq!(output, "/path/to//a/pathTemplates"); } #[test] fn retains_case_sensitivity() { env::set_var("tEstVar", "/a/path"); let input = "/path/${tEstVar}"; let output = expand_env_vars(input).unwrap(); assert_eq!(output, "/path//a/path"); } #[test] fn retains_unclosed_env_var() { let input = "/path/to/${UNCLOSED"; let output = expand_env_vars(input).unwrap(); assert_eq!(output, input); } #[test] fn ingores_markers() { let input = "path/{$/$}/${/to/{"; let output = expand_env_vars(input).unwrap(); assert_eq!(output, input); } #[test] fn errors_on_unset_env_var() { let input = "/path/to/${UNSET}"; let output = expand_env_vars(input); assert!(output.is_err()); } } } sailfish-compiler-0.9.0/src/error.rs000064400000000000000000000160601046102023000155350ustar 00000000000000use std::env; use std::fmt; use std::fs; use std::io; use std::path::{Path, PathBuf}; #[non_exhaustive] #[derive(Debug)] pub enum ErrorKind { FmtError(fmt::Error), IoError(io::Error), RustSyntaxError(syn::Error), ConfigError(String), ParseError(String), AnalyzeError(String), Unimplemented(String), Other(String), } impl fmt::Display for ErrorKind { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { ErrorKind::FmtError(ref e) => e.fmt(f), ErrorKind::IoError(ref e) => e.fmt(f), ErrorKind::RustSyntaxError(ref e) => write!(f, "Rust Syntax Error ({})", e), ErrorKind::ConfigError(ref e) => write!(f, "Invalid configuration ({})", e), ErrorKind::ParseError(ref msg) => write!(f, "Parse error ({})", msg), ErrorKind::AnalyzeError(ref msg) => write!(f, "Analyzation error ({})", msg), ErrorKind::Unimplemented(ref msg) => f.write_str(msg), ErrorKind::Other(ref msg) => f.write_str(msg), } } } macro_rules! impl_errorkind_conversion { ($source:ty, $kind:ident, $conv:expr, [ $($lifetimes:tt),* ]) => { impl<$($lifetimes),*> From<$source> for ErrorKind { #[inline] fn from(other: $source) -> Self { ErrorKind::$kind($conv(other)) } } }; ($source:ty, $kind:ident) => { impl_errorkind_conversion!($source, $kind, std::convert::identity, []); } } impl_errorkind_conversion!(fmt::Error, FmtError); impl_errorkind_conversion!(io::Error, IoError); impl_errorkind_conversion!(syn::Error, RustSyntaxError); impl_errorkind_conversion!(String, Other); impl_errorkind_conversion!(&'a str, Other, |s: &str| s.to_owned(), ['a]); #[derive(Debug, Default)] pub struct Error { pub(crate) source_file: Option, pub(crate) source: Option, pub(crate) offset: Option, pub(crate) chains: Vec, } impl Error { pub fn from_kind(kind: ErrorKind) -> Self { Self { chains: vec![kind], ..Self::default() } } pub fn kind(&self) -> &ErrorKind { self.chains.last().unwrap() } pub fn iter(&self) -> impl Iterator { self.chains.iter().rev() } } impl From for Error where ErrorKind: From, { fn from(other: T) -> Self { Self::from_kind(ErrorKind::from(other)) } } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let source = match (self.source.as_ref(), self.source_file.as_deref()) { (Some(s), _) => Some(s.to_owned()), (None, Some(f)) => fs::read_to_string(f).ok(), (None, None) => None, }; writeln!(f, "{}", self.chains.last().unwrap())?; for e in self.chains.iter().rev().skip(1) { writeln!(f, "caused by: {}", e)?; } f.write_str("\n")?; if let Some(ref source_file) = self.source_file { let source_file = if env::var("SAILFISH_INTEGRATION_TESTS").map_or(false, |s| s == "1") { match source_file.file_name() { Some(f) => Path::new(f), None => Path::new(""), } } else { source_file }; writeln!(f, "file: {}", source_file.display())?; } if let (Some(ref source), Some(offset)) = (source, self.offset) { let (lineno, colno) = into_line_column(source, offset); writeln!(f, "position: line {}, column {}\n", lineno, colno)?; // TODO: display adjacent lines let line = source.lines().nth(lineno - 1).unwrap(); let lpad = count_digits(lineno); writeln!(f, "{: { fn chain_err(self, kind: F) -> Result where F: FnOnce() -> EK, EK: Into; } impl ResultExt for Result { fn chain_err(self, kind: F) -> Result where F: FnOnce() -> EK, EK: Into, { self.map_err(|mut e| { e.chains.push(kind().into()); e }) } } impl> ResultExt for Result { fn chain_err(self, kind: F) -> Result where F: FnOnce() -> EK, EK: Into, { self.map_err(|e| { let mut e = Error::from(e.into()); e.chains.push(kind().into()); e }) } } fn into_line_column(source: &str, offset: usize) -> (usize, usize) { assert!( offset <= source.len(), "Internal error: error position offset overflow (error code: 56066)" ); let mut lineno = 1; let mut colno = 1; let mut current = 0; for line in source.lines() { let end = current + line.len() + 1; if offset < end { colno = offset - current + 1; break; } lineno += 1; current = end; } (lineno, colno) } fn count_digits(n: usize) -> usize { let mut current = 10; let mut digits = 1; while current <= n { current *= 10; digits += 1; } digits } macro_rules! make_error { ($kind:expr) => { $crate::Error::from_kind($kind) }; ($kind:expr, $($remain:tt)*) => {{ #[allow(unused_mut)] let mut err = $crate::Error::from_kind($kind); make_error!(@opt err $($remain)*); err }}; (@opt $var:ident $key:ident = $value:expr, $($remain:tt)*) => { $var.$key = Some($value.into()); make_error!(@opt $var $($remain)*); }; (@opt $var:ident $key:ident = $value:expr) => { $var.$key = Some($value.into()); }; (@opt $var:ident $key:ident, $($remain:tt)*) => { $var.$key = Some($key); make_error!(@opt $var $($remain)*); }; (@opt $var:ident $key:ident) => { $var.$key = Some($key); }; (@opt $var:ident) => {}; } #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; #[test] fn display_error() { let mut err = make_error!( ErrorKind::AnalyzeError("mismatched types".to_owned()), source_file = PathBuf::from("apple.rs"), source = "fn func() {\n 1\n}".to_owned(), offset = 16usize ); err.chains.push(ErrorKind::Other("some error".to_owned())); assert!(matches!(err.kind(), &ErrorKind::Other(_))); assert_eq!( err.to_string(), r#"some error caused by: Analyzation error (mismatched types) file: apple.rs position: line 2, column 5 | 2 | 1 | ^ "# ); } } sailfish-compiler-0.9.0/src/lib.rs000064400000000000000000000004661046102023000151550ustar 00000000000000#![forbid(unsafe_code)] #[macro_use] mod error; mod analyzer; mod compiler; mod config; mod optimizer; mod parser; mod resolver; mod translator; mod util; pub use compiler::Compiler; pub use config::Config; pub use error::{Error, ErrorKind}; #[cfg(feature = "procmacro")] #[doc(hidden)] pub mod procmacro; sailfish-compiler-0.9.0/src/optimizer.rs000064400000000000000000000146331046102023000164320ustar 00000000000000use proc_macro2::TokenStream; use quote::quote; use syn::parse::{Parse, ParseStream, Result as ParseResult}; use syn::visit_mut::VisitMut; use syn::{ Block, Expr, ExprBreak, ExprContinue, ExprMacro, Ident, LitStr, Macro, Stmt, StmtMacro, Token, }; pub struct Optimizer { rm_whitespace: bool, } impl Optimizer { #[inline] pub fn new() -> Self { Self { rm_whitespace: false, } } #[inline] pub fn rm_whitespace(mut self, new: bool) -> Self { self.rm_whitespace = new; self } #[inline] pub fn optimize(&self, i: &mut Block) { OptmizerImpl { rm_whitespace: self.rm_whitespace, } .visit_block_mut(i); } } struct OptmizerImpl { rm_whitespace: bool, } impl VisitMut for OptmizerImpl { fn visit_block_mut(&mut self, i: &mut Block) { let mut results = Vec::with_capacity(i.stmts.len()); for mut stmt in i.stmts.drain(..) { // process whole statement in advance syn::visit_mut::visit_stmt_mut(self, &mut stmt); // check if statement is for loop let fl = match stmt { Stmt::Expr(Expr::ForLoop(ref mut fl), _) => fl, _ => { results.push(stmt); continue; } }; // check if for loop contains 2 or more statements if fl.body.stmts.len() <= 1 { results.push(stmt); continue; } // check if for loop contains continue or break statement if block_has_continue_or_break(&mut fl.body) { results.push(stmt); continue; } // check if first and last statement inside for loop is render_text! macro let (sf, sl) = match ( fl.body .stmts .first() .and_then(get_rendertext_value_from_stmt), fl.body .stmts .last() .and_then(get_rendertext_value_from_stmt), ) { (Some(sf), Some(sl)) => (sf, sl), _ => { results.push(stmt); continue; } }; let sf_len = sf.len(); // optimize for loop contents let mut concat = sl; concat += sf.as_str(); let mut previous; if let Some(prev) = results.last().and_then(get_rendertext_value_from_stmt) { results.pop(); previous = prev; previous += sf.as_str(); } else { previous = sf; } fl.body.stmts.remove(0); *fl.body.stmts.last_mut().unwrap() = syn::parse2(quote! { __sf_rt::render_text!(__sf_buf, #concat); }) .unwrap(); let mut new_stmts = syn::parse2::(quote! {{ __sf_rt::render_text!(__sf_buf, #previous); #stmt unsafe { __sf_buf._set_len(__sf_buf.len() - #sf_len); } }}) .unwrap(); results.append(&mut new_stmts.stmts) } i.stmts = results; } fn visit_stmt_macro_mut(&mut self, i: &mut StmtMacro) { if self.rm_whitespace { if let Some(v) = get_rendertext_value(&i.mac) { let ts = match remove_whitespace(v) { Some(value) => value, None => return, }; i.mac.tokens = ts; return; } } syn::visit_mut::visit_stmt_macro_mut(self, i); } fn visit_expr_macro_mut(&mut self, i: &mut ExprMacro) { if self.rm_whitespace { if let Some(v) = get_rendertext_value(&i.mac) { let ts = match remove_whitespace(v) { Some(value) => value, None => return, }; i.mac.tokens = ts; return; } } syn::visit_mut::visit_expr_macro_mut(self, i); } } fn remove_whitespace(v: String) -> Option { let mut buffer = String::new(); let mut it = v.lines().peekable(); if let Some(line) = it.next() { if it.peek().is_some() { buffer.push_str(line.trim_end()); buffer.push('\n'); } else { return None; } } while let Some(line) = it.next() { if it.peek().is_some() { if !line.is_empty() { buffer.push_str(line.trim()); buffer.push('\n'); } else { // ignore empty line } } else { // last line buffer.push_str(line.trim_start()); } } Some(quote! { __sf_buf, #buffer }) } fn get_rendertext_value(mac: &Macro) -> Option { struct RenderTextMacroArgument { #[allow(dead_code)] context: Ident, arg: LitStr, } impl Parse for RenderTextMacroArgument { fn parse(s: ParseStream) -> ParseResult { let context = s.parse()?; s.parse::()?; let arg = s.parse()?; Ok(Self { context, arg }) } } let mut it = mac.path.segments.iter(); if it.next().map_or(false, |s| s.ident == "__sf_rt") && it.next().map_or(false, |s| s.ident == "render_text") && it.next().is_none() { let tokens = mac.tokens.clone(); if let Ok(macro_arg) = syn::parse2::(tokens) { return Some(macro_arg.arg.value()); } } None } fn get_rendertext_value_from_stmt(stmt: &Stmt) -> Option { let em = match stmt { Stmt::Expr(Expr::Macro(ref mac), Some(_)) => mac, _ => return None, }; get_rendertext_value(&em.mac) } fn block_has_continue_or_break(i: &mut Block) -> bool { #[derive(Default)] struct ContinueBreakFinder { found: bool, } impl VisitMut for ContinueBreakFinder { fn visit_expr_continue_mut(&mut self, _: &mut ExprContinue) { self.found = true; } fn visit_expr_break_mut(&mut self, _: &mut ExprBreak) { self.found = true; } } let mut finder = ContinueBreakFinder::default(); finder.visit_block_mut(i); finder.found } sailfish-compiler-0.9.0/src/parser.rs000064400000000000000000000325241046102023000157030ustar 00000000000000// TODO: Better error message (unbalanced rust delimiter, etc.) // TODO: disallow '<%' token inside code block use memchr::{memchr, memchr2, memchr3}; use std::convert::TryInto; use std::rc::Rc; use crate::{Error, ErrorKind}; macro_rules! unwrap_or_break { ($val:expr) => { match $val { Some(t) => t, None => break, } }; } #[derive(Clone, Debug)] pub struct Parser { delimiter: char, } impl Parser { pub fn new() -> Self { Self::default() } /// change delimiter pub fn delimiter(mut self, new: char) -> Self { self.delimiter = new; self } /// parse source string pub fn parse<'a>(&self, source: &'a str) -> ParseStream<'a> { let block_delimiter = Rc::new(( format!("<{}", self.delimiter), format!("{}>", self.delimiter), )); ParseStream { block_delimiter, original_source: source, source, delimiter: self.delimiter, } } } impl Default for Parser { fn default() -> Self { Self { delimiter: '%' } } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TokenKind { NestedTemplateOnce, BufferedCode { escape: bool }, Code, Comment, Text, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct Token<'a> { content: &'a str, offset: usize, kind: TokenKind, } impl<'a> Token<'a> { #[inline] pub fn new(content: &'a str, offset: usize, kind: TokenKind) -> Token<'a> { Token { content, offset, kind, } } #[inline] pub fn as_str(&self) -> &'a str { self.content } #[inline] pub fn offset(&self) -> usize { self.offset } #[inline] pub fn kind(&self) -> TokenKind { self.kind } } #[derive(Clone, Debug)] pub struct ParseStream<'a> { block_delimiter: Rc<(String, String)>, pub(crate) original_source: &'a str, source: &'a str, delimiter: char, } impl<'a> ParseStream<'a> { // /// Returns an empty `ParseStream` containing no tokens // pub fn new() -> Self { // Self::default() // } // // pub fn is_empty(&self) -> bool { // self.source.is_empty() // } pub fn into_vec(self) -> Result>, Error> { let mut vec = Vec::new(); for token in self { vec.push(token?); } Ok(vec) } fn error(&self, msg: &str) -> Error { let offset = self.original_source.len() - self.source.len(); make_error!( ErrorKind::ParseError(msg.to_owned()), source = self.original_source.to_owned(), offset ) } fn offset(&self) -> usize { self.original_source.len() - self.source.len() } fn take_n(&mut self, n: usize) -> &'a str { let (l, r) = self.source.split_at(n); self.source = r; l } fn tokenize_code(&mut self) -> Result, Error> { debug_assert!(self.source.starts_with(&*self.block_delimiter.0)); let mut start = self.block_delimiter.0.len(); let mut token_kind = TokenKind::Code; // read flags match self.source.as_bytes().get(start).copied() { Some(b'#') => { token_kind = TokenKind::Comment; start += 1; } Some(b'=') => { token_kind = TokenKind::BufferedCode { escape: true }; start += 1; } Some(b'-') => { token_kind = TokenKind::BufferedCode { escape: false }; start += 1; } Some(b'+') => { token_kind = TokenKind::NestedTemplateOnce; start += 1; } _ => {} } // skip whitespaces for ch in self.source.bytes().skip(start) { match ch { b' ' | b'\t' | b'\n'..=b'\r' => { start += 1; } _ => break, } } if token_kind == TokenKind::Comment { let pos = self.source[start..] .find(&*self.block_delimiter.1) .ok_or_else(|| self.error("Unterminated comment block"))?; self.take_n(start); let token = Token { content: self.source[..pos].trim_end(), offset: self.offset(), kind: token_kind, }; self.take_n(pos + self.block_delimiter.1.len()); return Ok(token); } // find closing bracket if let Some(pos) = find_block_end(&self.source[start..], &self.block_delimiter.1) { // closing bracket was found self.take_n(start); let s = &self.source[..pos - self.block_delimiter.1.len()].trim_end_matches( |c| matches!(c, ' ' | '\t' | '\r' | '\u{000B}' | '\u{000C}'), ); let token = Token { content: s, offset: self.offset(), kind: token_kind, }; self.take_n(pos); Ok(token) } else { Err(self.error("Unterminated code block")) } } fn tokenize_text(&mut self) -> Result, Error> { // TODO: allow buffer block inside code block let offset = self.offset(); let end = self .source .find(&*self.block_delimiter.0) .unwrap_or(self.source.len()); let token = Token { content: self.take_n(end), offset, kind: TokenKind::Text, }; Ok(token) } } impl<'a> Default for ParseStream<'a> { fn default() -> Self { Self { block_delimiter: Rc::new(("<%".to_owned(), "%>".to_owned())), original_source: "", source: "", delimiter: '%', } } } impl<'a> Iterator for ParseStream<'a> { type Item = Result, Error>; fn next(&mut self) -> Option { if self.source.is_empty() { return None; } let token = if self.source.starts_with(&*self.block_delimiter.0) { if !self.source[self.block_delimiter.0.len()..].starts_with(self.delimiter) { self.tokenize_code() } else { debug_assert_eq!( &self.source[..self.delimiter.len_utf8() * 2 + 1], format!("<{0}{0}", self.delimiter) ); // Escape '<%%' token let token = Token { content: &self.source[..self.block_delimiter.0.len()], offset: self.offset(), kind: TokenKind::Text, }; self.take_n(self.block_delimiter.0.len() * 2 - 1); Ok(token) } } else { self.tokenize_text() }; Some(token) } } impl<'a> TryInto>> for ParseStream<'a> { type Error = crate::Error; fn try_into(self) -> Result>, Error> { self.into_vec() } } fn find_block_end(haystack: &str, delimiter: &str) -> Option { let mut remain = haystack; 'outer: while let Some(pos) = memchr3(b'/', b'\"', delimiter.as_bytes()[0], remain.as_bytes()) { let skip_num = match remain.as_bytes()[pos] { b'/' => match remain.as_bytes().get(pos + 1).copied() { Some(b'/') => unwrap_or_break!(find_comment_end(&remain[pos..])), Some(b'*') => unwrap_or_break!(find_block_comment_end(&remain[pos..])), _ => 1, }, b'\"' => { // check if the literal is a raw string for (i, byte) in remain[..pos].as_bytes().iter().enumerate().rev() { match byte { b'#' => {} b'r' => { let skip_num = unwrap_or_break!(find_raw_string_end(&remain[i..])); remain = &remain[i + skip_num..]; continue 'outer; } _ => break, } } unwrap_or_break!(find_string_end(&remain[pos..])) } _ => { if remain[pos..].starts_with(delimiter) { return Some(haystack.len() - remain.len() + pos + delimiter.len()); } else { 1 } } }; remain = &remain[pos + skip_num..]; } None } fn find_comment_end(haystack: &str) -> Option { debug_assert!(haystack.starts_with("//")); memchr(b'\n', haystack.as_bytes()).map(|p| p + 1) } fn find_block_comment_end(haystack: &str) -> Option { debug_assert!(haystack.starts_with("/*")); let mut remain = &haystack[2..]; let mut depth = 1; while let Some(p) = memchr2(b'*', b'/', remain.as_bytes()) { let c = remain.as_bytes()[p]; let next = remain.as_bytes().get(p + 1); match (c, next) { (b'*', Some(b'/')) => { if depth == 1 { let offset = haystack.len() - (remain.len() - (p + 2)); return Some(offset); } depth -= 1; remain = &remain[p + 2..]; } (b'/', Some(b'*')) => { depth += 1; remain = &remain[p + 2..]; } _ => { remain = &remain[p + 1..]; } } } None } fn find_string_end(haystack: &str) -> Option { debug_assert!(haystack.starts_with('\"')); let mut bytes = &haystack.as_bytes()[1..]; while let Some(p) = memchr2(b'"', b'\\', bytes) { if bytes[p] == b'\"' { // string terminator found return Some(haystack.len() - (bytes.len() - p) + 1); } else if p + 2 < bytes.len() { // skip escape bytes = &bytes[p + 2..]; } else { break; } } None } fn find_raw_string_end(haystack: &str) -> Option { debug_assert!(haystack.starts_with('r')); let mut terminator = String::from("\""); for ch in haystack[1..].bytes() { match ch { b'#' => terminator.push('#'), b'"' => break, _ => { // is not a raw string literal return Some(1); } } } haystack[terminator.len() + 1..] .find(&terminator) .map(|p| p + terminator.len() * 2 + 1) } #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; #[test] fn nested_render_once() { let src = r#"outer <%+ inner|upper %> outer"#; let parser = Parser::default(); let tokens = parser.parse(src).into_vec().unwrap(); assert_eq!( &tokens, &[ Token { content: "outer ", offset: 0, kind: TokenKind::Text, }, Token { content: "inner|upper", offset: 10, kind: TokenKind::NestedTemplateOnce, }, Token { content: " outer", offset: 24, kind: TokenKind::Text, }, ] ); } #[test] fn non_ascii_delimiter() { let src = r##"foo <🍣# This is a comment 🍣> bar <🍣= r"🍣>" 🍣> baz <🍣🍣"##; let parser = Parser::new().delimiter('🍣'); let tokens = parser.parse(src).into_vec().unwrap(); assert_eq!( &tokens, &[ Token { content: "foo ", offset: 0, kind: TokenKind::Text }, Token { content: "This is a comment", offset: 11, kind: TokenKind::Comment }, Token { content: " bar ", offset: 34, kind: TokenKind::Text }, Token { content: "r\"🍣>\"", offset: 46, kind: TokenKind::BufferedCode { escape: true } }, Token { content: " baz ", offset: 60, kind: TokenKind::Text }, Token { content: "<🍣", offset: 65, kind: TokenKind::Text }, ] ); } #[test] fn comment_inside_block() { let src = "<% // %>\n %><%= /* %%>*/ 1 %>"; let parser = Parser::new(); let tokens = parser.parse(src).into_vec().unwrap(); assert_eq!( &tokens, &[ Token { content: "// %>\n", offset: 3, kind: TokenKind::Code }, Token { content: "/* %%>*/ 1", offset: 16, kind: TokenKind::BufferedCode { escape: true } }, ] ); } } sailfish-compiler-0.9.0/src/procmacro.rs000064400000000000000000000467421046102023000164030ustar 00000000000000use proc_macro2::{Span, TokenStream}; use quote::{quote, TokenStreamExt}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::io::Write; use std::iter; use std::path::{Path, PathBuf}; use std::time::Duration; use std::{env, thread}; use syn::parse::{ParseStream, Parser, Result as ParseResult}; use syn::punctuated::Punctuated; use syn::{Fields, Ident, ItemStruct, LitBool, LitChar, LitStr, Token}; use crate::compiler::Compiler; use crate::config::Config; use crate::error::*; use crate::util::filetime; // options for `template` attributes #[derive(Default)] struct DeriveTemplateOptions { found_keys: Vec, path: Option, delimiter: Option, escape: Option, rm_whitespace: Option, } impl DeriveTemplateOptions { fn parser(&mut self) -> impl Parser + '_ { move |s: ParseStream| -> ParseResult<()> { while !s.is_empty() { let key = s.parse::()?; s.parse::()?; // check if argument is repeated if self.found_keys.iter().any(|e| *e == key) { return Err(syn::Error::new( key.span(), format!("Argument `{}` was repeated.", key), )); } if key == "path" { self.path = Some(s.parse::()?); } else if key == "delimiter" { self.delimiter = Some(s.parse::()?); } else if key == "escape" { self.escape = Some(s.parse::()?); } else if key == "rm_whitespace" { self.rm_whitespace = Some(s.parse::()?); } else { return Err(syn::Error::new( key.span(), format!("Unknown option: `{}`", key), )); } self.found_keys.push(key); // consume comma token if s.is_empty() { break; } else { s.parse::()?; } } Ok(()) } } } fn merge_config_options(config: &mut Config, options: &DeriveTemplateOptions) { if let Some(ref delimiter) = options.delimiter { config.delimiter = delimiter.value(); } if let Some(ref escape) = options.escape { config.escape = escape.value; } if let Some(ref rm_whitespace) = options.rm_whitespace { config.rm_whitespace = rm_whitespace.value; } } fn resolve_template_file(path: &str, template_dirs: &[PathBuf]) -> Option { for template_dir in template_dirs.iter().rev() { let p = template_dir.join(path); if p.is_file() { return Some(p); } } let mut fallback = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect( "Internal error: environmental variable `CARGO_MANIFEST_DIR` is not set.", )); fallback.push("templates"); fallback.push(path); if fallback.is_file() { return Some(fallback); } None } fn filename_hash(path: &Path, config: &Config) -> String { let mut hasher = DefaultHasher::new(); config.hash(&mut hasher); let config_hash = hasher.finish(); path.hash(&mut hasher); let path_hash = hasher.finish(); format!("{:016x}-{:016x}", config_hash, path_hash) } fn with_compiler Result>( config: Config, apply: F, ) -> Result { struct FallbackScope {} impl FallbackScope { fn new() -> Self { // SAFETY: // Any token or span constructed after `proc_macro2::fallback::force()` must // not outlive after `unforce()` because it causes span mismatch error. In // this case, we must ensure that `compile_file` does not return any token or // span. proc_macro2::fallback::force(); FallbackScope {} } } impl Drop for FallbackScope { fn drop(&mut self) { proc_macro2::fallback::unforce(); } } let compiler = Compiler::with_config(config); let _scope = FallbackScope::new(); apply(compiler) } fn derive_template_common_impl( tokens: TokenStream, ) -> Result<(ItemStruct, TokenStream, String), syn::Error> { let strct = syn::parse2::(tokens)?; let mut all_options = DeriveTemplateOptions::default(); for attr in &strct.attrs { if attr.path().is_ident("template") { attr.parse_args_with(all_options.parser())?; } } #[cfg(feature = "config")] let mut config = { let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect( "Internal error: environmental variable `CARGO_MANIFEST_DIR` is not set.", )); Config::search_file_and_read(&manifest_dir) .map_err(|e| syn::Error::new(Span::call_site(), e))? }; #[cfg(not(feature = "config"))] let mut config = Config::default(); if env::var("SAILFISH_INTEGRATION_TESTS").map_or(false, |s| s == "1") { let template_dir = env::current_dir() .unwrap() .ancestors() .find(|p| p.join("LICENSE").exists()) .unwrap() .join("sailfish-tests") .join("integration-tests") .join("tests") .join("fails") .join("templates"); config.template_dirs.push(template_dir); } let input_file = { let path = all_options.path.as_ref().ok_or_else(|| { syn::Error::new(Span::call_site(), "`path` option must be specified.") })?; resolve_template_file(&path.value(), &config.template_dirs) .and_then(|path| path.canonicalize().ok()) .ok_or_else(|| { syn::Error::new( path.span(), format!("Template file {:?} not found", path.value()), ) })? }; merge_config_options(&mut config, &all_options); // Template compilation through this proc-macro uses a caching mechanism. Output file // names include a hash calculated from input file contents and compiler // configuration. This way, existing files never need updating and can simply be // re-used if they exist. let mut output_file = PathBuf::from(env!("OUT_DIR")); output_file.push("templates"); output_file.push(filename_hash(&input_file, &config)); std::fs::create_dir_all(output_file.parent().unwrap()).unwrap(); // This makes sure max 1 process creates a new file, "create_new" check+create is an // atomic operation. Cargo sometimes runs multiple macro invocations for the same // file in parallel, so that's important to prevent a race condition. struct Lock<'path> { path: &'path Path, } impl<'path> Lock<'path> { fn new(path: &'path Path) -> std::io::Result { std::fs::OpenOptions::new() .write(true) .create_new(true) .open(path) .map(|_| Lock { path }) } } impl<'path> Drop for Lock<'path> { fn drop(&mut self) { std::fs::remove_file(self.path) .expect("Failed to clean up lock file {}. Delete it manually, or run `cargo clean`."); } } let deps = with_compiler(config, |compiler| { let dep_path = output_file.with_extension("deps"); let lock_path = output_file.with_extension("lock"); let lock = Lock::new(&lock_path); match lock { Ok(lock) => { let (tsource, report) = compiler.resolve_file(&input_file)?; let output_filetime = filetime(&output_file); let input_filetime = iter::once(&input_file) .chain(&report.deps) .map(|path| filetime(path)) .max() .expect("Iterator contains at least `input_file`"); // Recompile template if any included templates were changed // since the last time we compiled. if input_filetime > output_filetime { compiler.compile_file(&input_file, tsource, &output_file)?; // Write access to `dep_path` is serialized by `lock`. let mut dep_file = std::fs::OpenOptions::new() .write(true) .create(true) .truncate(true) .open(&dep_path) .unwrap_or_else(|e| { panic!("Failed to open {:?}: {}", dep_path, e) }); // Write out dependencies for concurrent processes to reuse. for dep in &report.deps { writeln!(&mut dep_file, "{}", dep.to_str().unwrap()).unwrap(); } // Prevent output file from being tracked by Cargo. Without this hack, // every change to a template causes two recompilations: // // 1. Change a template at timestamp t. // 2. Cargo detects template change due to `include_bytes!` macro below. // 3. Sailfish compiler generates an output file with a later timestamp t'. // 4. Build finishes with timestamp t. // 5. Next cargo build detects output file with timestamp t' > t and rebuilds. // 6. Sailfish compiler does not regenerate output due to timestamp logic above. // 7. Build finishes with timestamp t'. let _ = filetime::set_file_times( &output_file, input_filetime, input_filetime, ); } drop(lock); Ok(report.deps) } // Lock file exists, template is already (currently being?) compiled. Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { let mut load_attempts = 0; while lock_path.exists() { load_attempts += 1; if load_attempts > 100 { panic!("Lock file {:?} is stuck. Try deleting it.", lock_path); } thread::sleep(Duration::from_millis(10)); } Ok(std::fs::read_to_string(&dep_path) .unwrap() .trim() .lines() .map(PathBuf::from) .collect()) } Err(e) => panic!("{:?}: {}. Maybe try `cargo clean`?", lock_path, e), } }) .map_err(|e| syn::Error::new(Span::call_site(), e))?; let input_file_string = input_file .to_str() .unwrap_or_else(|| panic!("Non UTF-8 file name: {:?}", input_file)); let output_file_string = output_file .to_str() .unwrap_or_else(|| panic!("Non UTF-8 file name: {:?}", output_file)); let mut include_bytes_seq = quote! { include_bytes!(#input_file_string); }; for dep in deps { if let Some(dep_string) = dep.to_str() { include_bytes_seq.extend(quote! { include_bytes!(#dep_string); }); } } Ok((strct, include_bytes_seq, output_file_string.to_string())) } fn derive_template_once_only_impl( strct: &ItemStruct, include_bytes_seq: &TokenStream, output_file_string: &String, ) -> TokenStream { let name = &strct.ident; let (impl_generics, ty_generics, where_clause) = strct.generics.split_for_impl(); // render_once method always results in the same code. // This method can be implemented in `sailfish` crate, but I found that performance // drops when the implementation is written in `sailfish` crate. quote! { impl #impl_generics sailfish::TemplateOnce for #name #ty_generics #where_clause { fn render_once(mut self) -> sailfish::RenderResult { use sailfish::runtime::{Buffer, SizeHint}; static SIZE_HINT: SizeHint = SizeHint::new(); let mut buf = Buffer::with_capacity(SIZE_HINT.get()); self.render_once_to(&mut buf)?; SIZE_HINT.update(buf.len()); Ok(buf.into_string()) } fn render_once_to(mut self, __sf_buf: &mut sailfish::runtime::Buffer) -> std::result::Result<(), sailfish::runtime::RenderError> { // This line is required for cargo to track child templates #include_bytes_seq; use sailfish::runtime as __sf_rt; include!(#output_file_string); Ok(()) } } } } fn derive_template_mut_only_impl( strct: &ItemStruct, include_bytes_seq: &TokenStream, output_file_string: &String, ) -> TokenStream { let name = &strct.ident; let (impl_generics, ty_generics, where_clause) = strct.generics.split_for_impl(); // This method can be implemented in `sailfish` crate, but I found that performance // drops when the implementation is written in `sailfish` crate. quote! { impl #impl_generics sailfish::TemplateMut for #name #ty_generics #where_clause { fn render_mut(&mut self) -> sailfish::RenderResult { use sailfish::runtime::{Buffer, SizeHint}; static SIZE_HINT: SizeHint = SizeHint::new(); let mut buf = Buffer::with_capacity(SIZE_HINT.get()); self.render_mut_to(&mut buf)?; SIZE_HINT.update(buf.len()); Ok(buf.into_string()) } fn render_mut_to(&mut self, __sf_buf: &mut sailfish::runtime::Buffer) -> std::result::Result<(), sailfish::runtime::RenderError> { // This line is required for cargo to track child templates #include_bytes_seq; use sailfish::runtime as __sf_rt; include!(#output_file_string); Ok(()) } } } } fn derive_template_only_impl( strct: &ItemStruct, include_bytes_seq: &TokenStream, output_file_string: &String, ) -> TokenStream { let name = &strct.ident; let (impl_generics, ty_generics, where_clause) = strct.generics.split_for_impl(); // This method can be implemented in `sailfish` crate, but I found that performance // drops when the implementation is written in `sailfish` crate. quote! { impl #impl_generics sailfish::Template for #name #ty_generics #where_clause { fn render(&self) -> sailfish::RenderResult { use sailfish::runtime::{Buffer, SizeHint}; static SIZE_HINT: SizeHint = SizeHint::new(); let mut buf = Buffer::with_capacity(SIZE_HINT.get()); self.render_to(&mut buf)?; SIZE_HINT.update(buf.len()); Ok(buf.into_string()) } fn render_to(&self, __sf_buf: &mut sailfish::runtime::Buffer) -> std::result::Result<(), sailfish::runtime::RenderError> { // This line is required for cargo to track child templates #include_bytes_seq; use sailfish::runtime as __sf_rt; include!(#output_file_string); Ok(()) } } } } fn derive_template_once_impl(tokens: TokenStream) -> Result { let (strct, include_bytes_seq, output_file_string) = derive_template_common_impl(tokens)?; let mut output = TokenStream::new(); output.append_all(derive_template_once_only_impl( &strct, &include_bytes_seq, &output_file_string, )); Ok(output) } fn derive_template_mut_impl(tokens: TokenStream) -> Result { let (strct, include_bytes_seq, output_file_string) = derive_template_common_impl(tokens)?; let mut output = TokenStream::new(); output.append_all(derive_template_once_only_impl( &strct, &include_bytes_seq, &output_file_string, )); output.append_all(derive_template_mut_only_impl( &strct, &include_bytes_seq, &output_file_string, )); Ok(output) } fn derive_template_impl(tokens: TokenStream) -> Result { let (strct, include_bytes_seq, output_file_string) = derive_template_common_impl(tokens)?; let mut output = TokenStream::new(); output.append_all(derive_template_once_only_impl( &strct, &include_bytes_seq, &output_file_string, )); output.append_all(derive_template_mut_only_impl( &strct, &include_bytes_seq, &output_file_string, )); output.append_all(derive_template_only_impl( &strct, &include_bytes_seq, &output_file_string, )); Ok(output) } fn derive_template_simple_impl(tokens: TokenStream) -> Result { let (strct, include_bytes_seq, output_file_string) = derive_template_common_impl(tokens)?; let name = &strct.ident; let field_names: Punctuated = match strct.fields { Fields::Named(fields) => fields .named .into_iter() .map(|f| { f.ident.expect( "Internal error: Failed to get field name (error code: 73621)", ) }) .collect(), Fields::Unit => Punctuated::new(), _ => { return Err(syn::Error::new( Span::call_site(), "You cannot derive `TemplateSimple` for tuple struct", )); } }; let (impl_generics, ty_generics, where_clause) = strct.generics.split_for_impl(); // render_once method always results in the same code. // This method can be implemented in `sailfish` crate, but I found that performance // drops when the implementation is written in `sailfish` crate. Ok(quote! { impl #impl_generics sailfish::TemplateSimple for #name #ty_generics #where_clause { fn render_once(self) -> sailfish::RenderResult { use sailfish::runtime::{Buffer, SizeHint}; static SIZE_HINT: SizeHint = SizeHint::new(); let mut buf = Buffer::with_capacity(SIZE_HINT.get()); self.render_once_to(&mut buf)?; SIZE_HINT.update(buf.len()); Ok(buf.into_string()) } fn render_once_to(self, __sf_buf: &mut sailfish::runtime::Buffer) -> std::result::Result<(), sailfish::runtime::RenderError> { // This line is required for cargo to track child templates #include_bytes_seq; use sailfish::runtime as __sf_rt; let #name { #field_names } = self; include!(#output_file_string); Ok(()) } } }) } pub fn derive_template_once(tokens: TokenStream) -> TokenStream { derive_template_once_impl(tokens).unwrap_or_else(|e| e.to_compile_error()) } pub fn derive_template_mut(tokens: TokenStream) -> TokenStream { derive_template_mut_impl(tokens).unwrap_or_else(|e| e.to_compile_error()) } pub fn derive_template(tokens: TokenStream) -> TokenStream { derive_template_impl(tokens).unwrap_or_else(|e| e.to_compile_error()) } pub fn derive_template_simple(tokens: TokenStream) -> TokenStream { derive_template_simple_impl(tokens).unwrap_or_else(|e| e.to_compile_error()) } sailfish-compiler-0.9.0/src/resolver.rs000064400000000000000000000120251046102023000162420ustar 00000000000000use quote::quote; use std::path::{Path, PathBuf}; use std::sync::Arc; use syn::visit_mut::VisitMut; use syn::{Block, Expr, ExprBlock, LitStr, Macro, Stmt}; use crate::error::*; macro_rules! matches_or_else { ($val:expr, $p:pat, $ok:expr, $else:expr) => { match $val { $p => $ok, _ => $else, } }; } macro_rules! return_if_some { ($val:expr) => { if $val.is_some() { return; } }; } pub type IncludeHandler<'h> = Arc Result>; #[derive(Clone)] pub struct Resolver<'h> { include_handler: IncludeHandler<'h>, } impl<'h> Resolver<'h> { pub fn new() -> Self { Self { include_handler: Arc::new(|_| { Err(make_error!(ErrorKind::AnalyzeError( "You cannot use `include` macro inside templates".to_owned() ))) }), } } #[inline] pub fn include_handler(mut self, new: IncludeHandler<'h>) -> Resolver<'h> { self.include_handler = new; self } #[inline] pub fn resolve( &self, input_file: &Path, ast: &mut Block, ) -> Result { let mut child = ResolverImpl { path_stack: vec![input_file.to_owned()], deps: Vec::new(), error: None, include_handler: Arc::clone(&self.include_handler), }; child.visit_block_mut(ast); if let Some(e) = child.error { Err(e) } else { Ok(ResolveReport { deps: child.deps }) } } } pub struct ResolveReport { pub deps: Vec, } struct ResolverImpl<'h> { path_stack: Vec, deps: Vec, error: Option, include_handler: IncludeHandler<'h>, } impl<'h> ResolverImpl<'h> { fn resolve_include(&mut self, mac: &Macro) -> Result { let arg = match syn::parse2::(mac.tokens.clone()) { Ok(l) => l.value(), Err(e) => { let mut e = Error::from(e); e.chains.push(ErrorKind::AnalyzeError( "invalid arguments for `include` macro".to_owned(), )); return Err(e); } }; // resolve include! for rust file if arg.ends_with(".rs") { let absolute_path = if Path::new(&*arg).is_absolute() { PathBuf::from(&arg[1..]) } else { self.path_stack.last().unwrap().parent().unwrap().join(arg) }; return if let Some(absolute_path_str) = absolute_path.to_str() { Ok(syn::parse2(quote! { include!(#absolute_path_str) }).unwrap()) } else { let msg = format!( "cannot include path with non UTF-8 character: {:?}", absolute_path ); Err(make_error!(ErrorKind::AnalyzeError(msg))) }; } // resolve the template file path // TODO: How should arguments be interpreted on Windows? let child_template_file = if Path::new(&*arg).is_absolute() { // absolute imclude PathBuf::from(&arg[1..]) } else { // relative include self.path_stack.last().unwrap().parent().unwrap().join(arg) }; // parse and translate the child template let mut blk = (*self.include_handler)(&child_template_file).chain_err(|| { format!("Failed to include {:?}", child_template_file.clone()) })?; self.path_stack.push(child_template_file); syn::visit_mut::visit_block_mut(self, &mut blk); let child_template_file = self.path_stack.pop().unwrap(); if self.deps.iter().all(|p| p != &child_template_file) { self.deps.push(child_template_file); } Ok(Expr::Block(ExprBlock { attrs: Vec::new(), label: None, block: blk, })) } } impl<'h> VisitMut for ResolverImpl<'h> { fn visit_stmt_mut(&mut self, i: &mut Stmt) { return_if_some!(self.error); let sm = matches_or_else!(*i, Stmt::Macro(ref mut sm), sm, { syn::visit_mut::visit_stmt_mut(self, i); return; }); if sm.mac.path.is_ident("include") { match self.resolve_include(&sm.mac) { Ok(e) => *i = Stmt::Expr(e, None), Err(e) => { self.error = Some(e); } } } } fn visit_expr_mut(&mut self, i: &mut Expr) { return_if_some!(self.error); let em = matches_or_else!(*i, Expr::Macro(ref mut em), em, { syn::visit_mut::visit_expr_mut(self, i); return; }); // resolve `include` if em.mac.path.is_ident("include") { match self.resolve_include(&em.mac) { Ok(e) => *i = e, Err(e) => { self.error = Some(e); } } } } } sailfish-compiler-0.9.0/src/translator.rs000064400000000000000000000320131046102023000165710ustar 00000000000000use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use syn::parse::{Parse, ParseStream as SynParseStream, Result as ParseResult}; use syn::{BinOp, Block, Expr}; use crate::error::*; use crate::parser::{ParseStream, Token, TokenKind}; // translate tokens into Rust code #[derive(Clone, Debug, Default)] pub struct Translator { escape: bool, } impl Translator { #[inline] pub fn new() -> Self { Self { escape: true } } #[inline] pub fn escape(mut self, new: bool) -> Self { self.escape = new; self } pub fn translate( &self, token_iter: ParseStream<'_>, ) -> Result { let original_source = token_iter.original_source; let mut ps = SourceBuilder::new(self.escape); ps.reserve(original_source.len()); ps.feed_tokens(token_iter)?; ps.finalize() } } pub struct TranslatedSource { pub ast: Block, pub source_map: SourceMap, } #[derive(Default)] pub struct SourceMap { entries: Vec, } #[derive(Clone)] pub struct SourceMapEntry { pub original: usize, pub new: usize, pub length: usize, } impl SourceMap { // #[inline] // pub fn entries(&self) -> &[SourceMapEntry] { // &*self.entries // } pub fn reverse_mapping(&self, offset: usize) -> Option { // find entry which satisfies entry.new <= offset < entry.new + entry.length let idx = self .entries .iter() .position(|entry| offset < entry.new + entry.length && entry.new <= offset)?; let entry = &self.entries[idx]; debug_assert!(entry.new <= offset); debug_assert!(offset < entry.new + entry.length); Some(entry.original + offset - entry.new) } } struct SourceBuilder { escape: bool, source: String, source_map: SourceMap, } impl SourceBuilder { fn new(escape: bool) -> SourceBuilder { SourceBuilder { escape, source: String::from("{\n"), source_map: SourceMap::default(), } } fn reserve(&mut self, additional: usize) { self.source.reserve(additional); } fn write_token(&mut self, token: &Token<'_>) { let entry = SourceMapEntry { original: token.offset(), new: self.source.len(), length: token.as_str().len(), }; self.source_map.entries.push(entry); self.source.push_str(token.as_str()); } fn write_code(&mut self, token: &Token<'_>) -> Result<(), Error> { // TODO: automatically add missing tokens (e.g. ';', '{') self.write_token(token); self.source.push('\n'); Ok(()) } fn write_text(&mut self, token: &Token<'_>) -> Result<(), Error> { use std::fmt::Write; // if error has occured at the first byte of `render_text!` macro, it // will be mapped to the first byte of text self.source_map.entries.push(SourceMapEntry { original: token.offset(), new: self.source.len(), length: 1, }); self.source.push_str("__sf_rt::render_text!(__sf_buf, "); // write text token with Debug::fmt write!(self.source, "{:?}", token.as_str()).unwrap(); self.source.push_str(");\n"); Ok(()) } fn write_buffered_code( &mut self, token: &Token<'_>, escape: bool, ) -> Result<(), Error> { self.write_buffered_code_with_suffix(token, escape, "") } fn write_buffered_code_with_suffix( &mut self, token: &Token<'_>, escape: bool, suffix: &str, ) -> Result<(), Error> { // parse and split off filter let code_block = syn::parse_str::(token.as_str()).map_err(|e| { let span = e.span(); let mut err = make_error!(ErrorKind::RustSyntaxError(e)); err.offset = into_offset(token.as_str(), span).map(|p| token.offset() + p); err })?; let method = if self.escape && escape { "render_escaped" } else { "render" }; self.source.push_str("__sf_rt::"); self.source.push_str(method); self.source.push_str("!(__sf_buf, "); if let Some(filter) = code_block.filter { let expr_str = format!("{}{}", code_block.expr.into_token_stream(), suffix); let (name, extra_args) = match filter { Filter::Ident(i) => (i.to_string(), None), Filter::Call(c) => ( c.func.into_token_stream().to_string(), Some(c.args.into_token_stream().to_string()), ), }; self.source.push_str("sailfish::runtime::filter::"); self.source.push_str(&name); self.source.push('('); // arguments to filter function { self.source.push_str("&("); let entry = SourceMapEntry { original: token.offset(), new: self.source.len(), length: expr_str.len(), }; self.source_map.entries.push(entry); self.source.push_str(&expr_str); self.source.push(')'); if let Some(extra_args) = extra_args { self.source.push_str(", "); self.source.push_str(&extra_args); } } self.source.push(')'); } else { self.write_token(token); self.source.push_str(suffix); } self.source.push_str(");\n"); Ok(()) } pub fn feed_tokens(&mut self, token_iter: ParseStream<'_>) -> Result<(), Error> { let mut it = token_iter.peekable(); while let Some(token) = it.next() { let token = token?; match token.kind() { TokenKind::Code => self.write_code(&token)?, TokenKind::Comment => {} TokenKind::BufferedCode { escape } => { self.write_buffered_code(&token, escape)? } TokenKind::NestedTemplateOnce => self.write_buffered_code_with_suffix( &token, false, ".render_once()?", )?, TokenKind::Text => { // concatenate repeated text token let offset = token.offset(); let mut concatenated = String::new(); concatenated.push_str(token.as_str()); while let Some(Ok(next_token)) = it.peek() { match next_token.kind() { TokenKind::Text => { concatenated.push_str(next_token.as_str()); it.next(); } TokenKind::Comment => { it.next(); } _ => break, } } let new_token = Token::new(&concatenated, offset, TokenKind::Text); self.write_text(&new_token)?; } } } Ok(()) } pub fn finalize(mut self) -> Result { self.source.push_str("\n}"); match syn::parse_str::(&self.source) { Ok(ast) => Ok(TranslatedSource { ast, source_map: self.source_map, }), Err(synerr) => { let span = synerr.span(); let original_offset = into_offset(&self.source, span) .and_then(|o| self.source_map.reverse_mapping(o)); let mut err = make_error!(ErrorKind::RustSyntaxError(synerr), source = self.source); err.offset = original_offset; Err(err) } } } } enum Filter { Ident(syn::Ident), Call(syn::ExprCall), } impl ToTokens for Filter { fn to_tokens(&self, tokens: &mut TokenStream) { match self { Filter::Ident(ident) => ident.to_tokens(tokens), Filter::Call(call) => call.to_tokens(tokens), } } } struct CodeBlock { #[allow(dead_code)] expr: Box, filter: Option, } impl Parse for CodeBlock { fn parse(s: SynParseStream) -> ParseResult { let main = s.parse::()?; let code_block = match main { Expr::Binary(b) if matches!(b.op, BinOp::BitOr(_)) => { match *b.right { Expr::Call(c) => { if let Expr::Path(ref p) = *c.func { if p.path.get_ident().is_some() { CodeBlock { expr: b.left, filter: Some(Filter::Call(c)), } } else { return Err(syn::Error::new_spanned( p, "Invalid filter name", )); } } else { // if function in right side is not a path, fallback to // normal evaluation block CodeBlock { expr: b.left, filter: None, } } } Expr::Path(p) => { if let Some(i) = p.path.get_ident() { CodeBlock { expr: b.left, filter: Some(Filter::Ident(i.clone())), } } else { return Err(syn::Error::new_spanned( p, "Invalid filter name", )); } } _ => { return Err(syn::Error::new_spanned(b, "Expected filter")); } } } _ => CodeBlock { expr: Box::new(main), filter: None, }, }; Ok(code_block) } } fn into_offset(source: &str, span: Span) -> Option { let lc = span.start(); if lc.line > 0 { Some( source .lines() .take(lc.line - 1) .fold(0, |s, e| s + e.len() + 1) + lc.column, ) } else { None } } #[cfg(test)] mod tests { use super::*; use crate::parser::Parser; #[test] #[ignore] fn translate() { let src = "<% pub fn sample() { %> <%% <%=//%>\n1%><% } %>"; let lexer = Parser::new(); let token_iter = lexer.parse(src); let mut ps = SourceBuilder { escape: true, source: String::with_capacity(token_iter.original_source.len()), source_map: SourceMap::default(), }; ps.feed_tokens(token_iter.clone()).unwrap(); Translator::new().translate(token_iter).unwrap(); } #[test] fn translate_nested_render_once() { let src = r#"outer <%+ inner %> outer"#; let lexer = Parser::new(); let token_iter = lexer.parse(src); let mut ps = SourceBuilder { escape: true, source: String::with_capacity(token_iter.original_source.len()), source_map: SourceMap::default(), }; ps.feed_tokens(token_iter.clone()).unwrap(); assert_eq!( &Translator::new() .translate(token_iter) .unwrap() .ast .into_token_stream() .to_string(), r#"{ __sf_rt :: render_text ! (__sf_buf , "outer ") ; __sf_rt :: render ! (__sf_buf , inner . render_once () ?) ; __sf_rt :: render_text ! (__sf_buf , " outer") ; }"# ); } #[test] fn translate_nested_render_once_with_filter() { let src = r#"outer <%+ inner|upper %> outer"#; let lexer = Parser::new(); let token_iter = lexer.parse(src); let mut ps = SourceBuilder { escape: true, source: String::with_capacity(token_iter.original_source.len()), source_map: SourceMap::default(), }; ps.feed_tokens(token_iter.clone()).unwrap(); assert_eq!( &Translator::new() .translate(token_iter) .unwrap() .ast .into_token_stream() .to_string(), r#"{ __sf_rt :: render_text ! (__sf_buf , "outer ") ; __sf_rt :: render ! (__sf_buf , sailfish :: runtime :: filter :: upper (& (inner . render_once () ?))) ; __sf_rt :: render_text ! (__sf_buf , " outer") ; }"# ); } } sailfish-compiler-0.9.0/src/util.rs000064400000000000000000000045231046102023000153620ustar 00000000000000use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; pub fn read_to_string(path: &Path) -> io::Result { let mut content = std::fs::read_to_string(path)?; // strip break line at file end if content.ends_with('\n') { content.truncate(content.len() - 1); if content.ends_with('\r') { content.truncate(content.len() - 1); } } Ok(content) } fn find_rustfmt() -> io::Result> { let mut toolchain_dir = home::rustup_home()?; toolchain_dir.push("toolchains"); for e in fs::read_dir(toolchain_dir)? { let mut path = e?.path(); path.push("bin"); path.push("rustfmt"); if path.exists() { return Ok(Some(path)); } } Ok(None) } /// Format block expression using `rustfmt` command pub fn rustfmt_block(source: &str) -> io::Result { let rustfmt = match find_rustfmt()? { Some(p) => p, None => { return Err(io::Error::new( io::ErrorKind::NotFound, "rustfmt command not found", )) } }; let mut new_source = String::with_capacity(source.len() + 11); new_source.push_str("fn render()"); new_source.push_str(source); let mut child = Command::new(rustfmt) .args(["--emit", "stdout", "--color", "never", "--quiet"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn()?; let stdin = child .stdin .as_mut() .ok_or_else(|| io::Error::from(io::ErrorKind::BrokenPipe))?; stdin.write_all(new_source.as_bytes())?; let output = child.wait_with_output()?; if output.status.success() { let mut s = String::from_utf8(output.stdout).expect("rustfmt output is non-UTF-8!"); let brace_offset = s.find('{').unwrap(); s.replace_range(..brace_offset, ""); Ok(s) } else { Err(io::Error::new( io::ErrorKind::Other, "rustfmt command failed", )) } } #[cfg(feature = "procmacro")] pub fn filetime(input: &Path) -> filetime::FileTime { use filetime::FileTime; fs::metadata(input) .and_then(|metadata| metadata.modified()) .map_or(FileTime::zero(), FileTime::from_system_time) }