diff --git a/example/src/lib.rs b/example/src/lib.rs index 788e66c..a9bf281 100644 --- a/example/src/lib.rs +++ b/example/src/lib.rs @@ -11,6 +11,10 @@ // // @matches "included[?id=='nested_modules::example_module'] \ // .relationships.parent" [] +/// An example module +/// ```rust +/// assert!(true); +/// ``` pub mod example_module { // For the child: diff --git a/src/error.rs b/src/error.rs index 955732c..c6b8090 100644 --- a/src/error.rs +++ b/src/error.rs @@ -44,3 +44,11 @@ pub struct MovedFlag { /// A message explaning where the flag moved to pub msg: String, } + +/// Thrown whenever creation of documentation tests fail +#[derive(Debug, Fail)] +#[fail(display = "Unable to test documentation: \"{}\"", output)] +pub struct DocTestErr { + /// The output of the Command that failed + pub output: String, +} diff --git a/src/lib.rs b/src/lib.rs index 4a16e33..7cadcad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,13 +36,11 @@ use std::env; use std::fs::{self, File}; use std::io::prelude::*; use std::io; -use std::iter; use std::path::{Path, PathBuf}; -use std::process::{self, Command, Stdio}; +use std::process::{Command, Stdio}; use cargo::Target; use json::Documentation; -use test::TestResult; use ui::Ui; pub use json::create_documentation; @@ -228,81 +226,33 @@ pub fn test(config: &Config) -> Result<()> { .map_err(|e| failure::Error::from(e.context("could not find generated documentation")))?; let docs: Documentation = serde_json::from_reader(doc_json)?; + // TODO a better way to find crate name? let krate = docs.data.as_ref().unwrap(); - let tests: Vec<_> = iter::once(krate) - .chain(docs.included.iter().flat_map(|data| data)) - .map(|data| (&data.id, test::gather_tests(&data))) - .collect(); - - // Run the tests. - static SUCCESS_MESSAGE: &str = "ok"; - static FAILURE_MESSAGE: &str = "FAILED"; - - let num_tests: usize = tests.iter().map(|&(_, ref tests)| tests.len()).sum(); - println!("running {} tests", num_tests); - - let mut passed = 0; - let mut failures = vec![]; - for (id, tests) in tests { - for (number, test) in tests.iter().enumerate() { - // FIXME: Make the name based off the file and line number. - let name = format!("{} - {}", id, number); - print!("test {} ... ", name); - io::stdout().flush()?; - - let message = match test::run_test(config, test)? { - TestResult::Success => { - passed += 1; - SUCCESS_MESSAGE - } - TestResult::Failure(output) => { - failures.push((name, output)); - FAILURE_MESSAGE - } - }; - - println!("{}", message); - } - } + let crate_name = krate.id.split("::").next().unwrap(); - if !failures.is_empty() { - // Print the output of each failure. - for &(ref name, ref output) in &failures { - let stdout = String::from_utf8_lossy(&output.stdout); - let stdout = stdout.trim(); - - if !stdout.is_empty() { - println!("\n---- {} stdout ----\n{}", name, stdout); - } - - let stderr = String::from_utf8_lossy(&output.stderr); - let stderr = stderr.trim(); - - if !stderr.is_empty() { - println!("\n---- {} stderr ----\n{}", name, stderr); - } - } + let location = config.output_path().join("tests"); + let tests = { + let task = config.ui.start_task("Finding tests"); + task.report("In Progress"); + test::find_tests(&docs) + }; - // Print a summary of all failures at the bottom. - println!("\nfailures:"); - for &(ref name, _) in &failures { - println!(" {}", name); - } + { + let task = config.ui.start_task("Saving tests"); + task.report("In Progress"); + test::save_tests(&tests, &location, &crate_name)?; } - println!( - "\ntest result: {}. {} passed; {} failed; 0 ignored; 0 measured; 0 filtered out", - if failures.is_empty() { - SUCCESS_MESSAGE - } else { - FAILURE_MESSAGE - }, - passed, - failures.len() - ); - - if !failures.is_empty() { - process::exit(1); + let binary = { + let task = config.ui.start_task("Compiling tests"); + task.report("In Progress"); + test::compile_tests(&config, &location)? + }; + + { + let task = config.ui.start_task("Executing tests"); + task.report("In Progress"); + test::execute_tests(&binary)?; } Ok(()) diff --git a/src/test.rs b/src/test.rs index 7451eba..e770227 100644 --- a/src/test.rs +++ b/src/test.rs @@ -1,30 +1,88 @@ //! Documentation tests. +use std::fs::{DirBuilder, File}; use std::io::prelude::*; -use std::process::{Command, Output, Stdio}; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::Command; use pulldown_cmark::{Event, Parser, Tag}; use quote::{ToTokens, Tokens}; use serde_json::{self, Value}; -use syn::{self, Block, Constness, FnDecl, FunctionRetTy, Generics, Ident, Item, ItemKind, Stmt, - Unsafety, Visibility, WhereClause}; -use tempdir::TempDir; +use syn::{self, AttrStyle, Attribute, Block, Constness, Expr, ExprKind, FnDecl, FunctionRetTy, + Generics, Ident, Item, ItemKind, MetaItem, PathParameters, PathSegment, Stmt, Unsafety, + Visibility, WhereClause}; use Config; use Result; -use json::Document; +use json::{Document, Documentation}; +use error; -/// The outcome of a documentation test. -pub enum TestResult { - /// The test passed. - Success, +struct Extern { + /// name of the external crate + name: String, - /// The test failed. Includes output generated by the failure. - Failure(Output), + /// Path to rlib + location: PathBuf, +} + +pub fn find_tests<'a>(docs: &'a Documentation) -> Vec<(&'a String, Vec)> { + let krate = docs.data.as_ref().unwrap(); + + iter::once(krate) + .chain(docs.included.iter().flat_map(|data| data)) + .map(|data| (&data.id, gather_tests(&data))) + .collect() +} + +/// Determine the dependencies and the location of the dependency artifacts +fn find_externs_for_crate(config: &Config) -> Result> { + let output = Command::new("cargo") + .arg("build") + .arg("--manifest-path") + .arg(&config.manifest_path) + .args(&["--message-format", "json"]) + .output()?; + if !output.status.success() { + return Err(format_err!( + "cargo did not exit successfully: {}", + output.status + )); + } + + let output = String::from_utf8(output.stdout).expect("cargo did not output valid utf-8"); + + let mut externs = vec![]; + + for message in output.lines() { + let message = serde_json::from_str::(message)?; + let is_compiler_artifact = message + .as_object() + .unwrap() + .get("reason") + .and_then(Value::as_str) + .map(|reason| reason == "compiler-artifact") + .unwrap_or_default(); + + if is_compiler_artifact { + let name = message + .pointer("/target/name") + .and_then(|v| Some(v.to_string())) + .unwrap(); + let location = message + .pointer("/filenames/0") + .and_then(|v| Some(PathBuf::from(v.as_str().unwrap()))) + .unwrap(); + + externs.push(Extern { name, location }); + } + } + + Ok(externs) } /// Find and prepare tests in the given document. -pub fn gather_tests(document: &Document) -> Vec { +fn gather_tests(document: &Document) -> Vec { if let Some(docs) = document.attributes.get("docs") { find_test_blocks(docs) .into_iter() @@ -87,160 +145,202 @@ fn find_test_blocks(docs: &str) -> Vec { /// /// Any crate attributes are preserved at the top level. fn preprocess(test: &str, crate_name: &str) -> String { - if let Ok(mut test_crate) = syn::parse_crate(test) { - let has_extern_crate = test_crate.items.iter().any(|item| match item.node { + if let Ok(mut ast) = syn::parse_crate(test) { + // TODO if the extern crate has `#[macro_use]` we need to strip it out + let has_extern_crate = ast.items.iter().any(|item| match item.node { ItemKind::ExternCrate(..) => true, _ => false, }); - let has_main_function = test_crate.items.iter().any(|item| match item.node { + let has_main_function = ast.items.iter().any(|item| match item.node { ItemKind::Fn(..) if item.ident == "main" => true, _ => false, }); - if !has_main_function { - let new_main = ItemKind::Fn( - Box::new(FnDecl { - inputs: vec![], - output: FunctionRetTy::Default, - variadic: false, - }), - Unsafety::Normal, - Constness::NotConst, - None, - Generics { - lifetimes: vec![], - ty_params: vec![], - where_clause: WhereClause::none(), - }, - Box::new(Block { - stmts: test_crate - .items - .drain(..) - .map(|item| Stmt::Item(Box::new(item))) - .collect(), - }), - ); - - test_crate.items.push(Item { - ident: Ident::new("main"), - vis: Visibility::Inherited, + let mut stmts = ast.items + .drain(..) + .map(|item| Stmt::Item(Box::new(item))) + .collect::>(); + + if has_main_function { + let main_fn_call = Stmt::Semi(Box::new(Expr { + node: ExprKind::Call( + Box::new(Expr { + node: ExprKind::Path( + None, + PathSegment { + ident: Ident::new("main"), + parameters: PathParameters::none(), + }.into(), + ), + attrs: vec![], + }), + vec![], + ), attrs: vec![], - node: new_main, - }); + })); + + stmts.push(main_fn_call); } // TODO: Handle `#![doc(test(no_crate_inject))]`? if !has_extern_crate && crate_name != "std" { - test_crate.items.insert( + stmts.insert( 0, - Item { + Stmt::Item(Box::new(Item { ident: Ident::new(crate_name), vis: Visibility::Inherited, attrs: vec![], node: ItemKind::ExternCrate(None), - }, + })), ) } + let a_doc_test = ItemKind::Fn( + Box::new(FnDecl { + inputs: vec![], + output: FunctionRetTy::Default, + variadic: false, + }), + Unsafety::Normal, + Constness::NotConst, + None, + Generics { + lifetimes: vec![], + ty_params: vec![], + where_clause: WhereClause::none(), + }, + Box::new(Block { stmts: stmts }), + ); + + let test_attr = Attribute { + style: AttrStyle::Outer, + value: MetaItem::Word(Ident::new("test")), + is_sugared_doc: false, + }; + + ast.items.push(Item { + ident: Ident::new("a_doc_test"), + vis: Visibility::Inherited, + attrs: vec![test_attr], + node: a_doc_test, + }); + let mut tokens = Tokens::new(); - test_crate.to_tokens(&mut tokens); + ast.to_tokens(&mut tokens); let program = tokens.to_string(); program } else { // If we couldn't parse the crate, then test compilation will fail anyways. Just wrap - // everything in a main function. - format!("fn main() {{\n{}\n}}", test) + // everything in a test function. + format!("#[test] fn a_doc_test() {{\n{}\n}}", test) } } -/// Execute a test. -pub fn run_test(config: &Config, program: &str) -> Result { - static TEST_NAME: &str = "rustdoc-test"; +pub fn save_tests( + tests: &Vec<(&String, Vec)>, + save_path: &Path, + crate_name: &str, +) -> Result<()> { + DirBuilder::new().recursive(true).create(save_path)?; - // First, determine the location of the dependency artifacts so we can pass them to rustc. - let output = Command::new("cargo") - .arg("build") - .arg("--manifest-path") - .arg(&config.manifest_path) - .args(&["--message-format", "json"]) - .output()?; - if !output.status.success() { - return Err(format_err!( - "cargo did not exit successfully: {}", - output.status - )); + let mut mods = vec![]; + + for &(ref id, ref tests) in tests { + for (number, test) in tests.iter().enumerate() { + // FIXME: Make the name based off the file and line number. + let name = format!("{}_{}", id, number); + + // TODO make this a different function + // filter test names into valid identifiers that can be put into `mod #ident` + let name = name.replace("::", "_"); + // + + let filename = save_path.join(&name).with_extension("rs"); + let mut file = File::create(filename)?; + file.write_all(test.as_bytes())?; + + mods.push(name); + } } - let output = String::from_utf8(output.stdout).expect("cargo did not output valid utf-8"); + // TODO use syn here as well? + let mut main = String::new(); - let mut externs = vec![]; + main.push_str(&format!("extern crate {};\n", crate_name)); + for m in mods { + main.push_str(&format!("mod {};\n", m)); + } + main.push_str("fn main() {}"); + let mut file = File::create(save_path.join("main.rs"))?; + file.write_all(main.as_bytes())?; - for message in output.lines() { - let message = serde_json::from_str::(message)?; - let is_compiler_artifact = message - .as_object() - .unwrap() - .get("reason") - .and_then(Value::as_str) - .map(|reason| reason == "compiler-artifact") - .unwrap_or_default(); + Ok(()) +} - if is_compiler_artifact { - let name = message - .pointer("/target/name") - .and_then(Value::as_str) - .unwrap(); - let rlib = message - .pointer("/filenames/0") - .and_then(Value::as_str) - .unwrap(); +fn find_search_path(crate_externs: &Vec) -> Result { + let e = crate_externs.first().ok_or::<::failure::Error>( + error::DocTestErr { + output: "No externs to get search path".to_string(), + }.into(), + )?; + let path = e.location.parent().ok_or::<::failure::Error>( + error::DocTestErr { + output: "No parent for extern path".to_string(), + }.into(), + )?; + Ok(path.to_path_buf()) +} - externs.push(format!("{}={}", name, rlib)); - } +pub fn compile_tests(config: &Config, save_path: &Path) -> Result { + static TEST_NAME: &str = "rustdoc-test"; + + let crate_externs = find_externs_for_crate(config)?; + + let mut externs = vec![]; + for c in crate_externs.iter() { + externs.push(format!("{}={}", &c.name, &c.location.to_string_lossy())); } + let search_path = find_search_path(&crate_externs)?; + let extern_args: Vec<_> = externs .into_iter() .flat_map(|arg| vec![String::from("--extern"), arg]) .collect(); - let tempdir = TempDir::new("rustdoc")?; - - // Compile the test. - // - // TODO: If the tests fail, the output we get will not include the name of the file that the - // test came from, since we're passing the code through stdin. - let mut rustc = Command::new("rustc") - .arg("-") + let output = Command::new("rustc") + .arg("main.rs") + .arg("--test") .args(&["-o", TEST_NAME]) .args(&["--cap-lints", "allow"]) + .arg("-L") + .arg(search_path.to_str().unwrap()) .args(extern_args) - .current_dir(&tempdir) - .stderr(Stdio::piped()) - .stdin(Stdio::piped()) - .spawn()?; - - rustc.stdin.as_mut().unwrap().write_all(program.as_bytes())?; + .current_dir(&save_path) + .output()?; - let rustc_output = rustc.wait_with_output()?; - if !rustc_output.status.success() { - return Ok(TestResult::Failure(rustc_output)); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + return Err(error::DocTestErr { output: stderr }.into()); } - // Run the test. - let test_output = Command::new(tempdir.as_ref().join(TEST_NAME)) - .current_dir(&tempdir) - .output()?; + Ok(save_path.join(TEST_NAME)) +} - let result = if test_output.status.success() { - TestResult::Success - } else { - TestResult::Failure(test_output) - }; +pub fn execute_tests(binary: &Path) -> Result<()> { + // spawn allows the test output to write to stdout so we are not waiting for all the tests to + // complete before showing the user output + let rustdoc_test = Command::new(binary).spawn()?; + let output = rustdoc_test.wait_with_output()?; - Ok(result) + if output.status.success() { + Ok(()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Err(error::DocTestErr { output: stderr }.into()) + } } #[cfg(test)] @@ -289,8 +389,9 @@ mod tests { assert_eq!( &super::preprocess("assert!(true);", "test_crate"), quote!{ - extern crate test_crate; - fn main() { + #[test] + fn a_doc_test() { + extern crate test_crate; assert!(true); } }.as_str() @@ -307,7 +408,8 @@ mod tests { "some_other_crate", ), quote!{ - fn main() { + #[test] + fn a_doc_test() { extern crate rustdoc; use rustdoc::build; } @@ -324,9 +426,13 @@ mod tests { "hello_world", ), quote!{ - extern crate hello_world; - fn main() { - println!("Hello, world!"); + #[test] + fn a_doc_test() { + extern crate hello_world; + fn main() { + println!("Hello, world!"); + } + main(); } }.as_str() );