From 38e5839a8abea71de6b7c8e7c2f018062d43398e Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Thu, 15 Jun 2017 17:01:59 +0530 Subject: [PATCH 1/7] Add `#[serial]' tests. --- src/librustdoc/test.rs | 1 + src/libsyntax/feature_gate.rs | 1 + src/libsyntax/test.rs | 11 ++- src/libtest/lib.rs | 147 +++++++++++++++++++++++++++++++++- 4 files changed, 155 insertions(+), 5 deletions(-) diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index f012fd974b574..e06b8308116c5 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -490,6 +490,7 @@ impl Collector { // compiler failures are test failures should_panic: testing::ShouldPanic::No, allow_fail: allow_fail, + serial: false, }, testfn: testing::DynTestFn(box move |()| { let panic = io::set_panic(None); diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index cb625b1ac066e..aed94f63ffce3 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -550,6 +550,7 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG ("derive", Normal, Ungated), ("should_panic", Normal, Ungated), ("ignore", Normal, Ungated), + ("serial", Normal, Ungated), ("no_implicit_prelude", Normal, Ungated), ("reexport_test_harness_main", Normal, Ungated), ("link_args", Normal, Ungated), diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs index 86f5f42eac796..67156dce05d64 100644 --- a/src/libsyntax/test.rs +++ b/src/libsyntax/test.rs @@ -54,6 +54,7 @@ struct Test { ignore: bool, should_panic: ShouldPanic, allow_fail: bool, + serial: bool, } struct TestCtxt<'a> { @@ -136,6 +137,7 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> { ignore: is_ignored(&i), should_panic: should_panic(&i, &self.cx), allow_fail: is_allowed_fail(&i), + serial: is_serial(&i), }; self.cx.testfns.push(test); self.tests.push(i.ident); @@ -389,6 +391,10 @@ fn is_allowed_fail(i: &ast::Item) -> bool { i.attrs.iter().any(|attr| attr.check_name("allow_fail")) } +fn is_serial(i: &ast::Item) -> bool { + i.attrs.iter().any(|attr| attr.check_name("serial")) +} + fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic { match i.attrs.iter().find(|attr| attr.check_name("should_panic")) { Some(attr) => { @@ -661,6 +667,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P { let should_panic_path = |name| { ecx.path(span, vec![self_id, test_id, ecx.ident_of("ShouldPanic"), ecx.ident_of(name)]) }; + let serial_expr = ecx.expr_bool(span, test.serial); let fail_expr = match test.should_panic { ShouldPanic::No => ecx.expr_path(should_panic_path("No")), ShouldPanic::Yes(msg) => { @@ -683,8 +690,8 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P { vec![field("name", name_expr), field("ignore", ignore_expr), field("should_panic", fail_expr), - field("allow_fail", allow_fail_expr)]); - + field("allow_fail", allow_fail_expr), + field("serial", serial_expr)]); let mut visible_path = match cx.toplevel_reexport { Some(id) => vec![id], diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 92cfb862b1669..79fe62f23f68f 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -213,6 +213,7 @@ pub struct TestDesc { pub ignore: bool, pub should_panic: ShouldPanic, pub allow_fail: bool, + pub serial: bool, } #[derive(Clone)] @@ -425,7 +426,10 @@ Test Attributes: #[ignore] - When applied to a function which is already attributed as a test, then the test runner will ignore these tests during normal test runs. Running with --ignored will run these - tests."#, + tests. + #[serial] - When applied to a function which is already attributed as a + test, then the test runner will not run these tests in + parallel with any other tests."#, usage = options.usage(&message)); } @@ -970,6 +974,7 @@ fn should_sort_failures_before_printing_them() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }; let test_b = TestDesc { @@ -977,6 +982,7 @@ fn should_sort_failures_before_printing_them() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }; let mut st = ConsoleTestState { @@ -1091,7 +1097,9 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) None => get_concurrency(), }; - let mut remaining = filtered_tests; + let (serial, mut remaining): (Vec<_>, Vec<_>) = filtered_tests + .into_iter() + .partition(|t| t.desc.serial); remaining.reverse(); let mut pending = 0; @@ -1161,6 +1169,34 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) pending -= 1; } + for test in serial { + callback(TeWait(test.desc.clone(), test.testfn.padding()))?; + let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); + running_tests.insert(test.desc.clone(), timeout); + run_test(opts, !opts.run_tests, test, tx.clone()); + + let mut res; + loop { + if let Some(timeout) = calc_timeout(&running_tests) { + res = rx.recv_timeout(timeout); + for test in get_timed_out_tests(&mut running_tests) { + callback(TeTimeout(test))?; + } + if res != Err(RecvTimeoutError::Timeout) { + break; + } + } else { + res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected); + break; + } + } + + let (desc, result, stdout) = res.unwrap(); + running_tests.remove(&desc); + + callback(TeResult(desc, result, stdout))?; + } + if opts.bench_benchmarks { // All benchmarks run at the end, in serial. // (this includes metric fns) @@ -1723,8 +1759,11 @@ pub mod bench { mod tests { use test::{TrFailed, TrFailedMsg, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName, - DynTestFn, ShouldPanic}; + DynTestFn, ShouldPanic, TestResult}; + use super::{TestEvent, run_tests}; use std::sync::mpsc::channel; + use std::sync::{Arc, RwLock}; + use std::thread::sleep; use bench; use Bencher; @@ -1739,6 +1778,7 @@ mod tests { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1757,6 +1797,7 @@ mod tests { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1777,6 +1818,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::Yes, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1797,6 +1839,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::YesWithMessage("error message"), allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1819,6 +1862,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::YesWithMessage(expected), allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1837,6 +1881,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::Yes, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| f())), }; @@ -1871,6 +1916,7 @@ mod tests { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| {})), }, @@ -1880,6 +1926,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| {})), }]; @@ -1904,6 +1951,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| {})) }) @@ -1986,6 +2034,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + serial: false, }, testfn: DynTestFn(Box::new(move |()| testfn())), }; @@ -2010,6 +2059,98 @@ mod tests { } } + #[test] + pub fn stress_test_serial_tests() { + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.test_threads = Some(100); + + let limit = 100; + + let lock = Arc::new(RwLock::new(0)); + + let tests = (0..limit) + .map(|n| { + let lock = lock.clone(); + + TestDescAndFn { + desc: TestDesc { + name: DynTestName(format!("stress_{:?}", n)), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + serial: true, + }, + testfn: DynTestFn(Box::new(move |()| { + let mut c = lock.write().unwrap(); + *c += 1; + })) + } + }) + .collect::>(); + + run_tests(&opts, tests, |e| { + match e { + TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"), + TestEvent::TeTimeout(_) => panic!("timeout"), + TestEvent::TeResult(_, ref result, _) if result != &TestResult::TrOk => + panic!("result not okay"), + _ => Ok(()) + } + }).unwrap(); + + assert_eq!(*(*lock).read().unwrap(), limit); + } + + #[test] + pub fn run_concurrent_tests_concurrently() { + use std::time::Duration; + + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.test_threads = Some(2); + + let (tx, rx) = channel::<()>(); + + let tests = vec![ + TestDescAndFn { + desc: TestDesc { + name: DynTestName("first".to_string()), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + serial: false, + }, + testfn: DynTestFn(Box::new(move |()| { + rx.recv_timeout(Duration::from_secs(1)).unwrap(); + })) + }, + TestDescAndFn { + desc: TestDesc { + name: DynTestName("second".to_string()), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + serial: false, + }, + testfn: DynTestFn(Box::new(move |()| { + sleep(Duration::from_millis(100)); + tx.send(()).unwrap(); + })) + }, + ]; + + run_tests(&opts, tests, |e| { + match e { + TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"), + TestEvent::TeTimeout(_) => panic!("timeout"), + TestEvent::TeResult(_, ref result, _) if result != &TestResult::TrOk => + panic!("result not okay"), + _ => Ok(()) + } + }).unwrap(); + } + #[test] pub fn test_metricmap_compare() { let mut m1 = MetricMap::new(); From ef5cf977fb3bc990caf3f1cef6dc9311737185dc Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Fri, 16 Jun 2017 00:01:23 +0530 Subject: [PATCH 2/7] Test serial tests without needing a hundred of them. --- src/libtest/lib.rs | 66 +++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 79fe62f23f68f..361d8058646d8 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -1764,6 +1764,8 @@ mod tests { use std::sync::mpsc::channel; use std::sync::{Arc, RwLock}; use std::thread::sleep; + use std::time::Duration; + use bench; use Bencher; @@ -2063,31 +2065,43 @@ mod tests { pub fn stress_test_serial_tests() { let mut opts = TestOpts::new(); opts.run_tests = true; - opts.test_threads = Some(100); - - let limit = 100; + opts.test_threads = Some(2); let lock = Arc::new(RwLock::new(0)); + let lock2 = lock.clone(); + let lock3 = lock.clone(); - let tests = (0..limit) - .map(|n| { - let lock = lock.clone(); - - TestDescAndFn { - desc: TestDesc { - name: DynTestName(format!("stress_{:?}", n)), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - serial: true, - }, - testfn: DynTestFn(Box::new(move |()| { - let mut c = lock.write().unwrap(); - *c += 1; - })) - } - }) - .collect::>(); + let tests = vec![ + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("first"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + serial: true + }, + testfn: DynTestFn(Box::new(move |()| { + let mut c = lock2.write().unwrap(); + sleep(Duration::from_millis(200)); + *c += 1; + assert_eq!(*c, 1); + })) + }, + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("second"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + serial: true + }, + testfn: DynTestFn(Box::new(move |()| { + let mut c = lock3.write().unwrap(); + assert_eq!(*c, 1); + *c += 1; + })) + } + ]; run_tests(&opts, tests, |e| { match e { @@ -2099,13 +2113,11 @@ mod tests { } }).unwrap(); - assert_eq!(*(*lock).read().unwrap(), limit); + assert_eq!(*(*lock).read().unwrap(), 2); } #[test] pub fn run_concurrent_tests_concurrently() { - use std::time::Duration; - let mut opts = TestOpts::new(); opts.run_tests = true; opts.test_threads = Some(2); @@ -2115,7 +2127,7 @@ mod tests { let tests = vec![ TestDescAndFn { desc: TestDesc { - name: DynTestName("first".to_string()), + name: StaticTestName("first"), ignore: false, should_panic: ShouldPanic::No, allow_fail: false, @@ -2127,7 +2139,7 @@ mod tests { }, TestDescAndFn { desc: TestDesc { - name: DynTestName("second".to_string()), + name: StaticTestName("second"), ignore: false, should_panic: ShouldPanic::No, allow_fail: false, From 3fa5e21416a36c07fadef385f1a752333f11cf0a Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Fri, 16 Jun 2017 01:19:55 +0530 Subject: [PATCH 3/7] Use simpler code to run serial tests and remove trailing whitespace. --- src/libtest/lib.rs | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 361d8058646d8..74ceacdb594ba 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -1171,29 +1171,8 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) for test in serial { callback(TeWait(test.desc.clone(), test.testfn.padding()))?; - let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); - running_tests.insert(test.desc.clone(), timeout); run_test(opts, !opts.run_tests, test, tx.clone()); - - let mut res; - loop { - if let Some(timeout) = calc_timeout(&running_tests) { - res = rx.recv_timeout(timeout); - for test in get_timed_out_tests(&mut running_tests) { - callback(TeTimeout(test))?; - } - if res != Err(RecvTimeoutError::Timeout) { - break; - } - } else { - res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected); - break; - } - } - - let (desc, result, stdout) = res.unwrap(); - running_tests.remove(&desc); - + let (desc, result, stdout) = rx.recv().unwrap(); callback(TeResult(desc, result, stdout))?; } From 161d0b5a6ffed2d0a47e9f9f5d800b24aa61943e Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Fri, 16 Jun 2017 03:50:44 +0530 Subject: [PATCH 4/7] Specify `serial: false' in `make_test' in `compiletest/src/main.rs'. --- src/tools/compiletest/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index cc42544ef02b7..f8064a07f5da8 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -475,6 +475,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn ignore: ignore, should_panic: should_panic, allow_fail: false, + serial: false, }, testfn: make_test_closure(config, testpaths), } From 9549b7d745e51e816e7a7a8861a4ab6fdd53ee0a Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Mon, 26 Jun 2017 00:04:15 +0530 Subject: [PATCH 5/7] Incorporate suggestions from @bjorn3. --- src/libtest/lib.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 74ceacdb594ba..239531ea7d71e 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -1740,6 +1740,7 @@ mod tests { TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName, DynTestFn, ShouldPanic, TestResult}; use super::{TestEvent, run_tests}; + use std::io; use std::sync::mpsc::channel; use std::sync::{Arc, RwLock}; use std::thread::sleep; @@ -2040,6 +2041,16 @@ mod tests { } } + fn panic_on_non_ok_result(result: &TestResult) -> io::Result<()> { + match *result { + TestResult::TrOk => Ok(()), + TestResult::TrFailed => panic!("test failed (no message)"), + TestResult::TrFailedMsg(ref s) => panic!("test failed: {}", s), + TestResult::TrIgnored => panic!("test ignored"), + _ => panic!("test returned unexpected result"), + } + } + #[test] pub fn stress_test_serial_tests() { let mut opts = TestOpts::new(); @@ -2057,7 +2068,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, - serial: true + serial: true, }, testfn: DynTestFn(Box::new(move |()| { let mut c = lock2.write().unwrap(); @@ -2072,7 +2083,7 @@ mod tests { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, - serial: true + serial: true, }, testfn: DynTestFn(Box::new(move |()| { let mut c = lock3.write().unwrap(); @@ -2086,8 +2097,8 @@ mod tests { match e { TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"), TestEvent::TeTimeout(_) => panic!("timeout"), - TestEvent::TeResult(_, ref result, _) if result != &TestResult::TrOk => - panic!("result not okay"), + TestEvent::TeResult(_, ref result, _) => + panic_on_non_ok_result(result), _ => Ok(()) } }).unwrap(); @@ -2135,8 +2146,8 @@ mod tests { match e { TestEvent::TeFilteredOut(n) if n > 0 => panic!("filtered out"), TestEvent::TeTimeout(_) => panic!("timeout"), - TestEvent::TeResult(_, ref result, _) if result != &TestResult::TrOk => - panic!("result not okay"), + TestEvent::TeResult(_, ref result, _) => + panic_on_non_ok_result(result), _ => Ok(()) } }).unwrap(); From f6e5c5e8f4c4a48581a939b048dee6d906deac40 Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Mon, 26 Jun 2017 01:57:25 +0530 Subject: [PATCH 6/7] Feature-gate `#[serial]` and add `compile-fail` test. --- src/libsyntax/feature_gate.rs | 9 ++++++++- .../compile-fail/feature-gate-serial_tests.rs | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 src/test/compile-fail/feature-gate-serial_tests.rs diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index aed94f63ffce3..f1f210f4ce73a 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -369,6 +369,9 @@ declare_features! ( // global allocators and their internals (active, global_allocator, "1.20.0", None), (active, allocator_internals, "1.20.0", None), + + // Forces a test to execute in serial rather than in parallel with others. + (active, serial_tests, "1.20.0", Some(42684)), ); declare_features! ( @@ -550,7 +553,6 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG ("derive", Normal, Ungated), ("should_panic", Normal, Ungated), ("ignore", Normal, Ungated), - ("serial", Normal, Ungated), ("no_implicit_prelude", Normal, Ungated), ("reexport_test_harness_main", Normal, Ungated), ("link_args", Normal, Ungated), @@ -840,6 +842,11 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG "allow_fail attribute is currently unstable", cfg_fn!(allow_fail))), + ("serial", Normal, Gated(Stability::Unstable, + "serial_tests", + "serial tests are currently unstable", + cfg_fn!(serial_tests))), + // Crate level attributes ("crate_name", CrateLevel, Ungated), ("crate_type", CrateLevel, Ungated), diff --git a/src/test/compile-fail/feature-gate-serial_tests.rs b/src/test/compile-fail/feature-gate-serial_tests.rs new file mode 100644 index 0000000000000..12774f6691464 --- /dev/null +++ b/src/test/compile-fail/feature-gate-serial_tests.rs @@ -0,0 +1,16 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// check that #[serial] is feature-gated + +#[serial] //~ ERROR serial tests are currently unstable +fn in_serial() { + assert!(true); +} \ No newline at end of file From 6cf05ab529052ef6ae62c4874628bcbecfe7b0cc Mon Sep 17 00:00:00 2001 From: Shiv Jha-Mathur Date: Mon, 26 Jun 2017 11:47:46 +0530 Subject: [PATCH 7/7] Add trailing newline to `feature-gate-serial_tests.rs'. --- src/test/compile-fail/feature-gate-serial_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/compile-fail/feature-gate-serial_tests.rs b/src/test/compile-fail/feature-gate-serial_tests.rs index 12774f6691464..55f86269495ec 100644 --- a/src/test/compile-fail/feature-gate-serial_tests.rs +++ b/src/test/compile-fail/feature-gate-serial_tests.rs @@ -13,4 +13,4 @@ #[serial] //~ ERROR serial tests are currently unstable fn in_serial() { assert!(true); -} \ No newline at end of file +}