libtest: Added UI tests for --format=json

libtest: Remove usage of jq

libtest: Fixed UI tests

- Now comparing to the right file.
- A python script checks for validity of JSON documents

Author: Gilad Naaman

src/test/run-make/libtest-json/Makefile

@@ -0,0 +1,14 @@
+-include ../tools.mk
+
+# Test expected libtest's JSON output
+
+OUTPUT_FILE := $(TMPDIR)/libtest-json-output.json
+
+all:
+	$(RUSTC) --test f.rs
+	$(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true
+
+	cat $(OUTPUT_FILE) | $(PYTHON) validate_json.py
+
+	# Compare to output file
+	diff output.json $(OUTPUT_FILE)

src/test/run-make/libtest-json/f.rs

@@ -0,0 +1,31 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[test]
+fn a() {
+    // Should pass
+}
+
+#[test]
+fn b() {
+    assert!(false)
+}
+
+#[test]
+#[should_panic]
+fn c() {
+    assert!(false);
+}
+
+#[test]
+#[ignore]
+fn d() {
+    assert!(false);
+}

src/test/run-make/libtest-json/output.json

@@ -0,0 +1,10 @@
+{ "type": "suite", "event": "started", "test_count": "4" }
+{ "type": "test", "event": "started", "name": "a" }
+{ "type": "test", "name": "a", "event": "ok" }
+{ "type": "test", "event": "started", "name": "b" }
+{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:4\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" }
+{ "type": "test", "event": "started", "name": "c" }
+{ "type": "test", "name": "c", "event": "ok" }
+{ "type": "test", "event": "started", "name": "d" }
+{ "type": "test", "name": "d", "event": "ignored" }
+{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" }

src/test/run-make/libtest-json/validate_json.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+
+import sys
+import json
+
+# Try to decode line in order to ensure it is a valid JSON document
+for line in sys.stdin:
+    json.loads(line)