build: Integrate SauceLabs for automated cross-browser testing

kamilogorek · kamilogorek · commit e8319b74393e · 2017-09-08T10:57:08.000+02:00
- Use grunt-saucelabs to run all the tests
- Add README badges for SauceLabs
- Update Grunt tasks CI tests
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -197,6 +197,32 @@ module.exports = function(grunt) {
       }
     },
 
+    'saucelabs-mocha': {
+      all: {
+        options: {
+          urls: [
+            'http://127.0.0.1:9999/test/index.html',
+            'http://127.0.0.1:9999/test/integration/index.html'
+          ],
+          sauceConfig: {
+            'record-video': false,
+            'record-screenshots': false
+          },
+          build: process.env.TRAVIS_BUILD_NUMBER,
+          testname:
+            'Raven.js' +
+            (process.env.TRAVIS_JOB_NUMBER ? ' #' + process.env.TRAVIS_JOB_NUMBER : ''),
+          browsers: [
+            ['macOS 10.12', 'chrome', 'latest'],
+            ['macOS 10.12', 'firefox', 'latest'],
+            ['macOS 10.12', 'safari', '10.0']
+          ],
+          public: 'public',
+          tunnelArgs: ['--verbose']
+        }
+      }
+    },
+
     release: {
       options: {
         npm: false,
@@ -229,6 +255,12 @@ module.exports = function(grunt) {
     },
 
     connect: {
+      ci: {
+        options: {
+          port: 9999
+        }
+      },
+
       test: {
         options: {
           port: 8000,
@@ -301,6 +333,14 @@ module.exports = function(grunt) {
     grunt.config.set('pkg', pkg);
   });
 
+  grunt.registerTask('config:ci', 'Verify CI config', function() {
+    if (!process.env.SAUCE_USERNAME)
+      console.warn('No SAUCE_USERNAME env variable defined.');
+    if (!process.env.SAUCE_ACCESS_KEY)
+      console.warn('No SAUCE_ACCESS_KEY env variable defined.');
+    if (!process.env.SAUCE_USERNAME || !process.env.SAUCE_ACCESS_KEY) process.exit(1);
+  });
+
   // Grunt contrib tasks
   grunt.loadNpmTasks('grunt-contrib-uglify');
   grunt.loadNpmTasks('grunt-contrib-clean');
@@ -313,6 +353,7 @@ module.exports = function(grunt) {
   grunt.loadNpmTasks('grunt-s3');
   grunt.loadNpmTasks('grunt-gitinfo');
   grunt.loadNpmTasks('grunt-sri');
+  grunt.loadNpmTasks('grunt-saucelabs');
 
   // Build tasks
   grunt.registerTask('_prep', ['clean', 'gitinfo', 'version']);
@@ -335,6 +376,13 @@ module.exports = function(grunt) {
   grunt.registerTask('build', ['build.plugins-combined']);
   grunt.registerTask('dist', ['build.core', 'copy:dist']);
 
+  grunt.registerTask('test:ci', [
+    'config:ci',
+    'build.test',
+    'connect:ci',
+    'saucelabs-mocha'
+  ]);
+
   // Webserver tasks
   grunt.registerTask('run:test', ['connect:test']);
   grunt.registerTask('run:docs', ['connect:docs']);
diff --git a/README.md b/README.md
@@ -6,7 +6,9 @@
     <h1>Raven.js - Sentry SDK for JavaScript</h1>
 </p>
 
-[![Build Status](https://travis-ci.org/getsentry/raven-js.svg?branch=master)](https://travis-ci.org/getsentry/raven-js) [![npm](https://img.shields.io/npm/v/raven-js.svg)](https://www.npmjs.com/package/raven-js) [![npm](https://img.shields.io/npm/dm/raven-js.svg)](https://www.npmjs.com/package/raven-js)
+[![Sauce Test Status](https://saucelabs.com/buildstatus/kamilsentry)](https://saucelabs.com/u/kamilsentry)[![Build Status](https://travis-ci.org/getsentry/raven-js.svg?branch=master)](https://travis-ci.org/getsentry/raven-js) [![npm](https://img.shields.io/npm/v/raven-js.svg)](https://www.npmjs.com/package/raven-js) [![npm](https://img.shields.io/npm/dm/raven-js.svg)](https://www.npmjs.com/package/raven-js)
+
+[![Sauce Test Status](https://saucelabs.com/browser-matrix/kamilsentry.svg)](https://saucelabs.com/u/kamilsentry)
 
 ## Requirements
 
diff --git a/package.json b/package.json
@@ -21,7 +21,8 @@
     "test": "npm run lint && grunt build.test && npm run test:unit && npm run test:integration && npm run test:typescript",
     "test:unit": "mocha-chrome test/index.html",
     "test:integration": "mocha-chrome test/integration/index.html --chrome-flags '[\"--disable-web-security\"]' --ignore-resource-errors --ignore-exceptions",
-    "test:typescript": "tsc --noEmit --noImplicitAny typescript/raven-tests.ts"
+    "test:typescript": "tsc --noEmit --noImplicitAny typescript/raven-tests.ts",
+    "test:ci": "npm run lint && grunt test:ci"
   },
   "devDependencies": {
     "bluebird": "^3.4.1",
@@ -42,6 +43,7 @@
     "grunt-gitinfo": "^0.1.7",
     "grunt-release": "^0.13.0",
     "grunt-s3": "0.2.0-alpha.3",
+    "grunt-saucelabs": "^9.0.0",
     "grunt-sri": "mattrobenolt/grunt-sri#pretty",
     "husky": "^0.14.3",
     "lint-staged": "^4.0.4",
diff --git a/test/index.html b/test/index.html
@@ -4,56 +4,65 @@
     <title>Raven.js Test Suite</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <link rel="stylesheet" href="../node_modules/mocha/mocha.css" />
-  </head>
-  <body>
-    <div id="mocha"></div>
+
     <!-- Mocha -->
     <script src="../node_modules/mocha/mocha.js"></script>
     <script src="../node_modules/chai/chai.js"></script>
     <script>
-        mocha.ui('bdd');
-        mocha.reporter('html');
-        var assert = chai.assert
+      mocha.ui('bdd');
+      mocha.reporter('html');
+      var assert = chai.assert
     </script>
 
     <!-- Mocking -->
     <script src="../node_modules/sinon/pkg/sinon.js"></script>
     <script>
-    window.__DEV__ = true;
-    beforeEach(function() {
+      window.__DEV__ = true;
+
+      beforeEach(function() {
         this.sinon = sinon.sandbox.create();
-    });
+      });
 
-    afterEach(function() {
+      afterEach(function() {
         this.sinon.restore();
-    });
+      });
     </script>
 
     <!-- Tests -->
     <script src="../build/raven.test.js"></script>
+  </head>
+  <body>
+    <div id="mocha"></div>
 
     <script>
-        (function(runner){
-          var failed = [];
-
-          runner.on('fail', function(test, err){
-            failed.push({
-              title: test.title,
-              fullTitle: test.fullTitle(),
-              error: {
-                message: err.message,
-                stack: err.stack
-              }
-            });
-          });
+      window.onload = function () {
+        var runner = mocha.run();
+        var failedTests = [];
 
-          runner.on('end', function(){
-            runner.stats.failed = failed;
-            if (typeof global !== "undefined") {
-              global.mochaResults = runner.stats;
-            }
+        var flattenTitles = function (test) {
+          var titles = [];
+          while (test.parent.title){
+            titles.push(test.parent.title);
+            test = test.parent;
+          }
+          return titles.reverse();
+        };
+
+        runner.on('fail', function (test, err) {
+          failedTests.push({
+            name: test.title,
+            result: false,
+            message: err.message,
+            stack: err.stack,
+            titles: flattenTitles(test)
           });
-        })(mocha.run());
+        });
+
+        runner.on('end', function () {
+          window.mochaResults = runner.stats;
+          window.mochaResults.reports = failedTests;
+        });
+      };
     </script>
   </body>
 </html>
diff --git a/test/integration/frame.html b/test/integration/frame.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html>
 <head>
   <meta charset="utf-8">
diff --git a/test/integration/index.html b/test/integration/index.html
@@ -4,61 +4,52 @@
     <title>Raven.js Test Suite</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <link rel="stylesheet" href="../../node_modules/mocha/mocha.css" />
-  </head>
-  <body>
-    <div id="mocha"></div>
+
     <!-- Mocha -->
     <script src="../../node_modules/mocha/mocha.js"></script>
     <script src="../../node_modules/chai/chai.js"></script>
     <script>
-        mocha.ui('bdd');
-        mocha.reporter('html');
-        var assert = chai.assert
-    </script>
-
-    <!-- Mocking -->
-    <script src="../../node_modules/sinon/pkg/sinon.js"></script>
-    <script src="../../build/raven.js"></script>
-    <script>
-    beforeEach(function() {
-        this.sinon = sinon.sandbox.create();
-        var xhr = sinon.useFakeXMLHttpRequest();
-        window.requests = this.requests = [];
-
-        xhr.onCreate = function (xhr) {
-          requests.push(xhr);
-        };
-    });
-
-    afterEach(function() {
-        this.sinon.restore();
-    });
+      mocha.ui('bdd');
+      mocha.reporter('html');
+      var assert = chai.assert
     </script>
 
     <!-- Tests -->
+    <script src="../../build/raven.js"></script>
     <script src="test.js"></script>
+  </head>
+  <body>
+    <div id="mocha"></div>
 
     <script>
-      (function (runner) {
-        window.runner = runner;
-        var failed = [];
+      window.onload = function () {
+        var runner = mocha.run();
+        var failedTests = [];
+
+        var flattenTitles = function (test) {
+          var titles = [];
+          while (test.parent.title){
+            titles.push(test.parent.title);
+            test = test.parent;
+          }
+          return titles.reverse();
+        };
 
         runner.on('fail', function (test, err) {
-          failed.push({
-            title: test.title,
-            fullTitle: test.fullTitle(),
-            error: {
-              message: err.message,
-              stack: err.stack
-            }
+          failedTests.push({
+            name: test.title,
+            result: false,
+            message: err.message,
+            stack: err.stack,
+            titles: flattenTitles(test)
           });
         });
 
         runner.on('end', function () {
-          runner.stats.failed = failed;
           window.mochaResults = runner.stats;
+          window.mochaResults.reports = failedTests;
         });
-      })(mocha.run());
+      };
     </script>
   </body>
 </html>

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-<!DOCTYPE html>`
	`1`	`+<!doctype html>`
`2`	`2`	`<html>`
`3`	`3`	`<head>`
`4`	`4`	`<meta charset="utf-8">`