From 0e3c61f0883059e9aeea840fcce5662b7969baaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:09:19 -0500
Subject: [PATCH 01/12] merge karma ciconf with (regular) conf

- use process.env.CIRCLECI to determine
  if karma is run on CI or not
- autoWatch is false on CI
- singleRun is true on CI
- Chrome 54 (on ubuntu 14.04) is now used on CI !!
---
 package.json                 |  2 +-
 test/jasmine/karma.ciconf.js | 42 ------------------------------------
 test/jasmine/karma.conf.js   | 16 ++++++++------
 3 files changed, 10 insertions(+), 50 deletions(-)
 delete mode 100644 test/jasmine/karma.ciconf.js

diff --git a/package.json b/package.json
index 788e9f0dfd8..ff1be246dc6 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
     "docker": "node tasks/docker.js",
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
-    "citest-jasmine": "karma start test/jasmine/karma.ciconf.js",
+    "citest-jasmine": "CIRCLECI=0 karma start test/jasmine/karma.conf.js",
     "test-image": "node tasks/test_image.js",
     "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue",
     "test-export": "node tasks/test_export.js",
diff --git a/test/jasmine/karma.ciconf.js b/test/jasmine/karma.ciconf.js
deleted file mode 100644
index 7b782e60cea..00000000000
--- a/test/jasmine/karma.ciconf.js
+++ /dev/null
@@ -1,42 +0,0 @@
-// Karma configuration
-
-function func(config) {
-
-    // level of logging
-    // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
-    func.defaultConfig.logLevel = config.LOG_INFO;
-
-    // Note: config.LOG_DEBUG may not be verbose enough to pin down the source of failed tests.
-    // See the note in CONTRIBUTING.md about karma-verbose-reporter:
-    // func.defaultConfig.reporters = ['verbose'];
-
-
-    // Continuous Integration mode
-
-    /*
-     * WebGL interaction test cases fail on the CircleCI
-     * most likely due to a WebGL/driver issue;
-     * exclude them from the CircleCI test bundle.
-     *
-     */
-    func.defaultConfig.exclude = [
-        'tests/gl_plot_interact_test.js',
-        'tests/gl_plot_interact_basic_test.js',
-        'tests/gl2d_scatterplot_contour_test.js',
-        'tests/gl2d_pointcloud_test.js'
-    ];
-
-    // if true, Karma captures browsers, runs the tests and exits
-    func.defaultConfig.singleRun = true;
-
-    func.defaultConfig.browserNoActivityTimeout = 30000; // 30 seconds
-
-    func.defaultConfig.autoWatch = false;
-
-    func.defaultConfig.browsers = ['Firefox_WindowSized'];
-
-    config.set(func.defaultConfig);
-}
-
-func.defaultConfig = require('./karma.conf').defaultConfig;
-module.exports = func;
diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js
index 5210abba9a5..f42a771fb46 100644
--- a/test/jasmine/karma.conf.js
+++ b/test/jasmine/karma.conf.js
@@ -17,6 +17,7 @@ var constants = require('../../tasks/util/constants');
 
 var arg = process.argv[4];
 
+var isCI = !!process.env.CIRCLECI;
 var testFileGlob = arg ? arg : 'tests/*_test.js';
 var isSingleSuiteRun = (arg && arg.indexOf('bundle_tests/') === -1);
 var isRequireJSTest = (arg && arg.indexOf('bundle_tests/requirejs') !== -1);
@@ -84,7 +85,13 @@ func.defaultConfig = {
     colors: true,
 
     // enable / disable watching file and executing tests whenever any file changes
-    autoWatch: true,
+    autoWatch: !isCI,
+
+    // if true, Karma captures browsers, runs the tests and exits
+    singleRun: isCI,
+
+    // how long will Karma wait for a message from a browser before disconnecting (30 ms)
+    browserNoActivityTimeout: 30000,
 
     // start these browsers
     // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
@@ -103,19 +110,14 @@ func.defaultConfig = {
         }
     },
 
-    // Continuous Integration mode
-    // if true, Karma captures browsers, runs the tests and exits
-    singleRun: false,
-
     browserify: {
         transform: ['../../tasks/util/shortcut_paths.js'],
         extensions: ['.js'],
-        watch: true,
+        watch: !isCI,
         debug: true
     }
 };
 
-
 // Add lib/index.js to single-suite runs,
 // to avoid import conflicts due to plotly.js
 // circular dependencies.

From aa1aaef85e420e8b15ffa8e47ecb6d6319824f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:13:57 -0500
Subject: [PATCH 02/12] comment & lint in karma conf

- Chrome is now used on CI
- highlight that --ignore-gpu-blacklist allow to
  test WebGL on CI (CircleCI already wraps the process with xfvb)
- DRY files array operations
---
 test/jasmine/karma.conf.js | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js
index f42a771fb46..b54df0708f7 100644
--- a/test/jasmine/karma.conf.js
+++ b/test/jasmine/karma.conf.js
@@ -61,6 +61,7 @@ func.defaultConfig = {
     // N.B. this field is filled below
     files: [],
 
+    // list of files / pattern to exclude
     exclude: [],
 
     // preprocess matching files before serving them to the browser
@@ -98,7 +99,10 @@ func.defaultConfig = {
     browsers: ['Chrome_WindowSized'],
 
     // custom browser options
+    //
     // window-size values came from observing default size
+    //
+    // '--ignore-gpu-blacklist' allow to test WebGL on CI (!!!)
     customLaunchers: {
         Chrome_WindowSized: {
             base: 'Chrome',
@@ -122,11 +126,10 @@ func.defaultConfig = {
 // to avoid import conflicts due to plotly.js
 // circular dependencies.
 if(isSingleSuiteRun) {
-    func.defaultConfig.files = [
+    func.defaultConfig.files.push(
         pathToJQuery,
-        pathToMain,
-        testFileGlob
-    ];
+        pathToMain
+    );
 
     func.defaultConfig.preprocessors[pathToMain] = ['browserify'];
     func.defaultConfig.preprocessors[testFileGlob] = ['browserify'];
@@ -134,8 +137,7 @@ if(isSingleSuiteRun) {
 else if(isRequireJSTest) {
     func.defaultConfig.files = [
         constants.pathToRequireJS,
-        constants.pathToRequireJSFixture,
-        testFileGlob
+        constants.pathToRequireJSFixture
     ];
 }
 else if(isIE9Test) {
@@ -143,20 +145,15 @@ else if(isIE9Test) {
     // to catch reference errors that could occur
     // when plotly.js is first loaded.
 
-    func.defaultConfig.files = [
-        './assets/ie9_mock.js',
-        testFileGlob
-    ];
-
+    func.defaultConfig.files.push('./assets/ie9_mock.js');
     func.defaultConfig.preprocessors[testFileGlob] = ['browserify'];
 }
 else {
-    func.defaultConfig.files = [
-        pathToJQuery,
-        testFileGlob
-    ];
-
+    func.defaultConfig.files.push(pathToJQuery);
     func.defaultConfig.preprocessors[testFileGlob] = ['browserify'];
 }
 
+// lastly, load test file glob
+func.defaultConfig.files.push(testFileGlob);
+
 module.exports = func;

From 62d33b90ab20f8d076f70465398b2263020b536d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:16:24 -0500
Subject: [PATCH 03/12] rm hasWebGLSupport early return in jasmine specs

- ... as WebGL is now supported on CI :tada:
---
 test/jasmine/assets/has_webgl_support.js       | 18 ------------------
 test/jasmine/tests/gl2d_click_test.js          |  3 ---
 .../tests/gl2d_date_axis_render_test.js        |  4 ----
 test/jasmine/tests/gl_plot_interact_test.js    |  6 ------
 test/jasmine/tests/mapbox_test.js              |  5 -----
 test/jasmine/tests/parcoords_test.js           |  3 ---
 test/jasmine/tests/scattermapbox_test.js       |  3 ---
 7 files changed, 42 deletions(-)
 delete mode 100644 test/jasmine/assets/has_webgl_support.js

diff --git a/test/jasmine/assets/has_webgl_support.js b/test/jasmine/assets/has_webgl_support.js
deleted file mode 100644
index c5702b53ff9..00000000000
--- a/test/jasmine/assets/has_webgl_support.js
+++ /dev/null
@@ -1,18 +0,0 @@
-'use strict';
-
-var getContext = require('webgl-context');
-
-module.exports = function hasWebGLSupport(testName) {
-    var gl, canvas;
-
-    canvas = document.createElement('canvas');
-    gl = getContext({canvas: canvas});
-
-    var hasSupport = !!gl;
-
-    if(!hasSupport) {
-        console.warn('Cannot get WebGL context. Skip test *' + testName + '*');
-    }
-
-    return hasSupport;
-};
diff --git a/test/jasmine/tests/gl2d_click_test.js b/test/jasmine/tests/gl2d_click_test.js
index fe14d214b34..e170b43fe2a 100644
--- a/test/jasmine/tests/gl2d_click_test.js
+++ b/test/jasmine/tests/gl2d_click_test.js
@@ -4,7 +4,6 @@ var Lib = require('@src/lib');
 var createGraphDiv = require('../assets/create_graph_div');
 var destroyGraphDiv = require('../assets/destroy_graph_div');
 var customMatchers = require('../assets/custom_matchers');
-var hasWebGLSupport = require('../assets/has_webgl_support');
 
 // cartesian click events events use the hover data
 // from the mousemove events and then simulate
@@ -19,8 +18,6 @@ Plotly.register([
 
 describe('Test hover and click interactions', function() {
 
-    if(!hasWebGLSupport('gl2d_click_test')) return;
-
     var mock = require('@mocks/gl2d_14.json');
     var mock2 = require('@mocks/gl2d_pointcloud-basic.json');
     var mock3 = {
diff --git a/test/jasmine/tests/gl2d_date_axis_render_test.js b/test/jasmine/tests/gl2d_date_axis_render_test.js
index 53392de3e76..52d98209e48 100644
--- a/test/jasmine/tests/gl2d_date_axis_render_test.js
+++ b/test/jasmine/tests/gl2d_date_axis_render_test.js
@@ -1,14 +1,10 @@
 var PlotlyInternal = require('@src/plotly');
 
-var hasWebGLSupport = require('../assets/has_webgl_support');
-
 var createGraphDiv = require('../assets/create_graph_div');
 var destroyGraphDiv = require('../assets/destroy_graph_div');
 
 describe('date axis', function() {
 
-    if(!hasWebGLSupport('axes_test date axis')) return;
-
     var gd;
 
     beforeEach(function() {
diff --git a/test/jasmine/tests/gl_plot_interact_test.js b/test/jasmine/tests/gl_plot_interact_test.js
index db4a57922a2..746c9c5474c 100644
--- a/test/jasmine/tests/gl_plot_interact_test.js
+++ b/test/jasmine/tests/gl_plot_interact_test.js
@@ -11,12 +11,6 @@ var mouseEvent = require('../assets/mouse_event');
 var selectButton = require('../assets/modebar_button');
 var customMatchers = require('../assets/custom_matchers');
 
-/*
- * WebGL interaction test cases fail on the CircleCI
- * most likely due to a WebGL/driver issue
- *
- */
-
 var MODEBAR_DELAY = 500;
 
 
diff --git a/test/jasmine/tests/mapbox_test.js b/test/jasmine/tests/mapbox_test.js
index 61a7e868a9b..c490b539257 100644
--- a/test/jasmine/tests/mapbox_test.js
+++ b/test/jasmine/tests/mapbox_test.js
@@ -7,7 +7,6 @@ var supplyLayoutDefaults = require('@src/plots/mapbox/layout_defaults');
 var d3 = require('d3');
 var createGraphDiv = require('../assets/create_graph_div');
 var destroyGraphDiv = require('../assets/destroy_graph_div');
-var hasWebGLSupport = require('../assets/has_webgl_support');
 var mouseEvent = require('../assets/mouse_event');
 var customMatchers = require('../assets/custom_matchers');
 var failTest = require('../assets/fail_test');
@@ -176,8 +175,6 @@ describe('mapbox defaults', function() {
 describe('mapbox credentials', function() {
     'use strict';
 
-    if(!hasWebGLSupport('mapbox credentials')) return;
-
     var dummyToken = 'asfdsa124331wersdsa1321q3';
     var gd;
 
@@ -282,8 +279,6 @@ describe('mapbox credentials', function() {
 describe('mapbox plots', function() {
     'use strict';
 
-    if(!hasWebGLSupport('mapbox plots')) return;
-
     var mock = require('@mocks/mapbox_0.json'),
         gd;
 
diff --git a/test/jasmine/tests/parcoords_test.js b/test/jasmine/tests/parcoords_test.js
index 5528983a27d..e06d304c603 100644
--- a/test/jasmine/tests/parcoords_test.js
+++ b/test/jasmine/tests/parcoords_test.js
@@ -7,7 +7,6 @@ var attributes = require('@src/traces/parcoords/attributes');
 
 var createGraphDiv = require('../assets/create_graph_div');
 var destroyGraphDiv = require('../assets/destroy_graph_div');
-var hasWebGLSupport = require('../assets/has_webgl_support');
 var mouseEvent = require('../assets/mouse_event');
 
 // mock with two dimensions (one panel); special case, e.g. left and right panel is obv. the same
@@ -222,8 +221,6 @@ describe('parcoords initialization tests', function() {
 
 describe('parcoords', function() {
 
-    if(!hasWebGLSupport('parcoords')) return;
-
     beforeAll(function() {
         mock.data[0].dimensions.forEach(function(d) {
             d.values = d.values.slice(lineStart, lineStart + lineCount);
diff --git a/test/jasmine/tests/scattermapbox_test.js b/test/jasmine/tests/scattermapbox_test.js
index 393bd4f5d27..ce41c3bb624 100644
--- a/test/jasmine/tests/scattermapbox_test.js
+++ b/test/jasmine/tests/scattermapbox_test.js
@@ -7,7 +7,6 @@ var convert = require('@src/traces/scattermapbox/convert');
 
 var createGraphDiv = require('../assets/create_graph_div');
 var destroyGraphDiv = require('../assets/destroy_graph_div');
-var hasWebGLSupport = require('../assets/has_webgl_support');
 var customMatchers = require('../assets/custom_matchers');
 
 Plotly.setPlotConfig({
@@ -455,8 +454,6 @@ describe('scattermapbox convert', function() {
 describe('scattermapbox hover', function() {
     'use strict';
 
-    if(!hasWebGLSupport('scattermapbox hover')) return;
-
     var hoverPoints = ScatterMapbox.hoverPoints;
 
     var gd;

From 100b993dc8427b68e8cb8327dc169e92d9098468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:17:32 -0500
Subject: [PATCH 04/12] rm unnecessary quotes in yml

---
 circle.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/circle.yml b/circle.yml
index 584761eefcb..f8ed1d33a2b 100644
--- a/circle.yml
+++ b/circle.yml
@@ -1,7 +1,7 @@
 general:
   artifacts:
-      - "build/test_images/"
-      - "build/test_images_diff/"
+    - build/test_images/
+    - build/test_images_diff/
 
 machine:
   node:

From 14621e122b5766e9a6360f0192cdd435cac40afd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:22:31 -0500
Subject: [PATCH 05/12] revamp circle.yml for ubuntu 14.04 with 2x parallel
 containers !!!

- add ci_tesh.sh that splits test command between 2 containers
- make `npm run lint` exit with exit code 1 if fails
- rm `eval ($node tasks/docker.js ---)` hack (no longer needed in 14.04)
- ping imagetest server from outside container,
  as pinging from inside was failling in 14.04
- add more robust `npm i` step
---
 circle.yml                       | 23 +++++++++--------------
 package.json                     |  4 ++--
 tasks/ci_test.sh                 | 22 ++++++++++++++++++++++
 tasks/docker.js                  | 11 ++---------
 tasks/test_export.js             |  4 +++-
 tasks/test_image.js              |  4 +++-
 tasks/util/container_commands.js |  2 --
 7 files changed, 41 insertions(+), 29 deletions(-)
 create mode 100755 tasks/ci_test.sh

diff --git a/circle.yml b/circle.yml
index f8ed1d33a2b..285ebd40ea8 100644
--- a/circle.yml
+++ b/circle.yml
@@ -12,21 +12,16 @@ machine:
     - docker
 
 dependencies:
-  pre:
-    - eval $(node tasks/docker.js pull)
-  post:
-    - eval $(node tasks/docker.js run)
-    - npm run cibuild
+  override:
+    - npm install && npm dedupe && npm prune && npm install
+    - npm ls || true
+    - npm run docker -- pull
     - npm run pretest
-    - eval $(node tasks/docker.js setup)
-    - npm prune && npm ls
+    - npm run docker -- run
+    - npm run cibuild
+    - npm run docker -- setup
 
 test:
   override:
-    - npm run test-image
-    - npm run test-image-gl2d
-    - npm run test-export
-    - npm run citest-jasmine
-    - npm run test-bundle
-    - npm run test-syntax
-    - eslint .
+    - ./tasks/ci_test.sh:
+        parallel: true
diff --git a/package.json b/package.json
index ff1be246dc6..6a8b21e7986 100644
--- a/package.json
+++ b/package.json
@@ -28,8 +28,8 @@
     "build": "npm run preprocess && npm run bundle && npm run header && npm run stats",
     "cibuild": "npm run preprocess && node tasks/cibundle.js",
     "watch": "node tasks/watch.js",
-    "lint": "eslint --version && eslint . || true",
-    "lint-fix": "eslint . --fix",
+    "lint": "eslint --version && eslint .",
+    "lint-fix": "eslint . --fix || true",
     "docker": "node tasks/docker.js",
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
diff --git a/tasks/ci_test.sh b/tasks/ci_test.sh
new file mode 100755
index 00000000000..e018bfdd044
--- /dev/null
+++ b/tasks/ci_test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+EXIT_STATE=0
+
+case $CIRCLE_NODE_INDEX in
+
+    0)
+        npm run test-image      || EXIT_STATE=$?
+        npm run test-image-gl2d || EXIT_STATE=$?
+        npm run test-export     || EXIT_STATE=$?
+        npm run test-syntax     || EXIT_STATE=$?
+        npm run lint            || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
+    1)
+        npm run citest-jasmine || EXIT_STATE=$?
+        npm run test-bundle    || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
+esac
diff --git a/tasks/docker.js b/tasks/docker.js
index d3ba12a0131..a767caa587c 100644
--- a/tasks/docker.js
+++ b/tasks/docker.js
@@ -46,12 +46,5 @@ switch(arg) {
         break;
 }
 
-// Log command string on CircleCI, to then `eval` them,
-// which appears to be more reliable then calling `child_process.exec()`
-if(isCI) {
-    console.log(cmd);
-}
-else {
-    console.log(msg);
-    common.execCmd(cmd, cb, errorCb);
-}
+console.log(msg);
+common.execCmd(cmd, cb, errorCb);
diff --git a/tasks/test_export.js b/tasks/test_export.js
index de08f97e7e7..d747d818b6e 100644
--- a/tasks/test_export.js
+++ b/tasks/test_export.js
@@ -14,4 +14,6 @@ var cmd = containerCommands.getRunCmd(
 );
 
 console.log(msg);
-common.execCmd(cmd);
+common.execCmd(containerCommands.ping, function() {
+    common.execCmd(cmd);
+});
diff --git a/tasks/test_image.js b/tasks/test_image.js
index 8156cabaead..eed7ccd9148 100644
--- a/tasks/test_image.js
+++ b/tasks/test_image.js
@@ -14,4 +14,6 @@ var cmd = containerCommands.getRunCmd(
 );
 
 console.log(msg);
-common.execCmd(cmd);
+common.execCmd(containerCommands.ping, function() {
+    common.execCmd(cmd);
+});
diff --git a/tasks/util/container_commands.js b/tasks/util/container_commands.js
index 11743613172..938cf694993 100644
--- a/tasks/util/container_commands.js
+++ b/tasks/util/container_commands.js
@@ -22,8 +22,6 @@ containerCommands.setup = [
     containerCommands.injectEnv,
     containerCommands.restart,
     'sleep 1',
-    containerCommands.ping,
-    'echo '
 ].join(' && ');
 
 containerCommands.dockerRun = [

From b42f021e691f14c5a8448a83e9ee5b704f5e805e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:25:47 -0500
Subject: [PATCH 06/12] use `karma-jasmine-spec-tags` to skip a few test on CI

- unfortunately, still some test need to be skipped on CI
  e.g. mapbox-gl takes too much resources ant blows up the run
  and similarly for some parcoords tests.
---
 package.json                             |  1 +
 test/jasmine/karma.conf.js               | 11 ++++++++++-
 test/jasmine/tests/gl2d_click_test.js    |  2 +-
 test/jasmine/tests/mapbox_test.js        |  2 +-
 test/jasmine/tests/parcoords_test.js     |  2 +-
 test/jasmine/tests/scattermapbox_test.js |  2 +-
 6 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 6a8b21e7986..a46654e1b46 100644
--- a/package.json
+++ b/package.json
@@ -119,6 +119,7 @@
     "karma-coverage": "^1.0.0",
     "karma-firefox-launcher": "^1.0.0",
     "karma-jasmine": "^1.1.0",
+    "karma-jasmine-spec-tags": "^1.0.1",
     "madge": "^1.6.0",
     "node-sass": "^4.5.0",
     "npm-link-check": "^1.2.0",
diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js
index b54df0708f7..d485d098e50 100644
--- a/test/jasmine/karma.conf.js
+++ b/test/jasmine/karma.conf.js
@@ -54,7 +54,7 @@ func.defaultConfig = {
 
     // frameworks to use
     // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
-    frameworks: ['jasmine', 'browserify'],
+    frameworks: ['jasmine', 'jasmine-spec-tags', 'browserify'],
 
     // list of files / patterns to load in the browser
     //
@@ -119,6 +119,15 @@ func.defaultConfig = {
         extensions: ['.js'],
         watch: !isCI,
         debug: true
+
+    // unfortunately a few tests don't behave well on CI
+    // using `karma-jasmine-spec-tags`
+    // add @noCI to the spec description to skip a spec on CI
+    client: {
+        tagPrefix: '@',
+        skipTags: isCI ? 'noCI' : null
+    },
+
     }
 };
 
diff --git a/test/jasmine/tests/gl2d_click_test.js b/test/jasmine/tests/gl2d_click_test.js
index e170b43fe2a..a7d679a09ed 100644
--- a/test/jasmine/tests/gl2d_click_test.js
+++ b/test/jasmine/tests/gl2d_click_test.js
@@ -16,7 +16,7 @@ Plotly.register([
     require('@lib/contourgl')
 ]);
 
-describe('Test hover and click interactions', function() {
+describe('@noCI Test hover and click interactions', function() {
 
     var mock = require('@mocks/gl2d_14.json');
     var mock2 = require('@mocks/gl2d_pointcloud-basic.json');
diff --git a/test/jasmine/tests/mapbox_test.js b/test/jasmine/tests/mapbox_test.js
index c490b539257..d494573ecb2 100644
--- a/test/jasmine/tests/mapbox_test.js
+++ b/test/jasmine/tests/mapbox_test.js
@@ -276,7 +276,7 @@ describe('mapbox credentials', function() {
     }, LONG_TIMEOUT_INTERVAL);
 });
 
-describe('mapbox plots', function() {
+describe('@noCI, mapbox plots', function() {
     'use strict';
 
     var mock = require('@mocks/mapbox_0.json'),
diff --git a/test/jasmine/tests/parcoords_test.js b/test/jasmine/tests/parcoords_test.js
index e06d304c603..6aa01532634 100644
--- a/test/jasmine/tests/parcoords_test.js
+++ b/test/jasmine/tests/parcoords_test.js
@@ -219,7 +219,7 @@ describe('parcoords initialization tests', function() {
     });
 });
 
-describe('parcoords', function() {
+describe('@noCI parcoords', function() {
 
     beforeAll(function() {
         mock.data[0].dimensions.forEach(function(d) {
diff --git a/test/jasmine/tests/scattermapbox_test.js b/test/jasmine/tests/scattermapbox_test.js
index ce41c3bb624..716af5581a3 100644
--- a/test/jasmine/tests/scattermapbox_test.js
+++ b/test/jasmine/tests/scattermapbox_test.js
@@ -451,7 +451,7 @@ describe('scattermapbox convert', function() {
     }
 });
 
-describe('scattermapbox hover', function() {
+describe('@noCI scattermapbox hover', function() {
     'use strict';
 
     var hoverPoints = ScatterMapbox.hoverPoints;

From 8d7171ff07fed768eaa33287292959836bbe9049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:26:52 -0500
Subject: [PATCH 07/12] add karma-verbose-reporter in dev deps

- which is very useful for debugging as discussed
  in CONTRIBUTING.md
---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index a46654e1b46..9851637b3fe 100644
--- a/package.json
+++ b/package.json
@@ -120,6 +120,7 @@
     "karma-firefox-launcher": "^1.0.0",
     "karma-jasmine": "^1.1.0",
     "karma-jasmine-spec-tags": "^1.0.1",
+    "karma-verbose-reporter": "0.0.6",
     "madge": "^1.6.0",
     "node-sass": "^4.5.0",
     "npm-link-check": "^1.2.0",

From b86bf069f24f4cb57d04f0b0455b30d14025597e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:30:17 -0500
Subject: [PATCH 08/12] add `karma-spec-reporter` to log info about skipped
 specs

- so that the CI logs show which tests were skipped
- use with 'dots' reporter when not targeting an individual suite
  for cleaner output.
---
 package.json               |  1 +
 test/jasmine/karma.conf.js | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 9851637b3fe..c168da608b1 100644
--- a/package.json
+++ b/package.json
@@ -120,6 +120,7 @@
     "karma-firefox-launcher": "^1.0.0",
     "karma-jasmine": "^1.1.0",
     "karma-jasmine-spec-tags": "^1.0.1",
+    "karma-spec-reporter": "0.0.30",
     "karma-verbose-reporter": "0.0.6",
     "madge": "^1.6.0",
     "node-sass": "^4.5.0",
diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js
index d485d098e50..9412e30666c 100644
--- a/test/jasmine/karma.conf.js
+++ b/test/jasmine/karma.conf.js
@@ -77,7 +77,7 @@ func.defaultConfig = {
     // See note in CONTRIBUTING.md about more verbose reporting via karma-verbose-reporter:
     // https://www.npmjs.com/package/karma-verbose-reporter ('verbose')
     //
-    reporters: ['progress'],
+    reporters: isSingleSuiteRun ? ['progress'] : ['dots', 'spec'],
 
     // web server port
     port: 9876,
@@ -119,6 +119,7 @@ func.defaultConfig = {
         extensions: ['.js'],
         watch: !isCI,
         debug: true
+    },
 
     // unfortunately a few tests don't behave well on CI
     // using `karma-jasmine-spec-tags`
@@ -128,6 +129,14 @@ func.defaultConfig = {
         skipTags: isCI ? 'noCI' : null
     },
 
+    // use 'karma-spec-reporter' to log info about skipped specs
+    specReporter: {
+        suppressErrorSummary: true,
+        suppressFailed: true,
+        suppressPassed: true,
+        suppressSkipped: false,
+        showSpecTiming: false,
+        failFast: false
     }
 };
 

From 631533d20eac0704bdd382f8a84db955aa013bfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:31:29 -0500
Subject: [PATCH 09/12] tweak precision to make tests pass on CI w/ Chrome 54

---
 test/jasmine/tests/annotations_test.js | 7 +++++--
 test/jasmine/tests/hover_label_test.js | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/jasmine/tests/annotations_test.js b/test/jasmine/tests/annotations_test.js
index d6701bd1f71..967b31323eb 100644
--- a/test/jasmine/tests/annotations_test.js
+++ b/test/jasmine/tests/annotations_test.js
@@ -514,8 +514,8 @@ describe('annotations autorange', function() {
 
     function assertRanges(x, y, x2, y2, x3, y3) {
         var fullLayout = gd._fullLayout;
-        var PREC = 1;
 
+        var PREC = 1;
         // xaxis2 need a bit more tolerance to pass on CI
         // this most likely due to the different text bounding box values
         // on headfull vs headless browsers.
@@ -523,6 +523,9 @@ describe('annotations autorange', function() {
         var PRECX2 = -10;
         // yaxis2 needs a bit more now too...
         var PRECY2 = 0.2;
+        // and xaxis3 too...
+        var PRECX3 = 0.2;
+
         var dateAx = fullLayout.xaxis2;
 
         expect(fullLayout.xaxis.range).toBeCloseToArray(x, PREC, '- xaxis');
@@ -530,7 +533,7 @@ describe('annotations autorange', function() {
         expect(Lib.simpleMap(dateAx.range, dateAx.r2l))
             .toBeCloseToArray(Lib.simpleMap(x2, dateAx.r2l), PRECX2, 'xaxis2 ' + dateAx.range);
         expect(fullLayout.yaxis2.range).toBeCloseToArray(y2, PRECY2, 'yaxis2');
-        expect(fullLayout.xaxis3.range).toBeCloseToArray(x3, PREC, 'xaxis3');
+        expect(fullLayout.xaxis3.range).toBeCloseToArray(x3, PRECX3, 'xaxis3');
         expect(fullLayout.yaxis3.range).toBeCloseToArray(y3, PREC, 'yaxis3');
     }
 
diff --git a/test/jasmine/tests/hover_label_test.js b/test/jasmine/tests/hover_label_test.js
index ebfd193ce8c..b8830036e8e 100644
--- a/test/jasmine/tests/hover_label_test.js
+++ b/test/jasmine/tests/hover_label_test.js
@@ -756,8 +756,8 @@ describe('hover on fill', function() {
                 var transformParts = hoverText.attr('transform').split('(');
                 expect(transformParts[0]).toEqual('translate');
                 var transformCoords = transformParts[1].split(')')[0].split(',');
-                expect(+transformCoords[0]).toBeCloseTo(labelPos[0], -1, labelText + ':x');
-                expect(+transformCoords[1]).toBeCloseTo(labelPos[1], -1, labelText + ':y');
+                expect(+transformCoords[0]).toBeCloseTo(labelPos[0], -1.2, labelText + ':x');
+                expect(+transformCoords[1]).toBeCloseTo(labelPos[1], -1.2, labelText + ':y');
 
                 resolve();
             }, constants.HOVERMINTIME);

From dbc6ca133818c216015541820c5624c0bb11cdf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:33:23 -0500
Subject: [PATCH 10/12] lock uglify-js to 2.7.x

- the new 2.8.x series breaks the `cwise` transform
- see https://github.com/plotly/plotly.js/pull/1450#issuecomment-284803443
- maybe we should make uglify-js a core dependencies then?
---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index c168da608b1..ab83fee5bf4 100644
--- a/package.json
+++ b/package.json
@@ -131,7 +131,7 @@
     "read-last-lines": "^1.1.0",
     "requirejs": "^2.3.1",
     "through2": "^2.0.3",
-    "uglify-js": "^2.7.5",
+    "uglify-js": "~2.7.5",
     "watchify": "^3.9.0",
     "xml2js": "^0.4.16"
   }

From e52f0208cf6144b332afd641b87f120650be15c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 09:34:21 -0500
Subject: [PATCH 11/12] add no_citest script

- to make it easier to run tests that are skipped on CI
  and easier to document.
---
 tasks/noci_test.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100755 tasks/noci_test.sh

diff --git a/tasks/noci_test.sh b/tasks/noci_test.sh
new file mode 100755
index 00000000000..46d0c779e43
--- /dev/null
+++ b/tasks/noci_test.sh
@@ -0,0 +1,16 @@
+#! /bin/bash
+
+EXIT_STATE=0
+
+# tests that aren't run on CI
+
+# jasmine specs with @noCI tag
+npm run citest-jasmine --  tests/*_test.js --tags noCI || EXIT_STATE=$?
+
+# mapbox image tests take too much resources on CI
+npm run test-image -- mapbox_* || EXIT_STATE=$?
+
+# run gl2d image test again (some mocks are skipped on CI)
+npm run test-image-gl2d || EXIT_STATE=$?
+
+exit $EXIT_STATE

From 33ae9639083af234a55002e75cd12d1162967f8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= <etienne@plot.ly>
Date: Thu, 9 Mar 2017 17:24:33 -0500
Subject: [PATCH 12/12] use (plain) npm run test-jasmine on CI

- no need to set the CIRCLECI env variable again.
---
 package.json     | 2 +-
 tasks/ci_test.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index ab83fee5bf4..d5399661250 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
     "docker": "node tasks/docker.js",
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
-    "citest-jasmine": "CIRCLECI=0 karma start test/jasmine/karma.conf.js",
+    "citest-jasmine": "CIRCLECI=1 karma start test/jasmine/karma.conf.js",
     "test-image": "node tasks/test_image.js",
     "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue",
     "test-export": "node tasks/test_export.js",
diff --git a/tasks/ci_test.sh b/tasks/ci_test.sh
index e018bfdd044..5e2b7d40a95 100755
--- a/tasks/ci_test.sh
+++ b/tasks/ci_test.sh
@@ -14,7 +14,7 @@ case $CIRCLE_NODE_INDEX in
         ;;
 
     1)
-        npm run citest-jasmine || EXIT_STATE=$?
+        npm run test-jasmine || EXIT_STATE=$?
         npm run test-bundle    || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;