Merge pull request #3634 from plotly/circleci-parallel

CircleCI: run jobs in parallel

Author: antoinerg

.circleci/config.yml

@@ -40,6 +40,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -52,6 +53,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -75,6 +77,7 @@ jobs:
   test-image:
     docker:
       - image: plotly/testbed:latest
+    parallelism: 4
     working_directory: /var/www/streambed/image_server/plotly.js/
     steps:
       - attach_workspace:
@@ -121,6 +124,18 @@ jobs:
           name: Run syntax tests
           command: ./.circleci/test.sh syntax
 
+  test-bundle:
+    docker:
+      # need '-browsers' version to test in real (xvfb-wrapped) browsers
+      - image: circleci/node:10.9.0-browsers
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Run test-bundle
+          command: ./.circleci/test.sh bundle
+
   publish:
     docker:
       - image: circleci/node:10.9.0
@@ -157,6 +172,9 @@ workflows:
   build-and-test:
     jobs:
       - build
+      - test-bundle:
+          requires:
+            - build
       - test-jasmine:
           requires:
             - build

.circleci/test.sh

@@ -44,19 +44,19 @@ case $1 in
     jasmine)
         set_tz
 
-        npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
-        npm run test-bundle || EXIT_STATE=$?
+        SUITE=$(circleci tests glob "$ROOT/test/jasmine/tests/*" | circleci tests split)
+        npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky --showSkipped || EXIT_STATE=$?
 
         exit $EXIT_STATE
         ;;
 
     jasmine2)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
@@ -65,23 +65,29 @@ case $1 in
     jasmine3)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
         ;;
 
     image)
-        npm run test-image      || EXIT_STATE=$?
+        SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split)
+        npm run test-image -- $SUITE --filter || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     image2)
         npm run test-export     || EXIT_STATE=$?
-        npm run test-image-gl2d || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
+    bundle)
+        set_tz
+        npm run test-bundle || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 

package.json

@@ -35,11 +35,10 @@
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
     "test-image": "node tasks/test_image.js",
-    "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue",
     "test-export": "node tasks/test_export.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",
-    "test": "npm run test-jasmine && npm run test-bundle && npm run test-image && npm run test-image-gl2d && npm run test-syntax && npm run lint",
+    "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
     "start-test_dashboard": "node devtools/test_dashboard/server.js",
     "start-image_viewer": "node devtools/image_viewer/server.js",
     "start": "npm run start-test_dashboard",

test/image/compare_pixels_test.js

@@ -1,4 +1,5 @@
 var fs = require('fs');
+var minimist = require('minimist');
 
 var common = require('../../tasks/util/common');
 var getMockList = require('./assets/get_mock_list');
@@ -48,45 +49,57 @@ var QUEUE_WAIT = 10;
  *      npm run test-image -- gl3d_* --queue
  */
 
-var pattern = process.argv[2];
-var mockList = getMockList(pattern);
-var isInQueue = (process.argv[3] === '--queue');
+var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]});
+var isInQueue = argv.queue;
+var filter = argv.filter;
 
-if(mockList.length === 0) {
-    throw new Error('No mocks found with pattern ' + pattern);
+var allMock = false;
+// If no pattern is provided, all mocks are compared
+if(argv._.length === 0) {
+    allMock = true;
+    argv._.push('');
 }
 
-// filter out untestable mocks if no pattern is specified
-if(!pattern) {
+// Build list of mocks to compare
+var allMockList = [];
+argv._.forEach(function(pattern) {
+    var mockList = getMockList(pattern);
+
+    if(mockList.length === 0) {
+        throw new Error('No mocks found with pattern ' + pattern);
+    }
+
+    allMockList = allMockList.concat(mockList);
+});
+
+// To get rid of duplicates
+function unique(value, index, self) {
+    return self.indexOf(value) === index;
+}
+allMockList = allMockList.filter(unique);
+
+// filter out untestable mocks if no pattern is specified (ie. we're testing all mocks)
+// or if flag '--filter' is provided
+if(allMock || filter) {
     console.log('Filtering out untestable mocks:');
-    mockList = mockList.filter(untestableFilter);
+    allMockList = allMockList.filter(untestableFilter);
     console.log('\n');
 }
 
-// gl2d have limited image-test support
-if(pattern === 'gl2d_*') {
-    if(!isInQueue) {
-        console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n');
-    }
-    console.log('\nSorting gl2d mocks to avoid gl-shader conflicts');
-    sortGl2dMockList(mockList);
-    console.log('');
-}
+sortGl2dMockList(allMockList);
 
 // main
 if(isInQueue) {
-    runInQueue(mockList);
+    runInQueue(allMockList);
 }
 else {
-    runInBatch(mockList);
+    runInBatch(allMockList);
 }
 
 /* Test cases:
  *
  * - font-wishlist
- * - all gl2d
  * - all mapbox
- * - gl3d_cone-*
  *
  * don't behave consistently from run-to-run and/or
  * machine-to-machine; skip over them for now.
@@ -95,7 +108,6 @@ else {
 function untestableFilter(mockName) {
     var cond = !(
         mockName === 'font-wishlist' ||
-        mockName.indexOf('gl2d_') !== -1 ||
         mockName.indexOf('mapbox_') !== -1
     );
 

test/jasmine/karma.conf.js

@@ -59,6 +59,7 @@ if(argv.info) {
         '  - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode',
         '  - `--failFast` (dflt: `false`): exit karma upon first test failure',
         '  - `--verbose` (dflt: `false`): show test result using verbose reporter',
+        '  - `--showSkipped` (dflt: `false`): show tests that are skipped',
         '  - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)',
         '  - `--width`(dflt: 1035): set width of the browser window',
         '  - `--height` (dflt: 617): set height of the browser window',
@@ -111,7 +112,7 @@ var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js');
 var pathToUnpolyfill = path.join(__dirname, 'assets', 'unpolyfill.js');
 var pathToMathJax = path.join(constants.pathToDist, 'extras', 'mathjax');
 
-var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'];
+var reporters = ((isFullSuite && !argv.tags) || argv.showSkipped) ? ['dots', 'spec'] : ['progress'];
 if(argv.failFast) reporters.push('fail-fast');
 if(argv.verbose) reporters.push('verbose');
 

test/jasmine/tests/sankey_test.js

@@ -1054,7 +1054,7 @@ describe('sankey tests', function() {
         });
 
         ['node', 'link'].forEach(function(obj) {
-            it('should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
+            it('@flaky should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
                 var fig = Lib.extendDeep({}, mock);
 
                 Plotly.plot(gd, fig)