From 165aa0ceb040b6ab602d08fae54b544d7aad09eb Mon Sep 17 00:00:00 2001
From: etienne <etienne@plot.ly>
Date: Thu, 15 Feb 2018 12:41:06 -0500
Subject: [PATCH 1/2] add auto-retry logic to ci test scripts

---
 .circleci/test.sh | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/.circleci/test.sh b/.circleci/test.sh
index 148d18d19b9..c8eaa3050c4 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -5,17 +5,36 @@ set +e
 set +o pipefail
 
 EXIT_STATE=0
+MAX_AUTO_RETRY=5
+
+# inspired by https://unix.stackexchange.com/a/82602
+retry () {
+    local n=0
+
+    until [ $n -ge $MAX_AUTO_RETRY ]; do
+        "$@" && break
+        n=$[$n+1]
+        echo ''
+        echo run $n of $MAX_AUTO_RETRY failed, trying again ...
+        echo ''
+        sleep 15
+    done
+
+    if [ $n -eq $MAX_AUTO_RETRY ]; then
+        EXIT_STATE=1
+    fi
+}
 
 case $1 in
 
     jasmine)
-        npm run test-jasmine -- --skip-tags=gl,noCI || EXIT_STATE=$?
+        retry npm run test-jasmine -- --skip-tags=gl,noCI
         exit $EXIT_STATE
         ;;
 
     jasmine2)
-        npm run test-jasmine -- --tags=gl --skip-tags=noCI || EXIT_STATE=$?
-        npm run test-bundle                                || EXIT_STATE=$?
+        retry npm run test-jasmine -- --tags=gl --skip-tags=noCI
+        retry npm run test-bundle
         exit $EXIT_STATE
         ;;
 

From e298acf800662314f851b2fbd65c67dc4c561f26 Mon Sep 17 00:00:00 2001
From: etienne <etienne@plot.ly>
Date: Fri, 16 Feb 2018 12:27:13 -0500
Subject: [PATCH 2/2] add @flaky tags to flaky jasmine tests, run them in retry
 loop on CI

---
 .circleci/test.sh                             | 7 ++++---
 test/jasmine/tests/animate_test.js            | 2 +-
 test/jasmine/tests/gl2d_plot_interact_test.js | 2 +-
 test/jasmine/tests/parcoords_test.js          | 2 +-
 test/jasmine/tests/select_test.js             | 2 +-
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.circleci/test.sh b/.circleci/test.sh
index c8eaa3050c4..c2e82b26442 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -28,13 +28,14 @@ retry () {
 case $1 in
 
     jasmine)
-        retry npm run test-jasmine -- --skip-tags=gl,noCI
+        npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     jasmine2)
-        retry npm run test-jasmine -- --tags=gl --skip-tags=noCI
-        retry npm run test-bundle
+        npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky || EXIT_STATE=$?
+        retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
+        npm run test-bundle || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
diff --git a/test/jasmine/tests/animate_test.js b/test/jasmine/tests/animate_test.js
index a8dca8d0b53..4c43332de3a 100644
--- a/test/jasmine/tests/animate_test.js
+++ b/test/jasmine/tests/animate_test.js
@@ -720,7 +720,7 @@ describe('Animating multiple axes', function() {
         destroyGraphDiv();
     });
 
-    it('updates ranges of secondary axes', function(done) {
+    it('@flaky updates ranges of secondary axes', function(done) {
         Plotly.plot(gd, [
             {y: [1, 2, 3]},
             {y: [1, 2, 3], yaxis: 'y2'}
diff --git a/test/jasmine/tests/gl2d_plot_interact_test.js b/test/jasmine/tests/gl2d_plot_interact_test.js
index 8c1063f3c3d..af33ca0e2e1 100644
--- a/test/jasmine/tests/gl2d_plot_interact_test.js
+++ b/test/jasmine/tests/gl2d_plot_interact_test.js
@@ -250,7 +250,7 @@ describe('@gl Test gl2d plots', function() {
         });
     }
 
-    it('should respond to drag interactions', function(done) {
+    it('@flaky should respond to drag interactions', function(done) {
         var _mock = Lib.extendDeep({}, mock);
 
         var relayoutCallback = jasmine.createSpy('relayoutCallback');
diff --git a/test/jasmine/tests/parcoords_test.js b/test/jasmine/tests/parcoords_test.js
index cebffb84195..a1dc36f1130 100644
--- a/test/jasmine/tests/parcoords_test.js
+++ b/test/jasmine/tests/parcoords_test.js
@@ -774,7 +774,7 @@ describe('@gl parcoords', function() {
 
         });
 
-        it('Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) {
+        it('@flaky Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) {
             Plotly.newPlot(gd, [{
                 type: 'parcoords',
                 line: {color: 'blue'},
diff --git a/test/jasmine/tests/select_test.js b/test/jasmine/tests/select_test.js
index 7d2af86b3e0..eb92824a06a 100644
--- a/test/jasmine/tests/select_test.js
+++ b/test/jasmine/tests/select_test.js
@@ -107,7 +107,7 @@ var BOXEVENTS = [1, 2, 1];
 // assumes 5 points in the lasso path
 var LASSOEVENTS = [4, 2, 1];
 
-describe('Test select box and lasso in general:', function() {
+describe('@flaky Test select box and lasso in general:', function() {
     var mock = require('@mocks/14.json');
     var selectPath = [[93, 193], [143, 193]];
     var lassoPath = [[316, 171], [318, 239], [335, 243], [328, 169]];