add baseline and jasmine tests using virtual-webgl

Author: archmoj

.circleci/config.yml

@@ -122,6 +122,26 @@ jobs:
           name: Run jasmine tests (part B)
           command: .circleci/test.sh webgl-jasmine
 
+  virtual-webgl-jasmine:
+    docker:
+      # need '-browsers' version to test in real (xvfb-wrapped) browsers
+      - image: cimg/node:16.17.1-browsers
+    environment:
+      # Alaska time (arbitrary timezone to test date logic)
+      TZ: "America/Anchorage"
+    parallelism: 8
+    working_directory: ~/plotly.js
+    steps:
+      - browser-tools/install-browser-tools: &browser-versions
+          chrome-version: 110.0.5481.100
+          install-firefox: false
+          install-geckodriver: false
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Run jasmine tests (part B)
+          command: .circleci/test.sh virtual-webgl-jasmine
+
   flaky-no-gl-jasmine:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
@@ -216,6 +236,25 @@ jobs:
           name: Test MathJax on firefox-latest
           command: .circleci/test.sh mathjax-firefox82+
 
+  make-baselines-virtual-webgl:
+    parallelism: 2
+    docker:
+      - image: circleci/python:3.8.9
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Install kaleido, plotly.io and required fonts
+          command: .circleci/env_image.sh
+      - run:
+          name: Create png files using virtual-webgl & WebGL v1
+          command: .circleci/test.sh make-baselines-virtual-webgl
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - plotly.js
+
   make-baselines-mathjax3:
     docker:
       - image: circleci/python:3.8.9
@@ -267,6 +306,20 @@ jobs:
           path: build
           destination: /
 
+  test-baselines-virtual-webgl:
+    docker:
+      - image: circleci/node:16.9.0
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Compare pixels
+          command: .circleci/test.sh test-image-virtual-webgl ; find build -maxdepth 1 -type f -delete
+      - store_artifacts:
+          path: build
+          destination: /
+
   test-baselines-mathjax3:
     docker:
       - image: circleci/node:16.9.0
@@ -441,9 +494,18 @@ workflows:
       - webgl-jasmine:
           requires:
             - install-and-cibuild
+      - virtual-webgl-jasmine:
+          requires:
+            - install-and-cibuild
       - flaky-no-gl-jasmine:
           requires:
             - install-and-cibuild
+      - make-baselines-virtual-webgl:
+          requires:
+            - install-and-cibuild
+      - test-baselines-virtual-webgl:
+          requires:
+            - make-baselines-virtual-webgl
       - make-baselines-mathjax3:
           requires:
             - install-and-cibuild

.circleci/test.sh

@@ -53,6 +53,16 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
+    virtual-webgl-jasmine)
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --limit=5 --tag=gl | circleci tests split))
+        for s in ${SHARDS[@]}; do
+            MAX_AUTO_RETRY=2
+            retry ./node_modules/karma/bin/karma start test/jasmine/karma.conf.js --virtualWebgl --tags=gl --skip-tags=noCI,noVirtualWebgl --doNotFailOnEmptyTestSuite -- "$s"
+        done
+
+        exit $EXIT_STATE
+        ;;
+
     flaky-no-gl-jasmine)
         SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --limit=1 --tag=flaky | circleci tests split))
 
@@ -82,6 +92,15 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
+    make-baselines-virtual-webgl)
+        SUITE=$({\
+                  find $ROOT/test/image/mocks/gl*     -type f -printf "%f\n"; \
+                  find $ROOT/test/image/mocks/mapbox* -type f -printf "%f\n"; \
+                } | sed 's/\.json$//1' | circleci tests split)
+        python3 test/image/make_baseline.py virtual-webgl $SUITE || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
     make-baselines-mathjax3)
         python3 test/image/make_baseline.py mathjax3    legend_mathjax_title_and_items mathjax parcats_grid_subplots table_latex_multitrace_scatter table_plain_birds table_wrapped_birds ternary-mathjax || EXIT_STATE=$?
         exit $EXIT_STATE
@@ -103,6 +122,11 @@ case $1 in
         exit $EXIT_STATE
         ;;
 
+    test-image-virtual-webgl)
+        node test/image/compare_pixels_test.js virtual-webgl || { tar -cvf build/baselines.tar build/test_images/*.png ; exit 1 ; } || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
     source-syntax)
         npm run lint        || EXIT_STATE=$?
         npm run test-syntax || EXIT_STATE=$?

test/image/compare_pixels_test.js

@@ -42,9 +42,13 @@ if(argv._.length === 0) {
 // Build list of mocks to compare
 var allMockList = [];
 var mathjax3;
+var virtualWebgl = false;
 argv._.forEach(function(pattern) {
     if(pattern === 'mathjax3') {
         mathjax3 = true;
+    } else if(pattern === 'virtual-webgl') {
+        virtualWebgl = true;
+        allMockList = getMockList('');
     } else {
         var mockList = getMockList(pattern);
 
@@ -66,6 +70,12 @@ allMockList = allMockList.filter(function(a) {
     );
 });
 
+if(virtualWebgl) {
+    allMockList = allMockList.filter(function(a) {
+        return a.slice(0, 2) === 'gl' || a.slice(0, 6) === 'mapbox';
+    });
+}
+
 if(mathjax3) {
     allMockList = [
         'legend_mathjax_title_and_items',
@@ -91,6 +101,7 @@ var fail = function(mockName) {
         failed.push(mockName);
     }
 };
+
 for(var i = 0; i < allMockList.length; i++) {
     var mockName = allMockList[i];
 
@@ -102,6 +113,18 @@ for(var i = 0; i < allMockList.length; i++) {
         continue;
     }
 
+    // the following mocks than only have regl-line2d lines (no morkers),
+    // do not render at first draw when using virtual webgl. TODO: find a fix
+    if(virtualWebgl && [
+        'gl2d_connect_gaps',
+        'gl2d_line_select',
+        'gl2d_lines_almost_horizontal_vertical',
+        'gl2d_scattergl_simple_line_reversed_ranges',
+        'gl2d_subplots_anchor'
+    ].indexOf(mockName) !== -1) {
+        continue;
+    }
+
     var isMapbox = mockName.substr(0, 7) === 'mapbox_';
     var isOtherFlaky = [
         // list flaky mocks other than mapbox:
@@ -151,15 +174,28 @@ for(var i = 0; i < allMockList.length; i++) {
 
     var shouldBePixelPerfect = !(isMapbox || isOtherFlaky);
 
+    var threshold = shouldBePixelPerfect ? 0 : [
+        // more flaky
+        'mapbox_angles',
+        'mapbox_layers',
+        'mapbox_custom-style',
+        'mapbox_geojson-attributes'
+    ].indexOf(mockName) !== -1 ? 1 : 0.15;
+
+    if(virtualWebgl) {
+        threshold = Math.max(0.4, threshold);
+        if([
+            'gl3d_ibm-plot',
+            'gl3d_isosurface_2surfaces-checker_spaceframe',
+            'gl3d_opacity-scaling-spikes',
+            'gl3d_cone-wind',
+            'gl3d_isosurface_math',
+            'gl3d_scatter3d-blank-text'
+        ].indexOf(mockName) !== -1) threshold = 0.7;
+    }
+
     var numDiffPixels = pixelmatch(img0.data, img1.data, diff.data, width, height, {
-        threshold: shouldBePixelPerfect ? 0 :
-            [
-                // more flaky
-                'mapbox_angles',
-                'mapbox_layers',
-                'mapbox_custom-style',
-                'mapbox_geojson-attributes'
-            ].indexOf(mockName) !== -1 ? 1 : 0.15
+        threshold: threshold
     });
 
     if(numDiffPixels) {

test/image/make_baseline.py

@@ -10,6 +10,11 @@
     args = sys.argv
 
 root = os.getcwd()
+
+virtual_webgl = os.path.join(root, 'node_modules', 'virtual-webgl', 'src', 'virtual-webgl.js')
+plotlyjs = os.path.join(root, 'build', 'plotly.js')
+plotlyjs_with_virtual_webgl = os.path.join(root, 'build', 'plotly_with_virtual-webgl.js')
+
 dirIn = os.path.join(root, 'test', 'image', 'mocks')
 dirOut = os.path.join(root, 'build', 'test_images')
 
@@ -33,8 +38,21 @@
     mathjax_version = 3
     print('Kaleido using MathJax v3')
 
+virtual_webgl_version = 0 # i.e. virtual-webgl is not used
+if 'virtual-webgl' in sys.argv or 'virtual-webgl=' in sys.argv :
+    virtual_webgl_version = 1
+    print('using virtual-webgl for WebGL v1')
+
+    with open(plotlyjs_with_virtual_webgl, 'w') as fileOut:
+        for filename in [virtual_webgl, plotlyjs]:
+            with open(filename, 'r') as fileIn:
+                for line in fileIn:
+                    fileOut.write(line)
+
+    plotlyjs = plotlyjs_with_virtual_webgl
+
+pio.kaleido.scope.plotlyjs = plotlyjs
 pio.templates.default = 'none'
-pio.kaleido.scope.plotlyjs = os.path.join(root, 'build', 'plotly.js')
 
 _credentials = open(os.path.join(root, 'build', 'credentials.json'), 'r')
 pio.kaleido.scope.mapbox_access_token = json.load(_credentials)['MAPBOX_ACCESS_TOKEN']

test/jasmine/karma.conf.js

@@ -127,6 +127,7 @@ var pathToUnpolyfill = path.join(__dirname, 'assets', 'unpolyfill.js');
 var pathToSaneTopojsonDist = path.join(__dirname, '..', '..', 'node_modules', 'sane-topojson', 'dist');
 var pathToMathJax2 = path.join(__dirname, '..', '..', 'node_modules', 'mathjax-v2');
 var pathToMathJax3 = path.join(__dirname, '..', '..', 'node_modules', 'mathjax-v3');
+var pathToVirtualWebgl = path.join(__dirname, '..', '..', 'node_modules', 'virtual-webgl', 'src', 'virtual-webgl.js');
 
 var reporters = [];
 if(argv['report-progress'] || argv['report-spec'] || argv['report-dots']) {
@@ -351,6 +352,13 @@ if(isBundleTest) {
     func.defaultConfig.files.push(pathToJQuery);
 }
 
+if(argv.virtualWebgl) {
+    // add virtual-webgl to the top
+    func.defaultConfig.files = [
+        pathToVirtualWebgl
+    ].concat(func.defaultConfig.files);
+}
+
 // lastly, load test file glob
 func.defaultConfig.files.push(testFileGlob);
 

test/jasmine/tests/gl3d_plot_interact_test.js

@@ -2156,7 +2156,7 @@ describe('Test removal of gl contexts', function() {
         .then(done, done.fail);
     });
 
-    it('@gl should fire *plotly_webglcontextlost* when on webgl context lost', function(done) {
+    it('@gl @noVirtualWebgl should fire *plotly_webglcontextlost* when on webgl context lost', function(done) {
         var _mock = Lib.extendDeep({}, require('../../image/mocks/gl3d_marker-arrays.json'));
 
         Plotly.newPlot(gd, _mock).then(function() {