double-check images that fail

Author: antoinerg

.circleci/orca-build-verify.sh

@@ -7,6 +7,17 @@ BASELINES=$ROOT/test/image/baselines
 TEST_IMAGES=$ROOT/build/test_images
 DIFF_IMAGES=$ROOT/build/test_images_diff
 
+get_mock_path()
+{
+  echo "$MOCKS/$1.json"
+}
+export -f get_mock_path
+
+get_image_path()
+{
+  echo "$TEST_IMAGES/$1.png"
+}
+
 # Deterministic shuffling (https://www.gnu.org/software/coreutils/manual/html_node/Random-sources.html)
 get_seeded_random()
 {
@@ -31,39 +42,58 @@ else
   CIRCLE_NODE_INDEX=0
 fi
 
+rm -f /tmp/all
 if [[ $# -eq 0 ]]; then
-  echo "No arguments provided"
-  FIGURE=$(ls $MOCKS/*.json)
+  ls $MOCKS/*.json | xargs -I{} -n1 basename {} .json > /tmp/all
 else
   for var in "$@"
   do
-      FIGURE="$FIGURE $MOCKS/$var.json "
+    echo "$var" >> /tmp/all
   done
 fi
 
-echo $FIGURE | tr " " "\n" | deterministic_shuffle > /tmp/all
-split -d -a3 -n l/$CIRCLE_NODE_TOTAL /tmp/all /tmp/queue
+cat /tmp/all | \
+  # Shuffle to distribute randomly slow and fast mocks
+  deterministic_shuffle > /tmp/shuffled_all
 
+# Split on each node
+split -d -a3 -n l/$CIRCLE_NODE_TOTAL /tmp/shuffled_all /tmp/queue
 NODE_QUEUE="/tmp/queue$(printf "%03d" $CIRCLE_NODE_INDEX)"
 
 echo ""
-echo "Generating test images"
-cat $NODE_QUEUE | awk '!/mapbox/' | \
-    # Shuffle to distribute randomly slow and fast mocks
-    deterministic_shuffle | \
-    # head -n 10 | \
-    # Split in chunks of 20
-    xargs -P1 -n20 xvfb-run -a orca graph --verbose --output-dir $TEST_IMAGES
+function generate()
+{
+  echo "Generating test images"
+  cat $1 | awk '!/mapbox/' | \
+      xargs -n1 -I{} echo "$MOCKS/{}.json" | \
+      # Split in chunks of 20
+      xargs -P1 -n20 xvfb-run -a orca graph --verbose --output-dir $TEST_IMAGES
+}
+generate $NODE_QUEUE
 
-echo "Comparing with baselines"
-find $TEST_IMAGES -type f -name "*.png" -printf "%f\n" | \
-  # shuf | \
-  # sort | \
-  # head -n 20 | \
-  # xargs -n1 -P16 -I {} bash -c "echo {} && ./node_modules/.bin/pixelmatch $1/{} $2/{} diff/{} 0 true" | tee results.txt
-  xargs -n1 -P`nproc` -I {} bash -c "compare -verbose -metric AE $TEST_IMAGES/{} $BASELINES/{} $DIFF_IMAGES/{} 2> $DIFF_IMAGES/{}.txt"
+function compare()
+{
+  echo "Comparing with baselines"
+  cat $1 | awk '!/mapbox/' | \
+  #find $TEST_IMAGES -type f -name "*.png" -printf "%f\n" | \
+    # shuf | \
+    # sort | \
+    # head -n 20 | \
+    # xargs -n1 -P16 -I {} bash -c "echo {} && ./node_modules/.bin/pixelmatch $1/{} $2/{} diff/{} 0 true" | tee results.txt
+    xargs -n1 -P`nproc` -I {} bash -c "compare -verbose -metric AE $TEST_IMAGES/{} $BASELINES/{} $DIFF_IMAGES/{} 2> $DIFF_IMAGES/{}.txt"
+}
+compare $NODE_QUEUE
 
 CODE=$(grep -R "all: [^0]" $DIFF_IMAGES/ | wc -l)
-grep -l -R "all: [^0]" $DIFF_IMAGES/ | gawk '{match($0, /diff\/([^.]*)/, arr); print arr[1]}'
+
+# If there are different images, retry
+if [ "$CODE" -ne "0" ]; then
+  grep -l -R "all: [^0]" $DIFF_IMAGES/ | gawk '{match($0, /diff\/([^.]*)/, arr); print arr[1]}' > /tmp/failed
+  echo "Retrying the following $CODE images"
+  cat /tmp/failed
+  generate /tmp/failed
+  compare /tmp/failed
+  CODE=$(grep -R "all: [^0]" $DIFF_IMAGES/ | wc -l)
+fi
 echo "$CODE different images"
 exit $CODE