|
| 1 | +#!/bin/bash |
| 2 | +# Copyright 2014 Cloudera, Inc. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | +# |
| 16 | +# Script which wraps running a test and redirects its output to a |
| 17 | +# test log directory. |
| 18 | + |
| 19 | +ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd) |
| 20 | + |
| 21 | +TEST_LOGDIR=$ROOT/build/test-logs |
| 22 | +mkdir -p $TEST_LOGDIR |
| 23 | + |
| 24 | +TEST_DEBUGDIR=$ROOT/build/test-debug |
| 25 | +mkdir -p $TEST_DEBUGDIR |
| 26 | + |
| 27 | +TEST_DIRNAME=$(cd $(dirname $1); pwd) |
| 28 | +TEST_FILENAME=$(basename $1) |
| 29 | +shift |
| 30 | +TEST_EXECUTABLE="$TEST_DIRNAME/$TEST_FILENAME" |
| 31 | +TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//') # Remove path and extension (if any). |
| 32 | + |
| 33 | +# We run each test in its own subdir to avoid core file related races. |
| 34 | +TEST_WORKDIR=$ROOT/build/test-work/$TEST_NAME |
| 35 | +mkdir -p $TEST_WORKDIR |
| 36 | +pushd $TEST_WORKDIR >/dev/null || exit 1 |
| 37 | +rm -f * |
| 38 | + |
| 39 | +set -o pipefail |
| 40 | + |
| 41 | +LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt |
| 42 | +XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml |
| 43 | + |
| 44 | +TEST_EXECUTION_ATTEMPTS=1 |
| 45 | + |
| 46 | +# Remove both the uncompressed output, so the developer doesn't accidentally get confused |
| 47 | +# and read output from a prior test run. |
| 48 | +rm -f $LOGFILE $LOGFILE.gz |
| 49 | + |
| 50 | +pipe_cmd=cat |
| 51 | + |
| 52 | +# Configure TSAN (ignored if this isn't a TSAN build). |
| 53 | +# |
| 54 | +# Deadlock detection (new in clang 3.5) is disabled because: |
| 55 | +# 1. The clang 3.5 deadlock detector crashes in some unit tests. It |
| 56 | +# needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others. |
| 57 | +# 2. Many unit tests report lock-order-inversion warnings; they should be |
| 58 | +# fixed before reenabling the detector. |
| 59 | +TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0" |
| 60 | +TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt" |
| 61 | +TSAN_OPTIONS="$TSAN_OPTIONS history_size=7" |
| 62 | +export TSAN_OPTIONS |
| 63 | + |
| 64 | +# Enable leak detection even under LLVM 3.4, where it was disabled by default. |
| 65 | +# This flag only takes effect when running an ASAN build. |
| 66 | +ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1" |
| 67 | +export ASAN_OPTIONS |
| 68 | + |
| 69 | +# Set up suppressions for LeakSanitizer |
| 70 | +LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt" |
| 71 | +export LSAN_OPTIONS |
| 72 | + |
| 73 | +# Suppressions require symbolization. We'll default to using the symbolizer in |
| 74 | +# thirdparty. |
| 75 | +if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then |
| 76 | + export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer) |
| 77 | +fi |
| 78 | + |
| 79 | +# Allow for collecting core dumps. |
| 80 | +PANDAS_TEST_ULIMIT_CORE=${PANDAS_TEST_ULIMIT_CORE:-0} |
| 81 | +ulimit -c $PANDAS_TEST_ULIMIT_CORE |
| 82 | + |
| 83 | +# Run the actual test. |
| 84 | +for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do |
| 85 | + if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then |
| 86 | + # If the test fails, the test output may or may not be left behind, |
| 87 | + # depending on whether the test cleaned up or exited immediately. Either |
| 88 | + # way we need to clean it up. We do this by comparing the data directory |
| 89 | + # contents before and after the test runs, and deleting anything new. |
| 90 | + # |
| 91 | + # The comm program requires that its two inputs be sorted. |
| 92 | + TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort) |
| 93 | + fi |
| 94 | + |
| 95 | + # gtest won't overwrite old junit test files, resulting in a build failure |
| 96 | + # even when retries are successful. |
| 97 | + rm -f $XMLFILE |
| 98 | + |
| 99 | + echo "Running $TEST_NAME, redirecting output into $LOGFILE" \ |
| 100 | + "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)" |
| 101 | + $TEST_EXECUTABLE "$@" 2>&1 \ |
| 102 | + | $ROOT/build-support/asan_symbolize.py \ |
| 103 | + | c++filt \ |
| 104 | + | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ |
| 105 | + | $pipe_cmd > $LOGFILE |
| 106 | + STATUS=$? |
| 107 | + |
| 108 | + # TSAN doesn't always exit with a non-zero exit code due to a bug: |
| 109 | + # mutex errors don't get reported through the normal error reporting infrastructure. |
| 110 | + # So we make sure to detect this and exit 1. |
| 111 | + # |
| 112 | + # Additionally, certain types of failures won't show up in the standard JUnit |
| 113 | + # XML output from gtest. We assume that gtest knows better than us and our |
| 114 | + # regexes in most cases, but for certain errors we delete the resulting xml |
| 115 | + # file and let our own post-processing step regenerate it. |
| 116 | + export GREP=$(which egrep) |
| 117 | + if zgrep --silent "ThreadSanitizer|Leak check.*detected leaks" $LOGFILE ; then |
| 118 | + echo ThreadSanitizer or leak check failures in $LOGFILE |
| 119 | + STATUS=1 |
| 120 | + rm -f $XMLFILE |
| 121 | + fi |
| 122 | + |
| 123 | + if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then |
| 124 | + # Now delete any new test output. |
| 125 | + TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort) |
| 126 | + DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \ |
| 127 | + <(echo "$TEST_TMPDIR_AFTER")) |
| 128 | + for DIR in $DIFF; do |
| 129 | + # Multiple tests may be running concurrently. To avoid deleting the |
| 130 | + # wrong directories, constrain to only directories beginning with the |
| 131 | + # test name. |
| 132 | + # |
| 133 | + # This may delete old test directories belonging to this test, but |
| 134 | + # that's not typically a concern when rerunning flaky tests. |
| 135 | + if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then |
| 136 | + echo Deleting leftover flaky test directory "$DIR" |
| 137 | + rm -Rf "$DIR" |
| 138 | + fi |
| 139 | + done |
| 140 | + fi |
| 141 | + |
| 142 | + if [ "$STATUS" -eq "0" ]; then |
| 143 | + break |
| 144 | + elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then |
| 145 | + echo Test failed attempt number $ATTEMPT_NUMBER |
| 146 | + echo Will retry... |
| 147 | + fi |
| 148 | +done |
| 149 | + |
| 150 | +# If we have a LeakSanitizer report, and XML reporting is configured, add a new test |
| 151 | +# case result to the XML file for the leak report. Otherwise Jenkins won't show |
| 152 | +# us which tests had LSAN errors. |
| 153 | +if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then |
| 154 | + echo Test had memory leaks. Editing XML |
| 155 | + perl -p -i -e ' |
| 156 | + if (m#</testsuite>#) { |
| 157 | + print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n"; |
| 158 | + print " <failure message=\"LeakSanitizer failed\" type=\"\">\n"; |
| 159 | + print " See txt log file for details\n"; |
| 160 | + print " </failure>\n"; |
| 161 | + print "</testcase>\n"; |
| 162 | + }' $XMLFILE |
| 163 | +fi |
| 164 | + |
| 165 | +# Capture and compress core file and binary. |
| 166 | +COREFILES=$(ls | grep ^core) |
| 167 | +if [ -n "$COREFILES" ]; then |
| 168 | + echo Found core dump. Saving executable and core files. |
| 169 | + gzip < $TEST_EXECUTABLE > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $? |
| 170 | + for COREFILE in $COREFILES; do |
| 171 | + gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $? |
| 172 | + done |
| 173 | + # Pull in any .so files as well. |
| 174 | + for LIB in $(ldd $TEST_EXECUTABLE | grep $ROOT | awk '{print $3}'); do |
| 175 | + LIB_NAME=$(basename $LIB) |
| 176 | + gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $? |
| 177 | + done |
| 178 | +fi |
| 179 | + |
| 180 | +popd |
| 181 | +rm -Rf $TEST_WORKDIR |
| 182 | + |
| 183 | +exit $STATUS |
0 commit comments