Configure test fixture using CTest for Windows build

- Added new test/test_suite.in with list of regression and unit tests
  previously in Makefile.am, to be shared between Unix and Windows
  builds.
- Updated regression.cc & unit.cc to return the number of failed tests
  to indicate to CTest that the test failed. Similarly, a crash or
  unhandled exception terminates the process with a non-zero exit code.
  - This change doesn't affect running the tests with autotest in Unix
    builds because this processes test output from custom-test-driver &
    test-suite.sh, and ignores the exit code of the test runner.
- Removed comment in test/test-cases/regression-offset-variable.json as
  this is not supported by JSON and prevents strict parsers to read and
  process the file.
- Minor change in regression.cc's clearAuditLog to replace std::ifstream
  with std::ofstream as the mode to open the flag applies to an output
  stream.
- Minor change in unit.cc to simplify code that deletes tests.
- Minor changes to test/custom-test-driver to correct usage information.

Author: eduar-hte

Makefile.am

@@ -151,6 +151,39 @@ add_regression_test_capability(WITH_CURL HAVE_CURL)
 add_regression_test_capability(WITH_LMDB HAVE_LMDB)
 add_regression_test_capability(WITH_MAXMIND HAVE_MAXMIND)
 
+enable_testing()
+
+file(READ ${BASE_DIR}/test/test-suite.in TEST_FILES_RAW)
+string(REPLACE "\n" ";" TEST_FILES ${TEST_FILES_RAW})
+
+foreach(TEST_FILE ${TEST_FILES})
+  # ignore comment lines
+  string(FIND ${TEST_FILE} "#" is_comment)
+  if(NOT is_comment EQUAL 0)
+    string(FIND ${TEST_FILE} "TESTS+=" is_valid_prefix)
+    if(NOT is_valid_prefix EQUAL 0)
+      message(FATAL_ERROR "Invalid prefix in line: ${TEST_FILE}")
+    endif()
+
+    # remove 'TESTS+=' prefix and 'test/' too because tests are launched
+    # from that directory
+    string(SUBSTRING ${TEST_FILE} 12 -1 TEST_FILE)
+
+    # test name
+    get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE)
+
+    # determine test runner based on test path prefix
+    string(FIND ${TEST_FILE} "test-cases/regression/" is_regression_test)
+    if(is_regression_test EQUAL 0)
+        set(TEST_RUNNER "regression_tests")
+    else()
+        set(TEST_RUNNER "unit_tests")
+    endif()
+
+    add_test(NAME ${TEST_NAME} COMMAND ${TEST_RUNNER} ${TEST_FILE} WORKING_DIRECTORY ${BASE_DIR}/test)
+  endif()
+endforeach()
+
 # benchmark
 add_executable(benchmark ${BASE_DIR}/test/benchmark/benchmark.cc)
 setTestTargetProperties(benchmark)

build/win32/CMakeLists.txt

@@ -77,6 +77,35 @@ ARG USE_ASAN=
 
 RUN C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && vcbuild.bat %BUILD_TYPE% %ARCH% %USE_ASAN%
 
+# test suite
+#
+
+# setup test environment
+RUN cmd.exe /C md \tmp
+RUN cmd.exe /C md \bin
+RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin" \bin > NUL
+RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin\echo.exe" \bin\echo > NUL
+
+# disable tests that don't work on windows
+ARG JQ_VERSION=1.7.1
+ARG JQ_BINARY=jq-windows-amd64.exe
+ARG JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/${JQ_BINARY}
+
+ARG JQ_BIN=C:\TEMP\jq.exe
+ADD ${JQ_URL} ${JQ_BIN}
+
+WORKDIR ${MOD_SECURITY_DIR}\test\test-cases\regression
+
+RUN %JQ_BIN% "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
+RUN %JQ_BIN% "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json
+
+# run tests
+WORKDIR ${MOD_SECURITY_DIR}\build\win32\build
+
+RUN C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && ctest -C %BUILD_TYPE% --output-on-failure
+
 # setup container's entrypoint
 #
 

build/win32/docker/Dockerfile

@@ -42,9 +42,9 @@ print_usage ()
 {
   cat <<END
 Usage:
-  test-driver --test-name=NAME --log-file=PATH --trs-file=PATH
-              [--expect-failure={yes|no}] [--color-tests={yes|no}]
-              [--enable-hard-errors={yes|no}] [--]
+  test-driver --test-name NAME --log-file PATH --trs-file PATH
+              [--expect-failure {yes|no}] [--color-tests {yes|no}]
+              [--enable-hard-errors {yes|no}] [--]
               TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS]
 The '--test-name', '--log-file' and '--trs-file' options are mandatory.
 END

test/custom-test-driver

@@ -64,13 +64,11 @@ bool contains(const std::string &s, const std::string &pattern) {
 
 void clearAuditLog(const std::string &filename) {
     if (!filename.empty()) {
-        std::ifstream file;
-        file.open(filename.c_str(), std::ifstream::out | std::ifstream::trunc);
+        std::ofstream file{filename.c_str(), std::ofstream::out | std::ofstream::trunc};
         if (!file.is_open() || file.fail()) {
             std::cout << std::endl << "Failed to clear previous contents of audit log: " \
                 << filename << std::endl;
         }
-        file.close();
     }
 }
 std::string getAuditLogContent(const std::string &filename) {
@@ -510,6 +508,7 @@ int main(int argc, char **argv) {
 #ifdef NO_LOGS
     std::cout << "Test utility cannot work without logging support." \
         << std::endl;
+    return 0;
 #else
     test.cmd_options(argc, argv);
     if (!test.m_automake_output && !test.m_count_all) {
@@ -609,6 +608,6 @@ int main(int argc, char **argv) {
         delete vec;
     }
 
+    return failed;
 #endif
-    return 0;
 }

test/regression/regression.cc

@@ -215,7 +215,6 @@
       ]
     },
     "expected":{
-      // should not match
     },
     "rules":[
       "SecRequestBodyAccess On",
@@ -248,7 +247,6 @@
       ]
     },
     "expected":{
-      // should not match
     },
     "rules":[
       "SecRequestBodyAccess On",