Add ability to use multiple compilers in a test

Author: ianfixes

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Definition of `LED_BUILTIN`, first reported by `dfrencham` on GitHub
 - Stubs for `tone` and `noTone`, first suggested by `dfrencham` on GitHub
+- Ability to specify multiple compilers for unit testing
 
 ### Changed
 

README.md

@@ -349,6 +349,10 @@ For your unit tests, in addition to setting specific libraries and platforms, yo
 
 ```yaml
 unittest:
+  compilers:
+    - g++      # default
+    - g++-4.9
+    - g++-7
   testfiles:
     select:
       - "*-*.*"

exe/arduino_ci_remote.rb

@@ -69,11 +69,15 @@ def assure(message, &block)
 attempt("Library installed at #{installed_library_path}") { true }
 
 # check GCC
-attempt_multiline("Checking GCC version") do
-  version = cpp_library.gcc_version
-  next nil unless version
-  puts version.split("\n").map { |l| "    #{l}" }.join("\n")
-  version
+compilers = config.compilers_to_use
+assure("The set of compilers (#{compilers.length}) isn't empty") { !compilers.empty? }
+compilers.each do |gcc_binary|
+  attempt_multiline("Checking #{gcc_binary} version") do
+    version = cpp_library.gcc_version(gcc_binary)
+    next nil unless version
+    puts version.split("\n").map { |l| "    #{l}" }.join("\n")
+    version
+  end
 end
 
 # gather up all required boards so we can install them up front.
@@ -121,20 +125,22 @@ def assure(message, &block)
     last_board = board
     cpp_library.test_files.each do |unittest_path|
       unittest_name = File.basename(unittest_path)
-      attempt_multiline("Unit testing #{unittest_name}") do
-        exe = cpp_library.build_for_test_with_configuration(
-          unittest_path,
-          config.aux_libraries_for_unittest,
-          config.gcc_config(p)
-        )
-        puts
-        unless exe
-          puts "Last command: #{cpp_library.last_cmd}"
-          puts cpp_library.last_out
-          puts cpp_library.last_err
-          next false
+      compilers.each do |gcc_binary|
+        attempt_multiline("Unit testing #{unittest_name} with #{gcc_binary}") do
+          exe = cpp_library.build_for_test_with_configuration(
+            unittest_path,
+            config.aux_libraries_for_unittest,
+            config.gcc_config(p)
+          )
+          puts
+          unless exe
+            puts "Last command: #{cpp_library.last_cmd}"
+            puts cpp_library.last_out
+            puts cpp_library.last_err
+            next false
+          end
+          cpp_library.run_test_file(exe)
         end
-        cpp_library.run_test_file(exe)
       end
     end
   end

lib/arduino_ci/ci_config.rb

@@ -25,6 +25,7 @@
 }.freeze
 
 UNITTEST_SCHEMA = {
+  compilers: Array,
   platforms: Array,
   libraries: Array,
   testfiles: {
@@ -197,6 +198,13 @@ def package_url(package)
       @package_info[package][:url]
     end
 
+    # compilers to build (unit tests) with
+    # @return [Array<String>] The compiler binary names (e.g. g++) to build with
+    def compilers_to_use
+      return [] if @unittest_info[:compilers].nil?
+      @unittest_info[:compilers]
+    end
+
     # platforms to build [the examples on]
     # @return [Array<String>] The platforms to build
     def platforms_to_build

lib/arduino_ci/cpp_library.rb

@@ -107,10 +107,9 @@ def header_dirs
     end
 
     # wrapper for the GCC command
-    def run_gcc(*args, **kwargs)
-      full_args = ["g++"] + args
+    def run_gcc(gcc_binary, *args, **kwargs)
+      full_args = [gcc_binary] + args
       @last_cmd = " $ #{full_args.join(' ')}"
-
       ret = Host.run_and_capture(*full_args, **kwargs)
       @last_err = ret[:err]
       @last_out = ret[:out]
@@ -119,8 +118,8 @@ def run_gcc(*args, **kwargs)
 
     # Return the GCC version
     # @return [String] the version reported by `gcc -v`
-    def gcc_version
-      return nil unless run_gcc("-v")
+    def gcc_version(gcc_binary)
+      return nil unless run_gcc(gcc_binary, "-v")
       @last_err
     end
 
@@ -183,7 +182,7 @@ def test_args(aux_libraries, ci_gcc_config)
     # @param aux_libraries [String] The external Arduino libraries required by this project
     # @param ci_gcc_config [Hash] The GCC config object
     # @return [String] path to the compiled test executable
-    def build_for_test_with_configuration(test_file, aux_libraries, ci_gcc_config)
+    def build_for_test_with_configuration(test_file, aux_libraries, gcc_binary, ci_gcc_config)
       base = File.basename(test_file)
       executable = File.expand_path("unittest_#{base}.bin")
       File.delete(executable) if File.exist?(executable)
@@ -192,7 +191,7 @@ def build_for_test_with_configuration(test_file, aux_libraries, ci_gcc_config)
         test_args(aux_libraries, ci_gcc_config),
         [test_file],
       ].flatten(1)
-      return nil unless run_gcc(*args)
+      return nil unless run_gcc(gcc_binary, *args)
       artifacts << executable
       executable
     end

misc/default.yml

@@ -80,6 +80,8 @@ compile:
     - leonardo
 
 unittest:
+  compilers:
+    - g++
   libraries: ~
   platforms:
     - uno

spec/ci_config_spec.rb

@@ -28,6 +28,8 @@
       expect(default_config.platforms_to_unittest).to match(["uno", "due", "zero", "leonardo"])
       expect(default_config.aux_libraries_for_build).to match([])
       expect(default_config.aux_libraries_for_unittest).to match([])
+
+      expect(default_config.compilers_to_use).to match(["g++"])
     end
   end
 
@@ -63,6 +65,9 @@
       expect(combined_config.platforms_to_unittest).to match(["bogo"])
       expect(combined_config.aux_libraries_for_build).to match(["Adafruit FONA Library"])
       expect(combined_config.aux_libraries_for_unittest).to match(["abc123", "def456"])
+
+      expect(combined_config.compilers_to_use).to match(["g++", "g++-7"])
+
     end
   end
 

spec/cpp_library_spec.rb

@@ -62,10 +62,12 @@
     test_files = cpp_library.test_files
     test_files.each do |path|
       expected = path.include?("good")
-      it "tests #{File.basename(path)} expecting #{expected}" do
-        exe = cpp_library.build_for_test_with_configuration(path, [], config.gcc_config("uno"))
-        expect(exe).not_to be nil
-        expect(cpp_library.run_test_file(exe)).to eq(expected)
+      config.compilers_to_use.each do |compiler|
+        it "tests #{File.basename(path)} with #{compiler} expecting #{expected}" do
+          exe = cpp_library.build_for_test_with_configuration(path, [], compiler, config.gcc_config("uno"))
+          expect(exe).not_to be nil
+          expect(cpp_library.run_test_file(exe)).to eq(expected)
+        end
       end
     end
   end

spec/testsomething_unittests_spec.rb

@@ -26,32 +26,40 @@
       expect(allowed_files.empty?).to be false
     end
 
+    it "has at least one compiler defined" do
+      expect(config.compilers_to_use.length.zero?).to be(false)
+    end
+
     test_files = config.allowable_unittest_files(cpp_library.test_files)
     test_files.each do |path|
       tfn = File.basename(path)
-      context "file #{tfn}" do
 
-        before(:all) do
-          @exe = cpp_library.build_for_test_with_configuration(path, [], config.gcc_config("uno"))
-        end
+      config.compilers_to_use.each do |compiler|
 
-        # extra debug for c++ failures
-        after(:each) do |example|
-          if example.exception
-            puts "Last command: #{cpp_library.last_cmd}"
-            puts "========== Stdout:"
-            puts cpp_library.last_out
-            puts "========== Stderr:"
-            puts cpp_library.last_err
+        context "file #{tfn} (using #{compiler})" do
+
+          before(:all) do
+            @exe = cpp_library.build_for_test_with_configuration(path, [], compiler, config.gcc_config("uno"))
           end
-        end
 
-        it "#{tfn} builds successfully" do
-          expect(@exe).not_to be nil
-        end
-        it "#{tfn} passes tests" do
-          skip "Can't run the test program because it failed to build" if @exe.nil?
-          expect(cpp_library.run_test_file(@exe)).to_not be_falsey
+          # extra debug for c++ failures
+          after(:each) do |example|
+            if example.exception
+              puts "Last command: #{cpp_library.last_cmd}"
+              puts "========== Stdout:"
+              puts cpp_library.last_out
+              puts "========== Stderr:"
+              puts cpp_library.last_err
+            end
+          end
+
+          it "#{tfn} builds successfully" do
+            expect(@exe).not_to be nil
+          end
+          it "#{tfn} passes tests" do
+            skip "Can't run the test program because it failed to build" if @exe.nil?
+            expect(cpp_library.run_test_file(@exe)).to_not be_falsey
+          end
         end
       end
     end

spec/yaml/o1.yaml

@@ -32,6 +32,9 @@ compile:
     - esp8266
 
 unittest:
+  compilers:
+    - g++
+    - g++-7
   testfiles:
     select:
       - "*-*.*"