Add test for incorrect check configurations

The enable/disable configuration of each check must be configured for every check mode.

The level configuration of each check must be configured for every check mode the check is enabled for.

Author: per1234

check/check.go

@@ -71,6 +71,10 @@ func shouldRun(checkConfiguration checkconfigurations.Type, currentProject proje
 		return false, nil
 	}
 
+	return IsEnabled(checkConfiguration, configurationCheckModes)
+}
+
+func IsEnabled(checkConfiguration checkconfigurations.Type, configurationCheckModes map[checkmode.Type]bool) (bool, error) {
 	for _, disableMode := range checkConfiguration.DisableModes {
 		if configurationCheckModes[disableMode] {
 			return false, nil

check/checkconfigurations/checkconfigurations_test.go

@@ -0,0 +1,40 @@
+// This file is part of arduino-check.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-check.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package checkconfigurations_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/arduino/arduino-check/check"
+	"github.com/arduino/arduino-check/check/checkconfigurations"
+	"github.com/arduino/arduino-check/check/checklevel"
+	"github.com/arduino/arduino-check/configuration/checkmode"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigurations(t *testing.T) {
+	for _, checkConfiguration := range checkconfigurations.Configurations() {
+		for checkMode := range checkmode.Types {
+			enabled, err := check.IsEnabled(checkConfiguration, map[checkmode.Type]bool{checkMode: true})
+			assert.Nil(t, err, fmt.Sprintf("Enable configuration of check %s doesn't resolve for check mode %s", checkConfiguration.ID, checkMode))
+			if err == nil && enabled {
+				_, err := checklevel.CheckLevelForCheckModes(checkConfiguration, map[checkmode.Type]bool{checkMode: true})
+				assert.Nil(t, err, fmt.Sprintf("Level configuration of check %s doesn't resolve for check mode %s", checkConfiguration.ID, checkMode))
+			}
+		}
+	}
+}

check/checklevel/checklevel.go

@@ -39,6 +39,10 @@ const (
 // CheckLevel determines the check level assigned to failure of the given check under the current tool configuration.
 func CheckLevel(checkConfiguration checkconfigurations.Type) (Type, error) {
 	configurationCheckModes := configuration.CheckModes(checkConfiguration.ProjectType)
+	return CheckLevelForCheckModes(checkConfiguration, configurationCheckModes)
+}
+
+func CheckLevelForCheckModes(checkConfiguration checkconfigurations.Type, configurationCheckModes map[checkmode.Type]bool) (Type, error) {
 	for _, errorMode := range checkConfiguration.ErrorModes {
 		if configurationCheckModes[errorMode] {
 			return Error, nil

configuration/checkmode/checkmode.go

@@ -38,6 +38,19 @@ const (
 	Default                              // default
 )
 
+var empty struct{}
+
+// Types provides an iterator and validator for Type.
+var Types = map[Type]struct{}{
+	Strict:                   empty,
+	Specification:            empty,
+	Permissive:               empty,
+	LibraryManagerSubmission: empty,
+	LibraryManagerIndexed:    empty,
+	Official:                 empty,
+	Default:                  empty,
+}
+
 // ComplianceModeFromString parses the --compliance flag value and returns the corresponding check mode settings.
 func ComplianceModeFromString(complianceModeString string) (bool, bool, bool, error) {
 	switch strings.ToLower(complianceModeString) {

configuration/checkmode/checkmode_test.go

@@ -23,6 +23,18 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestTypes(t *testing.T) {
+	for key := range Types {
+		_, valid := Types[key]
+		assert.True(t, valid)
+	}
+
+	_, valid := Types[Strict]
+	assert.True(t, valid)
+	_, valid = Types[42]
+	assert.False(t, valid)
+}
+
 func TestMode(t *testing.T) {
 	defaultCheckModes := map[projecttype.Type]map[Type]bool{
 		projecttype.Sketch: {