Merge pull request #1642 from gyohuangxin/all-features-conformance-flag

Add `all-features` flag to add ability to enable all supported feature conformance tests.

Author: k8s-ci-robot

conformance/conformance_test.go

@@ -27,6 +27,7 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/utils/flags"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -47,29 +48,30 @@ func TestConformance(t *testing.T) {
 	supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
 	exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
 	for feature := range exemptFeatures {
-		supportedFeatures[feature] = false
+		supportedFeatures.Delete(feature)
 	}
 
-	t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n supported features: [%v]\n exempt features: [%v]",
-		*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.SupportedFeatures, *flags.ExemptFeatures)
+	t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
+		*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)
 
 	cSuite := suite.New(suite.Options{
-		Client:               client,
-		GatewayClassName:     *flags.GatewayClassName,
-		Debug:                *flags.ShowDebug,
-		CleanupBaseResources: *flags.CleanupBaseResources,
-		SupportedFeatures:    supportedFeatures,
+		Client:                     client,
+		GatewayClassName:           *flags.GatewayClassName,
+		Debug:                      *flags.ShowDebug,
+		CleanupBaseResources:       *flags.CleanupBaseResources,
+		SupportedFeatures:          supportedFeatures,
+		EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
 	})
 	cSuite.Setup(t)
 	cSuite.Run(t, tests.ConformanceTests)
 }
 
 // parseSupportedFeatures parses flag arguments and converts the string to
 // map[suite.SupportedFeature]bool
-func parseSupportedFeatures(f string) map[suite.SupportedFeature]bool {
-	res := map[suite.SupportedFeature]bool{}
+func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
+	res := sets.Set[suite.SupportedFeature]{}
 	for _, value := range strings.Split(f, ",") {
-		res[suite.SupportedFeature(value)] = true
+		res.Insert(suite.SupportedFeature(value))
 	}
 	return res
 }

conformance/utils/flags/flags.go

@@ -24,9 +24,10 @@ import (
 )
 
 var (
-	GatewayClassName     = flag.String("gateway-class", "gateway-conformance", "Name of GatewayClass to use for tests")
-	ShowDebug            = flag.Bool("debug", false, "Whether to print debug logs")
-	CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
-	SupportedFeatures    = flag.String("supported-features", "", "Supported features included in conformance tests suites")
-	ExemptFeatures       = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
+	GatewayClassName           = flag.String("gateway-class", "gateway-conformance", "Name of GatewayClass to use for tests")
+	ShowDebug                  = flag.Bool("debug", false, "Whether to print debug logs")
+	CleanupBaseResources       = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
+	SupportedFeatures          = flag.String("supported-features", "", "Supported features included in conformance tests suites")
+	ExemptFeatures             = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
+	EnableAllSupportedFeatures = flag.Bool("all-features", false, "Whether to enable all supported features for conformance tests")
 )

conformance/utils/suite/suite.go

@@ -74,9 +74,29 @@ const (
 
 // StandardCoreFeatures are the features that are required to be conformant with
 // the Core API features that are part of the Standard release channel.
-var StandardCoreFeatures = map[SupportedFeature]bool{
-	SupportReferenceGrant: true,
-}
+var StandardCoreFeatures = sets.New(
+	SupportReferenceGrant,
+)
+
+// AllFeatures contains all the supported features and can be used to run all
+// conformance tests with `all-features` flag.
+//
+// Note that the AllFeatures must in sync with defined features when the
+// feature constants change.
+var AllFeatures = sets.New(
+	SupportReferenceGrant,
+	SupportTLSRoute,
+	SupportHTTPRouteQueryParamMatching,
+	SupportHTTPRouteMethodMatching,
+	SupportHTTPResponseHeaderModification,
+	SupportRouteDestinationPortMatching,
+	SupportGatewayClassObservedGenerationBump,
+	SupportHTTPRoutePortRedirect,
+	SupportHTTPRouteSchemeRedirect,
+	SupportHTTPRoutePathRedirect,
+	SupportHTTPRouteHostRewrite,
+	SupportHTTPRoutePathRewrite,
+)
 
 // ConformanceTestSuite defines the test suite used to run Gateway API
 // conformance tests.
@@ -89,7 +109,7 @@ type ConformanceTestSuite struct {
 	Cleanup           bool
 	BaseManifests     string
 	Applier           kubernetes.Applier
-	SupportedFeatures map[SupportedFeature]bool
+	SupportedFeatures sets.Set[SupportedFeature]
 	TimeoutConfig     config.TimeoutConfig
 	SkipTests         sets.Set[string]
 }
@@ -112,9 +132,10 @@ type Options struct {
 
 	// CleanupBaseResources indicates whether or not the base test
 	// resources such as Gateways should be cleaned up after the run.
-	CleanupBaseResources bool
-	SupportedFeatures    map[SupportedFeature]bool
-	TimeoutConfig        config.TimeoutConfig
+	CleanupBaseResources       bool
+	SupportedFeatures          sets.Set[SupportedFeature]
+	EnableAllSupportedFeatures bool
+	TimeoutConfig              config.TimeoutConfig
 	// SkipTests contains all the tests not to be run and can be used to opt out
 	// of specific tests
 	SkipTests []string
@@ -129,13 +150,13 @@ func New(s Options) *ConformanceTestSuite {
 		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
 	}
 
-	if s.SupportedFeatures == nil {
+	if s.EnableAllSupportedFeatures == true {
+		s.SupportedFeatures = AllFeatures
+	} else if s.SupportedFeatures == nil {
 		s.SupportedFeatures = StandardCoreFeatures
 	} else {
-		for feature, val := range StandardCoreFeatures {
-			if _, ok := s.SupportedFeatures[feature]; !ok {
-				s.SupportedFeatures[feature] = val
-			}
+		for feature := range StandardCoreFeatures {
+			s.SupportedFeatures.Insert(feature)
 		}
 	}
 
@@ -222,7 +243,7 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
 	// Check that all features exercised by the test have been opted into by
 	// the suite.
 	for _, feature := range test.Features {
-		if supported, ok := suite.SupportedFeatures[feature]; !ok || !supported {
+		if !suite.SupportedFeatures.Has(feature) {
 			t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature)
 		}
 	}