Support processing multiple HTTP listeners

Problem:
The Gateway needs to process a Gateway resource. Currently, that is
already supported by the 'state' package. However, that support is
limited. For example, the package doesn't allow processing  multiple
listeners. More over, the approach used in the 'state' package is hard
to extend to support multiple listeners. 
This prompts development of a better solution.

As a first deliverable, we want to support processing multiple HTTP
listeners assuming they all bind to port 80. 
We also assume that the Gateway only supports a single Gateway resource
with known namespace and name.

Solution - replace the existing types of the state package with
the new ones:
- Introduce Configuration, which is an internal representation of the
Gateway configuration. 
- Introduce Statuses, which holds status-related information about
processed resources.
- Introduce ChangeProcessor, which processes changes to Gateway API
resources and returns new Configuration and Statuses.

Author: pleshakov

README.md

@@ -34,7 +34,6 @@ Before you can build and run the NGINX Kubernetes Gateway, make sure you have th
 
    Set the `PREFIX` variable to the name of the registry you'd like to push the image to. By default, the image will be named `nginx-kubernetes-gateway:0.0.1`.
 
-
 1. Push the image to your container registry:
 
    ```
@@ -55,7 +54,6 @@ You can deploy NGINX Kubernetes Gateway on an existing Kubernetes 1.16+ cluster.
 
    Make sure to substitute the image name with the name of the image you built.
 
-
 1. Install the Gateway CRDs:
 
    ```
@@ -89,6 +87,11 @@ You can deploy NGINX Kubernetes Gateway on an existing Kubernetes 1.16+ cluster.
    NAME                             READY   STATUS    RESTARTS   AGE
    nginx-gateway-5d4f4c7db7-xk2kq   2/2     Running   0          112s
    ```
+1. Create the Gateway resource:
+
+   ```
+   kubectl apply -f deploy/manifests/gateway.yaml
+   ```
 
 ## Expose NGINX Kubernetes Gateway
 

cmd/gateway/main.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	flag "github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/nginxinc/nginx-kubernetes-gateway/internal/config"
@@ -36,6 +37,11 @@ func main() {
 	conf := config.Config{
 		GatewayCtlrName: *gatewayCtlrName,
 		Logger:          logger,
+		// FIXME(pleshakov) Allow the cluster operator to customize this value
+		GatewayNsName: types.NamespacedName{
+			Namespace: "nginx-gateway",
+			Name:      "gateway",
+		},
 	}
 
 	MustValidateArguments(

deploy/manifests/gateway.yaml

@@ -8,7 +8,6 @@ metadata:
 spec:
   gatewayClassName: nginx
   listeners:
-  - name: my-listener
-    hostname: example.com
-    port: 8080
+  - name: http
+    port: 80
     protocol: HTTP

deploy/manifests/nginx-gateway.yaml

@@ -36,6 +36,7 @@ rules:
   - gateway.networking.k8s.io
   resources:
   - httproutes/status
+  - gateways/status
   verbs:
   - update
 ---

examples/advanced-routing/cafe-routes.yaml

@@ -3,6 +3,10 @@ kind: HTTPRoute
 metadata:
   name: cafe
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:
@@ -15,6 +19,10 @@ kind: HTTPRoute
 metadata:
   name: coffee
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:
@@ -37,6 +45,10 @@ kind: HTTPRoute
 metadata:
   name: tea
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:

examples/cafe-example/cafe-routes.yaml

@@ -3,6 +3,10 @@ kind: HTTPRoute
 metadata:
   name: cafe
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:
@@ -15,6 +19,10 @@ kind: HTTPRoute
 metadata:
   name: coffee
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:
@@ -31,6 +39,10 @@ kind: HTTPRoute
 metadata:
   name: tea
 spec:
+  parentRefs:
+  - name: gateway
+    namespace: nginx-gateway
+    sectionName: http
   hostnames:
   - "cafe.example.com"
   rules:

internal/config/config.go

@@ -2,9 +2,13 @@ package config
 
 import (
 	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 type Config struct {
 	GatewayCtlrName string
 	Logger          logr.Logger
+	// GatewayNsName is the namespaced name of a Gateway resource that the Gateway will use.
+	// The Gateway will ignore all other Gateway resources.
+	GatewayNsName types.NamespacedName
 }

internal/events/loop.go

@@ -17,36 +17,36 @@ import (
 
 // EventLoop is the main event loop of the Gateway.
 type EventLoop struct {
-	conf            state.Configuration
+	processor       state.ChangeProcessor
 	serviceStore    state.ServiceStore
 	generator       config.Generator
 	eventCh         <-chan interface{}
 	logger          logr.Logger
-	statusUpdater   status.Updater
 	nginxFileMgr    file.Manager
 	nginxRuntimeMgr runtime.Manager
+	statusUpdater   status.Updater
 }
 
 // NewEventLoop creates a new EventLoop.
 func NewEventLoop(
-	conf state.Configuration,
+	processor state.ChangeProcessor,
 	serviceStore state.ServiceStore,
 	generator config.Generator,
 	eventCh <-chan interface{},
-	statusUpdater status.Updater,
 	logger logr.Logger,
 	nginxFileMgr file.Manager,
 	nginxRuntimeMgr runtime.Manager,
+	statusUpdater status.Updater,
 ) *EventLoop {
 	return &EventLoop{
-		conf:            conf,
+		processor:       processor,
 		serviceStore:    serviceStore,
 		generator:       generator,
 		eventCh:         eventCh,
-		statusUpdater:   statusUpdater,
 		logger:          logger.WithName("eventLoop"),
 		nginxFileMgr:    nginxFileMgr,
 		nginxRuntimeMgr: nginxRuntimeMgr,
+		statusUpdater:   statusUpdater,
 	}
 }
 
@@ -58,112 +58,88 @@ func (el *EventLoop) Start(ctx context.Context) error {
 	for {
 		select {
 		case <-ctx.Done():
+			// although we always return nil, Start must return it to satisfy
+			// "sigs.k8s.io/controller-runtime/pkg/manager".Runnable
 			return nil
 		case e := <-el.eventCh:
-			err := el.handleEvent(ctx, e)
-			if err != nil {
-				return err
-			}
+			el.handleEvent(ctx, e)
 		}
 	}
 }
 
 // FIXME(pleshakov): think about how to avoid using an interface{} here
-func (el *EventLoop) handleEvent(ctx context.Context, event interface{}) error {
-	var changes []state.Change
-	var updates []state.StatusUpdate
-	var err error
-
+func (el *EventLoop) handleEvent(ctx context.Context, event interface{}) {
 	switch e := event.(type) {
 	case *UpsertEvent:
-		changes, updates, err = el.propagateUpsert(e)
+		el.propagateUpsert(e)
 	case *DeleteEvent:
-		changes, updates, err = el.propagateDelete(e)
+		el.propagateDelete(e)
 	default:
-		// FIXME(pleshakov): panic because it is a coding error
-		return fmt.Errorf("unknown event type %T", e)
+		panic(fmt.Errorf("unknown event type %T", e))
+	}
+
+	changed, conf, statuses := el.processor.Process()
+	if !changed {
+		return
 	}
 
+	err := el.updateNginx(ctx, conf)
+	if err != nil {
+		el.logger.Error(err, "Failed to update NGINX configuration")
+	}
+
+	el.statusUpdater.Update(ctx, statuses)
+}
+
+func (el *EventLoop) updateNginx(ctx context.Context, conf state.Configuration) error {
+	cfg, warnings := el.generator.Generate(conf)
+
+	// For now, we keep all http servers in one config
+	// We might rethink that. For example, we can write each server to its file
+	// or group servers in some way.
+	err := el.nginxFileMgr.WriteHTTPServersConfig("http-servers", cfg)
 	if err != nil {
 		return err
 	}
 
-	el.processChangesAndStatusUpdates(ctx, changes, updates)
-	return nil
+	for obj, objWarnings := range warnings {
+		for _, w := range objWarnings {
+			// FIXME(pleshakov): report warnings via Object status
+			el.logger.Info("got warning while generating config",
+				"kind", obj.GetObjectKind().GroupVersionKind().Kind,
+				"namespace", obj.GetNamespace(),
+				"name", obj.GetName(),
+				"warning", w)
+		}
+	}
+
+	return el.nginxRuntimeMgr.Reload(ctx)
 }
 
-func (el *EventLoop) propagateUpsert(e *UpsertEvent) ([]state.Change, []state.StatusUpdate, error) {
+func (el *EventLoop) propagateUpsert(e *UpsertEvent) {
 	switch r := e.Resource.(type) {
+	case *v1alpha2.Gateway:
+		el.processor.CaptureUpsertChange(r)
 	case *v1alpha2.HTTPRoute:
-		changes, statusUpdates := el.conf.UpsertHTTPRoute(r)
-		return changes, statusUpdates, nil
+		el.processor.CaptureUpsertChange(r)
 	case *apiv1.Service:
-		el.serviceStore.Upsert(r)
 		// FIXME(pleshakov): make sure the affected hosts are updated
-		return nil, nil, nil
+		el.serviceStore.Upsert(r)
+	default:
+		panic(fmt.Errorf("unknown resource type %T", e.Resource))
 	}
-
-	// FIXME(pleshakov): panic because it is a coding error
-	return nil, nil, fmt.Errorf("unknown resource type %T", e.Resource)
 }
 
-func (el *EventLoop) propagateDelete(e *DeleteEvent) ([]state.Change, []state.StatusUpdate, error) {
+func (el *EventLoop) propagateDelete(e *DeleteEvent) {
 	switch e.Type.(type) {
+	case *v1alpha2.Gateway:
+		el.processor.CaptureDeleteChange(e.Type, e.NamespacedName)
 	case *v1alpha2.HTTPRoute:
-		changes, statusUpdates := el.conf.DeleteHTTPRoute(e.NamespacedName)
-		return changes, statusUpdates, nil
+		el.processor.CaptureDeleteChange(e.Type, e.NamespacedName)
 	case *apiv1.Service:
-		el.serviceStore.Delete(e.NamespacedName)
 		// FIXME(pleshakov): make sure the affected hosts are updated
-		return nil, nil, nil
-	}
-
-	// FIXME(pleshakov): panic because it is a coding error
-	return nil, nil, fmt.Errorf("unknown resource type %T", e.Type)
-}
-
-func (el *EventLoop) processChangesAndStatusUpdates(ctx context.Context, changes []state.Change, updates []state.StatusUpdate) {
-	for _, c := range changes {
-		el.logger.Info("Processing a change",
-			"host", c.Host.Value)
-
-		if c.Op == state.Upsert {
-			cfg, warnings := el.generator.GenerateForHost(c.Host)
-
-			for obj, objWarnings := range warnings {
-				for _, w := range objWarnings {
-					// FIXME(pleshakov): report warnings via Object status
-					el.logger.Info("got warning while generating config",
-						"kind", obj.GetObjectKind().GroupVersionKind().Kind,
-						"namespace", obj.GetNamespace(),
-						"name", obj.GetName(),
-						"warning", w)
-				}
-			}
-
-			el.logger.Info("Writing configuration",
-				"host", c.Host.Value)
-
-			err := el.nginxFileMgr.WriteServerConfig(c.Host.Value, cfg)
-			if err != nil {
-				el.logger.Error(err, "Failed to write configuration",
-					"host", c.Host.Value)
-			}
-		} else {
-			err := el.nginxFileMgr.DeleteServerConfig(c.Host.Value)
-			if err != nil {
-				el.logger.Error(err, "Failed to delete configuration",
-					"host", c.Host.Value)
-			}
-		}
-	}
-
-	if len(changes) > 0 {
-		err := el.nginxRuntimeMgr.Reload(ctx)
-		if err != nil {
-			el.logger.Error(err, "Failed to reload NGINX")
-		}
+		el.serviceStore.Delete(e.NamespacedName)
+	default:
+		panic(fmt.Errorf("unknown resource type %T", e.Type))
 	}
-
-	el.statusUpdater.ProcessStatusUpdates(ctx, updates)
 }