Add support for multiple InferencePool backends (#4439)

Add support for multiple InferencePool backends on a Route.

Problem: A route should be able to have multiple InferencePools in its backendRefs.

Solution: Add support for multiple InferencePool backends. Added logic to remove duplicated inference maps.

Testing: Added unit tests and enabled correlating GatewayWeightedAcrossTwoInferencePools conformance test. Manually tested situations for multiple inferencepool backends with and without http matches.

Author: bjee19

internal/controller/nginx/config/maps.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	gotemplate "text/template"
 
@@ -185,14 +186,21 @@ func createAddHeadersMap(name string) shared.Map {
 
 // buildInferenceMaps creates maps for InferencePool Backends.
 func buildInferenceMaps(groups []dataplane.BackendGroup) []shared.Map {
-	inferenceMaps := make([]shared.Map, 0, len(groups))
+	uniqueMaps := make(map[string]shared.Map)
 
 	for _, group := range groups {
 		for _, backend := range group.Backends {
 			if backend.EndpointPickerConfig == nil || backend.EndpointPickerConfig.EndpointPickerRef == nil {
 				continue
 			}
 
+			backendVarName := strings.ReplaceAll(backend.UpstreamName, "-", "_")
+			mapKey := backendVarName // Use this as the key to detect duplicates
+
+			// Skip if we've already processed this upstream
+			if _, exists := uniqueMaps[mapKey]; exists {
+				continue
+			}
 			// Decide what the map must return when the picker didn’t set a value.
 			var defaultResult string
 			switch backend.EndpointPickerConfig.EndpointPickerRef.FailureMode {
@@ -230,14 +238,26 @@ func buildInferenceMaps(groups []dataplane.BackendGroup) []shared.Map {
 				Result: defaultResult,
 			})
 
-			backendVarName := strings.ReplaceAll(backend.UpstreamName, "-", "_")
-
-			inferenceMaps = append(inferenceMaps, shared.Map{
+			uniqueMaps[mapKey] = shared.Map{
 				Source:     `$inference_workload_endpoint`,
 				Variable:   fmt.Sprintf("$inference_backend_%s", backendVarName),
 				Parameters: params,
-			})
+			}
 		}
 	}
+
+	// Sort the map keys to ensure deterministic ordering
+	mapKeys := make([]string, 0, len(uniqueMaps))
+	for key := range uniqueMaps {
+		mapKeys = append(mapKeys, key)
+	}
+	sort.Strings(mapKeys)
+
+	// Build the result slice in sorted order
+	inferenceMaps := make([]shared.Map, 0, len(uniqueMaps))
+	for _, key := range mapKeys {
+		inferenceMaps = append(inferenceMaps, uniqueMaps[key])
+	}
+
 	return inferenceMaps
 }

internal/controller/nginx/config/maps_test.go

@@ -397,55 +397,183 @@ func TestCreateStreamMapsWithEmpty(t *testing.T) {
 
 func TestBuildInferenceMaps(t *testing.T) {
 	t.Parallel()
-	g := NewWithT(t)
 
-	group := dataplane.BackendGroup{
-		Backends: []dataplane.Backend{
-			{
-				UpstreamName: "upstream1",
-				EndpointPickerConfig: &dataplane.EndpointPickerConfig{
-					NsName: "default",
-					EndpointPickerRef: &inference.EndpointPickerRef{
-						FailureMode: inference.EndpointPickerFailClose,
+	tests := []struct {
+		expectedConfig map[string]struct {
+			failureMode   inference.EndpointPickerFailureMode
+			defaultResult string
+		}
+		name          string
+		backendGroups []dataplane.BackendGroup
+		expectedMaps  int
+	}{
+		{
+			name: "unique backends with different failure modes, result is ordered by upstream name",
+			backendGroups: []dataplane.BackendGroup{
+				{
+					Backends: []dataplane.Backend{
+						{
+							UpstreamName: "upstream2",
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailOpen,
+								},
+							},
+						},
+						{
+							UpstreamName: "upstream1",
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailClose,
+								},
+							},
+						},
+						{
+							UpstreamName:         "upstream3",
+							EndpointPickerConfig: nil,
+						},
 					},
 				},
 			},
-			{
-				UpstreamName: "upstream2",
-				EndpointPickerConfig: &dataplane.EndpointPickerConfig{
-					NsName: "default",
-					EndpointPickerRef: &inference.EndpointPickerRef{
-						FailureMode: inference.EndpointPickerFailOpen,
+			expectedMaps: 2,
+			expectedConfig: map[string]struct {
+				failureMode   inference.EndpointPickerFailureMode
+				defaultResult string
+			}{
+				"upstream1": {
+					failureMode:   inference.EndpointPickerFailClose,
+					defaultResult: "invalid-backend-ref",
+				},
+				"upstream2": {
+					failureMode:   inference.EndpointPickerFailOpen,
+					defaultResult: "upstream2",
+				},
+			},
+		},
+		{
+			name: "duplicate upstreams should be deduplicated",
+			backendGroups: []dataplane.BackendGroup{
+				{
+					Backends: []dataplane.Backend{
+						{
+							UpstreamName: "upstream1",
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailClose,
+								},
+							},
+						},
+						{
+							UpstreamName: "upstream1", // Duplicate
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailClose,
+								},
+							},
+						},
+					},
+				},
+				{
+					Backends: []dataplane.Backend{
+						{
+							UpstreamName: "upstream1", // Another duplicate
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailClose,
+								},
+							},
+						},
+						{
+							UpstreamName: "upstream2",
+							EndpointPickerConfig: &dataplane.EndpointPickerConfig{
+								NsName: "default",
+								EndpointPickerRef: &inference.EndpointPickerRef{
+									FailureMode: inference.EndpointPickerFailOpen,
+								},
+							},
+						},
 					},
 				},
 			},
-			{
-				UpstreamName:         "upstream3",
-				EndpointPickerConfig: nil,
+			expectedMaps: 2, // Only 2 unique upstreams
+			expectedConfig: map[string]struct {
+				failureMode   inference.EndpointPickerFailureMode
+				defaultResult string
+			}{
+				"upstream1": {
+					failureMode:   inference.EndpointPickerFailClose,
+					defaultResult: "invalid-backend-ref",
+				},
+				"upstream2": {
+					failureMode:   inference.EndpointPickerFailOpen,
+					defaultResult: "upstream2",
+				},
 			},
 		},
+		{
+			name: "no endpoint picker configs",
+			backendGroups: []dataplane.BackendGroup{
+				{
+					Backends: []dataplane.Backend{
+						{
+							UpstreamName:         "upstream1",
+							EndpointPickerConfig: nil,
+						},
+						{
+							UpstreamName:         "upstream2",
+							EndpointPickerConfig: nil,
+						},
+					},
+				},
+			},
+			expectedMaps: 0,
+		},
 	}
 
-	maps := buildInferenceMaps([]dataplane.BackendGroup{group})
-	g.Expect(maps).To(HaveLen(2))
-	g.Expect(maps[0].Source).To(Equal("$inference_workload_endpoint"))
-	g.Expect(maps[0].Variable).To(Equal("$inference_backend_upstream1"))
-	g.Expect(maps[0].Parameters).To(HaveLen(3))
-	g.Expect(maps[0].Parameters[0].Value).To(Equal("\"\""))
-	g.Expect(maps[0].Parameters[0].Result).To(Equal("upstream1"))
-	g.Expect(maps[0].Parameters[1].Value).To(Equal("~.+"))
-	g.Expect(maps[0].Parameters[1].Result).To(Equal("$inference_workload_endpoint"))
-	g.Expect(maps[0].Parameters[2].Value).To(Equal("default"))
-	g.Expect(maps[0].Parameters[2].Result).To(Equal("invalid-backend-ref"))
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			g := NewWithT(t)
+
+			maps := buildInferenceMaps(tc.backendGroups)
+			g.Expect(maps).To(HaveLen(tc.expectedMaps))
 
-	// Check the second map
-	g.Expect(maps[1].Source).To(Equal("$inference_workload_endpoint"))
-	g.Expect(maps[1].Variable).To(Equal("$inference_backend_upstream2"))
-	g.Expect(maps[1].Parameters).To(HaveLen(3))
-	g.Expect(maps[1].Parameters[0].Value).To(Equal("\"\""))
-	g.Expect(maps[1].Parameters[0].Result).To(Equal("upstream2"))
-	g.Expect(maps[1].Parameters[1].Value).To(Equal("~.+"))
-	g.Expect(maps[1].Parameters[1].Result).To(Equal("$inference_workload_endpoint"))
-	g.Expect(maps[1].Parameters[2].Value).To(Equal("default"))
-	g.Expect(maps[1].Parameters[2].Result).To(Equal("upstream2"))
+			// Verify each map has the correct structure
+			seenUpstreams := make(map[string]bool)
+			for _, m := range maps {
+				g.Expect(m.Source).To(Equal("$inference_workload_endpoint"))
+				g.Expect(m.Parameters).To(HaveLen(3))
+
+				// Extract upstream name from variable name
+				varName := strings.TrimPrefix(m.Variable, "$inference_backend_")
+				upstreamName := strings.ReplaceAll(varName, "_", "-")
+
+				// Verify we haven't seen this upstream before (no duplicates)
+				g.Expect(seenUpstreams[upstreamName]).To(BeFalse(), "Duplicate upstream found: %s", upstreamName)
+				seenUpstreams[upstreamName] = true
+
+				// Verify parameter structure
+				g.Expect(m.Parameters[0].Value).To(Equal("\"\""))
+				g.Expect(m.Parameters[0].Result).To(Equal(upstreamName))
+				g.Expect(m.Parameters[1].Value).To(Equal("~.+"))
+				g.Expect(m.Parameters[1].Result).To(Equal("$inference_workload_endpoint"))
+				g.Expect(m.Parameters[2].Value).To(Equal("default"))
+
+				// Verify the default result matches expected failure mode
+				if expectedConfig, exists := tc.expectedConfig[upstreamName]; exists {
+					g.Expect(m.Parameters[2].Result).To(Equal(expectedConfig.defaultResult))
+				}
+			}
+
+			// Verify all expected upstreams are present
+			for expectedUpstream := range tc.expectedConfig {
+				g.Expect(seenUpstreams[expectedUpstream]).To(BeTrue(), "Expected upstream not found: %s", expectedUpstream)
+			}
+		})
+	}
 }