Skip to content

Commit 852585e

Browse files
salonichf5salonichf5
authored
Adds sticky sessions via ip_hash (Upstream Settings Policy) and Route-level sessionPersistence (sticky cookie, NGINX Plus experimental only), and expands support for all NGINX OSS/Plus load-balancing methods. (#4471)
Problem: Users need session persistence (“sticky sessions”) so traffic from the same client consistently routes to the same upstream instance for stateful apps. Solution: Adds two types of session persistence: IP-based persistence by allowing users to set the ip_hash load-balancing method via the Upstream Settings Policy, and cookie-based persistence by adding support for sessionPersistence configuration at the Route level to enable sticky sessions using a sticky cookie. Cookie-based persistence is available only to NGINX Plus users and is supported in experimental mode. Additionally, the PR expands load-balancing support to include all methods available in both NGINX OSS and NGINX Plus.
1 parent 5824970 commit 852585e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+4099
-398
lines changed

‎apis/v1alpha1/upstreamsettingspolicy_types.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ type UpstreamSettingsPolicyList struct {
3636
}
3737

3838
// UpstreamSettingsPolicySpec defines the desired state of the UpstreamSettingsPolicy.
39+
// +kubebuilder:validation:XValidation:rule="!(has(self.loadBalancingMethod) && (self.loadBalancingMethod == 'hash' || self.loadBalancingMethod == 'hash consistent')) || has(self.hashMethodKey)",message="hashMethodKey is required when loadBalancingMethod is 'hash' or 'hash consistent'"
40+
//
41+
//nolint:lll
3942
type UpstreamSettingsPolicySpec struct {
4043
// ZoneSize is the size of the shared memory zone used by the upstream. This memory zone is used to share
4144
// the upstream configuration between nginx worker processes. The more servers that an upstream has,
@@ -51,6 +54,19 @@ type UpstreamSettingsPolicySpec struct {
5154
// +optional
5255
KeepAlive *UpstreamKeepAlive `json:"keepAlive,omitempty"`
5356

57+
// LoadBalancingMethod specifies the load balancing algorithm to be used for the upstream.
58+
// If not specified, NGINX Gateway Fabric defaults to `random two least_conn`,
59+
// which differs from the standard NGINX default `round-robin`.
60+
//
61+
// +optional
62+
LoadBalancingMethod *LoadBalancingType `json:"loadBalancingMethod,omitempty"`
63+
64+
// HashMethodKey defines the key used for hash-based load balancing methods.
65+
// This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
66+
//
67+
// +optional
68+
HashMethodKey *HashMethodKey `json:"hashMethodKey,omitempty"`
69+
5470
// TargetRefs identifies API object(s) to apply the policy to.
5571
// Objects must be in the same namespace as the policy.
5672
// Support: Service
@@ -98,3 +114,99 @@ type UpstreamKeepAlive struct {
98114
// +optional
99115
Timeout *Duration `json:"timeout,omitempty"`
100116
}
117+
118+
// LoadBalancingType defines the supported load balancing methods.
119+
//
120+
// +kubebuilder:validation:Enum=round_robin;least_conn;ip_hash;hash;hash consistent;random;random two;random two least_conn;random two least_time=header;random two least_time=last_byte;least_time header;least_time last_byte;least_time header inflight;least_time last_byte inflight
121+
//
122+
//nolint:lll
123+
type LoadBalancingType string
124+
125+
const (
126+
// Combination of NGINX directive
127+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#random
128+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#least_conn
129+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#least_time
130+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream
131+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ip_hash
132+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#hash
133+
134+
// LoadBalancingMethods supported by NGINX OSS and NGINX Plus.
135+
136+
// LoadBalancingTypeRoundRobin enables round-robin load balancing,
137+
// distributing requests evenly across all upstream servers.
138+
LoadBalancingTypeRoundRobin LoadBalancingType = "round_robin"
139+
140+
// LoadBalancingTypeLeastConnection enables least-connections load balancing,
141+
// routing requests to the upstream server with the fewest active connections.
142+
LoadBalancingTypeLeastConnection LoadBalancingType = "least_conn"
143+
144+
// LoadBalancingTypeIPHash enables IP hash-based load balancing,
145+
// ensuring requests from the same client IP are routed to the same upstream server.
146+
LoadBalancingTypeIPHash LoadBalancingType = "ip_hash"
147+
148+
// LoadBalancingTypeHash enables generic hash-based load balancing,
149+
// routing requests to upstream servers based on a hash of a specified key
150+
// HashMethodKey field must be set when this method is selected.
151+
// Example configuration: hash $binary_remote_addr;.
152+
LoadBalancingTypeHash LoadBalancingType = "hash"
153+
154+
// LoadBalancingTypeHashConsistent enables consistent hash-based load balancing,
155+
// which minimizes the number of keys remapped when a server is added or removed.
156+
// HashMethodKey field must be set when this method is selected.
157+
// Example configuration: hash $binary_remote_addr consistent;.
158+
LoadBalancingTypeHashConsistent LoadBalancingType = "hash consistent"
159+
160+
// LoadBalancingTypeRandom enables random load balancing,
161+
// routing requests to upstream servers in a random manner.
162+
LoadBalancingTypeRandom LoadBalancingType = "random"
163+
164+
// LoadBalancingTypeRandomTwo enables a variation of random load balancing
165+
// that randomly selects two servers and forwards traffic to one of them.
166+
// The default method is least_conn which passes a request to a server with the least number of active connections.
167+
LoadBalancingTypeRandomTwo LoadBalancingType = "random two"
168+
169+
// LoadBalancingTypeRandomTwoLeastConnection enables a variation of least-connections
170+
// balancing that randomly selects two servers and forwards traffic to the one with
171+
// fewer active connections.
172+
LoadBalancingTypeRandomTwoLeastConnection LoadBalancingType = "random two least_conn"
173+
174+
// LoadBalancingMethods supported by NGINX Plus.
175+
176+
// LoadBalancingTypeRandomTwoLeastTimeHeader enables a variation of least-time load balancing
177+
// that randomly selects two servers and forwards traffic to the one with the least
178+
// time to receive the response header.
179+
LoadBalancingTypeRandomTwoLeastTimeHeader LoadBalancingType = "random two least_time=header"
180+
181+
// LoadBalancingTypeRandomTwoLeastTimeLastByte enables a variation of least-time load balancing
182+
// that randomly selects two servers and forwards traffic to the one with the least time
183+
// to receive the full response.
184+
LoadBalancingTypeRandomTwoLeastTimeLastByte LoadBalancingType = "random two least_time=last_byte"
185+
186+
// LoadBalancingTypeLeastTimeHeader enables least-time load balancing,
187+
// routing requests to the upstream server with the least time to receive the response header.
188+
LoadBalancingTypeLeastTimeHeader LoadBalancingType = "least_time header"
189+
190+
// LoadBalancingTypeLeastTimeLastByte enables least-time load balancing,
191+
// routing requests to the upstream server with the least time to receive the full response.
192+
LoadBalancingTypeLeastTimeLastByte LoadBalancingType = "least_time last_byte"
193+
194+
// LoadBalancingTypeLeastTimeHeaderInflight enables least-time load balancing,
195+
// routing requests to the upstream server with the least time to receive the response header,
196+
// considering the incomplete requests.
197+
LoadBalancingTypeLeastTimeHeaderInflight LoadBalancingType = "least_time header inflight"
198+
199+
// LoadBalancingTypeLeastTimeLastByteInflight enables least-time load balancing,
200+
// routing requests to the upstream server with the least time to receive the full response,
201+
// considering the incomplete requests.
202+
LoadBalancingTypeLeastTimeLastByteInflight LoadBalancingType = "least_time last_byte inflight"
203+
)
204+
205+
// HashMethodKey defines the key used for hash-based load balancing methods.
206+
// The key must be a valid NGINX variable name starting with '$' followed by lowercase
207+
// letters and underscores only.
208+
// For a full list of NGINX variables,
209+
// refer to: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables
210+
//
211+
// +kubebuilder:validation:Pattern=`^\$[a-z_]+$`
212+
type HashMethodKey string

‎apis/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/gateway.nginx.org_upstreamsettingspolicies.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ spec:
5151
spec:
5252
description: Spec defines the desired state of the UpstreamSettingsPolicy.
5353
properties:
54+
hashMethodKey:
55+
description: |-
56+
HashMethodKey defines the key used for hash-based load balancing methods.
57+
This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
58+
pattern: ^\$[a-z_]+$
59+
type: string
5460
keepAlive:
5561
description: KeepAlive defines the keep-alive settings.
5662
properties:
@@ -85,6 +91,27 @@ spec:
8591
pattern: ^[0-9]{1,4}(ms|s|m|h)?$
8692
type: string
8793
type: object
94+
loadBalancingMethod:
95+
description: |-
96+
LoadBalancingMethod specifies the load balancing algorithm to be used for the upstream.
97+
If not specified, NGINX Gateway Fabric defaults to `random two least_conn`,
98+
which differs from the standard NGINX default `round-robin`.
99+
enum:
100+
- round_robin
101+
- least_conn
102+
- ip_hash
103+
- hash
104+
- hash consistent
105+
- random
106+
- random two
107+
- random two least_conn
108+
- random two least_time=header
109+
- random two least_time=last_byte
110+
- least_time header
111+
- least_time last_byte
112+
- least_time header inflight
113+
- least_time last_byte inflight
114+
type: string
88115
targetRefs:
89116
description: |-
90117
TargetRefs identifies API object(s) to apply the policy to.
@@ -143,6 +170,12 @@ spec:
143170
required:
144171
- targetRefs
145172
type: object
173+
x-kubernetes-validations:
174+
- message: hashMethodKey is required when loadBalancingMethod is 'hash'
175+
or 'hash consistent'
176+
rule: '!(has(self.loadBalancingMethod) && (self.loadBalancingMethod
177+
== ''hash'' || self.loadBalancingMethod == ''hash consistent'')) ||
178+
has(self.hashMethodKey)'
146179
status:
147180
description: Status defines the state of the UpstreamSettingsPolicy.
148181
properties:

‎deploy/crds.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9578,6 +9578,12 @@ spec:
95789578
spec:
95799579
description: Spec defines the desired state of the UpstreamSettingsPolicy.
95809580
properties:
9581+
hashMethodKey:
9582+
description: |-
9583+
HashMethodKey defines the key used for hash-based load balancing methods.
9584+
This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
9585+
pattern: ^\$[a-z_]+$
9586+
type: string
95819587
keepAlive:
95829588
description: KeepAlive defines the keep-alive settings.
95839589
properties:
@@ -9612,6 +9618,27 @@ spec:
96129618
pattern: ^[0-9]{1,4}(ms|s|m|h)?$
96139619
type: string
96149620
type: object
9621+
loadBalancingMethod:
9622+
description: |-
9623+
LoadBalancingMethod specifies the load balancing algorithm to be used for the upstream.
9624+
If not specified, NGINX Gateway Fabric defaults to `random two least_conn`,
9625+
which differs from the standard NGINX default `round-robin`.
9626+
enum:
9627+
- round_robin
9628+
- least_conn
9629+
- ip_hash
9630+
- hash
9631+
- hash consistent
9632+
- random
9633+
- random two
9634+
- random two least_conn
9635+
- random two least_time=header
9636+
- random two least_time=last_byte
9637+
- least_time header
9638+
- least_time last_byte
9639+
- least_time header inflight
9640+
- least_time last_byte inflight
9641+
type: string
96159642
targetRefs:
96169643
description: |-
96179644
TargetRefs identifies API object(s) to apply the policy to.
@@ -9670,6 +9697,12 @@ spec:
96709697
required:
96719698
- targetRefs
96729699
type: object
9700+
x-kubernetes-validations:
9701+
- message: hashMethodKey is required when loadBalancingMethod is 'hash'
9702+
or 'hash consistent'
9703+
rule: '!(has(self.loadBalancingMethod) && (self.loadBalancingMethod
9704+
== ''hash'' || self.loadBalancingMethod == ''hash consistent'')) ||
9705+
has(self.hashMethodKey)'
96739706
status:
96749707
description: Status defines the state of the UpstreamSettingsPolicy.
96759708
properties:

‎internal/controller/manager.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func StartManager(cfg config.Config) error {
124124
mustExtractGVK := kinds.NewMustExtractGKV(scheme)
125125

126126
genericValidator := ngxvalidation.GenericValidator{}
127-
policyManager := createPolicyManager(mustExtractGVK, genericValidator)
127+
policyManager := createPolicyManager(mustExtractGVK, genericValidator, cfg.Plus)
128128

129129
plusSecrets, err := createPlusSecretMetadata(cfg, mgr.GetAPIReader())
130130
if err != nil {
@@ -140,10 +140,13 @@ func StartManager(cfg config.Config) error {
140140
GenericValidator: genericValidator,
141141
PolicyValidator: policyManager,
142142
},
143-
EventRecorder: recorder,
144-
MustExtractGVK: mustExtractGVK,
145-
PlusSecrets: plusSecrets,
146-
ExperimentalFeatures: cfg.ExperimentalFeatures,
143+
EventRecorder: recorder,
144+
MustExtractGVK: mustExtractGVK,
145+
PlusSecrets: plusSecrets,
146+
FeatureFlags: graph.FeatureFlags{
147+
Plus: cfg.Plus,
148+
Experimental: cfg.ExperimentalFeatures,
149+
},
147150
})
148151

149152
var handlerCollector handlerMetricsCollector = collectors.NewControllerNoopCollector()
@@ -323,6 +326,7 @@ func StartManager(cfg config.Config) error {
323326
func createPolicyManager(
324327
mustExtractGVK kinds.MustExtractGVK,
325328
validator validation.GenericValidator,
329+
plusEnabled bool,
326330
) *policies.CompositeValidator {
327331
cfgs := []policies.ManagerConfig{
328332
{
@@ -335,7 +339,7 @@ func createPolicyManager(
335339
},
336340
{
337341
GVK: mustExtractGVK(&ngfAPIv1alpha1.UpstreamSettingsPolicy{}),
338-
Validator: upstreamsettings.NewValidator(validator),
342+
Validator: upstreamsettings.NewValidator(validator, plusEnabled),
339343
},
340344
}
341345

‎internal/controller/nginx/config/http/config.go‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package http //nolint:revive // ignoring conflicting package name
22

33
import (
4+
ngfAPI "github.com/nginx/nginx-gateway-fabric/v2/apis/v1alpha1"
45
"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/nginx/config/shared"
56
)
67

@@ -119,11 +120,22 @@ const (
119120

120121
// Upstream holds all configuration for an HTTP upstream.
121122
type Upstream struct {
122-
Name string
123-
ZoneSize string // format: 512k, 1m
124-
StateFile string
125-
KeepAlive UpstreamKeepAlive
126-
Servers []UpstreamServer
123+
SessionPersistence UpstreamSessionPersistence
124+
Name string
125+
ZoneSize string // format: 512k, 1m
126+
StateFile string
127+
LoadBalancingMethod string
128+
HashMethodKey string
129+
KeepAlive UpstreamKeepAlive
130+
Servers []UpstreamServer
131+
}
132+
133+
// UpstreamSessionPersistence holds the session persistence configuration for an upstream.
134+
type UpstreamSessionPersistence struct {
135+
Name string
136+
Expiry string
137+
Path string
138+
SessionType string
127139
}
128140

129141
// UpstreamKeepAlive holds the keepalive configuration for an HTTP upstream.
@@ -166,3 +178,33 @@ type ServerConfig struct {
166178
Plus bool
167179
DisableSNIHostValidation bool
168180
}
181+
182+
var (
183+
OSSAllowedLBMethods = map[ngfAPI.LoadBalancingType]struct{}{
184+
ngfAPI.LoadBalancingTypeRoundRobin: {},
185+
ngfAPI.LoadBalancingTypeLeastConnection: {},
186+
ngfAPI.LoadBalancingTypeIPHash: {},
187+
ngfAPI.LoadBalancingTypeRandom: {},
188+
ngfAPI.LoadBalancingTypeHash: {},
189+
ngfAPI.LoadBalancingTypeHashConsistent: {},
190+
ngfAPI.LoadBalancingTypeRandomTwo: {},
191+
ngfAPI.LoadBalancingTypeRandomTwoLeastConnection: {},
192+
}
193+
194+
PlusAllowedLBMethods = map[ngfAPI.LoadBalancingType]struct{}{
195+
ngfAPI.LoadBalancingTypeRoundRobin: {},
196+
ngfAPI.LoadBalancingTypeLeastConnection: {},
197+
ngfAPI.LoadBalancingTypeIPHash: {},
198+
ngfAPI.LoadBalancingTypeRandom: {},
199+
ngfAPI.LoadBalancingTypeHash: {},
200+
ngfAPI.LoadBalancingTypeHashConsistent: {},
201+
ngfAPI.LoadBalancingTypeRandomTwo: {},
202+
ngfAPI.LoadBalancingTypeRandomTwoLeastConnection: {},
203+
ngfAPI.LoadBalancingTypeLeastTimeHeader: {},
204+
ngfAPI.LoadBalancingTypeLeastTimeLastByte: {},
205+
ngfAPI.LoadBalancingTypeLeastTimeHeaderInflight: {},
206+
ngfAPI.LoadBalancingTypeLeastTimeLastByteInflight: {},
207+
ngfAPI.LoadBalancingTypeRandomTwoLeastTimeHeader: {},
208+
ngfAPI.LoadBalancingTypeRandomTwoLeastTimeLastByte: {},
209+
}
210+
)

‎internal/controller/nginx/config/policies/upstreamsettings/processor.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ type Processor struct{}
1313
type UpstreamSettings struct {
1414
// ZoneSize is the zone size setting.
1515
ZoneSize string
16+
// LoadBalancingMethod is the load balancing method setting.
17+
LoadBalancingMethod string
18+
// HashMethodKey is the key to be used for hash-based load balancing methods.
19+
HashMethodKey string
1620
// KeepAlive contains the keepalive settings.
1721
KeepAlive http.UpstreamKeepAlive
1822
}
@@ -61,6 +65,14 @@ func processPolicies(pols []policies.Policy) UpstreamSettings {
6165
upstreamSettings.KeepAlive.Timeout = string(*usp.Spec.KeepAlive.Timeout)
6266
}
6367
}
68+
69+
if usp.Spec.LoadBalancingMethod != nil {
70+
upstreamSettings.LoadBalancingMethod = string(*usp.Spec.LoadBalancingMethod)
71+
}
72+
73+
if usp.Spec.HashMethodKey != nil {
74+
upstreamSettings.HashMethodKey = string(*usp.Spec.HashMethodKey)
75+
}
6476
}
6577

6678
return upstreamSettings

0 commit comments

Comments
 (0)