Use UUID instead of IP address for tracking agent (#4470)

build/Dockerfile.nginx

@@ -31,6 +31,9 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
+# Create empty /run/.containerenv file so agent can identify that it's running in a container
+RUN mkdir -p /run && touch /run/.containerenv
+
 RUN chown -R 101:1001 /etc/nginx /var/cache/nginx
 
 LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"

build/Dockerfile.nginxplus

@@ -34,6 +34,9 @@ COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
+# Create empty /run/.containerenv file so agent can identify that it's running in a container
+RUN mkdir -p /run && touch /run/.containerenv
+
 RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
 
 USER 101:1001

build/ubi/Dockerfile.nginx

@@ -63,6 +63,9 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
+# Create empty /run/.containerenv file so agent can identify that it's running in a container
+RUN mkdir -p /run && touch /run/.containerenv
+
 # Switch to non-root user
 USER 101:1001
 

build/ubi/Dockerfile.nginxplus

@@ -71,6 +71,9 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
+# Create empty /run/.containerenv file so agent can identify that it's running in a container
+RUN mkdir -p /run && touch /run/.containerenv
+
 # Switch to non-root user
 USER 101:1001
 

internal/controller/nginx/agent/command.go

@@ -84,9 +84,6 @@ func (cs *commandService) CreateConnection(
 
 	resource := req.GetResource()
 	podName := resource.GetContainerInfo().GetHostname()
-	if podName == "" {
-		podName = resource.GetHostInfo().GetHostname()
-	}
 	cs.logger.Info(fmt.Sprintf("Creating connection for nginx pod: %s", podName))
 
 	owner, _, err := cs.getPodOwner(podName)
@@ -107,7 +104,7 @@ func (cs *commandService) CreateConnection(
 		PodName:    podName,
 		InstanceID: getNginxInstanceID(resource.GetInstances()),
 	}
-	cs.connTracker.Track(gi.IPAddress, conn)
+	cs.connTracker.Track(gi.UUID, conn)
 
 	return &pb.CreateConnectionResponse{
 		Response: &pb.CommandResponse{
@@ -133,7 +130,7 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error
 	if !ok {
 		return agentgrpc.ErrStatusInvalidConnection
 	}
-	defer cs.connTracker.RemoveConnection(gi.IPAddress)
+	defer cs.connTracker.RemoveConnection(gi.UUID)
 
 	// wait for the agent to report itself and nginx
 	conn, deployment, err := cs.waitForConnection(ctx, gi)
@@ -261,7 +258,7 @@ func (cs *commandService) waitForConnection(
 		case <-timer.C:
 			return nil, nil, err
 		case <-ticker.C:
-			if conn := cs.connTracker.GetConnection(gi.IPAddress); conn.Ready() {
+			if conn := cs.connTracker.GetConnection(gi.UUID); conn.Ready() {
 				// connection has been established, now ensure that the deployment exists in the store
 				if deployment := cs.nginxDeployments.Get(conn.Parent); deployment != nil {
 					return &conn, deployment, nil
@@ -575,7 +572,7 @@ func (cs *commandService) UpdateDataPlaneStatus(
 		return nil, grpcStatus.Errorf(codes.InvalidArgument, "request does not contain nginx instanceID")
 	}
 
-	cs.connTracker.SetInstanceID(gi.IPAddress, instanceID)
+	cs.connTracker.SetInstanceID(gi.UUID, instanceID)
 
 	return &pb.UpdateDataPlaneStatusResponse{}, nil
 }

internal/controller/nginx/agent/command_test.go

@@ -73,15 +73,15 @@ func createFakeK8sClient(initObjs ...runtime.Object) (client.Client, error) {
 
 func createGrpcContext() context.Context {
 	return grpcContext.NewGrpcContext(context.Background(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 }
 
 func createGrpcContextWithCancel() (context.Context, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	return grpcContext.NewGrpcContext(ctx, grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	}), cancel
 }
 
@@ -163,32 +163,6 @@ func TestCreateConnection(t *testing.T) {
 				},
 			},
 		},
-		{
-			name: "uses regular hostname if container info not set",
-			ctx:  createGrpcContext(),
-			request: &pb.CreateConnectionRequest{
-				Resource: &pb.Resource{
-					Info: &pb.Resource_HostInfo{
-						HostInfo: &pb.HostInfo{
-							Hostname: "nginx-pod",
-						},
-					},
-					Instances: []*pb.Instance{
-						{
-							InstanceMeta: &pb.InstanceMeta{
-								InstanceId:   "nginx-id",
-								InstanceType: pb.InstanceMeta_INSTANCE_TYPE_NGINX,
-							},
-						},
-					},
-				},
-			},
-			response: &pb.CreateConnectionResponse{
-				Response: &pb.CommandResponse{
-					Status: pb.CommandResponse_COMMAND_STATUS_OK,
-				},
-			},
-		},
 		{
 			name:      "request is nil",
 			request:   nil,
@@ -268,7 +242,7 @@ func TestCreateConnection(t *testing.T) {
 			}
 
 			key, conn := connTracker.TrackArgsForCall(0)
-			g.Expect(key).To(Equal("127.0.0.1"))
+			g.Expect(key).To(Equal("1234567"))
 			g.Expect(conn).To(Equal(expConn))
 		})
 	}
@@ -1062,7 +1036,7 @@ func TestUpdateDataPlaneStatus(t *testing.T) {
 			g.Expect(connTracker.SetInstanceIDCallCount()).To(Equal(1))
 
 			key, id := connTracker.SetInstanceIDArgsForCall(0)
-			g.Expect(key).To(Equal("127.0.0.1"))
+			g.Expect(key).To(Equal("1234567"))
 			g.Expect(id).To(Equal(test.expID))
 		})
 	}

internal/controller/nginx/agent/file.go

@@ -65,7 +65,7 @@ func (fs *fileService) GetFile(
 		return nil, status.Error(codes.InvalidArgument, "invalid request")
 	}
 
-	contents, err := fs.getFileContents(req, gi.IPAddress)
+	contents, err := fs.getFileContents(req, gi.UUID)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +93,7 @@ func (fs *fileService) GetFileStream(
 		return status.Error(codes.InvalidArgument, "invalid request")
 	}
 
-	contents, err := fs.getFileContents(req, gi.IPAddress)
+	contents, err := fs.getFileContents(req, gi.UUID)
 	if err != nil {
 		return err
 	}
@@ -192,7 +192,7 @@ func (fs *fileService) UpdateOverview(
 		return &pb.UpdateOverviewResponse{}, agentgrpc.ErrStatusInvalidConnection
 	}
 
-	conn := fs.connTracker.GetConnection(gi.IPAddress)
+	conn := fs.connTracker.GetConnection(gi.UUID)
 	if conn.PodName == "" {
 		return &pb.UpdateOverviewResponse{}, status.Errorf(codes.NotFound, "connection not found")
 	}

internal/controller/nginx/agent/file_test.go

@@ -68,7 +68,7 @@ func TestGetFile(t *testing.T) {
 	fs := newFileService(logr.Discard(), depStore, connTracker)
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	req := &pb.GetFileRequest{
@@ -121,7 +121,7 @@ func TestGetFile_InvalidRequest(t *testing.T) {
 	fs := newFileService(logr.Discard(), depStore, connTracker)
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	req := &pb.GetFileRequest{
@@ -148,7 +148,7 @@ func TestGetFile_ConnectionNotFound(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	resp, err := fs.GetFile(ctx, req)
@@ -181,7 +181,7 @@ func TestGetFile_DeploymentNotFound(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	resp, err := fs.GetFile(ctx, req)
@@ -217,7 +217,7 @@ func TestGetFile_FileNotFound(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	resp, err := fs.GetFile(ctx, req)
@@ -264,7 +264,7 @@ func TestGetFileStream(t *testing.T) {
 	fs := newFileService(logr.Discard(), depStore, connTracker)
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	req := &pb.GetFileRequest{
@@ -324,7 +324,7 @@ func TestGetFileStream_InvalidRequest(t *testing.T) {
 	fs := newFileService(logr.Discard(), depStore, connTracker)
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	// no filemeta
@@ -397,7 +397,7 @@ func TestUpdateOverview(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	fs := newFileService(logr.Discard(), depStore, connTracker)
@@ -487,7 +487,7 @@ func TestUpdateOverview_ConnectionNotFound(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	resp, err := fs.UpdateOverview(ctx, req)
@@ -526,7 +526,7 @@ func TestUpdateOverview_DeploymentNotFound(t *testing.T) {
 	}
 
 	ctx := grpcContext.NewGrpcContext(t.Context(), grpcContext.GrpcInfo{
-		IPAddress: "127.0.0.1",
+		UUID: "1234567",
 	})
 
 	resp, err := fs.UpdateOverview(ctx, req)

internal/controller/nginx/agent/grpc/context/context.go

@@ -6,8 +6,8 @@ import (
 
 // GrpcInfo for storing identity information for the gRPC client.
 type GrpcInfo struct {
-	Token     string `json:"token"`      // auth token that was provided by the gRPC client
-	IPAddress string `json:"ip_address"` // ip address of the agent
+	UUID  string `json:"uuid"`  // unique identifier for the gRPC client
+	Token string `json:"token"` // auth token that was provided by the gRPC client
 }
 
 type contextGRPCKey struct{}

internal/controller/nginx/agent/grpc/context/context_test.go

@@ -13,7 +13,7 @@ func TestGrpcInfoInContext(t *testing.T) {
 	t.Parallel()
 	g := NewWithT(t)
 
-	grpcInfo := grpcContext.GrpcInfo{IPAddress: "192.168.1.1"}
+	grpcInfo := grpcContext.GrpcInfo{Token: "test"}
 
 	newCtx := grpcContext.NewGrpcContext(context.Background(), grpcInfo)
 	info, ok := grpcContext.GrpcInfoFromContext(newCtx)

internal/controller/nginx/agent/grpc/interceptor/interceptor.go

@@ -3,19 +3,16 @@ package interceptor
 import (
 	"context"
 	"fmt"
-	"net"
 	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
-	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 	authv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -111,19 +108,9 @@ func getGrpcInfo(ctx context.Context) (*grpcContext.GrpcInfo, error) {
 		return nil, status.Error(codes.Unauthenticated, "no authorization")
 	}
 
-	p, ok := peer.FromContext(ctx)
-	if !ok {
-		return nil, status.Error(codes.InvalidArgument, "no peer data")
-	}
-
-	addr, ok := p.Addr.(*net.TCPAddr)
-	if !ok {
-		panic(fmt.Sprintf("address %q was not of type net.TCPAddr", p.Addr.String()))
-	}
-
 	return &grpcContext.GrpcInfo{
-		Token:     auths[0],
-		IPAddress: addr.IP.String(),
+		UUID:  id[0],
+		Token: auths[0],
 	}, nil
 }
 
@@ -160,8 +147,7 @@ func (c ContextSetter) validateToken(ctx context.Context, gi *grpcContext.GrpcIn
 
 	var podList corev1.PodList
 	opts := &client.ListOptions{
-		FieldSelector: fields.SelectorFromSet(fields.Set{"status.podIP": gi.IPAddress}),
-		Namespace:     usernameItems[2],
+		Namespace: usernameItems[2],
 		LabelSelector: labels.Set(map[string]string{
 			controller.AppNameLabel: usernameItems[3],
 		}).AsSelector(),
@@ -178,8 +164,8 @@ func (c ContextSetter) validateToken(ctx context.Context, gi *grpcContext.GrpcIn
 		}
 	}
 
-	if runningCount != 1 {
-		msg := fmt.Sprintf("expected a single Running pod with IP address %q, but found %d", gi.IPAddress, runningCount)
+	if runningCount < 1 {
+		msg := fmt.Sprintf("no running pods found for service account %s/%s", usernameItems[2], usernameItems[3])
 		return nil, status.Error(codes.Unauthenticated, msg)
 	}