Misleading 404 logs in IMDS mode when querying EC2 Metadata without a public IP (#1058)

* assume public ip and hostname ec2 metadata can be absent

Signed-off-by: Xabier Napal <xabier.napal@dvzr.io>

* add ec2metadata new tests to check for optional paths

Signed-off-by: Xabier Napal <xabier.napal@dvzr.io>

---------

Signed-off-by: Xabier Napal <xabier.napal@dvzr.io>

Author: xabinapal

pkg/ec2metadata/ec2metadata.go

@@ -218,7 +218,7 @@ func (e *Service) GetASGTargetLifecycleState() (state string, err error) {
 }
 
 // GetMetadataInfo generic function for retrieving ec2 metadata
-func (e *Service) GetMetadataInfo(path string) (info string, err error) {
+func (e *Service) GetMetadataInfo(path string, allowMissing bool) (info string, err error) {
 	metadataInfo := ""
 	resp, err := e.Request(path)
 	if err != nil {
@@ -232,8 +232,12 @@ func (e *Service) GetMetadataInfo(path string) (info string, err error) {
 		}
 		metadataInfo = string(body)
 		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-			log.Info().Msgf("Metadata response status code: %d. Body: %s", resp.StatusCode, metadataInfo)
-			return "", fmt.Errorf("Metadata request received http status code: %d", resp.StatusCode)
+			if resp.StatusCode != 404 || !allowMissing {
+				log.Info().Msgf("Metadata response status code: %d. Body: %s", resp.StatusCode, metadataInfo)
+				return "", fmt.Errorf("Metadata request received http status code: %d", resp.StatusCode)
+			} else {
+				return "", nil
+			}
 		}
 	}
 	return metadataInfo, nil
@@ -351,26 +355,26 @@ func retry(attempts int, sleep time.Duration, httpReq func() (*http.Response, er
 // GetNodeMetadata attempts to gather additional ec2 instance information from the metadata service
 func (e *Service) GetNodeMetadata() NodeMetadata {
 	metadata := NodeMetadata{}
-	identityDoc, err := e.GetMetadataInfo(IdentityDocPath)
+	identityDoc, err := e.GetMetadataInfo(IdentityDocPath, false)
 	if err != nil {
 		log.Err(err).Msg("Unable to fetch metadata from IMDS")
 		return metadata
 	}
 	err = json.NewDecoder(strings.NewReader(identityDoc)).Decode(&metadata)
 	if err != nil {
 		log.Warn().Msg("Unable to fetch instance identity document from ec2 metadata")
-		metadata.InstanceID, _ = e.GetMetadataInfo(InstanceIDPath)
-		metadata.InstanceType, _ = e.GetMetadataInfo(InstanceTypePath)
-		metadata.LocalIP, _ = e.GetMetadataInfo(LocalIPPath)
-		metadata.AvailabilityZone, _ = e.GetMetadataInfo(AZPlacementPath)
+		metadata.InstanceID, _ = e.GetMetadataInfo(InstanceIDPath, false)
+		metadata.InstanceType, _ = e.GetMetadataInfo(InstanceTypePath, false)
+		metadata.LocalIP, _ = e.GetMetadataInfo(LocalIPPath, false)
+		metadata.AvailabilityZone, _ = e.GetMetadataInfo(AZPlacementPath, false)
 		if len(metadata.AvailabilityZone) > 1 {
 			metadata.Region = metadata.AvailabilityZone[0 : len(metadata.AvailabilityZone)-1]
 		}
 	}
-	metadata.InstanceLifeCycle, _ = e.GetMetadataInfo(InstanceLifeCycle)
-	metadata.LocalHostname, _ = e.GetMetadataInfo(LocalHostnamePath)
-	metadata.PublicHostname, _ = e.GetMetadataInfo(PublicHostnamePath)
-	metadata.PublicIP, _ = e.GetMetadataInfo(PublicIPPath)
+	metadata.InstanceLifeCycle, _ = e.GetMetadataInfo(InstanceLifeCycle, false)
+	metadata.LocalHostname, _ = e.GetMetadataInfo(LocalHostnamePath, false)
+	metadata.PublicHostname, _ = e.GetMetadataInfo(PublicHostnamePath, true)
+	metadata.PublicIP, _ = e.GetMetadataInfo(PublicIPPath, true)
 
 	log.Info().Interface("metadata", metadata).Msg("Startup Metadata Retrieved")
 

pkg/ec2metadata/ec2metadata_test.go

@@ -609,16 +609,66 @@ func TestGetMetadataServiceRequest404(t *testing.T) {
 	// Use URL from our local test server
 	imds := ec2metadata.New(server.URL, 1)
 
-	_, err := imds.GetMetadataInfo(requestPath)
+	_, err := imds.GetMetadataInfo(requestPath, false)
 
-	h.Assert(t, err != nil, "Expected error to be nil but it was not")
+	h.Assert(t, err != nil, "Error expected because request errored with 404")
+}
+
+func TestGetMetadataServiceRequest404AllowMissing(t *testing.T) {
+	var requestPath string = "/latest/meta-data/instance-type"
+
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Add("X-aws-ec2-metadata-token-ttl-seconds", "100")
+		if req.URL.String() == "/latest/api/token" {
+			rw.WriteHeader(200)
+			_, err := rw.Write([]byte(`token`))
+			h.Ok(t, err)
+			return
+		}
+		h.Equals(t, req.Header.Get("X-aws-ec2-metadata-token"), "token")
+		h.Equals(t, req.URL.String(), requestPath)
+		rw.WriteHeader(404)
+	}))
+	defer server.Close()
+
+	// Use URL from our local test server
+	imds := ec2metadata.New(server.URL, 1)
+
+	_, err := imds.GetMetadataInfo(requestPath, true)
+
+	h.Assert(t, err == nil, "Expected error to be nil but it was not")
+}
+
+func TestGetMetadataServiceRequest500AllowMissing(t *testing.T) {
+	var requestPath string = "/latest/meta-data/instance-type"
+
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Add("X-aws-ec2-metadata-token-ttl-seconds", "100")
+		if req.URL.String() == "/latest/api/token" {
+			rw.WriteHeader(200)
+			_, err := rw.Write([]byte(`token`))
+			h.Ok(t, err)
+			return
+		}
+		h.Equals(t, req.Header.Get("X-aws-ec2-metadata-token"), "token")
+		h.Equals(t, req.URL.String(), requestPath)
+		rw.WriteHeader(500)
+	}))
+	defer server.Close()
+
+	// Use URL from our local test server
+	imds := ec2metadata.New(server.URL, 1)
+
+	_, err := imds.GetMetadataInfo(requestPath, true)
+
+	h.Assert(t, err != nil, "Error expected because request errored with 500")
 }
 
 func TestGetMetadataServiceRequestFailure(t *testing.T) {
 	// Use URL from our local test server
 	imds := ec2metadata.New("/some-path-that-will-error", 1)
 
-	_, err := imds.GetMetadataInfo("/latest/meta-data/instance-type")
+	_, err := imds.GetMetadataInfo("/latest/meta-data/instance-type", false)
 	h.Assert(t, err != nil, "Error expected because no server should be running")
 }
 
@@ -643,7 +693,7 @@ func TestGetMetadataServiceSuccess(t *testing.T) {
 	// Use URL from our local test server
 	imds := ec2metadata.New(server.URL, 1)
 
-	resp, err := imds.GetMetadataInfo(requestPath)
+	resp, err := imds.GetMetadataInfo(requestPath, false)
 
 	h.Ok(t, err)
 	h.Equals(t, `x1.32xlarge`, resp)