[8.19](backport #6491) Fix checkin endpoint gzip handling (#6526)

* Fix checkin endpoint gzip handling (#6491)

Detect Content-Encoding: gzip header when validating checkin bodies and uncompress the body if detected.

(cherry picked from commit 122fac1)

# Conflicts:
#	internal/pkg/api/handleCheckin_test.go

* Fix backport

* Fix backport test

* Fix panic

---------

Co-authored-by: Michel Laterman <82832767+michel-laterman@users.noreply.github.com>
Co-authored-by: michel-laterman <michel.laterman@elastic.co>

Author: mergify[bot]

changelog/fragments/1772577135-Fix-checkin-compression-support.yaml

@@ -0,0 +1,33 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix checkin endpoint compression support
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Adds support for gzip compressed requests to the checkin endpoint.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: fleet-server
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/fleet-server/pull/6491
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/api/handleCheckin.go

@@ -181,9 +181,20 @@ func (ct *CheckinT) validateRequest(zlog zerolog.Logger, w http.ResponseWriter,
 	}
 	readCounter := datacounter.NewReaderCounter(body)
 
+	// Decompress the body when the client signals Content-Encoding: gzip.
+	var bodyReader io.Reader = readCounter
+	if r.Header.Get("Content-Encoding") == kEncodingGzip {
+		gr, err := gzip.NewReader(readCounter)
+		if err != nil {
+			return validatedCheckin{}, &BadRequestErr{msg: "unable to create gzip reader for request body", nextErr: err}
+		}
+		defer gr.Close()
+		bodyReader = gr
+	}
+
 	var val validatedCheckin
 	var req CheckinRequest
-	decoder := json.NewDecoder(readCounter)
+	decoder := json.NewDecoder(bodyReader)
 	if err := decoder.Decode(&req); err != nil {
 		return val, &BadRequestErr{msg: "unable to decode checkin request", nextErr: err}
 	}
@@ -643,6 +654,8 @@ func (ct *CheckinT) writeResponse(zlog zerolog.Logger, w http.ResponseWriter, r
 	return err
 }
 
+// acceptsEncoding reports whether the request includes the passed encoding.
+// Only an exact match is checked as that is all the agent will send.
 func acceptsEncoding(r *http.Request, encoding string) bool {
 	for _, v := range r.Header.Values("Accept-Encoding") {
 		if v == encoding {

internal/pkg/api/handleCheckin_test.go

@@ -7,7 +7,9 @@
 package api
 
 import (
+	"bytes"
 	"compress/flate"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -1073,6 +1075,44 @@ func TestValidateCheckinRequest(t *testing.T) {
 			},
 			expValid: validatedCheckin{},
 		},
+		{
+			name: "gzip-compressed request body is decompressed before JSON decoding",
+			req: func() *http.Request {
+				var buf bytes.Buffer
+				gz := gzip.NewWriter(&buf)
+				_, _ = gz.Write([]byte(`{"status": "online", "message": "test message"}`))
+				_ = gz.Close()
+				return &http.Request{
+					Header: http.Header{"Content-Encoding": []string{"gzip"}},
+					Body:   io.NopCloser(&buf),
+				}
+			}(),
+			cfg: &config.Server{
+				Limits: config.ServerLimits{
+					CheckinLimit: config.Limit{
+						MaxBody: 0,
+					},
+				},
+			},
+			expErr:   nil,
+			expValid: validatedCheckin{},
+		},
+		{
+			name: "invalid gzip request body returns bad request error",
+			req: &http.Request{
+				Header: http.Header{"Content-Encoding": []string{"gzip"}},
+				Body:   io.NopCloser(strings.NewReader(`not gzip data`)),
+			},
+			cfg: &config.Server{
+				Limits: config.ServerLimits{
+					CheckinLimit: config.Limit{
+						MaxBody: 0,
+					},
+				},
+			},
+			expErr:   &BadRequestErr{msg: "unable to create gzip reader for request body", nextErr: gzip.ErrHeader},
+			expValid: validatedCheckin{},
+		},
 	}
 
 	for _, tc := range tests {
@@ -1081,7 +1121,7 @@ func TestValidateCheckinRequest(t *testing.T) {
 			assert.NoError(t, err)
 			wr := httptest.NewRecorder()
 			logger := testlog.SetLogger(t)
-			valid, err := checkin.validateRequest(logger, wr, tc.req, time.Time{}, nil)
+			valid, err := checkin.validateRequest(logger, wr, tc.req, time.Time{}, &model.Agent{LocalMetadata: json.RawMessage(`{}`)})
 			if tc.expErr == nil {
 				assert.NoError(t, err)
 			} else {
@@ -1090,8 +1130,8 @@ func TestValidateCheckinRequest(t *testing.T) {
 				// we will end up with false positives.
 				assert.Equal(t, tc.expErr.Error(), err.Error())
 				assert.ErrorAs(t, err, &tc.expErr)
+				assert.Equal(t, tc.expValid, valid)
 			}
-			assert.Equal(t, tc.expValid, valid)
 		})
 	}
 }

internal/pkg/api/openapi.gen.go

@@ -50,7 +50,7 @@ const (
 	serverVersion = "8.0.0"
 	localhost     = "localhost"
 
-	testWaitServerUp = 3 * time.Second
+	testWaitServerUp = 10 * time.Second
 
 	enrollBody = `{
 	    "type": "PERMANENT",
@@ -242,6 +242,7 @@ func startTestServer(t *testing.T, ctx context.Context, policyD model.PolicyData
 }
 
 func (s *tserver) waitServerUp(ctx context.Context, dur time.Duration) error {
+	zlog := zerolog.Ctx(ctx)
 	ctx, cancel := context.WithTimeout(ctx, dur)
 	defer cancel()
 
@@ -268,6 +269,7 @@ func (s *tserver) waitServerUp(ctx context.Context, dur time.Duration) error {
 			return false, err
 		}
 
+		zlog.Info().Msgf("test wait for fleet-server up status: %s", status.Status)
 		return status.Status == "HEALTHY", nil
 	}
 

internal/pkg/server/fleet_integration_test.go

@@ -1372,11 +1372,18 @@ paths:
         - name: Accept-Encoding
           in: header
           description: |
-            If the agent is able to accept encoded responses.
-            Used to indicate if GZIP compression may be used by the server.
-            The elastic-agent does not use the accept-encoding header.
+            Indicates encodings the agent can accept. The only encoding supported by the server currently is gzip.
+            Comma-separated directive lists and RFC 7231 quality values (e.g. "gzip;q=1.0, deflate;q=0.5") are both accepted.
           schema:
             type: string
+        - name: Content-Encoding
+          in: header
+          description: |
+            Signals that the request body has been compressed. Server currently only supports gzip compression.
+          schema:
+            type: string
+            enum:
+              - gzip
         - $ref: "#/components/parameters/userAgent"
         - $ref: "#/components/parameters/requestId"
         - $ref: "#/components/parameters/apiVersion"
@@ -1412,12 +1419,19 @@ paths:
           description: Agent checkin successful. May include actions.
           headers:
             Content-Encoding:
-              description: Responses may be compressed if the accept encoding indicates it. Currently not used by the agent.
+              description: |
+                Present when fleet-server has gzip-compressed the response body.
+                Compression is applied when all three conditions are met: the serialised
+                response exceeds the configured compression threshold, the server compression
+                level is not NoCompression, and the request's Accept-Encoding header
+                advertises gzip support.
               schema:
                 type: string
+                enum:
+                  - gzip
               examples:
                 gzip:
-                  description: Response is gzip encoded as the request headers allowed it.
+                  description: Response body is gzip-compressed.
                   value: gzip
             Elastic-Api-Version:
               $ref: "#/components/headers/apiVersion"