mirror of
https://github.com/fleetdm/fleet
synced 2026-04-21 13:37:30 +00:00
Increase body size limits for osquerylog and osquery/dist/write endpoints (#40946)
Resolves #40813 * Added configurable body size limits for the `/api/osquery/log`, `/api/osquery/distributed/write` and `/api/osquery/config` endpoints. * Fixed false positive `PayloadTooLargeError` errors. --------- Co-authored-by: Lucas Manuel Rodriguez <lucas@fleetdm.com>
This commit is contained in:
parent
42678a7674
commit
64585f8bdd
9 changed files with 381 additions and 10 deletions
2
changes/40813-increase-osquery-log-endpoint-body-size
Normal file
2
changes/40813-increase-osquery-log-endpoint-body-size
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
* Added configurable body size limits for the `/api/osquery/log` and `/api/osquery/distributed/write` endpoints.
|
||||
* Fixed false positive `PayloadTooLargeError` errors.
|
||||
|
|
@ -1259,6 +1259,30 @@ The minimum time difference between the software's "last opened at" timestamp re
|
|||
min_software_last_opened_at_diff: 4h
|
||||
```
|
||||
|
||||
### osquery_max_log_write_body_size
|
||||
|
||||
Maximum HTTP request body size accepted by the `osquery/log` endpoint. Increase this if osquery agents are submitting log batches that exceed the default limit. Accepts a byte size with a unit suffix (e.g. `10MiB`, `500KB`). A value of `0` uses the built-in default. Values smaller than the server-wide minimum request body size are silently raised to that minimum.
|
||||
|
||||
- Default value: `10MiB`
|
||||
- Environment variable: `FLEET_OSQUERY_MAX_LOG_WRITE_BODY_SIZE`
|
||||
- Config file format:
|
||||
```yaml
|
||||
osquery:
|
||||
max_log_write_body_size: 20MiB
|
||||
```
|
||||
|
||||
### osquery_max_distributed_write_body_size
|
||||
|
||||
Maximum HTTP request body size accepted by the `osquery/distributed/write` endpoint. Increase this if osquery agents are submitting distributed query results that exceed the default limit. Accepts a byte size with a unit suffix (e.g. `10MiB`, `500KB`). A value of `0` uses the built-in default. Values smaller than the server-wide minimum request body size are silently raised to that minimum.
|
||||
|
||||
- Default value: `5MiB`
|
||||
- Environment variable: `FLEET_OSQUERY_MAX_DISTRIBUTED_WRITE_BODY_SIZE`
|
||||
- Config file format:
|
||||
```yaml
|
||||
osquery:
|
||||
max_distributed_write_body_size: 10MiB
|
||||
```
|
||||
|
||||
## External activity audit logging
|
||||
|
||||
> Available in Fleet Premium. Activity information is available for all Fleet Free and Fleet Premium instances using the [Activities API](https://fleetdm.com/docs/using-fleet/rest-api#activities).
|
||||
|
|
|
|||
|
|
@ -211,6 +211,13 @@ type OsqueryConfig struct {
|
|||
AsyncHostRedisPopCount int `yaml:"async_host_redis_pop_count"`
|
||||
AsyncHostRedisScanKeysCount int `yaml:"async_host_redis_scan_keys_count"`
|
||||
MinSoftwareLastOpenedAtDiff time.Duration `yaml:"min_software_last_opened_at_diff"`
|
||||
|
||||
// MaxLogWriteBodySize overrides the default body size limit for the
|
||||
// osquery/log endpoint. A value of 0 means use the built-in default.
|
||||
MaxLogWriteBodySize int64 `yaml:"max_log_write_body_size"`
|
||||
// MaxDistributedWriteBodySize overrides the default body size limit for the
|
||||
// osquery/distributed/write endpoint. A value of 0 means use the built-in default.
|
||||
MaxDistributedWriteBodySize int64 `yaml:"max_distributed_write_body_size"`
|
||||
}
|
||||
|
||||
// AsyncTaskName is the type of names that identify tasks supporting
|
||||
|
|
@ -1296,6 +1303,10 @@ func (man Manager) addConfigs() {
|
|||
"Batch size to scan redis keys in async collection")
|
||||
man.addConfigDuration("osquery.min_software_last_opened_at_diff", 1*time.Hour,
|
||||
"Minimum time difference of the software's last opened timestamp (compared to the last one saved) to trigger an update to the database")
|
||||
man.addConfigByteSize("osquery.max_log_write_body_size", "0",
|
||||
"Maximum body size for the osquery/log endpoint (e.g. 10MiB, 500KB). 0 means use the built-in default (10MiB). Values below the server minimum request body size are raised to that minimum.")
|
||||
man.addConfigByteSize("osquery.max_distributed_write_body_size", "0",
|
||||
"Maximum body size for the osquery/distributed/write endpoint (e.g. 10MiB, 500KB). 0 means use the built-in default (5MiB). Values below the server minimum request body size are raised to that minimum.")
|
||||
|
||||
// Activities
|
||||
man.addConfigBool("activity.enable_audit_log", false,
|
||||
|
|
@ -1733,6 +1744,8 @@ func (man Manager) LoadConfig() FleetConfig {
|
|||
AsyncHostRedisPopCount: man.getConfigInt("osquery.async_host_redis_pop_count"),
|
||||
AsyncHostRedisScanKeysCount: man.getConfigInt("osquery.async_host_redis_scan_keys_count"),
|
||||
MinSoftwareLastOpenedAtDiff: man.getConfigDuration("osquery.min_software_last_opened_at_diff"),
|
||||
MaxLogWriteBodySize: man.getConfigByteSize("osquery.max_log_write_body_size"),
|
||||
MaxDistributedWriteBodySize: man.getConfigByteSize("osquery.max_distributed_write_body_size"),
|
||||
},
|
||||
Activity: ActivityConfig{
|
||||
EnableAuditLog: man.getConfigBool("activity.enable_audit_log"),
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ const (
|
|||
MaxEULASize int64 = 25 * units.MiB
|
||||
MaxMDMCommandSize int64 = 2 * units.MiB
|
||||
// MaxMultiScriptQuerySize, sets a max size for payloads that take multiple scripts and SQL queries.
|
||||
MaxMultiScriptQuerySize int64 = 5 * units.MiB
|
||||
MaxMicrosoftMDMSize int64 = 2 * units.MiB
|
||||
MaxOsqueryDistributedWriteSize int64 = 5 * units.MiB
|
||||
MaxMultiScriptQuerySize int64 = 5 * units.MiB
|
||||
MaxMicrosoftMDMSize int64 = 2 * units.MiB
|
||||
|
||||
DefaultMaxOsqueryDistributedWriteSize int64 = 5 * units.MiB
|
||||
DefaultMaxOsqueryLogWriteSize int64 = 10 * units.MiB
|
||||
)
|
||||
|
|
|
|||
|
|
@ -373,6 +373,23 @@ type requestValidator interface {
|
|||
// query parameter decoding logic.
|
||||
//
|
||||
// If adding a new way to parse/decode the requset, make sure to wrap the body in a limited reader with the maxRequestBodySize
|
||||
|
||||
// limitExhaustedBody returns true when the LimitedReader was the cause of an
|
||||
// unexpected EOF — i.e. the underlying body actually had more data beyond the
|
||||
// limit. It does this by attempting a single-byte read from the underlying
|
||||
// reader (limitedReader.R) which bypasses the limit wrapper. A successful read
|
||||
// means the body exceeded the limit; an immediate EOF means the body ended
|
||||
// exactly at the limit (malformed JSON, not an oversized payload).
|
||||
//
|
||||
// This is used to avoid false-positive PayloadTooLargeError responses for
|
||||
// bodies whose JSON is malformed and happen to be exactly maxRequestBodySize
|
||||
// bytes long.
|
||||
func limitExhaustedBody(limitedReader *io.LimitedReader) bool {
|
||||
var peek [1]byte
|
||||
n, _ := limitedReader.R.Read(peek[:])
|
||||
return n > 0
|
||||
}
|
||||
|
||||
func MakeDecoder(
|
||||
iface interface{},
|
||||
jsonUnmarshal func(body io.Reader, req any) error,
|
||||
|
|
@ -389,8 +406,9 @@ func MakeDecoder(
|
|||
}
|
||||
if rd, ok := iface.(RequestDecoder); ok {
|
||||
return func(ctx context.Context, r *http.Request) (interface{}, error) {
|
||||
var limitedReader *io.LimitedReader
|
||||
if maxRequestBodySize != -1 {
|
||||
limitedReader := io.LimitReader(r.Body, maxRequestBodySize).(*io.LimitedReader)
|
||||
limitedReader = io.LimitReader(r.Body, maxRequestBodySize).(*io.LimitedReader)
|
||||
|
||||
r.Body = &LimitedReadCloser{
|
||||
LimitedReader: limitedReader,
|
||||
|
|
@ -398,7 +416,7 @@ func MakeDecoder(
|
|||
}
|
||||
}
|
||||
ret, err := rd.DecodeRequest(ctx, r)
|
||||
if err != nil && errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
if err != nil && errors.Is(err, io.ErrUnexpectedEOF) && limitedReader != nil && limitedReader.N == 0 && limitExhaustedBody(limitedReader) {
|
||||
return nil, platform_http.PayloadTooLargeError{ContentLength: r.Header.Get("Content-Length"), MaxRequestSize: maxRequestBodySize}
|
||||
}
|
||||
return ret, err
|
||||
|
|
@ -414,8 +432,9 @@ func MakeDecoder(
|
|||
v := reflect.New(t)
|
||||
nilBody := false
|
||||
|
||||
var limitedReader *io.LimitedReader
|
||||
if maxRequestBodySize != -1 {
|
||||
limitedReader := io.LimitReader(r.Body, maxRequestBodySize).(*io.LimitedReader)
|
||||
limitedReader = io.LimitReader(r.Body, maxRequestBodySize).(*io.LimitedReader)
|
||||
|
||||
r.Body = &LimitedReadCloser{
|
||||
LimitedReader: limitedReader,
|
||||
|
|
@ -441,7 +460,7 @@ func MakeDecoder(
|
|||
req := v.Interface()
|
||||
err := jsonUnmarshal(body, req)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) && limitedReader != nil && limitedReader.N == 0 && limitExhaustedBody(limitedReader) {
|
||||
return nil, platform_http.PayloadTooLargeError{ContentLength: r.Header.Get("Content-Length"), MaxRequestSize: maxRequestBodySize}
|
||||
}
|
||||
|
||||
|
|
@ -503,7 +522,10 @@ func MakeDecoder(
|
|||
err := decodeBody(ctx, r, v, body)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return nil, platform_http.PayloadTooLargeError{ContentLength: r.Header.Get("Content-Length"), MaxRequestSize: maxRequestBodySize}
|
||||
if limitedReader != nil && limitedReader.N == 0 && limitExhaustedBody(limitedReader) {
|
||||
return nil, platform_http.PayloadTooLargeError{ContentLength: r.Header.Get("Content-Length"), MaxRequestSize: maxRequestBodySize}
|
||||
}
|
||||
return nil, BadRequestErr("json decoder error", err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,14 @@ package endpointer
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
authz_ctx "github.com/fleetdm/fleet/v4/server/contexts/authz"
|
||||
|
|
@ -12,6 +17,7 @@ import (
|
|||
"github.com/go-kit/kit/endpoint"
|
||||
kithttp "github.com/go-kit/kit/transport/http"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
@ -123,3 +129,183 @@ func (n nopEP) CallHandlerFunc(f testHandlerFunc, ctx context.Context, request a
|
|||
func (n nopEP) Service() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRegisterDeprecatedPathAliases(t *testing.T) {
|
||||
// Set up a router and register a primary endpoint via CommonEndpointer.
|
||||
r := mux.NewRouter()
|
||||
registry := NewHandlerRegistry()
|
||||
versions := []string{"v1", "2022-04"}
|
||||
|
||||
authMiddleware := func(next endpoint.Endpoint) endpoint.Endpoint {
|
||||
return func(ctx context.Context, req any) (any, error) {
|
||||
if authctx, ok := authz_ctx.FromContext(ctx); ok {
|
||||
authctx.SetChecked()
|
||||
}
|
||||
return next(ctx, req)
|
||||
}
|
||||
}
|
||||
|
||||
ce := &CommonEndpointer[testHandlerFunc]{
|
||||
EP: nopEP{},
|
||||
MakeDecoderFn: func(iface any, requestBodySizeLimit int64) kithttp.DecodeRequestFunc {
|
||||
return func(ctx context.Context, r *http.Request) (request any, err error) {
|
||||
return nopRequest{}, nil
|
||||
}
|
||||
},
|
||||
EncodeFn: func(ctx context.Context, w http.ResponseWriter, i any) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return nil
|
||||
},
|
||||
AuthMiddleware: authMiddleware,
|
||||
Router: r,
|
||||
Versions: versions,
|
||||
HandlerRegistry: registry,
|
||||
}
|
||||
|
||||
// Register the primary endpoint.
|
||||
ce.GET("/api/_version_/fleet/fleets", func(ctx context.Context, request any) (platform_http.Errorer, error) {
|
||||
return nopResponse{}, nil
|
||||
}, nil)
|
||||
|
||||
// Register a deprecated alias for it.
|
||||
RegisterDeprecatedPathAliases(r, versions, registry, []DeprecatedPathAlias{
|
||||
{
|
||||
Method: "GET",
|
||||
PrimaryPath: "/api/_version_/fleet/fleets",
|
||||
DeprecatedPaths: []string{"/api/_version_/fleet/teams"},
|
||||
},
|
||||
})
|
||||
|
||||
s := httptest.NewServer(r)
|
||||
t.Cleanup(s.Close)
|
||||
|
||||
// Both the primary and deprecated paths should return 200.
|
||||
for _, path := range []string{"/api/v1/fleet/fleets", "/api/v1/fleet/teams", "/api/latest/fleet/teams"} {
|
||||
resp, err := http.Get(s.URL + path)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "path %s should return 200", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogDeprecatedPathAlias(t *testing.T) {
|
||||
// Without deprecated path info in context, LogDeprecatedPathAlias is a no-op.
|
||||
lc := &logging.LoggingContext{}
|
||||
ctx := logging.NewContext(context.Background(), lc)
|
||||
ctx2 := LogDeprecatedPathAlias(ctx, nil)
|
||||
require.Equal(t, ctx, ctx2, "should return same context when no deprecated path info")
|
||||
require.Empty(t, lc.Extras)
|
||||
|
||||
// With deprecated path info, it should set warn level and extras.
|
||||
ctx = context.WithValue(ctx, deprecatedPathInfoKey{}, deprecatedPathInfo{
|
||||
deprecatedPath: "/api/_version_/fleet/teams",
|
||||
primaryPath: "/api/_version_/fleet/fleets",
|
||||
})
|
||||
LogDeprecatedPathAlias(ctx, nil)
|
||||
|
||||
// Extras is a flat []interface{} of key-value pairs.
|
||||
require.Len(t, lc.Extras, 4) // "deprecated_path", value, "deprecation_warning", value
|
||||
require.Equal(t, "deprecated_path", lc.Extras[0])
|
||||
require.Equal(t, "/api/_version_/fleet/teams", lc.Extras[1])
|
||||
require.Equal(t, "deprecation_warning", lc.Extras[2])
|
||||
require.Contains(t, lc.Extras[3], "deprecated")
|
||||
|
||||
// ForceLevel should be set to Warn.
|
||||
require.NotNil(t, lc.ForceLevel)
|
||||
require.Equal(t, slog.LevelWarn, *lc.ForceLevel)
|
||||
}
|
||||
|
||||
func TestRegisterDeprecatedPathAliasesPanicsOnMissing(t *testing.T) {
|
||||
r := mux.NewRouter()
|
||||
registry := NewHandlerRegistry()
|
||||
versions := []string{"v1"}
|
||||
|
||||
require.Panics(t, func() {
|
||||
RegisterDeprecatedPathAliases(r, versions, registry, []DeprecatedPathAlias{
|
||||
{
|
||||
Method: "GET",
|
||||
PrimaryPath: "/api/_version_/fleet/nonexistent",
|
||||
DeprecatedPaths: []string{"/api/_version_/fleet/old"},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func defaultJSONUnmarshal(body io.Reader, req any) error {
|
||||
return json.NewDecoder(body).Decode(req)
|
||||
}
|
||||
|
||||
type testRequestDecoderType struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
func (d *testRequestDecoderType) DecodeRequest(ctx context.Context, r *http.Request) (any, error) {
|
||||
err := json.NewDecoder(r.Body).Decode(d)
|
||||
return d, err
|
||||
}
|
||||
|
||||
// TestMakeDecoderRequestDecoderFalsePositive verifies that a body containing
|
||||
// malformed JSON that is within the size limit does not produce a
|
||||
// PayloadTooLargeError (false positive)
|
||||
func TestMakeDecoderRequestDecoderFalsePositive(t *testing.T) {
|
||||
const limit = 50
|
||||
|
||||
makeDecoder := func(limit int64) kithttp.DecodeRequestFunc {
|
||||
return MakeDecoder(&testRequestDecoderType{}, defaultJSONUnmarshal, nil, nil, nil, nil, limit)
|
||||
}
|
||||
|
||||
t.Run("malformed JSON within limit returns decode error, not 413", func(t *testing.T) {
|
||||
body := strings.NewReader(`{"data": "truncated`) // malformed, within limit
|
||||
r := httptest.NewRequest("POST", "/", body)
|
||||
_, err := makeDecoder(limit)(context.Background(), r)
|
||||
require.Error(t, err)
|
||||
var ple platform_http.PayloadTooLargeError
|
||||
require.False(t, errors.As(err, &ple), "malformed body within limit must not produce PayloadTooLargeError")
|
||||
})
|
||||
|
||||
t.Run("body over limit returns 413", func(t *testing.T) {
|
||||
big := `{"data":"` + strings.Repeat("x", limit+10) + `"}`
|
||||
body := strings.NewReader(big)
|
||||
r := httptest.NewRequest("POST", "/", body)
|
||||
_, err := makeDecoder(limit)(context.Background(), r)
|
||||
require.Error(t, err)
|
||||
var ple platform_http.PayloadTooLargeError
|
||||
require.True(t, errors.As(err, &ple), "body over limit must produce PayloadTooLargeError, got: %v", err)
|
||||
})
|
||||
|
||||
t.Run("malformed JSON exactly at limit returns decode error, not 413", func(t *testing.T) {
|
||||
// Build a body of exactly `limit` bytes that is malformed JSON (no closing
|
||||
// brace). The LimitedReader is exhausted (N==0), but a peek at the
|
||||
// underlying reader returns EOF — the body ended at the limit, it was not
|
||||
// cut short. Must not produce PayloadTooLargeError.
|
||||
prefix := `{"data":"`
|
||||
body := strings.NewReader(prefix + strings.Repeat("x", limit-len(prefix))) // exactly limit bytes, no closing
|
||||
r := httptest.NewRequest("POST", "/", body)
|
||||
_, err := makeDecoder(limit)(context.Background(), r)
|
||||
require.Error(t, err)
|
||||
var ple platform_http.PayloadTooLargeError
|
||||
require.False(t, errors.As(err, &ple), "malformed body exactly at limit must not produce PayloadTooLargeError")
|
||||
})
|
||||
|
||||
t.Run("body over limit without Content-Length returns 413", func(t *testing.T) {
|
||||
// Simulate a chunked request (no Content-Length) whose body exceeds the
|
||||
// limit. The peek at the underlying reader finds more data → 413.
|
||||
big := `{"data":"` + strings.Repeat("x", limit+10) + `"}`
|
||||
r := httptest.NewRequest("POST", "/", strings.NewReader(big))
|
||||
r.ContentLength = -1 // strip the Content-Length that httptest set
|
||||
_, err := makeDecoder(limit)(context.Background(), r)
|
||||
require.Error(t, err)
|
||||
var ple platform_http.PayloadTooLargeError
|
||||
require.True(t, errors.As(err, &ple), "over-limit body without Content-Length must produce PayloadTooLargeError, got: %v", err)
|
||||
})
|
||||
|
||||
t.Run("valid body within limit is decoded successfully", func(t *testing.T) {
|
||||
body := strings.NewReader(`{"data":"hello"}`)
|
||||
r := httptest.NewRequest("POST", "/", body)
|
||||
result, err := makeDecoder(limit)(context.Background(), r)
|
||||
require.NoError(t, err)
|
||||
rd, ok := result.(*testRequestDecoderType)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "hello", rd.Data)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package service
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -263,6 +264,7 @@ func TestUniversalDecoderSizeLimit(t *testing.T) {
|
|||
}
|
||||
decoder := makeDecoder(universalStruct{}, platform_http.MaxRequestBodySize)
|
||||
|
||||
// Body larger than the limit should return PayloadTooLargeError.
|
||||
largeBody := `{"key": "` + strings.Repeat("A", int(platform_http.MaxRequestBodySize)+1) + `"}`
|
||||
req := httptest.NewRequest("POST", "/target?per_page=77&page=4", strings.NewReader(largeBody))
|
||||
req = mux.SetURLVars(req, map[string]string{"some-id": "123"})
|
||||
|
|
@ -271,6 +273,19 @@ func TestUniversalDecoderSizeLimit(t *testing.T) {
|
|||
require.Error(t, err)
|
||||
require.IsType(t, platform_http.PayloadTooLargeError{}, err)
|
||||
|
||||
// Body within the limit but with broken JSON
|
||||
incompleteBody := `{"key": "` + strings.Repeat("A", 100) // missing closing "}
|
||||
req = httptest.NewRequest("POST", "/target?per_page=77&page=4", strings.NewReader(incompleteBody))
|
||||
req = mux.SetURLVars(req, map[string]string{"some-id": "123"})
|
||||
|
||||
_, err = decoder(context.Background(), req)
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, io.ErrUnexpectedEOF), "expected io.ErrUnexpectedEOF, got %T: %v", err, err)
|
||||
_, isPayloadTooLarge := err.(platform_http.PayloadTooLargeError)
|
||||
require.False(t, isPayloadTooLarge, "incomplete body within size limit must not produce PayloadTooLargeError, got %T: %v", err, err)
|
||||
|
||||
// Body within the limit and complete ... OK
|
||||
|
||||
largeBody = `{"key": "` + strings.Repeat("A", int(platform_http.MaxRequestBodySize)-11) + `"}` // -11 to account for the wrapping JSON
|
||||
req = httptest.NewRequest("POST", "/target?per_page=77&page=4", strings.NewReader(largeBody))
|
||||
req = mux.SetURLVars(req, map[string]string{"some-id": "123"})
|
||||
|
|
|
|||
|
|
@ -927,11 +927,19 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC
|
|||
POST("/api/osquery/config", getClientConfigEndpoint, getClientConfigRequest{})
|
||||
he.WithAltPaths("/api/v1/osquery/distributed/read").
|
||||
POST("/api/osquery/distributed/read", getDistributedQueriesEndpoint, getDistributedQueriesRequest{})
|
||||
he.WithRequestBodySizeLimit(fleet.MaxOsqueryDistributedWriteSize).WithAltPaths("/api/v1/osquery/distributed/write").
|
||||
distWriteLimit := config.Osquery.MaxDistributedWriteBodySize
|
||||
if distWriteLimit == 0 {
|
||||
distWriteLimit = fleet.DefaultMaxOsqueryDistributedWriteSize
|
||||
}
|
||||
he.WithRequestBodySizeLimit(distWriteLimit).WithAltPaths("/api/v1/osquery/distributed/write").
|
||||
POST("/api/osquery/distributed/write", submitDistributedQueryResultsEndpoint, submitDistributedQueryResultsRequestShim{})
|
||||
he.WithAltPaths("/api/v1/osquery/carve/begin").
|
||||
POST("/api/osquery/carve/begin", carveBeginEndpoint, carveBeginRequest{})
|
||||
he.WithAltPaths("/api/v1/osquery/log").
|
||||
logWriteLimit := config.Osquery.MaxLogWriteBodySize
|
||||
if logWriteLimit == 0 {
|
||||
logWriteLimit = fleet.DefaultMaxOsqueryLogWriteSize
|
||||
}
|
||||
he.WithRequestBodySizeLimit(logWriteLimit).WithAltPaths("/api/v1/osquery/log").
|
||||
POST("/api/osquery/log", submitLogsEndpoint, submitLogsRequest{})
|
||||
he.WithAltPaths("/api/v1/osquery/yara/{name}").
|
||||
POST("/api/osquery/yara/{name}", getYaraEndpoint, getYaraRequest{})
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/WatchBeam/clock"
|
||||
"github.com/docker/go-units"
|
||||
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
|
||||
"github.com/fleetdm/fleet/v4/server"
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
|
|
@ -15707,3 +15708,101 @@ func (s *integrationTestSuite) TestDeleteCertificateTemplateSpec() {
|
|||
require.Equal(t, fleet.MDMOperationTypeRemove, profile.OperationType, "%s profile operation_type should be remove after deletion", tc.hostName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOsqueryBodySizeLimit verifies the body size limits on the
|
||||
// /api/osquery/log and /api/osquery/distributed/write endpoints:
|
||||
// - Bodies exceeding the default limit are rejected with HTTP 413.
|
||||
// - Bodies within the limit are accepted.
|
||||
// - A malformed (truncated) body within the limit is NOT reported as HTTP 413
|
||||
// (guards against the false-positive PayloadTooLargeError fix).
|
||||
// - Setting Osquery.MaxLogWriteBodySize / MaxDistributedWriteBodySize in the
|
||||
// server config overrides the built-in defaults.
|
||||
func (s *integrationTestSuite) TestOsqueryBodySizeLimit() {
|
||||
t := s.T()
|
||||
|
||||
host := createOrbitEnrolledHost(t, "linux", "body-limit", s.ds)
|
||||
|
||||
// Body over DefaultMaxOsqueryLogWriteSize must be rejected with 413. The padding
|
||||
// is inside a JSON string value so the body is syntactically valid up to
|
||||
// the point where the reader is cut off.
|
||||
logPrefix := fmt.Sprintf(`{"node_key":%q,"log_type":"status","data":["`, *host.NodeKey)
|
||||
logSuffix := `"]}`
|
||||
logPadSize := int(fleet.DefaultMaxOsqueryLogWriteSize) + 1 - len(logPrefix) - len(logSuffix)
|
||||
require.Positive(t, logPadSize, "padding must be positive; DefaultMaxOsqueryLogWriteSize may be too small")
|
||||
overLimitLog := []byte(logPrefix + strings.Repeat("x", logPadSize) + logSuffix)
|
||||
s.DoRawNoAuth("POST", "/api/osquery/log", overLimitLog, http.StatusRequestEntityTooLarge)
|
||||
|
||||
// A well-formed body within the limit is accepted.
|
||||
withinLimitLog, err := json.Marshal(submitLogsRequest{
|
||||
NodeKey: *host.NodeKey,
|
||||
LogType: "status",
|
||||
Data: []json.RawMessage{},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
s.DoRawNoAuth("POST", "/api/osquery/log", withinLimitLog, http.StatusOK)
|
||||
|
||||
// A truncated (malformed) body within the limit must NOT return 413.
|
||||
// Before the fix, io.ErrUnexpectedEOF from the JSON decoder was incorrectly
|
||||
// converted to PayloadTooLargeError even when the reader had not been exhausted.
|
||||
// The correct response is 400 Bad Request.
|
||||
truncatedLog := fmt.Appendf(nil, `{"node_key":%q,"log_type":"status","data":[`, *host.NodeKey) // missing closing ]}
|
||||
s.DoRawNoAuth("POST", "/api/osquery/log", truncatedLog, http.StatusBadRequest)
|
||||
|
||||
// Body over DefaultMaxOsqueryDistributedWriteSize must be rejected with 413.
|
||||
distPrefix := fmt.Sprintf(`{"node_key":%q,"queries":{"q1":[{"data":"`, *host.NodeKey)
|
||||
distSuffix := `"}]},"statuses":{"q1":0},"messages":{},"stats":{}}`
|
||||
distPadSize := int(fleet.DefaultMaxOsqueryDistributedWriteSize) + 1 - len(distPrefix) - len(distSuffix)
|
||||
require.Positive(t, distPadSize, "padding must be positive; DefaultMaxOsqueryDistributedWriteSize may be too small")
|
||||
overLimitDist := []byte(distPrefix + strings.Repeat("x", distPadSize) + distSuffix)
|
||||
s.DoRawNoAuth("POST", "/api/osquery/distributed/write", overLimitDist, http.StatusRequestEntityTooLarge)
|
||||
|
||||
// A well-formed body within the limit is accepted.
|
||||
withinLimitDist, err := json.Marshal(submitDistributedQueryResultsRequestShim{
|
||||
NodeKey: *host.NodeKey,
|
||||
Results: map[string]json.RawMessage{},
|
||||
Statuses: map[string]any{},
|
||||
Messages: map[string]string{},
|
||||
Stats: map[string]*fleet.Stats{},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
s.DoRawNoAuth("POST", "/api/osquery/distributed/write", withinLimitDist, http.StatusOK)
|
||||
|
||||
// A truncated body within the limit must NOT return 413 (same false-positive guard).
|
||||
// io.ErrUnexpectedEOF from the bodyDecoder path is now wrapped as BadRequestErr → 400.
|
||||
truncatedDist := fmt.Appendf(nil, `{"node_key":%q,"queries":{"q1":[`, *host.NodeKey) // missing closing
|
||||
s.DoRawNoAuth("POST", "/api/osquery/distributed/write", truncatedDist, http.StatusBadRequest)
|
||||
|
||||
// Verify that Osquery.MaxLogWriteBodySize and MaxDistributedWriteBodySize
|
||||
// in the server config override the built-in defaults.
|
||||
s.Run("config override", func() {
|
||||
const customLimit = 2 * units.MiB
|
||||
|
||||
cfg := config.TestConfig()
|
||||
cfg.Osquery.MaxLogWriteBodySize = customLimit
|
||||
cfg.Osquery.MaxDistributedWriteBodySize = customLimit
|
||||
|
||||
_, customServer := RunServerForTestsWithDS(s.T(), s.ds, &TestServerOpts{
|
||||
FleetConfig: &cfg,
|
||||
SkipCreateTestUsers: true,
|
||||
})
|
||||
s.T().Cleanup(customServer.Close)
|
||||
ts := withServer{server: customServer}
|
||||
ts.s = &s.Suite
|
||||
|
||||
// body over the custom limit must return 413.
|
||||
logPad := customLimit + 1 - len(logPrefix) - len(logSuffix)
|
||||
require.Positive(s.T(), logPad, "padding must be positive; customLimit may be too small")
|
||||
ts.DoRawNoAuth("POST", "/api/osquery/log", []byte(logPrefix+strings.Repeat("x", logPad)+logSuffix), http.StatusRequestEntityTooLarge)
|
||||
|
||||
// body within the custom limit must succeed.
|
||||
ts.DoRawNoAuth("POST", "/api/osquery/log", withinLimitLog, http.StatusOK)
|
||||
|
||||
// body over the custom limit must return 413.
|
||||
distPad := customLimit + 1 - len(distPrefix) - len(distSuffix)
|
||||
require.Positive(s.T(), distPad, "padding must be positive; customLimit may be too small")
|
||||
ts.DoRawNoAuth("POST", "/api/osquery/distributed/write", []byte(distPrefix+strings.Repeat("x", distPad)+distSuffix), http.StatusRequestEntityTooLarge)
|
||||
|
||||
// body within the custom limit must succeed.
|
||||
ts.DoRawNoAuth("POST", "/api/osquery/distributed/write", withinLimitDist, http.StatusOK)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue