diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 4ddce81210..9053f37312 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -1178,7 +1178,7 @@ the way that the Fleet server works. if err := rc.SetWriteDeadline(zeroTime); err != nil { level.Error(logger).Log("msg", "http middleware failed to override endpoint write timeout", "err", err) } - req.Body = http.MaxBytesReader(rw, req.Body, service.MaxSoftwareInstallerSize) + req.Body = http.MaxBytesReader(rw, req.Body, fleet.MaxSoftwareInstallerSize) } apiHandler.ServeHTTP(rw, req) }) diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index d38e32d637..241a6752f7 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -1708,7 +1708,8 @@ func TestGitOpsFullGlobalAndTeam(t *testing.T) { scepKey := tokenpki.PEMRSAPrivateKey(key) ds.GetAllMDMConfigAssetsByNameFunc = func(ctx context.Context, assetNames []fleet.MDMAssetName, - _ sqlx.QueryerContext) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) { + _ sqlx.QueryerContext, + ) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) { return map[fleet.MDMAssetName]fleet.MDMConfigAsset{ fleet.MDMAssetCACert: {Value: scepCert}, fleet.MDMAssetCAKey: {Value: scepKey}, @@ -1754,7 +1755,7 @@ func TestGitOpsTeamSofwareInstallers(t *testing.T) { }{ {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe, .deb or .rpm."}, - {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, + {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 3 GB"}, {"testdata/gitops/team_software_installer_valid.yml", ""}, {"testdata/gitops/team_software_installer_valid_apply.yml", ""}, {"testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, @@ -1809,7 +1810,7 @@ func TestGitOpsNoTeamSoftwareInstallers(t *testing.T) { }{ {"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/no_team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe, .deb or .rpm."}, - {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, + {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 3 GB"}, {"testdata/gitops/no_team_software_installer_valid.yml", ""}, {"testdata/gitops/no_team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, {"testdata/gitops/no_team_software_installer_pre_condition_not_found.yml", "no such file or directory"}, @@ -2006,7 +2007,7 @@ func startSoftwareInstallerServer(t *testing.T) { case strings.Contains(r.URL.Path, "toolarge"): w.Header().Set("Content-Type", "application/vnd.debian.binary-package") var sz int - for sz < 500*1024*1024 { + for sz < 3000*1024*1024 { n, _ := w.Write(b) sz += n } diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 5b5679efb7..8da5edddc6 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -1117,8 +1117,7 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f } const ( - maxInstallerSizeBytes int64 = 1024 * 1024 * 500 - batchSoftwarePrefix = "software_batch_" + batchSoftwarePrefix = "software_batch_" ) func (svc *Service) BatchSetSoftwareInstallers( @@ -1232,7 +1231,7 @@ func (svc *Service) softwareBatchUpload( downloadURLFn := func(ctx context.Context, url string) (http.Header, []byte, error) { client := fleethttp.NewClient() - client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) + client.Transport = fleethttp.NewSizeLimitTransport(fleet.MaxSoftwareInstallerSize) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -1245,7 +1244,7 @@ func (svc *Service) softwareBatchUpload( if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d GB", url, fleet.MaxSoftwareInstallerSize/(1000*1024*1024)), ) } @@ -1276,7 +1275,7 @@ func (svc *Service) softwareBatchUpload( if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d GB", url, fleet.MaxSoftwareInstallerSize/(1000*1024*1024)), ) } return nil, nil, fmt.Errorf("reading installer %q contents: %w", url, err) @@ -1286,7 +1285,7 @@ func (svc *Service) softwareBatchUpload( } var g errgroup.Group - g.SetLimit(3) // TODO: consider lowering this limit, see https://github.com/fleetdm/fleet/issues/22704#issuecomment-2397407837 + g.SetLimit(1) // TODO: consider whether we can increase this limit, see https://github.com/fleetdm/fleet/issues/22704#issuecomment-2397407837 // critical to avoid data race, the slice is pre-allocated and each // goroutine only writes to its index. installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) diff --git a/server/datastore/mysql/maintained_apps.go b/server/datastore/mysql/maintained_apps.go index e8dbb670a7..285db769cb 100644 --- a/server/datastore/mysql/maintained_apps.go +++ b/server/datastore/mysql/maintained_apps.go @@ -130,11 +130,6 @@ WHERE NOT EXISTS ( ) )` - // build the count statement before adding the pagination constraints to the - // default stmt. - dbReader := ds.reader(ctx) - getAppsCountStmt := fmt.Sprintf(`SELECT COUNT(DISTINCT s.id) FROM (%s) AS s`, stmt) - args := []any{teamID, teamID} if match := opt.MatchQuery; match != "" { @@ -143,6 +138,16 @@ WHERE NOT EXISTS ( args = append(args, match) } + // perform a second query to grab the counts. Build the count statement before + // adding the pagination constraints to the stmt but after including the + // MatchQuery option sql. + dbReader := ds.reader(ctx) + getAppsCountStmt := fmt.Sprintf(`SELECT COUNT(DISTINCT s.id) FROM (%s) AS s`, stmt) + var counts int + if err := sqlx.GetContext(ctx, dbReader, &counts, getAppsCountStmt, args...); err != nil { + return nil, nil, ctxerr.Wrap(ctx, err, "get fleet maintained apps count") + } + stmtPaged, args := appendListOptionsWithCursorToSQL(stmt, args, &opt) var avail []fleet.MaintainedApp @@ -150,12 +155,6 @@ WHERE NOT EXISTS ( return nil, nil, ctxerr.Wrap(ctx, err, "selecting available fleet managed apps") } - // perform a second query to grab the counts - var counts int - if err := sqlx.GetContext(ctx, dbReader, &counts, getAppsCountStmt, args...); err != nil { - return nil, nil, ctxerr.Wrap(ctx, err, "get fleet maintained apps count") - } - meta := &fleet.PaginationMetadata{HasPreviousResults: opt.Page > 0, TotalResults: uint(counts)} if len(avail) > int(opt.PerPage) { meta.HasNextResults = true diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index 963799e9c7..df479d72b5 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -9,10 +9,15 @@ import ( "strings" "time" + "github.com/docker/go-units" "github.com/fleetdm/fleet/v4/pkg/optjson" "github.com/fleetdm/fleet/v4/server/ptr" ) +// MaxSoftwareInstallerSize is the maximum size allowed for software +// installers. This is enforced by the endpoints that upload installers. +const MaxSoftwareInstallerSize = 3000 * units.MiB + // SoftwareInstallerStore is the interface to store and retrieve software // installer files. Fleet supports storing to the local filesystem and to an // S3 bucket. diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 5435f7f88e..c0fc8888fc 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -47,10 +47,6 @@ type uploadSoftwareInstallerResponse struct { Err error `json:"error,omitempty"` } -// MaxSoftwareInstallerSize is the maximum size allowed for software -// installers. This is enforced by the endpoints that upload installers. -const MaxSoftwareInstallerSize = 3000 * units.MiB - // TODO: We parse the whole body before running svc.authz.Authorize. // An authenticated but unauthorized user could abuse this. func (updateSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { @@ -88,7 +84,7 @@ func (updateSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http // unlike for uploadSoftwareInstallerRequest, every field is optional, including the file upload if r.MultipartForm.File["software"] != nil || len(r.MultipartForm.File["software"]) > 0 { decoded.File = r.MultipartForm.File["software"][0] - if decoded.File.Size > MaxSoftwareInstallerSize { + if decoded.File.Size > fleet.MaxSoftwareInstallerSize { // Should never happen here since the request's body is limited to the maximum size. return nil, &fleet.BadRequestError{ Message: "The maximum file size is 3 GB.", @@ -186,7 +182,7 @@ func (uploadSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http var mbe *http.MaxBytesError if errors.As(err, &mbe) { return nil, &fleet.BadRequestError{ - Message: "The maximum file size is 500 MB.", + Message: "The maximum file size is 3 GB.", InternalErr: err, } } @@ -211,11 +207,11 @@ func (uploadSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http } decoded.File = r.MultipartForm.File["software"][0] - if decoded.File.Size > MaxSoftwareInstallerSize { + if decoded.File.Size > fleet.MaxSoftwareInstallerSize { // Should never happen here since the request's body is limited to the // maximum size. return nil, &fleet.BadRequestError{ - Message: "The maximum file size is 500 MB.", + Message: "The maximum file size is 3 GB.", } }