From 9a6b2f655e0d44784851538d4b924e7091f2f36f Mon Sep 17 00:00:00 2001
From: Victor Lyuboslavsky <victor.lyuboslavsky@gmail.com>
Date: Tue, 7 May 2024 10:53:24 -0500
Subject: [PATCH] Fixed MySQL sort buffer overflow when fetching activities.
 (#18782)

#12619

Fixed MySQL sort buffer overflow when fetching activities. This issue
happened when activities contained very large details, such as large SQL
queries.

# Checklist for submitter
<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [ ] Added/updated tests
  - Existing tests provide sufficient coverage
- [x] Manual QA for all new/changed functionality
---
 ...2619-fixed-activities-sort-buffer-overflow |  1 +
 server/datastore/mysql/activities.go          | 40 +++++++++++++++++--
 2 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100644 changes/12619-fixed-activities-sort-buffer-overflow

diff --git a/changes/12619-fixed-activities-sort-buffer-overflow b/changes/12619-fixed-activities-sort-buffer-overflow
new file mode 100644
index 0000000000..93ba207fad
--- /dev/null
+++ b/changes/12619-fixed-activities-sort-buffer-overflow
@@ -0,0 +1 @@
+Fixed MySQL sort buffer overflow when fetching activities. This issue happened when activities contained very large details, such as large SQL queries.
diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go
index 3e6199dfaf..045976c346 100644
--- a/server/datastore/mysql/activities.go
+++ b/server/datastore/mysql/activities.go
@@ -10,6 +10,7 @@ import (
 	"github.com/fleetdm/fleet/v4/pkg/scripts"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/go-kit/log/level"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -83,7 +84,6 @@ func (ds *Datastore) ListActivities(ctx context.Context, opt fleet.ListActivitie
 			a.user_id,
 			a.created_at,
 			a.activity_type,
-			a.details,
 			a.user_name as name,
 			a.streamed,
 			a.user_email
@@ -100,12 +100,44 @@ func (ds *Datastore) ListActivities(ctx context.Context, opt fleet.ListActivitie
 	activitiesQ, args = appendListOptionsWithCursorToSQL(activitiesQ, args, &opt.ListOptions)
 
 	err := sqlx.SelectContext(ctx, ds.reader(ctx), &activities, activitiesQ, args...)
-	if err == sql.ErrNoRows {
-		return nil, nil, ctxerr.Wrap(ctx, notFound("Activity"))
-	} else if err != nil {
+	if err != nil {
 		return nil, nil, ctxerr.Wrap(ctx, err, "select activities")
 	}
 
+	if len(activities) > 0 {
+		// Fetch details as a separate query due to sort buffer issue triggered by large JSON details entries. Issue last reproduced on MySQL 8.0.36
+		// https://stackoverflow.com/questions/29575835/error-1038-out-of-sort-memory-consider-increasing-sort-buffer-size/67266529
+		IDs := make([]uint, 0, len(activities))
+		for _, a := range activities {
+			IDs = append(IDs, a.ID)
+		}
+		detailsStmt, detailsArgs, err := sqlx.In("SELECT id, details FROM activities WHERE id IN (?)", IDs)
+		if err != nil {
+			return nil, nil, ctxerr.Wrap(ctx, err, "Error binding activity IDs")
+		}
+		type activityDetails struct {
+			ID      uint             `db:"id"`
+			Details *json.RawMessage `db:"details"`
+		}
+		var details []activityDetails
+		err = sqlx.SelectContext(ctx, ds.reader(ctx), &details, detailsStmt, detailsArgs...)
+		if err != nil {
+			return nil, nil, ctxerr.Wrap(ctx, err, "select activities details")
+		}
+		detailsLookup := make(map[uint]*json.RawMessage, len(details))
+		for _, d := range details {
+			detailsLookup[d.ID] = d.Details
+		}
+		for _, a := range activities {
+			det, ok := detailsLookup[a.ID]
+			if !ok {
+				level.Warn(ds.logger).Log("msg", "Activity details not found", "activity_id", a.ID)
+				continue
+			}
+			a.Details = det
+		}
+	}
+
 	// Fetch users as a stand-alone query (because of performance reasons)
 
 	lookup := make(map[uint][]int)