Add script execs and software installs stats to osquery-perf (#26239)

.github/workflows/goreleaser-fleet.yaml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Run GoReleaser
         id: goreleaser
-        uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b
+        uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1
         with:
           distribution: goreleaser-pro
           version: "~> 2"

.github/workflows/goreleaser-snapshot-fleet.yaml

@@ -69,7 +69,7 @@ jobs:
         run: make deps
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b
+        uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1
         with:
           distribution: goreleaser-pro
           version: "~> 2"

cmd/osquery-perf/agent.go

@@ -1025,9 +1025,11 @@ func (a *agent) execScripts(execIDs []string, orbitClient *service.OrbitClient)
 			continue
 		}
 
+		a.stats.IncrementScriptExecs()
 		script, err := orbitClient.GetHostScript(execID)
 		if err != nil {
 			log.Println("get host script:", err)
+			a.stats.IncrementScriptExecErrs()
 			return
 		}
 
@@ -1047,6 +1049,7 @@ func (a *agent) execScripts(execIDs []string, orbitClient *service.OrbitClient)
 			ExitCode:    exitCode,
 		}); err != nil {
 			log.Println("save host script result:", err)
+			a.stats.IncrementScriptExecErrs()
 			return
 		}
 		log.Printf("did exec and save host script result: id=%s, output size=%d, runtime=%d, exit code=%d", execID, base64.StdEncoding.EncodedLen(n), runtime, exitCode)
@@ -1064,11 +1067,14 @@ func (a *agent) installSoftware(installerIDs []string, orbitClient *service.Orbi
 }
 
 func (a *agent) installSoftwareItem(installerID string, orbitClient *service.OrbitClient) {
+	a.stats.IncrementSoftwareInstalls()
+
 	payload := &fleet.HostSoftwareInstallResultPayload{}
 	payload.InstallUUID = installerID
 	installer, err := orbitClient.GetInstallerDetails(installerID)
 	if err != nil {
 		log.Println("get installer details:", err)
+		a.stats.IncrementSoftwareInstallErrs()
 		return
 	}
 	failed := false
@@ -1093,6 +1099,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb
 		// Download the file if needed to get its metadata
 		meta, cacheMiss, err = installerMetadataCache.Get(installer, orbitClient)
 		if err != nil {
+			a.stats.IncrementSoftwareInstallErrs()
 			return
 		}
 
@@ -1102,6 +1109,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb
 			err = orbitClient.DownloadAndDiscardSoftwareInstaller(installer.InstallerID)
 			if err != nil {
 				log.Println("download and discard software installer:", err)
+				a.stats.IncrementSoftwareInstallErrs()
 				return
 			}
 		}
@@ -1174,6 +1182,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb
 	err = orbitClient.SaveInstallerResult(payload)
 	if err != nil {
 		log.Println("save installer result:", err)
+		a.stats.IncrementSoftwareInstallErrs()
 		return
 	}
 }

cmd/osquery-perf/osquery_perf/stats.go

@@ -34,6 +34,10 @@ type Stats struct {
 	distributedWriteErrors     int
 	resultLogErrors            int
 	bufferedLogs               int
+	scriptExecs                int
+	scriptExecErrs             int
+	softwareInstalls           int
+	softwareInstallErrs        int
 
 	l sync.Mutex
 }
@@ -197,12 +201,36 @@ func (s *Stats) UpdateBufferedLogs(v int) {
 	}
 }
 
+func (s *Stats) IncrementScriptExecs() {
+	s.l.Lock()
+	defer s.l.Unlock()
+	s.scriptExecs++
+}
+
+func (s *Stats) IncrementScriptExecErrs() {
+	s.l.Lock()
+	defer s.l.Unlock()
+	s.scriptExecErrs++
+}
+
+func (s *Stats) IncrementSoftwareInstalls() {
+	s.l.Lock()
+	defer s.l.Unlock()
+	s.softwareInstalls++
+}
+
+func (s *Stats) IncrementSoftwareInstallErrs() {
+	s.l.Lock()
+	defer s.l.Unlock()
+	s.softwareInstalls++
+}
+
 func (s *Stats) Log() {
 	s.l.Lock()
 	defer s.l.Unlock()
 
 	log.Printf(
-		"uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d",
+		"uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d, script execs (errs): %d (%d), software installs (errs): %d (%d)",
 		time.Since(s.StartTime).Round(time.Second),
 		float64(s.errors)/float64(s.osqueryEnrollments),
 		s.osqueryEnrollments,
@@ -230,6 +258,10 @@ func (s *Stats) Log() {
 		s.ddmStatusSuccess,
 		s.ddmStatusErrors,
 		s.bufferedLogs,
+		s.scriptExecs,
+		s.scriptExecErrs,
+		s.softwareInstalls,
+		s.softwareInstallErrs,
 	)
 }
 

server/service/client_scripts.go

@@ -218,3 +218,14 @@ func (c *Client) uploadMacOSSetupScript(filename string, data []byte, teamID *ui
 
 	return nil
 }
+
+// ListScripts retrieves the saved scripts.
+func (c *Client) ListScripts(query string) ([]*fleet.Script, error) {
+	verb, path := "GET", "/api/latest/fleet/scripts"
+	var responseBody listScriptsResponse
+	err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
+	if err != nil {
+		return nil, err
+	}
+	return responseBody.Scripts, nil
+}

server/service/client_software.go

@@ -67,3 +67,11 @@ func (c *Client) applySoftwareInstallers(softwareInstallers []fleet.SoftwareInst
 		}
 	}
 }
+
+// InstallSoftware triggers a software installation (VPP or software package)
+// on the specified host.
+func (c *Client) InstallSoftware(hostID uint, softwareTitleID uint) error {
+	verb, path := "POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/%d/install", hostID, softwareTitleID)
+	var responseBody installSoftwareResponse
+	return c.authenticatedRequest(nil, verb, path, &responseBody)
+}

tools/loadtest/unified_queue/README.md

@@ -0,0 +1,5 @@
+# Load testing of the unified queue story
+
+This is the Go program used to run load tests for the [unified queue story](https://github.com/fleetdm/fleet/issues/22866).
+
+It expects some software to be available for install on both macOS and Windows (including VPP apps for macOS), and some scripts too, and it enqueues installs and script execution requests on every host in the Fleet deployment for an hour.

tools/loadtest/unified_queue/main.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"math/rand/v2"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/service"
+)
+
+func printf(format string, a ...any) {
+	fmt.Printf(time.Now().UTC().Format("2006-01-02T15:04:05Z")+": "+format, a...)
+}
+
+func main() {
+	fleetURL := flag.String("fleet_url", "", "URL (with protocol and port of Fleet server)")
+	apiToken := flag.String("api_token", "", "API authentication token to use on API calls")
+	debug := flag.Bool("debug", false, "Debug mode")
+
+	flag.Parse()
+
+	if *fleetURL == "" {
+		log.Fatal("missing fleet_url argument")
+	}
+	if *apiToken == "" {
+		log.Fatal("missing api_token argument")
+	}
+	var clientOpts []service.ClientOption
+	if *debug {
+		clientOpts = append(clientOpts, service.EnableClientDebug())
+	}
+	apiClient, err := service.NewClient(*fleetURL, true, "", "", clientOpts...)
+	if err != nil {
+		log.Fatal(err)
+	}
+	apiClient.SetToken(*apiToken)
+
+	printf("Fetching hosts...\n")
+	records, err := apiClient.GetHostsReport("id", "hostname", "platform")
+	if err != nil {
+		log.Fatal(err)
+	}
+	type smallHost struct {
+		ID       uint
+		Hostname string
+		Platform string
+	}
+	var (
+		macOSHosts   []smallHost
+		windowsHosts []smallHost
+		linuxHosts   []smallHost
+	)
+	for i, record := range records {
+		if i == 0 {
+			continue
+		}
+		hostID, _ := strconv.Atoi(record[0])
+		hostname := record[1]
+		platform := fleet.PlatformFromHost(record[2])
+		switch platform {
+		case "linux":
+			linuxHosts = append(linuxHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec
+		case "darwin":
+			macOSHosts = append(macOSHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec
+		case "windows":
+			windowsHosts = append(windowsHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec
+		}
+	}
+	printf("Got linux=%d, windows=%d, macOS=%d\n", len(linuxHosts), len(windowsHosts), len(macOSHosts))
+
+	titles, err := apiClient.ListSoftwareTitles("per_page=1000&team_id=0&available_for_install=1")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var (
+		macOSSoftware   []fleet.SoftwareTitleListResult
+		windowsSoftware []fleet.SoftwareTitleListResult
+	)
+	for _, title := range titles {
+		if title.AppStoreApp != nil {
+			macOSSoftware = append(macOSSoftware, title)
+		} else if title.SoftwarePackage != nil {
+			if ext := filepath.Ext(title.SoftwarePackage.Name); ext == ".exe" || ext == ".msi" {
+				windowsSoftware = append(windowsSoftware, title)
+			} else {
+				macOSSoftware = append(macOSSoftware, title)
+			}
+		}
+	}
+	printf("Got software titles windows=%d, macOS=%d\n", len(windowsSoftware), len(macOSSoftware))
+
+	scripts, err := apiClient.ListScripts("per_page=1000&team_id=0")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var (
+		macOSScripts   []string
+		windowsScripts []string
+	)
+	for _, script := range scripts {
+		if strings.HasSuffix(script.Name, ".sh") {
+			macOSScripts = append(macOSScripts, script.Name)
+		} else if strings.HasSuffix(script.Name, ".ps1") {
+			windowsScripts = append(windowsScripts, script.Name)
+		}
+	}
+	printf("Got scripts windows=%d, macOS=%d\n", len(windowsScripts), len(macOSScripts))
+
+	var queuedScripts, queuedInstalls, hostsTargeted, errors int
+	targetedHosts := append(macOSHosts, windowsHosts...) // nolint:gocritic
+	rand.Shuffle(len(targetedHosts), func(i, j int) {
+		targetedHosts[i], targetedHosts[j] = targetedHosts[j], targetedHosts[i]
+	})
+
+	tick := time.Tick(300 * time.Millisecond)
+	for i, host := range targetedHosts {
+		<-tick
+
+		if hostsTargeted > 0 && hostsTargeted%500 == 0 {
+			printf("In progress: queued scripts=%d, queued installs=%d, hosts targeted=%d, errors=%d\n", queuedScripts, queuedInstalls, hostsTargeted, errors)
+		}
+
+		switch host.Platform {
+		case "darwin":
+			hostsTargeted++
+
+			// enqueue a software install and a couple scripts
+			_, err := apiClient.RunHostScriptAsync(host.ID, nil, macOSScripts[i%len(macOSScripts)], 0)
+			if err != nil {
+				printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedScripts++
+			_, err = apiClient.RunHostScriptAsync(host.ID, nil, macOSScripts[(i+1)%len(macOSScripts)], 0)
+			if err != nil {
+				printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedScripts++
+
+			err = apiClient.InstallSoftware(host.ID, macOSSoftware[i%len(macOSSoftware)].ID)
+			if err != nil {
+				printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedInstalls++
+
+		case "windows":
+			hostsTargeted++
+
+			// enqueue a couple software installs and a script
+			err = apiClient.InstallSoftware(host.ID, windowsSoftware[i%len(windowsSoftware)].ID)
+			if err != nil {
+				printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedInstalls++
+
+			err = apiClient.InstallSoftware(host.ID, windowsSoftware[(i+1)%len(windowsSoftware)].ID)
+			if err != nil {
+				printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedInstalls++
+
+			_, err = apiClient.RunHostScriptAsync(host.ID, nil, windowsScripts[i%len(windowsScripts)], 0)
+			if err != nil {
+				printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err)
+				errors++
+				continue
+			}
+			queuedScripts++
+		}
+	}
+
+	printf("Done: queued scripts=%d, queued installs=%d, hosts targeted=%d, errors=%d\n", queuedScripts, queuedInstalls, hostsTargeted, errors)
+}