diff --git a/.github/workflows/goreleaser-fleet.yaml b/.github/workflows/goreleaser-fleet.yaml index 4d7e73104b..f7666ac5eb 100644 --- a/.github/workflows/goreleaser-fleet.yaml +++ b/.github/workflows/goreleaser-fleet.yaml @@ -76,7 +76,7 @@ jobs: - name: Run GoReleaser id: goreleaser - uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b + uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 with: distribution: goreleaser-pro version: "~> 2" diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml index c815a9902d..9bc99c8b38 100644 --- a/.github/workflows/goreleaser-snapshot-fleet.yaml +++ b/.github/workflows/goreleaser-snapshot-fleet.yaml @@ -69,7 +69,7 @@ jobs: run: make deps - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f82d6c1c344bcacabba2c841718984797f664a6b + uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 with: distribution: goreleaser-pro version: "~> 2" diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go index 3608403a09..a16c326e92 100644 --- a/cmd/osquery-perf/agent.go +++ b/cmd/osquery-perf/agent.go @@ -1025,9 +1025,11 @@ func (a *agent) execScripts(execIDs []string, orbitClient *service.OrbitClient) continue } + a.stats.IncrementScriptExecs() script, err := orbitClient.GetHostScript(execID) if err != nil { log.Println("get host script:", err) + a.stats.IncrementScriptExecErrs() return } @@ -1047,6 +1049,7 @@ func (a *agent) execScripts(execIDs []string, orbitClient *service.OrbitClient) ExitCode: exitCode, }); err != nil { log.Println("save host script result:", err) + a.stats.IncrementScriptExecErrs() return } log.Printf("did exec and save host script result: id=%s, output size=%d, runtime=%d, exit code=%d", execID, base64.StdEncoding.EncodedLen(n), runtime, exitCode) @@ -1064,11 +1067,14 @@ func (a *agent) installSoftware(installerIDs []string, orbitClient *service.Orbi } func (a *agent) installSoftwareItem(installerID string, orbitClient *service.OrbitClient) { + a.stats.IncrementSoftwareInstalls() + payload := &fleet.HostSoftwareInstallResultPayload{} payload.InstallUUID = installerID installer, err := orbitClient.GetInstallerDetails(installerID) if err != nil { log.Println("get installer details:", err) + a.stats.IncrementSoftwareInstallErrs() return } failed := false @@ -1093,6 +1099,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb // Download the file if needed to get its metadata meta, cacheMiss, err = installerMetadataCache.Get(installer, orbitClient) if err != nil { + a.stats.IncrementSoftwareInstallErrs() return } @@ -1102,6 +1109,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb err = orbitClient.DownloadAndDiscardSoftwareInstaller(installer.InstallerID) if err != nil { log.Println("download and discard software installer:", err) + a.stats.IncrementSoftwareInstallErrs() return } } @@ -1174,6 +1182,7 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb err = orbitClient.SaveInstallerResult(payload) if err != nil { log.Println("save installer result:", err) + a.stats.IncrementSoftwareInstallErrs() return } } diff --git a/cmd/osquery-perf/osquery_perf/stats.go b/cmd/osquery-perf/osquery_perf/stats.go index f20e851078..4569f888e1 100644 --- a/cmd/osquery-perf/osquery_perf/stats.go +++ b/cmd/osquery-perf/osquery_perf/stats.go @@ -34,6 +34,10 @@ type Stats struct { distributedWriteErrors int resultLogErrors int bufferedLogs int + scriptExecs int + scriptExecErrs int + softwareInstalls int + softwareInstallErrs int l sync.Mutex } @@ -197,12 +201,36 @@ func (s *Stats) UpdateBufferedLogs(v int) { } } +func (s *Stats) IncrementScriptExecs() { + s.l.Lock() + defer s.l.Unlock() + s.scriptExecs++ +} + +func (s *Stats) IncrementScriptExecErrs() { + s.l.Lock() + defer s.l.Unlock() + s.scriptExecErrs++ +} + +func (s *Stats) IncrementSoftwareInstalls() { + s.l.Lock() + defer s.l.Unlock() + s.softwareInstalls++ +} + +func (s *Stats) IncrementSoftwareInstallErrs() { + s.l.Lock() + defer s.l.Unlock() + s.softwareInstalls++ +} + func (s *Stats) Log() { s.l.Lock() defer s.l.Unlock() log.Printf( - "uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d", + "uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d, script execs (errs): %d (%d), software installs (errs): %d (%d)", time.Since(s.StartTime).Round(time.Second), float64(s.errors)/float64(s.osqueryEnrollments), s.osqueryEnrollments, @@ -230,6 +258,10 @@ func (s *Stats) Log() { s.ddmStatusSuccess, s.ddmStatusErrors, s.bufferedLogs, + s.scriptExecs, + s.scriptExecErrs, + s.softwareInstalls, + s.softwareInstallErrs, ) } diff --git a/server/service/client_scripts.go b/server/service/client_scripts.go index 9761fe24b2..893abd3faf 100644 --- a/server/service/client_scripts.go +++ b/server/service/client_scripts.go @@ -218,3 +218,14 @@ func (c *Client) uploadMacOSSetupScript(filename string, data []byte, teamID *ui return nil } + +// ListScripts retrieves the saved scripts. +func (c *Client) ListScripts(query string) ([]*fleet.Script, error) { + verb, path := "GET", "/api/latest/fleet/scripts" + var responseBody listScriptsResponse + err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query) + if err != nil { + return nil, err + } + return responseBody.Scripts, nil +} diff --git a/server/service/client_software.go b/server/service/client_software.go index 60a0911093..58ea6944eb 100644 --- a/server/service/client_software.go +++ b/server/service/client_software.go @@ -67,3 +67,11 @@ func (c *Client) applySoftwareInstallers(softwareInstallers []fleet.SoftwareInst } } } + +// InstallSoftware triggers a software installation (VPP or software package) +// on the specified host. +func (c *Client) InstallSoftware(hostID uint, softwareTitleID uint) error { + verb, path := "POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/%d/install", hostID, softwareTitleID) + var responseBody installSoftwareResponse + return c.authenticatedRequest(nil, verb, path, &responseBody) +} diff --git a/tools/loadtest/unified_queue/README.md b/tools/loadtest/unified_queue/README.md new file mode 100644 index 0000000000..ea4955c373 --- /dev/null +++ b/tools/loadtest/unified_queue/README.md @@ -0,0 +1,5 @@ +# Load testing of the unified queue story + +This is the Go program used to run load tests for the [unified queue story](https://github.com/fleetdm/fleet/issues/22866). + +It expects some software to be available for install on both macOS and Windows (including VPP apps for macOS), and some scripts too, and it enqueues installs and script execution requests on every host in the Fleet deployment for an hour. diff --git a/tools/loadtest/unified_queue/main.go b/tools/loadtest/unified_queue/main.go new file mode 100644 index 0000000000..575851cf64 --- /dev/null +++ b/tools/loadtest/unified_queue/main.go @@ -0,0 +1,190 @@ +package main + +import ( + "flag" + "fmt" + "log" + "math/rand/v2" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/service" +) + +func printf(format string, a ...any) { + fmt.Printf(time.Now().UTC().Format("2006-01-02T15:04:05Z")+": "+format, a...) +} + +func main() { + fleetURL := flag.String("fleet_url", "", "URL (with protocol and port of Fleet server)") + apiToken := flag.String("api_token", "", "API authentication token to use on API calls") + debug := flag.Bool("debug", false, "Debug mode") + + flag.Parse() + + if *fleetURL == "" { + log.Fatal("missing fleet_url argument") + } + if *apiToken == "" { + log.Fatal("missing api_token argument") + } + var clientOpts []service.ClientOption + if *debug { + clientOpts = append(clientOpts, service.EnableClientDebug()) + } + apiClient, err := service.NewClient(*fleetURL, true, "", "", clientOpts...) + if err != nil { + log.Fatal(err) + } + apiClient.SetToken(*apiToken) + + printf("Fetching hosts...\n") + records, err := apiClient.GetHostsReport("id", "hostname", "platform") + if err != nil { + log.Fatal(err) + } + type smallHost struct { + ID uint + Hostname string + Platform string + } + var ( + macOSHosts []smallHost + windowsHosts []smallHost + linuxHosts []smallHost + ) + for i, record := range records { + if i == 0 { + continue + } + hostID, _ := strconv.Atoi(record[0]) + hostname := record[1] + platform := fleet.PlatformFromHost(record[2]) + switch platform { + case "linux": + linuxHosts = append(linuxHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec + case "darwin": + macOSHosts = append(macOSHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec + case "windows": + windowsHosts = append(windowsHosts, smallHost{ID: uint(hostID), Hostname: hostname, Platform: platform}) // nolint:gosec + } + } + printf("Got linux=%d, windows=%d, macOS=%d\n", len(linuxHosts), len(windowsHosts), len(macOSHosts)) + + titles, err := apiClient.ListSoftwareTitles("per_page=1000&team_id=0&available_for_install=1") + if err != nil { + log.Fatal(err) + } + + var ( + macOSSoftware []fleet.SoftwareTitleListResult + windowsSoftware []fleet.SoftwareTitleListResult + ) + for _, title := range titles { + if title.AppStoreApp != nil { + macOSSoftware = append(macOSSoftware, title) + } else if title.SoftwarePackage != nil { + if ext := filepath.Ext(title.SoftwarePackage.Name); ext == ".exe" || ext == ".msi" { + windowsSoftware = append(windowsSoftware, title) + } else { + macOSSoftware = append(macOSSoftware, title) + } + } + } + printf("Got software titles windows=%d, macOS=%d\n", len(windowsSoftware), len(macOSSoftware)) + + scripts, err := apiClient.ListScripts("per_page=1000&team_id=0") + if err != nil { + log.Fatal(err) + } + + var ( + macOSScripts []string + windowsScripts []string + ) + for _, script := range scripts { + if strings.HasSuffix(script.Name, ".sh") { + macOSScripts = append(macOSScripts, script.Name) + } else if strings.HasSuffix(script.Name, ".ps1") { + windowsScripts = append(windowsScripts, script.Name) + } + } + printf("Got scripts windows=%d, macOS=%d\n", len(windowsScripts), len(macOSScripts)) + + var queuedScripts, queuedInstalls, hostsTargeted, errors int + targetedHosts := append(macOSHosts, windowsHosts...) // nolint:gocritic + rand.Shuffle(len(targetedHosts), func(i, j int) { + targetedHosts[i], targetedHosts[j] = targetedHosts[j], targetedHosts[i] + }) + + tick := time.Tick(300 * time.Millisecond) + for i, host := range targetedHosts { + <-tick + + if hostsTargeted > 0 && hostsTargeted%500 == 0 { + printf("In progress: queued scripts=%d, queued installs=%d, hosts targeted=%d, errors=%d\n", queuedScripts, queuedInstalls, hostsTargeted, errors) + } + + switch host.Platform { + case "darwin": + hostsTargeted++ + + // enqueue a software install and a couple scripts + _, err := apiClient.RunHostScriptAsync(host.ID, nil, macOSScripts[i%len(macOSScripts)], 0) + if err != nil { + printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedScripts++ + _, err = apiClient.RunHostScriptAsync(host.ID, nil, macOSScripts[(i+1)%len(macOSScripts)], 0) + if err != nil { + printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedScripts++ + + err = apiClient.InstallSoftware(host.ID, macOSSoftware[i%len(macOSSoftware)].ID) + if err != nil { + printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedInstalls++ + + case "windows": + hostsTargeted++ + + // enqueue a couple software installs and a script + err = apiClient.InstallSoftware(host.ID, windowsSoftware[i%len(windowsSoftware)].ID) + if err != nil { + printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedInstalls++ + + err = apiClient.InstallSoftware(host.ID, windowsSoftware[(i+1)%len(windowsSoftware)].ID) + if err != nil { + printf("Failed to install software on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedInstalls++ + + _, err = apiClient.RunHostScriptAsync(host.ID, nil, windowsScripts[i%len(windowsScripts)], 0) + if err != nil { + printf("Failed to run script on host %v (%v): %v\n", host.Hostname, host.Platform, err) + errors++ + continue + } + queuedScripts++ + } + } + + printf("Done: queued scripts=%d, queued installs=%d, hosts targeted=%d, errors=%d\n", queuedScripts, queuedInstalls, hostsTargeted, errors) +}