fleet/cmd/osquery-perf/agent.go

package main

import (
	"bytes"
	"compress/bzip2"
	cryptorand "crypto/rand"
	"crypto/sha1" // nolint:gosec
	"crypto/sha256"
	"crypto/tls"
	"embed"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"maps"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"text/template"
	"time"

	"github.com/fleetdm/fleet/v4/cmd/osquery-perf/hostidentity"
	"github.com/fleetdm/fleet/v4/cmd/osquery-perf/installer_cache"
	"github.com/fleetdm/fleet/v4/cmd/osquery-perf/osquery_perf"
	"github.com/fleetdm/fleet/v4/cmd/osquery-perf/softwaredb"
	"github.com/fleetdm/fleet/v4/pkg/file"
	"github.com/fleetdm/fleet/v4/pkg/mdm/mdmtest"
	"github.com/fleetdm/fleet/v4/server/fleet"
	apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple"
	"github.com/fleetdm/fleet/v4/server/mdm/microsoft/syncml"
	"github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm"
	"github.com/fleetdm/fleet/v4/server/ptr"
	"github.com/fleetdm/fleet/v4/server/service"
	"github.com/fleetdm/fleet/v4/server/service/contract"
	"github.com/google/uuid"
	"github.com/micromdm/plist"
	"github.com/remitly-oss/httpsig-go"
)

var (
	//go:embed *.tmpl
	templatesFS embed.FS

	//go:embed macos_vulnerable-software.json.bz2
	macOSVulnerableSoftwareFS embed.FS

	//go:embed vscode_extensions_vulnerable.software
	vsCodeExtensionsVulnerableSoftwareFS embed.FS

	//go:embed ubuntu_2204-software.json.bz2
	ubuntuSoftwareFS embed.FS
	//go:embed ubuntu_2204-kernels.json
	ubuntuKernelsFS embed.FS
	//go:embed windows_11-software.json.bz2
	windowsSoftwareFS embed.FS

	macosVulnerableSoftware            []fleet.Software
	vsCodeExtensionsVulnerableSoftware []fleet.Software
	windowsSoftware                    []map[string]string
	ubuntuSoftware                     []map[string]string
	ubuntuKernels                      []string

	// Software library database (loaded from SQLite if --software_db_path is specified)
	softwareDB *softwaredb.DB

	installerMetadataCache installer_cache.Metadata

	linuxRandomBuildNumber = randomString(8)
)

func loadMacOSVulnerableSoftware() {
	bz2, err := macOSVulnerableSoftwareFS.Open("macos_vulnerable-software.json.bz2")
	if err != nil {
		log.Fatal("open vulnerable macOS software file: ", err)
	}

	type vulnerableSoftware struct {
		Software []fleet.Software `json:"software"`
	}

	var vs vulnerableSoftware
	if err := json.NewDecoder(bzip2.NewReader(bz2)).Decode(&vs); err != nil { //nolint:gosec
		log.Fatal("unmarshaling vulnerable macOS software: ", err)
	}

	macosVulnerableSoftware = vs.Software

	log.Printf("Loaded %d vulnerable macOS software", len(macosVulnerableSoftware))
}

func loadExtraVulnerableSoftware() {
	vsCodeExtensionsVulnerableSoftwareData, err := vsCodeExtensionsVulnerableSoftwareFS.ReadFile("vscode_extensions_vulnerable.software")
	if err != nil {
		log.Fatal("reading vulnerable vscode_extensions software file: ", err)
	}
	lines := bytes.Split(vsCodeExtensionsVulnerableSoftwareData, []byte("\n"))
	for _, line := range lines {
		parts := bytes.Split(line, []byte("##"))
		if len(parts) < 3 {
			log.Println("skipping", string(line))
			continue
		}
		vsCodeExtensionsVulnerableSoftware = append(vsCodeExtensionsVulnerableSoftware, fleet.Software{
			Vendor:  strings.TrimSpace(string(parts[0])),
			Name:    strings.TrimSpace(string(parts[1])),
			Version: strings.TrimSpace(string(parts[2])),
			Source:  "vscode_extensions",
		})
	}
	log.Printf("Loaded %d vulnerable vscode_extensions software", len(vsCodeExtensionsVulnerableSoftware))
}

func loadSoftwareItems(fs embed.FS, path string, source string) []map[string]string {
	bz2, err := fs.Open(path)
	if err != nil {
		panic(err)
	}

	type softwareJSON struct {
		Name        string `json:"name"`
		Version     string `json:"version"`
		UpgradeCode string `json:"upgrade_code"`
		Release     string `json:"release,omitempty"`
		Arch        string `json:"arch,omitempty"`
	}
	var softwareList []softwareJSON
	// ignoring "G110: Potential DoS vulnerability via decompression bomb", as this is test code.
	if err := json.NewDecoder(bzip2.NewReader(bz2)).Decode(&softwareList); err != nil { //nolint:gosec
		panic(err)
	}

	softwareRows := make([]map[string]string, 0, len(softwareList))
	for _, s := range softwareList {
		softwareRows = append(softwareRows, map[string]string{
			"name":         s.Name,
			"version":      s.Version,
			"source":       source,
			"upgrade_code": s.UpgradeCode,
		})
	}
	return softwareRows
}

func loadKernelList(fs embed.FS, path string) []string {
	data, err := fs.ReadFile(path)
	if err != nil {
		panic(err)
	}

	var kernels []string
	if err := json.Unmarshal(data, &kernels); err != nil {
		panic(err)
	}

	return kernels
}

func init() {
	loadMacOSVulnerableSoftware()
	loadExtraVulnerableSoftware()
	windowsSoftware = loadSoftwareItems(windowsSoftwareFS, "windows_11-software.json.bz2", "programs")
	ubuntuSoftware = loadSoftwareItems(ubuntuSoftwareFS, "ubuntu_2204-software.json.bz2", "deb_packages")
	ubuntuKernels = loadKernelList(ubuntuKernelsFS, "ubuntu_2204-kernels.json")
}

type nodeKeyManager struct {
	filepath string

	l        sync.Mutex
	nodekeys []string
}

func (n *nodeKeyManager) LoadKeys() {
	if n.filepath == "" {
		return
	}

	n.l.Lock()
	defer n.l.Unlock()

	data, err := os.ReadFile(n.filepath)
	if err != nil {
		log.Println("WARNING (ignore if creating a new node key file): error loading nodekey file:", err)
		return
	}
	n.nodekeys = strings.Split(string(data), "\n")
	n.nodekeys = n.nodekeys[:len(n.nodekeys)-1] // remove last empty node key due to new line.
	log.Printf("loaded %d node keys", len(n.nodekeys))
}

func (n *nodeKeyManager) Get(i int) string {
	n.l.Lock()
	defer n.l.Unlock()

	if len(n.nodekeys) > i {
		return n.nodekeys[i]
	}
	return ""
}

func (n *nodeKeyManager) Add(nodekey string) {
	if n.filepath == "" {
		return
	}

	// we lock just to make sure we write one at a time
	n.l.Lock()
	defer n.l.Unlock()

	f, err := os.OpenFile(n.filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
	if err != nil {
		log.Printf("error opening nodekey file: %s", err.Error())
		return
	}
	defer f.Close()
	if _, err := f.WriteString(nodekey + "\n"); err != nil {
		log.Printf("error writing nodekey file: %s", err)
	}
}

type mdmAgent struct {
	agentIndex            int
	MDMCheckInInterval    time.Duration
	model                 string
	serverAddress         string
	softwareCount         softwareEntityCount
	stats                 *osquery_perf.Stats
	strings               map[string]string
	softwareVersionMap    map[rune]int // Maps first char to version option: 0=base, 1=alternate, 2-31=patch versions 0-29
	mdmProfileFailureProb float64
}

// stats, model, *serverURL, *mdmSCEPChallenge, *mdmCheckInInterval

func (a *mdmAgent) CachedString(key string) string {
	if val, ok := a.strings[key]; ok {
		return val
	}
	val := randomString(12)
	a.strings[key] = val
	return val
}

// selectSoftwareVersion returns a consistent version for a software package based on its name.
// Same implementation as agent.selectSoftwareVersion.
func (a *mdmAgent) selectSoftwareVersion(softwareName, baseVersion, alternateVersion string) string {
	if len(softwareName) == 0 {
		return baseVersion
	}

	firstChar := rune(softwareName[0])
	versionOption, exists := a.softwareVersionMap[firstChar]
	if !exists {
		r := rand.Float64()
		switch {
		case r < 0.99:
			versionOption = 0
		case r < 0.999:
			versionOption = 1
		default:
			versionOption = 2 + rand.Intn(30)
		}
		a.softwareVersionMap[firstChar] = versionOption
	}

	switch versionOption {
	case 0:
		return baseVersion
	case 1:
		return alternateVersion
	default:
		patchNum := versionOption - 2
		return fmt.Sprintf("%s.%d", baseVersion, patchNum)
	}
}

// adamIDsToSoftware is the set of VPP apps that we support in our mock VPP install flow.
var adamIDsToSoftware = map[int]*fleet.Software{
	406056744: {
		Name:             "Evernote",
		BundleIdentifier: "com.evernote.Evernote",
		Version:          "10.147.1",
		Installed:        false,
	},
	1091189122: {
		Name:             "Bear: Markdown Notes",
		BundleIdentifier: "net.shinyfrog.bear",
		Version:          "2.4.5",
		Installed:        false,
	},
	1487937127: {
		Name:             "Craft: Write docs, AI editing",
		BundleIdentifier: "com.lukilabs.lukiapp",
		Version:          "3.1.7",
		Installed:        false,
	},
	1444383602: {
		Name:             "Goodnotes 6: AI Notes & Docs",
		BundleIdentifier: "com.goodnotesapp.x",
		Version:          "6.7.2",
		Installed:        false,
	},
}

type agent struct {
	agentIndex                    int
	hostCount                     int
	totalHostCount                int
	hostIndexOffset               int
	softwareCount                 softwareEntityCount
	softwareVSCodeExtensionsCount softwareExtraEntityCount
	userCount                     entityCount
	policyPassProb                float64
	munkiIssueProb                float64
	munkiIssueCount               int
	liveQueryFailProb             float64
	liveQueryNoResultsProb        float64
	strings                       map[string]string
	serverAddress                 string
	stats                         *osquery_perf.Stats
	nodeKeyManager                *nodeKeyManager
	nodeKey                       string
	templates                     *template.Template
	os                            string
	// deviceAuthToken holds Fleet Desktop device authentication token.
	//
	// Non-nil means the agent is identified as orbit osquery,
	// nil means the agent is identified as vanilla osquery.
	deviceAuthToken *string
	orbitNodeKey    *string

	// macMDMClient and winMDMClient simulate a device running the MDM protocol
	// (client side) against Fleet MDM.
	macMDMClient *mdmtest.TestAppleMDMClient
	winMDMClient *mdmtest.TestWindowsMDMClient

	// isEnrolledToMDM is true when the mdmDevice has enrolled.
	isEnrolledToMDM bool
	// isEnrolledToMDMMu protects isEnrolledToMDM.
	isEnrolledToMDMMu sync.Mutex

	disableScriptExec   bool
	disableFleetDesktop bool
	loggerTLSMaxLines   int

	// atomic boolean is set to true when executing scripts, so that only a
	// single goroutine at a time can execute scripts.
	scriptExecRunning atomic.Bool

	softwareQueryFailureProb         float64
	softwareVSCodeExtensionsFailProb float64

	softwareInstaller softwareInstaller

	linuxUniqueSoftwareVersion bool
	linuxUniqueSoftwareTitle   bool

	// Software installed on the host via Fleet. Key is the software name + version + bundle identifier.
	installedSoftware sync.Map

	// Cached software indices (pointers into global softwareDB array for this agent's platform)
	cachedSoftwareIndices []uint32

	// Host identity client for HTTP message signatures
	hostIdentityClient *hostidentity.Client

	//
	// The following are exported to be used by the templates.
	//

	EnrollSecret          string
	UUID                  string
	SerialNumber          string
	defaultSerialProb     float64
	ConfigInterval        time.Duration
	LogInterval           time.Duration
	QueryInterval         time.Duration
	MDMCheckInInterval    time.Duration
	DiskEncryptionEnabled bool
	mdmProfileFailureProb float64
	OSPatchLevel          int                 // For Linux patches
	linuxKernels          []map[string]string // Pre-selected kernels for this agent
	softwareVersionMap    map[rune]int        // Maps first char to version option: 0=base, 1=alternate, 2-31=patch versions 0-29

	// Note that a sync.Map is safe for concurrent use, but we still need a mutex
	// because we read and write the field itself (not data in the map) from
	// different goroutines (the write is in a.config).
	scheduledQueryMapMutex sync.RWMutex
	scheduledQueryData     *sync.Map
	// bufferedResults contains result logs that are buffered when
	// /api/v1/osquery/log requests to the Fleet server fail.
	//
	// NOTE: We use a map instead of a slice to prevent the data structure to
	// increase indefinitely (we sacrifice accuracy of logs but that's
	// a-ok for osquery-perf and load testing).
	bufferedResults map[resultLog]int

	// cache of certificates returned by this agent. Note that this requires
	// a mutex even though only used in a.processQuery, that's because both
	// the runLoop and the live query goroutines may call DistributedWrite
	// (which calls processQuery).
	certificatesMutex        sync.RWMutex
	certificatesCache        []map[string]string
	commonSoftwareNameSuffix string

	entraIDDeviceID          string
	entraIDUserPrincipalName string
	installedAdamIDs         []int
}

func (a *agent) GetSerialNumber() string {
	if rand.Float64() <= a.defaultSerialProb {
		return "-1"
	}
	return a.SerialNumber
}

type entityCount struct {
	common int
	unique int
}

type softwareEntityCount struct {
	entityCount
	vulnerable                        int
	withLastOpened                    int
	lastOpenedProb                    float64
	commonSoftwareUninstallCount      int
	commonSoftwareUninstallProb       float64
	uniqueSoftwareUninstallCount      int
	uniqueSoftwareUninstallProb       float64
	duplicateBundleIdentifiersPercent int
	softwareRenaming                  bool
}
type softwareExtraEntityCount struct {
	entityCount
	commonSoftwareUninstallCount int
	commonSoftwareUninstallProb  float64
	uniqueSoftwareUninstallCount int
	uniqueSoftwareUninstallProb  float64
}
type softwareInstaller struct {
	preInstallFailureProb  float64
	installFailureProb     float64
	postInstallFailureProb float64
	mu                     *sync.Mutex
}

func newAgent(
	agentIndex int,
	hostCount int,
	totalHostCount int,
	hostIndexOffset int,
	serverAddress, enrollSecret string,
	templates *template.Template,
	configInterval, logInterval, queryInterval, mdmCheckInInterval time.Duration,
	softwareQueryFailureProb float64,
	softwareVSCodeExtensionsQueryFailureProb float64,
	softwareInstaller softwareInstaller,
	softwareCount softwareEntityCount,
	softwareVSCodeExtensionsCount softwareExtraEntityCount,
	userCount entityCount,
	policyPassProb float64,
	orbitProb float64,
	munkiIssueProb float64, munkiIssueCount int,
	emptySerialProb float64,
	defaultSerialProb float64,
	mdmProb float64,
	mdmSCEPChallenge string,
	liveQueryFailProb float64,
	liveQueryNoResultsProb float64,
	disableScriptExec bool,
	disableFleetDesktop bool,
	loggerTLSMaxLines int,
	linuxUniqueSoftwareVersion bool,
	linuxUniqueSoftwareTitle bool,
	commonSoftwareNameSuffix string,
	mdmProfileFailureProb float64,
	httpMessageSignatureProb float64,
	httpMessageSignatureP384Prob float64,
) *agent {
	var deviceAuthToken *string
	if rand.Float64() <= orbitProb {
		deviceAuthToken = ptr.String(uuid.NewString())
	}
	serialNumber := mdmtest.RandSerialNumber()
	if rand.Float64() <= emptySerialProb {
		serialNumber = ""
	}
	hostUUID := strings.ToUpper(uuid.New().String())

	// determine the simulated host's OS based on the template name (see
	// validTemplateNames below for the list of possible names, the OS is always
	// the part before the underscore). Note that it is the OS and not the
	// "normalized" platform, so "ubuntu" and not "linux", "macos" and not
	// "darwin".
	agentOS := strings.TrimRight(templates.Name(), ".tmpl")
	agentOS, _, _ = strings.Cut(agentOS, "_")

	var (
		macMDMClient *mdmtest.TestAppleMDMClient
		winMDMClient *mdmtest.TestWindowsMDMClient
	)
	if rand.Float64() < mdmProb {
		switch agentOS {
		case "macos":
			macMDMClient = mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{
				SCEPChallenge: mdmSCEPChallenge,
				SCEPURL:       serverAddress + apple_mdm.SCEPPath,
				MDMURL:        serverAddress + apple_mdm.MDMPath,
			}, "MacBookPro16,1")
			// Have the osquery agent match the MDM device serial number and UUID.
			serialNumber = macMDMClient.SerialNumber
			hostUUID = macMDMClient.UUID

		case "windows":
			// windows MDM enrollment requires orbit enrollment
			if deviceAuthToken == nil {
				deviceAuthToken = ptr.String(uuid.NewString())
			}
			// creating the Windows MDM client requires the orbit node key, but we
			// only get it after orbit enrollment. So here we just set the value to a
			// placeholder (non-nil) client, the actual usable client will be created
			// after orbit enrollment, and after receiving the enrollment
			// notification.
			winMDMClient = new(mdmtest.TestWindowsMDMClient)
		}
	}

	// Determine if this agent should use HTTP message signatures
	useHTTPSig := rand.Float64() < httpMessageSignatureProb // nolint:gosec // ignore weak randomizer

	agent := &agent{
		agentIndex:                    agentIndex,
		hostCount:                     hostCount,
		totalHostCount:                totalHostCount,
		hostIndexOffset:               hostIndexOffset,
		serverAddress:                 serverAddress,
		softwareCount:                 softwareCount,
		softwareVSCodeExtensionsCount: softwareVSCodeExtensionsCount,
		userCount:                     userCount,
		strings:                       make(map[string]string),
		policyPassProb:                policyPassProb,
		munkiIssueProb:                munkiIssueProb,
		munkiIssueCount:               munkiIssueCount,
		liveQueryFailProb:             liveQueryFailProb,
		liveQueryNoResultsProb:        liveQueryNoResultsProb,
		templates:                     templates,
		deviceAuthToken:               deviceAuthToken,
		os:                            agentOS,
		EnrollSecret:                  enrollSecret,
		ConfigInterval:                configInterval,
		LogInterval:                   logInterval,
		QueryInterval:                 queryInterval,
		MDMCheckInInterval:            mdmCheckInInterval,
		UUID:                          hostUUID,
		SerialNumber:                  serialNumber,
		defaultSerialProb:             defaultSerialProb,
		OSPatchLevel:                  rand.Intn(25), // Random patch level 0-24

		softwareQueryFailureProb:         softwareQueryFailureProb,
		softwareVSCodeExtensionsFailProb: softwareVSCodeExtensionsQueryFailureProb,
		softwareInstaller:                softwareInstaller,

		linuxUniqueSoftwareVersion: linuxUniqueSoftwareVersion,
		linuxUniqueSoftwareTitle:   linuxUniqueSoftwareTitle,

		macMDMClient: macMDMClient,
		winMDMClient: winMDMClient,

		disableScriptExec:        disableScriptExec,
		disableFleetDesktop:      disableFleetDesktop,
		loggerTLSMaxLines:        loggerTLSMaxLines,
		bufferedResults:          make(map[resultLog]int),
		scheduledQueryData:       new(sync.Map),
		softwareVersionMap:       make(map[rune]int),
		commonSoftwareNameSuffix: commonSoftwareNameSuffix,
		mdmProfileFailureProb:    mdmProfileFailureProb,

		entraIDDeviceID:          uuid.NewString(),
		entraIDUserPrincipalName: fmt.Sprintf("fake-%s@example.com", randomString(5)),
	}

	// Initialize host identity client
	agent.hostIdentityClient = hostidentity.NewClient(hostidentity.Config{
		ServerAddress: serverAddress,
		EnrollSecret:  enrollSecret,
		HostUUID:      hostUUID,
		AgentIndex:    agentIndex,
	}, useHTTPSig, httpMessageSignatureP384Prob)

	// Pre-select kernels for Ubuntu agents to ensure consistency across queries
	if agentOS == "ubuntu" {
		agent.linuxKernels = selectKernels(ubuntuKernels)
	}

	return agent
}

type enrollResponse struct {
	NodeKey string `json:"node_key"`
}

type distributedReadResponse struct {
	Queries map[string]string `json:"queries"`
}

type scheduledQuery struct {
	Query            string  `json:"query"`
	Name             string  `json:"name"`
	ScheduleInterval float64 `json:"interval"`
	Platform         string  `json:"platform"`
	Version          string  `json:"version"`
	Snapshot         bool    `json:"snapshot"`

	lastRun  int64
	numRows  uint
	packName string
}

func (a *agent) isOrbit() bool {
	return a.deviceAuthToken != nil
}

func (a *agent) runLoop(i int, onlyAlreadyEnrolled bool) {
	// Request host identity certificate if this agent uses HTTP message signatures
	if a.hostIdentityClient.IsEnabled() && !onlyAlreadyEnrolled {
		if err := a.hostIdentityClient.RequestCertificate(); err != nil {
			log.Printf("Agent %d: Failed to request host identity certificate: %v", a.agentIndex, err)
			return
		}
	}

	if a.isOrbit() {
		if err := a.orbitEnroll(); err != nil {
			// clean-up any placeholder mdm client that depended on orbit enrollment
			// - there's no concurrency yet for a given agent instance, runLoop is
			// the place where the goroutines will be started later on.
			a.winMDMClient = nil
			return
		}
		if a.winMDMClient != nil {
			a.winMDMClient = mdmtest.NewTestMDMClientWindowsProgramatic(a.serverAddress, *a.orbitNodeKey)
		}
	}

	if err := a.enroll(i, onlyAlreadyEnrolled); err != nil {
		return
	}

	_ = a.config()

	resp, err := a.DistributedRead()
	if err == nil {
		if len(resp.Queries) > 0 {
			_ = a.DistributedWrite(resp.Queries)
		}
	}

	if a.isOrbit() {
		go a.runOrbitLoop()
	}

	// NOTE: the windows MDM client enrollment is only done after receiving a
	// notification via the config in the runOrbitLoop.
	if a.macMDMClient != nil {
		if err := a.macMDMClient.Enroll(); err != nil {
			log.Printf("macOS MDM enroll failed: %s", err)
			a.stats.IncrementMDMErrors()
			return
		}
		a.setMDMEnrolled()
		a.stats.IncrementMDMEnrollments()
		go a.runMacosMDMLoop()
	}

	//
	// osquery runs three separate independent threads,
	//	- a thread for getting, running and submitting results for distributed queries (distributed).
	// 	- a thread for getting configuration from a remote server (config).
	//	- a thread for submitting log results (logger).
	//
	// Thus we try to simulate that as much as we can.

	// (1) distributed thread:
	go func() {
		liveQueryTicker := time.NewTicker(a.QueryInterval)
		defer liveQueryTicker.Stop()

		for range liveQueryTicker.C {
			if resp, err := a.DistributedRead(); err == nil && len(resp.Queries) > 0 {
				_ = a.DistributedWrite(resp.Queries)
			}
		}
	}()

	// (2) config thread:
	go func() {
		configTicker := time.NewTicker(a.ConfigInterval)
		defer configTicker.Stop()

		for range configTicker.C {
			_ = a.config()
		}
	}()

	// (3) logger thread:
	logTicker := time.NewTicker(a.LogInterval)
	defer logTicker.Stop()
	for range logTicker.C {
		// check if we have any scheduled queries that should be returning results
		var results []resultLog
		now := time.Now().Unix()
		prevCount := a.countBuffered()

		// NOTE The goroutine that pulls in new configurations
		// MAY replace this map if it happens to run at the
		// exact same time. The result would be. The result
		// would be that the query lastRun does not get
		// updated and cause the query to run more times than
		// expected.
		a.scheduledQueryMapMutex.RLock()
		queryData := a.scheduledQueryData
		a.scheduledQueryMapMutex.RUnlock()
		queryData.Range(func(key, value any) bool {
			queryName := key.(string)
			query := value.(scheduledQuery)

			if query.lastRun == 0 || now >= (query.lastRun+int64(query.ScheduleInterval)) {
				results = append(results, resultLog{
					packName:  query.packName,
					queryName: query.Name,
					numRows:   int(query.numRows),
				})
				// Update lastRun
				query.lastRun = now
				queryData.Store(queryName, query)
			}

			return true
		})
		if prevCount+len(results) < 1_000_000 { // osquery buffered_log_max is 1M
			a.addToBuffer(results)
		}
		a.sendLogsBatch()
		newBufferedCount := a.countBuffered() - prevCount
		a.stats.UpdateBufferedLogs(newBufferedCount)
	}
}

func (a *agent) countBuffered() int {
	var total int
	for _, count := range a.bufferedResults {
		total += count
	}
	return total
}

func (a *agent) addToBuffer(results []resultLog) {
	for _, result := range results {
		a.bufferedResults[result] += 1
	}
}

// getBatch returns a random set of logs from the buffered logs.
// NOTE: We sacrifice some accuracy in the name of CPU and memory efficiency.
func (a *agent) getBatch(batchSize int) []resultLog {
	results := make([]resultLog, 0, batchSize)
	for result, count := range a.bufferedResults {
		left := batchSize - len(results)
		if left <= 0 {
			return results
		}
		if count > left {
			count = left
		}
		for i := 0; i < count; i++ {
			results = append(results, result)
		}
	}
	return results
}

type resultLog struct {
	packName  string
	queryName string
	numRows   int
}

func (r resultLog) emit() []byte {
	return scheduledQueryResults(r.packName, r.queryName, r.numRows)
}

// sendLogsBatch sends up to loggerTLSMaxLines logs and updates the buffer.
func (a *agent) sendLogsBatch() {
	if len(a.bufferedResults) == 0 {
		return
	}

	batchSize := a.loggerTLSMaxLines
	if count := a.countBuffered(); count < batchSize {
		batchSize = count
	}
	batch := a.getBatch(batchSize)
	if err := a.submitLogs(batch); err != nil {
		return
	}
	a.removeBuffered(batchSize)
}

// removeBuffered removes a random set of logs from the buffered logs.
// NOTE: We sacrifice some accuracy in the name of CPU and memory efficiency.
func (a *agent) removeBuffered(batchSize int) {
	for b := batchSize; b > 0; {
		for result, count := range a.bufferedResults {
			if count > b {
				a.bufferedResults[result] -= b
				return
			}
			delete(a.bufferedResults, result)
			b -= count
		}
	}
}

func (a *agent) runOrbitLoop() {
	// Create signerWrapper if HTTP signatures are enabled
	var signerWrapper func(*http.Client) *http.Client
	if a.hostIdentityClient.IsEnabled() && a.hostIdentityClient.HasSigner() {
		signer := a.hostIdentityClient.GetSigner()
		signerWrapper = func(client *http.Client) *http.Client {
			return httpsig.NewHTTPClient(client, signer, nil)
		}
	}

	orbitClient, err := service.NewOrbitClient(
		"",
		a.serverAddress,
		"",
		true,
		a.EnrollSecret,
		nil,
		fleet.OrbitHostInfo{
			HardwareUUID:   a.UUID,
			HardwareSerial: a.SerialNumber,
			Hostname:       a.CachedString("hostname"),
		},
		nil,
		signerWrapper,
		"",
	)
	if err != nil {
		log.Println("creating orbit client: ", err)
	}

	orbitClient.TestNodeKey = *a.orbitNodeKey

	deviceClient, err := service.NewDeviceClient(a.serverAddress, true, "", nil, "")
	if err != nil {
		log.Fatal("creating device client: ", err)
	}

	// orbit does a config check when it starts
	if _, err := orbitClient.GetConfig(); err != nil {
		a.stats.IncrementOrbitErrors()
	}

	tokenRotationEnabled := false
	if !a.disableFleetDesktop {
		tokenRotationEnabled = orbitClient.GetServerCapabilities().Has(fleet.CapabilityOrbitEndpoints) &&
			orbitClient.GetServerCapabilities().Has(fleet.CapabilityTokenRotation)

		// it also writes and checks the device token
		if tokenRotationEnabled {
			if err := orbitClient.SetOrUpdateDeviceToken(*a.deviceAuthToken); err != nil {
				a.stats.IncrementOrbitErrors()
				log.Println("orbitClient.SetOrUpdateDeviceToken: ", err)
			}

			if err := deviceClient.CheckToken(*a.deviceAuthToken); err != nil {
				a.stats.IncrementOrbitErrors()
				log.Println("deviceClient.CheckToken: ", err)
			}
		}
	}

	// checkToken is used to simulate Fleet Desktop polling until a token is
	// valid, we make a random number of requests to properly emulate what
	// happens in the real world as there are delays that are not accounted by
	// the way this simulation is arranged.
	checkToken := func() {
		minVal := 1
		maxVal := 5
		numberOfRequests := rand.Intn(maxVal-minVal+1) + minVal
		ticker := time.NewTicker(5 * time.Second)
		defer ticker.Stop()

		for {
			<-ticker.C
			numberOfRequests--
			if err := deviceClient.CheckToken(*a.deviceAuthToken); err != nil {
				log.Println("deviceClient.CheckToken: ", err)
			}
			if numberOfRequests == 0 {
				break
			}
		}
	}

	// Fleet Desktop performs a burst of check token requests when it's initialized
	if !a.disableFleetDesktop {
		checkToken()
	}

	// orbit makes a call to check the config and update the CLI flags every 30
	// seconds
	orbitConfigTicker := time.Tick(30 * time.Second)
	// orbit makes a call every 5 minutes to check the validity of the device
	// token on the server
	orbitTokenRemoteCheckTicker := time.Tick(5 * time.Minute)
	// orbit pings the server every 1 hour to rotate the device token
	orbitTokenRotationTicker := time.Tick(1 * time.Hour)
	// orbit polls the /orbit/ping endpoint every 5 minutes to check if the
	// server capabilities have changed
	capabilitiesCheckerTicker := time.Tick(5 * time.Minute)
	// fleet desktop polls for policy compliance every 5 minutes
	fleetDesktopPolicyTicker := time.Tick(5 * time.Minute)
	// fleet desktop pings every 10s for connectivity check.
	fleetDesktopConnectivityCheck := time.Tick(10 * time.Second)

	const windowsMDMEnrollmentAttemptFrequency = time.Hour
	var lastEnrollAttempt time.Time

	for {
		select {
		case <-orbitConfigTicker:
			cfg, err := orbitClient.GetConfig()
			if err != nil {
				a.stats.IncrementOrbitErrors()
				continue
			}
			if len(cfg.Notifications.PendingScriptExecutionIDs) > 0 {
				// there are pending scripts to execute on this host, start a goroutine
				// that will simulate executing them.
				go a.execScripts(cfg.Notifications.PendingScriptExecutionIDs, orbitClient)
			}
			if len(cfg.Notifications.PendingSoftwareInstallerIDs) > 0 {
				// there are pending software installations on this host, start a
				// goroutine that will download the software
				go a.installSoftware(cfg.Notifications.PendingSoftwareInstallerIDs, orbitClient)
			}
			if cfg.Notifications.NeedsProgrammaticWindowsMDMEnrollment &&
				!a.mdmEnrolled() &&
				a.winMDMClient != nil &&
				time.Since(lastEnrollAttempt) > windowsMDMEnrollmentAttemptFrequency {
				lastEnrollAttempt = time.Now()
				if err := a.winMDMClient.Enroll(); err != nil {
					log.Printf("Windows MDM enroll failed: %s", err)
					a.stats.IncrementMDMErrors()
				} else {
					a.setMDMEnrolled()
					a.stats.IncrementMDMEnrollments()
					go a.runWindowsMDMLoop()
				}
			}
		case <-orbitTokenRemoteCheckTicker:
			if !a.disableFleetDesktop && tokenRotationEnabled {
				if err := deviceClient.CheckToken(*a.deviceAuthToken); err != nil {
					a.stats.IncrementOrbitErrors()
					log.Println("deviceClient.CheckToken: ", err)
					continue
				}
			}
		case <-orbitTokenRotationTicker:
			if !a.disableFleetDesktop && tokenRotationEnabled {
				newToken := ptr.String(uuid.NewString())
				if err := orbitClient.SetOrUpdateDeviceToken(*newToken); err != nil {
					a.stats.IncrementOrbitErrors()
					log.Println("orbitClient.SetOrUpdateDeviceToken: ", err)
					continue
				}
				a.deviceAuthToken = newToken
				// fleet desktop performs a burst of check token requests after a token is rotated
				checkToken()
			}
		case <-capabilitiesCheckerTicker:
			if err := orbitClient.Ping(); err != nil {
				a.stats.IncrementOrbitErrors()
				continue
			}
		case <-fleetDesktopPolicyTicker:
			if !a.disableFleetDesktop {
				if _, err := deviceClient.DesktopSummary(*a.deviceAuthToken); err != nil {
					a.stats.IncrementDesktopErrors()
					log.Println("deviceClient.NumberOfFailingPolicies: ", err)
					continue
				}
			}
		case <-fleetDesktopConnectivityCheck:
			if !a.disableFleetDesktop {
				if err := deviceClient.Ping(); err != nil {
					a.stats.IncrementDesktopErrors()
					log.Println("deviceClient.Ping: ", err)
					continue
				}
			}
		}
	}
}

func (a *agent) runMacosMDMLoop() {
	mdmCheckInTicker := time.Tick(a.MDMCheckInInterval)

	for range mdmCheckInTicker {
		mdmCommandPayload, err := a.macMDMClient.Idle()
		if err != nil {
			log.Printf("MDM Idle request failed: %s", err)
			a.stats.IncrementMDMErrors()
			continue
		}
		a.stats.IncrementMDMSessions()

	INNER_FOR_LOOP:
		for mdmCommandPayload != nil {
			a.stats.IncrementMDMCommandsReceived()

			switch mdmCommandPayload.Command.RequestType {
			case "InstallProfile":
				if a.mdmProfileFailureProb > 0.0 && rand.Float64() <= a.mdmProfileFailureProb {
					errChain := []mdm.ErrorChain{
						{
							ErrorCode:            89,
							ErrorDomain:          "ErrorDomain",
							LocalizedDescription: "The profile did not install",
						},
					}
					mdmCommandPayload, err = a.macMDMClient.Err(mdmCommandPayload.CommandUUID, errChain)
					if err != nil {
						log.Printf("MDM Error request failed: %s", err)
						a.stats.IncrementMDMErrors()
						break INNER_FOR_LOOP
					}
				} else {
					mdmCommandPayload, err = a.macMDMClient.Acknowledge(mdmCommandPayload.CommandUUID)
					if err != nil {
						log.Printf("MDM Acknowledge request failed: %s", err)
						a.stats.IncrementMDMErrors()
						break INNER_FOR_LOOP
					}
				}
			case "DeclarativeManagement":
				// Device immediately responds with Acknowledged status and then contacts the Declarations endpoints.
				nextMdmCommandPayload, err := a.macMDMClient.Acknowledge(mdmCommandPayload.CommandUUID)
				if err != nil {
					log.Printf("MDM Acknowledge request failed: %s", err)
					a.stats.IncrementMDMErrors()
					break INNER_FOR_LOOP
				}
				// Note: Declarative management could happen async while other MDM commands proceed. This is a potential enhancement.
				a.doDeclarativeManagement(mdmCommandPayload)
				mdmCommandPayload = nextMdmCommandPayload
			case "InstalledApplicationList":
				var installedVPPSoftware []fleet.Software
				// Our mock VPP apps start off as "not installed".
				// The first time we get a verification command, we flip the flag to "installed",
				// but don't include the software in the response.
				// This ensures that 2 verification commands will be sent per VPP install.
				for _, adamID := range a.installedAdamIDs {
					if sw, ok := adamIDsToSoftware[adamID]; ok && sw != nil {
						if sw.Installed {
							installedVPPSoftware = append(installedVPPSoftware, *sw)
						}

						sw.Installed = true
					}
				}
				nextMdmCommandPayload, err := a.macMDMClient.AcknowledgeInstalledApplicationList(
					a.macMDMClient.UUID,
					mdmCommandPayload.CommandUUID,
					installedVPPSoftware,
				)
				if err != nil {
					log.Printf("MDM Acknowledge InstalledApplicationList request failed: %s", err)
					a.stats.IncrementMDMErrors()
					break INNER_FOR_LOOP
				}

				mdmCommandPayload = nextMdmCommandPayload
			case "InstallApplication":
				var appRequest struct {
					Command map[string]any `plist:"Command"`
				}

				err = plist.Unmarshal(mdmCommandPayload.Raw, &appRequest)
				if err != nil {
					log.Printf("parsing InstallApplication request: %s", err)
					a.stats.IncrementMDMErrors()
					break INNER_FOR_LOOP
				}
				log.Printf("got install application command for %d", appRequest.Command["iTunesStoreID"])

				if adamID, ok := appRequest.Command["iTunesStoreID"].(uint64); ok {
					a.installedAdamIDs = append(a.installedAdamIDs, int(adamID))
				}

				mdmCommandPayload, err = a.macMDMClient.Acknowledge(mdmCommandPayload.CommandUUID)
				if err != nil {
					log.Printf("MDM Acknowledge request failed: %s", err)
					a.stats.IncrementMDMErrors()
					break INNER_FOR_LOOP
				}

			default:
				mdmCommandPayload, err = a.macMDMClient.Acknowledge(mdmCommandPayload.CommandUUID)
				if err != nil {
					log.Printf("MDM Acknowledge request failed: %s", err)
					a.stats.IncrementMDMErrors()
					break INNER_FOR_LOOP
				}
			}
		}
	}
}

func (a *agent) doDeclarativeManagement(cmd *mdm.Command) {
	// defer log.Printf("Exiting DeclarativeManagement for command %s", cmd.CommandUUID)

	// get declaration-items endpoint
	r, err := a.macMDMClient.DeclarativeManagement("declaration-items")
	if err != nil {
		log.Printf("DDM %s declaration-items request failed: %s", cmd.CommandUUID, err)
		a.stats.IncrementDDMDeclarationItemsErrors()
		return
	}
	body, err := io.ReadAll(r.Body)
	if err != nil {
		log.Printf("DDM %s declaration-items read body failed: %s", cmd.CommandUUID, err)
		a.stats.IncrementDDMDeclarationItemsErrors()
		return
	}
	var items fleet.MDMAppleDDMDeclarationItemsResponse
	err = json.Unmarshal(body, &items)
	if err != nil {
		log.Printf("DDM %s declaration-items unmarshal failed: %s", cmd.CommandUUID, err)
		a.stats.IncrementDDMDeclarationItemsErrors()
		return
	}
	a.stats.IncrementDDMDeclarationItemsSuccess()

	// get declaration/configuration/:identifer endpoint
	for _, d := range items.Declarations.Configurations {
		path := fmt.Sprintf("declaration/%s/%s", "configuration", d.Identifier)
		r, err := a.macMDMClient.DeclarativeManagement(path)
		if err != nil {
			log.Printf("DDM %s request failed: %s", path, err)
			a.stats.IncrementDDMConfigurationErrors()
			return
		}
		body, err := io.ReadAll(r.Body)
		if err != nil {
			log.Printf("DDM %s read body failed: %s", path, err)
			a.stats.IncrementDDMConfigurationErrors()
			return
		}
		var decl fleet.MDMAppleDeclaration
		err = json.Unmarshal(body, &decl)
		if err != nil {
			log.Printf("DDM %s unmarshal failed: %s", path, err)
			a.stats.IncrementDDMConfigurationErrors()
			return
		}
	}
	a.stats.IncrementDDMConfigurationSuccess()

	// get declaration/activation/:identifer endpoint
	for _, d := range items.Declarations.Activations {
		path := fmt.Sprintf("declaration/%s/%s", "activation", d.Identifier)
		r, err := a.macMDMClient.DeclarativeManagement(path)
		if err != nil {
			log.Printf("DDM %s request failed: %s", path, err)
			a.stats.IncrementDDMActivationErrors()
			return
		}
		body, err := io.ReadAll(r.Body)
		if err != nil {
			log.Printf("DDM %s read body failed: %s", path, err)
			a.stats.IncrementDDMActivationErrors()
			return
		}
		var act fleet.MDMAppleDDMActivation
		err = json.Unmarshal(body, &act)
		if err != nil {
			log.Printf("DDM %s unmarshal failed: %s", path, err)
			a.stats.IncrementDDMActivationErrors()
			return
		}
	}
	a.stats.IncrementDDMActivationSuccess()

	// sent status report
	for _, d := range items.Declarations.Configurations {
		report := fleet.MDMAppleDDMStatusReport{}
		report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{
			{Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: d.Identifier, ServerToken: d.ServerToken},
		}
		r, err := a.macMDMClient.DeclarativeManagement("status", report)
		if err != nil {
			log.Printf("DDM %s status request failed: %s", d.Identifier, err)
			a.stats.IncrementDDMStatusErrors()
			return
		}

		// Apple's documentation has some conflicting information about the expected status here so we'll
		// just check for both.
		//
		// https://developer.apple.com/documentation/devicemanagement/get_the_device_status#response-codes
		// https://developer.apple.com/documentation/devicemanagement/statusreport#discussion
		if r.StatusCode != http.StatusOK && r.StatusCode != http.StatusNoContent {
			log.Printf("DDM %s status response unexpected: %d", d.Identifier, r.StatusCode)
			a.stats.IncrementDDMStatusErrors()
			return
		}
	}
	a.stats.IncrementDDMStatusSuccess()
}

func (a *agent) runWindowsMDMLoop() {
	mdmCheckInTicker := time.Tick(a.MDMCheckInInterval)

	for range mdmCheckInTicker {
		cmds, err := a.winMDMClient.StartManagementSession()
		if err != nil {
			log.Printf("MDM check-in start session request failed: %s", err)
			a.stats.IncrementMDMErrors()
			continue
		}
		a.stats.IncrementMDMSessions()

		// send a successful ack for each command
		msgID, err := a.winMDMClient.GetCurrentMsgID()
		if err != nil {
			log.Printf("MDM get current MsgID failed: %s", err)
			a.stats.IncrementMDMErrors()
			continue
		}

		for _, c := range cmds {
			a.stats.IncrementMDMCommandsReceived()

			status := syncml.CmdStatusOK
			if a.mdmProfileFailureProb > 0.0 && rand.Float64() <= a.mdmProfileFailureProb {
				status = syncml.CmdStatusBadRequest
			}
			a.winMDMClient.AppendResponse(fleet.SyncMLCmd{
				XMLName: xml.Name{Local: fleet.CmdStatus},
				MsgRef:  &msgID,
				CmdRef:  &c.Cmd.CmdID.Value,
				Cmd:     ptr.String(c.Verb),
				Data:    &status,
				Items:   nil,
				CmdID:   fleet.CmdID{Value: uuid.NewString()},
			})
		}
		if _, err := a.winMDMClient.SendResponse(); err != nil {
			log.Printf("MDM send response request failed: %s", err)
			a.stats.IncrementMDMErrors()
			continue
		}
	}
}

func (a *agent) execScripts(execIDs []string, orbitClient *service.OrbitClient) {
	if a.scriptExecRunning.Swap(true) {
		// if Swap returns true, the goroutine was already running, exit
		return
	}
	defer a.scriptExecRunning.Store(false)

	log.Printf("running scripts: %v", execIDs)
	for _, execID := range execIDs {
		if a.disableScriptExec {
			// send a no-op result without executing if script exec is disabled
			if err := orbitClient.SaveHostScriptResult(&fleet.HostScriptResultPayload{
				ExecutionID: execID,
				Output:      "Scripts are disabled",
				Runtime:     0,
				ExitCode:    -2,
			}); err != nil {
				log.Println("save disabled host script result:", err)
				return
			}
			log.Printf("did save disabled host script result: id=%s", execID)
			continue
		}

		a.stats.IncrementScriptExecs()
		script, err := orbitClient.GetHostScript(execID)
		if err != nil {
			log.Println("get host script:", err)
			a.stats.IncrementScriptExecErrs()
			return
		}

		// simulate script execution
		outputLen := rand.Intn(11000) // base64 encoding will make the actual output a bit bigger
		buf := make([]byte, outputLen)
		n, _ := io.ReadFull(cryptorand.Reader, buf)
		exitCode := rand.Intn(2)
		runtime := rand.Intn(5)
		time.Sleep(time.Duration(runtime) * time.Second)

		if err := orbitClient.SaveHostScriptResult(&fleet.HostScriptResultPayload{
			HostID:      script.HostID,
			ExecutionID: script.ExecutionID,
			Output:      base64.StdEncoding.EncodeToString(buf[:n]),
			Runtime:     runtime,
			ExitCode:    exitCode,
		}); err != nil {
			log.Println("save host script result:", err)
			a.stats.IncrementScriptExecErrs()
			return
		}
		log.Printf("did exec and save host script result: id=%s, output size=%d, runtime=%d, exit code=%d", execID, base64.StdEncoding.EncodedLen(n), runtime, exitCode)
	}
}

func (a *agent) installSoftware(installerIDs []string, orbitClient *service.OrbitClient) {
	// Only allow one software install to happen at a time.
	if a.softwareInstaller.mu.TryLock() {
		defer a.softwareInstaller.mu.Unlock()
		for _, installerID := range installerIDs {
			a.installSoftwareItem(installerID, orbitClient)
		}
	}
}

func (a *agent) installSoftwareItem(installerID string, orbitClient *service.OrbitClient) {
	a.stats.IncrementSoftwareInstalls()

	payload := &fleet.HostSoftwareInstallResultPayload{}
	payload.InstallUUID = installerID
	installer, err := orbitClient.GetInstallerDetails(installerID)
	if err != nil {
		log.Println("get installer details:", err)
		a.stats.IncrementSoftwareInstallErrs()
		return
	}
	failed := false
	if installer.PreInstallCondition != "" {
		time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
		if installer.PreInstallCondition == "select 1" { //nolint:gocritic // ignore ifElseChain
			// Always pass
			payload.PreInstallConditionOutput = ptr.String("1")
		} else if installer.PreInstallCondition == "select 0" ||
			a.softwareInstaller.preInstallFailureProb > 0.0 && rand.Float64() <= a.softwareInstaller.preInstallFailureProb {
			// Fail
			payload.PreInstallConditionOutput = ptr.String("")
			failed = true
		} else {
			payload.PreInstallConditionOutput = ptr.String("1")
		}
	}

	var meta *file.InstallerMetadata
	if !failed {
		var cacheMiss bool
		// Download the file if needed to get its metadata
		meta, cacheMiss, err = installerMetadataCache.Get(installer, orbitClient)
		if err != nil {
			a.stats.IncrementSoftwareInstallErrs()
			return
		}

		if !cacheMiss && installer.SoftwareInstallerURL == nil {
			// If we didn't download and analyze the file, AND we did not use a CDN URL to get the file,
			// we do a download now and don't save the result. Doing this download adds realistic load on the server.
			err = orbitClient.DownloadAndDiscardSoftwareInstaller(installer.InstallerID)
			if err != nil {
				log.Println("download and discard software installer:", err)
				a.stats.IncrementSoftwareInstallErrs()
				return
			}
		}

		time.Sleep(time.Duration(rand.Intn(30)) * time.Second)
		if installer.InstallScript == "exit 0" { //nolint:gocritic // ignore ifElseChain
			// Always pass
			payload.InstallScriptExitCode = ptr.Int(0)
			payload.InstallScriptOutput = ptr.String("Installed on osquery-perf (always pass)")
		} else if installer.InstallScript == "exit 1" {
			payload.InstallScriptExitCode = ptr.Int(1)
			payload.InstallScriptOutput = ptr.String("Installed on osquery-perf (always fail)")
			failed = true
		} else if a.softwareInstaller.installFailureProb > 0.0 && rand.Float64() <= a.softwareInstaller.installFailureProb {
			payload.InstallScriptExitCode = ptr.Int(1)
			payload.InstallScriptOutput = ptr.String("Installed on osquery-perf (fail)")
			failed = true
		} else {
			payload.InstallScriptExitCode = ptr.Int(0)
			payload.InstallScriptOutput = ptr.String("Installed on osquery-perf (pass)")
		}
	}
	if !failed {
		if meta.Name == "" {
			log.Printf("WARNING: installer metadata is missing a name for installer:%d\n", installer.InstallerID)
		} else {
			key := meta.Name + "+" + meta.Version + "+" + meta.BundleIdentifier
			if _, ok := a.installedSoftware.Load(key); !ok {
				source := ""
				switch a.os {
				case "macos":
					source = "apps"
				case "windows":
					source = "programs"
				case "ubuntu":
					source = "deb_packages"
				default:
					log.Printf("unknown OS to software installer: %s", a.os)
					return
				}
				a.installedSoftware.Store(key, map[string]string{
					"name":              meta.Name,
					"version":           meta.Version,
					"bundle_identifier": meta.BundleIdentifier,
					"source":            source,
					"installed_path":    os.DevNull,
				})
			}
		}

		if installer.PostInstallScript != "" {
			time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
			if installer.PostInstallScript == "exit 0" { //nolint:gocritic // ignore ifElseChain
				// Always pass
				payload.PostInstallScriptExitCode = ptr.Int(0)
				payload.PostInstallScriptOutput = ptr.String("PostInstall on osquery-perf (always pass)")
			} else if installer.PostInstallScript == "exit 1" {
				payload.PostInstallScriptExitCode = ptr.Int(1)
				payload.PostInstallScriptOutput = ptr.String("PostInstall on osquery-perf (always fail)")
			} else if a.softwareInstaller.postInstallFailureProb > 0.0 && rand.Float64() <= a.softwareInstaller.postInstallFailureProb {
				payload.PostInstallScriptExitCode = ptr.Int(1)
				payload.PostInstallScriptOutput = ptr.String("PostInstall on osquery-perf (fail)")
			} else {
				payload.PostInstallScriptExitCode = ptr.Int(0)
				payload.PostInstallScriptOutput = ptr.String("PostInstall on osquery-perf (pass)")
			}
		}
	}

	err = orbitClient.SaveInstallerResult(payload)
	if err != nil {
		log.Println("save installer result:", err)
		a.stats.IncrementSoftwareInstallErrs()
		return
	}
}

// shouldSignRequest determines if a request should be signed based on its path
func (a *agent) shouldSignRequest(req *http.Request) bool {
	// Don't sign if HTTP signatures are not enabled
	if !a.hostIdentityClient.IsEnabled() || !a.hostIdentityClient.HasSigner() {
		return false
	}

	// Exclude ping endpoint from signing
	if strings.HasSuffix(req.URL.Path, "/api/fleet/orbit/ping") {
		return false
	}

	// Only sign specific API paths
	return strings.Contains(req.URL.Path, "/api/fleet/orbit/") || strings.Contains(req.URL.Path, "/osquery/")
}

// sign applies HTTP message signature to a request if needed
func (a *agent) sign(req *http.Request) *http.Request {
	// Apply HTTP message signature if this request should be signed
	if a.shouldSignRequest(req) {
		if err := a.hostIdentityClient.SignRequest(req); err != nil {
			log.Printf("Agent %d: Failed to sign HTTP request: %v", a.agentIndex, err)
		}
	}
	return req
}

func (a *agent) waitingDo(fn func() *http.Request) *http.Response {
	response, err := http.DefaultClient.Do(a.sign(fn()))
	for err != nil || response.StatusCode != http.StatusOK {
		if err != nil {
			log.Printf("failed to run request: %s", err)
		} else { // res.StatusCode() != http.StatusOK
			response.Body.Close()
			log.Printf("request failed: %d", response.StatusCode)
		}
		a.stats.IncrementErrors(1)
		<-time.Tick(time.Duration(rand.Intn(120)+1) * time.Second)
		response, err = http.DefaultClient.Do(a.sign(fn()))
	}
	return response
}

// TODO: add support to `alreadyEnrolled` akin to the `enroll` function.  for
// now, we assume that the agent is not already enrolled, if you kill the agent
// process then those Orbit node keys are gone.
func (a *agent) orbitEnroll() error {
	params := contract.EnrollOrbitRequest{
		EnrollSecret:   a.EnrollSecret,
		HardwareUUID:   a.UUID,
		HardwareSerial: a.SerialNumber,
		Hostname:       a.CachedString("hostname"),
	}
	jsonBytes, err := json.Marshal(params)
	if err != nil {
		log.Println("orbit json marshall:", err)
		return err
	}

	response := a.waitingDo(func() *http.Request {
		request, err := http.NewRequest("POST", a.serverAddress+"/api/fleet/orbit/enroll", bytes.NewReader(jsonBytes))
		if err != nil {
			panic(err)
		}
		request.Header.Add("Content-type", "application/json")
		return request
	})
	defer response.Body.Close()

	var parsedResp service.EnrollOrbitResponse
	if err := json.NewDecoder(response.Body).Decode(&parsedResp); err != nil {
		log.Println("orbit json parse:", err)
		return err
	}

	a.orbitNodeKey = &parsedResp.OrbitNodeKey
	a.stats.IncrementOrbitEnrollments()
	return nil
}

// This is an osquery enroll as opposed to an orbit enroll
func (a *agent) enroll(i int, onlyAlreadyEnrolled bool) error {
	a.nodeKey = a.nodeKeyManager.Get(i)
	if a.nodeKey != "" {
		a.stats.IncrementEnrollments()
		return nil
	}

	if onlyAlreadyEnrolled {
		return errors.New("not enrolled")
	}

	response := a.waitingDo(func() *http.Request {
		var body bytes.Buffer
		if err := a.templates.ExecuteTemplate(&body, "enroll", a); err != nil {
			panic(err)
		}
		request, err := http.NewRequest("POST", a.serverAddress+"/api/osquery/enroll", &body)
		if err != nil {
			panic(err)
		}
		request.Header.Add("Content-type", "application/json")
		return request
	})
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		log.Println("enroll status:", response.StatusCode)
		return fmt.Errorf("status code: %d", response.StatusCode)
	}

	var parsedResp enrollResponse
	if err := json.NewDecoder(response.Body).Decode(&parsedResp); err != nil {
		log.Println("json parse:", err)
		return err
	}

	a.nodeKey = parsedResp.NodeKey
	a.stats.IncrementEnrollments()

	a.nodeKeyManager.Add(a.nodeKey)

	return nil
}

func (a *agent) config() error {
	request, err := http.NewRequest("POST", a.serverAddress+"/api/osquery/config", bytes.NewReader([]byte(`{"node_key": "`+a.nodeKey+`"}`)))
	if err != nil {
		return err
	}
	request.Header.Add("Content-type", "application/json")

	response, err := http.DefaultClient.Do(a.sign(request))
	if err != nil {
		return fmt.Errorf("config request failed to run: %w", err)
	}
	defer response.Body.Close()

	a.stats.IncrementConfigRequests()

	statusCode := response.StatusCode
	if statusCode != http.StatusOK {
		a.stats.IncrementConfigErrors()
		return fmt.Errorf("config request failed: %d", statusCode)
	}

	parsedResp := struct {
		Packs map[string]struct {
			Queries map[string]interface{} `json:"queries"`
		} `json:"packs"`
	}{}
	if err := json.NewDecoder(response.Body).Decode(&parsedResp); err != nil {
		a.stats.IncrementConfigErrors()
		return fmt.Errorf("json parse at config: %w", err)
	}

	existingLastRunData := make(map[string]int64)

	a.scheduledQueryMapMutex.RLock()
	queryData := a.scheduledQueryData
	a.scheduledQueryMapMutex.RUnlock()
	queryData.Range(func(key, value any) bool {
		existingLastRunData[key.(string)] = value.(scheduledQuery).lastRun

		return true
	})

	newScheduledQueryData := new(sync.Map)

	for packName, pack := range parsedResp.Packs {
		for queryName, query := range pack.Queries {
			m, ok := query.(map[string]interface{})
			if !ok {
				return fmt.Errorf("processing scheduled query failed: %v", query)
			}

			q := scheduledQuery{}
			q.packName = packName
			q.Name = queryName

			// This allows us to set the number of rows returned by the query
			// by appending a number to the query name, e.g. "queryName_10"
			q.numRows = 1
			parts := strings.Split(q.Name, "_")
			if len(parts) == 2 {
				num, err := strconv.ParseInt(parts[1], 10, 32)
				if err != nil {
					num = 1
				}
				q.numRows = uint(num)
			}

			q.ScheduleInterval = m["interval"].(float64)
			q.Query = m["query"].(string)

			scheduledQueryName := packName + "_" + queryName
			if lastRun, ok := existingLastRunData[scheduledQueryName]; ok {
				q.lastRun = lastRun
			}
			newScheduledQueryData.Store(scheduledQueryName, q)
		}
	}

	a.scheduledQueryMapMutex.Lock()
	a.scheduledQueryData = newScheduledQueryData
	a.scheduledQueryMapMutex.Unlock()

	return nil
}

const stringVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_."

func randomString(n int) string {
	sb := strings.Builder{}
	sb.Grow(n)
	for i := 0; i < n; i++ {
		sb.WriteByte(stringVals[rand.Int63()%int64(len(stringVals))])
	}
	return sb.String()
}

// selectSoftwareVersion returns a consistent version for a software package based on its name.
// Uses a per-agent map that assigns each first character to one of 32 version options:
// - 0: base version (99% probability)
// - 1: alternate version (0.9% probability)
// - 2-31: patch versions (0.1% probability, split among 30 different patch numbers)
// This ensures each agent consistently reports the same version for software with the same first letter.
func (a *agent) selectSoftwareVersion(softwareName, baseVersion, alternateVersion string) string {
	if len(softwareName) == 0 {
		return baseVersion
	}

	// Get first character as a rune
	firstChar := rune(softwareName[0])

	// Check if we've already assigned a version option for this character
	versionOption, exists := a.softwareVersionMap[firstChar]
	if !exists {
		// Randomly assign option with distribution matching original randomizeVersion:
		// 99% base (0), 0.9% alternate (1), 0.1% patch versions (2-31)
		r := rand.Float64()
		switch {
		case r < 0.99:
			versionOption = 0 // base version
		case r < 0.999:
			versionOption = 1 // alternate version
		default:
			versionOption = 2 + rand.Intn(30) // patch version: 2-31 maps to patch 0-29
		}
		a.softwareVersionMap[firstChar] = versionOption
	}

	// Return the appropriate version based on the stored option
	switch versionOption {
	case 0:
		return baseVersion
	case 1:
		return alternateVersion
	default: // 2-31
		patchNum := versionOption - 2 // 0-29
		return fmt.Sprintf("%s.%d", baseVersion, patchNum)
	}
}

func (a *agent) CachedString(key string) string {
	if val, ok := a.strings[key]; ok {
		return val
	}
	val := randomString(12)
	a.strings[key] = val
	return val
}

func (a *agent) hostUsers() []map[string]string {
	groupNames := []string{"staff", "nobody", "wheel", "tty", "daemon"}
	shells := []string{"/bin/zsh", "/bin/sh", "/usr/bin/false", "/bin/bash"}
	commonUsers := make([]map[string]string, a.userCount.common)
	for i := 0; i < len(commonUsers); i++ {
		commonUsers[i] = map[string]string{
			"uid":       fmt.Sprint(i),
			"username":  fmt.Sprintf("Common_%d", i),
			"type":      "", // Empty for macOS.
			"groupname": groupNames[i%len(groupNames)],
			"shell":     shells[i%len(shells)],
		}
	}
	uniqueUsers := make([]map[string]string, a.userCount.unique)
	for i := 0; i < len(uniqueUsers); i++ {
		uniqueUsers[i] = map[string]string{
			"uid":       fmt.Sprint(i),
			"username":  fmt.Sprintf("Unique_%d_%d", a.agentIndex, i),
			"type":      "", // Empty for macOS.
			"groupname": groupNames[i%len(groupNames)],
			"shell":     shells[i%len(shells)],
		}
	}
	users := commonUsers
	users = append(users, uniqueUsers...)
	rand.Shuffle(len(users), func(i, j int) {
		users[i], users[j] = users[j], users[i]
	})
	return users
}

func (a *agent) softwareMacOS() []map[string]string {
	var lastOpenedCount int

	totalCommon := a.softwareCount.common
	totalDuplicates := (a.softwareCount.common * a.softwareCount.duplicateBundleIdentifiersPercent) / 100
	totalSoftware := totalCommon + totalDuplicates

	var startIdx, endIdx int

	if a.totalHostCount == 0 {
		// non-distributed mode, all hosts get the same software count
		startIdx = 0
		endIdx = totalSoftware
	} else {
		// distributed mode, distribute software across hosts
		globalAgentIndex := a.hostIndexOffset + (a.agentIndex - 1)

		perHostCount := totalSoftware / a.totalHostCount
		remainder := totalSoftware % a.totalHostCount

		startIdx = globalAgentIndex * perHostCount
		if globalAgentIndex < remainder {
			startIdx += globalAgentIndex
		} else {
			startIdx += remainder
		}

		endIdx = startIdx + perHostCount
		if globalAgentIndex < remainder {
			endIdx++
		}
	}

	commonSoftware := make([]map[string]string, 0)
	duplicateBundleSoftware := make([]map[string]string, 0)
	groupSize := 4

	for i := startIdx; i < endIdx; i++ {
		var lastOpenedAt string
		if l := a.genLastOpenedAt(&lastOpenedCount); l != nil {
			lastOpenedAt = fmt.Sprint(l.Unix())
		}

		if i < totalCommon {
			name := fmt.Sprintf("Common_%d%s", i, a.commonSoftwareNameSuffix)
			commonSoftware = append(commonSoftware, map[string]string{
				"name":              name,
				"version":           a.selectSoftwareVersion(name, "0.0.1", "0.0.2"),
				"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.common_%d", i),
				"source":            "apps",
				"last_opened_at":    lastOpenedAt,
				"installed_path":    fmt.Sprintf("/some/path/Common_%d.app", i),
			})
		} else {
			duplicateIdx := i - totalCommon
			bundleIDIndex := duplicateIdx / groupSize
			bundleID := fmt.Sprintf("com.fleetdm.osquery-perf.common_%d", bundleIDIndex%totalCommon)

			var name string
			if a.softwareCount.softwareRenaming {
				name = fmt.Sprintf("RENAMED_DuplicateBundle_%d", duplicateIdx)
			} else {
				name = fmt.Sprintf("DuplicateBundle_%d", duplicateIdx)
			}

			duplicateBundleSoftware = append(duplicateBundleSoftware, map[string]string{
				"name":              name,
				"version":           fmt.Sprintf("0.0.1%d", duplicateIdx),
				"bundle_identifier": bundleID,
				"source":            "apps",
				"installed_path":    fmt.Sprintf("/some/path/DuplicateBundle_%d.app", duplicateIdx),
			})
		}
	}

	// Unique Software (always per-host, not distributed)
	uniqueSoftware := make([]map[string]string, a.softwareCount.unique)
	for i := 0; i < len(uniqueSoftware); i++ {
		var lastOpenedAt string
		if l := a.genLastOpenedAt(&lastOpenedCount); l != nil {
			lastOpenedAt = l.Format(time.UnixDate)
		}
		name := fmt.Sprintf("Unique_%s_%d", a.CachedString("hostname"), i)
		uniqueSoftware[i] = map[string]string{
			"name":              name,
			"version":           a.selectSoftwareVersion(name, "1.1.1", "1.1.2"),
			"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.unique_%s_%d", a.CachedString("hostname"), i),
			"source":            "apps",
			"last_opened_at":    lastOpenedAt,
			"installed_path":    fmt.Sprintf("/some/path/Unique_%s_%d.app", a.CachedString("hostname"), i),
		}
	}
	if a.softwareCount.uniqueSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.uniqueSoftwareUninstallProb {
		rand.Shuffle(len(uniqueSoftware), func(i, j int) {
			uniqueSoftware[i], uniqueSoftware[j] = uniqueSoftware[j], uniqueSoftware[i]
		})
		uniqueSoftware = uniqueSoftware[:a.softwareCount.unique-a.softwareCount.uniqueSoftwareUninstallCount]
	}

	// Use database software 80% of the time if available; otherwise use legacy vulnerable software.
	var realSoftware []map[string]string
	if softwareDB != nil && len(softwareDB.Darwin) > 0 && rand.Float64() < 0.8 { // nolint:gosec,G404 // load testing, not security-sensitive
		// Initialize cached indices on first call, then mutate on subsequent calls
		if a.cachedSoftwareIndices == nil {
			// Select a random count between min-max, then pick that many random indices
			count := softwaredb.RandomSoftwareCount("darwin")
			perm := rand.Perm(len(softwareDB.Darwin))
			a.cachedSoftwareIndices = make([]uint32, count)
			for i := 0; i < count; i++ {
				a.cachedSoftwareIndices[i] = uint32(perm[i])
			}
		} else {
			a.cachedSoftwareIndices = softwaredb.MaybeMutateSoftware(a.cachedSoftwareIndices, len(softwareDB.Darwin))
		}
		realSoftware = softwareDB.DarwinToMaps(a.cachedSoftwareIndices)
	} else {
		// Vulnerable Software
		var vCount int
		if a.softwareCount.vulnerable < 0 {
			vCount = len(macosVulnerableSoftware)
		} else {
			vCount = a.softwareCount.vulnerable
		}

		realSoftware = make([]map[string]string, 0, vCount)
		randomIndices := rand.Perm(len(macosVulnerableSoftware)) // Randomize software selection
		var softwareLimit int

		switch {
		case a.softwareCount.vulnerable < 0: // Sequential assignment
			softwareLimit = len(macosVulnerableSoftware)
		case a.softwareCount.vulnerable == 0: // No vulnerable software
			softwareLimit = 0
		default: // Random assignment
			softwareLimit = min(a.softwareCount.vulnerable, len(macosVulnerableSoftware)) // Limit to available software
		}

		for i := range softwareLimit {
			var sw fleet.Software

			if a.softwareCount.vulnerable < 0 {
				sw = macosVulnerableSoftware[i]
			} else {
				sw = macosVulnerableSoftware[randomIndices[i]]
			}

			var lastOpenedAt string
			if l := a.genLastOpenedAt(&lastOpenedCount); l != nil {
				lastOpenedAt = l.Format(time.UnixDate)
			}

			realSoftware = append(realSoftware, map[string]string{
				"name":              sw.Name,
				"version":           sw.Version,
				"bundle_identifier": sw.BundleIdentifier,
				"source":            sw.Source,
				"last_opened_at":    lastOpenedAt,
				"installed_path":    fmt.Sprintf("/some/path/%s", sw.Name),
			})
		}
	}

	// Combine all software
	software := commonSoftware
	software = append(software, uniqueSoftware...)
	software = append(software, realSoftware...)
	software = append(software, duplicateBundleSoftware...)
	a.installedSoftware.Range(func(key, value interface{}) bool {
		software = append(software, value.(map[string]string))
		return true
	})
	rand.Shuffle(len(software), func(i, j int) {
		software[i], software[j] = software[j], software[i]
	})

	return software
}

func (a *mdmAgent) softwareIOSandIPadOS(source string) []fleet.Software {
	commonSoftware := make([]map[string]string, a.softwareCount.common)
	for i := 0; i < len(commonSoftware); i++ {
		name := fmt.Sprintf("Common_%d", i)
		commonSoftware[i] = map[string]string{
			"name":              name,
			"version":           a.selectSoftwareVersion(name, "0.0.1", "0.0.2"),
			"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.common_%d", i),
			"source":            source,
		}
	}
	if a.softwareCount.commonSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.commonSoftwareUninstallProb {
		rand.Shuffle(len(commonSoftware), func(i, j int) {
			commonSoftware[i], commonSoftware[j] = commonSoftware[j], commonSoftware[i]
		})
		commonSoftware = commonSoftware[:a.softwareCount.common-a.softwareCount.commonSoftwareUninstallCount]
	}
	uniqueSoftware := make([]map[string]string, a.softwareCount.unique)
	for i := 0; i < len(uniqueSoftware); i++ {
		name := fmt.Sprintf("Unique_%s_%d", a.CachedString("hostname"), i)
		uniqueSoftware[i] = map[string]string{
			"name":              name,
			"version":           a.selectSoftwareVersion(name, "1.1.1", "1.1.2"),
			"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.unique_%s_%d", a.CachedString("hostname"), i),
			"source":            source,
		}
	}
	if a.softwareCount.uniqueSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.uniqueSoftwareUninstallProb {
		rand.Shuffle(len(uniqueSoftware), func(i, j int) {
			uniqueSoftware[i], uniqueSoftware[j] = uniqueSoftware[j], uniqueSoftware[i]
		})
		uniqueSoftware = uniqueSoftware[:a.softwareCount.unique-a.softwareCount.uniqueSoftwareUninstallCount]
	}
	software := commonSoftware
	software = append(software, uniqueSoftware...)
	rand.Shuffle(len(software), func(i, j int) {
		software[i], software[j] = software[j], software[i]
	})
	fleetSoftware := make([]fleet.Software, len(software))
	for i, s := range software {
		fleetSoftware[i] = fleet.Software{
			Name:             s["name"],
			Version:          s["version"],
			BundleIdentifier: s["bundle_identifier"],
			Source:           s["source"],
		}
	}
	return fleetSoftware
}

func (a *agent) softwareVSCodeExtensions() []map[string]string {
	commonVSCodeExtensionsSoftware := make([]map[string]string, a.softwareVSCodeExtensionsCount.common)
	for i := 0; i < len(commonVSCodeExtensionsSoftware); i++ {
		name := fmt.Sprintf("common.extension_%d", i)
		commonVSCodeExtensionsSoftware[i] = map[string]string{
			"name":    name,
			"version": a.selectSoftwareVersion(name, "0.0.1", "0.0.2"),
			"source":  "vscode_extensions",
		}
	}
	if a.softwareVSCodeExtensionsCount.commonSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.commonSoftwareUninstallProb {
		rand.Shuffle(len(commonVSCodeExtensionsSoftware), func(i, j int) {
			commonVSCodeExtensionsSoftware[i], commonVSCodeExtensionsSoftware[j] = commonVSCodeExtensionsSoftware[j], commonVSCodeExtensionsSoftware[i]
		})
		commonVSCodeExtensionsSoftware = commonVSCodeExtensionsSoftware[:a.softwareVSCodeExtensionsCount.common-a.softwareVSCodeExtensionsCount.commonSoftwareUninstallCount]
	}
	uniqueVSCodeExtensionsSoftware := make([]map[string]string, a.softwareVSCodeExtensionsCount.unique)
	for i := 0; i < len(uniqueVSCodeExtensionsSoftware); i++ {
		name := fmt.Sprintf("unique.extension_%s_%d", a.CachedString("hostname"), i)
		uniqueVSCodeExtensionsSoftware[i] = map[string]string{
			"name":    name,
			"version": a.selectSoftwareVersion(name, "1.1.1", "1.1.2"),
			"source":  "vscode_extensions",
		}
	}
	if a.softwareVSCodeExtensionsCount.uniqueSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareVSCodeExtensionsCount.uniqueSoftwareUninstallProb {
		rand.Shuffle(len(uniqueVSCodeExtensionsSoftware), func(i, j int) {
			uniqueVSCodeExtensionsSoftware[i], uniqueVSCodeExtensionsSoftware[j] = uniqueVSCodeExtensionsSoftware[j], uniqueVSCodeExtensionsSoftware[i]
		})
		uniqueVSCodeExtensionsSoftware = uniqueVSCodeExtensionsSoftware[:a.softwareVSCodeExtensionsCount.unique-a.softwareVSCodeExtensionsCount.uniqueSoftwareUninstallCount]
	}
	var vulnerableVSCodeExtensionsSoftware []map[string]string
	for _, vsCodeExtension := range vsCodeExtensionsVulnerableSoftware {
		vulnerableVSCodeExtensionsSoftware = append(vulnerableVSCodeExtensionsSoftware, map[string]string{
			"name":    vsCodeExtension.Name,
			"version": vsCodeExtension.Version,
			"vendor":  vsCodeExtension.Vendor,
			"source":  vsCodeExtension.Source,
		})
	}
	software := commonVSCodeExtensionsSoftware
	software = append(software, uniqueVSCodeExtensionsSoftware...)
	software = append(software, vulnerableVSCodeExtensionsSoftware...)
	rand.Shuffle(len(software), func(i, j int) {
		software[i], software[j] = software[j], software[i]
	})
	return software
}

func selectKernels(kernelList []string) []map[string]string {
	// Determine number of kernels based on probability distribution
	r := rand.Float64()
	var numKernels int
	switch {
	case r < 0.05:
		numKernels = 0 // 5% - rare, fresh install
	case r < 0.45:
		numKernels = 1 // 40% - most common, single current kernel
	case r < 0.75:
		numKernels = 2 // 30% - common, current + previous
	case r < 0.90:
		numKernels = 3 // 15% - a few old kernels
	case r < 0.95:
		numKernels = 4 // 5% - rare
	case r < 0.98:
		numKernels = 5 // 3% - very rare
	default:
		numKernels = 6 // 2% - very rare, many old kernels not cleaned up
	}

	if numKernels == 0 || len(kernelList) == 0 {
		return nil
	}

	// Randomly select unique kernels
	indices := rand.Perm(len(kernelList))
	kernels := make([]map[string]string, 0, numKernels)

	for i := 0; i < numKernels && i < len(indices); i++ {
		kernelName := kernelList[indices[i]]
		// Extract version from name (remove "linux-image-" prefix)
		version := strings.TrimPrefix(kernelName, "linux-image-")

		kernels = append(kernels, map[string]string{
			"name":    kernelName,
			"version": version,
			"source":  "deb_packages",
		})
	}

	return kernels
}

func (a *agent) DistributedRead() (*distributedReadResponse, error) {
	request, err := http.NewRequest("POST", a.serverAddress+"/api/osquery/distributed/read", bytes.NewReader([]byte(`{"node_key": "`+a.nodeKey+`"}`)))
	if err != nil {
		return nil, err
	}
	request.Header.Add("Content-type", "application/json")

	response, err := http.DefaultClient.Do(a.sign(request))
	if err != nil {
		return nil, fmt.Errorf("distributed/read request failed to run: %w", err)
	}
	defer response.Body.Close()

	a.stats.IncrementDistributedReads()

	statusCode := response.StatusCode
	if statusCode != http.StatusOK {
		a.stats.IncrementDistributedReadErrors()
		return nil, fmt.Errorf("distributed/read request failed: %d", statusCode)
	}

	var parsedResp distributedReadResponse
	if err := json.NewDecoder(response.Body).Decode(&parsedResp); err != nil {
		a.stats.IncrementDistributedReadErrors()
		log.Printf("json parse: %s", err)
		return nil, err
	}

	return &parsedResp, nil
}

var defaultQueryResult = []map[string]string{
	{"foo": "bar"},
}

func (a *agent) genLastOpenedAt(count *int) *time.Time {
	if *count >= a.softwareCount.withLastOpened {
		return nil
	}
	*count++
	if rand.Float64() <= a.softwareCount.lastOpenedProb {
		now := time.Now()
		return &now
	}
	return nil
}

func (a *agent) runPolicy(query string) []map[string]string {
	// Used to control the pass or fail of a policy
	// in the UI by setting the query to "select 1"(pass)
	// or "select 0"(fail)
	query = strings.TrimRight(query, ";")
	query = strings.ToLower(query)

	switch query {
	case "select 1":
		return []map[string]string{
			{"1": "1"},
		}
	case "select 0":
		return []map[string]string{}
	}

	if rand.Float64() <= a.policyPassProb {
		return []map[string]string{
			{"1": "1"},
		}
	}
	return []map[string]string{}
}

func (a *agent) randomQueryStats() []map[string]string {
	var stats []map[string]string
	a.scheduledQueryMapMutex.RLock()
	queryData := a.scheduledQueryData
	a.scheduledQueryMapMutex.RUnlock()

	queryData.Range(func(key, value any) bool {
		queryName := key.(string)

		stats = append(stats, map[string]string{
			"name":           queryName,
			"delimiter":      "_",
			"average_memory": fmt.Sprint(rand.Intn(200) + 10),
			"denylisted":     "false",
			"executions":     fmt.Sprint(rand.Intn(100) + 1),
			"interval":       fmt.Sprint(rand.Intn(100) + 1),
			"last_executed":  fmt.Sprint(time.Now().Unix()),
			"output_size":    fmt.Sprint(rand.Intn(100) + 1),
			"system_time":    fmt.Sprint(rand.Intn(4000) + 10),
			"user_time":      fmt.Sprint(rand.Intn(4000) + 10),
			"wall_time":      fmt.Sprint(rand.Intn(4) + 1),
			"wall_time_ms":   fmt.Sprint(rand.Intn(4000) + 10),
		})

		return true
	})
	return stats
}

// mdmMac returns the results for the `mdm` table query.
//
// If the host is enrolled via MDM it will return installed_from_dep as false
// (which means the host will be identified as manually enrolled).
//
// NOTE: To support proper DEP simulation in a loadtest environment
// we may need to implement a mocked Apple DEP endpoint.
func (a *agent) mdmMac() []map[string]string {
	if !a.mdmEnrolled() {
		return []map[string]string{
			{"enrolled": "false", "server_url": "", "installed_from_dep": "false"},
		}
	}
	return []map[string]string{
		{
			"enrolled":           "true",
			"server_url":         a.macMDMClient.EnrollInfo.MDMURL,
			"installed_from_dep": "false",
			"payload_identifier": apple_mdm.FleetPayloadIdentifier,
		},
	}
}

func (a *agent) mdmConfigProfilesMac() []map[string]string {
	return []map[string]string{
		{
			"identifier":   "osquery-perf",
			"display_name": "OSQuery Perf Agent",
			"install_date": "2006-01-02 15:04:05 -0700",
		},
	}
}

func (a *agent) entraConditionalAccess() []map[string]string {
	return []map[string]string{
		{
			"device_id":           a.entraIDDeviceID,
			"user_principal_name": a.entraIDUserPrincipalName,
		},
	}
}

func (a *agent) mdmEnrolled() bool {
	a.isEnrolledToMDMMu.Lock()
	defer a.isEnrolledToMDMMu.Unlock()

	return a.isEnrolledToMDM
}

func (a *agent) setMDMEnrolled() {
	a.isEnrolledToMDMMu.Lock()
	defer a.isEnrolledToMDMMu.Unlock()

	a.isEnrolledToMDM = true
}

func (a *agent) mdmWindows() []map[string]string {
	if !a.mdmEnrolled() {
		return []map[string]string{
			// empty service url means not enrolled
			{"aad_resource_id": "", "discovery_service_url": "", "provider_id": "", "installation_type": "Client"},
		}
	}
	return []map[string]string{
		{
			"aad_resource_id":       "",
			"discovery_service_url": a.serverAddress,
			"provider_id":           fleet.WellKnownMDMFleet,
			"installation_type":     "Client",
		},
	}
}

var munkiIssues = func() []string {
	// generate a list of random munki issues (messages)
	issues := make([]string, 1000)
	for i := range issues {
		// message size: between 60 and 200, with spaces between each 10-char word so
		// that it can still make a bit of sense for UI tests.
		numParts := rand.Intn(15) + 6 // number between 0-14, add 6 to get between 6-20
		var sb strings.Builder
		for j := 0; j < numParts; j++ {
			if j > 0 {
				sb.WriteString(" ")
			}
			sb.WriteString(randomString(10))
		}
		issues[i] = sb.String()
	}
	return issues
}()

func (a *agent) munkiInfo() []map[string]string {
	var errors, warnings []string

	if rand.Float64() <= a.munkiIssueProb {
		for i := 0; i < a.munkiIssueCount; i++ {
			if rand.Intn(2) == 1 {
				errors = append(errors, munkiIssues[rand.Intn(len(munkiIssues))])
			} else {
				warnings = append(warnings, munkiIssues[rand.Intn(len(munkiIssues))])
			}
		}
	}

	errList := strings.Join(errors, ";")
	warnList := strings.Join(warnings, ";")
	return []map[string]string{
		{"version": "1.2.3", "errors": errList, "warnings": warnList},
	}
}

func (a *agent) googleChromeProfiles() []map[string]string {
	count := rand.Intn(5) // return between 0 and 4 emails
	result := make([]map[string]string, count)
	for i := range result {
		email := fmt.Sprintf("user%d@example.com", i)
		if i == len(result)-1 {
			// if the maximum number of emails is returned, set a random domain name
			// so that we have email addresses that match a lot of hosts, and some
			// that match few hosts.
			domainRand := rand.Intn(10)
			email = fmt.Sprintf("user%d@example%d.com", i, domainRand)
		}
		result[i] = map[string]string{"email": email}
	}
	return result
}

func (a *agent) batteries() []map[string]string {
	count := rand.Intn(3) // return between 0 and 2 batteries
	result := make([]map[string]string, count)
	for i := range result {
		max_capacity := 700 + rand.Intn(300) // between 700 and 1000 to ensure most batteries are healthy
		cycleCount := rand.Intn(1200)

		result[i] = map[string]string{
			"serial_number":     fmt.Sprintf("%04d", i),
			"cycle_count":       strconv.Itoa(cycleCount),
			"max_capacity":      strconv.Itoa(max_capacity),
			"designed_capacity": "1000",
		}
	}
	return result
}

func (a *agent) diskSpace() []map[string]string {
	// between 1-100 gigs, between 0-99 percentage available
	gigs := rand.Intn(100)
	gigs++
	pct := rand.Intn(100)
	available := gigs * pct / 100
	return []map[string]string{
		{
			"percent_disk_space_available": strconv.Itoa(pct),
			"gigs_disk_space_available":    strconv.Itoa(available),
			"gigs_total_disk_space":        strconv.Itoa(gigs),
		},
	}
}

func (a *agent) diskEncryption() []map[string]string {
	// 50% of results have encryption enabled
	a.DiskEncryptionEnabled = rand.Intn(2) == 1
	if a.DiskEncryptionEnabled {
		return []map[string]string{{"1": "1"}}
	}
	return []map[string]string{}
}

func (a *agent) diskEncryptionLinux() []map[string]string {
	// 50% of results have encryption enabled
	a.DiskEncryptionEnabled = rand.Intn(2) == 1
	if a.DiskEncryptionEnabled {
		return []map[string]string{
			{"path": "/etc", "encrypted": "0"},
			{"path": "/tmp", "encrypted": "0"},
			{"path": "/", "encrypted": "1"},
		}
	}
	return []map[string]string{
		{"path": "/etc", "encrypted": "0"},
		{"path": "/tmp", "encrypted": "0"},
	}
}

func (a *agent) certificatesDarwin() []map[string]string {
	a.certificatesMutex.RLock()
	cache := a.certificatesCache
	a.certificatesMutex.RUnlock()

	// 90% of the time certificates do not change
	if rand.Intn(100) < 90 && len(cache) > 0 {
		return cache
	}

	// between 2 and 10 certificates (probably impossible to have 0, quick check
	// on dogfood gives between 4-7)
	count := rand.Intn(9) + 2

	sources := []string{"system", "user"}
	users := a.hostUsers()
	const day = 24 * time.Hour

	results := make([]map[string]string, count)
	for i := range count {
		m := make(map[string]string, 12)
		m["ca"] = fmt.Sprint(rand.Intn(2))
		m["common_name"] = uuid.NewString()
		m["issuer"] = fmt.Sprintf("/C=US/O=Issuer %d Inc./CN=Issuer %d Common Name", i, i)
		m["subject"] = fmt.Sprintf("/C=US/O=Subject %d Inc./OU=Subject %d Org Unit/CN=Subject %d Common Name", i, i, i)
		m["key_algorithm"] = "rsaEncryption"
		m["key_strength"] = "2048"
		m["key_usage"] = "Data Encipherment, Key Encipherment, Digital Signature"
		m["serial"] = uuid.NewString()
		m["signing_algorithm"] = "sha256WithRSAEncryption"
		// generate so that it may be expired
		m["not_valid_after"] = fmt.Sprint(time.Now().Add(-1 * day).Add(time.Duration(rand.Intn(100)) * day).Unix())
		// notBefore is always in the past (1-10 days in the past)
		m["not_valid_before"] = fmt.Sprint(time.Now().Add(-time.Duration(rand.Intn(10)+1) * day).Unix())
		rawHash := sha1.Sum([]byte(m["serial"])) //nolint: gosec
		hash := hex.EncodeToString(rawHash[:])
		m["sha1"] = hash
		m["source"] = sources[rand.Intn(2)]

		if m["source"] == "user" {
			// Set username for user keychain certificates
			user := users[rand.Intn(len(users))]
			m["path"] = fmt.Sprintf(`/Users/%s/Library/Keychains/login.keychain-db`, user["username"])
		}

		results[i] = m
	}

	a.certificatesMutex.Lock()
	a.certificatesCache = results
	a.certificatesMutex.Unlock()
	return results
}

func (a *agent) certificatesWindows() []map[string]string {
	a.certificatesMutex.RLock()
	cache := a.certificatesCache
	a.certificatesMutex.RUnlock()

	// 90% of the time certificates do not change
	if rand.Intn(100) < 90 && len(cache) > 0 {
		return cache
	}

	const day = 24 * time.Hour

	// custom SCEP profile ID used for certs issued via custom SCEP profiles (inserted by
	// FLEET_VAR_SCEP_WINDOWS_CERTIFICATE_ID)
	//
	// TODO: make this configurable as a loadtest agent parameter? for now, just hardcode it and try
	// manipulating it in loadtest DB directly if needed.
	profileIDCustomSCEP := "w2a6fd2c4-0018-4bdc-8046-c7342962b576"

	// when windows hosts enroll to Fleet MDM, we issue them a unique cert during the WSTEP/SCEP process
	uuidFleetSCEP := uuid.NewString()

	// uuids that we'll use in serials and hashes to ensure uniqueness
	serial1 := uuid.NewString()
	s1 := sha1.Sum([]byte(serial1)) //nolint: gosec

	serial2 := uuid.NewString()
	s2 := sha1.Sum([]byte(serial2)) //nolint: gosec

	// Fleet SCEP cert example based on data from a real Windows host
	c1 := map[string]string{
		"ca":                "-1",
		"common_name":       uuidFleetSCEP,
		"subject":           "Fleet, " + uuidFleetSCEP,
		"issuer":            "\"\", scep-ca, SCEP CA, FleetDM",
		"key_algorithm":     "RSA",
		"key_strength":      "2160",
		"key_usage":         "CERT_KEY_ENCIPHERMENT_KEY_USAGE,CERT_DIGITAL_SIGNATURE_KEY_USAGE",
		"signing_algorithm": "sha256RSA",
		// generate so that it may be expired
		"not_valid_after": fmt.Sprint(time.Now().Add(-1 * day).Add(time.Duration(rand.Intn(100)) * day).Unix()),
		// notBefore is always in the past (1-10 days in the past)
		"not_valid_before": fmt.Sprint(time.Now().Add(-time.Duration(rand.Intn(10)+1) * day).Unix()),
		"serial":           serial1,
		"sha1":             hex.EncodeToString(s1[:]),
		"username":         "Admin",
		"path":             "Users\\S-1-5-21-1043593016-4249271388-1765263865-1000\\Personal",
	}
	// Custom SCEP cert example based on data from a real Windows host
	c2 := map[string]string{
		"ca":                "-1",
		"common_name":       fmt.Sprintf("%s User\n            CN", profileIDCustomSCEP),
		"subject":           fmt.Sprintf("fleet-%s, \"%s User\n            CN\"", profileIDCustomSCEP, profileIDCustomSCEP),
		"issuer":            "US, scep-ca, SCEP CA, MICROMDM SCEP CA",
		"key_algorithm":     "RSA",
		"key_strength":      "1120",
		"key_usage":         "CERT_DIGITAL_SIGNATURE_KEY_USAGE",
		"signing_algorithm": "sha256RSA",
		// generate so that it may be expired
		"not_valid_after": fmt.Sprint(time.Now().Add(-1 * day).Add(time.Duration(rand.Intn(100)) * day).Unix()),
		// notBefore is always in the past (1-10 days in the past)
		"not_valid_before": fmt.Sprint(time.Now().Add(-time.Duration(rand.Intn(10)+1) * day).Unix()),
		"serial":           serial2,
		"sha1":             hex.EncodeToString(s2[:]),
		"username":         "Admin",
		"path":             "Users\\S-1-5-21-1043593016-4249271388-1765263865-1000\\Personal",
	}

	// We'll use the examples above to create rows with minor variations, similar to what
	// we would get from a real Windows host.
	c3 := maps.Clone(c1)
	c3["username"] = "SYSTEM"
	c3["path"] = "Users\\S-1-5-18\\Personal"

	c4 := maps.Clone(c1)
	c4["username"] = "SYSTEM"
	c4["path"] = "CurrentUser\\Personal"

	c5 := maps.Clone(c1)
	c5["username"] = "SYSTEM"
	c5["path"] = "Users\\S-1-5-18\\Personal"

	c6 := maps.Clone(c1)
	c6["path"] = "Users\\S-1-5-21-1043593016-4249271388-1765263865-1000_Classes\\Personal"

	c7 := maps.Clone(c2)
	c7["path"] = "Users\\S-1-5-21-1043593016-4249271388-1765263865-1000_Classes\\Personal"

	rows := []map[string]string{c1, c2, c3, c4, c5, c6, c7}

	a.certificatesMutex.Lock()
	a.certificatesCache = rows
	a.certificatesMutex.Unlock()
	return rows
}

func (a *agent) orbitInfo() []map[string]string {
	version := "1.22.0"
	desktopVersion := version
	if a.disableFleetDesktop {
		desktopVersion = ""
	}
	deviceAuthToken := ""
	if a.deviceAuthToken != nil {
		deviceAuthToken = *a.deviceAuthToken
	}
	return []map[string]string{
		{
			"version":             version,
			"device_auth_token":   deviceAuthToken,
			"enrolled":            "true",
			"last_recorded_error": "",
			"orbit_channel":       "stable",
			"osqueryd_channel":    "stable",
			"desktop_channel":     "stable",
			"desktop_version":     desktopVersion,
			"uptime":              "10000",
			"scripts_enabled":     "1",
		},
	}
}

func (a *agent) runLiveQuery(query string) (results []map[string]string, status *fleet.OsqueryStatus, message *string, stats *fleet.Stats) {
	if a.liveQueryFailProb > 0.0 && rand.Float64() <= a.liveQueryFailProb {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String("live query failed with error foobar"), nil
	}
	if a.liveQueryNoResultsProb > 0.0 && rand.Float64() <= a.liveQueryNoResultsProb {
		ss := fleet.OsqueryStatus(0)
		return []map[string]string{}, &ss, nil, nil
	}

	// Switch based on contents of the query.
	lcQuery := strings.ToLower(query)
	switch {
	case strings.Contains(lcQuery, "from yara") && strings.Contains(lcQuery, "sigurl"):
		return a.runLiveYaraQuery(query)
	default:
		return a.runLiveMockQuery(query)
	}
}

func (a *agent) runLiveYaraQuery(query string) (results []map[string]string, status *fleet.OsqueryStatus, message *string, stats *fleet.Stats) {
	// Get the URL of the YARA rule to request (i.e. the sigurl).
	urlRegex := regexp.MustCompile(`sigurl=(["'])([^"']*)["']`)
	matches := urlRegex.FindStringSubmatch(query)
	var url string
	if len(matches) > 2 {
		url = matches[2]
	} else {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String("live yara query failed because a valid sigurl could not be found"), nil
	}

	// Osquery validates that the sigurl is one of a configured set, so that it's not
	// sending requests to just anywhere. We'll check that it's at least the same host
	// as the Fleet server.
	if !strings.HasPrefix(url, a.serverAddress) {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String("live yara query failed because sigurl host did not match server address"), nil
	}

	// Make the request.
	body := []byte(`{"node_key": "` + a.nodeKey + `"}`)
	request, err := http.NewRequest("POST", url, bytes.NewReader(body))
	if err != nil {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String("live yara query failed due to error creating request"), nil
	}
	request.Header.Add("Content-type", "application/json")

	// Make the request.
	response, err := http.DefaultClient.Do(a.sign(request))
	if err != nil {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String(fmt.Sprintf("yara request failed to run: %v", err)), nil
	}
	defer response.Body.Close()

	// For load testing purposes we don't actually care about the response, but check that we at least got one.
	if _, err := io.Copy(io.Discard, response.Body); err != nil {
		ss := fleet.OsqueryStatus(1)
		return []map[string]string{}, &ss, ptr.String(fmt.Sprintf("error reading response from yara API: %v", err)), nil
	}

	// Return a response indicating that the file is clean.
	ss := fleet.OsqueryStatus(0)
	return []map[string]string{
			{
				"count":     "0",
				"matches":   "",
				"strings":   "",
				"tags":      "",
				"sig_group": "",
				"sigfile":   "",
				"sigrule":   "",
				"sigurl":    url,
				// Could pull this from the query, but not necessary for load testing.
				"path": "/some/path",
			},
		}, &ss, nil, &fleet.Stats{
			WallTimeMs: uint64(rand.Intn(1000) * 1000),
			UserTime:   uint64(rand.Intn(1000)),
			SystemTime: uint64(rand.Intn(1000)),
			Memory:     uint64(rand.Intn(1000)),
		}
}

func (a *agent) runLiveMockQuery(query string) (results []map[string]string, status *fleet.OsqueryStatus, message *string, stats *fleet.Stats) {
	ss := fleet.OsqueryStatus(0)
	return []map[string]string{
			{
				"admindir":   "/var/lib/dpkg",
				"arch":       "amd64",
				"maintainer": "foobar",
				"name":       "netconf",
				"priority":   "optional",
				"revision":   "",
				"section":    "default",
				"size":       "112594",
				"source":     "",
				"status":     "install ok installed",
				"version":    "20230224000000",
			},
		}, &ss, nil, &fleet.Stats{
			WallTimeMs: uint64(rand.Intn(1000) * 1000),
			UserTime:   uint64(rand.Intn(1000)),
			SystemTime: uint64(rand.Intn(1000)),
			Memory:     uint64(rand.Intn(1000)),
		}
}

func (a *agent) processQuery(name, query string, cachedResults *cachedResults) (
	handled bool, results []map[string]string,
	status *fleet.OsqueryStatus, message *string, stats *fleet.Stats,
) {
	const (
		hostPolicyQueryPrefix = "fleet_policy_query_"
		hostDetailQueryPrefix = "fleet_detail_query_"
		liveQueryPrefix       = "fleet_distributed_query_"
	)
	statusOK := fleet.StatusOK
	statusNotOK := fleet.OsqueryStatus(1)
	results = []map[string]string{} // if a query fails, osquery returns empty results array

	switch {
	case strings.HasPrefix(name, liveQueryPrefix):
		results, status, message, stats = a.runLiveQuery(query)
		return true, results, status, message, stats
	case strings.HasPrefix(name, hostPolicyQueryPrefix):
		return true, a.runPolicy(query), &statusOK, nil, nil
	case name == hostDetailQueryPrefix+"scheduled_query_stats":
		return true, a.randomQueryStats(), &statusOK, nil, nil
	case name == hostDetailQueryPrefix+"mdm":
		ss := statusOK
		if rand.Intn(10) > 0 { // 90% success
			results = a.mdmMac()
		} else {
			ss = statusNotOK
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"mdm_config_profiles_darwin_with_user", name == hostDetailQueryPrefix+"mdm_config_profiles_darwin":
		ss := statusOK
		if rand.Intn(10) > 0 { // 90% success
			results = a.mdmConfigProfilesMac()
		} else {
			ss = statusNotOK
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"conditional_access_microsoft_device_id":
		ss := statusOK
		if rand.Intn(10) > 0 { // 90% success
			results = a.entraConditionalAccess()
		} else {
			ss = statusNotOK
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"mdm_windows":
		ss := statusOK
		if rand.Intn(10) > 0 { // 90% success
			results = a.mdmWindows()
		} else {
			ss = statusNotOK
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"munki_info":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.munkiInfo()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"google_chrome_profiles":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.googleChromeProfiles()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"battery":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.batteries()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"users":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.hostUsers()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_macos":
		ss := fleet.StatusOK
		if a.softwareQueryFailureProb > 0.0 && rand.Float64() <= a.softwareQueryFailureProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			results = a.softwareMacOS()
			cachedResults.software = results
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_macos_codesign":
		// Given queries run in lexicographic order software_macos already run and
		// cachedResults.software should have its results.
		ss := fleet.StatusOK
		if a.softwareQueryFailureProb > 0.0 && rand.Float64() <= a.softwareQueryFailureProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			if len(cachedResults.software) > 0 {
				for _, s := range cachedResults.software {
					if s["source"] != "apps" {
						continue
					}
					installedPath := s["installed_path"]
					teamIdentifier := s["name"] // use name to be fixed (more realistic than changing often).
					if len(teamIdentifier) > 10 {
						teamIdentifier = teamIdentifier[:10]
					}
					cdhashSHA256 := fmt.Sprintf("%x", sha1.Sum([]byte(installedPath))) // cdhash returns 40 characters, matching the length of the sha1 here
					results = append(results, map[string]string{
						"path":            installedPath,
						"team_identifier": teamIdentifier,
						"cdhash_sha256":   cdhashSHA256,
					})
				}
			}
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_macos_executable_sha256":
		ss := fleet.StatusOK
		if a.softwareQueryFailureProb > 0.0 && rand.Float64() <= a.softwareQueryFailureProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			if len(cachedResults.software) > 0 {
				for _, s := range cachedResults.software {
					if s["source"] != "apps" {
						continue
					}
					installedPath := s["installed_path"]
					// Generate mock executable path
					executablePath := installedPath + "/Contents/MacOS/" + strings.TrimSuffix(s["name"], ".app")
					// Generate a mock sha256 hash based on the executable path for consistency
					executableSHA256 := fmt.Sprintf("%x", sha256.Sum256([]byte(executablePath)))
					results = append(results, map[string]string{
						"path":              installedPath,
						"executable_path":   executablePath,
						"executable_sha256": executableSHA256,
					})
				}
			}
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_windows":
		ss := fleet.StatusOK
		if a.softwareQueryFailureProb > 0.0 && rand.Float64() <= a.softwareQueryFailureProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			// Use database software 80% of the time if available, otherwise use embedded data
			if softwareDB != nil && len(softwareDB.Windows) > 0 && rand.Float64() < 0.8 { // nolint:gosec,G404 // load testing, not security-sensitive
				// Initialize cached indices on first call, then mutate on subsequent calls
				if a.cachedSoftwareIndices == nil {
					// Select a random count between min-max, then pick that many random indices
					count := softwaredb.RandomSoftwareCount("windows")
					perm := rand.Perm(len(softwareDB.Windows))
					a.cachedSoftwareIndices = make([]uint32, count)
					for i := 0; i < count; i++ {
						a.cachedSoftwareIndices[i] = uint32(perm[i])
					}
				} else {
					a.cachedSoftwareIndices = softwaredb.MaybeMutateSoftware(a.cachedSoftwareIndices, len(softwareDB.Windows))
				}
				results = softwareDB.WindowsToMaps(a.cachedSoftwareIndices)
			} else {
				results = make([]map[string]string, 0, len(windowsSoftware))
				for _, s := range windowsSoftware {
					// Use consistent version based on software name's first character
					baseVersion := s["version"]
					alternateVersion := baseVersion + ".1"
					m := map[string]string{
						"name":         s["name"],
						"source":       s["source"],
						"version":      a.selectSoftwareVersion(s["name"], baseVersion, alternateVersion),
						"upgrade_code": s["upgrade_code"],
					}
					results = append(results, m)
				}
			}
			a.installedSoftware.Range(func(key, value interface{}) bool {
				results = append(results, value.(map[string]string))
				return true
			})
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_linux":
		ss := fleet.StatusOK
		if a.softwareQueryFailureProb > 0.0 && rand.Float64() <= a.softwareQueryFailureProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			switch a.os { //nolint:gocritic // ignore singleCaseSwitch
			case "ubuntu":
				// Use database software 80% of the time if available, otherwise use embedded data
				if softwareDB != nil && len(softwareDB.Ubuntu) > 0 && rand.Float64() < 0.8 { // nolint:gosec,G404 // load testing, not security-sensitive
					// Initialize cached indices on first call, then mutate on subsequent calls
					if a.cachedSoftwareIndices == nil {
						// Select a random count between min-max, then pick that many random indices
						count := softwaredb.RandomSoftwareCount("ubuntu")
						perm := rand.Perm(len(softwareDB.Ubuntu))
						a.cachedSoftwareIndices = make([]uint32, count)
						for i := 0; i < count; i++ {
							a.cachedSoftwareIndices[i] = uint32(perm[i])
						}
					} else {
						a.cachedSoftwareIndices = softwaredb.MaybeMutateSoftware(a.cachedSoftwareIndices, len(softwareDB.Ubuntu))
					}
					results = softwareDB.UbuntuToMaps(a.cachedSoftwareIndices)
				} else {
					results = make([]map[string]string, 0, len(ubuntuSoftware))
					for _, s := range ubuntuSoftware {
						softwareName := s["name"]
						if a.linuxUniqueSoftwareTitle {
							softwareName = fmt.Sprintf("%s-%d-%s", softwareName, a.agentIndex, linuxRandomBuildNumber)
						}
						var version string
						if a.linuxUniqueSoftwareVersion {
							version = fmt.Sprintf("1.2.%d-%s", a.agentIndex, linuxRandomBuildNumber)
						} else {
							// Use consistent version based on software name's first character
							baseVersion := s["version"]
							alternateVersion := baseVersion + ".1"
							version = a.selectSoftwareVersion(softwareName, baseVersion, alternateVersion)
						}
						m := map[string]string{
							"name":    softwareName,
							"source":  s["source"],
							"version": version,
						}
						results = append(results, m)
					}
				}

				// Add pre-selected kernels for this agent
				results = append(results, a.linuxKernels...)

				a.installedSoftware.Range(func(key, value interface{}) bool {
					results = append(results, value.(map[string]string))
					return true
				})
			}
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"software_vscode_extensions":
		ss := fleet.StatusOK
		if a.softwareVSCodeExtensionsFailProb > 0.0 && rand.Float64() <= a.softwareVSCodeExtensionsFailProb {
			ss = fleet.OsqueryStatus(1)
		}
		if ss == fleet.StatusOK {
			results = a.softwareVSCodeExtensions()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"disk_space_unix" || name == hostDetailQueryPrefix+"disk_space_windows":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.diskSpace()
		}
		return true, results, &ss, nil, nil

	case strings.HasPrefix(name, hostDetailQueryPrefix+"disk_encryption_linux"):
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.diskEncryptionLinux()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"disk_encryption_darwin" ||
		name == hostDetailQueryPrefix+"disk_encryption_windows":
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.diskEncryption()
		}
		return true, results, &ss, nil, nil
	case name == hostDetailQueryPrefix+"kubequery_info" && a.os != "kubequery":
		// Real osquery running on hosts would return no results if it was not
		// running kubequery (due to discovery query). Returning true here so that
		// the caller knows it is handled, will not try to return lorem-ipsum-style
		// results.
		return true, nil, &statusNotOK, nil, nil
	case name == hostDetailQueryPrefix+"orbit_info":
		if a.orbitNodeKey == nil {
			return true, nil, &statusNotOK, nil, nil
		}
		return true, a.orbitInfo(), &statusOK, nil, nil
	case strings.HasPrefix(name, hostDetailQueryPrefix+"certificates_darwin"):
		// NOTE: feels exaggerated to fail osquery 50% of the time but this is how
		// most other osquery queries are handled.
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.certificatesDarwin()
		}
		return true, results, &ss, nil, nil
	case strings.HasPrefix(name, hostDetailQueryPrefix+"certificates_windows"):
		// NOTE: feels exaggerated to fail osquery 50% of the time but this is how
		// most other osquery queries are handled.
		ss := fleet.OsqueryStatus(rand.Intn(2))
		if ss == fleet.StatusOK {
			results = a.certificatesWindows()
		}
		return true, results, &ss, nil, nil
	default:
		// Look for results in the template file.
		if t := a.templates.Lookup(name); t == nil {
			return false, nil, nil, nil, nil
		}
		var ni bytes.Buffer
		err := a.templates.ExecuteTemplate(&ni, name, a)
		if err != nil {
			panic(err)
		}
		err = json.Unmarshal(ni.Bytes(), &results)
		if err != nil {
			panic(err)
		}

		return true, results, &statusOK, nil, nil
	}
}

type cachedResults struct {
	software []map[string]string
}

func (a *agent) DistributedWrite(queries map[string]string) error {
	r := service.SubmitDistributedQueryResultsRequest{
		Results:  make(fleet.OsqueryDistributedQueryResults),
		Statuses: make(map[string]fleet.OsqueryStatus),
		Messages: make(map[string]string),
		Stats:    make(map[string]*fleet.Stats),
	}
	r.NodeKey = a.nodeKey

	cachedResults := cachedResults{}

	// Sort queries to be executed by lexicographic name order (for result processing
	// to be more predictable). This aligns to how osquery executes the queries.
	queryNames := make([]string, 0, len(queries))
	for name := range queries {
		queryNames = append(queryNames, name)
	}
	sort.Strings(queryNames)

	for _, name := range queryNames {
		query := queries[name]

		handled, results, status, message, stats := a.processQuery(name, query, &cachedResults)
		if !handled {
			// If osquery-perf does not handle the incoming query,
			// always return status OK and the default query result.
			r.Results[name] = defaultQueryResult
			r.Statuses[name] = fleet.StatusOK
		} else {
			if results != nil {
				r.Results[name] = results
			}
			if status != nil {
				r.Statuses[name] = *status
			}
			if message != nil {
				r.Messages[name] = *message
			}
			if stats != nil {
				r.Stats[name] = stats
			}
		}
	}
	body, err := json.Marshal(r)
	if err != nil {
		panic(err)
	}

	request, err := http.NewRequest("POST", a.serverAddress+"/api/osquery/distributed/write", bytes.NewReader(body))
	if err != nil {
		return err
	}
	request.Header.Add("Content-type", "application/json")

	response, err := http.DefaultClient.Do(a.sign(request))
	if err != nil {
		return fmt.Errorf("distributed/write request failed to run: %w", err)
	}
	defer response.Body.Close()

	a.stats.IncrementDistributedWrites()

	statusCode := response.StatusCode
	if statusCode != http.StatusOK {
		a.stats.IncrementDistributedWriteErrors()
		return fmt.Errorf("distributed/write request failed: %d", statusCode)
	}

	// No need to read the distributed write body
	return nil
}

func scheduledQueryResults(packName, queryName string, numResults int) []byte {
	return []byte(`{
  "snapshot": [` + rows(numResults) + `
  ],
  "action": "snapshot",
  "name": "pack/` + packName + `/` + queryName + `",
  "hostIdentifier": "EF9595F0-CE81-493A-9B06-D8A9D2CCB952",
  "calendarTime": "Fri Oct  6 18:13:04 2023 UTC",
  "unixTime": 1696615984,
  "epoch": 0,
  "counter": 0,
  "numerics": false,
  "decorations": {
    "host_uuid": "187c4d56-8e45-1a9d-8513-ac17efd2f0fd",
    "hostname": "osquery-perf"
  }
}`)
}

func (a *agent) connCheck() error {
	request, err := http.NewRequest("GET", a.serverAddress+"/version", nil)
	if err != nil {
		panic(err)
	}
	response, err := http.DefaultClient.Do(request)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return errors.New(http.StatusText(response.StatusCode))
	}
	return nil
}

func (a *agent) submitLogs(results []resultLog) error {
	// Connection check to prevent unnecessary JSON marshaling when the server is down.
	if err := a.connCheck(); err != nil {
		return fmt.Errorf("/version check failed: %w", err)
	}

	var resultLogs []byte
	for i, result := range results {
		if i > 0 {
			resultLogs = append(resultLogs, ',')
		}
		resultLogs = append(resultLogs, result.emit()...)
	}

	body := []byte(`{"node_key": "` + a.nodeKey + `", "log_type": "result", "data": [` + string(resultLogs) + `]}`)
	request, err := http.NewRequest("POST", a.serverAddress+"/api/osquery/log", bytes.NewReader(body))
	if err != nil {
		return err
	}
	request.Header.Add("Content-type", "application/json")

	response, err := http.DefaultClient.Do(a.sign(request))
	if err != nil {
		return fmt.Errorf("log request failed to run: %w", err)
	}
	defer response.Body.Close()

	a.stats.IncrementResultLogRequests()

	statusCode := response.StatusCode
	if statusCode != http.StatusOK {
		a.stats.IncrementResultLogErrors()
		return fmt.Errorf("log request failed: %d", statusCode)
	}

	return nil
}

func (a *mdmAgent) runAppleIDeviceMDMLoop(mdmSCEPChallenge string) {
	udid := mdmtest.RandUDID()

	mdmClient := mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{
		SCEPChallenge: mdmSCEPChallenge,
		SCEPURL:       a.serverAddress + apple_mdm.SCEPPath,
		MDMURL:        a.serverAddress + apple_mdm.MDMPath,
	}, a.model)
	mdmClient.UUID = udid
	mdmClient.SerialNumber = mdmtest.RandSerialNumber()
	deviceName := fmt.Sprintf("%s-%d", a.model, a.agentIndex)
	productName := a.model
	softwareSource := "ios_apps"
	if strings.HasPrefix(a.model, "iPad") {
		softwareSource = "ipados_apps"
	}

	if err := mdmClient.Enroll(); err != nil {
		log.Printf("%s MDM enroll failed: %s", a.model, err)
		a.stats.IncrementMDMErrors()
		return
	}

	a.stats.IncrementMDMEnrollments()

	mdmCheckInTicker := time.Tick(a.MDMCheckInInterval)

	for range mdmCheckInTicker {
		mdmCommandPayload, err := mdmClient.Idle()
		if err != nil {
			log.Printf("MDM Idle request failed: %s: %s", a.model, err)
			a.stats.IncrementMDMErrors()
			continue
		}
		a.stats.IncrementMDMSessions()

		for mdmCommandPayload != nil {
			a.stats.IncrementMDMCommandsReceived()
			switch mdmCommandPayload.Command.RequestType {
			case "DeviceInformation":
				mdmCommandPayload, err = mdmClient.AcknowledgeDeviceInformation(udid, mdmCommandPayload.CommandUUID, deviceName,
					productName, "America/Los_Angeles")
			case "InstalledApplicationList":
				software := a.softwareIOSandIPadOS(softwareSource)
				mdmCommandPayload, err = mdmClient.AcknowledgeInstalledApplicationList(udid, mdmCommandPayload.CommandUUID, software)
			case "InstallProfile":
				if a.mdmProfileFailureProb > 0.0 && rand.Float64() <= a.mdmProfileFailureProb {
					errChain := []mdm.ErrorChain{
						{
							ErrorCode:            89,
							ErrorDomain:          "ErrorDomain",
							LocalizedDescription: "The profile did not install",
						},
					}
					mdmCommandPayload, err = mdmClient.Err(mdmCommandPayload.CommandUUID, errChain)
				} else {
					mdmCommandPayload, err = mdmClient.Acknowledge(mdmCommandPayload.CommandUUID)
				}

			default:
				mdmCommandPayload, err = mdmClient.Acknowledge(mdmCommandPayload.CommandUUID)
			}
			if err != nil {
				log.Printf("MDM Acknowledge request failed: %s: %s", a.model, err)
				a.stats.IncrementMDMErrors()
				break
			}
		}
	}
}

// rows returns a set of rows for use in tests for query results.
func rows(num int) string {
	b := strings.Builder{}
	for i := 0; i < num; i++ {
		b.WriteString(`    {
      "build_distro": "centos7",
      "build_platform": "linux",
      "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d",
      "config_valid": "1",
      "extensions": "active",
      "instance_id": "e5799132-85ab-4cfa-89f3-03e0dd3c509a",
      "pid": "3574",
      "platform_mask": "9",
      "start_time": "1696502961",
      "uuid": "EF9595F0-CE81-493A-9B06-D8A9D2CCB95",
      "version": "5.9.2",
      "watcher": "3570"
    }`)
		if i != num-1 {
			b.WriteString(",")
		}
	}

	return b.String()
}

func main() {
	// Start HTTP server for pprof. See https://pkg.go.dev/net/http/pprof.
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	// #nosec (osquery-perf is only used for testing)
	tlsConfig := &tls.Config{
		InsecureSkipVerify: true,
	}
	tr := http.DefaultTransport.(*http.Transport).Clone()
	tr.TLSClientConfig = tlsConfig
	http.DefaultClient.Transport = tr

	validTemplateNames := map[string]bool{
		"macos_13.6.2.tmpl":         true,
		"macos_14.1.2.tmpl":         true,
		"windows_11.tmpl":           true,
		"windows_11_22H2_2861.tmpl": true,
		"windows_11_22H2_3007.tmpl": true,
		"ubuntu_22.04.tmpl":         true,
		"iphone_14.6.tmpl":          true,
		"ipad_13.18.tmpl":           true,
	}
	allowedTemplateNames := make([]string, 0, len(validTemplateNames))
	for k := range validTemplateNames {
		allowedTemplateNames = append(allowedTemplateNames, k)
	}

	var (
		serverURL       = flag.String("server_url", "https://localhost:8080", "URL (with protocol and port of osquery server)")
		enrollSecret    = flag.String("enroll_secret", "", "Enroll secret to authenticate enrollment")
		hostCount       = flag.Int("host_count", 10, "Number of hosts to start (default 10)")
		totalHostCount  = flag.Int("total_host_count", 0, "Total number of hosts across all containers (if 0, uses host_count)")
		hostIndexOffset = flag.Int("host_index_offset", 0, "Starting index offset for this container's hosts (default 0)")
		randSeed        = flag.Int64("seed", time.Now().UnixNano(), "Seed for random generator (default current time)")
		startPeriod     = flag.Duration("start_period", 10*time.Second, "Duration to spread start of hosts over")
		configInterval  = flag.Duration("config_interval", 1*time.Minute, "Interval for config requests")
		// Flag logger_tls_period defines how often to check for sending scheduled query results.
		// osquery-perf will send log requests with results only if there are scheduled queries configured AND it's their time to run.
		logInterval         = flag.Duration("logger_tls_period", 10*time.Second, "Interval for scheduled queries log requests")
		queryInterval       = flag.Duration("query_interval", 10*time.Second, "Interval for distributed query requests")
		mdmCheckInInterval  = flag.Duration("mdm_check_in_interval", 1*time.Minute, "Interval for performing MDM check-ins (applies to both macOS and Windows)")
		onlyAlreadyEnrolled = flag.Bool("only_already_enrolled", false, "Only start agents that are already enrolled")
		nodeKeyFile         = flag.String("node_key_file", "", "File with node keys to use")

		httpMessageSignatureProb     = flag.Float64("http_message_signature_prob", 0.1, "Probability of hosts using HTTP message signatures")
		httpMessageSignatureP384Prob = flag.Float64("http_message_signature_p384_prob", 0.5,
			"Probability of hosts using P384 elliptic curve (as opposed to P256) for HTTP message signatures")

		// 50% failure probability is not realistic but this is our current baseline for the osquery-perf setup.
		// We tried setting this to a more realistic value like 5% but it overloaded the MySQL Writer instance
		// during hosts enroll.
		softwareQueryFailureProb                 = flag.Float64("software_query_fail_prob", 0.5, "Probability of the software query failing")
		softwareVSCodeExtensionsQueryFailureProb = flag.Float64("software_vscode_extensions_query_fail_prob", 0.0, "Probability of the software vscode_extensions query failing")

		softwareInstallerPreInstallFailureProb = flag.Float64("software_installer_pre_install_fail_prob", 0.05,
			"Probability of the pre-install query failing")
		softwareInstallerInstallFailureProb = flag.Float64("software_installer_install_fail_prob", 0.05,
			"Probability of the install script failing")
		softwareInstallerPostInstallFailureProb = flag.Float64("software_installer_post_install_fail_prob", 0.05,
			"Probability of the post-install script failing")

		commonSoftwareCount                          = flag.Int("common_software_count", 10, "Number of common installed applications reported to fleet")
		commonVSCodeExtensionsSoftwareCount          = flag.Int("common_vscode_extensions_software_count", 5, "Number of common vscode_extensions installed applications reported to fleet")
		commonSoftwareUninstallCount                 = flag.Int("common_software_uninstall_count", 1, "Number of common software to uninstall")
		commonVSCodeExtensionsSoftwareUninstallCount = flag.Int("common_vscode_extensions_software_uninstall_count", 1, "Number of common vscode_extensions software to uninstall")
		commonSoftwareUninstallProb                  = flag.Float64("common_software_uninstall_prob", 0.1, "Probability of uninstalling common_software_uninstall_count unique software/s")
		commonVSCodeExtensionsSoftwareUninstallProb  = flag.Float64("common_vscode_extensions_software_uninstall_prob", 0.1, "Probability of uninstalling vscode_extensions common_software_uninstall_count unique software/s")

		uniqueSoftwareCount                          = flag.Int("unique_software_count", 1, "Number of unique software installed on each host")
		uniqueVSCodeExtensionsSoftwareCount          = flag.Int("unique_vscode_extensions_software_count", 1, "Number of unique vscode_extensions software installed on each host")
		uniqueSoftwareUninstallCount                 = flag.Int("unique_software_uninstall_count", 1, "Number of unique software to uninstall")
		uniqueVSCodeExtensionsSoftwareUninstallCount = flag.Int("unique_vscode_extensions_software_uninstall_count", 1, "Number of unique vscode_extensions software to uninstall")
		uniqueSoftwareUninstallProb                  = flag.Float64("unique_software_uninstall_prob", 0.1, "Probability of uninstalling unique_software_uninstall_count common software/s")
		uniqueVSCodeExtensionsSoftwareUninstallProb  = flag.Float64("unique_vscode_extensions_software_uninstall_prob", 0.1, "Probability of uninstalling unique_vscode_extensions_software_uninstall_count common software/s")

		duplicateBundleIdentifiersPercent = flag.Int("duplicate_bundle_identifiers_percent", 0, "Percentage of software with duplicate bundle identifiers (0-100)")
		softwareRenaming                  = flag.Bool("software_renaming", false, "Enable software renaming for duplicate bundle identifiers")
		// WARNING: This will generate massive amounts of entries in the software table,
		// because linux devices report many individual software items, ~1600, compared to Windows around ~100s or macOS around ~500s.
		//
		// This flag can be used to load test software ingestion for Linux during enrollment (during enrollment all devices
		// report software to Fleet, so the initial reads/inserts can be expensive).
		linuxUniqueSoftwareVersion = flag.Bool("linux_unique_software_version", false, "Make version of software items on linux hosts unique. WARNING: This will generate massive amounts of entries in the software table, because linux devices report many individual software items (compared to Windows/macOS).")
		// WARNING: This will generate massive amounts of entries in the software and software_titles tables,
		//
		// This flag can be used to load test software ingestion for Linux during enrollment (during enrollment all devices
		// report software to Fleet, so the initial reads/inserts can be expensive).
		linuxUniqueSoftwareTitle = flag.Bool("linux_unique_software_title", false, "Make name of software items on linux hosts unique. WARNING: This will generate massive amounts of titles which is not realistic but serves to test performance of software ingestion when processing large number of titles.")

		// This flag can be used to set the number of vulnerable software items reported by each host picked randomly from the
		// list of vulnerable software.  Use -1 to load all vulnerable software.
		vulnerableSoftwareCount     = flag.Int("vulnerable_software_count", 10, "Number of vulnerable installed applications reported to fleet.  Use -1 to load all vulnerable software.")
		withLastOpenedSoftwareCount = flag.Int("with_last_opened_software_count", 10, "Number of applications that may report a last opened timestamp to fleet")
		lastOpenedChangeProb        = flag.Float64("last_opened_change_prob", 0.1, "Probability of last opened timestamp to be reported as changed [0, 1]")
		commonUserCount             = flag.Int("common_user_count", 10, "Number of common host users reported to fleet")
		uniqueUserCount             = flag.Int("unique_user_count", 10, "Number of unique host users reported to fleet")
		policyPassProb              = flag.Float64("policy_pass_prob", 1.0, "Probability of policies to pass [0, 1]")
		orbitProb                   = flag.Float64("orbit_prob", 0.5, "Probability of a host being identified as orbit install [0, 1]")
		munkiIssueProb              = flag.Float64("munki_issue_prob", 0.5, "Probability of a host having munki issues (note that ~50% of hosts have munki installed) [0, 1]")
		munkiIssueCount             = flag.Int("munki_issue_count", 10, "Number of munki issues reported by hosts identified to have munki issues")
		// E.g. when running with `-host_count=10`, you can set host count for each template the following way:
		// `-os_templates=windows_11.tmpl:3,macos_14.1.2.tmpl:4,ubuntu_22.04.tmpl:3`
		osTemplates       = flag.String("os_templates", "macos_14.1.2", fmt.Sprintf("Comma-separated list of host OS templates to use and optionally their host count separated by ':' (any of %v, with or without the .tmpl extension)", allowedTemplateNames))
		emptySerialProb   = flag.Float64("empty_serial_prob", 0.1, "Probability of a host having no serial number [0, 1]")
		defaultSerialProb = flag.Float64("default_serial_prob", 0.05,
			"Probability of osquery returning a default (-1) serial number. See: #19789")

		mdmProb               = flag.Float64("mdm_prob", 0.0, "Probability of a host enrolling via Fleet MDM (applies for macOS and Windows hosts, implies orbit enrollment on Windows) [0, 1]")
		mdmSCEPChallenge      = flag.String("mdm_scep_challenge", "", "SCEP challenge to use when running macOS MDM enroll")
		mdmProfileFailureProb = flag.Float64("mdm_profile_failure_prob", 0.0, "Probability of an MDM profile to fail install [0, 1]")

		liveQueryFailProb      = flag.Float64("live_query_fail_prob", 0.0, "Probability of a live query failing execution in the host")
		liveQueryNoResultsProb = flag.Float64("live_query_no_results_prob", 0.2, "Probability of a live query returning no results")

		disableScriptExec = flag.Bool("disable_script_exec", false, "Disable script execution support")

		disableFleetDesktop = flag.Bool("disable_fleet_desktop", false, "Disable Fleet Desktop")
		// logger_tls_max_lines is simulating the osquery setting with the same name.
		loggerTLSMaxLines = flag.Int("logger_tls_max_lines", 1024,
			"Maximum number of buffered result log lines to send on every log request")
		commonSoftwareNameSuffix = flag.String("common_software_name_suffix", "", "Suffix to add to generated common software names")
		softwareDatabasePath     = flag.String("software_db_path", "software-library/software.db",
			"Path to software.db (SQLite database with realistic software data). Auto-generates from software.sql if missing.")
	)

	flag.Parse()
	rand.Seed(*randSeed)

	// Load software from database if path provided
	if *softwareDatabasePath != "" {
		db, err := softwaredb.LoadFromDatabase(*softwareDatabasePath)
		if err != nil {
			log.Fatalf("Failed to load software database: %v", err)
		}
		softwareDB = db
	} else {
		log.Println("No software database specified (--software_db_path). Using embedded software data.")
	}

	// There are two modes for osquery-perf:
	// 1. Non distributed mode (old behavior). All agents get all software specified. This is done when specifying --host_count and --common_software_count
	// Example --host_count 500 --common_software_count 1000 -> means 500 hosts each with 1000 pieces of software
	// 2. Distributed mode. All agents get a subset of the total software specified. This is done when specifying --total_host_count and --host_index_offset along with other params.
	// Example --host_count 500 --common_software_count 1000 --total_host_count 5000 --host_index_offset [0...N...1000]
	// This example means that each container will run 500 hosts, but each host will only get a subset of the total 5000 software requested.
	if *totalHostCount > 0 && *totalHostCount > *hostCount {
		log.Printf("WARNING: total_host_count (%d) > host_count (%d). You are trying to use distributed mode, ensure you have --host_index_offset specified for each container", *totalHostCount, *hostCount)
		log.Printf("         Container 0 should use: --host_index_offset 0")
		log.Printf("         Container 1 should use: --host_index_offset %d", *hostCount)
		log.Printf("         Container 2 should use: --host_index_offset %d", *hostCount*2)
		log.Printf("         Container N should use: --host_index_offset Y")
	}

	if *onlyAlreadyEnrolled {
		// Orbit enrollment does not support the "already enrolled" mode at the
		// moment (see TODO in this file).
		*orbitProb = 0
	}

	if *commonSoftwareUninstallCount > *commonSoftwareCount {
		log.Fatalf("Argument common_software_uninstall_count cannot be bigger than common_software_count")
	}
	if *uniqueSoftwareUninstallCount > *uniqueSoftwareCount {
		log.Fatalf("Argument unique_software_uninstall_count cannot be bigger than unique_software_count")
	}

	tmplsm := make(map[*template.Template]int)
	requestedTemplates := strings.Split(*osTemplates, ",")
	tmplsTotalHostCount := 0
	for _, nm := range requestedTemplates {
		numberOfHosts := 0
		if strings.Contains(nm, ":") {
			parts := strings.Split(nm, ":")
			nm = parts[0]
			hc, err := strconv.ParseInt(parts[1], 10, 64)
			if err != nil {
				log.Fatalf("Invalid template host count: %s", parts[1])
			}
			numberOfHosts = int(hc)
		}
		if !strings.HasSuffix(nm, ".tmpl") {
			nm += ".tmpl"
		}
		if !validTemplateNames[nm] {
			log.Fatalf("Invalid template name: %s (accepted values: %v)", nm, allowedTemplateNames)
		}

		tmpl, err := template.ParseFS(templatesFS, nm)
		if err != nil {
			log.Fatal("parse templates: ", err)
		}
		tmplsm[tmpl] = numberOfHosts
		tmplsTotalHostCount += numberOfHosts
	}
	if tmplsTotalHostCount != 0 && tmplsTotalHostCount != *hostCount {
		log.Fatalf("Invalid host count in templates: total=%d vs host_count=%d", tmplsTotalHostCount, *hostCount)
	}

	// Spread starts over the interval to prevent thundering herd
	sleepTime := *startPeriod / time.Duration(*hostCount)

	stats := &osquery_perf.Stats{
		StartTime: time.Now(),
	}
	go stats.RunLoop()
	installerMetadataCache.Stats = stats

	nodeKeyManager := &nodeKeyManager{}
	if nodeKeyFile != nil {
		nodeKeyManager.filepath = *nodeKeyFile
		nodeKeyManager.LoadKeys()
	}

	var tmplss []*template.Template
	for tmpl := range tmplsm {
		tmplss = append(tmplss, tmpl)
	}

	for i := 0; i < *hostCount; i++ {
		var tmpl *template.Template
		if tmplsTotalHostCount > 0 {
			for tmpl_, hostCount := range tmplsm {
				if hostCount > 0 {
					tmpl = tmpl_
					tmplsm[tmpl_]--
					break
				}
			}
			if tmpl == nil {
				log.Fatalf("Failed to determine template for host: %d", i)
			}
		} else {
			tmpl = tmplss[i%len(tmplss)]
		}

		if tmpl.Name() == "iphone_14.6.tmpl" || tmpl.Name() == "ipad_13.18.tmpl" {
			model := "iPhone 14,6"
			if tmpl.Name() == "ipad_13.18.tmpl" {
				model = "iPad 13,18"
			}
			mobileDevice := mdmAgent{
				agentIndex:         i + 1,
				MDMCheckInInterval: *mdmCheckInInterval,
				model:              model,
				serverAddress:      *serverURL,
				softwareCount: softwareEntityCount{
					entityCount: entityCount{
						common: *commonSoftwareCount,
						unique: *uniqueSoftwareCount,
					},
					vulnerable:                   *vulnerableSoftwareCount,
					commonSoftwareUninstallCount: *commonSoftwareUninstallCount,
					commonSoftwareUninstallProb:  *commonSoftwareUninstallProb,
					uniqueSoftwareUninstallCount: *uniqueSoftwareUninstallCount,
					uniqueSoftwareUninstallProb:  *uniqueSoftwareUninstallProb,
				},
				stats:                 stats,
				strings:               make(map[string]string),
				softwareVersionMap:    make(map[rune]int),
				mdmProfileFailureProb: *mdmProfileFailureProb,
			}
			go mobileDevice.runAppleIDeviceMDMLoop(*mdmSCEPChallenge)
			time.Sleep(sleepTime)
			continue
		}

		a := newAgent(i+1,
			*hostCount,
			*totalHostCount,
			*hostIndexOffset,
			*serverURL,
			*enrollSecret,
			tmpl,
			*configInterval,
			*logInterval,
			*queryInterval,
			*mdmCheckInInterval,
			*softwareQueryFailureProb,
			*softwareVSCodeExtensionsQueryFailureProb,
			softwareInstaller{
				preInstallFailureProb:  *softwareInstallerPreInstallFailureProb,
				installFailureProb:     *softwareInstallerInstallFailureProb,
				postInstallFailureProb: *softwareInstallerPostInstallFailureProb,
				mu:                     new(sync.Mutex),
			},
			softwareEntityCount{
				entityCount: entityCount{
					common: *commonSoftwareCount,
					unique: *uniqueSoftwareCount,
				},
				vulnerable:                        *vulnerableSoftwareCount,
				withLastOpened:                    *withLastOpenedSoftwareCount,
				lastOpenedProb:                    *lastOpenedChangeProb,
				commonSoftwareUninstallCount:      *commonSoftwareUninstallCount,
				commonSoftwareUninstallProb:       *commonSoftwareUninstallProb,
				uniqueSoftwareUninstallCount:      *uniqueSoftwareUninstallCount,
				uniqueSoftwareUninstallProb:       *uniqueSoftwareUninstallProb,
				duplicateBundleIdentifiersPercent: *duplicateBundleIdentifiersPercent,
				softwareRenaming:                  *softwareRenaming,
			},
			softwareExtraEntityCount{
				entityCount: entityCount{
					common: *commonVSCodeExtensionsSoftwareCount,
					unique: *uniqueVSCodeExtensionsSoftwareCount,
				},
				commonSoftwareUninstallCount: *commonVSCodeExtensionsSoftwareUninstallCount,
				commonSoftwareUninstallProb:  *commonVSCodeExtensionsSoftwareUninstallProb,
				uniqueSoftwareUninstallCount: *uniqueVSCodeExtensionsSoftwareUninstallCount,
				uniqueSoftwareUninstallProb:  *uniqueVSCodeExtensionsSoftwareUninstallProb,
			},
			entityCount{
				common: *commonUserCount,
				unique: *uniqueUserCount,
			},
			*policyPassProb,
			*orbitProb,
			*munkiIssueProb,
			*munkiIssueCount,
			*emptySerialProb,
			*defaultSerialProb,
			*mdmProb,
			*mdmSCEPChallenge,
			*liveQueryFailProb,
			*liveQueryNoResultsProb,
			*disableScriptExec,
			*disableFleetDesktop,
			*loggerTLSMaxLines,
			*linuxUniqueSoftwareVersion,
			*linuxUniqueSoftwareTitle,
			*commonSoftwareNameSuffix,
			*mdmProfileFailureProb,
			*httpMessageSignatureProb,
			*httpMessageSignatureP384Prob,
		)
		a.stats = stats
		a.nodeKeyManager = nodeKeyManager
		go a.runLoop(i, *onlyAlreadyEnrolled)
		time.Sleep(sleepTime)
	}

	log.Println("Agents running. Kill with C-c.")
	<-make(chan struct{})
}