fleet/pkg/spec/gitops.go

package spec

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"unicode"

	"github.com/bmatcuk/doublestar/v4"
	"github.com/fleetdm/fleet/v4/server/fleet"
	"github.com/fleetdm/fleet/v4/server/ptr"
	"github.com/ghodss/yaml"
	"github.com/hashicorp/go-multierror"
	"golang.org/x/text/unicode/norm"
)

const LabelAPIGlobalTeamName = "global"

// LabelChangesSummary carries extra context of the labels operations for a config.
type LabelChangesSummary struct {
	LabelsToUpdate  []string
	LabelsToAdd     []string
	LabelsToRemove  []string
	LabelsMovements []LabelMovement
}

func NewLabelChangesSummary(changes []LabelChange, moves []LabelMovement) LabelChangesSummary {
	r := LabelChangesSummary{
		LabelsMovements: moves,
	}

	lookUp := make(map[string]any)
	for _, m := range moves {
		lookUp[m.Name] = nil
	}

	for _, change := range changes {
		if _, ok := lookUp[change.Name]; ok {
			continue
		}
		switch change.Op {
		case "+":

			r.LabelsToAdd = append(r.LabelsToAdd, change.Name)
		case "-":
			r.LabelsToRemove = append(r.LabelsToRemove, change.Name)
		case "=":
			r.LabelsToUpdate = append(r.LabelsToUpdate, change.Name)
		}
	}
	return r
}

// LabelMovement specifies a label movement, a label is moved if its removed from one team and added to another
type LabelMovement struct {
	FromTeamName string // Source team name
	ToTeamName   string // Dest. team name
	Name         string // The globally unique label name
}

// LabelChange used for keeping track of label operations
type LabelChange struct {
	Name     string // The globally unique label name
	Op       string // What operation to perform on the label. +:add, -:remove, =:no-op
	TeamName string // The team this label belongs to.
	FileName string // The filename that contains the label change
}

type ParseTypeError struct {
	Filename string   // The name of the file being parsed
	Keys     []string // The complete path to the field
	Field    string   // The field we tried to assign to
	Type     string   // The type that we want to have
	Value    string   // The type of the value that we received
	err      error    // The original error
}

func (e *ParseTypeError) Error() string {
	var keyPath []string
	keyPath = append(keyPath, e.Keys...)
	if e.Field != "" {
		var clearFields []string
		fields := strings.Split(e.Field, ".")
	fieldcheck:
		for _, field := range fields {
			for _, r := range field {
				// Any field name that contains an upper case letter is probably an embedded struct,
				// remove it from the field path to reduce end user confusion
				if unicode.IsUpper(r) {
					continue fieldcheck
				}
			}
			clearFields = append(clearFields, field)
		}
		keyPath = append(keyPath, strings.Join(clearFields, "."))
	}
	return fmt.Sprintf("Couldn't edit \"%s\" at \"%s\", expected type %s but got %s", e.Filename, strings.Join(keyPath, "."), e.Type, e.Value)
}

func (e *ParseTypeError) Unwrap() error {
	return e.err
}

func MaybeParseTypeError(filename string, keysPath []string, err error) error {
	unmarshallErr := &json.UnmarshalTypeError{}
	if errors.As(err, &unmarshallErr) {
		return &ParseTypeError{
			Filename: filename,
			Keys:     keysPath,
			Field:    unmarshallErr.Field,
			Type:     unmarshallErr.Type.String(),
			Value:    unmarshallErr.Value,
			err:      err,
		}
	}

	return fmt.Errorf("failed to unmarshal file \"%s\" key \"%s\": %w", filename, strings.Join(keysPath, "."), err)
}

// YamlUnmarshal unmarshals YAML bytes into JSON and then into the output struct. We have to do this
// because the yaml package stringifys the JSON parsing error before returning it, so we can't
// extract field information to produce a helpful error for users.
func YamlUnmarshal(yamlBytes []byte, out any) error {
	jsonBytes, err := yaml.YAMLToJSON(yamlBytes)
	if err != nil {
		return fmt.Errorf("failed to unmarshal YAML to JSON: %w", err)
	}
	if err := json.Unmarshal(jsonBytes, out); err != nil {
		return fmt.Errorf("failed to unmarshal JSON bytes: %w", err)
	}

	return nil
}

type BaseItem struct {
	Path  *string `json:"path"`
	Paths *string `json:"paths"`
}

type GitOpsControls struct {
	BaseItem
	MacOSUpdates   any               `json:"macos_updates"`
	IOSUpdates     any               `json:"ios_updates"`
	IPadOSUpdates  any               `json:"ipados_updates"`
	MacOSSettings  any               `json:"macos_settings"`
	MacOSSetup     *fleet.MacOSSetup `json:"macos_setup"`
	MacOSMigration any               `json:"macos_migration"`

	WindowsUpdates                 any `json:"windows_updates"`
	WindowsSettings                any `json:"windows_settings"`
	WindowsEnabledAndConfigured    any `json:"windows_enabled_and_configured"`
	WindowsMigrationEnabled        any `json:"windows_migration_enabled"`
	EnableTurnOnWindowsMDMManually any `json:"enable_turn_on_windows_mdm_manually"`
	WindowsEntraTenantIDs          any `json:"windows_entra_tenant_ids"`

	AndroidEnabledAndConfigured any `json:"android_enabled_and_configured"`
	AndroidSettings             any `json:"android_settings"`

	EnableDiskEncryption any        `json:"enable_disk_encryption"`
	RequireBitLockerPIN  any        `json:"windows_require_bitlocker_pin,omitempty"`
	Scripts              []BaseItem `json:"scripts"`

	Defined bool
}

func (c GitOpsControls) Set() bool {
	return c.MacOSUpdates != nil || c.IOSUpdates != nil ||
		c.IPadOSUpdates != nil || c.MacOSSettings != nil ||
		c.MacOSSetup != nil || c.MacOSMigration != nil ||
		c.WindowsUpdates != nil || c.WindowsSettings != nil || c.WindowsEnabledAndConfigured != nil ||
		c.WindowsMigrationEnabled != nil || c.EnableDiskEncryption != nil || len(c.Scripts) > 0 ||
		c.AndroidEnabledAndConfigured != nil || c.AndroidSettings != nil
}

type Policy struct {
	BaseItem
	GitOpsPolicySpec
}

type GitOpsPolicySpec struct {
	fleet.PolicySpec
	RunScript       *PolicyRunScript       `json:"run_script"`
	InstallSoftware *PolicyInstallSoftware `json:"install_software"`
	// InstallSoftwareURL is populated after parsing the software installer yaml
	// referenced by InstallSoftware.PackagePath.
	InstallSoftwareURL string `json:"-"`
	// RunScriptName is populated after confirming the script exists on both the file system
	// and in the controls scripts list for the same team
	RunScriptName *string `json:"-"`
}

type PolicyRunScript struct {
	Path string `json:"path"`
}

type PolicyInstallSoftware struct {
	PackagePath string `json:"package_path"`
	AppStoreID  string `json:"app_store_id"`
	HashSHA256  string `json:"hash_sha256"`
}

type Query struct {
	BaseItem
	fleet.QuerySpec
}

type Label struct {
	BaseItem
	fleet.LabelSpec
}

type SoftwarePackage struct {
	BaseItem
	fleet.SoftwarePackageSpec
}

func (spec SoftwarePackage) HydrateToPackageLevel(packageLevel fleet.SoftwarePackageSpec) (fleet.SoftwarePackageSpec, error) {
	if spec.Icon.Path != "" || spec.InstallScript.Path != "" || spec.UninstallScript.Path != "" ||
		spec.PostInstallScript.Path != "" || spec.URL != "" || spec.SHA256 != "" || spec.PreInstallQuery.Path != "" {
		return packageLevel, fmt.Errorf("the software package defined in %s must not have icons, scripts, queries, URL, or hash specified at the team level", *spec.Path)
	}

	packageLevel.Categories = spec.Categories
	packageLevel.LabelsIncludeAny = spec.LabelsIncludeAny
	packageLevel.LabelsExcludeAny = spec.LabelsExcludeAny
	packageLevel.InstallDuringSetup = spec.InstallDuringSetup
	packageLevel.SelfService = spec.SelfService

	// This will only override display name set at path: path/to/software.yml level
	// if display_name is specified at the team level yml
	if spec.DisplayName != "" {
		packageLevel.DisplayName = spec.DisplayName
	}

	return packageLevel, nil
}

type Software struct {
	Packages            []SoftwarePackage           `json:"packages"`
	AppStoreApps        []fleet.TeamSpecAppStoreApp `json:"app_store_apps"`
	FleetMaintainedApps []fleet.MaintainedAppSpec   `json:"fleet_maintained_apps"`
}

type GitOps struct {
	TeamID       *uint
	TeamName     *string
	TeamSettings map[string]interface{}
	OrgSettings  map[string]interface{}
	AgentOptions *json.RawMessage
	Controls     GitOpsControls
	Policies     []*GitOpsPolicySpec
	Queries      []*fleet.QuerySpec

	Labels              []*fleet.LabelSpec
	LabelChangesSummary LabelChangesSummary

	// Software is only allowed on teams, not on global config.
	Software GitOpsSoftware
	// FleetSecrets is a map of secret names to their values, extracted from FLEET_SECRET_ environment variables used in profiles and scripts.
	FleetSecrets map[string]string
}

type GitOpsSoftware struct {
	Packages            []*fleet.SoftwarePackageSpec
	AppStoreApps        []*fleet.TeamSpecAppStoreApp
	FleetMaintainedApps []*fleet.MaintainedAppSpec
}

type Logf func(format string, a ...interface{})

// GitOpsFromFile parses a GitOps yaml file.
func GitOpsFromFile(filePath, baseDir string, appConfig *fleet.EnrichedAppConfig, logFn Logf) (*GitOps, error) {
	b, err := os.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file: %s: %w", filePath, err)
	}

	// Replace $var and ${var} with env values.
	b, err = ExpandEnvBytes(b)
	if err != nil {
		return nil, fmt.Errorf("failed to expand environment in file %s: %w", filePath, err)
	}

	// First unmarshal to map[string]any for deprecation handling
	var rawData map[string]any
	if err := yaml.Unmarshal(b, &rawData); err != nil {
		return nil, fmt.Errorf("failed to unmarshal file %w: \n", err)
	}

	// Apply deprecated key mappings (e.g., team_settings -> settings, queries -> reports)
	if err := ApplyDeprecatedKeyMappings(rawData, logFn); err != nil {
		return nil, fmt.Errorf("failed to process deprecated keys in file %s: %w", filePath, err)
	}

	// Re-marshal and unmarshal to map[string]json.RawMessage for existing parsing logic
	updatedBytes, err := json.Marshal(rawData)
	if err != nil {
		return nil, fmt.Errorf("failed to re-marshal file %s: %w", filePath, err)
	}
	var top map[string]json.RawMessage
	if err := json.Unmarshal(updatedBytes, &top); err != nil {
		return nil, fmt.Errorf("failed to unmarshal file %s: %w", filePath, err)
	}

	var multiError *multierror.Error
	result := &GitOps{}
	result.FleetSecrets = make(map[string]string)

	topKeys := []string{"name", "settings", "org_settings", "agent_options", "controls", "policies", "reports", "software", "labels"}
	for k := range top {
		if !slices.Contains(topKeys, k) {
			multiError = multierror.Append(multiError, fmt.Errorf("unknown top-level field: %s", k))
		}
	}

	// Figure out if this is an org or fleet settings file
	teamRaw, teamOk := top["name"]
	settingsRaw, settingsOk := top["settings"]
	orgSettingsRaw, orgOk := top["org_settings"]
	switch {
	case orgOk:
		if teamOk || settingsOk {
			multiError = multierror.Append(multiError, errors.New("'org_settings' cannot be used with 'name', 'settings'"))
		} else {
			multiError = parseOrgSettings(orgSettingsRaw, result, baseDir, filePath, multiError)
		}
	case teamOk:
		multiError = parseName(teamRaw, result, filePath, multiError)
		// If the file is no-team.yml, the name must be "No team".
		switch {
		case filepath.Base(filePath) == "no-team.yml" && !result.IsNoTeam():
			multiError = multierror.Append(multiError, fmt.Errorf("file %q must have team name 'No Team'", filePath))
			return result, multiError.ErrorOrNil()
		case filepath.Base(filePath) == "unassigned.yml" && !result.IsUnassignedTeam():
			multiError = multierror.Append(multiError, fmt.Errorf("file %q must have team name 'Unassigned'", filePath))
			return result, multiError.ErrorOrNil()
		case result.IsNoTeam() && filepath.Base(filePath) != "no-team.yml":
			multiError = multierror.Append(multiError, fmt.Errorf("file `%s` for No Team must be named `no-team.yml`", filePath))
			multiError = multierror.Append(multiError, errors.New("no-team.yml is deprecated; please rename the file to 'unassigned.yml' and update the team name to 'Unassigned'."))
			return result, multiError.ErrorOrNil()
		case result.IsUnassignedTeam() && filepath.Base(filePath) != "unassigned.yml":
			multiError = multierror.Append(multiError, fmt.Errorf("file `%s` for unassigned hosts must be named `unassigned.yml`", filePath))
			return result, multiError.ErrorOrNil()
		case result.IsNoTeam() || result.IsUnassignedTeam():
			// Coerce to "No Team" for easier processing.
			// TODO - Remove No Team in Fleet 5
			result.TeamName = ptr.String(noTeam)
			// For No Team, we allow settings but only process webhook_settings from it
			if settingsOk {
				multiError = parseNoTeamSettings(settingsRaw, result, filePath, multiError)
			}
		default:
			if !settingsOk {
				multiError = multierror.Append(multiError, errors.New("'settings' is required when 'name' is provided"))
			} else {
				multiError = parseTeamSettings(settingsRaw, result, baseDir, filePath, multiError)
			}
		}
	default:
		multiError = multierror.Append(multiError, errors.New("either 'org_settings' or 'name' and 'settings' is required"))
	}

	// Get the labels. If `labels:` is specified but no labels are listed, this will
	// set Labels as nil.  If `labels:` isn't present at all, it will be set as an
	// empty array.
	if _, ok := top["labels"]; !ok {
		result.Labels = make([]*fleet.LabelSpec, 0)
	} else {
		multiError = parseLabels(top, result, baseDir, filePath, multiError)
	}
	// Get other top-level entities.
	multiError = parseControls(top, result, multiError, filePath, logFn)
	multiError = parseAgentOptions(top, result, baseDir, logFn, filePath, multiError)
	multiError = parseReports(top, result, baseDir, logFn, filePath, multiError)

	if appConfig != nil && appConfig.License.IsPremium() {
		multiError = parseSoftware(top, result, baseDir, filePath, multiError)
	}

	// Policies can reference software installers and scripts, thus we parse them after parseSoftware and parseControls.
	multiError = parsePolicies(top, result, baseDir, filePath, multiError)

	return result, multiError.ErrorOrNil()
}

func parseName(raw json.RawMessage, result *GitOps, filePath string, multiError *multierror.Error) *multierror.Error {
	if err := json.Unmarshal(raw, &result.TeamName); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"name"}, err))
	}
	if result.TeamName == nil || *result.TeamName == "" {
		return multierror.Append(multiError, errors.New("team 'name' is required"))
	}
	// Normalize team name for full Unicode support, so that we can assume team names are unique going forward
	normalized := norm.NFC.String(*result.TeamName)
	result.TeamName = &normalized
	return multiError
}

func (g *GitOps) global() bool {
	return g.TeamName == nil || *g.TeamName == ""
}

func (g *GitOps) IsGlobal() bool {
	return g.global()
}

func (g *GitOps) IsNoTeam() bool {
	return g.TeamName != nil && strings.EqualFold(*g.TeamName, noTeam)
}

func (g *GitOps) IsUnassignedTeam() bool {
	return g.TeamName != nil && strings.EqualFold(*g.TeamName, unassignedTeamName)
}

func (g *GitOps) CoercedTeamName() string {
	if g.global() {
		return LabelAPIGlobalTeamName
	}
	return *g.TeamName
}

const (
	noTeam             = "No team"
	unassignedTeamName = "Unassigned"
)

func parseOrgSettings(raw json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
	var orgSettingsTop BaseItem
	if err := json.Unmarshal(raw, &orgSettingsTop); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"org_settings"}, err))
	}
	noError := true
	if orgSettingsTop.Path != nil {
		fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, *orgSettingsTop.Path))
		if err != nil {
			noError = false
			multiError = multierror.Append(multiError, fmt.Errorf("failed to read org settings file %s: %v", *orgSettingsTop.Path, err))
		} else {
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				noError = false
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *orgSettingsTop.Path, err),
				)
			} else {
				var pathOrgSettings BaseItem
				if err := YamlUnmarshal(fileBytes, &pathOrgSettings); err != nil {
					noError = false
					multiError = multierror.Append(
						multiError, MaybeParseTypeError(*orgSettingsTop.Path, []string{"org_settings"}, err),
					)
				} else {
					if pathOrgSettings.Path != nil {
						noError = false
						multiError = multierror.Append(
							multiError,
							fmt.Errorf("nested paths are not supported: %s in %s", *pathOrgSettings.Path, *orgSettingsTop.Path),
						)
					} else {
						raw = fileBytes
					}
				}
			}
		}
	}
	if noError {
		if err := YamlUnmarshal(raw, &result.OrgSettings); err != nil {
			// This error is currently unreachable because we know the file is valid YAML when we checked for nested path
			multiError = multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"org_settings"}, err))
		} else {
			multiError = parseSecrets(result, multiError)
		}
		// TODO: Validate that integrations.(jira|zendesk)[].api_token is not empty or fleet.MaskedPassword
	}
	return multiError
}

func parseTeamSettings(raw json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
	var teamSettingsTop BaseItem
	if err := json.Unmarshal(raw, &teamSettingsTop); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"settings"}, err))
	}
	noError := true
	if teamSettingsTop.Path != nil {
		fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, *teamSettingsTop.Path))
		if err != nil {
			noError = false
			multiError = multierror.Append(multiError, fmt.Errorf("failed to read team settings file %s: %v", *teamSettingsTop.Path, err))
		} else {
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				noError = false
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *teamSettingsTop.Path, err),
				)
			} else {
				var pathTeamSettings BaseItem
				if err := YamlUnmarshal(fileBytes, &pathTeamSettings); err != nil {
					noError = false
					multiError = multierror.Append(
						multiError, MaybeParseTypeError(*teamSettingsTop.Path, []string{"settings"}, err),
					)
				} else {
					if pathTeamSettings.Path != nil {
						noError = false
						multiError = multierror.Append(
							multiError,
							fmt.Errorf("nested paths are not supported: %s in %s", *pathTeamSettings.Path, *teamSettingsTop.Path),
						)
					} else {
						raw = fileBytes
					}
				}
			}
		}
	}
	if noError {
		if err := YamlUnmarshal(raw, &result.TeamSettings); err != nil {
			// This error is currently unreachable because we know the file is valid YAML when we checked for nested path
			multiError = multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"settings"}, err))
		} else {
			multiError = parseSecrets(result, multiError)
			// Validate webhook settings for regular teams
			multiError = validateTeamWebhookSettings(result.TeamSettings, multiError)
		}
	}
	return multiError
}

// validateTeamWebhookSettings validates webhook settings for regular teams
func validateTeamWebhookSettings(teamSettings map[string]any, multiError *multierror.Error) *multierror.Error {
	if webhookSettings, hasWebhook := teamSettings["webhook_settings"]; hasWebhook && webhookSettings != nil {
		webhookMap, ok := webhookSettings.(map[string]any)
		if !ok {
			return multierror.Append(multiError, errors.New("'settings.webhook_settings' must be an object or null"))
		}

		// Validate failing_policies_webhook if present
		if fpw, hasFPW := webhookMap["failing_policies_webhook"]; hasFPW && fpw != nil {
			fpwMap, ok := fpw.(map[string]any)
			if !ok {
				multiError = multierror.Append(multiError, errors.New("'settings.webhook_settings.failing_policies_webhook' must be an object or null"))
			} else {
				// Validate failing_policies_webhook structure
				if err := validateFailingPoliciesWebhook(fpwMap, "settings.webhook_settings.failing_policies_webhook"); err != nil {
					multiError = multierror.Append(multiError, err)
				}
			}
		}

		// Could add validation for other webhook types here in the future
		// e.g., host_status_webhook, vulnerabilities_webhook, etc.
	}
	return multiError
}

// validateFailingPoliciesWebhook validates the failing_policies_webhook configuration.
// It ensures policy_ids is an array if present.
func validateFailingPoliciesWebhook(fpwMap map[string]any, keyPath string) error {
	// Validate policy_ids is an array if present
	if policyIDs, hasPolicyIDs := fpwMap["policy_ids"]; hasPolicyIDs && policyIDs != nil {
		// Check if it's an array
		_, isArray := policyIDs.([]any)
		if !isArray {
			return fmt.Errorf("'%s.policy_ids' must be an array, got %T", keyPath, policyIDs)
		}
	}
	return nil
}

// parseNoTeamSettings parses settings for "No Team" files, but only processes webhook_settings
func parseNoTeamSettings(raw json.RawMessage, result *GitOps, filePath string, multiError *multierror.Error) *multierror.Error {
	// Parse the raw JSON into a map to extract only webhook_settings
	var teamSettingsMap map[string]interface{}
	if err := json.Unmarshal(raw, &teamSettingsMap); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"settings"}, err))
	}

	// For No Team, only webhook_settings is allowed in settings
	// Jira/Zendesk integrations are not supported in gitops: https://github.com/fleetdm/fleet/issues/20287
	// Check for any other keys and error if found
	for key := range teamSettingsMap {
		if key != "webhook_settings" {
			multiError = multierror.Append(multiError,
				fmt.Errorf("unsupported settings option '%s' in %s - only 'webhook_settings' is allowed", key, filepath.Base(filePath)))
		}
	}

	// Initialize TeamSettings if nil
	if result.TeamSettings == nil {
		result.TeamSettings = make(map[string]interface{})
	}

	// For No Team, we only care about webhook_settings
	if webhookRaw, ok := teamSettingsMap["webhook_settings"]; ok {
		// Handle null webhook_settings (which means clear webhook settings)
		if webhookRaw == nil {
			// Store as nil to indicate webhook settings should be cleared
			result.TeamSettings["webhook_settings"] = nil
		} else {
			webhookMap, ok := webhookRaw.(map[string]any)
			if !ok {
				return multierror.Append(multiError, errors.New("'settings.webhook_settings' must be an object or null"))
			}
			for key := range webhookMap {
				if key != "failing_policies_webhook" {
					multiError = multierror.Append(multiError,
						fmt.Errorf("unsupported webhook_settings option '%s' in %s - only 'failing_policies_webhook' is allowed", key, filepath.Base(filePath)))
				}
			}
			// If present, ensure failing_policies_webhook is an object or null
			if fpw, ok := webhookMap["failing_policies_webhook"]; ok && fpw != nil {
				fpwMap, ok := fpw.(map[string]any)
				if !ok {
					multiError = multierror.Append(multiError, errors.New("'settings.webhook_settings.failing_policies_webhook' must be an object or null"))
				} else {
					// Validate failing_policies_webhook structure
					if err := validateFailingPoliciesWebhook(fpwMap, "settings.webhook_settings.failing_policies_webhook"); err != nil {
						multiError = multierror.Append(multiError, err)
					}
				}
			}
			// Store the webhook settings for later processing
			result.TeamSettings["webhook_settings"] = webhookMap
		}
	}

	return multiError
}

func parseSecrets(result *GitOps, multiError *multierror.Error) *multierror.Error {
	var rawSecrets interface{}
	var ok bool
	if result.TeamName == nil {
		rawSecrets, ok = result.OrgSettings["secrets"]
	} else {
		rawSecrets, ok = result.TeamSettings["secrets"]
	}
	if !ok {
		// Allow omitting secrets key, resulting in a no-op for secrets.
		// Any secrets present on the server will be retained.
		return multiError
	}
	// When secrets slice is empty, all secrets are removed.
	enrollSecrets := make([]*fleet.EnrollSecret, 0)
	if rawSecrets != nil {
		secrets, ok := rawSecrets.([]interface{})
		if !ok {
			return multierror.Append(multiError, errors.New("'secrets' must be a list of secret items"))
		}
		for _, enrollSecret := range secrets {
			var secret string
			var secretInterface interface{}
			secretMap, ok := enrollSecret.(map[string]interface{})
			if ok {
				secretInterface, ok = secretMap["secret"]
			}
			if ok {
				secret, ok = secretInterface.(string)
			}
			if !ok || secret == "" {
				multiError = multierror.Append(
					multiError, errors.New("each item in 'secrets' must have a 'secret' key containing an ASCII string value"),
				)
				break
			}
			enrollSecrets = append(
				enrollSecrets, &fleet.EnrollSecret{Secret: secret},
			)
		}
	}
	if result.TeamName == nil {
		result.OrgSettings["secrets"] = enrollSecrets
	} else {
		result.TeamSettings["secrets"] = enrollSecrets
	}
	return multiError
}

func parseAgentOptions(top map[string]json.RawMessage, result *GitOps, baseDir string, logFn Logf, filePath string, multiError *multierror.Error) *multierror.Error {
	agentOptionsRaw, ok := top["agent_options"]
	if result.IsNoTeam() {
		if ok {
			logFn("[!] 'agent_options' is not supported in %s. This key will be ignored.\n", filepath.Base(filePath))
		}
		return multiError
	} else if !ok {
		return multierror.Append(multiError, errors.New("'agent_options' is required"))
	}
	var agentOptionsTop BaseItem
	if err := json.Unmarshal(agentOptionsRaw, &agentOptionsTop); err != nil {
		multiError = multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"agent_options"}, err))
	} else {
		if agentOptionsTop.Path == nil {
			result.AgentOptions = &agentOptionsRaw
		} else {
			fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, *agentOptionsTop.Path))
			if err != nil {
				return multierror.Append(multiError, fmt.Errorf("failed to read agent options file %s: %v", *agentOptionsTop.Path, err))
			}
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *agentOptionsTop.Path, err),
				)
			} else {
				var pathAgentOptions BaseItem
				if err := YamlUnmarshal(fileBytes, &pathAgentOptions); err != nil {
					return multierror.Append(
						multiError, MaybeParseTypeError(*agentOptionsTop.Path, []string{"agent_options"}, err),
					)
				}
				if pathAgentOptions.Path != nil {
					return multierror.Append(
						multiError,
						fmt.Errorf("nested paths are not supported: %s in %s", *pathAgentOptions.Path, *agentOptionsTop.Path),
					)
				}
				var raw json.RawMessage
				if err := YamlUnmarshal(fileBytes, &raw); err != nil {
					// This error is currently unreachable because we know the file is valid YAML when we checked for nested path
					return multierror.Append(
						multiError, MaybeParseTypeError(*agentOptionsTop.Path, []string{"agent_options"}, err),
					)
				}
				result.AgentOptions = &raw
			}
		}
	}
	return multiError
}

func parseControls(top map[string]json.RawMessage, result *GitOps, multiError *multierror.Error, yamlFilename string, logFn Logf) *multierror.Error {
	controlsRaw, ok := top["controls"]
	if !ok {
		// Nothing to do, return.
		return multiError
	}

	var controlsTop GitOpsControls
	if err := json.Unmarshal(controlsRaw, &controlsTop); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(yamlFilename, []string{"controls"}, err))
	}
	controlsTop.Defined = true
	controlsFilePath := yamlFilename
	err := processControlsPathIfNeeded(controlsTop, result, &controlsFilePath)
	if err != nil {
		return multierror.Append(multiError, err)
	}

	controlsDir := filepath.Dir(controlsFilePath)
	var scriptErrs []error
	result.Controls.Scripts, scriptErrs = resolveScriptPaths(result.Controls.Scripts, controlsDir, logFn)
	for _, err := range scriptErrs {
		multiError = multierror.Append(multiError, fmt.Errorf("failed to parse scripts list in %s: %v", controlsFilePath, err))
	}

	// Find Fleet secrets in scripts.
	for _, script := range result.Controls.Scripts {
		fileBytes, err := os.ReadFile(*script.Path)
		if err != nil {
			return multierror.Append(multiError, fmt.Errorf("failed to read scripts file %s: %v", *script.Path, err))
		}
		err = LookupEnvSecrets(string(fileBytes), result.FleetSecrets)
		if err != nil {
			return multierror.Append(multiError, err)
		}
	}

	// Find the Fleet Secrets in the macos setup script file
	if result.Controls.MacOSSetup != nil {
		if result.Controls.MacOSSetup.Script.Set {
			startupScriptPath := resolveApplyRelativePath(controlsDir, result.Controls.MacOSSetup.Script.Value)
			fileBytes, err := os.ReadFile(startupScriptPath)
			if err != nil {
				return multierror.Append(multiError, fmt.Errorf("failed to read macos_setup script file %s: %v", startupScriptPath, err))
			}
			err = LookupEnvSecrets(string(fileBytes), result.FleetSecrets)
			if err != nil {
				return multierror.Append(multiError, err)
			}
		}
	}

	// Find Fleet secrets in profiles
	if result.Controls.MacOSSettings != nil {
		// We are marshalling/unmarshalling to get the data into the fleet.MacOSSettings struct.
		// This is inefficient, but it is more robust and less error-prone.
		var macOSSettings fleet.MacOSSettings
		data, err := json.Marshal(result.Controls.MacOSSettings)
		if err != nil {
			return multierror.Append(multiError, fmt.Errorf("failed to process controls.macos_settings: %v", err))
		}
		err = json.Unmarshal(data, &macOSSettings)
		if err != nil {
			return multierror.Append(multiError, MaybeParseTypeError(controlsFilePath, []string{"controls", "macos_settings"}, err))
		}

		for i := range macOSSettings.CustomSettings {
			err := resolveAndUpdateProfilePathToAbsolute(controlsDir, &macOSSettings.CustomSettings[i], result)
			if err != nil {
				return multierror.Append(multiError, err)
			}
		}
		// Since we already unmarshalled and updated the path, we need to update the result struct.
		result.Controls.MacOSSettings = macOSSettings
	}
	if result.Controls.WindowsSettings != nil {
		// We are marshalling/unmarshalling to get the data into the fleet.WindowsSettings struct.
		// This is inefficient, but it is more robust and less error-prone.
		var windowsSettings fleet.WindowsSettings
		data, err := json.Marshal(result.Controls.WindowsSettings)
		if err != nil {
			return multierror.Append(multiError, fmt.Errorf("failed to process controls.windows_settings: %v", err))
		}
		err = json.Unmarshal(data, &windowsSettings)
		if err != nil {
			return multierror.Append(multiError, MaybeParseTypeError(controlsFilePath, []string{"controls", "windows_settings"}, err))
		}
		if windowsSettings.CustomSettings.Valid {
			for i := range windowsSettings.CustomSettings.Value {
				err := resolveAndUpdateProfilePathToAbsolute(controlsDir, &windowsSettings.CustomSettings.Value[i], result)
				if err != nil {
					return multierror.Append(multiError, err)
				}
			}
		}
		// Since we already unmarshalled and updated the path, we need to update the result struct.
		result.Controls.WindowsSettings = windowsSettings
	}

	if result.Controls.AndroidSettings != nil {
		// We are marshalling/unmarshalling to get the data into the fleet.AndroidSettings struct.
		// This is inefficient, but it is more robust and less error-prone.
		var androidSettings fleet.AndroidSettings
		data, err := json.Marshal(result.Controls.AndroidSettings)
		if err != nil {
			return multierror.Append(multiError, fmt.Errorf("failed to process controls.android_settings: %v", err))
		}
		err = json.Unmarshal(data, &androidSettings)
		if err != nil {
			return multierror.Append(multiError, MaybeParseTypeError(controlsFilePath, []string{"controls", "android_settings"}, err))
		}

		if androidSettings.CustomSettings.Valid {
			for i := range androidSettings.CustomSettings.Value {
				err := resolveAndUpdateProfilePathToAbsolute(controlsDir, &androidSettings.CustomSettings.Value[i], result)
				if err != nil {
					return multierror.Append(multiError, err)
				}
			}
		}

		if androidSettings.Certificates.Valid {
			for i, cert := range androidSettings.Certificates.Value {
				if cert.Name == "" {
					multiError = multierror.Append(multiError, fmt.Errorf("android_settings.certificates[%d]: name is required", i))
				}
				if cert.CertificateAuthorityName == "" {
					multiError = multierror.Append(multiError, fmt.Errorf("android_settings.certificates[%d]: certificate_authority_name is required", i))
				}
				if cert.SubjectName == "" {
					multiError = multierror.Append(multiError, fmt.Errorf("android_settings.certificates[%d]: subject_name is required", i))
				}
			}
		}

		// Since we already unmarshalled and updated the path, we need to update the result struct.
		result.Controls.AndroidSettings = androidSettings
	}

	return multiError
}

func processControlsPathIfNeeded(controlsTop GitOpsControls, result *GitOps, controlsFilePath *string) error {
	if controlsTop.Path == nil {
		result.Controls = controlsTop
		return nil
	}

	// There is a path attribute which points to the real controls section in a separate file, so we need to process that.
	controlsFilePath = ptr.String(resolveApplyRelativePath(filepath.Dir(*controlsFilePath), *controlsTop.Path))
	fileBytes, err := os.ReadFile(*controlsFilePath)
	if err != nil {
		return fmt.Errorf("failed to read controls file %s: %v", *controlsTop.Path, err)
	}

	// Replace $var and ${var} with env values.
	fileBytes, err = ExpandEnvBytes(fileBytes)
	if err != nil {
		return fmt.Errorf("failed to expand environment in file %s: %v", *controlsTop.Path, err)
	}

	var pathControls GitOpsControls
	if err := YamlUnmarshal(fileBytes, &pathControls); err != nil {
		return MaybeParseTypeError(*controlsTop.Path, []string{"controls"}, err)
	}
	if pathControls.Path != nil {
		return fmt.Errorf("nested paths are not supported: %s in %s", *pathControls.Path, *controlsTop.Path)
	}
	pathControls.Defined = true
	result.Controls = pathControls
	return nil
}

func resolveAndUpdateProfilePathToAbsolute(controlsDir string, profile *fleet.MDMProfileSpec, result *GitOps) error {
	resolvedPath := resolveApplyRelativePath(controlsDir, profile.Path)
	// We switch to absolute path so that we don't have to keep track of the base directory.
	// This is useful because controls section can come from either the global config file or the no-team file.
	var err error
	profile.Path, err = filepath.Abs(resolvedPath)
	if err != nil {
		return fmt.Errorf("failed to resolve profile path %s: %v", resolvedPath, err)
	}
	fileBytes, err := os.ReadFile(resolvedPath)
	if err != nil {
		return fmt.Errorf("failed to read profile file %s: %v", resolvedPath, err)
	}
	err = LookupEnvSecrets(string(fileBytes), result.FleetSecrets)
	if err != nil {
		return err
	}
	return nil
}

// defaultAllowedExtensions is the default set of file extensions allowed for
// glob expansion (YAML files). Entity types that need different extensions
// (e.g. scripts) should override this in their GlobExpandOptions.
var defaultAllowedExtensions = map[string]bool{
	".yml":  true,
	".yaml": true,
}

// allowedScriptExtensions is the set of file extensions allowed for scripts.
var allowedScriptExtensions = map[string]bool{
	".sh":  true,
	".ps1": true,
}

// GlobExpandOptions configures how flattenBaseItems expands glob patterns.
type GlobExpandOptions struct {
	// AllowedExtensions filters glob results to only these extensions.
	// Files with other extensions are skipped with a warning.
	// Defaults to {".yml", ".yaml"} if nil.
	AllowedExtensions map[string]bool
	// RequireUniqueBasenames, if true, returns an error when two items resolve to the
	// same filename (filepath.Base).
	RequireUniqueBasenames bool
	// Optional function to log warnings (e.g. about files skipped due to extension mismatch).
	LogFn Logf
}

func (o *GlobExpandOptions) setDefaults() {
	if o.AllowedExtensions == nil {
		o.AllowedExtensions = defaultAllowedExtensions
	}
	if o.LogFn == nil {
		o.LogFn = func(_ string, _ ...any) {}
	}
}

// containsGlobMeta returns true if the string contains glob metacharacters.
func containsGlobMeta(s string) bool {
	return strings.ContainsAny(s, "*?[{")
}

// expandGlobPattern expands a glob pattern relative to baseDir and returns
// all of the matching files with allowed extensions.
func expandGlobPattern(pattern string, baseDir string, entityType string, opts GlobExpandOptions) ([]string, error) {
	absPattern := resolveApplyRelativePath(baseDir, pattern)
	matches, err := doublestar.FilepathGlob(absPattern)
	if err != nil {
		return nil, fmt.Errorf("invalid glob pattern %q: %w", pattern, err)
	}

	var result []string
	for _, match := range matches {
		info, err := os.Stat(match)
		if err != nil {
			return nil, fmt.Errorf("failed to stat %s: %w", match, err)
		}
		if info.IsDir() {
			continue
		}
		ext := strings.ToLower(filepath.Ext(match))
		if !opts.AllowedExtensions[ext] {
			opts.LogFn("[!] glob pattern %q matched non-%s file %q, skipping\n", pattern, entityType, match)
			continue
		}
		result = append(result, match)
	}

	slices.Sort(result)
	return result, nil
}

// flattenBaseItems validates path/paths fields on each item, expands glob
// patterns in "paths" entries, and returns a flat list where every item has only
// Path set (resolved to an absolute path). Errors are collected rather than
// returned early, so callers get all problems in one pass.
func flattenBaseItems(input []BaseItem, baseDir string, entityType string, opts GlobExpandOptions) ([]BaseItem, []error) {
	opts.setDefaults()
	var result []BaseItem
	var errs []error
	seenBasenames := make(map[string]string) // basename -> source (path or pattern)

	for _, item := range input {
		hasPath := item.Path != nil
		hasPaths := item.Paths != nil

		switch {
		case hasPath && hasPaths:
			errs = append(errs, fmt.Errorf(`%s entry cannot have both "path" and "paths" fields`, entityType))
			continue
		// Inline item (no file reference) — pass through unchanged.
		case !hasPath && !hasPaths:
			errs = append(errs, fmt.Errorf(`%s entry has no "path" or "paths" field; check for a stray "-" in the list`, entityType))
			continue
		// Single path -- resolve to absolute path and add to result.
		case hasPath:
			if containsGlobMeta(*item.Path) {
				errs = append(errs, fmt.Errorf(`%s "path" %q contains glob characters; use "paths" for glob patterns`, entityType, *item.Path))
				continue
			}
			resolved := resolveApplyRelativePath(baseDir, *item.Path)
			// Check for duplicate filenames if requested.
			if opts.RequireUniqueBasenames {
				base := filepath.Base(resolved)
				if existing, ok := seenBasenames[base]; ok {
					errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Path))
					continue
				}
				seenBasenames[base] = *item.Path
			}
			result = append(result, BaseItem{Path: &resolved})
		// Glob -- expand and add files to result.
		case hasPaths:
			if !containsGlobMeta(*item.Paths) {
				errs = append(errs, fmt.Errorf(`%s "paths" %q does not contain glob characters; use "path" for a specific file`, entityType, *item.Paths))
				continue
			}
			expanded, err := expandGlobPattern(*item.Paths, baseDir, entityType, opts)
			if err != nil {
				errs = append(errs, err)
				continue
			}
			if len(expanded) == 0 {
				opts.LogFn("[!] glob pattern %q matched no %s files\n", *item.Paths, entityType)
				continue
			}
			for _, p := range expanded {
				// Check for duplicate filenames if requested.
				if opts.RequireUniqueBasenames {
					base := filepath.Base(p)
					if existing, ok := seenBasenames[base]; ok {
						errs = append(errs, fmt.Errorf("duplicate %s basename %q (from %q and %q)", entityType, base, existing, *item.Paths))
						continue
					}
					seenBasenames[base] = *item.Paths
				}
				result = append(result, BaseItem{Path: &p})
			}
		}
	}

	return result, errs
}

func resolveScriptPaths(input []BaseItem, baseDir string, logFn Logf) ([]BaseItem, []error) {
	return flattenBaseItems(input, baseDir, "script", GlobExpandOptions{
		AllowedExtensions:      allowedScriptExtensions,
		RequireUniqueBasenames: true,
		LogFn:                  logFn,
	})
}

func parseLabels(top map[string]json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
	labelsRaw, ok := top["labels"]

	// This shouldn't happen as we check for the property earlier,
	// but better safe than sorry.
	if !ok {
		return multiError
	}

	var labels []Label
	if err := json.Unmarshal(labelsRaw, &labels); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"labels"}, err))
	}
	for _, item := range labels {
		if item.Path == nil {
			result.Labels = append(result.Labels, &item.LabelSpec)
		} else {
			fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, *item.Path))
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to read labels file %s: %v", *item.Path, err))
				continue
			}
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *item.Path, err),
				)
			} else {
				var pathLabels []*Label
				if err := YamlUnmarshal(fileBytes, &pathLabels); err != nil {
					multiError = multierror.Append(multiError, MaybeParseTypeError(*item.Path, []string{"labels"}, err))
					continue
				}
				for _, pq := range pathLabels {
					pq := pq
					if pq != nil {
						if pq.Path != nil {
							multiError = multierror.Append(
								multiError, fmt.Errorf("nested paths are not supported: %s in %s", *pq.Path, *item.Path),
							)
						} else {
							result.Labels = append(result.Labels, &pq.LabelSpec)
						}
					}
				}
			}
		}
	}
	// Make sure team name is correct and do additional validation
	for _, l := range result.Labels {
		if l.Name == "" {
			multiError = multierror.Append(multiError, errors.New("name is required for each label"))
		}

		if l.LabelMembershipType != fleet.LabelMembershipTypeManual && l.Query == "" && l.HostVitalsCriteria == nil {
			multiError = multierror.Append(multiError, errors.New("a SQL query or host vitals criteria is required for each non-manual label"))
		}

		// Manual labels can have empty hosts lists, just make sure we initialize the empty list
		if l.LabelMembershipType == fleet.LabelMembershipTypeManual && l.Hosts == nil {
			l.Hosts = []string{}
		}
		// Don't use non-ASCII
		if !isASCII(l.Name) {
			multiError = multierror.Append(multiError, fmt.Errorf("label name must be in ASCII: %s", l.Name))
		}
		// Check that host vitals criteria is valid
		if l.HostVitalsCriteria != nil {
			criteriaJson, err := json.Marshal(l.HostVitalsCriteria)
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to marshal host vitals criteria for label %s: %v", l.Name, err))
				continue
			}
			label := fleet.Label{
				Name:               l.Name,
				HostVitalsCriteria: ptr.RawMessage(criteriaJson),
			}
			if _, _, err := label.CalculateHostVitalsQuery(); err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("invalid host vitals criteria for label %s: %v", l.Name, err))
			}
		}
	}
	duplicates := getDuplicateNames(
		result.Labels, func(l *fleet.LabelSpec) string {
			return l.Name
		},
	)
	if len(duplicates) > 0 {
		multiError = multierror.Append(multiError, fmt.Errorf("duplicate label names: %v", duplicates))
	}
	return multiError
}

func parsePolicies(top map[string]json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
	parentFilePath := filePath
	policiesRaw, ok := top["policies"]
	if !ok {
		return multierror.Append(multiError, errors.New("'policies' key is required"))
	}
	var policies []Policy
	if err := json.Unmarshal(policiesRaw, &policies); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"policies"}, err))
	}
	for _, item := range policies {
		if item.Path == nil {
			if err := parsePolicyInstallSoftware(baseDir, result.TeamName, &item, result.Software.Packages, result.Software.AppStoreApps); err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to parse policy install_software %q: %v", item.Name, err))
				continue
			}
			if err := parsePolicyRunScript(baseDir, parentFilePath, result.TeamName, &item, result.Controls.Scripts); err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to parse policy run_script %q: %v", item.Name, err))
				continue
			}
			result.Policies = append(result.Policies, &item.GitOpsPolicySpec)
		} else {
			filePath := resolveApplyRelativePath(baseDir, *item.Path)
			fileBytes, err := os.ReadFile(filePath)
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to read policies file %s: %v", *item.Path, err))
				continue
			}
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *item.Path, err),
				)
			} else {
				var pathPolicies []*Policy
				if err := YamlUnmarshal(fileBytes, &pathPolicies); err != nil {
					multiError = multierror.Append(multiError, MaybeParseTypeError(*item.Path, []string{"policies"}, err))
					continue
				}
				for _, pp := range pathPolicies {
					pp := pp
					if pp != nil {
						if pp.Path != nil {
							multiError = multierror.Append(
								multiError, fmt.Errorf("nested paths are not supported: %s in %s", *pp.Path, *item.Path),
							)
						} else {
							if err := parsePolicyInstallSoftware(filepath.Dir(filePath), result.TeamName, pp, result.Software.Packages, result.Software.AppStoreApps); err != nil {
								multiError = multierror.Append(multiError, fmt.Errorf("failed to parse policy install_software %q: %v", pp.Name, err))
								continue
							}
							if err := parsePolicyRunScript(filepath.Dir(filePath), parentFilePath, result.TeamName, pp, result.Controls.Scripts); err != nil {
								multiError = multierror.Append(multiError, fmt.Errorf("failed to parse policy run_script %q: %v", pp.Name, err))
								continue
							}
							result.Policies = append(result.Policies, &pp.GitOpsPolicySpec)
						}
					}
				}
			}
		}
	}
	// Make sure team name is correct, and do additional validation
	for _, item := range result.Policies {
		if item.Name == "" {
			multiError = multierror.Append(multiError, errors.New("policy name is required for each policy"))
		} else {
			item.Name = norm.NFC.String(item.Name)
		}
		if item.Query == "" {
			multiError = multierror.Append(multiError, errors.New("policy query is required for each policy"))
		}
		if result.TeamName != nil {
			item.Team = *result.TeamName
		} else {
			item.Team = ""
		}
		if item.CalendarEventsEnabled && result.IsNoTeam() {
			multiError = multierror.Append(multiError, fmt.Errorf("calendar events are not supported on policies included in `%s`: %q", filepath.Base(parentFilePath), item.Name))
		}
	}
	duplicates := getDuplicateNames(
		result.Policies, func(p *GitOpsPolicySpec) string {
			return p.Name
		},
	)
	if len(duplicates) > 0 {
		multiError = multierror.Append(multiError, fmt.Errorf("duplicate policy names: %v", duplicates))
	}
	return multiError
}

func parsePolicyRunScript(baseDir string, parentFilePath string, teamName *string, policy *Policy, scripts []BaseItem) error {
	if policy.RunScript == nil {
		policy.ScriptID = ptr.Uint(0) // unset the script
		return nil
	}
	if policy.RunScript != nil && policy.RunScript.Path != "" && teamName == nil {
		return errors.New("run_script can only be set on team policies")
	}

	if policy.RunScript.Path == "" {
		return errors.New("empty run_script path")
	}

	scriptPath := resolveApplyRelativePath(baseDir, policy.RunScript.Path)
	_, err := os.Stat(scriptPath)
	if err != nil {
		return fmt.Errorf("script file does not exist %q: %v", policy.RunScript.Path, err)
	}

	scriptOnTeamFound := false
	for _, script := range scripts {
		if scriptPath == *script.Path {
			scriptOnTeamFound = true
			break
		}
	}
	if !scriptOnTeamFound {
		if *teamName == noTeam {
			return fmt.Errorf("policy script %s was not defined in controls in %s", scriptPath, filepath.Base(parentFilePath))
		}
		return fmt.Errorf("policy script %s was not defined in controls for %s", scriptPath, *teamName)
	}

	scriptName := filepath.Base(policy.RunScript.Path)
	policy.RunScriptName = &scriptName

	return nil
}

func parsePolicyInstallSoftware(baseDir string, teamName *string, policy *Policy, packages []*fleet.SoftwarePackageSpec, appStoreApps []*fleet.TeamSpecAppStoreApp) error {
	if policy.InstallSoftware == nil {
		policy.SoftwareTitleID = ptr.Uint(0) // unset the installer
		return nil
	}
	if policy.InstallSoftware != nil && (policy.InstallSoftware.PackagePath != "" || policy.InstallSoftware.AppStoreID != "") && teamName == nil {
		return errors.New("install_software can only be set on team policies")
	}
	if policy.InstallSoftware.PackagePath == "" && policy.InstallSoftware.AppStoreID == "" && policy.InstallSoftware.HashSHA256 == "" {
		return errors.New("install_software must include either a package_path, an app_store_id or a hash_sha256")
	}
	if policy.InstallSoftware.PackagePath != "" && policy.InstallSoftware.AppStoreID != "" {
		return errors.New("install_software must have only one of package_path or app_store_id")
	}

	if policy.InstallSoftware.PackagePath != "" {
		fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, policy.InstallSoftware.PackagePath))
		if err != nil {
			return fmt.Errorf("failed to read install_software.package_path file %q: %v", policy.InstallSoftware.PackagePath, err)
		}
		var policyInstallSoftwareSpec fleet.SoftwarePackageSpec
		if err := YamlUnmarshal(fileBytes, &policyInstallSoftwareSpec); err != nil {
			// see if the issue is that a package path was passed in that references multiple packages
			var multiplePackages []fleet.SoftwarePackageSpec
			if err := YamlUnmarshal(fileBytes, &multiplePackages); err != nil || len(multiplePackages) == 0 {
				return fmt.Errorf("file %q does not contain a valid software package definition", policy.InstallSoftware.PackagePath)
			}

			if len(multiplePackages) > 1 {
				return fmt.Errorf("file %q contains multiple packages, so cannot be used as a target for policy automation", policy.InstallSoftware.PackagePath)
			}

			policyInstallSoftwareSpec = multiplePackages[0]
		}
		installerOnTeamFound := false
		for _, pkg := range packages {
			if (pkg.URL != "" && pkg.URL == policyInstallSoftwareSpec.URL) || (pkg.SHA256 != "" && pkg.SHA256 == policyInstallSoftwareSpec.SHA256) {
				installerOnTeamFound = true
				break
			}
		}
		if !installerOnTeamFound {
			if policyInstallSoftwareSpec.URL != "" {
				return fmt.Errorf("install_software.package_path URL %s not found on team: %s", policyInstallSoftwareSpec.URL, policy.InstallSoftware.PackagePath)
			}
			return fmt.Errorf("install_software.package_path SHA256 %s not found on team: %s", policyInstallSoftwareSpec.SHA256, policy.InstallSoftware.PackagePath)
		}

		policy.InstallSoftwareURL = policyInstallSoftwareSpec.URL
		policy.InstallSoftware.HashSHA256 = policyInstallSoftwareSpec.SHA256
	}

	if policy.InstallSoftware.AppStoreID != "" {
		appOnTeamFound := false
		for _, app := range appStoreApps {
			if app.AppStoreID == policy.InstallSoftware.AppStoreID {
				appOnTeamFound = true
				break
			}
		}
		if !appOnTeamFound {
			return fmt.Errorf("install_software.app_store_id %s not found on team %s", policy.InstallSoftware.AppStoreID, *teamName)
		}
	}

	return nil
}

func validateReport(r *fleet.QuerySpec, filePath string, itemPath string) error {
	if r.Name == "" {
		return fmt.Errorf("`name` is required for each report in %s at %s", filepath.Base(filePath), itemPath)
	}
	if r.Query == "" {
		return fmt.Errorf("`query` is required for each report in %s at %s", filepath.Base(filePath), itemPath)
	}
	if !isASCII(r.Name) {
		return fmt.Errorf("`name` must be in ASCII: %s in %s at %s", r.Name, filepath.Base(filePath), itemPath)
	}
	return nil
}

func parseReports(top map[string]json.RawMessage, result *GitOps, baseDir string, logFn Logf, filePath string, multiError *multierror.Error) *multierror.Error {
	reportsRaw, ok := top["reports"]
	if result.IsNoTeam() {
		if ok {
			logFn("[!] 'reports' is not supported in %s. This key will be ignored.\n", filepath.Base(filePath))
		}
		return multiError
	} else if !ok {
		return multierror.Append(multiError, errors.New("'reports' key is required"))
	}
	var queries []Query
	if err := json.Unmarshal(reportsRaw, &queries); err != nil {
		return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"reports"}, err))
	}
	for i, item := range queries {
		if item.Path == nil {
			if err := validateReport(&item.QuerySpec, filePath, fmt.Sprintf("reports[%d]", i)); err != nil {
				multiError = multierror.Append(multiError, err)
				continue
			}
			item.QuerySpec.TeamName = ptr.ValOrZero(result.TeamName)
			result.Queries = append(result.Queries, &item.QuerySpec)
		} else {
			fileBytes, err := os.ReadFile(resolveApplyRelativePath(baseDir, *item.Path))
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to read reports file %s: %v", *item.Path, err))
				continue
			}
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				multiError = multierror.Append(
					multiError, fmt.Errorf("failed to expand environment in file %s: %v", *item.Path, err),
				)
			} else {
				var pathQueries []*Query
				if err := YamlUnmarshal(fileBytes, &pathQueries); err != nil {
					multiError = multierror.Append(multiError, MaybeParseTypeError(*item.Path, []string{"reports"}, err))
					continue
				}
				for i, pq := range pathQueries {
					if pq != nil {
						if pq.Path != nil {
							multiError = multierror.Append(
								multiError, fmt.Errorf("nested paths are not supported: %s in %s", *pq.Path, *item.Path),
							)
						} else {
							if err := validateReport(&pq.QuerySpec, *item.Path, fmt.Sprintf("reports[%d]", i)); err != nil {
								multiError = multierror.Append(multiError, err)
								continue
							}
							pq.QuerySpec.TeamName = ptr.ValOrZero(result.TeamName)
							result.Queries = append(result.Queries, &pq.QuerySpec)
						}
					}
				}
			}
		}
	}
	duplicates := getDuplicateNames(
		result.Queries, func(q *fleet.QuerySpec) string {
			return q.Name
		},
	)
	if len(duplicates) > 0 {
		multiError = multierror.Append(multiError, fmt.Errorf("duplicate report names: %v", duplicates))
	}
	return multiError
}

var validSHA256Value = regexp.MustCompile(`\b[a-f0-9]{64}\b`)

func parseSoftware(top map[string]json.RawMessage, result *GitOps, baseDir string, filePath string, multiError *multierror.Error) *multierror.Error {
	softwareRaw, ok := top["software"]
	if result.global() {
		if ok && string(softwareRaw) != "null" {
			return multierror.Append(multiError, errors.New("'software' cannot be set on global file"))
		}
	} else if !ok {
		return multierror.Append(multiError, errors.New("'software' is required"))
	}
	var software Software
	if len(softwareRaw) > 0 {
		if err := json.Unmarshal(softwareRaw, &software); err != nil {
			return multierror.Append(multiError, MaybeParseTypeError(filePath, []string{"software"}, err))
		}
	}
	for _, item := range software.AppStoreApps {
		if item.AppStoreID == "" {
			multiError = multierror.Append(multiError, errors.New("software app store id required"))
			continue
		}

		if len(item.LabelsExcludeAny) > 0 && len(item.LabelsIncludeAny) > 0 {
			multiError = multierror.Append(multiError, fmt.Errorf(`only one of "labels_exclude_any" or "labels_include_any" can be specified for app store app %q`, item.AppStoreID))
			continue
		}

		// Validate display_name length (matches database VARCHAR(255))
		if len(item.DisplayName) > 255 {
			multiError = multierror.Append(multiError, fmt.Errorf("app_store_id %q display_name is too long (max 255 characters)", item.AppStoreID))
			continue
		}

		item = item.ResolvePaths(baseDir)

		result.Software.AppStoreApps = append(result.Software.AppStoreApps, &item)
	}
	for _, maintainedAppSpec := range software.FleetMaintainedApps {
		if maintainedAppSpec.Slug == "" {
			multiError = multierror.Append(multiError, errors.New("fleet maintained app slug is required"))
			continue
		}

		if len(maintainedAppSpec.LabelsExcludeAny) > 0 && len(maintainedAppSpec.LabelsIncludeAny) > 0 {
			multiError = multierror.Append(multiError, fmt.Errorf(`only one of "labels_exclude_any" or "labels_include_any" can be specified for fleet maintained app %q`, maintainedAppSpec.Slug))
			continue
		}

		maintainedAppSpec = maintainedAppSpec.ResolveSoftwarePackagePaths(baseDir)

		// handle secrets
		if maintainedAppSpec.InstallScript.Path != "" {
			if err := gatherFileSecrets(result, maintainedAppSpec.InstallScript.Path); err != nil {
				multiError = multierror.Append(multiError, err)
				continue
			}
		}
		if maintainedAppSpec.PostInstallScript.Path != "" {
			if err := gatherFileSecrets(result, maintainedAppSpec.PostInstallScript.Path); err != nil {
				multiError = multierror.Append(multiError, err)
				continue
			}
		}
		if maintainedAppSpec.UninstallScript.Path != "" {
			if err := gatherFileSecrets(result, maintainedAppSpec.UninstallScript.Path); err != nil {
				multiError = multierror.Append(multiError, err)
				continue
			}
		}

		result.Software.FleetMaintainedApps = append(result.Software.FleetMaintainedApps, &maintainedAppSpec)
	}
	for _, teamLevelPackage := range software.Packages {
		// A single item in Packages can result in multiple SoftwarePackageSpecs being generated
		var softwarePackageSpecs []*fleet.SoftwarePackageSpec
		if teamLevelPackage.Path != nil {
			yamlPath := resolveApplyRelativePath(baseDir, *teamLevelPackage.Path)
			fileBytes, err := os.ReadFile(yamlPath)
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to read software package file %s: %w", *teamLevelPackage.Path, err))
				continue
			}
			// Replace $var and ${var} with env values.
			fileBytes, err = ExpandEnvBytes(fileBytes)
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("failed to expand environment in file %s: %w", *teamLevelPackage.Path, err))
				continue
			}
			var singlePackageSpec SoftwarePackage
			singlePackageSpec.ReferencedYamlPath = yamlPath
			if err := YamlUnmarshal(fileBytes, &singlePackageSpec); err == nil {
				if singlePackageSpec.IncludesFieldsDisallowedInPackageFile() {
					multiError = multierror.Append(multiError, fmt.Errorf("labels, categories, setup_experience, and self_service values must be specified at the team level; package-level specified in %s", *teamLevelPackage.Path))
					continue
				}
				softwarePackageSpecs = append(softwarePackageSpecs, &singlePackageSpec.SoftwarePackageSpec)
			} else if err = YamlUnmarshal(fileBytes, &softwarePackageSpecs); err == nil {
				// Failing that, try to unmarshal as a list of SoftwarePackageSpecs
				for i, spec := range softwarePackageSpecs {
					if spec.IncludesFieldsDisallowedInPackageFile() {
						multiError = multierror.Append(multiError, fmt.Errorf("labels, categories, setup_experience, and self_service values must be specified at the team level; package-level specified in %s", *teamLevelPackage.Path))
						continue
					}

					softwarePackageSpecs[i].ReferencedYamlPath = yamlPath
				}
			} else {
				// If we reached here, we couldn't unmarshal as either format.
				multiError = multierror.Append(multiError, MaybeParseTypeError(*teamLevelPackage.Path, []string{"software", "packages"}, err))
				continue
			}

			for i, spec := range softwarePackageSpecs {
				softwarePackageSpec := spec.ResolveSoftwarePackagePaths(filepath.Dir(spec.ReferencedYamlPath))
				softwarePackageSpec, err = teamLevelPackage.HydrateToPackageLevel(softwarePackageSpec)
				if err != nil {
					multiError = multierror.Append(multiError, err)
					continue
				}
				softwarePackageSpecs[i] = &softwarePackageSpec
			}
		} else {
			softwarePackageSpec := teamLevelPackage.SoftwarePackageSpec.ResolveSoftwarePackagePaths(baseDir)
			softwarePackageSpecs = append(softwarePackageSpecs, &softwarePackageSpec)
		}

		for i, softwarePackageSpec := range softwarePackageSpecs {
			if softwarePackageSpec.InstallScript.Path != "" {
				if err := gatherFileSecrets(result, softwarePackageSpec.InstallScript.Path); err != nil {
					multiError = multierror.Append(multiError, err)
					continue
				}
			}
			if softwarePackageSpec.PostInstallScript.Path != "" {
				if err := gatherFileSecrets(result, softwarePackageSpec.PostInstallScript.Path); err != nil {
					multiError = multierror.Append(multiError, err)
					continue
				}
			}
			if softwarePackageSpec.UninstallScript.Path != "" {
				if err := gatherFileSecrets(result, softwarePackageSpec.UninstallScript.Path); err != nil {
					multiError = multierror.Append(multiError, err)
					continue
				}
			}
			if len(softwarePackageSpec.LabelsExcludeAny) > 0 && len(softwarePackageSpec.LabelsIncludeAny) > 0 {
				multiError = multierror.Append(multiError, fmt.Errorf(`only one of "labels_exclude_any" or "labels_include_any" can be specified for software URL %q`, softwarePackageSpec.URL))
				continue
			}
			if softwarePackageSpec.SHA256 != "" && !validSHA256Value.MatchString(softwarePackageSpec.SHA256) {
				multiError = multierror.Append(multiError, fmt.Errorf("hash_sha256 value %q must be a valid lower-case hex-encoded (64-character) SHA-256 hash value", softwarePackageSpec.SHA256))
				continue
			}
			if softwarePackageSpec.SHA256 == "" && softwarePackageSpec.URL == "" {
				errorMessage := "at least one of hash_sha256 or url is required for each software package"
				if softwarePackageSpec.ReferencedYamlPath != "" {
					errorMessage += fmt.Sprintf("; missing in %s", softwarePackageSpec.ReferencedYamlPath)
				}
				if len(softwarePackageSpecs) > 1 {
					errorMessage += fmt.Sprintf(", list item #%d", i+1)
				}

				multiError = multierror.Append(multiError, errors.New(errorMessage))
				continue
			}
			if len(softwarePackageSpec.URL) > fleet.SoftwareInstallerURLMaxLength {
				multiError = multierror.Append(multiError, fmt.Errorf("software URL %q is too long, must be %d characters or less", softwarePackageSpec.URL, fleet.SoftwareInstallerURLMaxLength))
				continue
			}
			parsedUrl, err := url.Parse(softwarePackageSpec.URL)
			if err != nil {
				multiError = multierror.Append(multiError, fmt.Errorf("software URL %s is not a valid URL", softwarePackageSpec.URL))
				continue
			}
			if softwarePackageSpec.InstallScript.Path == "" || softwarePackageSpec.UninstallScript.Path == "" {
				// URL checks won't catch everything, but might as well include a lightweight check here to fail fast if it's
				// certain that the package will fail later.
				if strings.HasSuffix(parsedUrl.Path, ".exe") {
					multiError = multierror.Append(multiError, fmt.Errorf("software URL %s refers to an .exe package, which requires both install_script and uninstall_script", softwarePackageSpec.URL))
					continue
				} else if strings.HasSuffix(parsedUrl.Path, ".tar.gz") || strings.HasSuffix(parsedUrl.Path, ".tgz") {
					multiError = multierror.Append(multiError, fmt.Errorf("software URL %s refers to a .tar.gz archive, which requires both install_script and uninstall_script", softwarePackageSpec.URL))
					continue
				}
			}

			// Validate display_name length (matches database VARCHAR(255))
			if len(softwarePackageSpec.DisplayName) > 255 {
				multiError = multierror.Append(multiError, fmt.Errorf("software package %q display_name is too long (max 255 characters)", softwarePackageSpec.URL))
				continue
			}

			result.Software.Packages = append(result.Software.Packages, softwarePackageSpec)
		}
	}

	return multiError
}

func gatherFileSecrets(result *GitOps, filePath string) error {
	fileBytes, err := os.ReadFile(filePath)
	if err != nil {
		return fmt.Errorf("failed to read file %s: %w", filePath, err)
	}

	err = LookupEnvSecrets(string(fileBytes), result.FleetSecrets)
	if err != nil {
		return fmt.Errorf("failed to lookup environment secrets for %s: %w", filePath, err)
	}

	return nil
}

func getDuplicateNames[T any](slice []T, getComparableString func(T) string) []string {
	// We are using the allKeys map as a set here. True means the item is a duplicate.
	allKeys := make(map[string]bool)
	var duplicates []string
	for _, item := range slice {
		name := getComparableString(item)
		if isDuplicate, exists := allKeys[name]; exists {
			// If this name hasn't already been marked as a duplicate.
			if !isDuplicate {
				duplicates = append(duplicates, name)
			}
			allKeys[name] = true
		} else {
			allKeys[name] = false
		}
	}
	return duplicates
}

func isASCII(s string) bool {
	for _, c := range s {
		if c > unicode.MaxASCII {
			return false
		}
	}
	return true
}

// resolves the paths to an absolute path relative to the baseDir, which should
// be the path of the YAML file where the relative paths were specified. If the
// path is already absolute, it is left untouched.
func resolveApplyRelativePath(baseDir string, path string) string {
	if baseDir == "" || filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(baseDir, path)
}