mirror of
https://github.com/fleetdm/fleet
synced 2026-04-21 21:47:20 +00:00
288ea58bce
# Overview This pull request resolves #31165, implementing command-line tooling to migrate GitOps YAML files following the [changes introduced in the upcoming 4.74 release](https://github.com/fleetdm/fleet/pull/32237/files#diff-8769f6e90e8bdf15faad8f390fdf3ffb6fd2238b7d6087d83518c21464109119R7). Aligning with the recommended steps in the `README`; [this is an example of the first step](https://github.com/Illbjorn/fleet/pull/3/files) (`gitops-migrate format`) and [this is an example of the second step](https://github.com/Illbjorn/fleet/pull/4/files) (`gitops-migrate migrate`). --------- Signed-off-by: Illbjorn <am@hades.so> Co-authored-by: Ian Littman <iansltx@gmail.com>
590 lines
18 KiB
Go
590 lines
18 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/fleetdm/fleet/v4/cmd/gitops-migrate/log"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
const (
|
|
cmdMigrate = "migrate"
|
|
|
|
// Define some well-known map keys we'll use further down.
|
|
keyPath = "path"
|
|
keyPackages = "packages"
|
|
keySoftware = "software"
|
|
keyCategories = "categories"
|
|
keySelfService = "self_service"
|
|
keyLabelsExclude = "labels_exclude_any"
|
|
keyLabelsInclude = "labels_include_any"
|
|
keySetupExperience = "setup_experience"
|
|
keyControls = "controls"
|
|
keyMacosSetup = "macos_setup"
|
|
keyPackagePath = "package_path"
|
|
)
|
|
|
|
func cmdMigrateExec(ctx context.Context, args Args) error {
|
|
if len(args.Commands) < 1 {
|
|
showUsageAndExit(
|
|
1,
|
|
"please specify the path to your Fleet GitOps YAML files",
|
|
)
|
|
}
|
|
from, err := filepath.Abs(args.Commands[0])
|
|
if err != nil {
|
|
return fmt.Errorf(
|
|
"failed to derive absolute input path(%s): %w",
|
|
args.Commands[0], err,
|
|
)
|
|
}
|
|
|
|
// Create a temp directory to which we'll write the backup archive.
|
|
tmpDir, err := mkBackupDir()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Backup the provided migration target.
|
|
_, err = backup(ctx, from, tmpDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Packages can belong to >1 team file. So, to avoid unnecessary i/o we
|
|
// capture the state we mutate(read: delete) for each package file here and
|
|
// refer back to it if we encounter the same item again.
|
|
known := make(map[string]map[string]any)
|
|
// Track successful/failed conversions.
|
|
success := 0
|
|
failed := 0
|
|
for item, err := range fsEnum(from) {
|
|
// Handle any iterator errors.
|
|
if err != nil {
|
|
log.Errorf("Encountered error in file system enumeration: %s.", err)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Ignore directories.
|
|
if item.Stats.IsDir() {
|
|
log.Debugf("Ignoring directory: %s.", item.Path)
|
|
continue
|
|
}
|
|
|
|
// Ignore non-YAML files.
|
|
if !strings.HasSuffix(item.Path, ".yml") &&
|
|
!strings.HasSuffix(item.Path, ".yaml") {
|
|
log.Debugf("Ignoring non-YAML file: %s.", item.Path)
|
|
continue
|
|
}
|
|
|
|
// Get a read-writable handle to the input file.
|
|
teamFile, err := os.OpenFile(item.Path, fileFlagsReadWrite, 0)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to get a read-writable handle to file.",
|
|
"File Path", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Unmarshal the file content.
|
|
team := make(map[string]any)
|
|
err = yaml.NewDecoder(teamFile).Decode(&team)
|
|
if err != nil {
|
|
if isYAMLUnmarshalSeqError(err) {
|
|
log.Debugf(
|
|
"Skipping file with array item(s) at the root: %s.",
|
|
item.Path,
|
|
)
|
|
continue
|
|
}
|
|
log.Error(
|
|
"Failed to unmarshal file.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// To avoid unnecessarily serializing a file back to disk when no changes
|
|
// have been performed, since this would blow away all comments and manual
|
|
// spacing, we need to count the number of changes which we actaully
|
|
// apply. If this count is zero, we simply skip the re-encode step.
|
|
teamChangeCount := 0
|
|
|
|
// NOTE(max): The below works but was de-scoped.
|
|
//
|
|
// Relocate '.controls.software.packages[].package_path' to
|
|
// '.software.packages[].setup_experience=true'.
|
|
//
|
|
// For any '.controls.software.packages' objects which hold a 'package_path'
|
|
// key:
|
|
// - Resolve the absolute path for the software package file referenced and
|
|
// record this absolute path to 'pkgsWithSetupExp'.
|
|
// - Delete the 'package_path' key.
|
|
// - If the 'package_path' key is the only key in this 'packages' array
|
|
// object, drop that array index.
|
|
// NOTE(max): The below works but was de-scoped.
|
|
//
|
|
// pkgsWithSetupExp := map[string]struct{}{}
|
|
// Retrieve the 'controls' key, assert as 'map[string]any'.
|
|
// if controls, ok := team[keyControls].(map[string]any); ok {
|
|
// // Retrieve the 'macos_setup' key, assert as 'map[string]any'.
|
|
// if macosSetup, ok := controls[keyMacosSetup].(map[string]any); ok {
|
|
// // Retrieve the 'software' key, assert as '[]any'.
|
|
// if pkgs, ok := macosSetup[keySoftware].([]any); ok {
|
|
// pkgChangeCount := 0
|
|
// // We iterate the slice backward so we can mutate it as we go.
|
|
// for i, pkg := range slices.Backward(pkgs) {
|
|
// // Assert the package as 'map[string]any'.
|
|
// if pkg, ok := pkg.(map[string]any); ok {
|
|
// // Retrieve the 'package_path' key, assert as 'string'.
|
|
// if pkgPath, ok := pkg[keyPackagePath].(string); ok {
|
|
// // Resolve the absolute path to the referenced package file.
|
|
// pkgPathAbs, err := resolvePackagePath(item.Path, pkgPath)
|
|
// if err != nil {
|
|
// log.Errorf("%s.", err)
|
|
// failed += 1
|
|
// continue
|
|
// }
|
|
// // Capture the absolute path as a key in our map.
|
|
// pkgsWithSetupExp[pkgPathAbs] = struct{}{}
|
|
// // Delete this map key.
|
|
// delete(pkg, keyPackagePath)
|
|
// // If 'package_path' was the final map key, delete the entire
|
|
// // slice item.
|
|
// if len(pkg) == 0 {
|
|
// pkgs = slices.Delete(pkgs, i, i+1)
|
|
// }
|
|
// // Signal mutation.
|
|
// pkgChangeCount += 1
|
|
// }
|
|
// }
|
|
// }
|
|
// // If we mutated 'pkgs', update the 'macosSetup' map key.
|
|
// if pkgChangeCount > 0 {
|
|
// if len(pkgs) == 0 {
|
|
// // If the resulting 'pkgs' slice is empty, just delete the key.
|
|
// delete(macosSetup, keySoftware)
|
|
// } else {
|
|
// // Otherwise, insert the updated slice.
|
|
// macosSetup[keySoftware] = pkgs
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
// Relocate keys 'self_service', 'labels_include', 'labels_exclude' and
|
|
// 'categories' from software package files to the '.software.packages'
|
|
// array object in the team file.
|
|
//
|
|
// For any '.software.packages' array items which contain a 'path' key:
|
|
// - Resolve the absolute path for the software package file referenced by
|
|
// the 'path' key's value.
|
|
// - Check the 'known' map for existence of this absolute path, if not found:
|
|
// - Read and unmarshal the software package file.
|
|
// - If the software package file contains any of the above keys:
|
|
// - Record the key's value.
|
|
// - Delete the key.
|
|
// * Increment the 'swPkgChangeCount' counter for each key deleted.
|
|
// - If the 'swPkgChangeCount' counter is > 0, serialize the software
|
|
// package file back to disk.
|
|
// - Apply any values we recorded from the software package file to this
|
|
// software package array object.
|
|
// * Increment the 'teamChangeCount' counter for each value added.
|
|
// - If the 'teamChangeCount' is > 0, serialize the team file back to
|
|
// disk.
|
|
|
|
// Retrieve the 'software' key, assert as 'map[string]any'.
|
|
if software, ok := team[keySoftware].(map[string]any); ok {
|
|
// Retrieve the 'packages' key, assert as '[]any'.
|
|
if pkgs, ok := software[keyPackages].([]any); ok {
|
|
// Iterate all packages.
|
|
for _, pkg := range pkgs {
|
|
// Assert the package as 'map[string]any'.
|
|
if pkg, ok := pkg.(map[string]any); ok {
|
|
// Retrieve the 'path' key, assert as 'string'.
|
|
if pkgPath, ok := pkg[keyPath].(string); ok {
|
|
// Construct an absolute path from the 'path' key's value.
|
|
pkgPath, err := resolvePackagePath(item.Path, pkgPath)
|
|
if err != nil {
|
|
log.Errorf("%s.", err)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Check if this is a package file we've processed previously.
|
|
//
|
|
// If not unmarshal the file, record & delete the fields we're
|
|
// relocating and serialize the updated package representation
|
|
// back to disk.
|
|
var state map[string]any
|
|
if state, ok = known[pkgPath]; !ok {
|
|
state = make(map[string]any)
|
|
// Track mutations to the package file.
|
|
//
|
|
// If we reach the serialization point and this count is zero we skip
|
|
// the re-encode to file to preserve formatting + comments where we can.
|
|
var pkgChangeCount int
|
|
|
|
// Get a read-writable handle to the package file.
|
|
packageFile, err := os.OpenFile(pkgPath, fileFlagsReadWrite, 0)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to get a read-writable handle to package file.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Decode the package file.
|
|
pkg := make(map[string]any)
|
|
err = yaml.NewDecoder(packageFile).Decode(pkg)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to decode package file.",
|
|
"Package File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Record and delete the fields we're migrating.
|
|
|
|
// Look for the 'self_service' value.
|
|
if v, ok := pkg[keySelfService]; ok {
|
|
if b, ok := v.(bool); ok {
|
|
state[keySelfService] = b
|
|
}
|
|
delete(pkg, keySelfService)
|
|
pkgChangeCount += 1
|
|
}
|
|
|
|
// Look for 'labels_include' items.
|
|
if v, ok := pkg[keyLabelsInclude]; ok {
|
|
if v, ok := v.([]any); ok {
|
|
labelIncludes := make([]string, 0, len(v))
|
|
for i := range len(v) {
|
|
if labelInclude, ok := v[i].(string); ok {
|
|
labelIncludes = append(labelIncludes, labelInclude)
|
|
}
|
|
}
|
|
if len(labelIncludes) > 0 {
|
|
state[keyLabelsInclude] = labelIncludes
|
|
}
|
|
}
|
|
delete(pkg, keyLabelsInclude)
|
|
pkgChangeCount += 1
|
|
}
|
|
|
|
// Look for 'labels_exclude' items.
|
|
if v, ok := pkg[keyLabelsExclude]; ok {
|
|
if v, ok := v.([]any); ok {
|
|
labelExcludes := make([]string, 0, len(v))
|
|
for i := range len(v) {
|
|
if labelExclude, ok := v[i].(string); ok {
|
|
labelExcludes = append(labelExcludes, labelExclude)
|
|
}
|
|
}
|
|
if len(labelExcludes) > 0 {
|
|
state[keyLabelsExclude] = labelExcludes
|
|
}
|
|
}
|
|
delete(pkg, keyLabelsExclude)
|
|
pkgChangeCount += 1
|
|
}
|
|
|
|
// Look for 'categories' items.
|
|
if v, ok := pkg[keyCategories]; ok {
|
|
if v, ok := v.([]any); ok {
|
|
categories := make([]string, 0, len(v))
|
|
for i := range len(v) {
|
|
if category, ok := v[i].(string); ok {
|
|
categories = append(categories, category)
|
|
}
|
|
}
|
|
if len(categories) > 0 {
|
|
state[keyCategories] = categories
|
|
}
|
|
}
|
|
delete(pkg, keyCategories)
|
|
pkgChangeCount += 1
|
|
}
|
|
|
|
// Only re-encode if we actually mutated something.
|
|
if pkgChangeCount > 0 {
|
|
// Seek to file start for re-encode.
|
|
_, err = packageFile.Seek(0, io.SeekStart)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to seek to file start for re-encode.",
|
|
"Team File", item.Path,
|
|
"Package File", pkgPath,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Re-encode the package file.
|
|
enc := yaml.NewEncoder(packageFile)
|
|
enc.SetIndent(2)
|
|
err = enc.Encode(pkg)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to re-encode package file.",
|
|
"Team File", item.Path,
|
|
"Package File", pkgPath,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Seek to identify the number of bytes we wrote during the
|
|
// YAML encode.
|
|
n, err := packageFile.Seek(0, io.SeekCurrent)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to seek after package file re-encode.",
|
|
"Team File", item.Path,
|
|
"Package File", pkgPath,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Truncate at the number of bytes we just wrote.
|
|
err = packageFile.Truncate(n)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to truncate package file after re-encode.",
|
|
"Team File", item.Path,
|
|
"Package File", pkgPath,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Close the package file.
|
|
err = packageFile.Close()
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to close package file after re-encode.",
|
|
"Team File", item.Path,
|
|
"Package File", pkgPath,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Store the package state.
|
|
known[pkgPath] = state
|
|
}
|
|
|
|
// Relocate any items we removed from the package file to this
|
|
// package entry.
|
|
|
|
// Key: 'self_service'.
|
|
if v, ok := state[keySelfService]; ok {
|
|
teamChangeCount += 1
|
|
pkg[keySelfService] = v
|
|
}
|
|
|
|
// Key: 'categories'.
|
|
if v, ok := state[keyCategories]; ok {
|
|
teamChangeCount += 1
|
|
pkg[keyCategories] = v
|
|
}
|
|
|
|
// Key: 'labels_include'.
|
|
if v, ok := state[keyLabelsInclude]; ok {
|
|
teamChangeCount += 1
|
|
pkg[keyLabelsInclude] = v
|
|
}
|
|
|
|
// Key: 'labels_exclude'.
|
|
if v, ok := state[keyLabelsExclude]; ok {
|
|
teamChangeCount += 1
|
|
pkg[keyLabelsExclude] = v
|
|
}
|
|
|
|
// NOTE(max): The below works but was de-scoped.
|
|
//
|
|
// If we found+removed a 'package_path' key for this item earlier,
|
|
// add the 'setup_experience' key with a value of 'true'.
|
|
//
|
|
// if _, ok := pkgsWithSetupExp[pkgPath]; ok {
|
|
// teamChangeCount += 1
|
|
// pkg[keySetupExperience] = true
|
|
// }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Only re-encode the team file if we actually changed something.
|
|
if teamChangeCount > 0 {
|
|
// Seek to the file start for the YAML-encode.
|
|
_, err = teamFile.Seek(0, io.SeekStart)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to seek to team file start for YAML encode.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Serialize the team file back to disk.
|
|
enc := yaml.NewEncoder(teamFile)
|
|
enc.SetIndent(2)
|
|
err = enc.Encode(team)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to YAML-encode updated team file back to disk.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Identify the number of bytes we just wrote.
|
|
n, err := teamFile.Seek(0, io.SeekCurrent)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to identify number of bytes written following team file "+
|
|
"YAML-encode.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Truncate the file at the number of bytes we wrote during the
|
|
// YAML-encode.
|
|
err = teamFile.Truncate(n)
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to truncate team file following YAML-encode.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Close the team YAML file.
|
|
err = teamFile.Close()
|
|
if err != nil {
|
|
log.Error(
|
|
"Failed to close team YAML file following YAML-encode.",
|
|
"Team File", item.Path,
|
|
"Error", err,
|
|
)
|
|
failed += 1
|
|
continue
|
|
}
|
|
|
|
// Success!
|
|
if teamChangeCount > 0 {
|
|
log.Info(
|
|
"Successfully applied transforms to team file.",
|
|
"Team File", item.Path,
|
|
"Migrated Fields", teamChangeCount,
|
|
)
|
|
success += 1
|
|
}
|
|
}
|
|
|
|
log.Info(
|
|
"Migration complete.",
|
|
"Successful", success,
|
|
"Failed", failed,
|
|
)
|
|
|
|
if f := failed; f > 0 {
|
|
return errors.New("encountered failures during attempted GitOps migration")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolvePackagePath resolves an absolute file path to the software package at
|
|
// 'pkgPath' relative to the parent directory of the 'teamFilePath'.
|
|
//
|
|
// Example:
|
|
// Absolute path to Fleet GitOps file root : /home/fleet
|
|
// teamFilePath : teams/workstations.yml
|
|
// pkgPath : ../software/firefox.yml
|
|
// Output : /home/fleet/software/firefox.yml
|
|
func resolvePackagePath(teamFilePath, pkgPathRaw string) (string, error) {
|
|
// Concat the package path (usually relative) to the parent directory of the
|
|
// team file path.
|
|
//
|
|
// If 'teamFile' is _not_ a file, skip the 'filepath.Dir' call.
|
|
if filepath.Ext(teamFilePath) != "" {
|
|
teamFilePath = filepath.Dir(teamFilePath)
|
|
}
|
|
pkgPathRel := filepath.Join(teamFilePath, pkgPathRaw)
|
|
|
|
// Resolve the absolute path for this software package file.
|
|
pkgPathAbs, err := filepath.Abs(pkgPathRel)
|
|
if err != nil {
|
|
return "", fmt.Errorf(
|
|
"failed to resolve absolute software package file path from team file "+
|
|
"[%s] and software package path [%s]",
|
|
teamFilePath, pkgPathRaw,
|
|
)
|
|
}
|
|
|
|
return pkgPathAbs, nil
|
|
}
|
|
|
|
// isYAMLUnmarshalSeqError simply reports whether the provided error (returned
|
|
// by 'yaml.NewDecoder().Decode' or 'yaml.Unmarshal') is in fact a
|
|
// 'yaml.TypeError' which contains a single message regarding a failed unmarshal
|
|
// of a sequence (array) to a map.
|
|
//
|
|
// Certain GitOps YAML files (ex: 'it-and-security\lib\windows\queries\all-x86-hosts.yml')
|
|
// use an array as the outermost data type. These cases are rare but exist, and
|
|
// this will break the YAML unmarshal since we expect objects at the outermost
|
|
// level for the purposes of this tool. This function just reports whether an
|
|
// encountered error is one such case.
|
|
func isYAMLUnmarshalSeqError(err error) bool {
|
|
var typeErr *yaml.TypeError
|
|
if errors.As(err, &typeErr) {
|
|
// This sure is an ugly way to check error conditions, but in the case of
|
|
// this package it builds a slice of errors as strings so we have no choice.
|
|
if len(typeErr.Errors) == 1 && strings.Contains(
|
|
typeErr.Errors[0], "cannot unmarshal !!seq into map[string]interface",
|
|
) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|