Move specs parsing functionality to a new pkg/spec package (#7050)

This commit is contained in:
Lucas Manuel Rodriguez 2022-08-05 19:07:32 -03:00 committed by GitHub
parent 272691c6e5
commit 6dcff28be0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 297 additions and 260 deletions

View file

@ -1,142 +1,16 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"regexp"
"strings"
"os"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/service"
"github.com/ghodss/yaml"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/urfave/cli/v2"
)
var (
yamlSeparator = regexp.MustCompile(`(?m:^---[\t ]*)`)
)
type specMetadata struct {
Kind string `json:"kind"`
Version string `json:"apiVersion"`
Spec json.RawMessage `json:"spec"`
}
type specGroup struct {
Queries []*fleet.QuerySpec
Teams []*fleet.TeamSpec
Packs []*fleet.PackSpec
Labels []*fleet.LabelSpec
Policies []*fleet.PolicySpec
// This needs to be interface{} to allow for the patch logic. Otherwise we send a request that looks to the
// server like the user explicitly set the zero values.
AppConfig interface{}
EnrollSecret *fleet.EnrollSecretSpec
UsersRoles *fleet.UsersRoleSpec
}
type TeamSpec struct {
Team *fleet.TeamSpec `json:"team"`
}
func specGroupFromBytes(b []byte) (*specGroup, error) {
specs := &specGroup{
Queries: []*fleet.QuerySpec{},
Packs: []*fleet.PackSpec{},
Labels: []*fleet.LabelSpec{},
}
for _, spec := range splitYaml(string(b)) {
var s specMetadata
if err := yaml.Unmarshal([]byte(spec), &s); err != nil {
return nil, err
}
if s.Spec == nil {
return nil, fmt.Errorf("no spec field on %q document", s.Kind)
}
kind := strings.ToLower(s.Kind)
switch kind {
case fleet.QueryKind:
var querySpec *fleet.QuerySpec
if err := yaml.Unmarshal(s.Spec, &querySpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Queries = append(specs.Queries, querySpec)
case fleet.PackKind:
var packSpec *fleet.PackSpec
if err := yaml.Unmarshal(s.Spec, &packSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Packs = append(specs.Packs, packSpec)
case fleet.LabelKind:
var labelSpec *fleet.LabelSpec
if err := yaml.Unmarshal(s.Spec, &labelSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Labels = append(specs.Labels, labelSpec)
case fleet.PolicyKind:
var policySpec *fleet.PolicySpec
if err := yaml.Unmarshal(s.Spec, &policySpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Policies = append(specs.Policies, policySpec)
case fleet.AppConfigKind:
if specs.AppConfig != nil {
return nil, errors.New("config defined twice in the same file")
}
var appConfigSpec interface{}
if err := yaml.Unmarshal(s.Spec, &appConfigSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.AppConfig = appConfigSpec
case fleet.EnrollSecretKind:
if specs.AppConfig != nil {
return nil, errors.New("enroll_secret defined twice in the same file")
}
var enrollSecretSpec *fleet.EnrollSecretSpec
if err := yaml.Unmarshal(s.Spec, &enrollSecretSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.EnrollSecret = enrollSecretSpec
case fleet.UserRolesKind:
var userRoleSpec *fleet.UsersRoleSpec
if err := yaml.Unmarshal(s.Spec, &userRoleSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.UsersRoles = userRoleSpec
case fleet.TeamKind:
var teamSpec TeamSpec
if err := yaml.Unmarshal(s.Spec, &teamSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Teams = append(specs.Teams, teamSpec.Team)
default:
return nil, fmt.Errorf("unknown kind %q", s.Kind)
}
}
return specs, nil
}
func applyCommand() *cli.Command {
var (
flFilename string
)
var flFilename string
return &cli.Command{
Name: "apply",
Usage: "Apply files to declaratively manage osquery configurations",
@ -157,88 +31,26 @@ func applyCommand() *cli.Command {
if flFilename == "" {
return errors.New("-f must be specified")
}
b, err := ioutil.ReadFile(flFilename)
b, err := os.ReadFile(flFilename)
if err != nil {
return err
}
fleetClient, err := clientFromCLI(c)
if err != nil {
return err
}
err = applyYamlBytes(c, b, fleetClient)
specs, err := spec.GroupFromBytes(b)
if err != nil {
return err
}
logf := func(format string, a ...interface{}) {
fmt.Fprintf(c.App.Writer, format, a...)
}
err = fleetClient.ApplyGroup(c.Context, specs, logf)
if err != nil {
return err
}
return nil
},
}
}
func applyYamlBytes(c *cli.Context, b []byte, fleetClient *service.Client) error {
specs, err := specGroupFromBytes(b)
if err != nil {
return err
}
if len(specs.Queries) > 0 {
if err := fleetClient.ApplyQueries(specs.Queries); err != nil {
return fmt.Errorf("applying queries: %w", err)
}
logf(c, "[+] applied %d queries\n", len(specs.Queries))
}
if len(specs.Labels) > 0 {
if err := fleetClient.ApplyLabels(specs.Labels); err != nil {
return fmt.Errorf("applying labels: %w", err)
}
logf(c, "[+] applied %d labels\n", len(specs.Labels))
}
if len(specs.Policies) > 0 {
if err := fleetClient.ApplyPolicies(specs.Policies); err != nil {
return fmt.Errorf("applying policies: %w", err)
}
logf(c, "[+] applied %d policies\n", len(specs.Policies))
}
if len(specs.Packs) > 0 {
if err := fleetClient.ApplyPacks(specs.Packs); err != nil {
return fmt.Errorf("applying packs: %w", err)
}
logf(c, "[+] applied %d packs\n", len(specs.Packs))
}
if specs.AppConfig != nil {
if err := fleetClient.ApplyAppConfig(specs.AppConfig); err != nil {
return fmt.Errorf("applying fleet config: %w", err)
}
log(c, "[+] applied fleet config\n")
}
if specs.EnrollSecret != nil {
if err := fleetClient.ApplyEnrollSecretSpec(specs.EnrollSecret); err != nil {
return fmt.Errorf("applying enroll secrets: %w", err)
}
log(c, "[+] applied enroll secrets\n")
}
if len(specs.Teams) > 0 {
if err := fleetClient.ApplyTeams(specs.Teams); err != nil {
return fmt.Errorf("applying teams: %w", err)
}
logf(c, "[+] applied %d teams\n", len(specs.Teams))
}
if specs.UsersRoles != nil {
if err := fleetClient.ApplyUsersRoleSecretSpec(specs.UsersRoles); err != nil {
return fmt.Errorf("applying user roles: %w", err)
}
log(c, "[+] applied user roles\n")
}
return nil
}

View file

@ -13,13 +13,14 @@ import (
"strconv"
"strings"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/ghodss/yaml"
"github.com/urfave/cli/v2"
)
func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*specGroup, error) {
specs := &specGroup{
func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spec.Group, error) {
specs := &spec.Group{
Queries: []*fleet.QuerySpec{},
Packs: []*fleet.PackSpec{},
Labels: []*fleet.LabelSpec{},
@ -123,7 +124,7 @@ func convertCommand() *cli.Command {
re := regexp.MustCompile(`\s*\\\n`)
b = re.ReplaceAll(b, []byte(`\n`))
var specs *specGroup
var specs *spec.Group
var pack fleet.PermissivePackContent
if err := json.Unmarshal(b, &pack); err != nil {
@ -151,15 +152,15 @@ func convertCommand() *cli.Command {
}
for _, pack := range specs.Packs {
spec, err := json.Marshal(pack)
specBytes, err := json.Marshal(pack)
if err != nil {
return err
}
meta := specMetadata{
meta := spec.Metadata{
Kind: fleet.PackKind,
Version: fleet.ApiVersion,
Spec: spec,
Spec: specBytes,
}
out, err := yaml.Marshal(meta)
@ -172,15 +173,15 @@ func convertCommand() *cli.Command {
}
for _, query := range specs.Queries {
spec, err := json.Marshal(query)
specBytes, err := json.Marshal(query)
if err != nil {
return err
}
meta := specMetadata{
meta := spec.Metadata{
Kind: fleet.QueryKind,
Version: fleet.ApiVersion,
Spec: spec,
Spec: specBytes,
}
out, err := yaml.Marshal(meta)

View file

@ -5,15 +5,14 @@ import (
"fmt"
"io/ioutil"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/service"
"github.com/urfave/cli/v2"
)
func deleteCommand() *cli.Command {
var (
flFilename string
)
var flFilename string
return &cli.Command{
Name: "delete",
Usage: "Specify files to declaratively batch delete osquery configurations",
@ -45,7 +44,7 @@ func deleteCommand() *cli.Command {
return err
}
specs, err := specGroupFromBytes(b)
specs, err := spec.GroupFromBytes(b)
if err != nil {
return err
}

View file

@ -823,10 +823,6 @@ func log(c *cli.Context, msg ...interface{}) {
fmt.Fprint(c.App.Writer, msg...)
}
func logf(c *cli.Context, format string, a ...interface{}) {
fmt.Fprintf(c.App.Writer, format, a...)
}
func getUserRolesCommand() *cli.Command {
return &cli.Command{
Name: "user_roles",

View file

@ -11,6 +11,7 @@ import (
"github.com/ghodss/yaml"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/fleetdm/fleet/v4/server/service"
@ -406,7 +407,7 @@ func TestGetHosts(t *testing.T) {
name: "get hosts --yaml test_host",
goldenFile: "expectedHostDetailResponseYaml.yml",
scanner: func(s string) []string {
return splitYaml(s)
return spec.SplitYaml(s)
},
args: []string{"get", "hosts", "--yaml", "test_host"},
prettifier: yamlPrettify,

View file

@ -21,6 +21,7 @@ import (
"github.com/fleetdm/fleet/v4/orbit/pkg/update"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/pkg/open"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/service"
"github.com/mitchellh/go-ps"
@ -297,7 +298,14 @@ Use the stop and reset subcommands to manage the server and dependencies once st
}
}
err = applyYamlBytes(c, buf, client)
specs, err := spec.GroupFromBytes(buf)
if err != nil {
return err
}
logf := func(format string, a ...interface{}) {
fmt.Fprintf(c.App.Writer, format, a...)
}
err = client.ApplyGroup(c.Context, specs, logf)
if err != nil {
return err
}

View file

@ -1,16 +0,0 @@
package main
import "strings"
// splitYaml splits a text file into separate yaml documents divided by ---
func splitYaml(in string) []string {
var out []string
for _, chunk := range yamlSeparator.Split(in, -1) {
chunk = strings.TrimSpace(chunk)
if chunk == "" {
continue
}
out = append(out, chunk)
}
return out
}

View file

@ -1,25 +0,0 @@
package main
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitYaml(t *testing.T) {
in := `
---
- Document
#---
--- Document2
---
Document3
`
docs := splitYaml(in)
require.Equal(t, 3, len(docs))
assert.Equal(t, "- Document\n#---", docs[0])
assert.Equal(t, "Document2", docs[1])
assert.Equal(t, "Document3", docs[2])
}

142
pkg/spec/spec.go Normal file
View file

@ -0,0 +1,142 @@
// Package spec contains functionality to parse "Fleet specs" yaml files
// (which are concatenated yaml files) that can be applied to a Fleet server.
package spec
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/ghodss/yaml"
)
var yamlSeparator = regexp.MustCompile(`(?m:^---[\t ]*)`)
// Group holds a set of "specs" that can be applied to a Fleet server.
type Group struct {
Queries []*fleet.QuerySpec
Teams []*fleet.TeamSpec
Packs []*fleet.PackSpec
Labels []*fleet.LabelSpec
Policies []*fleet.PolicySpec
// This needs to be interface{} to allow for the patch logic. Otherwise we send a request that looks to the
// server like the user explicitly set the zero values.
AppConfig interface{}
EnrollSecret *fleet.EnrollSecretSpec
UsersRoles *fleet.UsersRoleSpec
}
// Metadata holds the metadata for a single YAML section/item.
type Metadata struct {
Kind string `json:"kind"`
Version string `json:"apiVersion"`
Spec json.RawMessage `json:"spec"`
}
// TeamSpec holds a spec to be applied to a team.
type TeamSpec struct {
Team *fleet.TeamSpec `json:"team"`
}
// GroupFromBytes parses a Group from concatenated YAML specs.
func GroupFromBytes(b []byte) (*Group, error) {
specs := &Group{}
for _, specItem := range SplitYaml(string(b)) {
var s Metadata
if err := yaml.Unmarshal([]byte(specItem), &s); err != nil {
return nil, err
}
if s.Spec == nil {
return nil, fmt.Errorf("no spec field on %q document", s.Kind)
}
kind := strings.ToLower(s.Kind)
switch kind {
case fleet.QueryKind:
var querySpec *fleet.QuerySpec
if err := yaml.Unmarshal(s.Spec, &querySpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Queries = append(specs.Queries, querySpec)
case fleet.PackKind:
var packSpec *fleet.PackSpec
if err := yaml.Unmarshal(s.Spec, &packSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Packs = append(specs.Packs, packSpec)
case fleet.LabelKind:
var labelSpec *fleet.LabelSpec
if err := yaml.Unmarshal(s.Spec, &labelSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Labels = append(specs.Labels, labelSpec)
case fleet.PolicyKind:
var policySpec *fleet.PolicySpec
if err := yaml.Unmarshal(s.Spec, &policySpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Policies = append(specs.Policies, policySpec)
case fleet.AppConfigKind:
if specs.AppConfig != nil {
return nil, errors.New("config defined twice in the same file")
}
var appConfigSpec interface{}
if err := yaml.Unmarshal(s.Spec, &appConfigSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.AppConfig = appConfigSpec
case fleet.EnrollSecretKind:
if specs.AppConfig != nil {
return nil, errors.New("enroll_secret defined twice in the same file")
}
var enrollSecretSpec *fleet.EnrollSecretSpec
if err := yaml.Unmarshal(s.Spec, &enrollSecretSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.EnrollSecret = enrollSecretSpec
case fleet.UserRolesKind:
var userRoleSpec *fleet.UsersRoleSpec
if err := yaml.Unmarshal(s.Spec, &userRoleSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.UsersRoles = userRoleSpec
case fleet.TeamKind:
var teamSpec TeamSpec
if err := yaml.Unmarshal(s.Spec, &teamSpec); err != nil {
return nil, fmt.Errorf("unmarshaling %s spec: %w", kind, err)
}
specs.Teams = append(specs.Teams, teamSpec.Team)
default:
return nil, fmt.Errorf("unknown kind %q", s.Kind)
}
}
return specs, nil
}
// SplitYaml splits a text file into separate yaml documents divided by ---
func SplitYaml(in string) []string {
var out []string
for _, chunk := range yamlSeparator.Split(in, -1) {
chunk = strings.TrimSpace(chunk)
if chunk == "" {
continue
}
out = append(out, chunk)
}
return out
}

52
pkg/spec/spec_test.go Normal file
View file

@ -0,0 +1,52 @@
package spec
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitYaml(t *testing.T) {
in := `
---
- Document
#---
--- Document2
---
Document3
`
docs := SplitYaml(in)
require.Equal(t, 3, len(docs))
assert.Equal(t, "- Document\n#---", docs[0])
assert.Equal(t, "Document2", docs[1])
assert.Equal(t, "Document3", docs[2])
}
func gitRootPath(t *testing.T) string {
path, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
require.NoError(t, err)
return strings.TrimSpace(string(path))
}
func loadStdQueryLibrary(t *testing.T) []byte {
b, err := os.ReadFile(filepath.Join(
gitRootPath(t),
"docs", "01-Using-Fleet", "standard-query-library", "standard-query-library.yml",
))
require.NoError(t, err)
return b
}
func TestGroupFromBytes(t *testing.T) {
stdQueryLib := loadStdQueryLibrary(t)
g, err := GroupFromBytes(stdQueryLib)
require.NoError(t, err)
require.NotEmpty(t, g.Queries)
require.NotEmpty(t, g.Policies)
}

View file

@ -11,6 +11,7 @@ import (
"os"
"time"
"github.com/fleetdm/fleet/v4/pkg/spec"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
)
@ -215,3 +216,69 @@ func (c *Client) authenticatedRequestWithQuery(params interface{}, verb string,
func (c *Client) authenticatedRequest(params interface{}, verb string, path string, responseDest interface{}) error {
return c.authenticatedRequestWithQuery(params, verb, path, responseDest, "")
}
// ApplyGroup applies the given spec group to Fleet.
func (c *Client) ApplyGroup(ctx context.Context, specs *spec.Group, logf func(format string, args ...interface{})) error {
logfn := func(format string, args ...interface{}) {
if logf != nil {
logf(format, args...)
}
}
if len(specs.Queries) > 0 {
if err := c.ApplyQueries(specs.Queries); err != nil {
return fmt.Errorf("applying queries: %w", err)
}
logfn("[+] applied %d queries\n", len(specs.Queries))
}
if len(specs.Labels) > 0 {
if err := c.ApplyLabels(specs.Labels); err != nil {
return fmt.Errorf("applying labels: %w", err)
}
logfn("[+] applied %d labels\n", len(specs.Labels))
}
if len(specs.Policies) > 0 {
if err := c.ApplyPolicies(specs.Policies); err != nil {
return fmt.Errorf("applying policies: %w", err)
}
logfn("[+] applied %d policies\n", len(specs.Policies))
}
if len(specs.Packs) > 0 {
if err := c.ApplyPacks(specs.Packs); err != nil {
return fmt.Errorf("applying packs: %w", err)
}
logfn("[+] applied %d packs\n", len(specs.Packs))
}
if specs.AppConfig != nil {
if err := c.ApplyAppConfig(specs.AppConfig); err != nil {
return fmt.Errorf("applying fleet config: %w", err)
}
logfn("[+] applied fleet config\n")
}
if specs.EnrollSecret != nil {
if err := c.ApplyEnrollSecretSpec(specs.EnrollSecret); err != nil {
return fmt.Errorf("applying enroll secrets: %w", err)
}
logfn("[+] applied enroll secrets\n")
}
if len(specs.Teams) > 0 {
if err := c.ApplyTeams(specs.Teams); err != nil {
return fmt.Errorf("applying teams: %w", err)
}
logfn("[+] applied %d teams\n", len(specs.Teams))
}
if specs.UsersRoles != nil {
if err := c.ApplyUsersRoleSecretSpec(specs.UsersRoles); err != nil {
return fmt.Errorf("applying user roles: %w", err)
}
logfn("[+] applied user roles\n")
}
return nil
}