Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-20 23:48:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Update query and pack interfaces for fleetctl (#1670)

Browse source 
- Add new Apply spec methods for queries and packs
- Remove now extraneous datastore/service methods
- Remove import service (unused, and had many dependencies that this breaks)
- Refactor tests as appropriate
This commit is contained in:
Zachary Wasserman 2018-01-03 11:18:05 -08:00 committed by GitHub
parent d6b4de3874
commit 26dc30bd25
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 1057 additions and 2295 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

106
server/datastore/datastore_packs_test.go
View file

@ -55,8 +55,6 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
t.Skip("inmem is deprecated")
}
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
mockClock := clock.NewMockClock()
p1, err := ds.NewPack(&kolide.Pack{
@ -64,23 +62,6 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
})
require.Nil(t, err)
q1, err := ds.NewQuery(&kolide.Query{
Name: "foo",
Query: "foo",
AuthorID: user.ID,
})
require.Nil(t, err)
q2, err := ds.NewQuery(&kolide.Query{
Name: "bar",
Query: "bar",
AuthorID: user.ID,
})
require.Nil(t, err)
test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, false, false)
l1, err := ds.NewLabel(&kolide.Label{
Name: "foo",
})
@ -151,3 +132,90 @@ func testAddLabelToPackTwice(t *testing.T, ds kolide.Datastore) {
assert.Nil(t, err)
assert.Len(t, labels, 1)
}
func testApplyPackSpecRoundtrip(t *testing.T, ds kolide.Datastore) {
zwass := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
queries := []*kolide.Query{
{Name: "foo", Description: "get the foos", Query: "select * from foo"},
{Name: "bar", Description: "do some bars", Query: "select baz from bar"},
}
// Zach creates some queries
err := ds.ApplyQueries(zwass.ID, queries)
require.Nil(t, err)
test.NewLabel(t, ds, "foo", "select * from foo")
test.NewLabel(t, ds, "bar", "select * from bar")
test.NewLabel(t, ds, "bing", "select * from bing")
boolPtr := func(b bool) *bool { return &b }
uintPtr := func(x uint) *uint { return &x }
stringPtr := func(s string) *string { return &s }
specs := []*kolide.PackSpec{
&kolide.PackSpec{
ID: 1,
Name: "test_pack",
Targets: kolide.PackSpecTargets{
Labels: []string{
"foo",
"bar",
"bing",
},
},
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Description: "test_foo",
Interval: 42,
},
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Name: "foo_snapshot",
Interval: 600,
Snapshot: boolPtr(true),
},
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Interval: 600,
Removed: boolPtr(false),
Shard: uintPtr(73),
Platform: stringPtr("foobar"),
Version: stringPtr("0.0.0.0.0.1"),
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
gotSpec, err := ds.GetPackSpecs()
require.Nil(t, err)
assert.Equal(t, specs, gotSpec)
}
func testApplyPackSpecMissingQueries(t *testing.T, ds kolide.Datastore) {
// Do not define queries mentioned in spec
specs := []*kolide.PackSpec{
&kolide.PackSpec{
ID: 1,
Name: "test_pack",
Targets: kolide.PackSpecTargets{
Labels: []string{},
},
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: "bar",
Interval: 600,
},
},
},
}
// Should error due to unkown query
err := ds.ApplyPackSpecs(specs)
if assert.NotNil(t, err) {
assert.Contains(t, err.Error(), "unknown query 'bar'")
}
}

217
server/datastore/datastore_queries_test.go
View file

@ -2,15 +2,76 @@ package datastore
import (
"fmt"
"sort"
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/kolide/fleet/server/test"
"github.com/patrickmn/sortutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testApplyQueries(t *testing.T, ds kolide.Datastore) {
zwass := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
groob := test.NewUser(t, ds, "Victor", "groob", "victor@kolide.co", true)
expectedQueries := []*kolide.Query{
{Name: "foo", Description: "get the foos", Query: "select * from foo"},
{Name: "bar", Description: "do some bars", Query: "select baz from bar"},
}
// Zach creates some queries
err := ds.ApplyQueries(zwass.ID, expectedQueries)
require.Nil(t, err)
queries, err := ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, len(expectedQueries))
for i, q := range queries {
comp := expectedQueries[i]
assert.Equal(t, comp.Name, q.Name)
assert.Equal(t, comp.Description, q.Description)
assert.Equal(t, comp.Query, q.Query)
assert.Equal(t, zwass.ID, q.AuthorID)
}
// Victor modifies a query (but also pushes the same version of the
// first query)
expectedQueries[1].Query = "not really a valid query ;)"
err = ds.ApplyQueries(groob.ID, expectedQueries)
require.Nil(t, err)
queries, err = ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, len(expectedQueries))
for i, q := range queries {
comp := expectedQueries[i]
assert.Equal(t, comp.Name, q.Name)
assert.Equal(t, comp.Description, q.Description)
assert.Equal(t, comp.Query, q.Query)
assert.Equal(t, groob.ID, q.AuthorID)
}
// Zach adds a third query (but does not re-apply the others)
expectedQueries = append(expectedQueries,
&kolide.Query{Name: "trouble", Description: "Look out!", Query: "select * from time"},
)
err = ds.ApplyQueries(zwass.ID, []*kolide.Query{expectedQueries[2]})
require.Nil(t, err)
queries, err = ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, len(expectedQueries))
for i, q := range queries {
comp := expectedQueries[i]
assert.Equal(t, comp.Name, q.Name)
assert.Equal(t, comp.Description, q.Description)
assert.Equal(t, comp.Query, q.Query)
}
assert.Equal(t, groob.ID, queries[0].AuthorID)
assert.Equal(t, groob.ID, queries[1].AuthorID)
assert.Equal(t, zwass.ID, queries[2].AuthorID)
}
func testDeleteQuery(t *testing.T, ds kolide.Datastore) {
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
@ -137,51 +198,133 @@ func testListQuery(t *testing.T, ds kolide.Datastore) {
assert.Equal(t, 10, len(results))
}
func checkPacks(t *testing.T, expected []kolide.Pack, actual []kolide.Pack) {
sortutil.AscByField(expected, "ID")
sortutil.AscByField(actual, "ID")
assert.Equal(t, expected, actual)
}
func testLoadPacksForQueries(t *testing.T, ds kolide.Datastore) {
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
q1 := test.NewQuery(t, ds, "q1", "select * from time", user.ID, true)
q2 := test.NewQuery(t, ds, "q2", "select * from osquery_info", user.ID, true)
p1 := test.NewPack(t, ds, "p1")
p2 := test.NewPack(t, ds, "p2")
p3 := test.NewPack(t, ds, "p3")
var err error
test.NewScheduledQuery(t, ds, p2.ID, q1.ID, 60, false, false)
q1, err = ds.Query(q1.ID)
zwass := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
queries := []*kolide.Query{
{Name: "q1", Query: "select * from time"},
{Name: "q2", Query: "select * from osquery_info"},
}
err := ds.ApplyQueries(zwass.ID, queries)
require.Nil(t, err)
q2, err = ds.Query(q2.ID)
require.Nil(t, err)
checkPacks(t, []kolide.Pack{*p2}, q1.Packs)
checkPacks(t, []kolide.Pack{}, q2.Packs)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, false, false)
test.NewScheduledQuery(t, ds, p3.ID, q2.ID, 60, false, false)
specs := []*kolide.PackSpec{
&kolide.PackSpec{Name: "p1"},
&kolide.PackSpec{Name: "p2"},
&kolide.PackSpec{Name: "p3"},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
q1, err = ds.Query(q1.ID)
q0, exists, err := ds.QueryByName(queries[0].Name)
require.Nil(t, err)
q2, err = ds.Query(q2.ID)
require.Nil(t, err)
checkPacks(t, []kolide.Pack{*p2}, q1.Packs)
checkPacks(t, []kolide.Pack{*p1, *p3}, q2.Packs)
require.True(t, exists)
assert.Empty(t, q0.Packs)
test.NewScheduledQuery(t, ds, p3.ID, q1.ID, 60, false, false)
q1, exists, err := ds.QueryByName(queries[1].Name)
require.Nil(t, err)
require.True(t, exists)
assert.Empty(t, q1.Packs)
q1, err = ds.Query(q1.ID)
specs = []*kolide.PackSpec{
&kolide.PackSpec{
Name: "p2",
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Interval: 60,
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
q2, err = ds.Query(q2.ID)
q0, exists, err = ds.QueryByName(queries[0].Name)
require.Nil(t, err)
checkPacks(t, []kolide.Pack{*p2, *p3}, q1.Packs)
checkPacks(t, []kolide.Pack{*p1, *p3}, q2.Packs)
require.True(t, exists)
if assert.Len(t, q0.Packs, 1) {
assert.Equal(t, "p2", q0.Packs[0].Name)
}
q1, exists, err = ds.QueryByName(queries[1].Name)
require.Nil(t, err)
require.True(t, exists)
assert.Empty(t, q1.Packs)
specs = []*kolide.PackSpec{
&kolide.PackSpec{
Name: "p1",
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Interval: 60,
},
},
},
&kolide.PackSpec{
Name: "p3",
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Interval: 60,
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
q0, exists, err = ds.QueryByName(queries[0].Name)
require.Nil(t, err)
require.True(t, exists)
if assert.Len(t, q0.Packs, 1) {
assert.Equal(t, "p2", q0.Packs[0].Name)
}
q1, exists, err = ds.QueryByName(queries[1].Name)
require.Nil(t, err)
require.True(t, exists)
if assert.Len(t, q1.Packs, 2) {
sort.Slice(q1.Packs, func(i, j int) bool { return q1.Packs[i].Name < q1.Packs[j].Name })
assert.Equal(t, "p1", q1.Packs[0].Name)
assert.Equal(t, "p3", q1.Packs[1].Name)
}
specs = []*kolide.PackSpec{
&kolide.PackSpec{
Name: "p3",
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Interval: 60,
},
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Interval: 60,
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
q0, exists, err = ds.QueryByName(queries[0].Name)
require.Nil(t, err)
require.True(t, exists)
if assert.Len(t, q0.Packs, 2) {
sort.Slice(q0.Packs, func(i, j int) bool { return q0.Packs[i].Name < q0.Packs[j].Name })
assert.Equal(t, "p2", q0.Packs[0].Name)
assert.Equal(t, "p3", q0.Packs[1].Name)
}
q1, exists, err = ds.QueryByName(queries[1].Name)
require.Nil(t, err)
require.True(t, exists)
if assert.Len(t, q1.Packs, 2) {
sort.Slice(q1.Packs, func(i, j int) bool { return q1.Packs[i].Name < q1.Packs[j].Name })
assert.Equal(t, "p1", q1.Packs[0].Name)
assert.Equal(t, "p3", q1.Packs[1].Name)
}
}
func testDuplicateNewQuery(t *testing.T, ds kolide.Datastore) {

164
server/datastore/datastore_scheduled_queries_test.go
View file

@ -9,113 +9,69 @@ import (
"github.com/stretchr/testify/require"
)
func testNewScheduledQuery(t *testing.T, ds kolide.Datastore) {
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
query, err := ds.NewScheduledQuery(&kolide.ScheduledQuery{
PackID: p1.ID,
QueryID: q1.ID,
})
require.Nil(t, err)
assert.Equal(t, "foo", query.Name)
assert.Equal(t, "select * from time;", query.Query)
}
func testScheduledQuery(t *testing.T, ds kolide.Datastore) {
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := ds.ScheduledQuery(sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
}
func testDeleteScheduledQuery(t *testing.T, ds kolide.Datastore) {
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := ds.ScheduledQuery(sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
err = ds.DeleteScheduledQuery(sq1.ID)
require.Nil(t, err)
_, err = ds.ScheduledQuery(sq1.ID)
require.NotNil(t, err)
}
func testListScheduledQueriesInPack(t *testing.T, ds kolide.Datastore) {
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
q2 := test.NewQuery(t, ds, "bar", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
queries, err := ds.ListScheduledQueriesInPack(p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, 1)
assert.Equal(t, uint(60), queries[0].Interval)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, false, false)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, true, false)
queries, err = ds.ListScheduledQueriesInPack(p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, 3)
}
func testSaveScheduledQuery(t *testing.T, ds kolide.Datastore) {
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := ds.ScheduledQuery(sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
query.Interval = uint(120)
query, err = ds.SaveScheduledQuery(query)
require.Nil(t, err)
assert.Equal(t, uint(120), query.Interval)
queryVerify, err := ds.ScheduledQuery(sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(120), queryVerify.Interval)
}
func testScheduledQueryWithDeletedPack(t *testing.T, ds kolide.Datastore) {
// When a pack is soft-deleted, it should not appear in the list of
// packs associated with a query.
if ds.Name() == "inmem" {
t.Skip("inmem is being deprecated, test skipped")
zwass := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
queries := []*kolide.Query{
{Name: "foo", Description: "get the foos", Query: "select * from foo"},
{Name: "bar", Description: "do some bars", Query: "select baz from bar"},
}
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
query := test.NewQuery(t, ds, "q1", "select 1", user.ID, true)
pack := test.NewPack(t, ds, "foobar_pack")
test.NewScheduledQuery(t, ds, pack.ID, query.ID, 60, false, false)
actual, err := ds.Query(query.ID)
err := ds.ApplyQueries(zwass.ID, queries)
require.Nil(t, err)
assert.Equal(t, "q1", actual.Name)
assert.Equal(t, "select 1", actual.Query)
assert.Equal(t, []kolide.Pack{*pack}, actual.Packs)
require.Nil(t, ds.DeletePack(pack.ID))
actual, err = ds.Query(query.ID)
specs := []*kolide.PackSpec{
&kolide.PackSpec{
Name: "baz",
Targets: kolide.PackSpecTargets{Labels: []string{}},
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Description: "test_foo",
Interval: 60,
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
assert.Equal(t, "q1", actual.Name)
assert.Equal(t, "select 1", actual.Query)
assert.Empty(t, actual.Packs)
gotQueries, err := ds.ListScheduledQueriesInPack(1, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, gotQueries, 1)
assert.Equal(t, uint(60), gotQueries[0].Interval)
assert.Equal(t, "test_foo", gotQueries[0].Description)
assert.Equal(t, "select * from foo", gotQueries[0].Query)
boolPtr := func(b bool) *bool { return &b }
specs = []*kolide.PackSpec{
&kolide.PackSpec{
Name: "baz",
Targets: kolide.PackSpecTargets{Labels: []string{}},
Queries: []kolide.PackSpecQuery{
kolide.PackSpecQuery{
QueryName: queries[0].Name,
Description: "test_foo",
Interval: 60,
},
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Name: "test bar",
Description: "test_bar",
Interval: 60,
},
kolide.PackSpecQuery{
QueryName: queries[1].Name,
Name: "test bar snapshot",
Description: "test_bar",
Interval: 60,
Snapshot: boolPtr(true),
},
},
},
}
err = ds.ApplyPackSpecs(specs)
require.Nil(t, err)
gotQueries, err = ds.ListScheduledQueriesInPack(1, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, gotQueries, 3)
}

8
server/datastore/datastore_test.go
View file

@ -52,16 +52,11 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testCleanupDistributedQueryCampaigns,
testBuiltInLabels,
testLoadPacksForQueries,
testScheduledQuery,
testDeleteScheduledQuery,
testListScheduledQueriesInPack,
testSaveScheduledQuery,
testOptions,
testNewScheduledQuery,
testOptionsToConfig,
testGetPackByName,
testGetQueryByName,
testScheduledQueryWithDeletedPack,
testDecorators,
testFileIntegrityMonitoring,
testYARAStore,
@ -82,4 +77,7 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testApplyOsqueryOptions,
testApplyOsqueryOptionsNoOverrides,
testOsqueryOptionsForHost,
testApplyQueries,
testApplyPackSpecRoundtrip,
testApplyPackSpecMissingQueries,
}

31
server/datastore/inmem/inmem.go
View file

@ -39,8 +39,14 @@ type Datastore struct {
yaraSignatureGroups map[uint]*kolide.YARASignatureGroup
appConfig *kolide.AppConfig
config *config.KolideConfig
// Embedded interfaces to avoid implementing new methods for (now
// deprecated) inmem.
kolide.TargetStore
kolide.OsqueryOptionsStore
kolide.QueryStore
kolide.PackStore
kolide.ScheduledQueryStore
}
func New(config config.KolideConfig) (*Datastore, error) {
@ -243,31 +249,6 @@ func (d *Datastore) createDevPacksAndQueries() error {
return err
}
_, err = d.NewScheduledQuery(&kolide.ScheduledQuery{
QueryID: query1.ID,
PackID: pack1.ID,
Interval: 60,
})
if err != nil {
return err
}
t := true
_, err = d.NewScheduledQuery(&kolide.ScheduledQuery{
QueryID: query3.ID,
PackID: pack1.ID,
Interval: 60,
Snapshot: &t,
})
if err != nil {
return err
}
_, err = d.NewScheduledQuery(&kolide.ScheduledQuery{
QueryID: query2.ID,
PackID: pack2.ID,
Interval: 60,
})
return err
}

2
server/datastore/inmem/queries.go
View file

@ -158,7 +158,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error {
for _, q := range queries {
q.Packs = make([]kolide.Pack, 0)
for _, sq := range d.scheduledQueries {
if sq.QueryID == q.ID {
if sq.QueryName == q.Name {
q.Packs = append(q.Packs, *d.packs[sq.PackID])
}
}

113
server/datastore/inmem/scheduled_queries.go
View file

@ -1,113 +0,0 @@
package inmem
import (
"sort"
"github.com/kolide/fleet/server/kolide"
)
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery, opts ...kolide.OptionalArg) (*kolide.ScheduledQuery, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
newScheduledQuery := *sq
newScheduledQuery.ID = d.nextID(newScheduledQuery)
d.scheduledQueries[newScheduledQuery.ID] = &newScheduledQuery
newScheduledQuery.Query = d.queries[newScheduledQuery.QueryID].Query
newScheduledQuery.Name = d.queries[newScheduledQuery.QueryID].Name
return &newScheduledQuery, nil
}
func (d *Datastore) SaveScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.scheduledQueries[sq.ID]; !ok {
return nil, notFound("ScheduledQuery").WithID(sq.ID)
}
d.scheduledQueries[sq.ID] = sq
return sq, nil
}
func (d *Datastore) DeleteScheduledQuery(id uint) error {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.scheduledQueries[id]; !ok {
return notFound("ScheduledQuery").WithID(id)
}
delete(d.scheduledQueries, id)
return nil
}
func (d *Datastore) ScheduledQuery(id uint) (*kolide.ScheduledQuery, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
sq, ok := d.scheduledQueries[id]
if !ok {
return nil, notFound("ScheduledQuery").WithID(id)
}
sq.Name = d.queries[sq.QueryID].Name
sq.Query = d.queries[sq.QueryID].Query
return sq, nil
}
func (d *Datastore) ListScheduledQueriesInPack(id uint, opt kolide.ListOptions) ([]*kolide.ScheduledQuery, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
// We need to sort by keys to provide reliable ordering
keys := []int{}
for k, sq := range d.scheduledQueries {
if sq.PackID == id {
keys = append(keys, int(k))
}
}
if len(keys) == 0 {
return []*kolide.ScheduledQuery{}, nil
}
sort.Ints(keys)
scheduledQueries := []*kolide.ScheduledQuery{}
for _, k := range keys {
q := d.scheduledQueries[uint(k)]
q.Name = d.queries[q.QueryID].Name
q.Query = d.queries[q.QueryID].Query
scheduledQueries = append(scheduledQueries, q)
}
// Apply ordering
if opt.OrderKey != "" {
var fields = map[string]string{
"id": "ID",
"created_at": "CreatedAt",
"updated_at": "UpdatedAt",
"name": "Name",
"query": "Query",
"interval": "Interval",
"snapshot": "Snapshot",
"removed": "Removed",
"platform": "Platform",
"version": "Version",
}
if err := sortResults(scheduledQueries, opt, fields); err != nil {
return nil, err
}
}
// Apply limit/offset
low, high := d.getLimitOffsetSliceBounds(opt, len(scheduledQueries))
scheduledQueries = scheduledQueries[low:high]
return scheduledQueries, nil
}

74
server/datastore/mysql/migrations/tables/20171219164727_UpdateScheduledQueryKey.go Normal file
View file

@ -0,0 +1,74 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20171219164727, Down_20171219164727)
}
func Up_20171219164727(tx *sql.Tx) error {
// Add query name column
query := `
ALTER TABLE scheduled_queries
ADD COLUMN query_name varchar(255)
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "adding query_name column")
}
// Populate query name column via join with query ID
query = `
UPDATE scheduled_queries
SET query_name = (SELECT name from queries where id = query_id)
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "populating query_name column")
}
// Add not null constraint
query = `
ALTER TABLE scheduled_queries
MODIFY query_name varchar(255) NOT NULL
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "adding not null")
}
// Add foreign key constraint
query = `
ALTER TABLE scheduled_queries
ADD FOREIGN KEY (query_name) REFERENCES queries (name)
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "adding foreign key to query_name column")
}
// Add `name` column to scheduled_queries
query = `
ALTER TABLE scheduled_queries
ADD COLUMN name varchar(255),
ADD COLUMN description varchar(255)
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "adding name to scheduled_queries")
}
query = `
ALTER TABLE packs
DROP COLUMN created_by
`
if _, err := tx.Exec(query); err != nil {
return errors.Wrap(err, "removing created_by from packs")
}
return nil
}
func Down_20171219164727(tx *sql.Tx) error {
return nil
}

13
server/datastore/mysql/mysql.go
View file

@ -282,3 +282,16 @@ func generateMysqlConnectionString(conf config.MysqlConfig) string {
return dsn
}
// isForeignKeyError checks if the provided error is a MySQL child foreign key
// error (Error #1452)
func isChildForeignKeyError(err error) bool {
mysqlErr, ok := err.(*mysql.MySQLError)
if !ok {
return false
}
// https://dev.mysql.com/doc/refman/5.7/en/error-messages-server.html#error_er_no_referenced_row_2
const ER_NO_REFERENCED_ROW_2 = 1452
return mysqlErr.Number == ER_NO_REFERENCED_ROW_2
}

186
server/datastore/mysql/packs.go
View file

@ -4,10 +4,186 @@ import (
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) ApplyPackSpecs(specs []*kolide.PackSpec) (err error) {
tx, err := d.db.Beginx()
if err != nil {
return errors.Wrap(err, "begin ApplyPackSpec transaction")
}
defer func() {
if err != nil {
rbErr := tx.Rollback()
// It seems possible that there might be a case in
// which the error we are dealing with here was thrown
// by the call to tx.Commit(), and the docs suggest
// this call would then result in sql.ErrTxDone.
if rbErr != nil && rbErr != sql.ErrTxDone {
panic(fmt.Sprintf("got err '%s' rolling back after err '%s'", rbErr, err))
}
}
}()
for _, spec := range specs {
err = applyPackSpec(tx, spec)
if err != nil {
return errors.Wrapf(err, "applying pack '%s'", spec.Name)
}
}
err = tx.Commit()
return errors.Wrap(err, "commit transaction")
}
func applyPackSpec(tx *sqlx.Tx, spec *kolide.PackSpec) error {
// Insert/update pack
query := `
INSERT INTO packs (name, description, platform)
VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE
name = VALUES(name),
description = VALUES(description),
platform = VALUES(platform),
deleted = false
`
if _, err := tx.Exec(query, spec.Name, spec.Description, spec.Platform); err != nil {
return errors.Wrap(err, "insert/update pack")
}
// Get Pack ID
// This is necessary because MySQL last_insert_id does not return a value
// if no update was made.
var packID uint
query = "SELECT id FROM packs WHERE name = ?"
if err := tx.Get(&packID, query, spec.Name); err != nil {
return errors.Wrap(err, "getting pack ID")
}
// Delete existing scheduled queries for pack
query = "DELETE FROM scheduled_queries WHERE pack_id = ?"
if _, err := tx.Exec(query, packID); err != nil {
return errors.Wrap(err, "delete existing scheduled queries")
}
// Insert new scheduled queries for pack
for _, q := range spec.Queries {
query = `
INSERT INTO scheduled_queries (
pack_id, query_name, name, description, ` + "`interval`" + `,
snapshot, removed, shard, platform, version
)
VALUES (
?, ?, ?, ?, ?,
?, ?, ?, ?, ?
)
`
_, err := tx.Exec(query,
packID, q.QueryName, q.Name, q.Description, q.Interval,
q.Snapshot, q.Removed, q.Shard, q.Platform, q.Version,
)
switch {
case isChildForeignKeyError(err):
return errors.Errorf("cannot schedule unknown query '%s'", q.QueryName)
case err != nil:
return errors.Wrapf(err, "adding query %s referencing %s", q.Name, q.QueryName)
}
}
// Delete existing targets
query = "DELETE FROM pack_targets WHERE pack_id = ?"
if _, err := tx.Exec(query, packID); err != nil {
return errors.Wrap(err, "delete existing targets")
}
// Insert targets
for _, l := range spec.Targets.Labels {
query = `
INSERT INTO pack_targets (pack_id, type, target_id)
VALUES (?, ?, (SELECT id FROM labels WHERE name = ?))
`
if _, err := tx.Exec(query, packID, kolide.TargetLabel, l); err != nil {
return errors.Wrap(err, "adding label to pack")
}
}
return nil
}
func (d *Datastore) GetPackSpecs() (specs []*kolide.PackSpec, err error) {
tx, err := d.db.Beginx()
if err != nil {
return nil, errors.Wrap(err, "begin GetPackSpecs transaction")
}
defer func() {
if err != nil {
rbErr := tx.Rollback()
// It seems possible that there might be a case in
// which the error we are dealing with here was thrown
// by the call to tx.Commit(), and the docs suggest
// this call would then result in sql.ErrTxDone.
if rbErr != nil && rbErr != sql.ErrTxDone {
panic(fmt.Sprintf("got err '%s' rolling back after err '%s'", rbErr, err))
}
}
}()
// Get basic specs
query := "SELECT id, name, description, platform FROM packs"
if err := tx.Select(&specs, query); err != nil {
fmt.Println(err)
return nil, errors.Wrap(err, "get packs")
}
// Load targets
for _, spec := range specs {
query = `
SELECT l.name
FROM labels l JOIN pack_targets pt
WHERE pack_id = ? AND pt.type = ? AND pt.target_id = l.id
`
if err := tx.Select(&spec.Targets.Labels, query, spec.ID, kolide.TargetLabel); err != nil {
return nil, errors.Wrap(err, "get pack targets")
}
}
// query = `
// INSERT INTO scheduled_queries (
// pack_id, query_name, name, description, ` + "`interval`" + `,
// snapshot, removed, shard, platform, version
// )
// VALUES (
// ?, ?, ?, ?, ?,
// ?, ?, ?, ?, ?
// )
// `
// Load queries
for _, spec := range specs {
query = `
SELECT
query_name, name, description, ` + "`interval`" + `,
snapshot, removed, shard, platform, version
FROM scheduled_queries
WHERE pack_id = ?
`
if err := tx.Select(&spec.Queries, query, spec.ID); err != nil {
return nil, errors.Wrap(err, "get pack queries")
}
}
err = tx.Commit()
if err != nil {
return nil, errors.Wrap(err, "commit transaction")
}
return specs, nil
}
func (d *Datastore) PackByName(name string, opts ...kolide.OptionalArg) (*kolide.Pack, bool, error) {
db := d.getTransaction(opts)
sqlStatement := `
@ -40,21 +216,21 @@ func (d *Datastore) NewPack(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kol
case nil:
query = `
REPLACE INTO packs
( name, description, platform, created_by, disabled, deleted)
VALUES ( ?, ?, ?, ?, ?, ?)
( name, description, platform, disabled, deleted)
VALUES ( ?, ?, ?, ?, ?)
`
case sql.ErrNoRows:
query = `
INSERT INTO packs
( name, description, platform, created_by, disabled, deleted)
VALUES ( ?, ?, ?, ?, ?, ?)
( name, description, platform, disabled, deleted)
VALUES ( ?, ?, ?, ?, ?)
`
default:
return nil, errors.Wrap(err, "check for existing pack")
}
deleted := false
result, err := db.Exec(query, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled, deleted)
result, err := db.Exec(query, pack.Name, pack.Description, pack.Platform, pack.Disabled, deleted)
if err != nil && isDuplicate(err) {
return nil, alreadyExists("Pack", deletedPack.ID)
} else if err != nil {

77
server/datastore/mysql/queries.go
View file

@ -2,12 +2,65 @@ package mysql
import (
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) ApplyQueries(authorID uint, queries []*kolide.Query) (err error) {
tx, err := d.db.Begin()
if err != nil {
return errors.Wrap(err, "begin ApplyQueries transaction")
}
defer func() {
if err != nil {
rbErr := tx.Rollback()
// It seems possible that there might be a case in
// which the error we are dealing with here was thrown
// by the call to tx.Commit(), and the docs suggest
// this call would then result in sql.ErrTxDone.
if rbErr != nil && rbErr != sql.ErrTxDone {
panic(fmt.Sprintf("got err '%s' rolling back after err '%s'", rbErr, err))
}
}
}()
sql := `
INSERT INTO queries (
name,
description,
query,
author_id,
saved,
deleted
) VALUES ( ?, ?, ?, ?, true, false )
ON DUPLICATE KEY UPDATE
name = VALUES(name),
description = VALUES(description),
query = VALUES(query),
author_id = VALUES(author_id),
saved = VALUES(saved),
deleted = VALUES(deleted)
`
stmt, err := tx.Prepare(sql)
if err != nil {
return errors.Wrap(err, "prepare ApplyQueries insert")
}
for _, q := range queries {
_, err := stmt.Exec(q.Name, q.Description, q.Query, authorID)
if err != nil {
return errors.Wrap(err, "exec ApplyQueries insert")
}
}
err = tx.Commit()
return errors.Wrap(err, "commit ApplyQueries transaction")
}
func (d *Datastore) QueryByName(name string, opts ...kolide.OptionalArg) (*kolide.Query, bool, error) {
db := d.getTransaction(opts)
sqlStatement := `
@ -23,6 +76,11 @@ func (d *Datastore) QueryByName(name string, opts ...kolide.OptionalArg) (*kolid
}
return nil, false, errors.Wrap(err, "selecting query by name")
}
if err := d.loadPacksForQueries([]*kolide.Query{&query}); err != nil {
return nil, false, errors.Wrap(err, "loading packs for query")
}
return &query, true, nil
}
@ -175,7 +233,6 @@ func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error)
}
return results, nil
}
// loadPacksForQueries loads the packs associated with the provided queries
@ -185,31 +242,31 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error {
}
sql := `
SELECT p.*, sq.query_id AS query_id
SELECT p.*, sq.query_name AS query_name
FROM packs p
JOIN scheduled_queries sq
ON p.id = sq.pack_id
WHERE query_id IN (?)
WHERE query_name IN (?)
AND NOT p.deleted
`
// Used to map the results
id_queries := map[uint]*kolide.Query{}
name_queries := map[string]*kolide.Query{}
// Used for the IN clause
ids := []uint{}
names := []string{}
for _, q := range queries {
q.Packs = make([]kolide.Pack, 0)
ids = append(ids, q.ID)
id_queries[q.ID] = q
names = append(names, q.Name)
name_queries[q.Name] = q
}
query, args, err := sqlx.In(sql, ids)
query, args, err := sqlx.In(sql, names)
if err != nil {
return errors.Wrap(err, "building query in load packs for queries")
}
rows := []struct {
QueryID uint `db:"query_id"`
QueryName string `db:"query_name"`
kolide.Pack
}{}
@ -219,7 +276,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error {
}
for _, row := range rows {
q := id_queries[row.QueryID]
q := name_queries[row.QueryName]
q.Packs = append(q.Packs, row.Pack)
}

95
server/datastore/mysql/scheduled_queries.go
View file

@ -1,104 +1,19 @@
package mysql
import (
"database/sql"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery, opts ...kolide.OptionalArg) (*kolide.ScheduledQuery, error) {
db := d.getTransaction(opts)
query := `
INSERT INTO scheduled_queries (
pack_id,
query_id,
snapshot,
removed,
` + "`interval`" + `,
platform,
version,
shard
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`
result, err := db.Exec(query, sq.PackID, sq.QueryID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard)
if err != nil {
return nil, errors.Wrap(err, "inserting scheduled query")
}
id, _ := result.LastInsertId()
sq.ID = uint(id)
query = `SELECT query, name FROM queries WHERE id = ? LIMIT 1`
metadata := []struct {
Query string
Name string
}{}
err = db.Select(&metadata, query, sq.QueryID)
if err != nil && err == sql.ErrNoRows {
return nil, notFound("Query").WithID(sq.QueryID)
} else if err != nil {
return nil, errors.Wrap(err, "select query by ID")
}
if len(metadata) != 1 {
return nil, errors.Wrap(err, "wrong number of results returned from database")
}
sq.Query = metadata[0].Query
sq.Name = metadata[0].Name
return sq, nil
}
func (d *Datastore) SaveScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
query := `
UPDATE scheduled_queries
SET pack_id = ?, query_id = ?, ` + "`interval`" + ` = ?, snapshot = ?, removed = ?, platform = ?, version = ?, shard = ?
WHERE id = ? AND NOT deleted
`
result, err := d.db.Exec(query, sq.PackID, sq.QueryID, sq.Interval, sq.Snapshot, sq.Removed, sq.Platform, sq.Version, sq.Shard, sq.ID)
if err != nil {
return nil, errors.Wrap(err, "saving a scheduled query")
}
rows, err := result.RowsAffected()
if err != nil {
return nil, errors.Wrap(err, "rows affected saving a scheduled query")
}
if rows == 0 {
return nil, notFound("ScheduledQueries").WithID(sq.ID)
}
return sq, nil
}
func (d *Datastore) DeleteScheduledQuery(id uint) error {
return d.deleteEntity("scheduled_queries", id)
}
func (d *Datastore) ScheduledQuery(id uint) (*kolide.ScheduledQuery, error) {
query := `
SELECT sq.*, q.query, q.name
FROM scheduled_queries sq
JOIN queries q
ON sq.query_id = q.id
WHERE sq.id = ?
AND NOT sq.deleted
`
sq := &kolide.ScheduledQuery{}
if err := d.db.Get(sq, query, id); err != nil {
return nil, errors.Wrap(err, "selecting a scheduled query")
}
return sq, nil
}
func (d *Datastore) ListScheduledQueriesInPack(id uint, opts kolide.ListOptions) ([]*kolide.ScheduledQuery, error) {
query := `
SELECT sq.*, q.query, q.name
SELECT
sq.id, sq.pack_id, sq.name, sq.query_name, q.query,
sq.description, sq.interval, sq.snapshot, sq.removed, sq.platform,
sq.version, sq.shard
FROM scheduled_queries sq
JOIN queries q
ON sq.query_id = q.id
ON sq.query_name = q.name
WHERE sq.pack_id = ?
AND NOT sq.deleted
`

29
server/kolide/packs.go
View file

@ -6,6 +6,9 @@ import (
// PackStore is the datastore interface for managing query packs.
type PackStore interface {
ApplyPackSpecs(specs []*PackSpec) error
GetPackSpecs() ([]*PackSpec, error)
// NewPack creates a new pack in the datastore.
NewPack(pack *Pack, opts ...OptionalArg) (*Pack, error)
@ -104,7 +107,6 @@ type Pack struct {
Name string `json:"name"`
Description string `json:"description"`
Platform string `json:"platform"`
CreatedBy uint `json:"created_by" db:"created_by"`
Disabled bool `json:"disabled"`
}
@ -118,6 +120,31 @@ type PackPayload struct {
LabelIDs *[]uint `json:"label_ids"`
}
type PackSpec struct {
ID uint
Name string `json:"name"`
Description string `json:"description"`
Platform string `json:"platform"`
Targets PackSpecTargets `json:"targets"`
Queries []PackSpecQuery `json:"queries"`
}
type PackSpecTargets struct {
Labels []string `json:"labels"`
}
type PackSpecQuery struct {
QueryName string `json:"query" db:"query_name"`
Name string `json:"name"`
Description string `json:"description"`
Interval uint `json:"interval"`
Snapshot *bool `json:"snapshot,omitempty"`
Removed *bool `json:"removed,omitempty"`
Shard *uint `json:"shard,omitempty"`
Platform *string `json:"platform,omitempty"`
Version *string `json:"version,omitempty"`
}
// PackTarget associates a pack with either a host or a label
type PackTarget struct {
ID uint

76
server/kolide/queries.go
View file

@ -2,9 +2,19 @@ package kolide
import (
"context"
"strings"
"github.com/pkg/errors"
"github.com/ghodss/yaml"
)
type QueryStore interface {
// ApplyQueries applies a list of queries (likely from a yaml file) to
// the datastore. Existing queries are updated, and new queries are
// created.
ApplyQueries(authorID uint, queries []*Query) error
// NewQuery creates a new query object in thie datastore. The returned
// query should have the ID updated.
NewQuery(query *Query, opts ...OptionalArg) (*Query, error)
@ -28,6 +38,12 @@ type QueryStore interface {
}
type QueryService interface {
// ApplyQuerySpecs applies a list of queries (creating or updating
// them as necessary)
ApplyQuerySpecs(ctx context.Context, specs []*QuerySpec) error
// GetQuerySpecs gets the YAML file representing all the stored queries.
GetQuerySpecs(ctx context.Context) ([]*QuerySpec, error)
// ListQueries returns a list of saved queries. Note only saved queries
// should be returned (those that are created for distributed queries
// but not saved should not be returned).
@ -64,3 +80,63 @@ type Query struct {
// table in the MySQL backend.
Packs []Pack `json:"packs" db:"-"`
}
const (
QueryKind = "Query"
)
type QueryObject struct {
ObjectMetadata
Spec QuerySpec `json:"spec"`
}
type QuerySpec struct {
Name string `json:"name"`
Description string `json:"description"`
Query string `json:"query"`
}
func LoadQueriesFromYaml(yml string) ([]*Query, error) {
queries := []*Query{}
for _, s := range strings.Split(yml, "---") {
s = strings.TrimSpace(s)
if len(s) == 0 {
continue
}
var q QueryObject
err := yaml.Unmarshal([]byte(s), &q)
if err != nil {
return nil, errors.Wrap(err, "unmarshal yaml")
}
queries = append(queries,
&Query{Name: q.Spec.Name, Description: q.Spec.Description, Query: q.Spec.Query},
)
}
return queries, nil
}
func WriteQueriesToYaml(queries []*Query) (string, error) {
ymlStrings := []string{}
for _, q := range queries {
qYaml := QueryObject{
ObjectMetadata: ObjectMetadata{
ApiVersion: ApiVersion,
Kind: QueryKind,
},
Spec: QuerySpec{
Name: q.Name,
Description: q.Description,
Query: q.Query,
},
}
yml, err := yaml.Marshal(qYaml)
if err != nil {
return "", errors.Wrap(err, "marshal YAML")
}
ymlStrings = append(ymlStrings, string(yml))
}
return strings.Join(ymlStrings, "---\n"), nil
}

104
server/kolide/queries_test.go Normal file
View file

@ -0,0 +1,104 @@
package kolide
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadQueriesFromYamlStrings(t *testing.T) {
var testCases = []struct {
yaml string
queries []*Query
shouldErr bool
}{
{"notyaml", []*Query{}, true},
{"", []*Query{}, false},
{"---", []*Query{}, false},
{
`
---
apiVersion: k8s.kolide.com/v1alpha1
kind: OsqueryQuery
spec:
name: osquery_version
description: The version of the Launcher and Osquery process
query: select launcher.version, osquery.version from kolide_launcher_info
support:
launcher: 0.3.0
osquery: 2.9.0
---
apiVersion: k8s.kolide.com/v1alpha1
kind: OsqueryQuery
spec:
name: osquery_schedule
description: Report performance stats for each file in the query schedule.
query: select name, interval, executions, output_size, wall_time, (user_time
---
apiVersion: k8s.kolide.com/v1alpha1
kind: OsqueryQuery
spec:
name: foobar
description: froobing
query: select fizz from frog
`,
[]*Query{
&Query{
Name: "osquery_version",
Description: "The version of the Launcher and Osquery process",
Query: "select launcher.version, osquery.version from kolide_launcher_info",
},
&Query{
Name: "osquery_schedule",
Description: "Report performance stats for each file in the query schedule.",
Query: "select name, interval, executions, output_size, wall_time, (user_time",
},
&Query{
Name: "foobar",
Description: "froobing",
Query: "select fizz from frog",
},
},
false,
},
}
for _, tt := range testCases {
t.Run("", func(t *testing.T) {
queries, err := LoadQueriesFromYaml(tt.yaml)
if tt.shouldErr {
require.NotNil(t, err)
} else {
require.Nil(t, err)
assert.Equal(t, tt.queries, queries)
}
})
}
}
func TestRoundtripQueriesYaml(t *testing.T) {
var testCases = []struct{ queries []*Query }{
{[]*Query{{Name: "froob", Description: "bing", Query: "blong"}}},
{
[]*Query{
{Name: "froob", Description: "bing", Query: "blong"},
{Name: "mant", Description: "smump", Query: "tmit"},
{Name: "gorm", Description: "", Query: "blirz"},
{Name: "blob", Description: "shmoo", Query: "smarle"},
},
},
}
for _, tt := range testCases {
t.Run("", func(t *testing.T) {
yml, err := WriteQueriesToYaml(tt.queries)
require.Nil(t, err)
queries, err := LoadQueriesFromYaml(yml)
require.Nil(t, err)
assert.Equal(t, tt.queries, queries)
})
}
}

31
server/kolide/scheduled_queries.go
View file

@ -5,35 +5,28 @@ import (
)
type ScheduledQueryStore interface {
NewScheduledQuery(sq *ScheduledQuery, opts ...OptionalArg) (*ScheduledQuery, error)
SaveScheduledQuery(sq *ScheduledQuery) (*ScheduledQuery, error)
DeleteScheduledQuery(id uint) error
ScheduledQuery(id uint) (*ScheduledQuery, error)
ListScheduledQueriesInPack(id uint, opts ListOptions) ([]*ScheduledQuery, error)
}
type ScheduledQueryService interface {
GetScheduledQuery(ctx context.Context, id uint) (query *ScheduledQuery, err error)
GetScheduledQueriesInPack(ctx context.Context, id uint, opts ListOptions) (queries []*ScheduledQuery, err error)
ScheduleQuery(ctx context.Context, sq *ScheduledQuery) (query *ScheduledQuery, err error)
DeleteScheduledQuery(ctx context.Context, id uint) (err error)
ModifyScheduledQuery(ctx context.Context, id uint, p ScheduledQueryPayload) (query *ScheduledQuery, err error)
}
type ScheduledQuery struct {
UpdateCreateTimestamps
DeleteFields
ID uint `json:"id"`
PackID uint `json:"pack_id" db:"pack_id"`
QueryID uint `json:"query_id" db:"query_id"`
Query string `json:"query"` // populated via a join on queries
Name string `json:"name"` // populated via a join on queries
Interval uint `json:"interval"`
Snapshot *bool `json:"snapshot"`
Removed *bool `json:"removed"`
Platform *string `json:"platform"`
Version *string `json:"version"`
Shard *uint `json:"shard"`
ID uint `json:"id"`
PackID uint `json:"pack_id" db:"pack_id"`
Name string `json:"name"`
QueryName string `json:"query_name" db:"query_name"`
Query string `json:"query"` // populated via a join on queries
Description string `json:"description"`
Interval uint `json:"interval"`
Snapshot *bool `json:"snapshot"`
Removed *bool `json:"removed"`
Platform *string `json:"platform"`
Version *string `json:"version"`
Shard *uint `json:"shard"`
}
type ScheduledQueryPayload struct {

1
server/kolide/service.go
View file

@ -15,7 +15,6 @@ type Service interface {
TargetService
ScheduledQueryService
OptionService
ImportConfigService
DecoratorService
FileIntegrityMonitoringService
}

3
server/mock/datastore.go
View file

@ -11,6 +11,7 @@ package mock
//go:generate mockimpl -o datastore_fim.go "s *FileIntegrityMonitoringStore" "kolide.FileIntegrityMonitoringStore"
//go:generate mockimpl -o datastore_osquery_options.go "s *OsqueryOptionsStore" "kolide.OsqueryOptionsStore"
//go:generate mockimpl -o datastore_scheduled_queries.go "s *ScheduledQueryStore" "kolide.ScheduledQueryStore"
//go:generate mockimpl -o datastore_queries.go "s *QueryStore" "kolide.QueryStore"
import "github.com/kolide/fleet/server/kolide"
@ -20,7 +21,6 @@ type Store struct {
kolide.CampaignStore
kolide.SessionStore
kolide.PasswordResetStore
kolide.QueryStore
kolide.YARAStore
kolide.TargetStore
ScheduledQueryStore
@ -34,6 +34,7 @@ type Store struct {
OptionStore
PackStore
UserStore
QueryStore
}
func (m *Store) Drop() error {

20
server/mock/datastore_packs.go
View file

@ -6,6 +6,10 @@ import "github.com/kolide/fleet/server/kolide"
var _ kolide.PackStore = (*PackStore)(nil)
type ApplyPackSpecsFunc func(specs []*kolide.PackSpec) error
type GetPackSpecsFunc func() ([]*kolide.PackSpec, error)
type NewPackFunc func(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error)
type SavePackFunc func(pack *kolide.Pack) error
@ -33,6 +37,12 @@ type ListHostsInPackFunc func(pid uint, opt kolide.ListOptions) ([]uint, error)
type ListExplicitHostsInPackFunc func(pid uint, opt kolide.ListOptions) ([]uint, error)
type PackStore struct {
ApplyPackSpecsFunc ApplyPackSpecsFunc
ApplyPackSpecsFuncInvoked bool
GetPackSpecsFunc GetPackSpecsFunc
GetPackSpecsFuncInvoked bool
NewPackFunc NewPackFunc
NewPackFuncInvoked bool
@ -73,6 +83,16 @@ type PackStore struct {
ListExplicitHostsInPackFuncInvoked bool
}
func (s *PackStore) ApplyPackSpecs(specs []*kolide.PackSpec) error {
s.ApplyPackSpecsFuncInvoked = true
return s.ApplyPackSpecsFunc(specs)
}
func (s *PackStore) GetPackSpecs() ([]*kolide.PackSpec, error) {
s.GetPackSpecsFuncInvoked = true
return s.GetPackSpecsFunc()
}
func (s *PackStore) NewPack(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error) {
s.NewPackFuncInvoked = true
return s.NewPackFunc(pack, opts...)

89
server/mock/datastore_queries.go Normal file
View file

@ -0,0 +1,89 @@
// Automatically generated by mockimpl. DO NOT EDIT!
package mock
import "github.com/kolide/fleet/server/kolide"
var _ kolide.QueryStore = (*QueryStore)(nil)
type ApplyQueriesFunc func(authorID uint, queries []*kolide.Query) error
type NewQueryFunc func(query *kolide.Query, opts ...kolide.OptionalArg) (*kolide.Query, error)
type SaveQueryFunc func(query *kolide.Query) error
type DeleteQueryFunc func(qid uint) error
type DeleteQueriesFunc func(ids []uint) (uint, error)
type QueryFunc func(id uint) (*kolide.Query, error)
type ListQueriesFunc func(opt kolide.ListOptions) ([]*kolide.Query, error)
type QueryByNameFunc func(name string, opts ...kolide.OptionalArg) (*kolide.Query, bool, error)
type QueryStore struct {
ApplyQueriesFunc ApplyQueriesFunc
ApplyQueriesFuncInvoked bool
NewQueryFunc NewQueryFunc
NewQueryFuncInvoked bool
SaveQueryFunc SaveQueryFunc
SaveQueryFuncInvoked bool
DeleteQueryFunc DeleteQueryFunc
DeleteQueryFuncInvoked bool
DeleteQueriesFunc DeleteQueriesFunc
DeleteQueriesFuncInvoked bool
QueryFunc QueryFunc
QueryFuncInvoked bool
ListQueriesFunc ListQueriesFunc
ListQueriesFuncInvoked bool
QueryByNameFunc QueryByNameFunc
QueryByNameFuncInvoked bool
}
func (s *QueryStore) ApplyQueries(authorID uint, queries []*kolide.Query) error {
s.ApplyQueriesFuncInvoked = true
return s.ApplyQueriesFunc(authorID, queries)
}
func (s *QueryStore) NewQuery(query *kolide.Query, opts ...kolide.OptionalArg) (*kolide.Query, error) {
s.NewQueryFuncInvoked = true
return s.NewQueryFunc(query, opts...)
}
func (s *QueryStore) SaveQuery(query *kolide.Query) error {
s.SaveQueryFuncInvoked = true
return s.SaveQueryFunc(query)
}
func (s *QueryStore) DeleteQuery(qid uint) error {
s.DeleteQueryFuncInvoked = true
return s.DeleteQueryFunc(qid)
}
func (s *QueryStore) DeleteQueries(ids []uint) (uint, error) {
s.DeleteQueriesFuncInvoked = true
return s.DeleteQueriesFunc(ids)
}
func (s *QueryStore) Query(id uint) (*kolide.Query, error) {
s.QueryFuncInvoked = true
return s.QueryFunc(id)
}
func (s *QueryStore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) {
s.ListQueriesFuncInvoked = true
return s.ListQueriesFunc(opt)
}
func (s *QueryStore) QueryByName(name string, opts ...kolide.OptionalArg) (*kolide.Query, bool, error) {
s.QueryByNameFuncInvoked = true
return s.QueryByNameFunc(name, opts...)
}

43
server/service/endpoint_import_config.go
View file

@ -1,43 +0,0 @@
package service
import (
"context"
"github.com/go-kit/kit/endpoint"
"github.com/kolide/fleet/server/kolide"
)
type importRequest struct {
DryRun bool `json:"dry_run"`
// Config contains a JSON osquery config supplied by the end user
Config string `json:"config"`
// ExternalPackConfigs contains a map of external Pack configs keyed by
// Pack name, this includes external packs referenced by the globbing
// feature. Not in the case of globbed packs, we expect the user to
// generate unique pack names since we don't know what they are, these
// names must be included in the GlobPackNames field so that we can
// validate that they've been accounted for.
ExternalPackConfigs map[string]string `json:"external_pack_configs"`
// GlobPackNames list of user generated names for external packs
// referenced by the glob feature, the JSON for the globbed packs
// is stored in ExternalPackConfigs keyed by the GlobPackName
GlobPackNames []string `json:"glob_pack_names"`
}
type importResponse struct {
Response *kolide.ImportConfigResponse `json:"response,omitempty"`
Err error `json:"error,omitempty"`
}
func (ir importResponse) error() error { return ir.Err }
func makeImportConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
config := request.(kolide.ImportConfig)
resp, err := svc.ImportConfig(ctx, &config)
if err != nil {
return importResponse{Err: err}, nil
}
return importResponse{Response: resp}, nil
}
}

157
server/service/endpoint_import_config_test.go
View file

@ -1,157 +0,0 @@
package service
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testImportConfigWithGlob(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"glob\":[\"globpack\"],\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}",
"globpack": "{\"discovery\":[\"select pid from processes where name = 'zip';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from other;\",\"interval\":1200,\"description\":\"Check other.\"}}}"
},
"glob_pack_names": ["globpack"]
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var impResponse importResponse
err = json.NewDecoder(resp.Body).Decode(&impResponse)
require.Nil(t, err)
assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.PacksSection].ImportCount)
}
func testImportConfigWithInvalidPlatform(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"foo\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"glob\":[\"globpack\"],\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}",
"globpack": "{\"discovery\":[\"select pid from processes where name = 'zip';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from other;\",\"interval\":1200,\"description\":\"Check other.\"}}}"
},
"glob_pack_names": ["globpack"]
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var v mockValidationError
err = json.NewDecoder(resp.Body).Decode(&v)
require.Nil(t, err)
require.Len(t, v.Errors, 1)
assert.Equal(t, "'foo' is not a valid platform", v.Errors[0].Reason)
}
func testImportConfigWithMissingGlob(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}"
}
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var v mockValidationError
err = json.NewDecoder(resp.Body).Decode(&v)
require.Nil(t, err)
require.Len(t, v.Errors, 1)
assert.Equal(t, "missing glob packs", v.Errors[0].Reason)
}
func testImportConfigWithIntAsString(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":\"10\"},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":\"1200\",\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}"
}
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var impResponse importResponse
err = json.NewDecoder(resp.Body).Decode(&impResponse)
require.Nil(t, err)
assert.Equal(t, 2, impResponse.Response.ImportStatusBySection[kolide.YARASigSection].ImportCount)
assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.DecoratorsSection].ImportCount)
}
func testImportConfig(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}"
}
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var impResponse importResponse
err = json.NewDecoder(resp.Body).Decode(&impResponse)
require.Nil(t, err)
assert.Equal(t, 2, impResponse.Response.ImportStatusBySection[kolide.YARASigSection].ImportCount)
assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.DecoratorsSection].ImportCount)
}
func testImportConfigMissingExternal(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3603\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}"
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var v mockValidationError
err = json.NewDecoder(resp.Body).Decode(&v)
require.Nil(t, err)
require.Len(t, v.Errors, 2)
assert.Equal(t, "missing content for 'external_pack'", v.Errors[0].Reason)
assert.Equal(t, "interval '3603' must be divisible by 60", v.Errors[1].Reason)
}

143
server/service/endpoint_scheduled_queries.go
View file

@ -7,42 +7,6 @@ import (
"github.com/kolide/fleet/server/kolide"
)
////////////////////////////////////////////////////////////////////////////////
// Get Scheduled Query
////////////////////////////////////////////////////////////////////////////////
type getScheduledQueryRequest struct {
ID uint
}
type scheduledQueryResponse struct {
kolide.ScheduledQuery
}
type getScheduledQueryResponse struct {
Scheduled scheduledQueryResponse `json:"scheduled,omitempty"`
Err error `json:"error,omitempty"`
}
func (r getScheduledQueryResponse) error() error { return r.Err }
func makeGetScheduledQueryEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(getScheduledQueryRequest)
sq, err := svc.GetScheduledQuery(ctx, req.ID)
if err != nil {
return getScheduledQueryResponse{Err: err}, nil
}
return getScheduledQueryResponse{
Scheduled: scheduledQueryResponse{
ScheduledQuery: *sq,
},
}, nil
}
}
////////////////////////////////////////////////////////////////////////////////
// Get Scheduled Queries In Pack
////////////////////////////////////////////////////////////////////////////////
@ -52,6 +16,10 @@ type getScheduledQueriesInPackRequest struct {
ListOptions kolide.ListOptions
}
type scheduledQueryResponse struct {
kolide.ScheduledQuery
}
type getScheduledQueriesInPackResponse struct {
Scheduled []scheduledQueryResponse `json:"scheduled"`
Err error `json:"error,omitempty"`
@ -78,106 +46,3 @@ func makeGetScheduledQueriesInPackEndpoint(svc kolide.Service) endpoint.Endpoint
return resp, nil
}
}
////////////////////////////////////////////////////////////////////////////////
// Schedule Query
////////////////////////////////////////////////////////////////////////////////
type scheduleQueryRequest struct {
PackID uint `json:"pack_id"`
QueryID uint `json:"query_id"`
Interval uint `json:"interval"`
Snapshot *bool `json:"snapshot"`
Removed *bool `json:"removed"`
Platform *string `json:"platform"`
Version *string `json:"version"`
Shard *uint `json:"shard"`
}
type scheduleQueryResponse struct {
Scheduled scheduledQueryResponse `json:"scheduled"`
Err error `json:"error,omitempty"`
}
func makeScheduleQueryEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(scheduleQueryRequest)
scheduled, err := svc.ScheduleQuery(ctx, &kolide.ScheduledQuery{
PackID: req.PackID,
QueryID: req.QueryID,
Interval: req.Interval,
Snapshot: req.Snapshot,
Removed: req.Removed,
Platform: req.Platform,
Version: req.Version,
Shard: req.Shard,
})
if err != nil {
return scheduleQueryResponse{Err: err}, nil
}
return scheduleQueryResponse{Scheduled: scheduledQueryResponse{
ScheduledQuery: *scheduled,
}}, nil
}
}
////////////////////////////////////////////////////////////////////////////////
// Modify Scheduled Query
////////////////////////////////////////////////////////////////////////////////
type modifyScheduledQueryRequest struct {
ID uint
payload kolide.ScheduledQueryPayload
}
type modifyScheduledQueryResponse struct {
Scheduled scheduledQueryResponse `json:"scheduled,omitempty"`
Err error `json:"error,omitempty"`
}
func (r modifyScheduledQueryResponse) error() error { return r.Err }
func makeModifyScheduledQueryEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(modifyScheduledQueryRequest)
sq, err := svc.ModifyScheduledQuery(ctx, req.ID, req.payload)
if err != nil {
return modifyScheduledQueryResponse{Err: err}, nil
}
return modifyScheduledQueryResponse{
Scheduled: scheduledQueryResponse{
ScheduledQuery: *sq,
},
}, nil
}
}
////////////////////////////////////////////////////////////////////////////////
// Delete Scheduled Query
////////////////////////////////////////////////////////////////////////////////
type deleteScheduledQueryRequest struct {
ID uint
}
type deleteScheduledQueryResponse struct {
Err error `json:"error,omitempty"`
}
func (r deleteScheduledQueryResponse) error() error { return r.Err }
func makeDeleteScheduledQueryEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(deleteScheduledQueryRequest)
err := svc.DeleteScheduledQuery(ctx, req.ID)
if err != nil {
return deleteScheduledQueryResponse{Err: err}, nil
}
return deleteScheduledQueryResponse{}, nil
}
}

6
server/service/endpoint_test.go
View file

@ -107,11 +107,6 @@ var testFunctions = [...]func(*testing.T, *testResource){
testModifyOptions,
testModifyOptionsValidationFail,
testOptionNotFound,
testImportConfig,
testImportConfigMissingExternal,
testImportConfigWithMissingGlob,
testImportConfigWithGlob,
testImportConfigWithIntAsString,
testAdminUserSetAdmin,
testNonAdminUserSetAdmin,
testAdminUserSetEnabled,
@ -123,7 +118,6 @@ var testFunctions = [...]func(*testing.T, *testResource){
testNewDecoratorFailValidation,
testDeleteDecorator,
testModifyDecoratorNoChanges,
testImportConfigWithInvalidPlatform,
}
func TestEndpoints(t *testing.T) {

26
server/service/handler.go
View file

@ -51,11 +51,7 @@ type KolideEndpoints struct {
CreatePack endpoint.Endpoint
ModifyPack endpoint.Endpoint
DeletePack endpoint.Endpoint
ScheduleQuery endpoint.Endpoint
GetScheduledQueriesInPack endpoint.Endpoint
GetScheduledQuery endpoint.Endpoint
ModifyScheduledQuery endpoint.Endpoint
DeleteScheduledQuery endpoint.Endpoint
EnrollAgent endpoint.Endpoint
GetClientConfig endpoint.Endpoint
GetDistributedQueries endpoint.Endpoint
@ -78,7 +74,6 @@ type KolideEndpoints struct {
GetOptions endpoint.Endpoint
ModifyOptions endpoint.Endpoint
ResetOptions endpoint.Endpoint
ImportConfig endpoint.Endpoint
GetCertificate endpoint.Endpoint
ChangeEmail endpoint.Endpoint
InitiateSSO endpoint.Endpoint
@ -140,11 +135,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint
CreatePack: authenticatedUser(jwtKey, svc, makeCreatePackEndpoint(svc)),
ModifyPack: authenticatedUser(jwtKey, svc, makeModifyPackEndpoint(svc)),
DeletePack: authenticatedUser(jwtKey, svc, makeDeletePackEndpoint(svc)),
ScheduleQuery: authenticatedUser(jwtKey, svc, makeScheduleQueryEndpoint(svc)),
GetScheduledQueriesInPack: authenticatedUser(jwtKey, svc, makeGetScheduledQueriesInPackEndpoint(svc)),
GetScheduledQuery: authenticatedUser(jwtKey, svc, makeGetScheduledQueryEndpoint(svc)),
ModifyScheduledQuery: authenticatedUser(jwtKey, svc, makeModifyScheduledQueryEndpoint(svc)),
DeleteScheduledQuery: authenticatedUser(jwtKey, svc, makeDeleteScheduledQueryEndpoint(svc)),
GetHost: authenticatedUser(jwtKey, svc, makeGetHostEndpoint(svc)),
ListHosts: authenticatedUser(jwtKey, svc, makeListHostsEndpoint(svc)),
GetHostSummary: authenticatedUser(jwtKey, svc, makeGetHostSummaryEndpoint(svc)),
@ -162,7 +153,6 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint
GetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeGetOptionsEndpoint(svc))),
ModifyOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeModifyOptionsEndpoint(svc))),
ResetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeResetOptionsEndpoint(svc))),
ImportConfig: authenticatedUser(jwtKey, svc, makeImportConfigEndpoint(svc)),
GetCertificate: authenticatedUser(jwtKey, svc, makeCertificateEndpoint(svc)),
ChangeEmail: authenticatedUser(jwtKey, svc, makeChangeEmailEndpoint(svc)),
GetFIM: authenticatedUser(jwtKey, svc, makeGetFIMEndpoint(svc)),
@ -214,11 +204,7 @@ type kolideHandlers struct {
CreatePack http.Handler
ModifyPack http.Handler
DeletePack http.Handler
ScheduleQuery http.Handler
GetScheduledQueriesInPack http.Handler
GetScheduledQuery http.Handler
ModifyScheduledQuery http.Handler
DeleteScheduledQuery http.Handler
EnrollAgent http.Handler
GetClientConfig http.Handler
GetDistributedQueries http.Handler
@ -241,7 +227,6 @@ type kolideHandlers struct {
GetOptions http.Handler
ModifyOptions http.Handler
ResetOptions http.Handler
ImportConfig http.Handler
GetCertificate http.Handler
ChangeEmail http.Handler
InitiateSSO http.Handler
@ -292,11 +277,7 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
CreatePack: newServer(e.CreatePack, decodeCreatePackRequest),
ModifyPack: newServer(e.ModifyPack, decodeModifyPackRequest),
DeletePack: newServer(e.DeletePack, decodeDeletePackRequest),
ScheduleQuery: newServer(e.ScheduleQuery, decodeScheduleQueryRequest),
GetScheduledQueriesInPack: newServer(e.GetScheduledQueriesInPack, decodeGetScheduledQueriesInPackRequest),
GetScheduledQuery: newServer(e.GetScheduledQuery, decodeGetScheduledQueryRequest),
ModifyScheduledQuery: newServer(e.ModifyScheduledQuery, decodeModifyScheduledQueryRequest),
DeleteScheduledQuery: newServer(e.DeleteScheduledQuery, decodeDeleteScheduledQueryRequest),
EnrollAgent: newServer(e.EnrollAgent, decodeEnrollAgentRequest),
GetClientConfig: newServer(e.GetClientConfig, decodeGetClientConfigRequest),
GetDistributedQueries: newServer(e.GetDistributedQueries, decodeGetDistributedQueriesRequest),
@ -319,7 +300,6 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
GetOptions: newServer(e.GetOptions, decodeNoParamsRequest),
ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest),
ResetOptions: newServer(e.ResetOptions, decodeNoParamsRequest),
ImportConfig: newServer(e.ImportConfig, decodeImportConfigRequest),
GetCertificate: newServer(e.GetCertificate, decodeNoParamsRequest),
ChangeEmail: newServer(e.ChangeEmail, decodeChangeEmailRequest),
InitiateSSO: newServer(e.InitiateSSO, decodeInitiateSSORequest),
@ -416,10 +396,6 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/packs/{id}", h.ModifyPack).Methods("PATCH").Name("modify_pack")
r.Handle("/api/v1/kolide/packs/{id}", h.DeletePack).Methods("DELETE").Name("delete_pack")
r.Handle("/api/v1/kolide/packs/{id}/scheduled", h.GetScheduledQueriesInPack).Methods("GET").Name("get_scheduled_queries_in_pack")
r.Handle("/api/v1/kolide/schedule", h.ScheduleQuery).Methods("POST").Name("schedule_query")
r.Handle("/api/v1/kolide/schedule/{id}", h.GetScheduledQuery).Methods("GET").Name("get_scheduled_query")
r.Handle("/api/v1/kolide/schedule/{id}", h.ModifyScheduledQuery).Methods("PATCH").Name("modify_scheduled_query")
r.Handle("/api/v1/kolide/schedule/{id}", h.DeleteScheduledQuery).Methods("DELETE").Name("delete_scheduled_query")
r.Handle("/api/v1/kolide/labels/{id}", h.GetLabel).Methods("GET").Name("get_label")
r.Handle("/api/v1/kolide/labels", h.ListLabels).Methods("GET").Name("list_labels")
r.Handle("/api/v1/kolide/labels", h.CreateLabel).Methods("POST").Name("create_label")
@ -445,8 +421,6 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/targets", h.SearchTargets).Methods("POST").Name("search_targets")
r.Handle("/api/v1/kolide/osquery/config/import", h.ImportConfig).Methods("POST").Name("import_config")
r.Handle("/api/v1/osquery/enroll", h.EnrollAgent).Methods("POST").Name("enroll_agent")
r.Handle("/api/v1/osquery/config", h.GetClientConfig).Methods("POST").Name("get_client_config")
r.Handle("/api/v1/osquery/distributed/read", h.GetDistributedQueries).Methods("POST").Name("get_distributed_queries")

12
server/service/handler_test.go
View file

@ -133,18 +133,6 @@ func TestAPIRoutes(t *testing.T) {
verb: "GET",
uri: "/api/v1/kolide/packs/1/scheduled",
},
{
verb: "POST",
uri: "/api/v1/kolide/schedule",
},
{
verb: "DELETE",
uri: "/api/v1/kolide/schedule/1",
},
{
verb: "PATCH",
uri: "/api/v1/kolide/schedule/1",
},
{
verb: "POST",
uri: "/api/v1/osquery/enroll",

27
server/service/logging_import_config.go
View file

@ -1,27 +0,0 @@
package service
import (
"context"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw loggingMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var (
resp *kolide.ImportConfigResponse
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "ImportConfig",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
resp, err = mw.Service.ImportConfig(ctx, cfg)
return resp, err
}

71
server/service/logging_scheduled_queries.go
View file

@ -7,24 +7,6 @@ import (
"github.com/kolide/fleet/server/kolide"
)
func (mw loggingMiddleware) GetScheduledQuery(ctx context.Context, id uint) (*kolide.ScheduledQuery, error) {
var (
query *kolide.ScheduledQuery
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "GetScheduledQuery",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
query, err = mw.Service.GetScheduledQuery(ctx, id)
return query, err
}
func (mw loggingMiddleware) GetScheduledQueriesInPack(ctx context.Context, id uint, opts kolide.ListOptions) ([]*kolide.ScheduledQuery, error) {
var (
queries []*kolide.ScheduledQuery
@ -42,56 +24,3 @@ func (mw loggingMiddleware) GetScheduledQueriesInPack(ctx context.Context, id ui
queries, err = mw.Service.GetScheduledQueriesInPack(ctx, id, opts)
return queries, err
}
func (mw loggingMiddleware) ScheduleQuery(ctx context.Context, sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
var (
query *kolide.ScheduledQuery
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "ScheduleQuery",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
query, err = mw.Service.ScheduleQuery(ctx, sq)
return query, err
}
func (mw loggingMiddleware) DeleteScheduledQuery(ctx context.Context, id uint) error {
var (
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "DeleteScheduledQuery",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
err = mw.Service.DeleteScheduledQuery(ctx, id)
return err
}
func (mw loggingMiddleware) ModifyScheduledQuery(ctx context.Context, id uint, p kolide.ScheduledQueryPayload) (*kolide.ScheduledQuery, error) {
var (
query *kolide.ScheduledQuery
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "ModifyScheduledQuery",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
query, err = mw.Service.ModifyScheduledQuery(ctx, id, p)
return query, err
}

23
server/service/metrics_import_config.go
View file

@ -1,23 +0,0 @@
package service
import (
"context"
"fmt"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw metricsMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var (
resp *kolide.ImportConfigResponse
err error
)
defer func(begin time.Time) {
lvs := []string{"method", "ImportConfig", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
resp, err = mw.Service.ImportConfig(ctx, cfg)
return resp, err
}

533
server/service/service_import_config.go
View file

@ -1,533 +0,0 @@
package service
import (
"context"
"crypto/md5"
"fmt"
"strconv"
"strings"
"github.com/kolide/fleet/server/contexts/viewer"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (svc service) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
vc, ok := viewer.FromContext(ctx)
if !ok {
return nil, errors.New("internal error, unable to fetch user")
}
tx, err := svc.ds.Begin()
if err != nil {
return nil, err
}
if err := svc.importOptions(cfg.Options, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importOptions")
return nil, errors.Wrap(err, "importOptions failed")
}
if err := svc.importPacks(vc.UserID(), cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importPacks")
return nil, errors.Wrap(err, "importPacks failed")
}
if err := svc.importScheduledQueries(vc.UserID(), cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importScheduledQueries")
return nil, errors.Wrap(err, "importScheduledQueries failed")
}
if err := svc.importDecorators(cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importDecorators")
return nil, errors.Wrap(err, "importDecorators")
}
if err := svc.importFIMSections(cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importFIMSections")
return nil, errors.Wrap(err, "importFIMSections")
}
if cfg.DryRun {
if err := tx.Rollback(); err != nil {
return nil, errors.Wrap(err, "dry run rollback failed")
}
return resp, nil
}
if err := tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit failed")
}
return resp, nil
}
func (svc service) rollbackImportConfig(tx kolide.Transaction, method string) {
if err := tx.Rollback(); err != nil {
svc.logger.Log(
"method", method,
"err", errors.Wrap(err, fmt.Sprintf("db rollback failed in %s", method)),
)
}
}
func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.YARA != nil {
for sig, paths := range cfg.YARA.Signatures {
ysg := &kolide.YARASignatureGroup{
SignatureName: sig,
Paths: paths,
}
_, err := svc.ds.NewYARASignatureGroup(ysg, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.YARAFileSection).SkipCount++
resp.Status(kolide.YARAFileSection).Warning(kolide.YARADuplicate, "skipped '%s', already exists", sig)
continue
}
if err != nil {
return err
}
resp.Status(kolide.YARASigSection).ImportCount++
resp.Status(kolide.YARASigSection).Message("imported '%s'", sig)
}
for section, sigs := range cfg.YARA.FilePaths {
for _, sig := range sigs {
err := svc.ds.NewYARAFilePath(section, sig, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.YARAFileSection).SkipCount++
resp.Status(kolide.YARAFileSection).Warning(kolide.YARADuplicate, "skipped '%s', already exists", section)
continue
}
if err != nil {
return err
}
resp.Status(kolide.YARAFileSection).ImportCount++
resp.Status(kolide.YARAFileSection).Message("imported '%s'", section)
}
}
}
return nil
}
type dbDuplicateError interface {
IsExists() bool
}
func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.FileIntegrityMonitoring != nil {
for sectionName, paths := range cfg.FileIntegrityMonitoring {
fp := &kolide.FIMSection{
SectionName: sectionName,
Description: "imported",
Paths: paths,
}
_, err := svc.ds.NewFIMSection(fp, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.FilePathsSection).SkipCount++
resp.Status(kolide.FilePathsSection).Warning(kolide.FIMDuplicate, "skipped '%s', already exists", sectionName)
continue
}
if err != nil {
return err
}
resp.Status(kolide.FilePathsSection).ImportCount++
resp.Status(kolide.FilePathsSection).Message("imported '%s'", sectionName)
}
}
// this has to happen AFTER fim section, because it requires file paths
return svc.importYARA(cfg, resp, tx)
}
func (svc service) getExistingDecoratorQueries(tx kolide.Transaction) (map[string]int, error) {
decs, err := svc.ds.ListDecorators(kolide.HasTransaction(tx))
if err != nil {
return nil, err
}
queryHashes := map[string]int{}
for _, dec := range decs {
hash := fmt.Sprintf("%x", md5.Sum([]byte(dec.Query)))
queryHashes[hash] = 0
}
return queryHashes, nil
}
func decoratorExists(query string, queryHashes map[string]int) bool {
hash := fmt.Sprintf("%x", md5.Sum([]byte(query)))
_, exists := queryHashes[hash]
return exists
}
func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.Decorators != nil {
queryHashes, err := svc.getExistingDecoratorQueries(tx)
if err != nil {
return errors.Wrap(err, "getting existing queries")
}
for _, query := range cfg.Decorators.Load {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped load '%s'", query)
continue
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorLoad,
}
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Warning("imported load '%s'", query)
}
for _, query := range cfg.Decorators.Always {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped always '%s'", query)
continue
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorAlways,
}
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported always '%s'", query)
}
for key, queries := range cfg.Decorators.Interval {
for _, query := range queries {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped interval '%s'", query)
continue
}
interval, err := strconv.ParseInt(key, 10, 32)
if err != nil {
return err
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorInterval,
Interval: uint(interval),
}
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported interval %d '%s'", interval, query)
}
}
}
return nil
}
func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
_, ok, err := svc.ds.PackByName(kolide.ImportPackName, kolide.HasTransaction(tx))
if ok {
resp.Status(kolide.PacksSection).Warning(
kolide.PackDuplicate, "skipped '%s' already exists", kolide.ImportPackName,
)
resp.Status(kolide.PacksSection).SkipCount++
return nil
}
// create import pack to hold imported scheduled queries
pack := &kolide.Pack{
Name: kolide.ImportPackName,
Description: "holds imported scheduled queries",
CreatedBy: uid,
Disabled: false,
}
pack, err = svc.ds.NewPack(pack, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.PacksSection).ImportCount++
resp.Status(kolide.PacksSection).Message("created import pack")
for queryName, queryDetails := range cfg.Schedule {
var query *kolide.Query
query, ok, err = svc.ds.QueryByName(queryName, kolide.HasTransaction(tx))
// if we find the query check to see if the import query matches the
// query we have, if it doesn't skip it
if ok {
if hashQuery("", query.Query) != hashQuery("", queryDetails.Query) {
resp.Status(kolide.PacksSection).Warning(
kolide.DifferentQuerySameName,
"queries named '%s' have different statements and won't be added to '%s'",
queryName,
pack.Name,
)
continue
}
resp.Status(kolide.QueriesSection).Warning(
kolide.QueryDuplicate, "skipped '%s' different query of same name already exists", queryName,
)
resp.Status(kolide.QueriesSection).SkipCount++
} else {
// if query doesn't exist, create it
query = &kolide.Query{
Name: queryName,
Description: "imported",
Query: queryDetails.Query,
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.QueriesSection).ImportCount++
resp.Status(kolide.QueriesSection).Message(
"imported scheduled query '%s'", query.Name,
)
}
sq := &kolide.ScheduledQuery{
PackID: pack.ID,
QueryID: query.ID,
Interval: uint(queryDetails.Interval),
Snapshot: queryDetails.Snapshot,
Removed: queryDetails.Removed,
Platform: queryDetails.Platform,
Version: queryDetails.Version,
Shard: configInt2Ptr(queryDetails.Shard),
}
_, err = svc.ds.NewScheduledQuery(sq, kolide.HasTransaction(tx))
if err != nil {
return nil
}
resp.Status(kolide.PacksSection).Message(
"added query '%s' to '%s'", query.Name, pack.Name,
)
}
return nil
}
func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
labelCache := map[string]*kolide.Label{}
packs, err := cfg.CollectPacks()
if err != nil {
return err
}
for packName, packDetails := range packs {
_, ok, err := svc.ds.PackByName(packName, kolide.HasTransaction(tx))
if err != nil {
return err
}
if ok {
resp.Status(kolide.PacksSection).Warning(
kolide.PackDuplicate, "skipped '%s' already exists", packName,
)
resp.Status(kolide.PacksSection).SkipCount++
continue
}
// import new pack
if packDetails.Shard != nil {
resp.Status(kolide.PacksSection).Warning(
kolide.Unsupported,
"shard for pack '%s'",
packName,
)
}
if packDetails.Version != nil {
resp.Status(kolide.PacksSection).Warning(
kolide.Unsupported,
"version for pack '%s'",
packName,
)
}
pack := &kolide.Pack{
Name: packName,
Description: "Imported pack",
Platform: packDetails.Platform,
}
pack, err = svc.ds.NewPack(pack, kolide.HasTransaction(tx))
if err != nil {
return err
}
err = svc.createLabelsForPack(pack, &packDetails, labelCache, resp, tx)
if err != nil {
return err
}
err = svc.createQueriesForPack(uid, pack, &packDetails, resp, tx)
if err != nil {
return err
}
resp.Status(kolide.PacksSection).ImportCount++
resp.Status(kolide.PacksSection).Message("imported '%s'", packName)
}
return nil
}
func hashQuery(platform, query string) string {
s := strings.Replace(query, " ", "", -1)
s = strings.Replace(s, "\t", "", -1)
s = strings.Replace(s, "\n", "", -1)
s = strings.Trim(s, ";")
s = platform + s
return strings.ToLower(s)
}
func uniqueImportName() (string, error) {
random, err := kolide.RandomText(6)
if err != nil {
return "", err
}
return "import_" + random, nil
}
func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *kolide.PackDetails,
resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
for queryName, queryDetails := range details.Queries {
query, ok, err := svc.ds.QueryByName(queryName, kolide.HasTransaction(tx))
if err != nil {
return err
}
// if the query isn't already in the database, create it
if !ok {
query = &kolide.Query{
Name: queryName,
Description: "imported",
Query: queryDetails.Query,
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.QueriesSection).Message(
"created '%s' as part of pack '%s'", queryName, pack.Name,
)
resp.Status(kolide.QueriesSection).ImportCount++
}
// associate query with pack
scheduledQuery := &kolide.ScheduledQuery{
PackID: pack.ID,
QueryID: query.ID,
Interval: uint(queryDetails.Interval),
Platform: queryDetails.Platform,
Snapshot: queryDetails.Snapshot,
Removed: queryDetails.Removed,
Version: queryDetails.Version,
Shard: configInt2Ptr(queryDetails.Shard),
}
_, err = svc.ds.NewScheduledQuery(scheduledQuery, kolide.HasTransaction(tx))
if err != nil {
return nil
}
resp.Status(kolide.PacksSection).Message("added query '%s'", query.Name)
}
return nil
}
// createLabelsForPack Iterates through discover queries, creates a label for
// each query and assigns it to the pack passed as an argument. Once a Label is created we cache
// it for reuse.
func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDetails,
cache map[string]*kolide.Label, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
for _, query := range details.Discovery {
hash := hashQuery(details.Platform, query)
label, ok := cache[hash]
// add existing label to pack
if ok {
err := svc.ds.AddLabelToPack(label.ID, pack.ID, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.PacksSection).Message(
"added label '%s' to pack '%s'", label.Name, pack.Name,
)
continue
}
// create new label and add it to pack
labelName, err := uniqueImportName()
if err != nil {
return err
}
label = &kolide.Label{
Name: labelName,
Query: query,
Description: "imported",
LabelType: kolide.LabelTypeRegular,
Platform: details.Platform,
}
label, err = svc.ds.NewLabel(label, kolide.HasTransaction(tx))
if err != nil {
return err
}
// hang on to label so we can reuse it for other packs if needed
cache[hash] = label
err = svc.ds.AddLabelToPack(label.ID, pack.ID, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.PacksSection).Message(
"added label '%s' to '%s'", label.Name, pack.Name,
)
}
return nil
}
func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
var updateOptions []kolide.Option
for optName, optValue := range opts {
opt, err := svc.ds.OptionByName(optName, kolide.HasTransaction(tx))
if err != nil {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionUnknown, "skipped '%s' can't find option", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
if opt.ReadOnly {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionReadonly, "skipped '%s' can't change read only option", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
if opt.OptionSet() {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionAlreadySet, "skipped '%s' can't change option that is already set", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
opt.SetValue(optValue)
resp.Status(kolide.OptionsSection).Message("set %s value to %v", optName, optValue)
resp.Status(kolide.OptionsSection).ImportCount++
updateOptions = append(updateOptions, *opt)
}
if len(updateOptions) > 0 {
if err := svc.ds.SaveOptions(updateOptions, kolide.HasTransaction(tx)); err != nil {
return err
}
}
return nil
}
func configInt2Ptr(ci *kolide.OsQueryConfigInt) *uint {
if ci == nil {
return nil
}
ui := uint(*ci)
return &ui
}

345
server/service/service_import_config_test.go
View file

@ -1,345 +0,0 @@
package service
import (
"testing"
"github.com/kolide/fleet/server/config"
"github.com/kolide/fleet/server/datastore/inmem"
"github.com/kolide/fleet/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func createServiceMockForImport(t *testing.T) *service {
ds, err := inmem.New(config.TestConfig())
require.Nil(t, err)
err = ds.MigrateData()
require.Nil(t, err)
return &service{
ds: ds,
}
}
func TestHashQuery(t *testing.T) {
q1 := `SELECT * FROM t1 INNER JOIN ON
t1.id = t2.t1id
WHERE t1.name = 'foo' `
q2 := "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t1.name = 'foo';"
h1 := hashQuery("platform", q1)
h2 := hashQuery("platform", q2)
assert.Equal(t, h1, h2)
q2 = "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t2.name = 'foo';"
h2 = hashQuery("platform", q2)
assert.NotEqual(t, h1, h2)
}
func TestImportFilePaths(t *testing.T) {
cfg := &kolide.ImportConfig{
FileIntegrityMonitoring: kolide.FIMCategoryToPaths{
"files1": []string{
"path1",
"path2",
},
"files2": []string{
"path3",
},
},
YARA: &kolide.YARAConfig{
Signatures: map[string][]string{
"sig1": []string{
"path4",
"path5",
},
"sig2": []string{
"path6",
},
},
FilePaths: map[string][]string{
"files1": []string{
"sig1",
"sig2",
},
"files2": []string{
"sig1",
},
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
err := svc.importFIMSections(cfg, resp, tx)
require.Nil(t, err)
assert.Equal(t, 2, resp.Status(kolide.FilePathsSection).ImportCount)
sections, err := svc.ds.FIMSections()
require.Nil(t, err)
assert.Len(t, sections, 2)
yara, err := svc.ds.YARASection()
require.Nil(t, err)
assert.Len(t, yara.Signatures, 2)
assert.Len(t, yara.FilePaths, 2)
}
func TestImportDecorators(t *testing.T) {
cfg := &kolide.ImportConfig{
Decorators: &kolide.DecoratorConfig{
Load: []string{
"select from foo",
"select from bar",
},
Always: []string{
"select from always",
},
Interval: map[string][]string{
"100": []string{
"select from 100",
},
"200": []string{
"select from 200",
},
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
err := svc.importDecorators(cfg, resp, tx)
require.Nil(t, err)
assert.Equal(t, 5, resp.Status(kolide.DecoratorsSection).ImportCount)
dec, err := svc.ds.ListDecorators()
require.Nil(t, err)
assert.Len(t, dec, 5)
}
func TestImportScheduledQueries(t *testing.T) {
cfg := &kolide.ImportConfig{
Schedule: kolide.QueryNameToQueryDetailsMap{
"q1": kolide.QueryDetails{
Query: "select pid from processes",
Interval: 60,
Platform: stringPtr("linux"),
},
"q2": kolide.QueryDetails{
Query: "select uid from users",
Interval: 120,
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
},
"q3": kolide.QueryDetails{
Query: "select name from os",
Interval: 240,
Platform: stringPtr("linux"),
Snapshot: boolPtr(true),
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
user := &kolide.User{
Username: "bob",
Password: []byte("secret"),
Email: "bob@something.com",
Admin: false,
AdminForcedPasswordReset: false,
}
user, err := svc.ds.NewUser(user)
require.Nil(t, err)
skipQuery := &kolide.Query{
Name: "q3",
Query: "select version from os",
Description: "should be skipped",
Saved: true,
AuthorID: user.ID,
}
_, err = svc.ds.NewQuery(skipQuery)
require.Nil(t, err)
noskipQuery := &kolide.Query{
Name: "q2",
Query: "select uid from users",
Saved: true,
AuthorID: user.ID,
}
_, err = svc.ds.NewQuery(noskipQuery)
require.Nil(t, err)
tx, _ := svc.ds.Begin()
err = svc.importScheduledQueries(user.ID, cfg, resp, tx)
require.Nil(t, err)
_, ok, err := svc.ds.QueryByName("q1")
require.Nil(t, err)
require.True(t, ok)
_, ok, err = svc.ds.QueryByName("q2")
require.Nil(t, err)
require.True(t, ok)
_, ok, err = svc.ds.QueryByName("q3")
require.Nil(t, err)
require.True(t, ok)
}
func TestOptionsImportConfig(t *testing.T) {
opts := kolide.OptionNameToValueMap{
"aws_access_key_id": "foo",
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
err := svc.importOptions(opts, resp, tx)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
assert.Equal(t, 1, status.ImportCount)
opt, err := svc.ds.OptionByName("aws_access_key_id")
require.Nil(t, err)
assert.Equal(t, "foo", opt.GetValue())
require.Len(t, status.Messages, 1)
assert.Equal(t, "set aws_access_key_id value to foo", status.Messages[0])
}
func TestOptionsImportConfigWithSkips(t *testing.T) {
opts := kolide.OptionNameToValueMap{
"aws_access_key_id": "foo",
"aws_secret_access_key": "secret",
// this should be skipped because it's already set
"aws_firehose_period": 100,
// these should be skipped because it's read only
"disable_distributed": false,
"pack_delimiter": "x",
// this should be skipped because it's not an option we know about
"wombat": "not venomous",
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
// set option val, it should be skipped
opt, err := svc.ds.OptionByName("aws_firehose_period")
require.Nil(t, err)
opt.SetValue(23)
err = svc.ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
err = svc.importOptions(opts, resp, tx)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
assert.Equal(t, 2, status.ImportCount)
assert.Equal(t, 4, status.SkipCount)
assert.Len(t, status.Warnings[kolide.OptionAlreadySet], 1)
assert.Len(t, status.Warnings[kolide.OptionReadonly], 2)
assert.Len(t, status.Warnings[kolide.OptionUnknown], 1)
assert.Len(t, status.Messages, 2)
}
func TestPacksImportConfig(t *testing.T) {
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
p := &kolide.Pack{
Name: "dup",
}
_, err := svc.ds.NewPack(p)
require.Nil(t, err)
q1 := kolide.QueryDetails{
Query: "select * from foo",
Interval: 100,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
q2 := kolide.QueryDetails{
Query: "select * from bar",
Interval: 50,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
q3 := kolide.QueryDetails{
Query: "select * from baz",
Interval: 500,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
importConfig := kolide.ImportConfig{
Packs: kolide.PackNameMap{
"ext1": "/home/usr/ext1.json",
"pack1": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
"q2": q2,
},
Discovery: []string{
"select * from zz",
"select id, xx from yy",
},
},
"dup": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
"q2": q2,
},
},
"*": "/home/usr/packs/*",
},
ExternalPacks: kolide.PackNameToPackDetails{
"ext1": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
},
Discovery: []string{
"select * from zz",
"select a, b, c from processes",
},
},
"ext2": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q3": q3,
},
},
},
GlobPackNames: []string{"ext2"},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
user := &kolide.User{
Username: "bob",
Password: []byte("secret"),
Email: "bob@something.com",
Admin: false,
AdminForcedPasswordReset: false,
}
user, err = svc.ds.NewUser(user)
require.Nil(t, err)
packs, err := importConfig.CollectPacks()
require.Nil(t, err)
assert.Len(t, packs, 4)
err = svc.importPacks(user.ID, &importConfig, resp, tx)
require.Nil(t, err)
queries, err := svc.ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, queries, 3)
pack, ok, err := svc.ds.PackByName("pack1")
require.Nil(t, err)
require.True(t, ok)
sqs, err := svc.ds.ListScheduledQueriesInPack(pack.ID, kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, sqs, 2)
labels, err := svc.ds.ListLabels(kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, labels, 8)
assert.Equal(t, 3, resp.Status(kolide.PacksSection).ImportCount)
assert.Equal(t, 1, resp.Status(kolide.PacksSection).SkipCount)
assert.Equal(t, 3, resp.Status(kolide.QueriesSection).ImportCount)
}

8
server/service/service_packs.go
View file

@ -3,7 +3,6 @@ package service
import (
"context"
"github.com/kolide/fleet/server/contexts/viewer"
"github.com/kolide/fleet/server/kolide"
)
@ -34,13 +33,6 @@ func (svc service) NewPack(ctx context.Context, p kolide.PackPayload) (*kolide.P
pack.Disabled = *p.Disabled
}
vc, ok := viewer.FromContext(ctx)
if ok {
if createdBy := vc.UserID(); createdBy != uint(0) {
pack.CreatedBy = createdBy
}
}
_, err := svc.ds.NewPack(&pack)
if err != nil {
return nil, err

45
server/service/service_queries.go
View file

@ -5,8 +5,53 @@ import (
"github.com/kolide/fleet/server/contexts/viewer"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func queryFromSpec(spec *kolide.QuerySpec) *kolide.Query {
return &kolide.Query{
Name: spec.Name,
Description: spec.Description,
Query: spec.Query,
}
}
func specFromQuery(query *kolide.Query) *kolide.QuerySpec {
return &kolide.QuerySpec{
Name: query.Name,
Description: query.Description,
Query: query.Query,
}
}
func (svc service) ApplyQuerySpecs(ctx context.Context, specs []*kolide.QuerySpec) error {
vc, ok := viewer.FromContext(ctx)
if !ok {
return errors.New("user must be authenticated to apply queries")
}
queries := []*kolide.Query{}
for _, spec := range specs {
queries = append(queries, queryFromSpec(spec))
}
err := svc.ds.ApplyQueries(vc.UserID(), queries)
return errors.Wrap(err, "applying queries")
}
func (svc service) GetQuerySpecs(ctx context.Context) ([]*kolide.QuerySpec, error) {
queries, err := svc.ds.ListQueries(kolide.ListOptions{})
if err != nil {
return nil, errors.Wrap(err, "getting queries")
}
specs := []*kolide.QuerySpec{}
for _, query := range queries {
specs = append(specs, specFromQuery(query))
}
return specs, nil
}
func (svc service) ListQueries(ctx context.Context, opt kolide.ListOptions) ([]*kolide.Query, error) {
return svc.ds.ListQueries(opt)
}

54
server/service/service_scheduled_queries.go
View file

@ -4,62 +4,8 @@ import (
"context"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (svc service) GetScheduledQuery(ctx context.Context, id uint) (*kolide.ScheduledQuery, error) {
return svc.ds.ScheduledQuery(id)
}
func (svc service) GetScheduledQueriesInPack(ctx context.Context, id uint, opts kolide.ListOptions) ([]*kolide.ScheduledQuery, error) {
return svc.ds.ListScheduledQueriesInPack(id, opts)
}
func (svc service) ScheduleQuery(ctx context.Context, sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
return svc.ds.NewScheduledQuery(sq)
}
func (svc service) ModifyScheduledQuery(ctx context.Context, id uint, p kolide.ScheduledQueryPayload) (*kolide.ScheduledQuery, error) {
sq, err := svc.GetScheduledQuery(ctx, id)
if err != nil {
return nil, errors.Wrap(err, "getting scheduled query to modify")
}
if p.PackID != nil {
sq.PackID = *p.PackID
}
if p.QueryID != nil {
sq.QueryID = *p.QueryID
}
if p.Interval != nil {
sq.Interval = *p.Interval
}
if p.Snapshot != nil {
sq.Snapshot = p.Snapshot
}
if p.Removed != nil {
sq.Removed = p.Removed
}
if p.Platform != nil {
sq.Platform = p.Platform
}
if p.Version != nil {
sq.Version = p.Version
}
if p.Shard != nil {
sq.Shard = p.Shard
}
return svc.ds.SaveScheduledQuery(sq)
}
func (svc service) DeleteScheduledQuery(ctx context.Context, id uint) error {
return svc.ds.DeleteScheduledQuery(id)
}

107
server/service/service_scheduled_queries_test.go
View file

@ -1,107 +0,0 @@
package service
import (
"context"
"testing"
"github.com/kolide/fleet/server/config"
"github.com/kolide/fleet/server/datastore/inmem"
"github.com/kolide/fleet/server/kolide"
"github.com/kolide/fleet/server/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetScheduledQueriesInPack(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
assert.Nil(t, err)
svc, err := newTestService(ds, nil)
assert.Nil(t, err)
ctx := context.Background()
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
q2 := test.NewQuery(t, ds, "bar", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
queries, err := svc.GetScheduledQueriesInPack(ctx, p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, 1)
assert.Equal(t, sq1.ID, queries[0].ID)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, false, false)
test.NewScheduledQuery(t, ds, p1.ID, q2.ID, 60, true, false)
queries, err = svc.GetScheduledQueriesInPack(ctx, p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, queries, 3)
}
func TestGetScheduledQuery(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
assert.Nil(t, err)
svc, err := newTestService(ds, nil)
assert.Nil(t, err)
ctx := context.Background()
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := svc.GetScheduledQuery(ctx, sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
}
func TestModifyScheduledQuery(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
assert.Nil(t, err)
svc, err := newTestService(ds, nil)
assert.Nil(t, err)
ctx := context.Background()
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := svc.GetScheduledQuery(ctx, sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
interval := uint(120)
queryPayload := kolide.ScheduledQueryPayload{
Interval: &interval,
}
query, err = svc.ModifyScheduledQuery(ctx, sq1.ID, queryPayload)
assert.Equal(t, uint(120), query.Interval)
queryVerify, err := svc.GetScheduledQuery(ctx, sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(120), queryVerify.Interval)
}
func TestDeleteScheduledQuery(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
assert.Nil(t, err)
svc, err := newTestService(ds, nil)
assert.Nil(t, err)
ctx := context.Background()
u1 := test.NewUser(t, ds, "Admin", "admin", "admin@kolide.co", true)
q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true)
p1 := test.NewPack(t, ds, "baz")
sq1 := test.NewScheduledQuery(t, ds, p1.ID, q1.ID, 60, false, false)
query, err := svc.GetScheduledQuery(ctx, sq1.ID)
require.Nil(t, err)
assert.Equal(t, uint(60), query.Interval)
err = svc.DeleteScheduledQuery(ctx, sq1.ID)
require.Nil(t, err)
_, err = svc.GetScheduledQuery(ctx, sq1.ID)
require.NotNil(t, err)
}

35
server/service/transport_import_config.go
View file

@ -1,35 +0,0 @@
package service
import (
"context"
"encoding/json"
"net/http"
"github.com/kolide/fleet/server/kolide"
)
func decodeImportConfigRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var req importRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
return nil, err
}
// Unmarshal main config
conf := kolide.ImportConfig{
DryRun: req.DryRun,
Packs: make(kolide.PackNameMap),
ExternalPacks: make(kolide.PackNameToPackDetails),
}
if err := json.Unmarshal([]byte(req.Config), &conf); err != nil {
return nil, err
}
// Unmarshal external packs
for packName, packConfig := range req.ExternalPackConfigs {
var pack kolide.PackDetails
if err := json.Unmarshal([]byte(packConfig), &pack); err != nil {
return nil, err
}
conf.ExternalPacks[packName] = pack
}
conf.GlobPackNames = req.GlobPackNames
return conf, nil
}

46
server/service/transport_scheduled_queries.go
View file

@ -2,55 +2,9 @@ package service
import (
"context"
"encoding/json"
"net/http"
)
func decodeScheduleQueryRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var req scheduleQueryRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
return nil, err
}
return req, nil
}
func decodeModifyScheduledQueryRequest(ctx context.Context, r *http.Request) (interface{}, error) {
id, err := idFromRequest(r, "id")
if err != nil {
return nil, err
}
var req modifyScheduledQueryRequest
if err := json.NewDecoder(r.Body).Decode(&req.payload); err != nil {
return nil, err
}
req.ID = id
return req, nil
}
func decodeDeleteScheduledQueryRequest(ctx context.Context, r *http.Request) (interface{}, error) {
id, err := idFromRequest(r, "id")
if err != nil {
return nil, err
}
var req deleteScheduledQueryRequest
req.ID = id
return req, nil
}
func decodeGetScheduledQueryRequest(ctx context.Context, r *http.Request) (interface{}, error) {
id, err := idFromRequest(r, "id")
if err != nil {
return nil, err
}
var req getScheduledQueryRequest
req.ID = id
return req, nil
}
func decodeGetScheduledQueriesInPackRequest(ctx context.Context, r *http.Request) (interface{}, error) {
id, err := idFromRequest(r, "id")
if err != nil {

91
server/service/transport_scheduled_queries_test.go
View file

@ -1,7 +1,6 @@
package service
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
@ -9,98 +8,8 @@ import (
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDecodeScheduleQueryRequest(t *testing.T) {
router := mux.NewRouter()
router.HandleFunc("/api/v1/kolide/schedule", func(writer http.ResponseWriter, request *http.Request) {
r, err := decodeScheduleQueryRequest(context.Background(), request)
require.Nil(t, err)
params := r.(scheduleQueryRequest)
assert.Equal(t, uint(5), params.PackID)
assert.Equal(t, uint(1), params.QueryID)
assert.Equal(t, uint(60), params.Interval)
assert.Equal(t, true, *params.Snapshot)
}).Methods("POST")
var body bytes.Buffer
body.Write([]byte(`{
"pack_id": 5,
"query_id": 1,
"interval": 60,
"snapshot": true
}`))
router.ServeHTTP(
httptest.NewRecorder(),
httptest.NewRequest("POST", "/api/v1/kolide/schedule", &body),
)
}
func TestDecodeModifyScheduledQueryRequest(t *testing.T) {
router := mux.NewRouter()
router.HandleFunc("/api/v1/kolide/scheduled/{id}", func(writer http.ResponseWriter, request *http.Request) {
r, err := decodeModifyScheduledQueryRequest(context.Background(), request)
assert.Nil(t, err)
params := r.(modifyScheduledQueryRequest)
assert.Equal(t, uint(1), params.ID)
assert.Equal(t, uint(5), *params.payload.PackID)
assert.Equal(t, uint(6), *params.payload.QueryID)
assert.Equal(t, true, *params.payload.Removed)
assert.Equal(t, uint(60), *params.payload.Interval)
assert.Equal(t, uint(1), *params.payload.Shard)
}).Methods("PATCH")
var body bytes.Buffer
body.Write([]byte(`{
"pack_id": 5,
"query_id": 6,
"removed": true,
"interval": 60,
"shard": 1
}`))
router.ServeHTTP(
httptest.NewRecorder(),
httptest.NewRequest("PATCH", "/api/v1/kolide/scheduled/1", &body),
)
}
func TestDecodeDeleteScheduledQueryRequest(t *testing.T) {
router := mux.NewRouter()
router.HandleFunc("/api/v1/kolide/scheduled/{id}", func(writer http.ResponseWriter, request *http.Request) {
r, err := decodeDeleteScheduledQueryRequest(context.Background(), request)
assert.Nil(t, err)
params := r.(deleteScheduledQueryRequest)
assert.Equal(t, uint(1), params.ID)
}).Methods("DELETE")
router.ServeHTTP(
httptest.NewRecorder(),
httptest.NewRequest("DELETE", "/api/v1/kolide/scheduled/1", nil),
)
}
func TestDecodeGetScheduledQueryRequest(t *testing.T) {
router := mux.NewRouter()
router.HandleFunc("/api/v1/kolide/scheduled/{id}", func(writer http.ResponseWriter, request *http.Request) {
r, err := decodeGetScheduledQueryRequest(context.Background(), request)
assert.Nil(t, err)
params := r.(getScheduledQueryRequest)
assert.Equal(t, uint(1), params.ID)
}).Methods("GET")
router.ServeHTTP(
httptest.NewRecorder(),
httptest.NewRequest("GET", "/api/v1/kolide/scheduled/1", nil),
)
}
func TestDecodeGetScheduledQueriesInPackRequest(t *testing.T) {
router := mux.NewRouter()
router.HandleFunc("/api/v1/kolide/packs/{id}/scheduled", func(writer http.ResponseWriter, request *http.Request) {

127
server/service/validation_import_config.go
View file

@ -1,127 +0,0 @@
package service
import (
"context"
"strconv"
"github.com/kolide/fleet/server/kolide"
)
func (vm validationMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var invalid invalidArgumentError
vm.validateConfigOptions(cfg, &invalid)
vm.validatePacks(cfg, &invalid)
vm.validateDecorator(cfg, &invalid)
vm.validateYARA(cfg, &invalid)
if invalid.HasErrors() {
return nil, invalid
}
return vm.Service.ImportConfig(ctx, cfg)
}
func (vm validationMiddleware) validateYARA(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.YARA != nil {
if cfg.YARA.FilePaths == nil {
argErrs.Append("yara", "missing file_paths")
return
}
if cfg.YARA.Signatures == nil {
argErrs.Append("yara", "missing signatures")
}
for fileSection, sigs := range cfg.YARA.FilePaths {
if cfg.FileIntegrityMonitoring == nil {
argErrs.Append("yara", "missing file paths section")
return
}
if _, ok := cfg.FileIntegrityMonitoring[fileSection]; !ok {
argErrs.Appendf("yara", "missing referenced file_paths section '%s'", fileSection)
}
for _, sig := range sigs {
if _, ok := cfg.YARA.Signatures[sig]; !ok {
argErrs.Appendf(
"yara",
"missing signature '%s' referenced in '%s'",
sig,
fileSection,
)
}
}
}
}
}
func (vm validationMiddleware) validateDecorator(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Decorators != nil {
for str := range cfg.Decorators.Interval {
val, err := strconv.ParseInt(str, 10, 32)
if err != nil {
argErrs.Appendf("decorators", "interval '%s' must be an integer", str)
continue
}
if val%60 != 0 {
argErrs.Appendf("decorators", "interval '%d' must be divisible by 60", val)
}
}
}
}
func (vm validationMiddleware) validateConfigOptions(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Options != nil {
for optName, optValue := range cfg.Options {
opt, err := vm.ds.OptionByName(string(optName))
if err != nil {
// skip validation for an option we don't know about, this will generate
// a warning in the service layer
continue
}
if !opt.SameType(optValue) {
argErrs.Appendf("options", "invalid type for '%s'", optName)
}
}
}
}
func (vm validationMiddleware) validatePacks(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Packs != nil {
for packName, pack := range cfg.Packs {
// if glob packs is defined we expect at least one external pack
if packName == kolide.GlobPacks {
if len(cfg.GlobPackNames) == 0 {
argErrs.Append("external_packs", "missing glob packs")
continue
}
// make sure that each glob pack has JSON content
for _, p := range cfg.GlobPackNames {
if pd, ok := cfg.ExternalPacks[p]; !ok {
argErrs.Appendf("external_packs", "missing content for '%s'", p)
} else {
vm.validatePackContents(p, pd, argErrs)
}
}
continue
}
// if value is a string we expect a file path, in this case, the user has to supply the
// contents of said file which we store in ExternalPacks, if it's not there we need to
// raise an error
switch val := pack.(type) {
case string:
if pd, ok := cfg.ExternalPacks[packName]; !ok {
argErrs.Appendf("external_packs", "missing content for '%s'", packName)
} else {
vm.validatePackContents(packName, pd, argErrs)
}
case kolide.PackDetails:
vm.validatePackContents(packName, val, argErrs)
}
}
}
}
func (vm validationMiddleware) validatePackContents(packName string, pack kolide.PackDetails, argErrs *invalidArgumentError) {
switch pack.Platform {
case "", "darwin", "freebsd", "windows", "linux", "any", "all":
default:
argErrs.Appendf("pack", "'%s' is not a valid platform", pack.Platform)
}
}

14
server/test/new_objects.go
View file

@ -128,17 +128,3 @@ func NewUser(t *testing.T, ds kolide.Datastore, name, username, email string, ad
return u
}
func NewScheduledQuery(t *testing.T, ds kolide.Datastore, pid, qid, interval uint, snapshot, removed bool) *kolide.ScheduledQuery {
sq, err := ds.NewScheduledQuery(&kolide.ScheduledQuery{
PackID: pid,
QueryID: qid,
Interval: interval,
Snapshot: &snapshot,
Removed: &removed,
})
require.Nil(t, err)
require.NotZero(t, sq.ID)
return sq
}