Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-24 09:28:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 52

Add attributes to packs (#524)

Browse source 
* Adds created_by attribute to packs

This PR also updated the distributed query code to use the pattern
established here (service checks context)

* add enable/disable state to packs

* add query_count to packs API responses

* add host_count to packs API responses (very, very poorly)

* pack description should not be required

* counting hosts in packs via mysql

* removing extraneous newline in test

* Switch case instead of if/if else

* add description to update query for SavePack method

* change AND to WHERE in query as per @zwass

* add ordering and list options as per @murphybytes' suggestion
This commit is contained in:
Mike Arpaia 2016-11-22 13:56:05 -08:00 committed by GitHub
parent f6e5084d0d
commit a036c5da9f
14 changed files with 267 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -132,6 +132,7 @@ deps:
npm install
go get github.com/jteeuwen/go-bindata/...
go get github.com/Masterminds/glide
go get github.com/pressly/goose
glide install
distclean:

4
server/datastore/datastore_labels_test.go
View file

@ -157,7 +157,7 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) {
err = ds.AddLabelToPack(mysqlLabel.ID, monitoringPack.ID)
require.Nil(t, err)
labels, err := ds.ListLabelsForPack(monitoringPack)
labels, err := ds.ListLabelsForPack(monitoringPack.ID)
require.Nil(t, err)
if assert.Len(t, labels, 1) {
assert.Equal(t, "MySQL Monitoring", labels[0].Name)
@ -173,7 +173,7 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) {
err = ds.AddLabelToPack(osqueryLabel.ID, monitoringPack.ID)
require.Nil(t, err)
labels, err = ds.ListLabelsForPack(monitoringPack)
labels, err = ds.ListLabelsForPack(monitoringPack.ID)
require.Nil(t, err)
assert.Len(t, labels, 2)
}

77
server/datastore/datastore_packs_test.go
View file

@ -1,8 +1,10 @@
package datastore
import (
"fmt"
"testing"
"github.com/WatchBeam/clock"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -70,3 +72,78 @@ func testAddAndRemoveQueryFromPack(t *testing.T, ds kolide.Datastore) {
assert.Nil(t, err)
assert.Len(t, queries, 1)
}
func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
mockClock := clock.NewMockClock()
p1, err := ds.NewPack(&kolide.Pack{
Name: "foo",
})
require.Nil(t, err)
q1, err := ds.NewQuery(&kolide.Query{
Name: "foo",
Query: "foo",
})
require.Nil(t, err)
q2, err := ds.NewQuery(&kolide.Query{
Name: "bar",
Query: "bar",
})
require.Nil(t, err)
err = ds.AddQueryToPack(q1.ID, p1.ID)
require.Nil(t, err)
err = ds.AddQueryToPack(q2.ID, p1.ID)
require.Nil(t, err)
l1, err := ds.NewLabel(&kolide.Label{
Name: "foo",
})
require.Nil(t, err)
err = ds.AddLabelToPack(l1.ID, p1.ID)
require.Nil(t, err)
h1, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
HostName: "foobar.local",
NodeKey: "1",
UUID: "1",
PrimaryIP: "192.168.1.1",
})
require.Nil(t, err)
err = ds.RecordLabelQueryExecutions(
h1,
map[string]bool{fmt.Sprintf("%d", l1.ID): true},
mockClock.Now(),
)
require.Nil(t, err)
hostsInPack, err := ds.ListHostsInPack(p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, hostsInPack, 1)
h2, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
HostName: "foobaz.local",
NodeKey: "2",
UUID: "2",
PrimaryIP: "192.168.1.2",
})
require.Nil(t, err)
err = ds.RecordLabelQueryExecutions(
h2,
map[string]bool{fmt.Sprintf("%d", l1.ID): true},
mockClock.Now(),
)
require.Nil(t, err)
hostsInPack, err = ds.ListHostsInPack(p1.ID, kolide.ListOptions{})
require.Nil(t, err)
require.Len(t, hostsInPack, 2)
}

1
server/datastore/datastore_test.go
View file

@ -45,4 +45,5 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testSaveHosts,
testDeleteHost,
testListHost,
testGetHostsInPack,
}

60
server/datastore/inmem/packs.go
View file

@ -158,12 +158,12 @@ func (orm *Datastore) AddLabelToPack(lid uint, pid uint) error {
return nil
}
func (orm *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) {
func (orm *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) {
var labels []*kolide.Label
orm.mtx.Lock()
for _, pt := range orm.packTargets {
if pt.Type == kolide.TargetLabel && pt.PackID == pack.ID {
if pt.Type == kolide.TargetLabel && pt.PackID == pid {
labels = append(labels, orm.labels[pt.TargetID])
}
}
@ -189,3 +189,59 @@ func (orm *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack
return nil
}
func (orm *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) {
hosts := []*kolide.Host{}
hostLookup := map[uint]bool{}
orm.mtx.Lock()
for _, pt := range orm.packTargets {
if pt.PackID != pid {
continue
}
switch pt.Type {
case kolide.TargetHost:
if !hostLookup[pt.TargetID] {
hostLookup[pt.TargetID] = true
hosts = append(hosts, orm.hosts[pt.TargetID])
}
case kolide.TargetLabel:
for _, lqe := range orm.labelQueryExecutions {
if lqe.LabelID == pt.TargetID && lqe.Matches && !hostLookup[lqe.HostID] {
hostLookup[lqe.HostID] = true
hosts = append(hosts, orm.hosts[lqe.HostID])
}
}
}
}
orm.mtx.Unlock()
// Apply ordering
if opt.OrderKey != "" {
var fields = map[string]string{
"id": "ID",
"created_at": "CreatedAt",
"updated_at": "UpdatedAt",
"detail_update_time": "DetailUpdateTime",
"hostname": "HostName",
"uuid": "UUID",
"platform": "Platform",
"osquery_version": "OsqueryVersion",
"os_version": "OSVersion",
"uptime": "Uptime",
"memory": "PhysicalMemory",
"mac": "PrimaryMAC",
"ip": "PrimaryIP",
}
if err := sortResults(hosts, opt, fields); err != nil {
return nil, err
}
}
// Apply limit/offset
low, high := orm.getLimitOffsetSliceBounds(opt, len(hosts))
hosts = hosts[low:high]
return hosts, nil
}

3
server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go
View file

@ -18,8 +18,11 @@ func Up_20161118212630(tx *sql.Tx) error {
"`updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," +
"`deleted_at` timestamp NULL DEFAULT NULL," +
"`deleted` tinyint(1) NOT NULL DEFAULT FALSE," +
"`disabled` tinyint(1) NOT NULL DEFAULT FALSE," +
"`name` varchar(255) NOT NULL," +
"`description` varchar(255) DEFAULT NULL," +
"`platform` varchar(255) DEFAULT NULL," +
"`created_by` int(10) unsigned DEFAULT NULL," +
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_pack_unique_name` (`name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",

39
server/datastore/mysql/packs.go
View file

@ -9,11 +9,11 @@ import (
func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
sql := `
INSERT INTO packs ( name, platform )
VALUES ( ?, ?)
INSERT INTO packs ( name, description, platform, created_by, disabled )
VALUES ( ?, ?, ?, ?, ?)
`
result, err := d.db.Exec(sql, pack.Name, pack.Platform)
result, err := d.db.Exec(sql, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled)
if err != nil {
return nil, errors.DatabaseError(err)
}
@ -28,11 +28,11 @@ func (d *Datastore) SavePack(pack *kolide.Pack) error {
sql := `
UPDATE packs
SET name = ?, platform = ?
SET name = ?, platform = ?, disabled = ?, description = ?,
WHERE id = ? AND NOT deleted
`
_, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.ID)
_, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.Disabled, pack.Description, pack.ID)
if err != nil {
return errors.DatabaseError(err)
}
@ -156,7 +156,7 @@ func (d *Datastore) AddLabelToPack(lid uint, pid uint) error {
}
// ListLabelsForPack will return a list of kolide.Label records associated with kolide.Pack
func (d *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) {
func (d *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) {
sql := `
SELECT
l.id,
@ -178,7 +178,7 @@ func (d *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error
labels := []*kolide.Label{}
if err := d.db.Select(&labels, sql, kolide.TargetLabel, pack.ID); err != nil {
if err := d.db.Select(&labels, sql, kolide.TargetLabel, pid); err != nil {
return nil, errors.DatabaseError(err)
}
@ -198,3 +198,28 @@ func (d *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack)
return nil
}
func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) {
sql := `
SELECT DISTINCT h.*
FROM hosts h
JOIN pack_targets pt
JOIN label_query_executions lqe
ON (
pt.target_id = lqe.label_id
AND lqe.host_id = h.id
AND lqe.matches
AND pt.type = ?
) OR (
pt.target_id = h.id
AND pt.type = ?
)
WHERE pt.pack_id = ?
`
sql = appendListOptionsToSQL(sql, opt)
hosts := []*kolide.Host{}
if err := d.db.Select(&hosts, sql, kolide.TargetLabel, kolide.TargetHost, pid); err != nil {
return nil, errors.DatabaseError(err)
}
return hosts, nil
}

8
server/kolide/packs.go
View file

@ -21,8 +21,10 @@ type PackStore interface {
// Modifying the labels for packs
AddLabelToPack(lid uint, pid uint) error
ListLabelsForPack(pack *Pack) ([]*Label, error)
ListLabelsForPack(pid uint) ([]*Label, error)
RemoveLabelFromPack(label *Label, pack *Pack) error
ListHostsInPack(pid uint, opt ListOptions) ([]*Host, error)
}
type PackService interface {
@ -41,6 +43,7 @@ type PackService interface {
RemoveLabelFromPack(ctx context.Context, lid, pid uint) error
ListPacksForHost(ctx context.Context, hid uint) ([]*Pack, error)
ListHostsInPack(ctx context.Context, pid uint, opt ListOptions) ([]*Host, error)
}
type Pack struct {
@ -50,12 +53,15 @@ type Pack struct {
Name string `json:"name"`
Description string `json:"description"`
Platform string `json:"platform"`
CreatedBy uint `json:"created_by" db:"created_by"`
Disabled bool `json:"disabled"`
}
type PackPayload struct {
Name *string
Description *string
Platform *string
Disabled *bool
}
type PackQuery struct {

2
server/kolide/queries.go
View file

@ -33,7 +33,7 @@ type QueryService interface {
NewQuery(ctx context.Context, p QueryPayload) (*Query, error)
ModifyQuery(ctx context.Context, id uint, p QueryPayload) (*Query, error)
DeleteQuery(ctx context.Context, id uint) error
NewDistributedQueryCampaign(ctx context.Context, userID uint, queryString string, hosts []uint, labels []uint) (*DistributedQueryCampaign, error)
NewDistributedQueryCampaign(ctx context.Context, queryString string, hosts []uint, labels []uint) (*DistributedQueryCampaign, error)
}
type QueryPayload struct {

48
server/service/endpoint_packs.go
View file

@ -14,8 +14,14 @@ type getPackRequest struct {
ID uint
}
type packResponse struct {
kolide.Pack
QueryCount uint `json:"query_count"`
HostCount uint `json:"host_count"`
}
type getPackResponse struct {
Pack *kolide.Pack `json:"pack,omitempty"`
Pack packResponse `json:"pack,omitempty"`
Err error `json:"error,omitempty"`
}
@ -24,11 +30,29 @@ func (r getPackResponse) error() error { return r.Err }
func makeGetPackEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(getPackRequest)
pack, err := svc.GetPack(ctx, req.ID)
if err != nil {
return getPackResponse{Err: err}, nil
}
return getPackResponse{pack, nil}, nil
queries, err := svc.ListQueriesInPack(ctx, pack.ID)
if err != nil {
return getPackResponse{Err: err}, nil
}
hosts, err := svc.ListHostsInPack(ctx, pack.ID, kolide.ListOptions{})
if err != nil {
return getPackResponse{Err: err}, nil
}
return getPackResponse{
Pack: packResponse{
Pack: *pack,
QueryCount: uint(len(queries)),
HostCount: uint(len(hosts)),
},
}, nil
}
}
@ -41,8 +65,8 @@ type listPacksRequest struct {
}
type listPacksResponse struct {
Packs []kolide.Pack `json:"packs"`
Err error `json:"error,omitempty"`
Packs []packResponse `json:"packs"`
Err error `json:"error,omitempty"`
}
func (r listPacksResponse) error() error { return r.Err }
@ -55,9 +79,21 @@ func makeListPacksEndpoint(svc kolide.Service) endpoint.Endpoint {
return getPackResponse{Err: err}, nil
}
resp := listPacksResponse{Packs: []kolide.Pack{}}
resp := listPacksResponse{Packs: []packResponse{}}
for _, pack := range packs {
resp.Packs = append(resp.Packs, *pack)
queries, err := svc.ListQueriesInPack(ctx, pack.ID)
if err != nil {
return getPackResponse{Err: err}, nil
}
hosts, err := svc.ListHostsInPack(ctx, pack.ID, kolide.ListOptions{})
if err != nil {
return getPackResponse{Err: err}, nil
}
resp.Packs = append(resp.Packs, packResponse{
Pack: *pack,
QueryCount: uint(len(queries)),
HostCount: uint(len(hosts)),
})
}
return resp, nil
}

8
server/service/endpoint_queries.go
View file

@ -2,7 +2,6 @@ package service
import (
"github.com/go-kit/kit/endpoint"
"github.com/kolide/kolide-ose/server/contexts/viewer"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
@ -163,13 +162,8 @@ func (r createDistributedQueryCampaignResponse) error() error { return r.Err }
func makeCreateDistributedQueryCampaignEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
vc, ok := viewer.FromContext(ctx)
if !ok {
return nil, errNoContext
}
req := request.(createDistributedQueryCampaignRequest)
campaign, err := svc.NewDistributedQueryCampaign(ctx, vc.UserID(), req.Query, req.Selected.Hosts, req.Selected.Labels)
campaign, err := svc.NewDistributedQueryCampaign(ctx, req.Query, req.Selected.Hosts, req.Selected.Labels)
if err != nil {
return createQueryResponse{Err: err}, nil
}

8
server/service/service_osquery_test.go
View file

@ -14,6 +14,7 @@ import (
"github.com/WatchBeam/clock"
hostctx "github.com/kolide/kolide-ose/server/contexts/host"
"github.com/kolide/kolide-ose/server/contexts/viewer"
"github.com/kolide/kolide-ose/server/datastore/inmem"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/kolide/kolide-ose/server/pubsub"
@ -627,11 +628,16 @@ func TestDistributedQueries(t *testing.T) {
labelId := strconv.Itoa(int(label.ID))
// Record match with label
ctx = viewer.NewContext(ctx, viewer.Viewer{
User: &kolide.User{
ID: 0,
},
})
err = ds.RecordLabelQueryExecutions(host, map[string]bool{labelId: true}, mockClock.Now())
require.Nil(t, err)
q = "select year, month, day, hour, minutes, seconds from time"
campaign, err := svc.NewDistributedQueryCampaign(ctx, 0, q, []uint{}, []uint{label.ID})
campaign, err := svc.NewDistributedQueryCampaign(ctx, q, []uint{}, []uint{label.ID})
require.Nil(t, err)
queryKey := fmt.Sprintf("%s%d", hostDistributedQueryPrefix, campaign.ID)

34
server/service/service_packs.go
View file

@ -1,6 +1,7 @@
package service
import (
"github.com/kolide/kolide-ose/server/contexts/viewer"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
@ -28,6 +29,17 @@ func (svc service) NewPack(ctx context.Context, p kolide.PackPayload) (*kolide.P
pack.Platform = *p.Platform
}
if p.Disabled != nil {
pack.Disabled = *p.Disabled
}
vc, ok := viewer.FromContext(ctx)
if ok {
if createdBy := vc.UserID(); createdBy != uint(0) {
pack.CreatedBy = createdBy
}
}
_, err := svc.ds.NewPack(&pack)
if err != nil {
return nil, err
@ -53,6 +65,10 @@ func (svc service) ModifyPack(ctx context.Context, id uint, p kolide.PackPayload
pack.Platform = *p.Platform
}
if p.Disabled != nil {
pack.Disabled = *p.Disabled
}
err = svc.ds.SavePack(pack)
if err != nil {
return nil, err
@ -106,12 +122,7 @@ func (svc service) AddLabelToPack(ctx context.Context, lid, pid uint) error {
}
func (svc service) ListLabelsForPack(ctx context.Context, pid uint) ([]*kolide.Label, error) {
pack, err := svc.ds.Pack(pid)
if err != nil {
return nil, err
}
labels, err := svc.ds.ListLabelsForPack(pack)
labels, err := svc.ds.ListLabelsForPack(pid)
if err != nil {
return nil, err
}
@ -138,6 +149,10 @@ func (svc service) RemoveLabelFromPack(ctx context.Context, lid, pid uint) error
return nil
}
func (svc service) ListHostsInPack(ctx context.Context, pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) {
return svc.ds.ListHostsInPack(pid, opt)
}
func (svc service) ListPacksForHost(ctx context.Context, hid uint) ([]*kolide.Pack, error) {
packs := []*kolide.Pack{}
@ -162,9 +177,14 @@ func (svc service) ListPacksForHost(ctx context.Context, hid uint) ([]*kolide.Pa
}
for _, pack := range allPacks {
// don't include packs which have been disabled
if pack.Disabled {
continue
}
// for each pack, we must know what labels have been assigned to that
// pack
labelsForPack, err := svc.ds.ListLabelsForPack(pack)
labelsForPack, err := svc.ds.ListLabelsForPack(pack.ID)
if err != nil {
return nil, err
}

10
server/service/service_queries.go
View file

@ -1,6 +1,7 @@
package service
import (
"github.com/kolide/kolide-ose/server/contexts/viewer"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
@ -115,7 +116,12 @@ func (svc service) DeleteQuery(ctx context.Context, id uint) error {
return nil
}
func (svc service) NewDistributedQueryCampaign(ctx context.Context, userID uint, queryString string, hosts []uint, labels []uint) (*kolide.DistributedQueryCampaign, error) {
func (svc service) NewDistributedQueryCampaign(ctx context.Context, queryString string, hosts []uint, labels []uint) (*kolide.DistributedQueryCampaign, error) {
vc, ok := viewer.FromContext(ctx)
if !ok {
return nil, errNoContext
}
query, err := svc.NewQuery(ctx, kolide.QueryPayload{
Name: &queryString,
Query: &queryString,
@ -127,7 +133,7 @@ func (svc service) NewDistributedQueryCampaign(ctx context.Context, userID uint,
campaign, err := svc.ds.NewDistributedQueryCampaign(&kolide.DistributedQueryCampaign{
QueryID: query.ID,
Status: kolide.QueryRunning,
UserID: userID,
UserID: vc.UserID(),
})
if err != nil {
return nil, err