diff --git a/Makefile b/Makefile index 0e7f2655fb..a0ff6e0883 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,7 @@ deps: npm install go get github.com/jteeuwen/go-bindata/... go get github.com/Masterminds/glide + go get github.com/pressly/goose glide install distclean: diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index 66c72eedb4..28bd6c9748 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -157,7 +157,7 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) { err = ds.AddLabelToPack(mysqlLabel.ID, monitoringPack.ID) require.Nil(t, err) - labels, err := ds.ListLabelsForPack(monitoringPack) + labels, err := ds.ListLabelsForPack(monitoringPack.ID) require.Nil(t, err) if assert.Len(t, labels, 1) { assert.Equal(t, "MySQL Monitoring", labels[0].Name) @@ -173,7 +173,7 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) { err = ds.AddLabelToPack(osqueryLabel.ID, monitoringPack.ID) require.Nil(t, err) - labels, err = ds.ListLabelsForPack(monitoringPack) + labels, err = ds.ListLabelsForPack(monitoringPack.ID) require.Nil(t, err) assert.Len(t, labels, 2) } diff --git a/server/datastore/datastore_packs_test.go b/server/datastore/datastore_packs_test.go index 9b8ddef47f..00f015c424 100644 --- a/server/datastore/datastore_packs_test.go +++ b/server/datastore/datastore_packs_test.go @@ -1,8 +1,10 @@ package datastore import ( + "fmt" "testing" + "github.com/WatchBeam/clock" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -70,3 +72,78 @@ func testAddAndRemoveQueryFromPack(t *testing.T, ds kolide.Datastore) { assert.Nil(t, err) assert.Len(t, queries, 1) } + +func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { + mockClock := clock.NewMockClock() + + p1, err := ds.NewPack(&kolide.Pack{ + Name: "foo", + }) + require.Nil(t, err) + + q1, err := ds.NewQuery(&kolide.Query{ + Name: "foo", + Query: "foo", + }) + require.Nil(t, err) + + q2, err := ds.NewQuery(&kolide.Query{ + Name: "bar", + Query: "bar", + }) + require.Nil(t, err) + + err = ds.AddQueryToPack(q1.ID, p1.ID) + require.Nil(t, err) + + err = ds.AddQueryToPack(q2.ID, p1.ID) + require.Nil(t, err) + + l1, err := ds.NewLabel(&kolide.Label{ + Name: "foo", + }) + require.Nil(t, err) + + err = ds.AddLabelToPack(l1.ID, p1.ID) + require.Nil(t, err) + + h1, err := ds.NewHost(&kolide.Host{ + DetailUpdateTime: mockClock.Now(), + HostName: "foobar.local", + NodeKey: "1", + UUID: "1", + PrimaryIP: "192.168.1.1", + }) + require.Nil(t, err) + + err = ds.RecordLabelQueryExecutions( + h1, + map[string]bool{fmt.Sprintf("%d", l1.ID): true}, + mockClock.Now(), + ) + require.Nil(t, err) + + hostsInPack, err := ds.ListHostsInPack(p1.ID, kolide.ListOptions{}) + require.Nil(t, err) + require.Len(t, hostsInPack, 1) + + h2, err := ds.NewHost(&kolide.Host{ + DetailUpdateTime: mockClock.Now(), + HostName: "foobaz.local", + NodeKey: "2", + UUID: "2", + PrimaryIP: "192.168.1.2", + }) + require.Nil(t, err) + + err = ds.RecordLabelQueryExecutions( + h2, + map[string]bool{fmt.Sprintf("%d", l1.ID): true}, + mockClock.Now(), + ) + require.Nil(t, err) + + hostsInPack, err = ds.ListHostsInPack(p1.ID, kolide.ListOptions{}) + require.Nil(t, err) + require.Len(t, hostsInPack, 2) +} diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index 9f98037fba..5b7fcc7b6b 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -45,4 +45,5 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testSaveHosts, testDeleteHost, testListHost, + testGetHostsInPack, } diff --git a/server/datastore/inmem/packs.go b/server/datastore/inmem/packs.go index c10c336d60..2bf334351e 100644 --- a/server/datastore/inmem/packs.go +++ b/server/datastore/inmem/packs.go @@ -158,12 +158,12 @@ func (orm *Datastore) AddLabelToPack(lid uint, pid uint) error { return nil } -func (orm *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { +func (orm *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) { var labels []*kolide.Label orm.mtx.Lock() for _, pt := range orm.packTargets { - if pt.Type == kolide.TargetLabel && pt.PackID == pack.ID { + if pt.Type == kolide.TargetLabel && pt.PackID == pid { labels = append(labels, orm.labels[pt.TargetID]) } } @@ -189,3 +189,59 @@ func (orm *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack return nil } + +func (orm *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + hosts := []*kolide.Host{} + hostLookup := map[uint]bool{} + + orm.mtx.Lock() + for _, pt := range orm.packTargets { + if pt.PackID != pid { + continue + } + + switch pt.Type { + case kolide.TargetHost: + if !hostLookup[pt.TargetID] { + hostLookup[pt.TargetID] = true + hosts = append(hosts, orm.hosts[pt.TargetID]) + } + case kolide.TargetLabel: + for _, lqe := range orm.labelQueryExecutions { + if lqe.LabelID == pt.TargetID && lqe.Matches && !hostLookup[lqe.HostID] { + hostLookup[lqe.HostID] = true + hosts = append(hosts, orm.hosts[lqe.HostID]) + } + } + } + } + orm.mtx.Unlock() + + // Apply ordering + if opt.OrderKey != "" { + var fields = map[string]string{ + "id": "ID", + "created_at": "CreatedAt", + "updated_at": "UpdatedAt", + "detail_update_time": "DetailUpdateTime", + "hostname": "HostName", + "uuid": "UUID", + "platform": "Platform", + "osquery_version": "OsqueryVersion", + "os_version": "OSVersion", + "uptime": "Uptime", + "memory": "PhysicalMemory", + "mac": "PrimaryMAC", + "ip": "PrimaryIP", + } + if err := sortResults(hosts, opt, fields); err != nil { + return nil, err + } + } + + // Apply limit/offset + low, high := orm.getLimitOffsetSliceBounds(opt, len(hosts)) + hosts = hosts[low:high] + + return hosts, nil +} diff --git a/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go b/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go index 4163d5c3ca..59d8ddbf43 100644 --- a/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go +++ b/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go @@ -18,8 +18,11 @@ func Up_20161118212630(tx *sql.Tx) error { "`updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," + "`deleted_at` timestamp NULL DEFAULT NULL," + "`deleted` tinyint(1) NOT NULL DEFAULT FALSE," + + "`disabled` tinyint(1) NOT NULL DEFAULT FALSE," + "`name` varchar(255) NOT NULL," + + "`description` varchar(255) DEFAULT NULL," + "`platform` varchar(255) DEFAULT NULL," + + "`created_by` int(10) unsigned DEFAULT NULL," + "PRIMARY KEY (`id`)," + "UNIQUE KEY `idx_pack_unique_name` (`name`)" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index dc99ff6621..7b4aa8e9a5 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -9,11 +9,11 @@ import ( func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { sql := ` - INSERT INTO packs ( name, platform ) - VALUES ( ?, ?) + INSERT INTO packs ( name, description, platform, created_by, disabled ) + VALUES ( ?, ?, ?, ?, ?) ` - result, err := d.db.Exec(sql, pack.Name, pack.Platform) + result, err := d.db.Exec(sql, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled) if err != nil { return nil, errors.DatabaseError(err) } @@ -28,11 +28,11 @@ func (d *Datastore) SavePack(pack *kolide.Pack) error { sql := ` UPDATE packs - SET name = ?, platform = ? + SET name = ?, platform = ?, disabled = ?, description = ?, WHERE id = ? AND NOT deleted ` - _, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.ID) + _, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.Disabled, pack.Description, pack.ID) if err != nil { return errors.DatabaseError(err) } @@ -156,7 +156,7 @@ func (d *Datastore) AddLabelToPack(lid uint, pid uint) error { } // ListLabelsForPack will return a list of kolide.Label records associated with kolide.Pack -func (d *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { +func (d *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) { sql := ` SELECT l.id, @@ -178,7 +178,7 @@ func (d *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error labels := []*kolide.Label{} - if err := d.db.Select(&labels, sql, kolide.TargetLabel, pack.ID); err != nil { + if err := d.db.Select(&labels, sql, kolide.TargetLabel, pid); err != nil { return nil, errors.DatabaseError(err) } @@ -198,3 +198,28 @@ func (d *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) return nil } + +func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + sql := ` + SELECT DISTINCT h.* + FROM hosts h + JOIN pack_targets pt + JOIN label_query_executions lqe + ON ( + pt.target_id = lqe.label_id + AND lqe.host_id = h.id + AND lqe.matches + AND pt.type = ? + ) OR ( + pt.target_id = h.id + AND pt.type = ? + ) + WHERE pt.pack_id = ? + ` + sql = appendListOptionsToSQL(sql, opt) + hosts := []*kolide.Host{} + if err := d.db.Select(&hosts, sql, kolide.TargetLabel, kolide.TargetHost, pid); err != nil { + return nil, errors.DatabaseError(err) + } + return hosts, nil +} diff --git a/server/kolide/packs.go b/server/kolide/packs.go index 29a753287f..88dce9a65a 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -21,8 +21,10 @@ type PackStore interface { // Modifying the labels for packs AddLabelToPack(lid uint, pid uint) error - ListLabelsForPack(pack *Pack) ([]*Label, error) + ListLabelsForPack(pid uint) ([]*Label, error) RemoveLabelFromPack(label *Label, pack *Pack) error + + ListHostsInPack(pid uint, opt ListOptions) ([]*Host, error) } type PackService interface { @@ -41,6 +43,7 @@ type PackService interface { RemoveLabelFromPack(ctx context.Context, lid, pid uint) error ListPacksForHost(ctx context.Context, hid uint) ([]*Pack, error) + ListHostsInPack(ctx context.Context, pid uint, opt ListOptions) ([]*Host, error) } type Pack struct { @@ -50,12 +53,15 @@ type Pack struct { Name string `json:"name"` Description string `json:"description"` Platform string `json:"platform"` + CreatedBy uint `json:"created_by" db:"created_by"` + Disabled bool `json:"disabled"` } type PackPayload struct { Name *string Description *string Platform *string + Disabled *bool } type PackQuery struct { diff --git a/server/kolide/queries.go b/server/kolide/queries.go index a64b25ae2a..ff10dc7ff0 100644 --- a/server/kolide/queries.go +++ b/server/kolide/queries.go @@ -33,7 +33,7 @@ type QueryService interface { NewQuery(ctx context.Context, p QueryPayload) (*Query, error) ModifyQuery(ctx context.Context, id uint, p QueryPayload) (*Query, error) DeleteQuery(ctx context.Context, id uint) error - NewDistributedQueryCampaign(ctx context.Context, userID uint, queryString string, hosts []uint, labels []uint) (*DistributedQueryCampaign, error) + NewDistributedQueryCampaign(ctx context.Context, queryString string, hosts []uint, labels []uint) (*DistributedQueryCampaign, error) } type QueryPayload struct { diff --git a/server/service/endpoint_packs.go b/server/service/endpoint_packs.go index 22a93f892c..f02bb950a2 100644 --- a/server/service/endpoint_packs.go +++ b/server/service/endpoint_packs.go @@ -14,8 +14,14 @@ type getPackRequest struct { ID uint } +type packResponse struct { + kolide.Pack + QueryCount uint `json:"query_count"` + HostCount uint `json:"host_count"` +} + type getPackResponse struct { - Pack *kolide.Pack `json:"pack,omitempty"` + Pack packResponse `json:"pack,omitempty"` Err error `json:"error,omitempty"` } @@ -24,11 +30,29 @@ func (r getPackResponse) error() error { return r.Err } func makeGetPackEndpoint(svc kolide.Service) endpoint.Endpoint { return func(ctx context.Context, request interface{}) (interface{}, error) { req := request.(getPackRequest) + pack, err := svc.GetPack(ctx, req.ID) if err != nil { return getPackResponse{Err: err}, nil } - return getPackResponse{pack, nil}, nil + + queries, err := svc.ListQueriesInPack(ctx, pack.ID) + if err != nil { + return getPackResponse{Err: err}, nil + } + + hosts, err := svc.ListHostsInPack(ctx, pack.ID, kolide.ListOptions{}) + if err != nil { + return getPackResponse{Err: err}, nil + } + + return getPackResponse{ + Pack: packResponse{ + Pack: *pack, + QueryCount: uint(len(queries)), + HostCount: uint(len(hosts)), + }, + }, nil } } @@ -41,8 +65,8 @@ type listPacksRequest struct { } type listPacksResponse struct { - Packs []kolide.Pack `json:"packs"` - Err error `json:"error,omitempty"` + Packs []packResponse `json:"packs"` + Err error `json:"error,omitempty"` } func (r listPacksResponse) error() error { return r.Err } @@ -55,9 +79,21 @@ func makeListPacksEndpoint(svc kolide.Service) endpoint.Endpoint { return getPackResponse{Err: err}, nil } - resp := listPacksResponse{Packs: []kolide.Pack{}} + resp := listPacksResponse{Packs: []packResponse{}} for _, pack := range packs { - resp.Packs = append(resp.Packs, *pack) + queries, err := svc.ListQueriesInPack(ctx, pack.ID) + if err != nil { + return getPackResponse{Err: err}, nil + } + hosts, err := svc.ListHostsInPack(ctx, pack.ID, kolide.ListOptions{}) + if err != nil { + return getPackResponse{Err: err}, nil + } + resp.Packs = append(resp.Packs, packResponse{ + Pack: *pack, + QueryCount: uint(len(queries)), + HostCount: uint(len(hosts)), + }) } return resp, nil } diff --git a/server/service/endpoint_queries.go b/server/service/endpoint_queries.go index 25e58310b9..814f382559 100644 --- a/server/service/endpoint_queries.go +++ b/server/service/endpoint_queries.go @@ -2,7 +2,6 @@ package service import ( "github.com/go-kit/kit/endpoint" - "github.com/kolide/kolide-ose/server/contexts/viewer" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" ) @@ -163,13 +162,8 @@ func (r createDistributedQueryCampaignResponse) error() error { return r.Err } func makeCreateDistributedQueryCampaignEndpoint(svc kolide.Service) endpoint.Endpoint { return func(ctx context.Context, request interface{}) (interface{}, error) { - vc, ok := viewer.FromContext(ctx) - if !ok { - return nil, errNoContext - } - req := request.(createDistributedQueryCampaignRequest) - campaign, err := svc.NewDistributedQueryCampaign(ctx, vc.UserID(), req.Query, req.Selected.Hosts, req.Selected.Labels) + campaign, err := svc.NewDistributedQueryCampaign(ctx, req.Query, req.Selected.Hosts, req.Selected.Labels) if err != nil { return createQueryResponse{Err: err}, nil } diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index f5c0ac8909..21e94544d9 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -14,6 +14,7 @@ import ( "github.com/WatchBeam/clock" hostctx "github.com/kolide/kolide-ose/server/contexts/host" + "github.com/kolide/kolide-ose/server/contexts/viewer" "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/kolide/kolide-ose/server/pubsub" @@ -627,11 +628,16 @@ func TestDistributedQueries(t *testing.T) { labelId := strconv.Itoa(int(label.ID)) // Record match with label + ctx = viewer.NewContext(ctx, viewer.Viewer{ + User: &kolide.User{ + ID: 0, + }, + }) err = ds.RecordLabelQueryExecutions(host, map[string]bool{labelId: true}, mockClock.Now()) require.Nil(t, err) q = "select year, month, day, hour, minutes, seconds from time" - campaign, err := svc.NewDistributedQueryCampaign(ctx, 0, q, []uint{}, []uint{label.ID}) + campaign, err := svc.NewDistributedQueryCampaign(ctx, q, []uint{}, []uint{label.ID}) require.Nil(t, err) queryKey := fmt.Sprintf("%s%d", hostDistributedQueryPrefix, campaign.ID) diff --git a/server/service/service_packs.go b/server/service/service_packs.go index fba0567ab5..958f7ea738 100644 --- a/server/service/service_packs.go +++ b/server/service/service_packs.go @@ -1,6 +1,7 @@ package service import ( + "github.com/kolide/kolide-ose/server/contexts/viewer" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" ) @@ -28,6 +29,17 @@ func (svc service) NewPack(ctx context.Context, p kolide.PackPayload) (*kolide.P pack.Platform = *p.Platform } + if p.Disabled != nil { + pack.Disabled = *p.Disabled + } + + vc, ok := viewer.FromContext(ctx) + if ok { + if createdBy := vc.UserID(); createdBy != uint(0) { + pack.CreatedBy = createdBy + } + } + _, err := svc.ds.NewPack(&pack) if err != nil { return nil, err @@ -53,6 +65,10 @@ func (svc service) ModifyPack(ctx context.Context, id uint, p kolide.PackPayload pack.Platform = *p.Platform } + if p.Disabled != nil { + pack.Disabled = *p.Disabled + } + err = svc.ds.SavePack(pack) if err != nil { return nil, err @@ -106,12 +122,7 @@ func (svc service) AddLabelToPack(ctx context.Context, lid, pid uint) error { } func (svc service) ListLabelsForPack(ctx context.Context, pid uint) ([]*kolide.Label, error) { - pack, err := svc.ds.Pack(pid) - if err != nil { - return nil, err - } - - labels, err := svc.ds.ListLabelsForPack(pack) + labels, err := svc.ds.ListLabelsForPack(pid) if err != nil { return nil, err } @@ -138,6 +149,10 @@ func (svc service) RemoveLabelFromPack(ctx context.Context, lid, pid uint) error return nil } +func (svc service) ListHostsInPack(ctx context.Context, pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + return svc.ds.ListHostsInPack(pid, opt) +} + func (svc service) ListPacksForHost(ctx context.Context, hid uint) ([]*kolide.Pack, error) { packs := []*kolide.Pack{} @@ -162,9 +177,14 @@ func (svc service) ListPacksForHost(ctx context.Context, hid uint) ([]*kolide.Pa } for _, pack := range allPacks { + // don't include packs which have been disabled + if pack.Disabled { + continue + } + // for each pack, we must know what labels have been assigned to that // pack - labelsForPack, err := svc.ds.ListLabelsForPack(pack) + labelsForPack, err := svc.ds.ListLabelsForPack(pack.ID) if err != nil { return nil, err } diff --git a/server/service/service_queries.go b/server/service/service_queries.go index 4cc4bbf2a9..db7bd6e65b 100644 --- a/server/service/service_queries.go +++ b/server/service/service_queries.go @@ -1,6 +1,7 @@ package service import ( + "github.com/kolide/kolide-ose/server/contexts/viewer" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" ) @@ -115,7 +116,12 @@ func (svc service) DeleteQuery(ctx context.Context, id uint) error { return nil } -func (svc service) NewDistributedQueryCampaign(ctx context.Context, userID uint, queryString string, hosts []uint, labels []uint) (*kolide.DistributedQueryCampaign, error) { +func (svc service) NewDistributedQueryCampaign(ctx context.Context, queryString string, hosts []uint, labels []uint) (*kolide.DistributedQueryCampaign, error) { + vc, ok := viewer.FromContext(ctx) + if !ok { + return nil, errNoContext + } + query, err := svc.NewQuery(ctx, kolide.QueryPayload{ Name: &queryString, Query: &queryString, @@ -127,7 +133,7 @@ func (svc service) NewDistributedQueryCampaign(ctx context.Context, userID uint, campaign, err := svc.ds.NewDistributedQueryCampaign(&kolide.DistributedQueryCampaign{ QueryID: query.ID, Status: kolide.QueryRunning, - UserID: userID, + UserID: vc.UserID(), }) if err != nil { return nil, err