From cf805aa66c0cd306fcae11b70f5047b836f764fe Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Wed, 11 Jan 2017 13:33:30 -0700 Subject: [PATCH] Only return hosts which have been explicitly scheduled from packs API (#909) * Only return hosts which have been explicitly scheduled from packs API close #903 * better error handling * documentation --- server/datastore/inmem/packs.go | 49 ++++++++++++++ server/datastore/mysql/packs.go | 109 +++++++++++++++++++++---------- server/kolide/packs.go | 65 ++++++++++++++++-- server/service/endpoint_packs.go | 2 +- server/service/service_packs.go | 4 ++ 5 files changed, 186 insertions(+), 43 deletions(-) diff --git a/server/datastore/inmem/packs.go b/server/datastore/inmem/packs.go index c85c0d358f..1041311950 100644 --- a/server/datastore/inmem/packs.go +++ b/server/datastore/inmem/packs.go @@ -252,3 +252,52 @@ func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide return hosts, nil } + +func (d *Datastore) ListExplicitHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + + hosts := []*kolide.Host{} + hostLookup := map[uint]bool{} + + for _, pt := range d.packTargets { + if pt.PackID != pid { + continue + } + + if pt.Type == kolide.TargetHost { + if !hostLookup[pt.TargetID] { + hostLookup[pt.TargetID] = true + hosts = append(hosts, d.hosts[pt.TargetID]) + } + } + } + + // Apply ordering + if opt.OrderKey != "" { + var fields = map[string]string{ + "id": "ID", + "created_at": "CreatedAt", + "updated_at": "UpdatedAt", + "detail_update_time": "DetailUpdateTime", + "hostname": "HostName", + "uuid": "UUID", + "platform": "Platform", + "osquery_version": "OsqueryVersion", + "os_version": "OSVersion", + "uptime": "Uptime", + "memory": "PhysicalMemory", + "mac": "PrimaryMAC", + "ip": "PrimaryIP", + } + if err := sortResults(hosts, opt, fields); err != nil { + return nil, err + } + } + + // Apply limit/offset + low, high := d.getLimitOffsetSliceBounds(opt, len(hosts)) + hosts = hosts[low:high] + + return hosts, nil +} diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index 90882475d7..b1f64dff69 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -1,21 +1,24 @@ package mysql import ( - "github.com/kolide/kolide-ose/server/errors" + "database/sql" + "fmt" + "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) // NewPack creates a new Pack func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { - sql := ` + query := ` INSERT INTO packs ( name, description, platform, created_by, disabled ) VALUES ( ?, ?, ?, ?, ?) ` - result, err := d.db.Exec(sql, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled) + result, err := d.db.Exec(query, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "creating new pack") } id, _ := result.LastInsertId() @@ -25,16 +28,17 @@ func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { // SavePack stores changes to pack func (d *Datastore) SavePack(pack *kolide.Pack) error { - - sql := ` + query := ` UPDATE packs SET name = ?, platform = ?, disabled = ?, description = ? WHERE id = ? AND NOT deleted ` - _, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.Disabled, pack.Description, pack.ID) - if err != nil { - return errors.DatabaseError(err) + _, err := d.db.Exec(query, pack.Name, pack.Platform, pack.Disabled, pack.Description, pack.ID) + if err == sql.ErrNoRows { + return notFound("Pack").WithID(pack.ID) + } else if err != nil { + return errors.Wrap(err, "update pack") } return nil @@ -42,15 +46,24 @@ func (d *Datastore) SavePack(pack *kolide.Pack) error { // DeletePack soft deletes a kolide.Pack so that it won't show up in results func (d *Datastore) DeletePack(pid uint) error { - return d.deleteEntity("packs", pid) + err := d.deleteEntity("packs", pid) + if err == sql.ErrNoRows { + return notFound("Pack").WithID(pid) + } else if err != nil { + return errors.Wrap(err, "delete pack") + } + return nil } // Pack fetch kolide.Pack with matching ID func (d *Datastore) Pack(pid uint) (*kolide.Pack, error) { - sql := `SELECT * FROM packs WHERE id = ? AND NOT deleted` + query := `SELECT * FROM packs WHERE id = ? AND NOT deleted` pack := &kolide.Pack{} - if err := d.db.Get(pack, sql, pid); err != nil { - return nil, errors.DatabaseError(err) + err := d.db.Get(pack, query, pid) + if err == sql.ErrNoRows { + return nil, notFound("Pack").WithID(pid) + } else if err != nil { + return nil, errors.Wrap(err, "getting pack") } return pack, nil @@ -58,25 +71,25 @@ func (d *Datastore) Pack(pid uint) (*kolide.Pack, error) { // ListPacks returns all kolide.Pack records limited and sorted by kolide.ListOptions func (d *Datastore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { - sql := `SELECT * FROM packs WHERE NOT deleted` - sql = appendListOptionsToSQL(sql, opt) + query := `SELECT * FROM packs WHERE NOT deleted` packs := []*kolide.Pack{} - if err := d.db.Select(&packs, sql); err != nil { - return nil, errors.DatabaseError(err) + err := d.db.Select(&packs, appendListOptionsToSQL(query, opt)) + if err != nil && err != sql.ErrNoRows { + return nil, errors.Wrap(err, "listing packs") } return packs, nil } // AddLabelToPack associates a kolide.Label with a kolide.Pack func (d *Datastore) AddLabelToPack(lid uint, pid uint) error { - sql := ` + query := ` INSERT INTO pack_targets ( pack_id, type, target_id ) VALUES ( ?, ?, ? ) ON DUPLICATE KEY UPDATE id=id ` - _, err := d.db.Exec(sql, pid, kolide.TargetLabel, lid) + _, err := d.db.Exec(query, pid, kolide.TargetLabel, lid) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "adding label to pack") } return nil @@ -84,14 +97,14 @@ func (d *Datastore) AddLabelToPack(lid uint, pid uint) error { // AddHostToPack associates a kolide.Host with a kolide.Pack func (d *Datastore) AddHostToPack(hid, pid uint) error { - sql := ` + query := ` INSERT INTO pack_targets ( pack_id, type, target_id ) VALUES ( ?, ?, ? ) ON DUPLICATE KEY UPDATE id=id ` - _, err := d.db.Exec(sql, pid, kolide.TargetHost, hid) + _, err := d.db.Exec(query, pid, kolide.TargetHost, hid) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "adding host to pack") } return nil @@ -99,7 +112,7 @@ func (d *Datastore) AddHostToPack(hid, pid uint) error { // ListLabelsForPack will return a list of kolide.Label records associated with kolide.Pack func (d *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) { - sql := ` + query := ` SELECT l.id, l.created_at, @@ -120,8 +133,8 @@ func (d *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) { labels := []*kolide.Label{} - if err := d.db.Select(&labels, sql, kolide.TargetLabel, pid); err != nil { - return nil, errors.DatabaseError(err) + if err := d.db.Select(&labels, query, kolide.TargetLabel, pid); err != nil && err != sql.ErrNoRows { + return nil, errors.Wrap(err, "listing labels for pack") } return labels, nil @@ -130,12 +143,15 @@ func (d *Datastore) ListLabelsForPack(pid uint) ([]*kolide.Label, error) { // RemoreLabelFromPack will remove the association between a kolide.Label and // a kolide.Pack func (d *Datastore) RemoveLabelFromPack(lid, pid uint) error { - sql := ` + query := ` DELETE FROM pack_targets WHERE target_id = ? AND pack_id = ? AND type = ? ` - if _, err := d.db.Exec(sql, lid, pid, kolide.TargetLabel); err != nil { - return errors.DatabaseError(err) + _, err := d.db.Exec(query, lid, pid, kolide.TargetLabel) + if err == sql.ErrNoRows { + return notFound("PackTarget").WithMessage(fmt.Sprintf("label ID: %d, pack ID: %d", lid, pid)) + } else if err != nil { + return errors.Wrap(err, "removing label from pack") } return nil @@ -144,12 +160,15 @@ func (d *Datastore) RemoveLabelFromPack(lid, pid uint) error { // RemoveHostFromPack will remove the association between a kolide.Host and a // kolide.Pack func (d *Datastore) RemoveHostFromPack(hid, pid uint) error { - sql := ` + query := ` DELETE FROM pack_targets WHERE target_id = ? AND pack_id = ? AND type = ? ` - if _, err := d.db.Exec(sql, hid, pid, kolide.TargetHost); err != nil { - return errors.DatabaseError(err) + _, err := d.db.Exec(query, hid, pid, kolide.TargetHost) + if err == sql.ErrNoRows { + return notFound("PackTarget").WithMessage(fmt.Sprintf("host ID: %d, pack ID: %d", hid, pid)) + } else if err != nil { + return errors.Wrap(err, "removing host from pack") } return nil @@ -157,7 +176,7 @@ func (d *Datastore) RemoveHostFromPack(hid, pid uint) error { } func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { - sql := ` + query := ` SELECT DISTINCT h.* FROM hosts h JOIN pack_targets pt @@ -173,10 +192,28 @@ func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide ) WHERE pt.pack_id = ? ` - sql = appendListOptionsToSQL(sql, opt) hosts := []*kolide.Host{} - if err := d.db.Select(&hosts, sql, kolide.TargetLabel, kolide.TargetHost, pid); err != nil { - return nil, errors.DatabaseError(err) + if err := d.db.Select(&hosts, appendListOptionsToSQL(query, opt), kolide.TargetLabel, kolide.TargetHost, pid); err != nil && err != sql.ErrNoRows { + return nil, errors.Wrap(err, "listing hosts in pack") } return hosts, nil } + +func (d *Datastore) ListExplicitHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + query := ` + SELECT DISTINCT h.* + FROM hosts h + JOIN pack_targets pt + ON ( + pt.target_id = h.id + AND pt.type = ? + ) + WHERE pt.pack_id = ? + ` + hosts := []*kolide.Host{} + if err := d.db.Select(&hosts, appendListOptionsToSQL(query, opt), kolide.TargetHost, pid); err != nil && err != sql.ErrNoRows { + return nil, errors.Wrap(err, "listing explicit hosts in pack") + } + return hosts, nil + +} diff --git a/server/kolide/packs.go b/server/kolide/packs.go index d419f479b8..9f25097cdc 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -4,45 +4,96 @@ import ( "golang.org/x/net/context" ) +// PackStore is the datastore interface for managing query packs. type PackStore interface { - // Pack methods + // NewPack creates a new pack in the datastore. NewPack(pack *Pack) (*Pack, error) + + // SavePack updates an existing pack in the datastore. SavePack(pack *Pack) error + + // DeletePack deletes a pack record from the datastore. DeletePack(pid uint) error + + // Pack retrieves a pack from the datastore by ID. Pack(pid uint) (*Pack, error) + + // ListPacks lists all packs in the datastore. ListPacks(opt ListOptions) ([]*Pack, error) - // Modifying the labels for packs + // AddLabelToPack adds an existing label to an existing pack, both by ID. AddLabelToPack(lid, pid uint) error + + // RemoveLabelFromPack removes an existing label from it's association with + // an existing pack, both by ID. RemoveLabelFromPack(lid, pid uint) error + + // ListLabelsForPack lists all labels that are associated with a pack. ListLabelsForPack(pid uint) ([]*Label, error) - // Modifying the hosts for packs + // AddHostToPack adds an existing host to an existing pack, both by ID. AddHostToPack(hid uint, pid uint) error + + // RemoveHostFromPack removes an existing host from it's association with + // an existing pack, both by ID. RemoveHostFromPack(hid uint, pid uint) error + + // ListHostsInPack lists all hosts that are associated with a pack, both + // through labels and manual associations. ListHostsInPack(pid uint, opt ListOptions) ([]*Host, error) + + // ListExplicitHostsInPack lists hosts that have been manually associated + // with a query pack. + ListExplicitHostsInPack(pid uint, opt ListOptions) ([]*Host, error) } +// PackService is the service interface for managing query packs. type PackService interface { - // Pack methods + // ListPacks lists all packs in the application. ListPacks(ctx context.Context, opt ListOptions) (packs []*Pack, err error) + + // GetPack retrieves a pack by ID. GetPack(ctx context.Context, id uint) (pack *Pack, err error) + + // NewPack creates a new pack in the datastore. NewPack(ctx context.Context, p PackPayload) (pack *Pack, err error) + + // ModifyPack modifies an existing pack in the datastore. ModifyPack(ctx context.Context, id uint, p PackPayload) (pack *Pack, err error) + + // DeletePack deletes a pack record from the datastore. DeletePack(ctx context.Context, id uint) (err error) - // Modifying the labels for packs + // AddLabelToPack adds an existing label to an existing pack, both by ID. AddLabelToPack(ctx context.Context, lid, pid uint) (err error) + + // RemoveLabelFromPack removes an existing label from it's association with + // an existing pack, both by ID. RemoveLabelFromPack(ctx context.Context, lid, pid uint) (err error) + + // ListLabelsForPack lists all labels that are associated with a pack. ListLabelsForPack(ctx context.Context, pid uint) (labels []*Label, err error) - // Modifying the hosts for packs + // AddHostToPack adds an existing host to an existing pack, both by ID. AddHostToPack(ctx context.Context, hid, pid uint) (err error) + + // RemoveHostFromPack removes an existing host from it's association with + // an existing pack, both by ID. RemoveHostFromPack(ctx context.Context, hid, pid uint) (err error) + + // ListPacksForHost lists the packs that a host should execute. ListPacksForHost(ctx context.Context, hid uint) (packs []*Pack, err error) + + // ListHostsInPack lists all hosts that are associated with a pack, both + // through labels and manual associations. ListHostsInPack(ctx context.Context, pid uint, opt ListOptions) (hosts []*Host, err error) + + // ListExplicitHostsInPack lists hosts that have been manually associated + // with a query pack. + ListExplicitHostsInPack(ctx context.Context, pid uint, opt ListOptions) (hosts []*Host, err error) } +// Pack is the structure which represents an osquery query pack. type Pack struct { UpdateCreateTimestamps DeleteFields @@ -54,6 +105,7 @@ type Pack struct { Disabled bool `json:"disabled"` } +// PackPayload is the struct which is used to create/update packs. type PackPayload struct { Name *string `json:"name"` Description *string `json:"description"` @@ -63,6 +115,7 @@ type PackPayload struct { LabelIDs *[]uint `json:"label_ids"` } +// PackTarget associates a pack with either a host or a label type PackTarget struct { ID uint PackID uint diff --git a/server/service/endpoint_packs.go b/server/service/endpoint_packs.go index 260477ac0e..a2c225a494 100644 --- a/server/service/endpoint_packs.go +++ b/server/service/endpoint_packs.go @@ -19,7 +19,7 @@ func packResponseForPack(ctx context.Context, svc kolide.Service, pack kolide.Pa if err != nil { return nil, err } - hosts, err := svc.ListHostsInPack(ctx, pack.ID, kolide.ListOptions{}) + hosts, err := svc.ListExplicitHostsInPack(ctx, pack.ID, kolide.ListOptions{}) if err != nil { return nil, err } diff --git a/server/service/service_packs.go b/server/service/service_packs.go index 4efc564b6e..e7e44811da 100644 --- a/server/service/service_packs.go +++ b/server/service/service_packs.go @@ -220,6 +220,10 @@ func (svc service) ListHostsInPack(ctx context.Context, pid uint, opt kolide.Lis return svc.ds.ListHostsInPack(pid, opt) } +func (svc service) ListExplicitHostsInPack(ctx context.Context, pid uint, opt kolide.ListOptions) ([]*kolide.Host, error) { + return svc.ds.ListExplicitHostsInPack(pid, opt) +} + func (svc service) ListPacksForHost(ctx context.Context, hid uint) ([]*kolide.Pack, error) { packs := []*kolide.Pack{}