From 7ebebbb7b19b2447b04365a8f35298c00dcf7e7f Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Wed, 2 Nov 2016 10:59:53 -0400 Subject: [PATCH] Target search endpoint (#339) --- frontend/kolide/endpoints.js | 1 + server/datastore/datastore_hosts_test.go | 61 +++++ server/datastore/datastore_labels_test.go | 177 ++++++++++++++ server/datastore/datastore_test.go | 6 + server/datastore/gorm.go | 29 ++- server/datastore/gorm_hosts.go | 25 ++ server/datastore/gorm_labels.go | 66 ++++++ server/datastore/gorm_packs.go | 16 +- server/datastore/inmem_hosts.go | 25 ++ server/datastore/inmem_labels.go | 66 ++++++ server/datastore/inmem_packs.go | 8 +- server/kolide/hosts.go | 5 +- server/kolide/labels.go | 7 +- server/kolide/packs.go | 10 +- server/kolide/service.go | 1 + server/kolide/target.go | 31 +++ server/service/endpoint_targets.go | 86 +++++++ server/service/handler.go | 22 +- server/service/service_targets.go | 45 ++++ server/service/service_targets_test.go | 268 ++++++++++++++++++++++ server/service/transport_targets.go | 17 ++ server/service/transport_targets_test.go | 46 ++++ 22 files changed, 984 insertions(+), 34 deletions(-) create mode 100644 server/kolide/target.go create mode 100644 server/service/endpoint_targets.go create mode 100644 server/service/service_targets.go create mode 100644 server/service/service_targets_test.go create mode 100644 server/service/transport_targets.go create mode 100644 server/service/transport_targets_test.go diff --git a/frontend/kolide/endpoints.js b/frontend/kolide/endpoints.js index 27bb6b9417..f421ba97da 100644 --- a/frontend/kolide/endpoints.js +++ b/frontend/kolide/endpoints.js @@ -10,5 +10,6 @@ export default { LOGOUT: '/v1/kolide/logout', ME: '/v1/kolide/me', RESET_PASSWORD: '/v1/kolide/reset_password', + TARGETS: '/v1/kolide/targets', USERS: '/v1/kolide/users', }; diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index 1740b733d8..9a3fa4c4e4 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -3,9 +3,11 @@ package datastore import ( "fmt" "testing" + "time" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var enrollTests = []struct { @@ -81,3 +83,62 @@ func testAuthenticateHost(t *testing.T, db kolide.Datastore) { _, err = db.AuthenticateHost("") assert.NotNil(t, err) } + +func testSearchHosts(t *testing.T, db kolide.Datastore) { + _, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + require.Nil(t, err) + + _, err = db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "2", + UUID: "2", + HostName: "bar.local", + PrimaryIP: "192.168.1.11", + }) + require.Nil(t, err) + + h3, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "3", + UUID: "3", + HostName: "foo-bar.local", + PrimaryIP: "192.168.1.12", + }) + require.Nil(t, err) + + hosts, err := db.SearchHosts("foo", nil) + assert.Nil(t, err) + assert.Len(t, hosts, 2) + + host, err := db.SearchHosts("foo", []uint{h3.ID}) + assert.Nil(t, err) + assert.Len(t, host, 1) + assert.Equal(t, "foo.local", host[0].HostName) + + none, err := db.SearchHosts("xxx", nil) + assert.Nil(t, err) + assert.Len(t, none, 0) +} + +func testSearchHostsLimit(t *testing.T, db kolide.Datastore) { + for i := 0; i < 15; i++ { + _, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: fmt.Sprintf("%d", i), + UUID: fmt.Sprintf("%d", i), + HostName: fmt.Sprintf("foo.%d.local", i), + PrimaryIP: fmt.Sprintf("192.168.1.%d", i+1), + }) + require.Nil(t, err) + } + + hosts, err := db.SearchHosts("foo", nil) + require.Nil(t, err) + assert.Len(t, hosts, 10) +} diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index 067a7f96b2..61eed2f485 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -1,12 +1,14 @@ package datastore import ( + "fmt" "sort" "testing" "time" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func testLabels(t *testing.T, db kolide.Datastore) { @@ -219,3 +221,178 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) { assert.Nil(t, err) assert.Len(t, labels, 2) } + +func testSearchLabels(t *testing.T, db kolide.Datastore) { + _, err := db.NewLabel(&kolide.Label{ + Name: "foo", + }) + require.Nil(t, err) + + _, err = db.NewLabel(&kolide.Label{ + Name: "bar", + }) + require.Nil(t, err) + + l3, err := db.NewLabel(&kolide.Label{ + Name: "foo-bar", + }) + require.Nil(t, err) + + labels, err := db.SearchLabels("foo", nil) + assert.Nil(t, err) + assert.Len(t, labels, 2) + + label, err := db.SearchLabels("foo", []uint{l3.ID}) + assert.Nil(t, err) + assert.Len(t, label, 1) + assert.Equal(t, "foo", label[0].Name) + + none, err := db.SearchLabels("xxx", nil) + assert.Nil(t, err) + assert.Len(t, none, 0) +} + +func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) { + for i := 0; i < 15; i++ { + _, err := db.NewLabel(&kolide.Label{ + Name: fmt.Sprintf("foo-%d", i), + }) + require.Nil(t, err) + } + + labels, err := db.SearchLabels("foo", nil) + require.Nil(t, err) + assert.Len(t, labels, 10) +} + +func testListHostsInLabel(t *testing.T, db kolide.Datastore) { + h1, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + require.Nil(t, err) + + h2, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "2", + UUID: "2", + HostName: "bar.local", + PrimaryIP: "192.168.1.11", + }) + require.Nil(t, err) + + h3, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "3", + UUID: "3", + HostName: "baz.local", + PrimaryIP: "192.168.1.12", + }) + require.Nil(t, err) + + l1, err := db.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + require.Nil(t, err) + require.NotZero(t, l1.ID) + l1ID := fmt.Sprintf("%d", l1.ID) + + { + + hosts, err := db.ListHostsInLabel(l1.ID) + require.Nil(t, err) + assert.Len(t, hosts, 0) + } + + for _, h := range []*kolide.Host{h1, h2, h3} { + err = db.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + assert.Nil(t, err) + } + + { + hosts, err := db.ListHostsInLabel(l1.ID) + require.Nil(t, err) + assert.Len(t, hosts, 3) + } +} + +func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) { + h1, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + require.Nil(t, err) + + h2, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "2", + UUID: "2", + HostName: "bar.local", + PrimaryIP: "192.168.1.11", + }) + require.Nil(t, err) + + h3, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "3", + UUID: "3", + HostName: "baz.local", + PrimaryIP: "192.168.1.12", + }) + require.Nil(t, err) + + h4, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "4", + UUID: "4", + HostName: "xxx.local", + PrimaryIP: "192.168.1.13", + }) + require.Nil(t, err) + + h5, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "5", + UUID: "5", + HostName: "yyy.local", + PrimaryIP: "192.168.1.14", + }) + require.Nil(t, err) + + l1, err := db.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + require.Nil(t, err) + require.NotZero(t, l1.ID) + l1ID := fmt.Sprintf("%d", l1.ID) + + l2, err := db.NewLabel(&kolide.Label{ + Name: "label bar", + QueryID: 2, + }) + require.Nil(t, err) + require.NotZero(t, l2.ID) + l2ID := fmt.Sprintf("%d", l2.ID) + + for _, h := range []*kolide.Host{h1, h2, h3} { + err = db.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + assert.Nil(t, err) + } + + for _, h := range []*kolide.Host{h3, h4, h5} { + err = db.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now()) + assert.Nil(t, err) + } + + hosts, err := db.ListUniqueHostsInLabels([]uint{l1.ID, l2.ID}) + assert.Nil(t, err) + assert.Len(t, hosts, 5) +} diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index a4a8b63baa..91c9cbf150 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -31,4 +31,10 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testSaveUser, testUserByID, testPasswordResetRequests, + testSearchHosts, + testSearchHostsLimit, + testSearchLabels, + testSearchLabelsLimit, + testListHostsInLabel, + testListUniqueHostsInLabels, } diff --git a/server/datastore/gorm.go b/server/datastore/gorm.go index 62f6e399a0..acf8979ba2 100644 --- a/server/datastore/gorm.go +++ b/server/datastore/gorm.go @@ -48,7 +48,34 @@ func (orm gormDB) Migrate() error { } // Have to manually add indexes. Yuck! - orm.DB.Model(&kolide.LabelQueryExecution{}).AddUniqueIndex("idx_lqe_label_host", "label_id", "host_id") + err := orm.DB.Model(&kolide.LabelQueryExecution{}).AddUniqueIndex("idx_lqe_label_host", "label_id", "host_id").Error + if err != nil { + return err + } + + indexes := []interface{}{} + err = orm.DB.Raw("SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = 'kolide' AND INDEX_NAME = 'hosts_search';").Scan(&indexes).Error + if err != nil { + return err + } + if len(indexes) == 0 { + err = orm.DB.Exec("CREATE FULLTEXT INDEX hosts_search ON hosts(host_name, primary_ip);").Error + if err != nil { + return err + } + } + + indexes = []interface{}{} + err = orm.DB.Raw("SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = 'kolide' AND INDEX_NAME = 'labels_search';").Scan(&indexes).Error + if err != nil { + return err + } + if len(indexes) == 0 { + err = orm.DB.Exec("CREATE FULLTEXT INDEX labels_search ON labels(name);").Error + if err != nil { + return err + } + } return nil } diff --git a/server/datastore/gorm_hosts.go b/server/datastore/gorm_hosts.go index 80fcbe7d4e..ef5a6d7531 100644 --- a/server/datastore/gorm_hosts.go +++ b/server/datastore/gorm_hosts.go @@ -131,3 +131,28 @@ func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error { host.UpdatedAt = t return nil } + +func (orm gormDB) SearchHosts(query string, omit []uint) ([]kolide.Host, error) { + sql := ` +SELECT * +FROM hosts +WHERE MATCH(host_name, primary_ip) +AGAINST(? IN BOOLEAN MODE) +` + results := []kolide.Host{} + + var db *gorm.DB + if len(omit) > 0 { + sql += "AND id NOT IN (?) LIMIT 10;" + db = orm.DB.Raw(sql, query+"*", omit) + } else { + sql += "LIMIT 10;" + db = orm.DB.Raw(sql, query+"*") + } + + err := db.Scan(&results).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, errors.DatabaseError(err) + } + return results, nil +} diff --git a/server/datastore/gorm_labels.go b/server/datastore/gorm_labels.go index eb26403dfa..56da3ec2f2 100644 --- a/server/datastore/gorm_labels.go +++ b/server/datastore/gorm_labels.go @@ -150,3 +150,69 @@ AND lqe.matches return results, nil } + +func (orm gormDB) SearchLabels(query string, omit []uint) ([]kolide.Label, error) { + sql := ` +SELECT * +FROM labels +WHERE MATCH(name) +AGAINST(? IN BOOLEAN MODE) +` + results := []kolide.Label{} + + var db *gorm.DB + if len(omit) > 0 { + sql += "AND id NOT IN (?) LIMIT 10;" + db = orm.DB.Raw(sql, query+"*", omit) + } else { + sql += "LIMIT 10;" + db = orm.DB.Raw(sql, query+"*") + } + + err := db.Scan(&results).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, errors.DatabaseError(err) + } + return results, nil +} + +func (orm gormDB) ListHostsInLabel(lid uint) ([]kolide.Host, error) { + results := []kolide.Host{} + err := orm.DB.Raw(` +SELECT h.* +FROM label_query_executions lqe +JOIN hosts h +ON lqe.host_id = h.id +WHERE lqe.label_id = ? +AND lqe.matches = 1; +`, lid).Scan(&results).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, errors.DatabaseError(err) + } + + return results, nil +} + +func (orm gormDB) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { + if labels == nil || len(labels) == 0 { + return nil, nil + } + + results := []kolide.Host{} + err := orm.DB.Raw(` +SELECT h.* +FROM label_query_executions lqe +JOIN hosts h +ON lqe.host_id = h.id +WHERE lqe.label_id in (?) +AND lqe.matches = 1 +GROUP BY h.id; +`, labels).Scan(&results).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, errors.DatabaseError(err) + } + + return results, nil +} diff --git a/server/datastore/gorm_packs.go b/server/datastore/gorm_packs.go index 6f9ad6a4a9..50a673f325 100644 --- a/server/datastore/gorm_packs.go +++ b/server/datastore/gorm_packs.go @@ -138,9 +138,11 @@ func (orm gormDB) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) er func (orm gormDB) AddLabelToPack(lid uint, pid uint) error { pt := &kolide.PackTarget{ - Type: kolide.TargetLabel, - PackID: pid, - TargetID: lid, + PackID: pid, + Target: kolide.Target{ + Type: kolide.TargetLabel, + TargetID: lid, + }, } return orm.DB.Create(pt).Error @@ -191,11 +193,5 @@ func (orm gormDB) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) er ) } - pt := &kolide.PackTarget{ - Type: kolide.TargetLabel, - PackID: pack.ID, - TargetID: label.ID, - } - - return orm.DB.Delete(pt).Error + return orm.DB.Where("pack_id = ? AND type = ? AND target_id = ?", pack.ID, kolide.TargetLabel, label.ID).Delete(&kolide.PackTarget{}).Error } diff --git a/server/datastore/inmem_hosts.go b/server/datastore/inmem_hosts.go index 7f5948a8a1..06ea019114 100644 --- a/server/datastore/inmem_hosts.go +++ b/server/datastore/inmem_hosts.go @@ -3,6 +3,7 @@ package datastore import ( "errors" "sort" + "strings" "time" "github.com/kolide/kolide-ose/server/kolide" @@ -178,3 +179,27 @@ func (orm *inmem) MarkHostSeen(host *kolide.Host, t time.Time) error { } return nil } + +func (orm *inmem) SearchHosts(query string, omit []uint) ([]kolide.Host, error) { + omitLookup := map[uint]bool{} + for _, o := range omit { + omitLookup[o] = true + } + + var results []kolide.Host + + orm.mtx.Lock() + defer orm.mtx.Unlock() + + for _, h := range orm.hosts { + if len(results) == 10 { + break + } + + if (strings.Contains(h.HostName, query) || strings.Contains(h.PrimaryIP, query)) && !omitLookup[h.ID] { + results = append(results, *h) + } + } + + return results, nil +} diff --git a/server/datastore/inmem_labels.go b/server/datastore/inmem_labels.go index 291d2f6c8b..34f0f98d3b 100644 --- a/server/datastore/inmem_labels.go +++ b/server/datastore/inmem_labels.go @@ -4,6 +4,7 @@ import ( "errors" "sort" "strconv" + "strings" "time" "github.com/kolide/kolide-ose/server/kolide" @@ -193,3 +194,68 @@ func (orm *inmem) SaveLabel(label *kolide.Label) error { return nil } + +func (orm *inmem) SearchLabels(query string, omit []uint) ([]kolide.Label, error) { + omitLookup := map[uint]bool{} + for _, o := range omit { + omitLookup[o] = true + } + + var results []kolide.Label + + orm.mtx.Lock() + defer orm.mtx.Unlock() + + for _, l := range orm.labels { + if len(results) == 10 { + break + } + + if strings.Contains(l.Name, query) && !omitLookup[l.ID] { + results = append(results, *l) + continue + } + } + + return results, nil +} + +func (orm *inmem) ListHostsInLabel(lid uint) ([]kolide.Host, error) { + var hosts []kolide.Host + + orm.mtx.Lock() + defer orm.mtx.Unlock() + + for _, lqe := range orm.labelQueryExecutions { + if lqe.LabelID == lid && lqe.Matches { + hosts = append(hosts, *orm.hosts[lqe.HostID]) + } + } + + return hosts, nil +} + +func (orm *inmem) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { + var hosts []kolide.Host + + labelSet := map[uint]bool{} + hostSet := map[uint]bool{} + + for _, label := range labels { + labelSet[label] = true + } + + orm.mtx.Lock() + defer orm.mtx.Unlock() + + for _, lqe := range orm.labelQueryExecutions { + if labelSet[lqe.LabelID] && lqe.Matches { + if !hostSet[lqe.HostID] { + hosts = append(hosts, *orm.hosts[lqe.HostID]) + hostSet[lqe.HostID] = true + } + } + } + + return hosts, nil +} diff --git a/server/datastore/inmem_packs.go b/server/datastore/inmem_packs.go index e0a69e9451..40754f8278 100644 --- a/server/datastore/inmem_packs.go +++ b/server/datastore/inmem_packs.go @@ -143,9 +143,11 @@ func (orm *inmem) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) er func (orm *inmem) AddLabelToPack(lid uint, pid uint) error { pt := &kolide.PackTarget{ - Type: kolide.TargetLabel, - PackID: pid, - TargetID: lid, + PackID: pid, + Target: kolide.Target{ + Type: kolide.TargetLabel, + TargetID: lid, + }, } orm.mtx.Lock() diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index 65ea0cebf9..3fd28f9a00 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -15,6 +15,7 @@ type HostStore interface { EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error) AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error + SearchHosts(query string, omit []uint) ([]Host, error) } type HostService interface { @@ -30,7 +31,7 @@ type Host struct { UpdatedAt time.Time `json:"updated_at"` DetailUpdateTime time.Time `json:"detail_updated_at"` // Time that the host details were last updated NodeKey string `json:"-" gorm:"unique_index:idx_host_unique_nodekey"` - HostName string `json:"hostname"` + HostName string `json:"hostname"` // there is a fulltext index on this field UUID string `json:"uuid" gorm:"unique_index:idx_host_unique_uuid"` Platform string `json:"platform"` OsqueryVersion string `json:"osquery_version"` @@ -38,5 +39,5 @@ type Host struct { Uptime time.Duration `json:"uptime"` PhysicalMemory int `json:"memory" sql:"type:bigint"` PrimaryMAC string `json:"mac"` - PrimaryIP string `json:"ip"` + PrimaryIP string `json:"ip"` // there is a fulltext index on this field } diff --git a/server/kolide/labels.go b/server/kolide/labels.go index 55ae311af6..4d02d069bc 100644 --- a/server/kolide/labels.go +++ b/server/kolide/labels.go @@ -29,6 +29,11 @@ type LabelStore interface { // LabelsForHost returns the labels that the given host is in. ListLabelsForHost(hid uint) ([]Label, error) + + ListHostsInLabel(lid uint) ([]Host, error) + ListUniqueHostsInLabels(labels []uint) ([]Host, error) + + SearchLabels(query string, omit []uint) ([]Label, error) } type LabelService interface { @@ -48,7 +53,7 @@ type Label struct { ID uint `json:"id" gorm:"primary_key"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - Name string `json:"name" gorm:"not null;unique_index:idx_label_unique_name"` + Name string `json:"name" gorm:"not null;unique_index:idx_label_unique_name"` // there is a fulltext index on this field QueryID uint `json:"query_id"` } diff --git a/server/kolide/packs.go b/server/kolide/packs.go index 97db37c7ea..00cf0d6a2d 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -59,16 +59,8 @@ type PackQuery struct { QueryID uint } -type TargetType int - -const ( - TargetLabel TargetType = iota - TargetHost -) - type PackTarget struct { ID uint `gorm:"primary_key"` - Type TargetType PackID uint - TargetID uint + Target } diff --git a/server/kolide/service.go b/server/kolide/service.go index 0700c07071..5cc8db1bf0 100644 --- a/server/kolide/service.go +++ b/server/kolide/service.go @@ -11,4 +11,5 @@ type Service interface { HostService AppConfigService InviteService + TargetService } diff --git a/server/kolide/target.go b/server/kolide/target.go new file mode 100644 index 0000000000..1aa53ac79a --- /dev/null +++ b/server/kolide/target.go @@ -0,0 +1,31 @@ +package kolide + +import ( + "golang.org/x/net/context" +) + +type TargetSearchResults struct { + Hosts []Host + Labels []Label +} + +type TargetService interface { + // SearchTargets will accept a search query, a slice of IDs of hosts to omit, + // and a slice of IDs of labels to omit, and it will return a set of targets + // (hosts and label) which match the supplied search query. + SearchTargets(ctx context.Context, query string, selectedHostIDs []uint, selectedLabelIDs []uint) (*TargetSearchResults, error) + + CountHostsInTargets(ctx context.Context, hosts []uint, labels []uint) (uint, error) +} + +type TargetType int + +const ( + TargetLabel TargetType = iota + TargetHost +) + +type Target struct { + Type TargetType + TargetID uint +} diff --git a/server/service/endpoint_targets.go b/server/service/endpoint_targets.go new file mode 100644 index 0000000000..c364c3e0e0 --- /dev/null +++ b/server/service/endpoint_targets.go @@ -0,0 +1,86 @@ +package service + +import ( + "github.com/go-kit/kit/endpoint" + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +//////////////////////////////////////////////////////////////////////////////// +// Search Targrets +//////////////////////////////////////////////////////////////////////////////// + +type searchTargetsRequest struct { + Query string `json:"query"` + Selected struct { + Labels []uint `json:"labels"` + Hosts []uint `json:"hosts"` + } `json:"selected"` +} + +type hostSearchResult struct { + hostResponse + DisplayText string `json:"display_text"` +} + +type labelSearchResult struct { + kolide.Label + DisplayText string `json:"display_text"` +} + +type targetsData struct { + Hosts []hostSearchResult `json:"hosts"` + Labels []labelSearchResult `json:"labels"` +} + +type searchTargetsResponse struct { + Targets *targetsData `json:"targets,omitempty"` + SelectedTargetsCount uint `json:"selected_targets_count,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r searchTargetsResponse) error() error { return r.Err } + +func makeSearchTargetsEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(searchTargetsRequest) + + results, err := svc.SearchTargets(ctx, req.Query, req.Selected.Hosts, req.Selected.Labels) + if err != nil { + return searchTargetsResponse{Err: err}, nil + } + + count, err := svc.CountHostsInTargets(ctx, req.Selected.Hosts, req.Selected.Labels) + if err != nil { + return searchTargetsResponse{Err: err}, nil + } + + targets := &targetsData{ + Hosts: []hostSearchResult{}, + Labels: []labelSearchResult{}, + } + + for _, host := range results.Hosts { + targets.Hosts = append(targets.Hosts, + hostSearchResult{ + hostResponse{host, svc.HostStatus(ctx, host)}, + host.HostName, + }, + ) + } + + for _, label := range results.Labels { + targets.Labels = append(targets.Labels, + labelSearchResult{ + label, + label.Name, + }, + ) + } + + return searchTargetsResponse{ + Targets: targets, + SelectedTargetsCount: count, + }, nil + } +} diff --git a/server/service/handler.go b/server/service/handler.go index 18a5d3f9a1..1afc23cc8d 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -60,6 +60,7 @@ type KolideEndpoints struct { GetHost endpoint.Endpoint DeleteHost endpoint.Endpoint ListHosts endpoint.Endpoint + SearchTargets endpoint.Endpoint } // MakeKolideServerEndpoints creates the Kolide API endpoints. @@ -101,6 +102,15 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint GetHost: authenticatedUser(jwtKey, svc, makeGetHostEndpoint(svc)), ListHosts: authenticatedUser(jwtKey, svc, makeListHostsEndpoint(svc)), DeleteHost: authenticatedUser(jwtKey, svc, makeDeleteHostEndpoint(svc)), + GetLabel: authenticatedUser(jwtKey, svc, makeGetLabelEndpoint(svc)), + ListLabels: authenticatedUser(jwtKey, svc, makeListLabelsEndpoint(svc)), + CreateLabel: authenticatedUser(jwtKey, svc, makeCreateLabelEndpoint(svc)), + ModifyLabel: authenticatedUser(jwtKey, svc, makeModifyLabelEndpoint(svc)), + DeleteLabel: authenticatedUser(jwtKey, svc, makeDeleteLabelEndpoint(svc)), + AddLabelToPack: authenticatedUser(jwtKey, svc, makeAddLabelToPackEndpoint(svc)), + GetLabelsForPack: authenticatedUser(jwtKey, svc, makeGetLabelsForPackEndpoint(svc)), + DeleteLabelFromPack: authenticatedUser(jwtKey, svc, makeDeleteLabelFromPackEndpoint(svc)), + SearchTargets: authenticatedUser(jwtKey, svc, makeSearchTargetsEndpoint(svc)), // Osquery endpoints EnrollAgent: makeEnrollAgentEndpoint(svc), @@ -108,14 +118,6 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint GetDistributedQueries: authenticatedHost(svc, makeGetDistributedQueriesEndpoint(svc)), SubmitDistributedQueryResults: authenticatedHost(svc, makeSubmitDistributedQueryResultsEndpoint(svc)), SubmitLogs: authenticatedHost(svc, makeSubmitLogsEndpoint(svc)), - GetLabel: authenticatedUser(jwtKey, svc, makeGetLabelEndpoint(svc)), - ListLabels: authenticatedUser(jwtKey, svc, makeListLabelsEndpoint(svc)), - CreateLabel: authenticatedUser(jwtKey, svc, makeCreateLabelEndpoint(svc)), - ModifyLabel: authenticatedUser(jwtKey, svc, makeModifyLabelEndpoint(svc)), - DeleteLabel: authenticatedUser(jwtKey, svc, makeDeleteLabelEndpoint(svc)), - AddLabelToPack: authenticatedUser(jwtKey, svc, makeAddLabelToPackEndpoint(svc)), - GetLabelsForPack: authenticatedUser(jwtKey, svc, makeGetLabelsForPackEndpoint(svc)), - DeleteLabelFromPack: authenticatedUser(jwtKey, svc, makeDeleteLabelFromPackEndpoint(svc)), } } @@ -167,6 +169,7 @@ type kolideHandlers struct { GetHost *kithttp.Server DeleteHost *kithttp.Server ListHosts *kithttp.Server + SearchTargets *kithttp.Server } func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithttp.ServerOption) kolideHandlers { @@ -221,6 +224,7 @@ func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithtt GetHost: newServer(e.GetHost, decodeGetHostRequest), DeleteHost: newServer(e.DeleteHost, decodeDeleteHostRequest), ListHosts: newServer(e.ListHosts, decodeListHostsRequest), + SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest), } } @@ -296,6 +300,8 @@ func attachKolideAPIRoutes(r *mux.Router, h kolideHandlers) { r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET") r.Handle("/api/v1/kolide/hosts/{id}", h.DeleteHost).Methods("DELETE") + r.Handle("/api/v1/kolide/targets", h.SearchTargets).Methods("POST") + r.Handle("/api/v1/osquery/enroll", h.EnrollAgent).Methods("POST") r.Handle("/api/v1/osquery/config", h.GetClientConfig).Methods("POST") r.Handle("/api/v1/osquery/distributed/read", h.GetDistributedQueries).Methods("POST") diff --git a/server/service/service_targets.go b/server/service/service_targets.go new file mode 100644 index 0000000000..dff92bf36a --- /dev/null +++ b/server/service/service_targets.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +func (svc service) SearchTargets(ctx context.Context, query string, selectedHostIDs []uint, selectedLabelIDs []uint) (*kolide.TargetSearchResults, error) { + results := &kolide.TargetSearchResults{} + + hosts, err := svc.ds.SearchHosts(query, selectedHostIDs) + if err != nil { + return nil, err + } + results.Hosts = hosts + + labels, err := svc.ds.SearchLabels(query, selectedLabelIDs) + if err != nil { + return nil, err + } + results.Labels = labels + + return results, nil +} + +func (svc service) CountHostsInTargets(ctx context.Context, hosts []uint, labels []uint) (uint, error) { + hostsInLabels, err := svc.ds.ListUniqueHostsInLabels(labels) + if err != nil { + return 0, err + } + + hostLookup := map[uint]bool{} + + for _, host := range hosts { + hostLookup[host] = true + } + + for _, host := range hostsInLabels { + if !hostLookup[host.ID] { + hostLookup[host.ID] = true + } + } + + return uint(len(hostLookup)), nil +} diff --git a/server/service/service_targets_test.go b/server/service/service_targets_test.go new file mode 100644 index 0000000000..acef6686bc --- /dev/null +++ b/server/service/service_targets_test.go @@ -0,0 +1,268 @@ +package service + +import ( + "fmt" + "testing" + "time" + + "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +func TestSearchTargets(t *testing.T) { + ds, err := datastore.New("inmem", "") + require.Nil(t, err) + + svc, err := newTestService(ds) + require.Nil(t, err) + + ctx := context.Background() + + h1, err := ds.NewHost(&kolide.Host{ + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + require.Nil(t, err) + + l1, err := ds.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + require.Nil(t, err) + + results, err := svc.SearchTargets(ctx, "foo", nil, nil) + require.Nil(t, err) + + require.Len(t, results.Hosts, 1) + assert.Equal(t, h1.HostName, results.Hosts[0].HostName) + + require.Len(t, results.Labels, 1) + assert.Equal(t, l1.Name, results.Labels[0].Name) +} + +func TestCountHostsInTargets(t *testing.T) { + ds, err := datastore.New("inmem", "") + require.Nil(t, err) + + svc, err := newTestService(ds) + require.Nil(t, err) + + ctx := context.Background() + + h1, err := ds.NewHost(&kolide.Host{ + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + NodeKey: "1", + UUID: "1", + }) + require.Nil(t, err) + + h2, err := ds.NewHost(&kolide.Host{ + HostName: "bar.local", + PrimaryIP: "192.168.1.11", + NodeKey: "2", + UUID: "2", + }) + require.Nil(t, err) + + h3, err := ds.NewHost(&kolide.Host{ + HostName: "baz.local", + PrimaryIP: "192.168.1.12", + NodeKey: "3", + UUID: "3", + }) + require.Nil(t, err) + + h4, err := ds.NewHost(&kolide.Host{ + HostName: "xxx.local", + PrimaryIP: "192.168.1.13", + NodeKey: "4", + UUID: "4", + }) + require.Nil(t, err) + + h5, err := ds.NewHost(&kolide.Host{ + HostName: "yyy.local", + PrimaryIP: "192.168.1.14", + NodeKey: "5", + UUID: "5", + }) + require.Nil(t, err) + + l1, err := ds.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + require.Nil(t, err) + require.NotZero(t, l1.ID) + l1ID := fmt.Sprintf("%d", l1.ID) + + l2, err := ds.NewLabel(&kolide.Label{ + Name: "label bar", + QueryID: 1, + }) + require.Nil(t, err) + require.NotZero(t, l2.ID) + l2ID := fmt.Sprintf("%d", l2.ID) + + for _, h := range []*kolide.Host{h1, h2, h3} { + err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + assert.Nil(t, err) + } + + for _, h := range []*kolide.Host{h3, h4, h5} { + err = ds.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now()) + assert.Nil(t, err) + } + + count, err := svc.CountHostsInTargets(ctx, nil, []uint{l1.ID, l2.ID}) + assert.Nil(t, err) + assert.Equal(t, uint(5), count) + + count, err = svc.CountHostsInTargets(ctx, []uint{h1.ID, h2.ID}, []uint{l1.ID, l2.ID}) + assert.Nil(t, err) + assert.Equal(t, uint(5), count) + + count, err = svc.CountHostsInTargets(ctx, []uint{h1.ID, h2.ID}, nil) + assert.Nil(t, err) + assert.Equal(t, uint(2), count) + + count, err = svc.CountHostsInTargets(ctx, []uint{h1.ID}, []uint{l2.ID}) + assert.Nil(t, err) + assert.Equal(t, uint(4), count) + + count, err = svc.CountHostsInTargets(ctx, nil, nil) + assert.Nil(t, err) + assert.Equal(t, uint(0), count) +} + +func TestSearchWithOmit(t *testing.T) { + ds, err := datastore.New("inmem", "") + require.Nil(t, err) + + svc, err := newTestService(ds) + require.Nil(t, err) + + ctx := context.Background() + + h1, err := ds.NewHost(&kolide.Host{ + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + NodeKey: "1", + UUID: "1", + }) + require.Nil(t, err) + + h2, err := ds.NewHost(&kolide.Host{ + HostName: "foobar.local", + PrimaryIP: "192.168.1.11", + NodeKey: "2", + UUID: "2", + }) + require.Nil(t, err) + + l1, err := ds.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + + { + results, err := svc.SearchTargets(ctx, "foo", nil, nil) + require.Nil(t, err) + + require.Len(t, results.Hosts, 2) + + require.Len(t, results.Labels, 1) + assert.Equal(t, l1.Name, results.Labels[0].Name) + } + + { + results, err := svc.SearchTargets(ctx, "foo", []uint{h2.ID}, nil) + require.Nil(t, err) + + require.Len(t, results.Hosts, 1) + assert.Equal(t, h1.HostName, results.Hosts[0].HostName) + + require.Len(t, results.Labels, 1) + assert.Equal(t, l1.Name, results.Labels[0].Name) + } +} + +func TestSearchHostsInLabels(t *testing.T) { + ds, err := datastore.New("inmem", "") + require.Nil(t, err) + + svc, err := newTestService(ds) + require.Nil(t, err) + + ctx := context.Background() + + h1, err := ds.NewHost(&kolide.Host{ + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + NodeKey: "1", + UUID: "1", + }) + require.Nil(t, err) + + h2, err := ds.NewHost(&kolide.Host{ + HostName: "bar.local", + PrimaryIP: "192.168.1.11", + NodeKey: "2", + UUID: "2", + }) + require.Nil(t, err) + + h3, err := ds.NewHost(&kolide.Host{ + HostName: "baz.local", + PrimaryIP: "192.168.1.12", + NodeKey: "3", + UUID: "3", + }) + require.Nil(t, err) + + l1, err := ds.NewLabel(&kolide.Label{ + Name: "label foo", + QueryID: 1, + }) + require.Nil(t, err) + require.NotZero(t, l1.ID) + l1ID := fmt.Sprintf("%d", l1.ID) + + for _, h := range []*kolide.Host{h1, h2, h3} { + err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + assert.Nil(t, err) + } + + results, err := svc.SearchTargets(ctx, "baz", nil, nil) + require.Nil(t, err) + + require.Len(t, results.Hosts, 1) + assert.Equal(t, h3.HostName, results.Hosts[0].HostName) +} + +func TestSearchResultsLimit(t *testing.T) { + ds, err := datastore.New("inmem", "") + require.Nil(t, err) + + svc, err := newTestService(ds) + require.Nil(t, err) + + ctx := context.Background() + + for i := 0; i < 15; i++ { + _, err := ds.NewHost(&kolide.Host{ + HostName: fmt.Sprintf("foo.%d.local", i), + PrimaryIP: fmt.Sprintf("192.168.1.%d", i+1), + NodeKey: fmt.Sprintf("%d", i+1), + UUID: fmt.Sprintf("%d", i+1), + }) + require.Nil(t, err) + } + targets, err := svc.SearchTargets(ctx, "foo", nil, nil) + require.Nil(t, err) + assert.Len(t, targets.Hosts, 10) +} diff --git a/server/service/transport_targets.go b/server/service/transport_targets.go new file mode 100644 index 0000000000..2f15923c02 --- /dev/null +++ b/server/service/transport_targets.go @@ -0,0 +1,17 @@ +package service + +import ( + "encoding/json" + "net/http" + + "golang.org/x/net/context" +) + +func decodeSearchTargetsRequest(ctx context.Context, r *http.Request) (interface{}, error) { + var req searchTargetsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return nil, err + } + + return req, nil +} diff --git a/server/service/transport_targets_test.go b/server/service/transport_targets_test.go new file mode 100644 index 0000000000..2337f153f0 --- /dev/null +++ b/server/service/transport_targets_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +func TestDecodeSearchTargetsRequest(t *testing.T) { + router := mux.NewRouter() + router.HandleFunc("/api/v1/kolide/targets", func(writer http.ResponseWriter, request *http.Request) { + r, err := decodeSearchTargetsRequest(context.Background(), request) + assert.Nil(t, err) + + params := r.(searchTargetsRequest) + assert.Equal(t, "bar", params.Query) + assert.Len(t, params.Selected.Hosts, 3) + assert.Len(t, params.Selected.Labels, 2) + }).Methods("POST") + var body bytes.Buffer + + body.Write([]byte(`{ + "query": "bar", + "selected": { + "hosts": [ + 1, + 2, + 3 + ], + "labels": [ + 1, + 2 + ] + } + }`)) + + router.ServeHTTP( + httptest.NewRecorder(), + httptest.NewRequest("POST", "/api/v1/kolide/targets", &body), + ) +}