diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index ef2d3714ed..50f3a3e845 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -2,6 +2,7 @@ package datastore import ( "fmt" + "strconv" "testing" "time" @@ -96,6 +97,16 @@ func testSaveHosts(t *testing.T, db kolide.Datastore) { assert.Equal(t, host.NetworkInterfaces[0].ID, *host.PrimaryNetworkInterfaceID) assert.Equal(t, 1, len(host.NetworkInterfaces)) + // remove all nics primary nic should be nil + host.NetworkInterfaces = []*kolide.NetworkInterface{} + err = db.SaveHost(host) + require.Nil(t, err) + assert.Nil(t, host.PrimaryNetworkInterfaceID) + host, err = db.Host(host.ID) + require.Nil(t, err) + require.NotNil(t, host) + assert.Nil(t, host.PrimaryNetworkInterfaceID) + err = db.DeleteHost(host) assert.Nil(t, err) @@ -111,8 +122,8 @@ func testDeleteHost(t *testing.T, db kolide.Datastore) { UUID: "1", HostName: "foo.local", }) - assert.Nil(t, err) - assert.NotNil(t, host) + require.Nil(t, err) + require.NotNil(t, host) err = db.DeleteHost(host) assert.Nil(t, err) @@ -126,6 +137,7 @@ func testListHost(t *testing.T, db kolide.Datastore) { for i := 0; i < 10; i++ { host, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), UUID: fmt.Sprintf("%d", i), HostName: fmt.Sprintf("foo.local%d", i), @@ -137,9 +149,39 @@ func testListHost(t *testing.T, db kolide.Datastore) { hosts = append(hosts, host) } + hosts[1].NetworkInterfaces = []*kolide.NetworkInterface{ + &kolide.NetworkInterface{ + Interface: "en0", + IPAddress: "99.100.101.102", + }, + &kolide.NetworkInterface{ + Interface: "en1", + IPAddress: "99.100.101.103", + }, + } + + err := db.SaveHost(hosts[1]) + require.Nil(t, err) + + hosts[3].NetworkInterfaces = []*kolide.NetworkInterface{ + &kolide.NetworkInterface{ + Interface: "en2", + IPAddress: "99.100.101.104", + }, + } + err = db.SaveHost(hosts[3]) + require.Nil(t, err) + hosts2, err := db.ListHosts(kolide.ListOptions{}) require.Nil(t, err) assert.Equal(t, len(hosts), len(hosts2)) + + require.Equal(t, 2, len(hosts2[1].NetworkInterfaces)) + require.Equal(t, 0, len(hosts2[2].NetworkInterfaces)) + require.Equal(t, 1, len(hosts2[3].NetworkInterfaces)) + assert.Equal(t, "en1", hosts2[1].NetworkInterfaces[1].Interface) + assert.Equal(t, "en2", hosts2[3].NetworkInterfaces[0].Interface) + err = db.DeleteHost(hosts[0]) require.Nil(t, err) hosts2, err = db.ListHosts(kolide.ListOptions{}) @@ -173,32 +215,19 @@ func testListHost(t *testing.T, db kolide.Datastore) { func testEnrollHost(t *testing.T, db kolide.Datastore) { var hosts []*kolide.Host for _, tt := range enrollTests { - h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.platform, tt.nodeKeySize) + h, err := db.EnrollHost(tt.uuid, tt.nodeKeySize) require.Nil(t, err) hosts = append(hosts, h) - assert.Equal(t, tt.uuid, h.UUID) - assert.Equal(t, tt.hostname, h.HostName) - assert.Equal(t, tt.platform, h.Platform) + assert.Equal(t, tt.uuid, h.OsqueryHostID) assert.NotEmpty(t, h.NodeKey) } - for _, enrolled := range hosts { - oldNodeKey := enrolled.NodeKey - newhostname := fmt.Sprintf("changed.%s", enrolled.HostName) - - h, err := db.EnrollHost(enrolled.UUID, newhostname, enrolled.Platform, 15) - assert.Nil(t, err) - assert.Equal(t, enrolled.UUID, h.UUID) - assert.NotEmpty(t, h.NodeKey) - assert.NotEqual(t, oldNodeKey, h.NodeKey) - } - } func testAuthenticateHost(t *testing.T, db kolide.Datastore) { for _, tt := range enrollTests { - h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.platform, tt.nodeKeySize) + h, err := db.EnrollHost(tt.uuid, tt.nodeKeySize) require.Nil(t, err) returned, err := db.AuthenticateHost(h.NodeKey) @@ -215,6 +244,7 @@ func testAuthenticateHost(t *testing.T, db kolide.Datastore) { func testSearchHosts(t *testing.T, db kolide.Datastore) { _, err := db.NewHost(&kolide.Host{ + OsqueryHostID: "1234", DetailUpdateTime: time.Now(), NodeKey: "1", UUID: "1", @@ -223,6 +253,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) { require.Nil(t, err) h2, err := db.NewHost(&kolide.Host{ + OsqueryHostID: "5679", DetailUpdateTime: time.Now(), NodeKey: "2", UUID: "2", @@ -231,6 +262,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) { require.Nil(t, err) h3, err := db.NewHost(&kolide.Host{ + OsqueryHostID: "99999", DetailUpdateTime: time.Now(), NodeKey: "3", UUID: "3", @@ -286,7 +318,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) { h3.NetworkInterfaces = []*kolide.NetworkInterface{ &kolide.NetworkInterface{ - Interface: "en0", + Interface: "en3", IPAddress: "99.100.101.104", }, } @@ -295,6 +327,8 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) { hits, err = db.SearchHosts("99.100.101") require.Nil(t, err) assert.Equal(t, 2, len(hits)) + assert.Equal(t, 2, len(hits[0].NetworkInterfaces)) + assert.Equal(t, "en0", hits[0].NetworkInterfaces[0].Interface) hits, err = db.SearchHosts("99.100.101", h3.ID) require.Nil(t, err) @@ -306,6 +340,7 @@ func testSearchHostsLimit(t *testing.T, db kolide.Datastore) { for i := 0; i < 15; i++ { _, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), UUID: fmt.Sprintf("%d", i), HostName: fmt.Sprintf("foo.%d.local", i), @@ -320,6 +355,7 @@ func testSearchHostsLimit(t *testing.T, db kolide.Datastore) { func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { h1, err := db.NewHost(&kolide.Host{ + OsqueryHostID: "1", DetailUpdateTime: time.Now(), NodeKey: "1", UUID: "1", @@ -328,6 +364,7 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { require.Nil(t, err) h2, err := db.NewHost(&kolide.Host{ + OsqueryHostID: "2", DetailUpdateTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index cb66ba8926..e5d85c787d 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -3,6 +3,7 @@ package datastore import ( "fmt" "sort" + "strconv" "testing" "time" @@ -18,8 +19,8 @@ func testLabels(t *testing.T, db kolide.Datastore) { var host *kolide.Host var err error for i := 0; i < 10; i++ { - host, err = db.EnrollHost(string(i), "foo", "", 10) - assert.Nil(t, err, "enrollment should succeed") + host, err = db.EnrollHost(string(i), 10) + require.Nil(t, err, "enrollment should succeed") hosts = append(hosts, *host) } @@ -231,6 +232,7 @@ func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) { func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h1, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + OsqueryHostID: "1", NodeKey: "1", UUID: "1", HostName: "foo.local", @@ -239,6 +241,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h2, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + OsqueryHostID: "2", NodeKey: "2", UUID: "2", HostName: "bar.local", @@ -247,6 +250,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h3, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + OsqueryHostID: "3", NodeKey: "3", UUID: "3", HostName: "baz.local", @@ -297,45 +301,19 @@ func testBuiltInLabels(t *testing.T, db kolide.Datastore) { } func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) { - h1, err := db.NewHost(&kolide.Host{ - DetailUpdateTime: time.Now(), - NodeKey: "1", - UUID: "1", - HostName: "foo.local", - }) - require.Nil(t, err) - - h2, err := db.NewHost(&kolide.Host{ - DetailUpdateTime: time.Now(), - NodeKey: "2", - UUID: "2", - HostName: "bar.local", - }) - require.Nil(t, err) - - h3, err := db.NewHost(&kolide.Host{ - DetailUpdateTime: time.Now(), - NodeKey: "3", - UUID: "3", - HostName: "baz.local", - }) - require.Nil(t, err) - - h4, err := db.NewHost(&kolide.Host{ - DetailUpdateTime: time.Now(), - NodeKey: "4", - UUID: "4", - HostName: "xxx.local", - }) - require.Nil(t, err) - - h5, err := db.NewHost(&kolide.Host{ - DetailUpdateTime: time.Now(), - NodeKey: "5", - UUID: "5", - HostName: "yyy.local", - }) - require.Nil(t, err) + hosts := []*kolide.Host{} + for i := 0; i < 4; i++ { + h, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + OsqueryHostID: strconv.Itoa(i), + NodeKey: strconv.Itoa(i), + UUID: strconv.Itoa(i), + HostName: fmt.Sprintf("host_%d", i), + }) + require.Nil(t, err) + require.NotNil(t, h) + hosts = append(hosts, h) + } l1, err := db.NewLabel(&kolide.Label{ Name: "label foo", @@ -353,17 +331,17 @@ func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) { require.NotZero(t, l2.ID) l2ID := fmt.Sprintf("%d", l2.ID) - for _, h := range []*kolide.Host{h1, h2, h3} { - err = db.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + for i := 0; i < 3; i++ { + err = db.RecordLabelQueryExecutions(hosts[i], map[string]bool{l1ID: true}, time.Now()) + assert.Nil(t, err) + } + // host 2 executes twice + for i := 2; i < len(hosts); i++ { + err = db.RecordLabelQueryExecutions(hosts[i], map[string]bool{l2ID: true}, time.Now()) assert.Nil(t, err) } - for _, h := range []*kolide.Host{h3, h4, h5} { - err = db.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now()) - assert.Nil(t, err) - } - - hosts, err := db.ListUniqueHostsInLabels([]uint{l1.ID, l2.ID}) + uniqueHosts, err := db.ListUniqueHostsInLabels([]uint{l1.ID, l2.ID}) assert.Nil(t, err) - assert.Len(t, hosts, 5) + assert.Equal(t, len(hosts), len(uniqueHosts)) } diff --git a/server/datastore/datastore_packs_test.go b/server/datastore/datastore_packs_test.go index 925aa16b49..61aac42c8d 100644 --- a/server/datastore/datastore_packs_test.go +++ b/server/datastore/datastore_packs_test.go @@ -110,6 +110,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { h1, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: mockClock.Now(), HostName: "foobar.local", + OsqueryHostID: "1", NodeKey: "1", UUID: "1", }) @@ -129,6 +130,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { h2, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: mockClock.Now(), HostName: "foobaz.local", + OsqueryHostID: "2", NodeKey: "2", UUID: "2", }) diff --git a/server/datastore/inmem/hosts.go b/server/datastore/inmem/hosts.go index 0954bf1d81..f1174fb917 100644 --- a/server/datastore/inmem/hosts.go +++ b/server/datastore/inmem/hosts.go @@ -114,39 +114,33 @@ func (orm *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) return hosts, nil } -func (orm *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*kolide.Host, error) { +func (orm *Datastore) EnrollHost(osQueryHostID string, nodeKeySize int) (*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() - if uuid == "" { - return nil, errors.New("missing uuid for host enrollment") + if osQueryHostID == "" { + return nil, errors.New("missing host identifier from osquery for host enrollment") } - host := kolide.Host{ - UUID: uuid, - HostName: hostname, - Platform: platform, - DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour), - } - for _, h := range orm.hosts { - if h.UUID == uuid { - host = *h - break - } - } - - var err error - host.NodeKey, err = kolide.RandomText(nodeKeySize) + nodeKey, err := kolide.RandomText(nodeKeySize) if err != nil { return nil, err } - if hostname != "" { - host.HostName = hostname + host := kolide.Host{ + OsqueryHostID: osQueryHostID, + NodeKey: nodeKey, + DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour), } - if platform != "" { - host.Platform = platform + host.CreatedAt = time.Now().UTC() + host.UpdatedAt = host.CreatedAt + + for _, h := range orm.hosts { + if h.OsqueryHostID == osQueryHostID { + host = *h + break + } } if host.ID == 0 { @@ -185,13 +179,13 @@ func (orm *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { return nil } -func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) { +func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) { omitLookup := map[uint]bool{} for _, o := range omit { omitLookup[o] = true } - var results []kolide.Host + var results []*kolide.Host orm.mtx.Lock() defer orm.mtx.Unlock() @@ -202,13 +196,13 @@ func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, er } if strings.Contains(h.HostName, query) && !omitLookup[h.ID] { - results = append(results, *h) + results = append(results, h) continue } for _, nic := range h.NetworkInterfaces { if strings.Contains(nic.IPAddress, query) && !omitLookup[nic.HostID] { - results = append(results, *h) + results = append(results, h) break } } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 6aadd4eefd..467f1db326 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -9,11 +9,13 @@ import ( "github.com/jmoiron/sqlx" "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/patrickmn/sortutil" ) func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { sqlStatement := ` INSERT INTO hosts ( + osquery_host_id, detail_update_time, node_key, host_name, @@ -24,9 +26,9 @@ func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { uptime, physical_memory ) - VALUES( ?,?,?,?,?,?,?,?,? ) + VALUES( ?,?,?,?,?,?,?,?,?,? ) ` - result, err := d.db.Exec(sqlStatement, host.DetailUpdateTime, + result, err := d.db.Exec(sqlStatement, host.OsqueryHostID, host.DetailUpdateTime, host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion, host.OSVersion, host.Uptime, host.PhysicalMemory) if err != nil { @@ -211,6 +213,7 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { tx.Rollback() return errors.DatabaseError(err) } + if err = removedUnusedNics(tx, host); err != nil { tx.Rollback() return errors.DatabaseError(err) @@ -219,9 +222,10 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { if needsUpdate := host.ResetPrimaryNetwork(); needsUpdate { _, err = tx.Exec( "UPDATE hosts SET primary_ip_id = ? WHERE id = ?", - *host.PrimaryNetworkInterfaceID, + host.PrimaryNetworkInterfaceID, host.ID, ) + if err != nil { tx.Rollback() return errors.DatabaseError(err) @@ -235,7 +239,6 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { return nil } -// TODO needs test func (d *Datastore) DeleteHost(host *kolide.Host) error { sqlStatement := ` UPDATE hosts SET @@ -263,13 +266,8 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) { return nil, errors.DatabaseError(err) } - sqlStatement = ` - SELECT * from network_interfaces - WHERE host_id = ? - ` - err = d.db.Select(&host.NetworkInterfaces, sqlStatement, id) - if err != nil { - return nil, errors.DatabaseError(err) + if err := d.getNetInterfacesForHost(host); err != nil { + return nil, err } return host, nil @@ -287,17 +285,65 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return nil, errors.DatabaseError(err) } - // TODO: This should be changed so that hosts and network interfaces - // are grabbed in a single sql request, - for _, host := range hosts { - if err := d.getNetInterfacesForHost(host); err != nil { - return nil, errors.DatabaseError(err) - } + if err := d.getNetInterfacesForHosts(hosts); err != nil { + return nil, err } return hosts, nil } +// Optimized network interface fetch for sets of hosts. Instead of looping +// through hosts and doing a select for each host to get nics, we get all +// nics at once, so 2 db calls, and then assign nics to hosts here. +func (d *Datastore) getNetInterfacesForHosts(hosts []*kolide.Host) error { + if len(hosts) == 0 { + return nil + } + + sqlStatement := ` + SELECT * + FROM network_interfaces + WHERE host_id IN (:hosts) + ORDER BY host_id ASC + ` + hostIDs := make([]interface{}, len(hosts)) + + for _, host := range hosts { + hostIDs = append(hostIDs, host.ID) + } + + arg := map[string]interface{}{ + "hosts": hostIDs, + } + query, args, err := sqlx.Named(sqlStatement, arg) + if err != nil { + return err + } + + query, args, err = sqlx.In(query, args...) + if err != nil { + return err + } + + query = d.db.Rebind(query) + nics := []*kolide.NetworkInterface{} + err = d.db.Select(&nics, query, args...) + if err != nil { + return err + } + + sortutil.AscByField(hosts, "ID") + + i := 0 + for _, host := range hosts { + for ; i < len(nics) && host.ID == nics[i].HostID; i++ { + host.NetworkInterfaces = append(host.NetworkInterfaces, nics[i]) + } + } + + return nil +} + func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error { sqlStatement := ` SELECT * FROM network_interfaces @@ -311,40 +357,31 @@ func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error { } // EnrollHost enrolls a host -func (d *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*kolide.Host, error) { - if uuid == "" { - return nil, errors.New("missing uuid for host enrollment", "programmer error") +func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.Host, error) { + if osqueryHostID == "" { + return nil, errors.InternalServerError(fmt.Errorf("missing osquery host identifier")) } - // REVIEW If a deleted host is enrolled, it is undeleted + + detailUpdateTime := time.Unix(0, 0).Add(24 * time.Hour) + nodeKey, err := kolide.RandomText(nodeKeySize) + if err != nil { + return nil, errors.InternalServerError(err) + } + sqlInsert := ` INSERT INTO hosts ( detail_update_time, - node_key, - host_name, - uuid, - platform - ) VALUES (?, ?, ?, ?, ? ) + osquery_host_id, + node_key + ) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE - updated_at = VALUES(updated_at), - detail_update_time = VALUES(detail_update_time), node_key = VALUES(node_key), - host_name = VALUES(host_name), - platform = VALUES(platform), deleted = FALSE ` - args := []interface{}{} - args = append(args, time.Unix(0, 0).Add(24*time.Hour)) - - nodeKey, err := kolide.RandomText(nodeKeySize) - - args = append(args, nodeKey) - args = append(args, hostname) - args = append(args, uuid) - args = append(args, platform) var result sql.Result - result, err = d.db.Exec(sqlInsert, args...) + result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, nodeKey) if err != nil { return nil, errors.DatabaseError(err) @@ -366,9 +403,12 @@ func (d *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int) func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { sqlStatement := ` - SELECT * FROM hosts - WHERE node_key = ? AND NOT DELETED LIMIT 1 + SELECT * + FROM hosts + WHERE node_key = ? AND NOT deleted + LIMIT 1 ` + host := &kolide.Host{} if err := d.db.Get(host, sqlStatement, nodeKey); err != nil { switch err { @@ -381,8 +421,11 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { } } - return host, nil + if err := d.getNetInterfacesForHost(host); err != nil { + return nil, errors.DatabaseError(err) + } + return host, nil } func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { @@ -401,7 +444,7 @@ func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { return nil } -func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.Host, error) { +func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.Host, error) { hostnameQuery := query if len(hostnameQuery) > 0 { hostnameQuery += "*" @@ -440,16 +483,15 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.H } sql = d.db.Rebind(sql) - hosts := []kolide.Host{} + hosts := []*kolide.Host{} + err = d.db.Select(&hosts, sql, args...) if err != nil { return nil, errors.DatabaseError(err) } - for i := 0; i < len(hosts); i++ { - if err := d.getNetInterfacesForHost(&hosts[i]); err != nil { - return nil, errors.DatabaseError(err) - } + if err := d.getNetInterfacesForHosts(hosts); err != nil { + return nil, err } return hosts, nil @@ -457,7 +499,7 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.H // SearchHosts find hosts by query containing an IP address or a host name. Optionally // pass a list of IDs to omit from the search -func (d *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) { +func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) { if len(omit) > 0 { return d.searchHostsWithOmits(query, omit...) } @@ -493,17 +535,14 @@ func (d *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, erro AND NOT deleted LIMIT 10 ` - hosts := []kolide.Host{} + hosts := []*kolide.Host{} if err := d.db.Select(&hosts, sqlStatement, hostnameQuery, ipQuery); err != nil { return nil, errors.DatabaseError(err) } - // We're not using range to avoid having to copy the hosts slice - // when we populate the host NetworkInterfaces - for i := 0; i < len(hosts); i++ { - if err := d.getNetInterfacesForHost(&hosts[i]); err != nil { - return nil, errors.DatabaseError(err) - } + + if err := d.getNetInterfacesForHosts(hosts); err != nil { + return nil, err } return hosts, nil diff --git a/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go b/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go index 49d6d4b524..20d0b28061 100644 --- a/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go +++ b/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go @@ -14,15 +14,16 @@ func Up_20161118212528(tx *sql.Tx) error { _, err := tx.Exec( "CREATE TABLE `hosts` (" + "`id` int(10) unsigned NOT NULL AUTO_INCREMENT," + + "`osquery_host_id` varchar(255) NOT NULL," + "`created_at` timestamp DEFAULT CURRENT_TIMESTAMP," + "`updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," + "`deleted_at` timestamp NULL DEFAULT NULL," + "`deleted` tinyint(1) NOT NULL DEFAULT FALSE," + "`detail_update_time` timestamp NULL DEFAULT NULL," + "`node_key` varchar(255) DEFAULT NULL," + - "`host_name` varchar(255) DEFAULT NULL," + - "`uuid` varchar(255) DEFAULT NULL," + - "`platform` varchar(255) DEFAULT NULL," + + "`host_name` varchar(255) NOT NULL DEFAULT ''," + + "`uuid` varchar(255) NOT NULL DEFAULT ''," + + "`platform` varchar(255) NOT NULL DEFAULT ''," + "`osquery_version` varchar(255) NOT NULL DEFAULT ''," + "`os_version` varchar(255) NOT NULL DEFAULT ''," + "`build` varchar(255) NOT NULL DEFAULT ''," + @@ -43,7 +44,7 @@ func Up_20161118212528(tx *sql.Tx) error { "`primary_ip_id` INT(10) UNSIGNED DEFAULT NULL, " + "PRIMARY KEY (`id`)," + "UNIQUE KEY `idx_host_unique_nodekey` (`node_key`)," + - "UNIQUE KEY `idx_host_unique_uuid` (`uuid`)," + + "UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`)," + "FULLTEXT KEY `hosts_search` (`host_name`)" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", ) diff --git a/server/datastore/test_util.go b/server/datastore/test_util.go index 7040d6aa40..3b044e76a7 100644 --- a/server/datastore/test_util.go +++ b/server/datastore/test_util.go @@ -90,11 +90,13 @@ func newExecution(t *testing.T, ds kolide.Datastore, campaignID uint, hostID uin } func newHost(t *testing.T, ds kolide.Datastore, name, ip, key, uuid string, now time.Time) *kolide.Host { + osqueryHostID, _ := kolide.RandomText(10) h, err := ds.NewHost(&kolide.Host{ HostName: name, NodeKey: key, UUID: uuid, DetailUpdateTime: now, + OsqueryHostID: osqueryHostID, }) require.Nil(t, err) diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index cbcfcf2192..2c9808bb37 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -14,10 +14,10 @@ type HostStore interface { DeleteHost(host *Host) error Host(id uint) (*Host, error) ListHosts(opt ListOptions) ([]*Host, error) - EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*Host, error) + EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error) AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error - SearchHosts(query string, omit ...uint) ([]Host, error) + SearchHosts(query string, omit ...uint) ([]*Host, error) // DistributedQueriesForHost retrieves the distributed queries that the // given host should run. The result map is a mapping from campaign ID // to query text. @@ -34,7 +34,11 @@ type HostService interface { type Host struct { UpdateCreateTimestamps DeleteFields - ID uint `json:"id"` + ID uint `json:"id"` + // OsqueryHostID is the key used in the request context that is + // used to retrieve host information. It is sent from osquery and may currently be + // a GUID or a Host Name, but in either case, it MUST be unique + OsqueryHostID string `json:"-" db:"osquery_host_id"` DetailUpdateTime time.Time `json:"detail_updated_at" db:"detail_update_time"` // Time that the host details were last updated NodeKey string `json:"-" db:"node_key"` HostName string `json:"hostname" db:"host_name"` // there is a fulltext index on this field @@ -72,7 +76,7 @@ type Host struct { // change should be written back to the database func (h *Host) ResetPrimaryNetwork() bool { if h.PrimaryNetworkInterfaceID != nil { - // No nic (should never happen) + // This *may* happen if other details of the host are fetched before network_interfaces if len(h.NetworkInterfaces) == 0 { h.PrimaryNetworkInterfaceID = nil return true diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index e62faa8bcd..046e0b75a7 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -49,7 +49,7 @@ func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier return "", osqueryError{message: "invalid enroll secret", nodeInvalid: true} } - host, err := svc.ds.EnrollHost(hostIdentifier, "", "", svc.config.Osquery.NodeKeySize) + host, err := svc.ds.EnrollHost(hostIdentifier, svc.config.Osquery.NodeKeySize) if err != nil { return "", osqueryError{message: "enrollment failed: " + err.Error(), nodeInvalid: true} } @@ -313,7 +313,6 @@ var detailQueries = map[string]struct { if nic.Type, err = strconv.Atoi(row["type"]); err != nil { return err } - networkInterfaces = append(networkInterfaces, &nic) } @@ -441,6 +440,7 @@ func (svc service) ingestDistributedQuery(host kolide.Host, name string, rows [] func (svc service) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults) error { host, ok := hostctx.FromContext(ctx) + if !ok { return osqueryError{message: "internal error: missing host from request context"} } @@ -455,13 +455,10 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko switch { case strings.HasPrefix(query, hostDetailQueryPrefix): err = svc.ingestDetailQuery(&host, query, rows) - case strings.HasPrefix(query, hostLabelQueryPrefix): err = svc.ingestLabelQuery(host, query, rows, labelResults) - case strings.HasPrefix(query, hostDistributedQueryPrefix): err = svc.ingestDistributedQuery(host, query, rows) - default: err = osqueryError{message: "unknown query prefix: " + query} } diff --git a/server/service/service_targets.go b/server/service/service_targets.go index 7cb5acd6f2..c1a7ae40c2 100644 --- a/server/service/service_targets.go +++ b/server/service/service_targets.go @@ -12,7 +12,10 @@ func (svc service) SearchTargets(ctx context.Context, query string, selectedHost if err != nil { return nil, err } - results.Hosts = hosts + + for _, h := range hosts { + results.Hosts = append(results.Hosts, *h) + } labels, err := svc.ds.SearchLabels(query, selectedLabelIDs...) if err != nil {