Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-24 09:28:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 52

Fixes host detail updates with MySQL backend (#568)

Browse source
This commit is contained in:
John Murphy 2016-12-07 03:51:11 +08:00 committed by GitHub
parent 691eb55cf2
commit 7812b2f3bd
10 changed files with 223 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
server/datastore/datastore_hosts_test.go
View file

@ -2,6 +2,7 @@ package datastore
import (
"fmt"
"strconv"
"testing"
"time"
@ -96,6 +97,16 @@ func testSaveHosts(t *testing.T, db kolide.Datastore) {
assert.Equal(t, host.NetworkInterfaces[0].ID, *host.PrimaryNetworkInterfaceID)
assert.Equal(t, 1, len(host.NetworkInterfaces))
// remove all nics primary nic should be nil
host.NetworkInterfaces = []*kolide.NetworkInterface{}
err = db.SaveHost(host)
require.Nil(t, err)
assert.Nil(t, host.PrimaryNetworkInterfaceID)
host, err = db.Host(host.ID)
require.Nil(t, err)
require.NotNil(t, host)
assert.Nil(t, host.PrimaryNetworkInterfaceID)
err = db.DeleteHost(host)
assert.Nil(t, err)
@ -111,8 +122,8 @@ func testDeleteHost(t *testing.T, db kolide.Datastore) {
UUID: "1",
HostName: "foo.local",
})
assert.Nil(t, err)
assert.NotNil(t, host)
require.Nil(t, err)
require.NotNil(t, host)
err = db.DeleteHost(host)
assert.Nil(t, err)
@ -126,6 +137,7 @@ func testListHost(t *testing.T, db kolide.Datastore) {
for i := 0; i < 10; i++ {
host, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: strconv.Itoa(i),
NodeKey: fmt.Sprintf("%d", i),
UUID: fmt.Sprintf("%d", i),
HostName: fmt.Sprintf("foo.local%d", i),
@ -137,9 +149,39 @@ func testListHost(t *testing.T, db kolide.Datastore) {
hosts = append(hosts, host)
}
hosts[1].NetworkInterfaces = []*kolide.NetworkInterface{
&kolide.NetworkInterface{
Interface: "en0",
IPAddress: "99.100.101.102",
},
&kolide.NetworkInterface{
Interface: "en1",
IPAddress: "99.100.101.103",
},
}
err := db.SaveHost(hosts[1])
require.Nil(t, err)
hosts[3].NetworkInterfaces = []*kolide.NetworkInterface{
&kolide.NetworkInterface{
Interface: "en2",
IPAddress: "99.100.101.104",
},
}
err = db.SaveHost(hosts[3])
require.Nil(t, err)
hosts2, err := db.ListHosts(kolide.ListOptions{})
require.Nil(t, err)
assert.Equal(t, len(hosts), len(hosts2))
require.Equal(t, 2, len(hosts2[1].NetworkInterfaces))
require.Equal(t, 0, len(hosts2[2].NetworkInterfaces))
require.Equal(t, 1, len(hosts2[3].NetworkInterfaces))
assert.Equal(t, "en1", hosts2[1].NetworkInterfaces[1].Interface)
assert.Equal(t, "en2", hosts2[3].NetworkInterfaces[0].Interface)
err = db.DeleteHost(hosts[0])
require.Nil(t, err)
hosts2, err = db.ListHosts(kolide.ListOptions{})
@ -173,32 +215,19 @@ func testListHost(t *testing.T, db kolide.Datastore) {
func testEnrollHost(t *testing.T, db kolide.Datastore) {
var hosts []*kolide.Host
for _, tt := range enrollTests {
h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.platform, tt.nodeKeySize)
h, err := db.EnrollHost(tt.uuid, tt.nodeKeySize)
require.Nil(t, err)
hosts = append(hosts, h)
assert.Equal(t, tt.uuid, h.UUID)
assert.Equal(t, tt.hostname, h.HostName)
assert.Equal(t, tt.platform, h.Platform)
assert.Equal(t, tt.uuid, h.OsqueryHostID)
assert.NotEmpty(t, h.NodeKey)
}
for _, enrolled := range hosts {
oldNodeKey := enrolled.NodeKey
newhostname := fmt.Sprintf("changed.%s", enrolled.HostName)
h, err := db.EnrollHost(enrolled.UUID, newhostname, enrolled.Platform, 15)
assert.Nil(t, err)
assert.Equal(t, enrolled.UUID, h.UUID)
assert.NotEmpty(t, h.NodeKey)
assert.NotEqual(t, oldNodeKey, h.NodeKey)
}
}
func testAuthenticateHost(t *testing.T, db kolide.Datastore) {
for _, tt := range enrollTests {
h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.platform, tt.nodeKeySize)
h, err := db.EnrollHost(tt.uuid, tt.nodeKeySize)
require.Nil(t, err)
returned, err := db.AuthenticateHost(h.NodeKey)
@ -215,6 +244,7 @@ func testAuthenticateHost(t *testing.T, db kolide.Datastore) {
func testSearchHosts(t *testing.T, db kolide.Datastore) {
_, err := db.NewHost(&kolide.Host{
OsqueryHostID: "1234",
DetailUpdateTime: time.Now(),
NodeKey: "1",
UUID: "1",
@ -223,6 +253,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) {
require.Nil(t, err)
h2, err := db.NewHost(&kolide.Host{
OsqueryHostID: "5679",
DetailUpdateTime: time.Now(),
NodeKey: "2",
UUID: "2",
@ -231,6 +262,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) {
require.Nil(t, err)
h3, err := db.NewHost(&kolide.Host{
OsqueryHostID: "99999",
DetailUpdateTime: time.Now(),
NodeKey: "3",
UUID: "3",
@ -286,7 +318,7 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) {
h3.NetworkInterfaces = []*kolide.NetworkInterface{
&kolide.NetworkInterface{
Interface: "en0",
Interface: "en3",
IPAddress: "99.100.101.104",
},
}
@ -295,6 +327,8 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) {
hits, err = db.SearchHosts("99.100.101")
require.Nil(t, err)
assert.Equal(t, 2, len(hits))
assert.Equal(t, 2, len(hits[0].NetworkInterfaces))
assert.Equal(t, "en0", hits[0].NetworkInterfaces[0].Interface)
hits, err = db.SearchHosts("99.100.101", h3.ID)
require.Nil(t, err)
@ -306,6 +340,7 @@ func testSearchHostsLimit(t *testing.T, db kolide.Datastore) {
for i := 0; i < 15; i++ {
_, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: fmt.Sprintf("host%d", i),
NodeKey: fmt.Sprintf("%d", i),
UUID: fmt.Sprintf("%d", i),
HostName: fmt.Sprintf("foo.%d.local", i),
@ -320,6 +355,7 @@ func testSearchHostsLimit(t *testing.T, db kolide.Datastore) {
func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) {
h1, err := db.NewHost(&kolide.Host{
OsqueryHostID: "1",
DetailUpdateTime: time.Now(),
NodeKey: "1",
UUID: "1",
@ -328,6 +364,7 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) {
require.Nil(t, err)
h2, err := db.NewHost(&kolide.Host{
OsqueryHostID: "2",
DetailUpdateTime: time.Now(),
NodeKey: "2",
UUID: "2",

78
server/datastore/datastore_labels_test.go
View file

@ -3,6 +3,7 @@ package datastore
import (
"fmt"
"sort"
"strconv"
"testing"
"time"
@ -18,8 +19,8 @@ func testLabels(t *testing.T, db kolide.Datastore) {
var host *kolide.Host
var err error
for i := 0; i < 10; i++ {
host, err = db.EnrollHost(string(i), "foo", "", 10)
assert.Nil(t, err, "enrollment should succeed")
host, err = db.EnrollHost(string(i), 10)
require.Nil(t, err, "enrollment should succeed")
hosts = append(hosts, *host)
}
@ -231,6 +232,7 @@ func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) {
func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h1, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: "1",
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
@ -239,6 +241,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h2, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: "2",
NodeKey: "2",
UUID: "2",
HostName: "bar.local",
@ -247,6 +250,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h3, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: "3",
NodeKey: "3",
UUID: "3",
HostName: "baz.local",
@ -297,45 +301,19 @@ func testBuiltInLabels(t *testing.T, db kolide.Datastore) {
}
func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) {
h1, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
})
require.Nil(t, err)
h2, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
NodeKey: "2",
UUID: "2",
HostName: "bar.local",
})
require.Nil(t, err)
h3, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
NodeKey: "3",
UUID: "3",
HostName: "baz.local",
})
require.Nil(t, err)
h4, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
NodeKey: "4",
UUID: "4",
HostName: "xxx.local",
})
require.Nil(t, err)
h5, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
NodeKey: "5",
UUID: "5",
HostName: "yyy.local",
})
require.Nil(t, err)
hosts := []*kolide.Host{}
for i := 0; i < 4; i++ {
h, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
OsqueryHostID: strconv.Itoa(i),
NodeKey: strconv.Itoa(i),
UUID: strconv.Itoa(i),
HostName: fmt.Sprintf("host_%d", i),
})
require.Nil(t, err)
require.NotNil(t, h)
hosts = append(hosts, h)
}
l1, err := db.NewLabel(&kolide.Label{
Name: "label foo",
@ -353,17 +331,17 @@ func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) {
require.NotZero(t, l2.ID)
l2ID := fmt.Sprintf("%d", l2.ID)
for _, h := range []*kolide.Host{h1, h2, h3} {
err = db.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now())
for i := 0; i < 3; i++ {
err = db.RecordLabelQueryExecutions(hosts[i], map[string]bool{l1ID: true}, time.Now())
assert.Nil(t, err)
}
// host 2 executes twice
for i := 2; i < len(hosts); i++ {
err = db.RecordLabelQueryExecutions(hosts[i], map[string]bool{l2ID: true}, time.Now())
assert.Nil(t, err)
}
for _, h := range []*kolide.Host{h3, h4, h5} {
err = db.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now())
assert.Nil(t, err)
}
hosts, err := db.ListUniqueHostsInLabels([]uint{l1.ID, l2.ID})
uniqueHosts, err := db.ListUniqueHostsInLabels([]uint{l1.ID, l2.ID})
assert.Nil(t, err)
assert.Len(t, hosts, 5)
assert.Equal(t, len(hosts), len(uniqueHosts))
}

2
server/datastore/datastore_packs_test.go
View file

@ -110,6 +110,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
h1, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
HostName: "foobar.local",
OsqueryHostID: "1",
NodeKey: "1",
UUID: "1",
})
@ -129,6 +130,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
h2, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
HostName: "foobaz.local",
OsqueryHostID: "2",
NodeKey: "2",
UUID: "2",
})

46
server/datastore/inmem/hosts.go
View file

@ -114,39 +114,33 @@ func (orm *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error)
return hosts, nil
}
func (orm *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*kolide.Host, error) {
func (orm *Datastore) EnrollHost(osQueryHostID string, nodeKeySize int) (*kolide.Host, error) {
orm.mtx.Lock()
defer orm.mtx.Unlock()
if uuid == "" {
return nil, errors.New("missing uuid for host enrollment")
if osQueryHostID == "" {
return nil, errors.New("missing host identifier from osquery for host enrollment")
}
host := kolide.Host{
UUID: uuid,
HostName: hostname,
Platform: platform,
DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour),
}
for _, h := range orm.hosts {
if h.UUID == uuid {
host = *h
break
}
}
var err error
host.NodeKey, err = kolide.RandomText(nodeKeySize)
nodeKey, err := kolide.RandomText(nodeKeySize)
if err != nil {
return nil, err
}
if hostname != "" {
host.HostName = hostname
host := kolide.Host{
OsqueryHostID: osQueryHostID,
NodeKey: nodeKey,
DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour),
}
if platform != "" {
host.Platform = platform
host.CreatedAt = time.Now().UTC()
host.UpdatedAt = host.CreatedAt
for _, h := range orm.hosts {
if h.OsqueryHostID == osQueryHostID {
host = *h
break
}
}
if host.ID == 0 {
@ -185,13 +179,13 @@ func (orm *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
return nil
}
func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) {
func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) {
omitLookup := map[uint]bool{}
for _, o := range omit {
omitLookup[o] = true
}
var results []kolide.Host
var results []*kolide.Host
orm.mtx.Lock()
defer orm.mtx.Unlock()
@ -202,13 +196,13 @@ func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, er
}
if strings.Contains(h.HostName, query) && !omitLookup[h.ID] {
results = append(results, *h)
results = append(results, h)
continue
}
for _, nic := range h.NetworkInterfaces {
if strings.Contains(nic.IPAddress, query) && !omitLookup[nic.HostID] {
results = append(results, *h)
results = append(results, h)
break
}
}

153
server/datastore/mysql/hosts.go
View file

@ -9,11 +9,13 @@ import (
"github.com/jmoiron/sqlx"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/patrickmn/sortutil"
)
func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) {
sqlStatement := `
INSERT INTO hosts (
osquery_host_id,
detail_update_time,
node_key,
host_name,
@ -24,9 +26,9 @@ func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) {
uptime,
physical_memory
)
VALUES( ?,?,?,?,?,?,?,?,? )
VALUES( ?,?,?,?,?,?,?,?,?,? )
`
result, err := d.db.Exec(sqlStatement, host.DetailUpdateTime,
result, err := d.db.Exec(sqlStatement, host.OsqueryHostID, host.DetailUpdateTime,
host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion,
host.OSVersion, host.Uptime, host.PhysicalMemory)
if err != nil {
@ -211,6 +213,7 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
tx.Rollback()
return errors.DatabaseError(err)
}
if err = removedUnusedNics(tx, host); err != nil {
tx.Rollback()
return errors.DatabaseError(err)
@ -219,9 +222,10 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
if needsUpdate := host.ResetPrimaryNetwork(); needsUpdate {
_, err = tx.Exec(
"UPDATE hosts SET primary_ip_id = ? WHERE id = ?",
*host.PrimaryNetworkInterfaceID,
host.PrimaryNetworkInterfaceID,
host.ID,
)
if err != nil {
tx.Rollback()
return errors.DatabaseError(err)
@ -235,7 +239,6 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
return nil
}
// TODO needs test
func (d *Datastore) DeleteHost(host *kolide.Host) error {
sqlStatement := `
UPDATE hosts SET
@ -263,13 +266,8 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) {
return nil, errors.DatabaseError(err)
}
sqlStatement = `
SELECT * from network_interfaces
WHERE host_id = ?
`
err = d.db.Select(&host.NetworkInterfaces, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
if err := d.getNetInterfacesForHost(host); err != nil {
return nil, err
}
return host, nil
@ -287,17 +285,65 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
return nil, errors.DatabaseError(err)
}
// TODO: This should be changed so that hosts and network interfaces
// are grabbed in a single sql request,
for _, host := range hosts {
if err := d.getNetInterfacesForHost(host); err != nil {
return nil, errors.DatabaseError(err)
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
return nil, err
}
return hosts, nil
}
// Optimized network interface fetch for sets of hosts. Instead of looping
// through hosts and doing a select for each host to get nics, we get all
// nics at once, so 2 db calls, and then assign nics to hosts here.
func (d *Datastore) getNetInterfacesForHosts(hosts []*kolide.Host) error {
if len(hosts) == 0 {
return nil
}
sqlStatement := `
SELECT *
FROM network_interfaces
WHERE host_id IN (:hosts)
ORDER BY host_id ASC
`
hostIDs := make([]interface{}, len(hosts))
for _, host := range hosts {
hostIDs = append(hostIDs, host.ID)
}
arg := map[string]interface{}{
"hosts": hostIDs,
}
query, args, err := sqlx.Named(sqlStatement, arg)
if err != nil {
return err
}
query, args, err = sqlx.In(query, args...)
if err != nil {
return err
}
query = d.db.Rebind(query)
nics := []*kolide.NetworkInterface{}
err = d.db.Select(&nics, query, args...)
if err != nil {
return err
}
sortutil.AscByField(hosts, "ID")
i := 0
for _, host := range hosts {
for ; i < len(nics) && host.ID == nics[i].HostID; i++ {
host.NetworkInterfaces = append(host.NetworkInterfaces, nics[i])
}
}
return nil
}
func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error {
sqlStatement := `
SELECT * FROM network_interfaces
@ -311,40 +357,31 @@ func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error {
}
// EnrollHost enrolls a host
func (d *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*kolide.Host, error) {
if uuid == "" {
return nil, errors.New("missing uuid for host enrollment", "programmer error")
func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.Host, error) {
if osqueryHostID == "" {
return nil, errors.InternalServerError(fmt.Errorf("missing osquery host identifier"))
}
// REVIEW If a deleted host is enrolled, it is undeleted
detailUpdateTime := time.Unix(0, 0).Add(24 * time.Hour)
nodeKey, err := kolide.RandomText(nodeKeySize)
if err != nil {
return nil, errors.InternalServerError(err)
}
sqlInsert := `
INSERT INTO hosts (
detail_update_time,
node_key,
host_name,
uuid,
platform
) VALUES (?, ?, ?, ?, ? )
osquery_host_id,
node_key
) VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE
updated_at = VALUES(updated_at),
detail_update_time = VALUES(detail_update_time),
node_key = VALUES(node_key),
host_name = VALUES(host_name),
platform = VALUES(platform),
deleted = FALSE
`
args := []interface{}{}
args = append(args, time.Unix(0, 0).Add(24*time.Hour))
nodeKey, err := kolide.RandomText(nodeKeySize)
args = append(args, nodeKey)
args = append(args, hostname)
args = append(args, uuid)
args = append(args, platform)
var result sql.Result
result, err = d.db.Exec(sqlInsert, args...)
result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, nodeKey)
if err != nil {
return nil, errors.DatabaseError(err)
@ -366,9 +403,12 @@ func (d *Datastore) EnrollHost(uuid, hostname, platform string, nodeKeySize int)
func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
sqlStatement := `
SELECT * FROM hosts
WHERE node_key = ? AND NOT DELETED LIMIT 1
SELECT *
FROM hosts
WHERE node_key = ? AND NOT deleted
LIMIT 1
`
host := &kolide.Host{}
if err := d.db.Get(host, sqlStatement, nodeKey); err != nil {
switch err {
@ -381,8 +421,11 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
}
}
return host, nil
if err := d.getNetInterfacesForHost(host); err != nil {
return nil, errors.DatabaseError(err)
}
return host, nil
}
func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
@ -401,7 +444,7 @@ func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
return nil
}
func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.Host, error) {
func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.Host, error) {
hostnameQuery := query
if len(hostnameQuery) > 0 {
hostnameQuery += "*"
@ -440,16 +483,15 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.H
}
sql = d.db.Rebind(sql)
hosts := []kolide.Host{}
hosts := []*kolide.Host{}
err = d.db.Select(&hosts, sql, args...)
if err != nil {
return nil, errors.DatabaseError(err)
}
for i := 0; i < len(hosts); i++ {
if err := d.getNetInterfacesForHost(&hosts[i]); err != nil {
return nil, errors.DatabaseError(err)
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
return nil, err
}
return hosts, nil
@ -457,7 +499,7 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]kolide.H
// SearchHosts find hosts by query containing an IP address or a host name. Optionally
// pass a list of IDs to omit from the search
func (d *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) {
func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) {
if len(omit) > 0 {
return d.searchHostsWithOmits(query, omit...)
}
@ -493,17 +535,14 @@ func (d *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, erro
AND NOT deleted
LIMIT 10
`
hosts := []kolide.Host{}
hosts := []*kolide.Host{}
if err := d.db.Select(&hosts, sqlStatement, hostnameQuery, ipQuery); err != nil {
return nil, errors.DatabaseError(err)
}
// We're not using range to avoid having to copy the hosts slice
// when we populate the host NetworkInterfaces
for i := 0; i < len(hosts); i++ {
if err := d.getNetInterfacesForHost(&hosts[i]); err != nil {
return nil, errors.DatabaseError(err)
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
return nil, err
}
return hosts, nil

9
server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go
View file

@ -14,15 +14,16 @@ func Up_20161118212528(tx *sql.Tx) error {
_, err := tx.Exec(
"CREATE TABLE `hosts` (" +
"`id` int(10) unsigned NOT NULL AUTO_INCREMENT," +
"`osquery_host_id` varchar(255) NOT NULL," +
"`created_at` timestamp DEFAULT CURRENT_TIMESTAMP," +
"`updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," +
"`deleted_at` timestamp NULL DEFAULT NULL," +
"`deleted` tinyint(1) NOT NULL DEFAULT FALSE," +
"`detail_update_time` timestamp NULL DEFAULT NULL," +
"`node_key` varchar(255) DEFAULT NULL," +
"`host_name` varchar(255) DEFAULT NULL," +
"`uuid` varchar(255) DEFAULT NULL," +
"`platform` varchar(255) DEFAULT NULL," +
"`host_name` varchar(255) NOT NULL DEFAULT ''," +
"`uuid` varchar(255) NOT NULL DEFAULT ''," +
"`platform` varchar(255) NOT NULL DEFAULT ''," +
"`osquery_version` varchar(255) NOT NULL DEFAULT ''," +
"`os_version` varchar(255) NOT NULL DEFAULT ''," +
"`build` varchar(255) NOT NULL DEFAULT ''," +
@ -43,7 +44,7 @@ func Up_20161118212528(tx *sql.Tx) error {
"`primary_ip_id` INT(10) UNSIGNED DEFAULT NULL, " +
"PRIMARY KEY (`id`)," +
"UNIQUE KEY `idx_host_unique_nodekey` (`node_key`)," +
"UNIQUE KEY `idx_host_unique_uuid` (`uuid`)," +
"UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`)," +
"FULLTEXT KEY `hosts_search` (`host_name`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
)

2
server/datastore/test_util.go
View file

@ -90,11 +90,13 @@ func newExecution(t *testing.T, ds kolide.Datastore, campaignID uint, hostID uin
}
func newHost(t *testing.T, ds kolide.Datastore, name, ip, key, uuid string, now time.Time) *kolide.Host {
osqueryHostID, _ := kolide.RandomText(10)
h, err := ds.NewHost(&kolide.Host{
HostName: name,
NodeKey: key,
UUID: uuid,
DetailUpdateTime: now,
OsqueryHostID: osqueryHostID,
})
require.Nil(t, err)

12
server/kolide/hosts.go
View file

@ -14,10 +14,10 @@ type HostStore interface {
DeleteHost(host *Host) error
Host(id uint) (*Host, error)
ListHosts(opt ListOptions) ([]*Host, error)
EnrollHost(uuid, hostname, platform string, nodeKeySize int) (*Host, error)
EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error)
AuthenticateHost(nodeKey string) (*Host, error)
MarkHostSeen(host *Host, t time.Time) error
SearchHosts(query string, omit ...uint) ([]Host, error)
SearchHosts(query string, omit ...uint) ([]*Host, error)
// DistributedQueriesForHost retrieves the distributed queries that the
// given host should run. The result map is a mapping from campaign ID
// to query text.
@ -34,7 +34,11 @@ type HostService interface {
type Host struct {
UpdateCreateTimestamps
DeleteFields
ID uint `json:"id"`
ID uint `json:"id"`
// OsqueryHostID is the key used in the request context that is
// used to retrieve host information. It is sent from osquery and may currently be
// a GUID or a Host Name, but in either case, it MUST be unique
OsqueryHostID string `json:"-" db:"osquery_host_id"`
DetailUpdateTime time.Time `json:"detail_updated_at" db:"detail_update_time"` // Time that the host details were last updated
NodeKey string `json:"-" db:"node_key"`
HostName string `json:"hostname" db:"host_name"` // there is a fulltext index on this field
@ -72,7 +76,7 @@ type Host struct {
// change should be written back to the database
func (h *Host) ResetPrimaryNetwork() bool {
if h.PrimaryNetworkInterfaceID != nil {
// No nic (should never happen)
// This *may* happen if other details of the host are fetched before network_interfaces
if len(h.NetworkInterfaces) == 0 {
h.PrimaryNetworkInterfaceID = nil
return true

7
server/service/service_osquery.go
View file

@ -49,7 +49,7 @@ func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier
return "", osqueryError{message: "invalid enroll secret", nodeInvalid: true}
}
host, err := svc.ds.EnrollHost(hostIdentifier, "", "", svc.config.Osquery.NodeKeySize)
host, err := svc.ds.EnrollHost(hostIdentifier, svc.config.Osquery.NodeKeySize)
if err != nil {
return "", osqueryError{message: "enrollment failed: " + err.Error(), nodeInvalid: true}
}
@ -313,7 +313,6 @@ var detailQueries = map[string]struct {
if nic.Type, err = strconv.Atoi(row["type"]); err != nil {
return err
}
networkInterfaces = append(networkInterfaces, &nic)
}
@ -441,6 +440,7 @@ func (svc service) ingestDistributedQuery(host kolide.Host, name string, rows []
func (svc service) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults) error {
host, ok := hostctx.FromContext(ctx)
if !ok {
return osqueryError{message: "internal error: missing host from request context"}
}
@ -455,13 +455,10 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko
switch {
case strings.HasPrefix(query, hostDetailQueryPrefix):
err = svc.ingestDetailQuery(&host, query, rows)
case strings.HasPrefix(query, hostLabelQueryPrefix):
err = svc.ingestLabelQuery(host, query, rows, labelResults)
case strings.HasPrefix(query, hostDistributedQueryPrefix):
err = svc.ingestDistributedQuery(host, query, rows)
default:
err = osqueryError{message: "unknown query prefix: " + query}
}

5
server/service/service_targets.go
View file

@ -12,7 +12,10 @@ func (svc service) SearchTargets(ctx context.Context, query string, selectedHost
if err != nil {
return nil, err
}
results.Hosts = hosts
for _, h := range hosts {
results.Hosts = append(results.Hosts, *h)
}
labels, err := svc.ds.SearchLabels(query, selectedLabelIDs...)
if err != nil {