Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-23 00:49:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 143

Host summary endpoint (#742)

Browse source 
* Initial scaffolding of the host summary endpoint

* inmem datastore implementation of GenerateHostStatusStatistics

* HostSummary docstring

* changing the url of the host summary endpoint

* datastore tests for GenerateHostStatusStatistics

* MySQL datastore implementation of GenerateHostStatusStatistics

* <= and >= to catch exact time edge case

* removing clock interface method

* lowercase error wraps

* removin superfluous whitespace

* use updated_at

* adding a seen_at column to the hosts table

* moving the update of seen_time to the caller

* using db.Get instead of db.Select
This commit is contained in:
Mike Arpaia 2017-01-04 14:16:17 -07:00 committed by GitHub
parent 046f75295e
commit 704ddd424b
23 changed files with 364 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
server/datastore/datastore_hosts_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/WatchBeam/clock"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/kolide/kolide-ose/server/test"
"github.com/stretchr/testify/assert"
@ -41,6 +42,7 @@ var enrollTests = []struct {
func testSaveHosts(t *testing.T, ds kolide.Datastore) {
host, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
@ -119,6 +121,7 @@ func testSaveHosts(t *testing.T, ds kolide.Datastore) {
func testDeleteHost(t *testing.T, ds kolide.Datastore) {
host, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
@ -138,6 +141,7 @@ func testListHost(t *testing.T, ds kolide.Datastore) {
for i := 0; i < 10; i++ {
host, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: strconv.Itoa(i),
NodeKey: fmt.Sprintf("%d", i),
UUID: fmt.Sprintf("%d", i),
@ -247,6 +251,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) {
_, err := ds.NewHost(&kolide.Host{
OsqueryHostID: "1234",
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
@ -256,6 +261,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) {
h2, err := ds.NewHost(&kolide.Host{
OsqueryHostID: "5679",
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "2",
UUID: "2",
HostName: "bar.local",
@ -265,6 +271,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) {
h3, err := ds.NewHost(&kolide.Host{
OsqueryHostID: "99999",
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "3",
UUID: "3",
HostName: "foo-bar.local",
@ -337,6 +344,7 @@ func testSearchHostsLimit(t *testing.T, ds kolide.Datastore) {
for i := 0; i < 15; i++ {
_, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: fmt.Sprintf("host%d", i),
NodeKey: fmt.Sprintf("%d", i),
UUID: fmt.Sprintf("%d", i),
@ -356,6 +364,7 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) {
h1, err := ds.NewHost(&kolide.Host{
OsqueryHostID: "1",
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "1",
UUID: "1",
HostName: "foo.local",
@ -365,6 +374,7 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) {
h2, err := ds.NewHost(&kolide.Host{
OsqueryHostID: "2",
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
NodeKey: "2",
UUID: "2",
HostName: "bar.local",
@ -487,5 +497,92 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) {
queries, err = ds.DistributedQueriesForHost(h2)
require.Nil(t, err)
assert.Empty(t, queries)
}
func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) {
mockClock := clock.NewMockClock()
// Online
_, err := ds.NewHost(&kolide.Host{
ID: 1,
OsqueryHostID: "1",
UUID: "1",
NodeKey: "1",
DetailUpdateTime: mockClock.Now(),
SeenTime: mockClock.Now(),
})
assert.Nil(t, err)
// Online
_, err = ds.NewHost(&kolide.Host{
ID: 2,
OsqueryHostID: "2",
UUID: "2",
NodeKey: "2",
DetailUpdateTime: mockClock.Now().Add(-1 * time.Minute),
SeenTime: mockClock.Now().Add(-1 * time.Minute),
})
assert.Nil(t, err)
// Offline
_, err = ds.NewHost(&kolide.Host{
ID: 3,
OsqueryHostID: "3",
UUID: "3",
NodeKey: "3",
DetailUpdateTime: mockClock.Now().Add(-1 * time.Hour),
SeenTime: mockClock.Now().Add(-1 * time.Hour),
})
assert.Nil(t, err)
// MIA
_, err = ds.NewHost(&kolide.Host{
ID: 4,
OsqueryHostID: "4",
UUID: "4",
NodeKey: "4",
DetailUpdateTime: mockClock.Now().Add(-35 * (24 * time.Hour)),
SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)),
})
assert.Nil(t, err)
online, offline, mia, err := ds.GenerateHostStatusStatistics(mockClock.Now())
assert.Nil(t, err)
assert.Equal(t, uint(2), online)
assert.Equal(t, uint(1), offline)
assert.Equal(t, uint(1), mia)
}
func testMarkHostSeen(t *testing.T, ds kolide.Datastore) {
mockClock := clock.NewMockClock()
anHourAgo := mockClock.Now().Add(-1 * time.Hour).UTC()
aDayAgo := mockClock.Now().Add(-24 * time.Hour).UTC()
h1, err := ds.NewHost(&kolide.Host{
ID: 1,
OsqueryHostID: "1",
UUID: "1",
NodeKey: "1",
DetailUpdateTime: aDayAgo,
SeenTime: aDayAgo,
})
assert.Nil(t, err)
{
h1Verify, err := ds.Host(1)
assert.Nil(t, err)
require.NotNil(t, h1Verify)
assert.WithinDuration(t, aDayAgo, h1Verify.SeenTime, time.Second)
}
err = ds.MarkHostSeen(h1, anHourAgo)
assert.Nil(t, err)
{
h1Verify, err := ds.Host(1)
assert.Nil(t, err)
require.NotNil(t, h1Verify)
assert.WithinDuration(t, anHourAgo, h1Verify.SeenTime, time.Second)
}
}

4
server/datastore/datastore_labels_test.go
View file

@ -242,6 +242,7 @@ func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) {
func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h1, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: "1",
NodeKey: "1",
UUID: "1",
@ -251,6 +252,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h2, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: "2",
NodeKey: "2",
UUID: "2",
@ -260,6 +262,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) {
h3, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: "3",
NodeKey: "3",
UUID: "3",
@ -317,6 +320,7 @@ func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) {
for i := 0; i < 4; i++ {
h, err := db.NewHost(&kolide.Host{
DetailUpdateTime: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: strconv.Itoa(i),
NodeKey: strconv.Itoa(i),
UUID: strconv.Itoa(i),

2
server/datastore/datastore_packs_test.go
View file

@ -67,6 +67,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
h1, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
SeenTime: mockClock.Now(),
HostName: "foobar.local",
OsqueryHostID: "1",
NodeKey: "1",
@ -87,6 +88,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
h2, err := ds.NewHost(&kolide.Host{
DetailUpdateTime: mockClock.Now(),
SeenTime: mockClock.Now(),
HostName: "foobaz.local",
OsqueryHostID: "2",
NodeKey: "2",

2
server/datastore/datastore_password_reset_test.go
View file

@ -10,7 +10,7 @@ import (
func testPasswordResetRequests(t *testing.T, db kolide.Datastore) {
createTestUsers(t, db)
now := time.Now()
now := time.Now().UTC()
tomorrow := now.Add(time.Hour * 24)
var passwordResetTests = []struct {
userID uint

2
server/datastore/datastore_test.go
View file

@ -60,4 +60,6 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testNewScheduledQuery,
testOptionsToConfig,
testAddLabelToPackTwice,
testGenerateHostStatusStatistics,
testMarkHostSeen,
}

22
server/datastore/inmem/hosts.go
View file

@ -114,6 +114,25 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
return hosts, nil
}
func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, host := range d.hosts {
status := host.Status(now)
switch status {
case kolide.StatusMIA:
mia++
case kolide.StatusOffline:
offline++
default:
online++
}
}
return online, offline, mia, nil
}
func (d *Datastore) EnrollHost(osQueryHostID string, nodeKeySize int) (*kolide.Host, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
@ -168,11 +187,10 @@ func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
d.mtx.Lock()
defer d.mtx.Unlock()
host.UpdatedAt = t
for _, h := range d.hosts {
if h.ID == host.ID {
h.UpdatedAt = t
h.SeenTime = t
break
}
}

2
server/datastore/mysql/datastore.go
View file

@ -6,10 +6,8 @@ import (
"github.com/WatchBeam/clock"
"github.com/go-kit/kit/log"
"github.com/jmoiron/sqlx"
"github.com/kolide/kolide-ose/server/config"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pressly/goose"
)

109
server/datastore/mysql/hosts.go
View file

@ -3,13 +3,12 @@ package mysql
import (
"database/sql"
"fmt"
"net/http"
"time"
"github.com/jmoiron/sqlx"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/patrickmn/sortutil"
"github.com/pkg/errors"
)
func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) {
@ -24,15 +23,16 @@ func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) {
osquery_version,
os_version,
uptime,
physical_memory
physical_memory,
seen_time
)
VALUES( ?,?,?,?,?,?,?,?,?,? )
VALUES( ?,?,?,?,?,?,?,?,?,?,? )
`
result, err := d.db.Exec(sqlStatement, host.OsqueryHostID, host.DetailUpdateTime,
host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion,
host.OSVersion, host.Uptime, host.PhysicalMemory)
host.OSVersion, host.Uptime, host.PhysicalMemory, host.SeenTime)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "new host")
}
id, _ := result.LastInsertId()
host.ID = uint(id)
@ -169,13 +169,14 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
build = ?,
platform_like = ?,
code_name = ?,
cpu_logical_cores = ?
cpu_logical_cores = ?,
seen_time = ?
WHERE id = ?
`
tx, err := d.db.Beginx()
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "creating transaction")
}
_, err = tx.Exec(sqlStatement,
@ -202,21 +203,22 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
host.PlatformLike,
host.CodeName,
host.CPULogicalCores,
host.SeenTime,
host.ID)
if err != nil {
tx.Rollback()
return errors.DatabaseError(err)
return errors.Wrap(err, "executing main SQL statement")
}
host.NetworkInterfaces, err = updateNicsForHost(tx, host)
if err != nil {
tx.Rollback()
return errors.DatabaseError(err)
return errors.Wrap(err, "updating nics")
}
if err = removedUnusedNics(tx, host); err != nil {
tx.Rollback()
return errors.DatabaseError(err)
return errors.Wrap(err, "removing unused nics")
}
if needsUpdate := host.ResetPrimaryNetwork(); needsUpdate {
@ -228,13 +230,13 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
if err != nil {
tx.Rollback()
return errors.DatabaseError(err)
return errors.Wrap(err, "resetting primary network")
}
}
if err = tx.Commit(); err != nil {
tx.Rollback()
return errors.DatabaseError(err)
return errors.Wrap(err, "committing transaction")
}
return nil
}
@ -252,7 +254,7 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) {
host := &kolide.Host{}
err := d.db.Get(host, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "getting host by id")
}
if err := d.getNetInterfacesForHost(host); err != nil {
@ -271,7 +273,7 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
sqlStatement = appendListOptionsToSQL(sqlStatement, opt)
hosts := []*kolide.Host{}
if err := d.db.Select(&hosts, sqlStatement); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "list hosts")
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
@ -281,6 +283,44 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
return hosts, nil
}
func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, e error) {
sqlStatement := `
SELECT (
SELECT count(id)
FROM hosts
WHERE DATE_ADD(seen_time, INTERVAL 30 DAY) <= ?
) AS mia,
(
SELECT count(id)
FROM hosts
WHERE DATE_ADD(seen_time, INTERVAL 30 MINUTE) <= ?
AND DATE_ADD(seen_time, INTERVAL 30 DAY) >= ?
) AS offline,
(
SELECT count(id)
FROM hosts
WHERE DATE_ADD(seen_time, INTERVAL 30 MINUTE) > ?
) AS online
FROM hosts
LIMIT 1;
`
counts := struct {
MIA uint `db:"mia"`
Offline uint `db:"offline"`
Online uint `db:"online"`
}{}
if err := d.db.Get(&counts, sqlStatement, now, now, now, now); err != nil {
e = errors.Wrap(err, "generating host statistics")
return
}
mia = counts.MIA
offline = counts.Offline
online = counts.Online
return online, offline, mia, nil
}
// Optimized network interface fetch for sets of hosts. Instead of looping
// through hosts and doing a select for each host to get nics, we get all
// nics at once, so 2 db calls, and then assign nics to hosts here.
@ -348,21 +388,22 @@ func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error {
// EnrollHost enrolls a host
func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.Host, error) {
if osqueryHostID == "" {
return nil, errors.InternalServerError(fmt.Errorf("missing osquery host identifier"))
return nil, fmt.Errorf("missing osquery host identifier")
}
detailUpdateTime := time.Unix(0, 0).Add(24 * time.Hour)
nodeKey, err := kolide.RandomText(nodeKeySize)
if err != nil {
return nil, errors.InternalServerError(err)
return nil, errors.Wrap(err, "generating random text")
}
sqlInsert := `
INSERT INTO hosts (
detail_update_time,
osquery_host_id,
seen_time,
node_key
) VALUES (?, ?, ?)
) VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
node_key = VALUES(node_key),
deleted = FALSE
@ -370,10 +411,10 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H
var result sql.Result
result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, nodeKey)
result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, time.Now().UTC(), nodeKey)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting")
}
id, _ := result.LastInsertId()
@ -383,7 +424,7 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H
host := &kolide.Host{}
err = d.db.Get(host, sqlSelect, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "getting the host to return")
}
return host, nil
@ -402,16 +443,14 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
if err := d.db.Get(host, sqlStatement, nodeKey); err != nil {
switch err {
case sql.ErrNoRows:
e := errors.NewFromError(err, http.StatusUnauthorized, "invalid node key")
e.Extra = map[string]interface{}{"node_invalid": "true"}
return nil, e
return nil, errors.Wrap(err, "host not found")
default:
return nil, errors.DatabaseError(err)
return nil, errors.New("finding host")
}
}
if err := d.getNetInterfacesForHost(host); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "getting interfaces")
}
return host, nil
@ -420,13 +459,13 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
sqlStatement := `
UPDATE hosts SET
updated_at = ?
seen_time = ?
WHERE node_key=?
`
_, err := d.db.Exec(sqlStatement, t, host.NodeKey)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "marking host seen")
}
host.UpdatedAt = t
@ -468,7 +507,7 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.
sql, args, err := sqlx.In(sqlStatement, hostnameQuery, ipQuery, omit)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "searching hosts")
}
sql = d.db.Rebind(sql)
@ -476,11 +515,11 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.
err = d.db.Select(&hosts, sql, args...)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "searching hosts rebound")
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
return nil, err
return nil, errors.Wrap(err, "getting network interfaces")
}
return hosts, nil
@ -527,11 +566,11 @@ func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, err
hosts := []*kolide.Host{}
if err := d.db.Select(&hosts, sqlStatement, hostnameQuery, ipQuery); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "searching hosts")
}
if err := d.getNetInterfacesForHosts(hosts); err != nil {
return nil, err
return nil, errors.Wrap(err, "getting interfaces")
}
return hosts, nil
@ -559,7 +598,7 @@ func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]strin
rows, err := d.db.Query(sqlStatement, kolide.TargetLabel, kolide.TargetLabel,
kolide.TargetHost, kolide.QueryRunning, host.ID)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "finding distributed queries for host")
}
defer rows.Close()
@ -572,7 +611,7 @@ func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]strin
)
err = rows.Scan(&id, &query)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "scanning query results")
}
results[id] = query

1
server/datastore/mysql/labels.go
View file

@ -133,7 +133,6 @@ func (d *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[st
}
return nil
}
// ListLabelsForHost returns a list of kolide.Label for a given host id.

27
server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go Normal file
View file

@ -0,0 +1,27 @@
package migration
import (
"database/sql"
"github.com/pressly/goose"
)
func init() {
goose.AddMigration(Up_20170104113816, Down_20170104113816)
}
func Up_20170104113816(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `hosts` " +
"ADD COLUMN `seen_time` timestamp NULL DEFAULT NULL;",
)
return err
}
func Down_20170104113816(tx *sql.Tx) error {
_, err := tx.Exec(
"ALTER TABLE `hosts` " +
"DROP COLUMN `seen_time`;",
)
return err
}

39
server/kolide/hosts.go
View file

@ -8,6 +8,21 @@ import (
"golang.org/x/net/context"
)
const (
// StatusOnline host is active
StatusOnline string = "online"
// StatusOffline no communication with host for OfflineDuration
StatusOffline string = "offline"
// StatusMIA no communition with host for MIADuration
StatusMIA string = "mia"
// OfflineDuration if a host hasn't been in communition for this
// period it is considered offline
OfflineDuration time.Duration = 30 * time.Minute
// OfflineDuration if a host hasn't been in communition for this
// period it is considered MIA
MIADuration time.Duration = 30 * 24 * time.Hour
)
type HostStore interface {
NewHost(host *Host) (*Host, error)
SaveHost(host *Host) error
@ -17,6 +32,7 @@ type HostStore interface {
EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error)
AuthenticateHost(nodeKey string) (*Host, error)
MarkHostSeen(host *Host, t time.Time) error
GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error)
SearchHosts(query string, omit ...uint) ([]*Host, error)
// DistributedQueriesForHost retrieves the distributed queries that the
// given host should run. The result map is a mapping from campaign ID
@ -27,7 +43,7 @@ type HostStore interface {
type HostService interface {
ListHosts(ctx context.Context, opt ListOptions) ([]*Host, error)
GetHost(ctx context.Context, id uint) (*Host, error)
HostStatus(ctx context.Context, host Host) string
GetHostSummary(ctx context.Context) (*HostSummary, error)
DeleteHost(ctx context.Context, id uint) error
}
@ -40,6 +56,7 @@ type Host struct {
// a GUID or a Host Name, but in either case, it MUST be unique
OsqueryHostID string `json:"-" db:"osquery_host_id"`
DetailUpdateTime time.Time `json:"detail_updated_at" db:"detail_update_time"` // Time that the host details were last updated
SeenTime time.Time `json:"seen_time" db:"seen_time"` // Time that the host was last "seen"
NodeKey string `json:"-" db:"node_key"`
HostName string `json:"hostname" db:"host_name"` // there is a fulltext index on this field
UUID string `json:"uuid"`
@ -68,6 +85,15 @@ type Host struct {
NetworkInterfaces []*NetworkInterface `json:"network_interfaces" db:"-"`
}
// HostSummary is a structure which represents a data summary about the total
// set of hosts in the database. This structure is returned by the HostService
// method GetHostSummary
type HostSummary struct {
OnlineCount uint `json:"online_count"`
OfflineCount uint `json:"offline_count"`
MIACount uint `json:"mia_count"`
}
// ResetPrimaryNetwork will determine if the PrimaryNetworkInterfaceID
// needs to change. If it has not been set, it will default to the interface
// with the most IO. If it doesn't match an existing nic (as in the nic got changed)
@ -111,3 +137,14 @@ func RandomText(keySize int) (string, error) {
return base64.StdEncoding.EncodeToString(key), nil
}
func (h *Host) Status(now time.Time) string {
switch {
case h.SeenTime.Add(MIADuration).Before(now):
return StatusMIA
case h.SeenTime.Add(OfflineDuration).Before(now):
return StatusOffline
default:
return StatusOnline
}
}

20
server/kolide/hosts_test.go
View file

@ -2,7 +2,9 @@ package kolide
import (
"testing"
"time"
"github.com/WatchBeam/clock"
"github.com/stretchr/testify/assert"
)
@ -41,3 +43,21 @@ func TestResetHosts(t *testing.T) {
assert.True(t, result)
assert.Nil(t, host.PrimaryNetworkInterfaceID)
}
func TestHostStatus(t *testing.T) {
mockClock := clock.NewMockClock()
host := Host{}
host.SeenTime = mockClock.Now()
assert.Equal(t, StatusOnline, host.Status(mockClock.Now()))
host.SeenTime = mockClock.Now().Add(-1 * time.Minute)
assert.Equal(t, StatusOnline, host.Status(mockClock.Now()))
host.SeenTime = mockClock.Now().Add(-1 * time.Hour)
assert.Equal(t, StatusOffline, host.Status(mockClock.Now()))
host.SeenTime = mockClock.Now().Add(-35 * (24 * time.Hour)) // 35 days
assert.Equal(t, StatusMIA, host.Status(mockClock.Now()))
}

56
server/service/endpoint_hosts.go
View file

@ -8,21 +8,6 @@ import (
"golang.org/x/net/context"
)
const (
// StatusOnline host is active
StatusOnline string = "online"
// StatusOffline no communication with host for OfflineDuration
StatusOffline string = "offline"
// StatusMIA no communition with host for MIADuration
StatusMIA string = "mia"
// OfflineDuration if a host hasn't been in communition for this
// period it is considered offline
OfflineDuration time.Duration = 30 * time.Minute
// OfflineDuration if a host hasn't been in communition for this
// period it is considered MIA
MIADuration time.Duration = 30 * 24 * time.Hour
)
type hostResponse struct {
kolide.Host
Status string `json:"status"`
@ -50,7 +35,13 @@ func makeGetHostEndpoint(svc kolide.Service) endpoint.Endpoint {
if err != nil {
return getHostResponse{Err: err}, nil
}
return getHostResponse{&hostResponse{*host, svc.HostStatus(ctx, *host)}, nil}, nil
return getHostResponse{
&hostResponse{
Host: *host,
Status: host.Status(time.Now()),
},
nil,
}, nil
}
}
@ -79,7 +70,38 @@ func makeListHostsEndpoint(svc kolide.Service) endpoint.Endpoint {
resp := listHostsResponse{Hosts: []hostResponse{}}
for _, host := range hosts {
resp.Hosts = append(resp.Hosts, hostResponse{*host, svc.HostStatus(ctx, *host)})
resp.Hosts = append(
resp.Hosts,
hostResponse{
Host: *host,
Status: host.Status(time.Now()),
},
)
}
return resp, nil
}
}
////////////////////////////////////////////////////////////////////////////////
// Get Host Summary
////////////////////////////////////////////////////////////////////////////////
type getHostSummaryResponse struct {
kolide.HostSummary
Err error `json:"error,omitempty"`
}
func (r getHostSummaryResponse) error() error { return r.Err }
func makeGetHostSummaryEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
summary, err := svc.GetHostSummary(ctx)
if err != nil {
return getHostSummaryResponse{Err: err}, nil
}
resp := getHostSummaryResponse{
HostSummary: *summary,
}
return resp, nil
}

7
server/service/endpoint_targets.go
View file

@ -1,6 +1,8 @@
package service
import (
"time"
"github.com/go-kit/kit/endpoint"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
@ -65,7 +67,10 @@ func makeSearchTargetsEndpoint(svc kolide.Service) endpoint.Endpoint {
for _, host := range results.Hosts {
targets.Hosts = append(targets.Hosts,
hostSearchResult{
hostResponse{host, svc.HostStatus(ctx, host)},
hostResponse{
Host: host,
Status: host.Status(time.Now()),
},
host.HostName,
},
)

5
server/service/handler.go
View file

@ -63,6 +63,7 @@ type KolideEndpoints struct {
GetHost endpoint.Endpoint
DeleteHost endpoint.Endpoint
ListHosts endpoint.Endpoint
GetHostSummary endpoint.Endpoint
SearchTargets endpoint.Endpoint
GetOptions endpoint.Endpoint
ModifyOptions endpoint.Endpoint
@ -118,6 +119,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint
DeleteScheduledQuery: authenticatedUser(jwtKey, svc, makeDeleteScheduledQueryEndpoint(svc)),
GetHost: authenticatedUser(jwtKey, svc, makeGetHostEndpoint(svc)),
ListHosts: authenticatedUser(jwtKey, svc, makeListHostsEndpoint(svc)),
GetHostSummary: authenticatedUser(jwtKey, svc, makeGetHostSummaryEndpoint(svc)),
DeleteHost: authenticatedUser(jwtKey, svc, makeDeleteHostEndpoint(svc)),
GetLabel: authenticatedUser(jwtKey, svc, makeGetLabelEndpoint(svc)),
ListLabels: authenticatedUser(jwtKey, svc, makeListLabelsEndpoint(svc)),
@ -186,6 +188,7 @@ type kolideHandlers struct {
GetHost http.Handler
DeleteHost http.Handler
ListHosts http.Handler
GetHostSummary http.Handler
SearchTargets http.Handler
GetOptions http.Handler
ModifyOptions http.Handler
@ -245,6 +248,7 @@ func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithtt
GetHost: newServer(e.GetHost, decodeGetHostRequest),
DeleteHost: newServer(e.DeleteHost, decodeDeleteHostRequest),
ListHosts: newServer(e.ListHosts, decodeListHostsRequest),
GetHostSummary: newServer(e.GetHostSummary, decodeNoParamsRequest),
SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest),
GetOptions: newServer(e.GetOptions, decodeNoParamsRequest),
ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest),
@ -337,6 +341,7 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/labels/{id}", h.DeleteLabel).Methods("DELETE").Name("delete_label")
r.Handle("/api/v1/kolide/hosts", h.ListHosts).Methods("GET").Name("list_hosts")
r.Handle("/api/v1/kolide/host_summary", h.GetHostSummary).Methods("GET").Name("get_host_summary")
r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET").Name("get_host")
r.Handle("/api/v1/kolide/hosts/{id}", h.DeleteHost).Methods("DELETE").Name("delete_host")

4
server/service/handler_test.go
View file

@ -196,6 +196,10 @@ func TestAPIRoutes(t *testing.T) {
verb: "DELETE",
uri: "/api/v1/kolide/hosts/1",
},
{
verb: "GET",
uri: "/api/v1/kolide/host_summary",
},
}
for _, route := range routes {

4
server/service/service.go
View file

@ -56,3 +56,7 @@ type service struct {
func (s service) SendEmail(mail kolide.Email) error {
return s.mailService.SendEmail(mail)
}
func (s service) Clock() clock.Clock {
return s.clock
}

17
server/service/service_hosts.go
View file

@ -13,15 +13,16 @@ func (svc service) GetHost(ctx context.Context, id uint) (*kolide.Host, error) {
return svc.ds.Host(id)
}
func (svc service) HostStatus(ctx context.Context, host kolide.Host) string {
switch {
case host.UpdatedAt.Add(MIADuration).Before(svc.clock.Now()):
return StatusMIA
case host.UpdatedAt.Add(OfflineDuration).Before(svc.clock.Now()):
return StatusOffline
default:
return StatusOnline
func (svc service) GetHostSummary(ctx context.Context) (*kolide.HostSummary, error) {
online, offline, mia, err := svc.ds.GenerateHostStatusStatistics(svc.clock.Now())
if err != nil {
return nil, err
}
return &kolide.HostSummary{
OnlineCount: online,
OfflineCount: offline,
MIACount: mia,
}, nil
}
func (svc service) DeleteHost(ctx context.Context, id uint) error {

26
server/service/service_hosts_test.go
View file

@ -2,14 +2,11 @@ package service
import (
"testing"
"time"
"github.com/WatchBeam/clock"
"github.com/kolide/kolide-ose/server/config"
"github.com/kolide/kolide-ose/server/datastore/inmem"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/context"
)
@ -80,26 +77,3 @@ func TestDeleteHost(t *testing.T) {
assert.Len(t, hosts, 0)
}
func TestHostStatus(t *testing.T) {
mockClock := clock.NewMockClock()
svc, err := newTestServiceWithClock(nil, nil, mockClock)
require.Nil(t, err)
assert.Nil(t, err)
ctx := context.Background()
host := kolide.Host{}
host.UpdatedAt = mockClock.Now()
assert.Equal(t, StatusOnline, svc.HostStatus(ctx, host))
host.UpdatedAt = mockClock.Now().Add(-1 * time.Minute)
assert.Equal(t, StatusOnline, svc.HostStatus(ctx, host))
host.UpdatedAt = mockClock.Now().Add(-1 * time.Hour)
assert.Equal(t, StatusOffline, svc.HostStatus(ctx, host))
host.UpdatedAt = mockClock.Now().Add(-24 * 35 * time.Hour) // 35 days
assert.Equal(t, StatusMIA, svc.HostStatus(ctx, host))
}

4
server/service/service_osquery_test.go
View file

@ -619,6 +619,8 @@ func TestDistributedQueries(t *testing.T) {
host, err := ds.AuthenticateHost(nodeKey)
require.Nil(t, err)
err = ds.MarkHostSeen(host, mockClock.Now())
require.Nil(t, err)
ctx = hostctx.NewContext(ctx, *host)
@ -640,6 +642,8 @@ func TestDistributedQueries(t *testing.T) {
})
err = ds.RecordLabelQueryExecutions(host, map[string]bool{labelId: true}, mockClock.Now())
require.Nil(t, err)
err = ds.MarkHostSeen(host, mockClock.Now())
require.Nil(t, err)
q = "select year, month, day, hour, minutes, seconds from time"
campaign, err := svc.NewDistributedQueryCampaign(ctx, q, []uint{}, []uint{label.ID})

8
server/service/service_targets.go
View file

@ -47,12 +47,12 @@ func (svc service) CountHostsInTargets(ctx context.Context, hostIDs []uint, labe
for _, host := range hosts {
if !hostLookup[host.ID] {
hostLookup[host.ID] = true
switch svc.HostStatus(ctx, host) {
case StatusOnline:
switch host.Status(svc.clock.Now().UTC()) {
case kolide.StatusOnline:
result.OnlineHosts++
case StatusOffline:
case kolide.StatusOffline:
result.OfflineHosts++
case StatusMIA:
case kolide.StatusMIA:
result.MissingInActionHosts++
}
}

4
server/service/service_targets_test.go
View file

@ -122,12 +122,12 @@ func TestCountHostsInTargets(t *testing.T) {
l2ID := fmt.Sprintf("%d", l2.ID)
for _, h := range []*kolide.Host{h1, h2, h3, h6} {
err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now())
err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, mockClock.Now())
assert.Nil(t, err)
}
for _, h := range []*kolide.Host{h3, h4, h5} {
err = ds.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now())
err = ds.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, mockClock.Now())
assert.Nil(t, err)
}

1
server/test/new_objects.go
View file

@ -93,6 +93,7 @@ func NewHost(t *testing.T, ds kolide.Datastore, name, ip, key, uuid string, now
NodeKey: key,
UUID: uuid,
DetailUpdateTime: now,
SeenTime: now,
OsqueryHostID: osqueryHostID,
})