diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index 351ea59f9a..ef9ec14a3f 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/WatchBeam/clock" "github.com/kolide/kolide-ose/server/kolide" "github.com/kolide/kolide-ose/server/test" "github.com/stretchr/testify/assert" @@ -41,6 +42,7 @@ var enrollTests = []struct { func testSaveHosts(t *testing.T, ds kolide.Datastore) { host, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "1", UUID: "1", HostName: "foo.local", @@ -119,6 +121,7 @@ func testSaveHosts(t *testing.T, ds kolide.Datastore) { func testDeleteHost(t *testing.T, ds kolide.Datastore) { host, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "1", UUID: "1", HostName: "foo.local", @@ -138,6 +141,7 @@ func testListHost(t *testing.T, ds kolide.Datastore) { for i := 0; i < 10; i++ { host, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), UUID: fmt.Sprintf("%d", i), @@ -247,6 +251,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) { _, err := ds.NewHost(&kolide.Host{ OsqueryHostID: "1234", DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "1", UUID: "1", HostName: "foo.local", @@ -256,6 +261,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) { h2, err := ds.NewHost(&kolide.Host{ OsqueryHostID: "5679", DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "2", UUID: "2", HostName: "bar.local", @@ -265,6 +271,7 @@ func testSearchHosts(t *testing.T, ds kolide.Datastore) { h3, err := ds.NewHost(&kolide.Host{ OsqueryHostID: "99999", DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "3", UUID: "3", HostName: "foo-bar.local", @@ -337,6 +344,7 @@ func testSearchHostsLimit(t *testing.T, ds kolide.Datastore) { for i := 0; i < 15; i++ { _, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), UUID: fmt.Sprintf("%d", i), @@ -356,6 +364,7 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) { h1, err := ds.NewHost(&kolide.Host{ OsqueryHostID: "1", DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "1", UUID: "1", HostName: "foo.local", @@ -365,6 +374,7 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) { h2, err := ds.NewHost(&kolide.Host{ OsqueryHostID: "2", DetailUpdateTime: time.Now(), + SeenTime: time.Now(), NodeKey: "2", UUID: "2", HostName: "bar.local", @@ -487,5 +497,92 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) { queries, err = ds.DistributedQueriesForHost(h2) require.Nil(t, err) assert.Empty(t, queries) - +} + +func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { + mockClock := clock.NewMockClock() + + // Online + _, err := ds.NewHost(&kolide.Host{ + ID: 1, + OsqueryHostID: "1", + UUID: "1", + NodeKey: "1", + DetailUpdateTime: mockClock.Now(), + SeenTime: mockClock.Now(), + }) + assert.Nil(t, err) + + // Online + _, err = ds.NewHost(&kolide.Host{ + ID: 2, + OsqueryHostID: "2", + UUID: "2", + NodeKey: "2", + DetailUpdateTime: mockClock.Now().Add(-1 * time.Minute), + SeenTime: mockClock.Now().Add(-1 * time.Minute), + }) + assert.Nil(t, err) + + // Offline + _, err = ds.NewHost(&kolide.Host{ + ID: 3, + OsqueryHostID: "3", + UUID: "3", + NodeKey: "3", + DetailUpdateTime: mockClock.Now().Add(-1 * time.Hour), + SeenTime: mockClock.Now().Add(-1 * time.Hour), + }) + assert.Nil(t, err) + + // MIA + _, err = ds.NewHost(&kolide.Host{ + ID: 4, + OsqueryHostID: "4", + UUID: "4", + NodeKey: "4", + DetailUpdateTime: mockClock.Now().Add(-35 * (24 * time.Hour)), + SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)), + }) + assert.Nil(t, err) + + online, offline, mia, err := ds.GenerateHostStatusStatistics(mockClock.Now()) + assert.Nil(t, err) + assert.Equal(t, uint(2), online) + assert.Equal(t, uint(1), offline) + assert.Equal(t, uint(1), mia) +} + +func testMarkHostSeen(t *testing.T, ds kolide.Datastore) { + mockClock := clock.NewMockClock() + + anHourAgo := mockClock.Now().Add(-1 * time.Hour).UTC() + aDayAgo := mockClock.Now().Add(-24 * time.Hour).UTC() + + h1, err := ds.NewHost(&kolide.Host{ + ID: 1, + OsqueryHostID: "1", + UUID: "1", + NodeKey: "1", + DetailUpdateTime: aDayAgo, + SeenTime: aDayAgo, + }) + assert.Nil(t, err) + + { + h1Verify, err := ds.Host(1) + assert.Nil(t, err) + require.NotNil(t, h1Verify) + assert.WithinDuration(t, aDayAgo, h1Verify.SeenTime, time.Second) + } + + err = ds.MarkHostSeen(h1, anHourAgo) + assert.Nil(t, err) + + { + h1Verify, err := ds.Host(1) + assert.Nil(t, err) + require.NotNil(t, h1Verify) + assert.WithinDuration(t, anHourAgo, h1Verify.SeenTime, time.Second) + } } diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index 5a815df3ed..cfba572b63 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -242,6 +242,7 @@ func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) { func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h1, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", UUID: "1", @@ -251,6 +252,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h2, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: "2", NodeKey: "2", UUID: "2", @@ -260,6 +262,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) { h3, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: "3", NodeKey: "3", UUID: "3", @@ -317,6 +320,7 @@ func testListUniqueHostsInLabels(t *testing.T, db kolide.Datastore) { for i := 0; i < 4; i++ { h, err := db.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), + SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), UUID: strconv.Itoa(i), diff --git a/server/datastore/datastore_packs_test.go b/server/datastore/datastore_packs_test.go index 44d0389db9..904c7e5a04 100644 --- a/server/datastore/datastore_packs_test.go +++ b/server/datastore/datastore_packs_test.go @@ -67,6 +67,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { h1, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: mockClock.Now(), + SeenTime: mockClock.Now(), HostName: "foobar.local", OsqueryHostID: "1", NodeKey: "1", @@ -87,6 +88,7 @@ func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { h2, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: mockClock.Now(), + SeenTime: mockClock.Now(), HostName: "foobaz.local", OsqueryHostID: "2", NodeKey: "2", diff --git a/server/datastore/datastore_password_reset_test.go b/server/datastore/datastore_password_reset_test.go index 861b4ed1ae..b23a5e6292 100644 --- a/server/datastore/datastore_password_reset_test.go +++ b/server/datastore/datastore_password_reset_test.go @@ -10,7 +10,7 @@ import ( func testPasswordResetRequests(t *testing.T, db kolide.Datastore) { createTestUsers(t, db) - now := time.Now() + now := time.Now().UTC() tomorrow := now.Add(time.Hour * 24) var passwordResetTests = []struct { userID uint diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index f5825142e4..86767afc87 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -60,4 +60,6 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testNewScheduledQuery, testOptionsToConfig, testAddLabelToPackTwice, + testGenerateHostStatusStatistics, + testMarkHostSeen, } diff --git a/server/datastore/inmem/hosts.go b/server/datastore/inmem/hosts.go index e047aea9cf..a775168c61 100644 --- a/server/datastore/inmem/hosts.go +++ b/server/datastore/inmem/hosts.go @@ -114,6 +114,25 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return hosts, nil } +func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error) { + d.mtx.Lock() + defer d.mtx.Unlock() + + for _, host := range d.hosts { + status := host.Status(now) + switch status { + case kolide.StatusMIA: + mia++ + case kolide.StatusOffline: + offline++ + default: + online++ + } + } + + return online, offline, mia, nil +} + func (d *Datastore) EnrollHost(osQueryHostID string, nodeKeySize int) (*kolide.Host, error) { d.mtx.Lock() defer d.mtx.Unlock() @@ -168,11 +187,10 @@ func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { d.mtx.Lock() defer d.mtx.Unlock() - host.UpdatedAt = t - for _, h := range d.hosts { if h.ID == host.ID { h.UpdatedAt = t + h.SeenTime = t break } } diff --git a/server/datastore/mysql/datastore.go b/server/datastore/mysql/datastore.go index b5a7080543..fa363dfbd3 100644 --- a/server/datastore/mysql/datastore.go +++ b/server/datastore/mysql/datastore.go @@ -6,10 +6,8 @@ import ( "github.com/WatchBeam/clock" "github.com/go-kit/kit/log" - "github.com/jmoiron/sqlx" "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/kolide" "github.com/pressly/goose" ) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index f4e8b087a6..71a9e1a800 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -3,13 +3,12 @@ package mysql import ( "database/sql" "fmt" - "net/http" "time" "github.com/jmoiron/sqlx" - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" "github.com/patrickmn/sortutil" + "github.com/pkg/errors" ) func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { @@ -24,15 +23,16 @@ func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { osquery_version, os_version, uptime, - physical_memory + physical_memory, + seen_time ) - VALUES( ?,?,?,?,?,?,?,?,?,? ) + VALUES( ?,?,?,?,?,?,?,?,?,?,? ) ` result, err := d.db.Exec(sqlStatement, host.OsqueryHostID, host.DetailUpdateTime, host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion, - host.OSVersion, host.Uptime, host.PhysicalMemory) + host.OSVersion, host.Uptime, host.PhysicalMemory, host.SeenTime) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "new host") } id, _ := result.LastInsertId() host.ID = uint(id) @@ -169,13 +169,14 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { build = ?, platform_like = ?, code_name = ?, - cpu_logical_cores = ? + cpu_logical_cores = ?, + seen_time = ? WHERE id = ? ` tx, err := d.db.Beginx() if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "creating transaction") } _, err = tx.Exec(sqlStatement, @@ -202,21 +203,22 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { host.PlatformLike, host.CodeName, host.CPULogicalCores, + host.SeenTime, host.ID) if err != nil { tx.Rollback() - return errors.DatabaseError(err) + return errors.Wrap(err, "executing main SQL statement") } host.NetworkInterfaces, err = updateNicsForHost(tx, host) if err != nil { tx.Rollback() - return errors.DatabaseError(err) + return errors.Wrap(err, "updating nics") } if err = removedUnusedNics(tx, host); err != nil { tx.Rollback() - return errors.DatabaseError(err) + return errors.Wrap(err, "removing unused nics") } if needsUpdate := host.ResetPrimaryNetwork(); needsUpdate { @@ -228,13 +230,13 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { if err != nil { tx.Rollback() - return errors.DatabaseError(err) + return errors.Wrap(err, "resetting primary network") } } if err = tx.Commit(); err != nil { tx.Rollback() - return errors.DatabaseError(err) + return errors.Wrap(err, "committing transaction") } return nil } @@ -252,7 +254,7 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) { host := &kolide.Host{} err := d.db.Get(host, sqlStatement, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "getting host by id") } if err := d.getNetInterfacesForHost(host); err != nil { @@ -271,7 +273,7 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { sqlStatement = appendListOptionsToSQL(sqlStatement, opt) hosts := []*kolide.Host{} if err := d.db.Select(&hosts, sqlStatement); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "list hosts") } if err := d.getNetInterfacesForHosts(hosts); err != nil { @@ -281,6 +283,44 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return hosts, nil } +func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, e error) { + sqlStatement := ` + SELECT ( + SELECT count(id) + FROM hosts + WHERE DATE_ADD(seen_time, INTERVAL 30 DAY) <= ? + ) AS mia, + ( + SELECT count(id) + FROM hosts + WHERE DATE_ADD(seen_time, INTERVAL 30 MINUTE) <= ? + AND DATE_ADD(seen_time, INTERVAL 30 DAY) >= ? + ) AS offline, + ( + SELECT count(id) + FROM hosts + WHERE DATE_ADD(seen_time, INTERVAL 30 MINUTE) > ? + ) AS online + FROM hosts + LIMIT 1; + ` + + counts := struct { + MIA uint `db:"mia"` + Offline uint `db:"offline"` + Online uint `db:"online"` + }{} + if err := d.db.Get(&counts, sqlStatement, now, now, now, now); err != nil { + e = errors.Wrap(err, "generating host statistics") + return + } + + mia = counts.MIA + offline = counts.Offline + online = counts.Online + return online, offline, mia, nil +} + // Optimized network interface fetch for sets of hosts. Instead of looping // through hosts and doing a select for each host to get nics, we get all // nics at once, so 2 db calls, and then assign nics to hosts here. @@ -348,21 +388,22 @@ func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error { // EnrollHost enrolls a host func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.Host, error) { if osqueryHostID == "" { - return nil, errors.InternalServerError(fmt.Errorf("missing osquery host identifier")) + return nil, fmt.Errorf("missing osquery host identifier") } detailUpdateTime := time.Unix(0, 0).Add(24 * time.Hour) nodeKey, err := kolide.RandomText(nodeKeySize) if err != nil { - return nil, errors.InternalServerError(err) + return nil, errors.Wrap(err, "generating random text") } sqlInsert := ` INSERT INTO hosts ( detail_update_time, osquery_host_id, + seen_time, node_key - ) VALUES (?, ?, ?) + ) VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE node_key = VALUES(node_key), deleted = FALSE @@ -370,10 +411,10 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H var result sql.Result - result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, nodeKey) + result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, time.Now().UTC(), nodeKey) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting") } id, _ := result.LastInsertId() @@ -383,7 +424,7 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H host := &kolide.Host{} err = d.db.Get(host, sqlSelect, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "getting the host to return") } return host, nil @@ -402,16 +443,14 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { if err := d.db.Get(host, sqlStatement, nodeKey); err != nil { switch err { case sql.ErrNoRows: - e := errors.NewFromError(err, http.StatusUnauthorized, "invalid node key") - e.Extra = map[string]interface{}{"node_invalid": "true"} - return nil, e + return nil, errors.Wrap(err, "host not found") default: - return nil, errors.DatabaseError(err) + return nil, errors.New("finding host") } } if err := d.getNetInterfacesForHost(host); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "getting interfaces") } return host, nil @@ -420,13 +459,13 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { sqlStatement := ` UPDATE hosts SET - updated_at = ? + seen_time = ? WHERE node_key=? ` _, err := d.db.Exec(sqlStatement, t, host.NodeKey) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "marking host seen") } host.UpdatedAt = t @@ -468,7 +507,7 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide. sql, args, err := sqlx.In(sqlStatement, hostnameQuery, ipQuery, omit) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "searching hosts") } sql = d.db.Rebind(sql) @@ -476,11 +515,11 @@ func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide. err = d.db.Select(&hosts, sql, args...) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "searching hosts rebound") } if err := d.getNetInterfacesForHosts(hosts); err != nil { - return nil, err + return nil, errors.Wrap(err, "getting network interfaces") } return hosts, nil @@ -527,11 +566,11 @@ func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, err hosts := []*kolide.Host{} if err := d.db.Select(&hosts, sqlStatement, hostnameQuery, ipQuery); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "searching hosts") } if err := d.getNetInterfacesForHosts(hosts); err != nil { - return nil, err + return nil, errors.Wrap(err, "getting interfaces") } return hosts, nil @@ -559,7 +598,7 @@ func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]strin rows, err := d.db.Query(sqlStatement, kolide.TargetLabel, kolide.TargetLabel, kolide.TargetHost, kolide.QueryRunning, host.ID) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "finding distributed queries for host") } defer rows.Close() @@ -572,7 +611,7 @@ func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]strin ) err = rows.Scan(&id, &query) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "scanning query results") } results[id] = query diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 70257c7757..b12d628f3d 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -133,7 +133,6 @@ func (d *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[st } return nil - } // ListLabelsForHost returns a list of kolide.Label for a given host id. diff --git a/server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go b/server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go new file mode 100644 index 0000000000..f70dbb82b8 --- /dev/null +++ b/server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go @@ -0,0 +1,27 @@ +package migration + +import ( + "database/sql" + + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up_20170104113816, Down_20170104113816) +} + +func Up_20170104113816(tx *sql.Tx) error { + _, err := tx.Exec( + "ALTER TABLE `hosts` " + + "ADD COLUMN `seen_time` timestamp NULL DEFAULT NULL;", + ) + return err +} + +func Down_20170104113816(tx *sql.Tx) error { + _, err := tx.Exec( + "ALTER TABLE `hosts` " + + "DROP COLUMN `seen_time`;", + ) + return err +} diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index 0deae099a7..7f697594f7 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -8,6 +8,21 @@ import ( "golang.org/x/net/context" ) +const ( + // StatusOnline host is active + StatusOnline string = "online" + // StatusOffline no communication with host for OfflineDuration + StatusOffline string = "offline" + // StatusMIA no communition with host for MIADuration + StatusMIA string = "mia" + // OfflineDuration if a host hasn't been in communition for this + // period it is considered offline + OfflineDuration time.Duration = 30 * time.Minute + // OfflineDuration if a host hasn't been in communition for this + // period it is considered MIA + MIADuration time.Duration = 30 * 24 * time.Hour +) + type HostStore interface { NewHost(host *Host) (*Host, error) SaveHost(host *Host) error @@ -17,6 +32,7 @@ type HostStore interface { EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error) AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error + GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error) SearchHosts(query string, omit ...uint) ([]*Host, error) // DistributedQueriesForHost retrieves the distributed queries that the // given host should run. The result map is a mapping from campaign ID @@ -27,7 +43,7 @@ type HostStore interface { type HostService interface { ListHosts(ctx context.Context, opt ListOptions) ([]*Host, error) GetHost(ctx context.Context, id uint) (*Host, error) - HostStatus(ctx context.Context, host Host) string + GetHostSummary(ctx context.Context) (*HostSummary, error) DeleteHost(ctx context.Context, id uint) error } @@ -40,6 +56,7 @@ type Host struct { // a GUID or a Host Name, but in either case, it MUST be unique OsqueryHostID string `json:"-" db:"osquery_host_id"` DetailUpdateTime time.Time `json:"detail_updated_at" db:"detail_update_time"` // Time that the host details were last updated + SeenTime time.Time `json:"seen_time" db:"seen_time"` // Time that the host was last "seen" NodeKey string `json:"-" db:"node_key"` HostName string `json:"hostname" db:"host_name"` // there is a fulltext index on this field UUID string `json:"uuid"` @@ -68,6 +85,15 @@ type Host struct { NetworkInterfaces []*NetworkInterface `json:"network_interfaces" db:"-"` } +// HostSummary is a structure which represents a data summary about the total +// set of hosts in the database. This structure is returned by the HostService +// method GetHostSummary +type HostSummary struct { + OnlineCount uint `json:"online_count"` + OfflineCount uint `json:"offline_count"` + MIACount uint `json:"mia_count"` +} + // ResetPrimaryNetwork will determine if the PrimaryNetworkInterfaceID // needs to change. If it has not been set, it will default to the interface // with the most IO. If it doesn't match an existing nic (as in the nic got changed) @@ -111,3 +137,14 @@ func RandomText(keySize int) (string, error) { return base64.StdEncoding.EncodeToString(key), nil } + +func (h *Host) Status(now time.Time) string { + switch { + case h.SeenTime.Add(MIADuration).Before(now): + return StatusMIA + case h.SeenTime.Add(OfflineDuration).Before(now): + return StatusOffline + default: + return StatusOnline + } +} diff --git a/server/kolide/hosts_test.go b/server/kolide/hosts_test.go index 2079ff77cb..1dbf648bb9 100644 --- a/server/kolide/hosts_test.go +++ b/server/kolide/hosts_test.go @@ -2,7 +2,9 @@ package kolide import ( "testing" + "time" + "github.com/WatchBeam/clock" "github.com/stretchr/testify/assert" ) @@ -41,3 +43,21 @@ func TestResetHosts(t *testing.T) { assert.True(t, result) assert.Nil(t, host.PrimaryNetworkInterfaceID) } + +func TestHostStatus(t *testing.T) { + mockClock := clock.NewMockClock() + + host := Host{} + + host.SeenTime = mockClock.Now() + assert.Equal(t, StatusOnline, host.Status(mockClock.Now())) + + host.SeenTime = mockClock.Now().Add(-1 * time.Minute) + assert.Equal(t, StatusOnline, host.Status(mockClock.Now())) + + host.SeenTime = mockClock.Now().Add(-1 * time.Hour) + assert.Equal(t, StatusOffline, host.Status(mockClock.Now())) + + host.SeenTime = mockClock.Now().Add(-35 * (24 * time.Hour)) // 35 days + assert.Equal(t, StatusMIA, host.Status(mockClock.Now())) +} diff --git a/server/service/endpoint_hosts.go b/server/service/endpoint_hosts.go index 412201e554..8f13ce0e58 100644 --- a/server/service/endpoint_hosts.go +++ b/server/service/endpoint_hosts.go @@ -8,21 +8,6 @@ import ( "golang.org/x/net/context" ) -const ( - // StatusOnline host is active - StatusOnline string = "online" - // StatusOffline no communication with host for OfflineDuration - StatusOffline string = "offline" - // StatusMIA no communition with host for MIADuration - StatusMIA string = "mia" - // OfflineDuration if a host hasn't been in communition for this - // period it is considered offline - OfflineDuration time.Duration = 30 * time.Minute - // OfflineDuration if a host hasn't been in communition for this - // period it is considered MIA - MIADuration time.Duration = 30 * 24 * time.Hour -) - type hostResponse struct { kolide.Host Status string `json:"status"` @@ -50,7 +35,13 @@ func makeGetHostEndpoint(svc kolide.Service) endpoint.Endpoint { if err != nil { return getHostResponse{Err: err}, nil } - return getHostResponse{&hostResponse{*host, svc.HostStatus(ctx, *host)}, nil}, nil + return getHostResponse{ + &hostResponse{ + Host: *host, + Status: host.Status(time.Now()), + }, + nil, + }, nil } } @@ -79,7 +70,38 @@ func makeListHostsEndpoint(svc kolide.Service) endpoint.Endpoint { resp := listHostsResponse{Hosts: []hostResponse{}} for _, host := range hosts { - resp.Hosts = append(resp.Hosts, hostResponse{*host, svc.HostStatus(ctx, *host)}) + resp.Hosts = append( + resp.Hosts, + hostResponse{ + Host: *host, + Status: host.Status(time.Now()), + }, + ) + } + return resp, nil + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Get Host Summary +//////////////////////////////////////////////////////////////////////////////// + +type getHostSummaryResponse struct { + kolide.HostSummary + Err error `json:"error,omitempty"` +} + +func (r getHostSummaryResponse) error() error { return r.Err } + +func makeGetHostSummaryEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + summary, err := svc.GetHostSummary(ctx) + if err != nil { + return getHostSummaryResponse{Err: err}, nil + } + + resp := getHostSummaryResponse{ + HostSummary: *summary, } return resp, nil } diff --git a/server/service/endpoint_targets.go b/server/service/endpoint_targets.go index 38d1798ca9..c346611b13 100644 --- a/server/service/endpoint_targets.go +++ b/server/service/endpoint_targets.go @@ -1,6 +1,8 @@ package service import ( + "time" + "github.com/go-kit/kit/endpoint" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" @@ -65,7 +67,10 @@ func makeSearchTargetsEndpoint(svc kolide.Service) endpoint.Endpoint { for _, host := range results.Hosts { targets.Hosts = append(targets.Hosts, hostSearchResult{ - hostResponse{host, svc.HostStatus(ctx, host)}, + hostResponse{ + Host: host, + Status: host.Status(time.Now()), + }, host.HostName, }, ) diff --git a/server/service/handler.go b/server/service/handler.go index 3b54d26e27..ea90174bc2 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -63,6 +63,7 @@ type KolideEndpoints struct { GetHost endpoint.Endpoint DeleteHost endpoint.Endpoint ListHosts endpoint.Endpoint + GetHostSummary endpoint.Endpoint SearchTargets endpoint.Endpoint GetOptions endpoint.Endpoint ModifyOptions endpoint.Endpoint @@ -118,6 +119,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint DeleteScheduledQuery: authenticatedUser(jwtKey, svc, makeDeleteScheduledQueryEndpoint(svc)), GetHost: authenticatedUser(jwtKey, svc, makeGetHostEndpoint(svc)), ListHosts: authenticatedUser(jwtKey, svc, makeListHostsEndpoint(svc)), + GetHostSummary: authenticatedUser(jwtKey, svc, makeGetHostSummaryEndpoint(svc)), DeleteHost: authenticatedUser(jwtKey, svc, makeDeleteHostEndpoint(svc)), GetLabel: authenticatedUser(jwtKey, svc, makeGetLabelEndpoint(svc)), ListLabels: authenticatedUser(jwtKey, svc, makeListLabelsEndpoint(svc)), @@ -186,6 +188,7 @@ type kolideHandlers struct { GetHost http.Handler DeleteHost http.Handler ListHosts http.Handler + GetHostSummary http.Handler SearchTargets http.Handler GetOptions http.Handler ModifyOptions http.Handler @@ -245,6 +248,7 @@ func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithtt GetHost: newServer(e.GetHost, decodeGetHostRequest), DeleteHost: newServer(e.DeleteHost, decodeDeleteHostRequest), ListHosts: newServer(e.ListHosts, decodeListHostsRequest), + GetHostSummary: newServer(e.GetHostSummary, decodeNoParamsRequest), SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest), GetOptions: newServer(e.GetOptions, decodeNoParamsRequest), ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest), @@ -337,6 +341,7 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) { r.Handle("/api/v1/kolide/labels/{id}", h.DeleteLabel).Methods("DELETE").Name("delete_label") r.Handle("/api/v1/kolide/hosts", h.ListHosts).Methods("GET").Name("list_hosts") + r.Handle("/api/v1/kolide/host_summary", h.GetHostSummary).Methods("GET").Name("get_host_summary") r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET").Name("get_host") r.Handle("/api/v1/kolide/hosts/{id}", h.DeleteHost).Methods("DELETE").Name("delete_host") diff --git a/server/service/handler_test.go b/server/service/handler_test.go index 32a95aa216..c113d4a3c4 100644 --- a/server/service/handler_test.go +++ b/server/service/handler_test.go @@ -196,6 +196,10 @@ func TestAPIRoutes(t *testing.T) { verb: "DELETE", uri: "/api/v1/kolide/hosts/1", }, + { + verb: "GET", + uri: "/api/v1/kolide/host_summary", + }, } for _, route := range routes { diff --git a/server/service/service.go b/server/service/service.go index 0e3f233301..0bca0bf0f7 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -56,3 +56,7 @@ type service struct { func (s service) SendEmail(mail kolide.Email) error { return s.mailService.SendEmail(mail) } + +func (s service) Clock() clock.Clock { + return s.clock +} diff --git a/server/service/service_hosts.go b/server/service/service_hosts.go index e9b0321851..b27acc1329 100644 --- a/server/service/service_hosts.go +++ b/server/service/service_hosts.go @@ -13,15 +13,16 @@ func (svc service) GetHost(ctx context.Context, id uint) (*kolide.Host, error) { return svc.ds.Host(id) } -func (svc service) HostStatus(ctx context.Context, host kolide.Host) string { - switch { - case host.UpdatedAt.Add(MIADuration).Before(svc.clock.Now()): - return StatusMIA - case host.UpdatedAt.Add(OfflineDuration).Before(svc.clock.Now()): - return StatusOffline - default: - return StatusOnline +func (svc service) GetHostSummary(ctx context.Context) (*kolide.HostSummary, error) { + online, offline, mia, err := svc.ds.GenerateHostStatusStatistics(svc.clock.Now()) + if err != nil { + return nil, err } + return &kolide.HostSummary{ + OnlineCount: online, + OfflineCount: offline, + MIACount: mia, + }, nil } func (svc service) DeleteHost(ctx context.Context, id uint) error { diff --git a/server/service/service_hosts_test.go b/server/service/service_hosts_test.go index e84a8266d3..960d850540 100644 --- a/server/service/service_hosts_test.go +++ b/server/service/service_hosts_test.go @@ -2,14 +2,11 @@ package service import ( "testing" - "time" - "github.com/WatchBeam/clock" "github.com/kolide/kolide-ose/server/config" "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "golang.org/x/net/context" ) @@ -80,26 +77,3 @@ func TestDeleteHost(t *testing.T) { assert.Len(t, hosts, 0) } - -func TestHostStatus(t *testing.T) { - mockClock := clock.NewMockClock() - svc, err := newTestServiceWithClock(nil, nil, mockClock) - require.Nil(t, err) - - assert.Nil(t, err) - ctx := context.Background() - - host := kolide.Host{} - - host.UpdatedAt = mockClock.Now() - assert.Equal(t, StatusOnline, svc.HostStatus(ctx, host)) - - host.UpdatedAt = mockClock.Now().Add(-1 * time.Minute) - assert.Equal(t, StatusOnline, svc.HostStatus(ctx, host)) - - host.UpdatedAt = mockClock.Now().Add(-1 * time.Hour) - assert.Equal(t, StatusOffline, svc.HostStatus(ctx, host)) - - host.UpdatedAt = mockClock.Now().Add(-24 * 35 * time.Hour) // 35 days - assert.Equal(t, StatusMIA, svc.HostStatus(ctx, host)) -} diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 72bc8dc5d0..514b248288 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -619,6 +619,8 @@ func TestDistributedQueries(t *testing.T) { host, err := ds.AuthenticateHost(nodeKey) require.Nil(t, err) + err = ds.MarkHostSeen(host, mockClock.Now()) + require.Nil(t, err) ctx = hostctx.NewContext(ctx, *host) @@ -640,6 +642,8 @@ func TestDistributedQueries(t *testing.T) { }) err = ds.RecordLabelQueryExecutions(host, map[string]bool{labelId: true}, mockClock.Now()) require.Nil(t, err) + err = ds.MarkHostSeen(host, mockClock.Now()) + require.Nil(t, err) q = "select year, month, day, hour, minutes, seconds from time" campaign, err := svc.NewDistributedQueryCampaign(ctx, q, []uint{}, []uint{label.ID}) diff --git a/server/service/service_targets.go b/server/service/service_targets.go index c1a7ae40c2..3b5d15ded0 100644 --- a/server/service/service_targets.go +++ b/server/service/service_targets.go @@ -47,12 +47,12 @@ func (svc service) CountHostsInTargets(ctx context.Context, hostIDs []uint, labe for _, host := range hosts { if !hostLookup[host.ID] { hostLookup[host.ID] = true - switch svc.HostStatus(ctx, host) { - case StatusOnline: + switch host.Status(svc.clock.Now().UTC()) { + case kolide.StatusOnline: result.OnlineHosts++ - case StatusOffline: + case kolide.StatusOffline: result.OfflineHosts++ - case StatusMIA: + case kolide.StatusMIA: result.MissingInActionHosts++ } } diff --git a/server/service/service_targets_test.go b/server/service/service_targets_test.go index e205fe64c3..bb088de96d 100644 --- a/server/service/service_targets_test.go +++ b/server/service/service_targets_test.go @@ -122,12 +122,12 @@ func TestCountHostsInTargets(t *testing.T) { l2ID := fmt.Sprintf("%d", l2.ID) for _, h := range []*kolide.Host{h1, h2, h3, h6} { - err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, time.Now()) + err = ds.RecordLabelQueryExecutions(h, map[string]bool{l1ID: true}, mockClock.Now()) assert.Nil(t, err) } for _, h := range []*kolide.Host{h3, h4, h5} { - err = ds.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, time.Now()) + err = ds.RecordLabelQueryExecutions(h, map[string]bool{l2ID: true}, mockClock.Now()) assert.Nil(t, err) } diff --git a/server/test/new_objects.go b/server/test/new_objects.go index 92aacf3ccd..19d26aab80 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -93,6 +93,7 @@ func NewHost(t *testing.T, ds kolide.Datastore, name, ip, key, uuid string, now NodeKey: key, UUID: uuid, DetailUpdateTime: now, + SeenTime: now, OsqueryHostID: osqueryHostID, })