From eac718e9376da5609f8faad3ce4e0a2aadb0906d Mon Sep 17 00:00:00 2001 From: Victor Vrantchan Date: Fri, 20 Jan 2017 08:57:47 -0500 Subject: [PATCH] add new status to host summary endpoint (#1057) new_count shows all hosts that have been added to kolide in the last 24 hours --- server/datastore/datastore_hosts_test.go | 18 +++++++++-- server/datastore/inmem/hosts.go | 8 +++-- server/datastore/mysql/hosts.go | 15 ++++++--- server/kolide/hosts.go | 40 +++++++++++++++++------- server/kolide/hosts_test.go | 12 +++++++ server/service/service_hosts.go | 3 +- 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index f41a8e78a3..a615d174c6 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -501,11 +501,12 @@ func testDistributedQueriesForHost(t *testing.T, ds kolide.Datastore) { func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { mockClock := clock.NewMockClock() - online, offline, mia, err := ds.GenerateHostStatusStatistics(mockClock.Now()) + online, offline, mia, new, err := ds.GenerateHostStatusStatistics(mockClock.Now()) assert.Nil(t, err) assert.Equal(t, uint(0), online) assert.Equal(t, uint(0), offline) assert.Equal(t, uint(0), mia) + assert.Equal(t, uint(0), new) // Online _, err = ds.NewHost(&kolide.Host{ @@ -515,6 +516,9 @@ func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { NodeKey: "1", DetailUpdateTime: mockClock.Now(), SeenTime: mockClock.Now(), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{CreatedAt: mockClock.Now()}, + }, }) assert.Nil(t, err) @@ -526,6 +530,9 @@ func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { NodeKey: "2", DetailUpdateTime: mockClock.Now().Add(-1 * time.Minute), SeenTime: mockClock.Now().Add(-1 * time.Minute), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{CreatedAt: mockClock.Now()}, + }, }) assert.Nil(t, err) @@ -537,6 +544,9 @@ func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { NodeKey: "3", DetailUpdateTime: mockClock.Now().Add(-1 * time.Hour), SeenTime: mockClock.Now().Add(-1 * time.Hour), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{CreatedAt: mockClock.Now()}, + }, }) assert.Nil(t, err) @@ -548,14 +558,18 @@ func testGenerateHostStatusStatistics(t *testing.T, ds kolide.Datastore) { NodeKey: "4", DetailUpdateTime: mockClock.Now().Add(-35 * (24 * time.Hour)), SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{CreatedAt: mockClock.Now()}, + }, }) assert.Nil(t, err) - online, offline, mia, err = ds.GenerateHostStatusStatistics(mockClock.Now()) + online, offline, mia, new, err = ds.GenerateHostStatusStatistics(mockClock.Now()) assert.Nil(t, err) assert.Equal(t, uint(2), online) assert.Equal(t, uint(1), offline) assert.Equal(t, uint(1), mia) + assert.Equal(t, uint(4), new) } func testMarkHostSeen(t *testing.T, ds kolide.Datastore) { diff --git a/server/datastore/inmem/hosts.go b/server/datastore/inmem/hosts.go index a775168c61..28fde68cb0 100644 --- a/server/datastore/inmem/hosts.go +++ b/server/datastore/inmem/hosts.go @@ -114,11 +114,15 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return hosts, nil } -func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error) { +func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia, new uint, err error) { d.mtx.Lock() defer d.mtx.Unlock() for _, host := range d.hosts { + if host.IsNew(now) { + new++ + } + status := host.Status(now) switch status { case kolide.StatusMIA: @@ -130,7 +134,7 @@ func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline } } - return online, offline, mia, nil + return online, offline, mia, new, nil } func (d *Datastore) EnrollHost(osQueryHostID string, nodeKeySize int) (*kolide.Host, error) { diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index b78afbb742..0dcadfd514 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -283,7 +283,7 @@ func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return hosts, nil } -func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, e error) { +func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia, new uint, e error) { sqlStatement := ` SELECT ( SELECT count(id) @@ -300,7 +300,12 @@ func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline SELECT count(id) FROM hosts WHERE DATE_ADD(seen_time, INTERVAL 30 MINUTE) > ? - ) AS online + ) AS online, + ( + SELECT count(id) + FROM hosts + WHERE DATE_ADD(created_at, INTERVAL 1 DAY) >= ? + ) AS new FROM hosts LIMIT 1; ` @@ -309,8 +314,9 @@ func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline MIA uint `db:"mia"` Offline uint `db:"offline"` Online uint `db:"online"` + New uint `db:"new"` }{} - err := d.db.Get(&counts, sqlStatement, now, now, now, now) + err := d.db.Get(&counts, sqlStatement, now, now, now, now, now) if err != nil && err != sql.ErrNoRows { e = errors.Wrap(err, "generating host statistics") return @@ -319,7 +325,8 @@ func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline mia = counts.MIA offline = counts.Offline online = counts.Online - return online, offline, mia, nil + new = counts.New + return online, offline, mia, new, nil } // Optimized network interface fetch for sets of hosts. Instead of looping diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index eda3755c24..f308a95e53 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -9,18 +9,26 @@ import ( ) const ( - // StatusOnline host is active - StatusOnline string = "online" - // StatusOffline no communication with host for OfflineDuration - StatusOffline string = "offline" - // StatusMIA no communition with host for MIADuration - StatusMIA string = "mia" + // StatusOnline host is active. + StatusOnline = "online" + + // StatusOffline no communication with host for OfflineDuration. + StatusOffline = "offline" + + // StatusMIA no communication with host for MIADuration. + StatusMIA = "mia" + + // NewDuration if a host has been created within this time period it's + // considered new. + NewDuration = 24 * time.Hour + // OfflineDuration if a host hasn't been in communition for this - // period it is considered offline - OfflineDuration time.Duration = 30 * time.Minute + // period it is considered offline. + OfflineDuration = 30 * time.Minute + // OfflineDuration if a host hasn't been in communition for this - // period it is considered MIA - MIADuration time.Duration = 30 * 24 * time.Hour + // period it is considered MIA. + MIADuration = 30 * 24 * time.Hour ) type HostStore interface { @@ -32,7 +40,7 @@ type HostStore interface { EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error) AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error - GenerateHostStatusStatistics(now time.Time) (online, offline, mia uint, err error) + GenerateHostStatusStatistics(now time.Time) (online, offline, mia, new uint, err error) SearchHosts(query string, omit ...uint) ([]*Host, error) // DistributedQueriesForHost retrieves the distributed queries that the // given host should run. The result map is a mapping from campaign ID @@ -92,6 +100,7 @@ type HostSummary struct { OnlineCount uint `json:"online_count"` OfflineCount uint `json:"offline_count"` MIACount uint `json:"mia_count"` + NewCount uint `json:"new_count"` } // ResetPrimaryNetwork will determine if the PrimaryNetworkInterfaceID @@ -148,3 +157,12 @@ func (h *Host) Status(now time.Time) string { return StatusOnline } } + +func (h *Host) IsNew(now time.Time) bool { + withDuration := h.CreatedAt.Add(NewDuration) + if withDuration.After(now) || + withDuration.Equal(now) { + return true + } + return false +} diff --git a/server/kolide/hosts_test.go b/server/kolide/hosts_test.go index 1dbf648bb9..1058d219f5 100644 --- a/server/kolide/hosts_test.go +++ b/server/kolide/hosts_test.go @@ -61,3 +61,15 @@ func TestHostStatus(t *testing.T) { host.SeenTime = mockClock.Now().Add(-35 * (24 * time.Hour)) // 35 days assert.Equal(t, StatusMIA, host.Status(mockClock.Now())) } + +func TestHostIsNew(t *testing.T) { + mockClock := clock.NewMockClock() + + host := Host{} + + host.CreatedAt = mockClock.Now().AddDate(0, 0, -1) + assert.True(t, host.IsNew(mockClock.Now())) + + host.CreatedAt = mockClock.Now().AddDate(0, 0, -2) + assert.False(t, host.IsNew(mockClock.Now())) +} diff --git a/server/service/service_hosts.go b/server/service/service_hosts.go index b27acc1329..b048af6217 100644 --- a/server/service/service_hosts.go +++ b/server/service/service_hosts.go @@ -14,7 +14,7 @@ func (svc service) GetHost(ctx context.Context, id uint) (*kolide.Host, error) { } func (svc service) GetHostSummary(ctx context.Context) (*kolide.HostSummary, error) { - online, offline, mia, err := svc.ds.GenerateHostStatusStatistics(svc.clock.Now()) + online, offline, mia, new, err := svc.ds.GenerateHostStatusStatistics(svc.clock.Now()) if err != nil { return nil, err } @@ -22,6 +22,7 @@ func (svc service) GetHostSummary(ctx context.Context) (*kolide.HostSummary, err OnlineCount: online, OfflineCount: offline, MIACount: mia, + NewCount: new, }, nil }