diff --git a/changes/issue-3095-fix-seen-times b/changes/issue-3095-fix-seen-times new file mode 100644 index 0000000000..53d15efefc --- /dev/null +++ b/changes/issue-3095-fix-seen-times @@ -0,0 +1 @@ +* Fix scan issue with missing (or NULL) `host_seen_times.seen_time` on some hosts. diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 4888179af3..cb47b950be 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -407,7 +407,7 @@ func (d *Datastore) Host(ctx context.Context, id uint, skipLoadingExtras bool) ( sqlStatement := fmt.Sprintf(` SELECT h.*, - hst.seen_time, + COALESCE(hst.seen_time, h.created_at) AS seen_time, t.name AS team_name, (SELECT additional FROM host_additional WHERE host_id = h.id) AS additional %s @@ -448,7 +448,7 @@ func amountEnrolledHostsDB(db sqlx.Queryer) (int, error) { func (d *Datastore) ListHosts(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) { sql := `SELECT h.*, - hst.seen_time, + COALESCE(hst.seen_time, h.created_at) AS seen_time, t.name AS team_name ` @@ -558,13 +558,13 @@ func filterHostsByStatus(sql string, opt fleet.HostListOptions, params []interfa sql += "AND DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ?" params = append(params, time.Now()) case "online": - sql += fmt.Sprintf("AND DATE_ADD(hst.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) > ?", fleet.OnlineIntervalBuffer) + sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) > ?", fleet.OnlineIntervalBuffer) params = append(params, time.Now()) case "offline": - sql += fmt.Sprintf("AND DATE_ADD(hst.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ?", fleet.OnlineIntervalBuffer) + sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ?", fleet.OnlineIntervalBuffer) params = append(params, time.Now(), time.Now()) case "mia": - sql += "AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ?" + sql += "AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ?" params = append(params, time.Now()) } return sql, params @@ -610,9 +610,9 @@ func (d *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fle sqlStatement := fmt.Sprintf(` SELECT COUNT(*) total, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new FROM hosts h LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) WHERE %s LIMIT 1; @@ -655,12 +655,11 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { zeroTime := time.Unix(0, 0).Add(24 * time.Hour) - var id int64 + var hostID int64 err := sqlx.GetContext(ctx, tx, &host, `SELECT id, last_enrolled_at FROM hosts WHERE osquery_host_id = ?`, osqueryHostID) switch { case err != nil && !errors.Is(err, sql.ErrNoRows): return ctxerr.Wrap(ctx, err, "check existing") - case errors.Is(err, sql.ErrNoRows): // Create new host record sqlInsert := ` @@ -677,14 +676,7 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin if err != nil { return ctxerr.Wrap(ctx, err, "insert host") } - - id, _ = result.LastInsertId() - - _, err = d.writer.ExecContext(ctx, `INSERT INTO host_seen_times (host_id, seen_time) VALUES (?,?)`, id, time.Now().UTC()) - if err != nil { - return ctxerr.Wrap(ctx, err, "new host seen time") - } - + hostID, _ = result.LastInsertId() default: // Prevent hosts from enrolling too often with the same identifier. // Prior to adding this we saw many hosts (probably VMs) with the @@ -692,7 +684,7 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin if cooldown > 0 && time.Since(host.LastEnrolledAt) < cooldown { return backoff.Permanent(ctxerr.Errorf(ctx, "host identified by %s enrolling too often", osqueryHostID)) } - id = int64(host.ID) + hostID = int64(host.ID) // Update existing host record sqlUpdate := ` UPDATE hosts @@ -706,20 +698,24 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin return ctxerr.Wrap(ctx, err, "update host") } } - + _, err = tx.ExecContext(ctx, ` + INSERT INTO host_seen_times (host_id, seen_time) VALUES (?,?) + ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + hostID, time.Now().UTC()) + if err != nil { + return ctxerr.Wrap(ctx, err, "new host seen time") + } sqlSelect := ` SELECT * FROM hosts WHERE id = ? LIMIT 1 ` - err = sqlx.GetContext(ctx, tx, &host, sqlSelect, id) + err = sqlx.GetContext(ctx, tx, &host, sqlSelect, hostID) if err != nil { return ctxerr.Wrap(ctx, err, "getting the host to return") } - - _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, id) + _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, hostID) if err != nil { return ctxerr.Wrap(ctx, err, "insert new host into all hosts label") } - return nil }) if err != nil { @@ -745,34 +741,25 @@ func (d *Datastore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee return host, nil } -func (d *Datastore) MarkHostSeen(ctx context.Context, host *fleet.Host, t time.Time) error { - sqlStatement := `UPDATE host_seen_times SET seen_time = ? WHERE host_id=?` - - _, err := d.writer.ExecContext(ctx, sqlStatement, t, host.ID) - if err != nil { - return ctxerr.Wrap(ctx, err, "marking host seen") - } - - host.UpdatedAt = t - return nil -} - func (d *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error { if len(hostIDs) == 0 { return nil } if err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - query := `UPDATE host_seen_times SET seen_time = ? WHERE host_id IN (?)` - query, args, err := sqlx.In(query, t, hostIDs) - if err != nil { - return ctxerr.Wrap(ctx, err, "sqlx in") + var insertArgs []interface{} + for _, hostID := range hostIDs { + insertArgs = append(insertArgs, hostID, t) } - query = tx.Rebind(query) - if _, err := tx.ExecContext(ctx, query, args...); err != nil { + insertValues := strings.TrimSuffix(strings.Repeat("(?, ?),", len(hostIDs)), ",") + query := fmt.Sprintf(` + INSERT INTO host_seen_times (host_id, seen_time) VALUES %s + ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + insertValues, + ) + if _, err := tx.ExecContext(ctx, query, insertArgs...); err != nil { return ctxerr.Wrap(ctx, err, "exec update") } - return nil }); err != nil { return ctxerr.Wrap(ctx, err, "MarkHostsSeen transaction") @@ -789,7 +776,12 @@ func (d *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Ti // - An optional list of IDs to omit from the search. func (d *Datastore) SearchHosts(ctx context.Context, filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error) { var sqlb strings.Builder - sqlb.WriteString("SELECT h.*, hst.seen_time FROM hosts h LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) WHERE") + sqlb.WriteString(`SELECT + h.*, + COALESCE(hst.seen_time, h.created_at) AS seen_time + FROM hosts h + LEFT JOIN host_seen_times hst + ON (h.id=hst.host_id) WHERE`) var args []interface{} if len(query) > 0 { @@ -816,7 +808,7 @@ func (d *Datastore) SearchHosts(ctx context.Context, filter fleet.TeamFilter, qu args = append(args, in) sqlb.WriteString(" id NOT IN (?) AND ") sqlb.WriteString(d.whereFilterHostsByTeams(filter, "h")) - sqlb.WriteString(` ORDER BY hst.seen_time DESC LIMIT 10`) + sqlb.WriteString(` ORDER BY COALESCE(hst.seen_time, h.created_at) DESC LIMIT 10`) sql, args, err := sqlx.In(sqlb.String(), args...) if err != nil { @@ -969,22 +961,25 @@ func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host) return nil } -func (d *Datastore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (int, int, error) { - var totalCount, unseenCount int - err := sqlx.GetContext(ctx, d.reader, &totalCount, "SELECT count(*) FROM hosts") - if err != nil { - return 0, 0, ctxerr.Wrap(ctx, err, "getting total host count") +func (d *Datastore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) { + var counts struct { + Total int `db:"total"` + Unseen int `db:"unseen"` } - - err = sqlx.GetContext(ctx, d.reader, &unseenCount, - "SELECT count(*) FROM host_seen_times WHERE DATEDIFF(CURRENT_DATE, seen_time) >= ?", + err = sqlx.GetContext(ctx, d.reader, &counts, + `SELECT + COUNT(*) as total, + SUM(IF(DATEDIFF(CURRENT_DATE, COALESCE(hst.seen_time, h.created_at)) >= ?, 1, 0)) as unseen + FROM hosts h + LEFT JOIN host_seen_times hst + ON h.id = hst.host_id`, daysCount, ) if err != nil { - return 0, 0, ctxerr.Wrap(ctx, err, "getting unseen host count") + return 0, 0, ctxerr.Wrap(ctx, err, "getting total and unseen host counts") } - return totalCount, unseenCount, nil + return counts.Total, counts.Unseen, nil } func (d *Datastore) DeleteHosts(ctx context.Context, ids []uint) error { @@ -1050,7 +1045,10 @@ func (d *Datastore) CleanupExpiredHosts(ctx context.Context) error { rows, err := d.writer.QueryContext( ctx, - `SELECT host_id FROM host_seen_times WHERE seen_time < DATE_SUB(NOW(), INTERVAL ? DAY)`, + `SELECT h.id FROM hosts h + LEFT JOIN host_seen_times hst + ON h.id = hst.host_id + WHERE COALESCE(hst.seen_time, h.created_at) < DATE_SUB(NOW(), INTERVAL ? DAY)`, ac.HostExpirySettings.HostExpiryWindow, ) if err != nil { diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index e7f005f145..b74a9fb2bf 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -95,6 +95,7 @@ func TestHosts(t *testing.T) { {"HostsPackStatsMultipleHosts", testHostsPackStatsMultipleHosts}, {"HostsPackStatsForPlatform", testHostsPackStatsForPlatform}, {"HostsReadsLessRows", testHostsReadsLessRows}, + {"HostsNoSeenTime", testHostsNoSeenTime}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -1157,7 +1158,7 @@ func testHostsMarkSeen(t *testing.T, ds *Datastore) { assert.WithinDuration(t, aDayAgo, h1Verify.SeenTime, time.Second) } - err = ds.MarkHostSeen(context.Background(), h1, anHourAgo) + err = ds.MarkHostsSeen(context.Background(), []uint{h1.ID}, anHourAgo) assert.Nil(t, err) { @@ -2941,3 +2942,187 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { } require.ElementsMatch(t, packStats2[0].QueryStats, zeroStats) } + +// testHostsNoSeenTime tests all changes around the seen_time issue #3095. +func testHostsNoSeenTime(t *testing.T, ds *Datastore) { + h1, err := ds.NewHost(context.Background(), &fleet.Host{ + ID: 1, + OsqueryHostID: "1", + NodeKey: "1", + Platform: "linux", + Hostname: "host1", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + require.NoError(t, err) + + removeHostSeenTimes := func(hostID uint) { + result, err := ds.writer.Exec("DELETE FROM host_seen_times WHERE host_id = ?", hostID) + require.NoError(t, err) + rowsAffected, err := result.RowsAffected() + require.NoError(t, err) + require.EqualValues(t, 1, rowsAffected) + } + removeHostSeenTimes(h1.ID) + + h1, err = ds.Host(context.Background(), h1.ID, true) + require.NoError(t, err) + require.Equal(t, h1.CreatedAt, h1.SeenTime) + + teamFilter := fleet.TeamFilter{User: test.UserAdmin} + hosts, err := ds.ListHosts(context.Background(), teamFilter, fleet.HostListOptions{}) + require.NoError(t, err) + hostsLen := len(hosts) + require.Equal(t, hostsLen, 1) + var foundHost *fleet.Host + for _, host := range hosts { + if host.ID == h1.ID { + foundHost = host + break + } + } + require.NotNil(t, foundHost) + require.Equal(t, foundHost.CreatedAt, foundHost.SeenTime) + hostCount, err := ds.CountHosts(context.Background(), teamFilter, fleet.HostListOptions{}) + require.NoError(t, err) + require.Equal(t, hostsLen, hostCount) + + labelID := uint(1) + l1 := &fleet.LabelSpec{ + ID: labelID, + Name: "label foo", + Query: "query1", + } + err = ds.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{l1}) + require.NoError(t, err) + err = ds.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false) + require.NoError(t, err) + listHostsInLabelCheckCount(t, ds, fleet.TeamFilter{ + User: test.UserAdmin, + }, labelID, fleet.HostListOptions{}, 1) + + mockClock := clock.NewMockClock() + summary, err := ds.GenerateHostStatusStatistics(context.Background(), teamFilter, mockClock.Now()) + assert.NoError(t, err) + assert.Nil(t, summary.TeamID) + assert.Equal(t, uint(1), summary.TotalsHostsCount) + assert.Equal(t, uint(1), summary.OnlineCount) + assert.Equal(t, uint(0), summary.OfflineCount) + assert.Equal(t, uint(0), summary.MIACount) + assert.Equal(t, uint(1), summary.NewCount) + + var count []int + err = ds.writer.Select(&count, "SELECT COUNT(*) FROM host_seen_times") + require.NoError(t, err) + require.Len(t, count, 1) + require.Zero(t, count[0]) + + // Enroll existing host. + _, err = ds.EnrollHost(context.Background(), "1", "1", nil, 0) + require.NoError(t, err) + + var seenTime1 []time.Time + err = ds.writer.Select(&seenTime1, "SELECT seen_time FROM host_seen_times WHERE host_id = ?", h1.ID) + require.NoError(t, err) + require.Len(t, seenTime1, 1) + require.NotZero(t, seenTime1[0]) + + time.Sleep(1 * time.Second) + + // Enroll again to trigger an update of host_seen_times. + _, err = ds.EnrollHost(context.Background(), "1", "1", nil, 0) + require.NoError(t, err) + + var seenTime2 []time.Time + err = ds.writer.Select(&seenTime2, "SELECT seen_time FROM host_seen_times WHERE host_id = ?", h1.ID) + require.NoError(t, err) + require.Len(t, seenTime2, 1) + require.NotZero(t, seenTime2[0]) + + require.True(t, seenTime2[0].After(seenTime1[0]), "%s vs. %s", seenTime1[0], seenTime2[0]) + + removeHostSeenTimes(h1.ID) + + h2, err := ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: "2", + NodeKey: "2", + Platform: "windows", + Hostname: "host2", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + require.NoError(t, err) + + t1 := time.Now().UTC() + // h1 has no host_seen_times entry, h2 does. + err = ds.MarkHostsSeen(context.Background(), []uint{h1.ID, h2.ID}, t1) + require.NoError(t, err) + + // Reload hosts. + h1, err = ds.Host(context.Background(), h1.ID, true) + require.NoError(t, err) + h2, err = ds.Host(context.Background(), h2.ID, true) + require.NoError(t, err) + + // Equal doesn't work, it looks like a time.Time scanned from + // the database is different from the original in some fields + // (wall and ext). + require.WithinDuration(t, t1, h1.SeenTime, time.Second) + require.WithinDuration(t, t1, h2.SeenTime, time.Second) + + removeHostSeenTimes(h1.ID) + + foundHosts, err := ds.SearchHosts(context.Background(), teamFilter, "") + require.NoError(t, err) + require.Len(t, foundHosts, 2) + // SearchHosts orders by seen time. + require.Equal(t, h2.ID, foundHosts[0].ID) + require.WithinDuration(t, t1, foundHosts[0].SeenTime, time.Second) + require.Equal(t, h1.ID, foundHosts[1].ID) + require.Equal(t, foundHosts[1].SeenTime, foundHosts[1].CreatedAt) + + total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) + require.NoError(t, err) + require.Equal(t, total, 2) + require.Equal(t, unseen, 0) + + h3, err := ds.NewHost(context.Background(), &fleet.Host{ + ID: 3, + OsqueryHostID: "3", + NodeKey: "3", + Platform: "darwin", + Hostname: "host3", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + require.NoError(t, err) + + removeHostSeenTimes(h3.ID) + + err = ds.CleanupExpiredHosts(context.Background()) + require.NoError(t, err) + + hosts, err = ds.ListHosts(context.Background(), teamFilter, fleet.HostListOptions{}) + require.NoError(t, err) + require.Len(t, hosts, 3) + + err = ds.RecordLabelQueryExecutions(context.Background(), h2, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false) + require.NoError(t, err) + err = ds.RecordLabelQueryExecutions(context.Background(), h3, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false) + require.NoError(t, err) + metrics, err := ds.CountHostsInTargets(context.Background(), teamFilter, fleet.HostTargets{ + LabelIDs: []uint{l1.ID}, + }, mockClock.Now()) + require.NoError(t, err) + assert.Equal(t, uint(3), metrics.TotalHosts) + assert.Equal(t, uint(0), metrics.OfflineHosts) + assert.Equal(t, uint(3), metrics.OnlineHosts) + assert.Equal(t, uint(0), metrics.MissingInActionHosts) +} diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 9ebd8077a6..cb602f3aa7 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -424,7 +424,10 @@ func (d *Datastore) ListLabelsForHost(ctx context.Context, hid uint) ([]*fleet.L // with fleet.Label referened by Label ID func (d *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilter, lid uint, opt fleet.HostListOptions) ([]*fleet.Host, error) { query := ` - SELECT h.*, hst.seen_time, (SELECT name FROM teams t WHERE t.id = h.team_id) AS team_name + SELECT + h.*, + COALESCE(hst.seen_time, h.created_at) as seen_time, + (SELECT name FROM teams t WHERE t.id = h.team_id) AS team_name FROM label_membership lm JOIN hosts h ON (lm.host_id = h.id) LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) diff --git a/server/datastore/mysql/targets.go b/server/datastore/mysql/targets.go index 870e3cb582..6feaeddc8a 100644 --- a/server/datastore/mysql/targets.go +++ b/server/datastore/mysql/targets.go @@ -23,12 +23,12 @@ func (d *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamFi sql := fmt.Sprintf(` SELECT COUNT(*) total, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline, - COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline, + COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new FROM hosts h - JOIN host_seen_times hst ON (h.id=hst.host_id) + LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) WHERE (id IN (?) OR (id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id IN (?))) OR team_id IN (?)) AND %s `, fleet.OnlineIntervalBuffer, fleet.OnlineIntervalBuffer, d.whereFilterHostsByTeams(filter, "h")) diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index 6bb42a3bfe..493b2530fb 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -54,8 +54,8 @@ func testTargetsCountHosts(t *testing.T, ds *Datastore) { ConfigTLSRefresh: configTLSRefresh, TeamID: teamID, }) - require.Nil(t, err) - require.Nil(t, ds.MarkHostSeen(context.Background(), h, seenTime)) + require.NoError(t, err) + require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, seenTime)) return h } @@ -228,7 +228,7 @@ func testTargetsHostStatus(t *testing.T, ds *Datastore) { expectOffline := fleet.TargetMetrics{TotalHosts: 1, OfflineHosts: 1} expectMIA := fleet.TargetMetrics{TotalHosts: 1, MissingInActionHosts: 1} - var testCases = []struct { + testCases := []struct { seenTime time.Time distributedInterval uint configTLSRefresh uint @@ -257,14 +257,14 @@ func testTargetsHostStatus(t *testing.T, ds *Datastore) { // Save interval values h.DistributedInterval = tt.distributedInterval h.ConfigTLSRefresh = tt.configTLSRefresh - require.Nil(t, ds.SaveHost(context.Background(), h)) + require.NoError(t, ds.SaveHost(context.Background(), h)) // Mark seen - require.Nil(t, ds.MarkHostSeen(context.Background(), h, tt.seenTime)) + require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, tt.seenTime)) // Verify status metrics, err := ds.CountHostsInTargets(context.Background(), filter, fleet.HostTargets{HostIDs: []uint{h.ID}}, mockClock.Now()) - require.Nil(t, err) + require.NoError(t, err) assert.Equal(t, tt.metrics, metrics) }) } @@ -382,8 +382,8 @@ func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { ConfigTLSRefresh: configTLSRefresh, TeamID: teamID, }) - require.Nil(t, err) - require.Nil(t, ds.MarkHostSeen(context.Background(), h, seenTime)) + require.NoError(t, err) + require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, seenTime)) return h } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 5e67c89e3c..ebb413e784 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -188,7 +188,6 @@ type Datastore interface { // "additional" information as this is not typically necessary for the operations performed by the osquery // endpoints. AuthenticateHost(ctx context.Context, nodeKey string) (*Host, error) - MarkHostSeen(ctx context.Context, host *Host, t time.Time) error MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error SearchHosts(ctx context.Context, filter TeamFilter, query string, omit ...uint) ([]*Host, error) // CleanupIncomingHosts deletes hosts that have enrolled but never updated their status details. This clears dead @@ -206,7 +205,7 @@ type Datastore interface { // AddHostsToTeam adds hosts to an existing team, clearing their team settings if teamID is nil. AddHostsToTeam(ctx context.Context, teamID *uint, hostIDs []uint) error - TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (int, int, error) + TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) DeleteHosts(ctx context.Context, ids []uint) error diff --git a/server/fleet/hosts_test.go b/server/fleet/hosts_test.go index 45865a5fdb..1d67519a94 100644 --- a/server/fleet/hosts_test.go +++ b/server/fleet/hosts_test.go @@ -11,7 +11,7 @@ import ( func TestHostStatus(t *testing.T) { mockClock := clock.NewMockClock() - var testCases = []struct { + testCases := []struct { seenTime time.Time distributedInterval uint configTLSRefresh uint @@ -47,7 +47,6 @@ func TestHostStatus(t *testing.T) { assert.Equal(t, tt.status, h.Status(mockClock.Now())) }) } - } func TestHostIsNew(t *testing.T) { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 4fa3292a8d..597bc545f2 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -153,8 +153,6 @@ type ListHostsFunc func(ctx context.Context, filter fleet.TeamFilter, opt fleet. type AuthenticateHostFunc func(ctx context.Context, nodeKey string) (*fleet.Host, error) -type MarkHostSeenFunc func(ctx context.Context, host *fleet.Host, t time.Time) error - type MarkHostsSeenFunc func(ctx context.Context, hostIDs []uint, t time.Time) error type SearchHostsFunc func(ctx context.Context, filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error) @@ -537,9 +535,6 @@ type DataStore struct { AuthenticateHostFunc AuthenticateHostFunc AuthenticateHostFuncInvoked bool - MarkHostSeenFunc MarkHostSeenFunc - MarkHostSeenFuncInvoked bool - MarkHostsSeenFunc MarkHostsSeenFunc MarkHostsSeenFuncInvoked bool @@ -1148,11 +1143,6 @@ func (s *DataStore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee return s.AuthenticateHostFunc(ctx, nodeKey) } -func (s *DataStore) MarkHostSeen(ctx context.Context, host *fleet.Host, t time.Time) error { - s.MarkHostSeenFuncInvoked = true - return s.MarkHostSeenFunc(ctx, host, t) -} - func (s *DataStore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error { s.MarkHostsSeenFuncInvoked = true return s.MarkHostsSeenFunc(ctx, hostIDs, t) diff --git a/server/service/endpoint_middleware_test.go b/server/service/endpoint_middleware_test.go index 5b4e3643d4..8304f713fd 100644 --- a/server/service/endpoint_middleware_test.go +++ b/server/service/endpoint_middleware_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "testing" - "time" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/contexts/viewer" @@ -147,7 +146,7 @@ func TestGetNodeKey(t *testing.T) { NodeKey int } - var getNodeKeyTests = []struct { + getNodeKeyTests := []struct { i interface{} expectKey string shouldErr bool @@ -206,9 +205,6 @@ func TestAuthenticatedHost(t *testing.T) { } } - ds.MarkHostSeenFunc = func(ctx context.Context, host *fleet.Host, t time.Time) error { - return nil - } endpoint := authenticatedHost( svc, @@ -221,7 +217,7 @@ func TestAuthenticatedHost(t *testing.T) { }, ) - var authenticatedHostTests = []struct { + authenticatedHostTests := []struct { nodeKey string shouldErr bool }{ @@ -241,7 +237,7 @@ func TestAuthenticatedHost(t *testing.T) { for _, tt := range authenticatedHostTests { t.Run("", func(t *testing.T) { - var r = struct{ NodeKey string }{NodeKey: tt.nodeKey} + r := struct{ NodeKey string }{NodeKey: tt.nodeKey} _, err := endpoint(context.Background(), r) if tt.shouldErr { assert.IsType(t, osqueryError{}, err) diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 5bd4f4a703..a2d449d7d0 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -1566,9 +1566,6 @@ func (e notFoundError) IsNotFound() bool { func TestAuthenticationErrors(t *testing.T) { ms := new(mock.Store) - ms.MarkHostSeenFunc = func(context.Context, *fleet.Host, time.Time) error { - return nil - } ms.AuthenticateHostFunc = func(ctx context.Context, nodeKey string) (*fleet.Host, error) { return nil, nil } diff --git a/server/test/new_objects.go b/server/test/new_objects.go index 6b70abcc2b..0057cc144a 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -114,7 +114,7 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t require.NoError(t, err) require.NotZero(t, h.ID) - require.NoError(t, ds.MarkHostSeen(context.Background(), h, now)) + require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now)) return h }