mirror of
https://github.com/fleetdm/fleet
synced 2026-05-23 08:58:41 +00:00
Add COALESCEs on all host_seen_times JOINs (#3147)
* Add COALESCEs on all host_seen_times JOINs * Use tx instead of d.writer * Add unit tests * Fix compile test
This commit is contained in:
Lucas Manuel Rodriguez
GitHub
parent
4ef9cfdf63
commit
e64a88d8b1
12 changed files with 263 additions and 95 deletions
1
changes/issue-3095-fix-seen-times
Normal file
1
changes/issue-3095-fix-seen-times
Normal file
|
|
@ -0,0 +1 @@
|
|||
* Fix scan issue with missing (or NULL) `host_seen_times.seen_time` on some hosts.
|
||||
|
|
@ -407,7 +407,7 @@ func (d *Datastore) Host(ctx context.Context, id uint, skipLoadingExtras bool) (
|
|||
sqlStatement := fmt.Sprintf(`
|
||||
SELECT
|
||||
h.*,
|
||||
hst.seen_time,
|
||||
COALESCE(hst.seen_time, h.created_at) AS seen_time,
|
||||
t.name AS team_name,
|
||||
(SELECT additional FROM host_additional WHERE host_id = h.id) AS additional
|
||||
%s
|
||||
|
|
@ -448,7 +448,7 @@ func amountEnrolledHostsDB(db sqlx.Queryer) (int, error) {
|
|||
func (d *Datastore) ListHosts(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) {
|
||||
sql := `SELECT
|
||||
h.*,
|
||||
hst.seen_time,
|
||||
COALESCE(hst.seen_time, h.created_at) AS seen_time,
|
||||
t.name AS team_name
|
||||
`
|
||||
|
||||
|
|
@ -558,13 +558,13 @@ func filterHostsByStatus(sql string, opt fleet.HostListOptions, params []interfa
|
|||
sql += "AND DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ?"
|
||||
params = append(params, time.Now())
|
||||
case "online":
|
||||
sql += fmt.Sprintf("AND DATE_ADD(hst.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) > ?", fleet.OnlineIntervalBuffer)
|
||||
sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) > ?", fleet.OnlineIntervalBuffer)
|
||||
params = append(params, time.Now())
|
||||
case "offline":
|
||||
sql += fmt.Sprintf("AND DATE_ADD(hst.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ?", fleet.OnlineIntervalBuffer)
|
||||
sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ?", fleet.OnlineIntervalBuffer)
|
||||
params = append(params, time.Now(), time.Now())
|
||||
case "mia":
|
||||
sql += "AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ?"
|
||||
sql += "AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ?"
|
||||
params = append(params, time.Now())
|
||||
}
|
||||
return sql, params
|
||||
|
|
@ -610,9 +610,9 @@ func (d *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fle
|
|||
sqlStatement := fmt.Sprintf(`
|
||||
SELECT
|
||||
COUNT(*) total,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new
|
||||
FROM hosts h LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) WHERE %s
|
||||
LIMIT 1;
|
||||
|
|
@ -655,12 +655,11 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin
|
|||
err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
|
||||
zeroTime := time.Unix(0, 0).Add(24 * time.Hour)
|
||||
|
||||
var id int64
|
||||
var hostID int64
|
||||
err := sqlx.GetContext(ctx, tx, &host, `SELECT id, last_enrolled_at FROM hosts WHERE osquery_host_id = ?`, osqueryHostID)
|
||||
switch {
|
||||
case err != nil && !errors.Is(err, sql.ErrNoRows):
|
||||
return ctxerr.Wrap(ctx, err, "check existing")
|
||||
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
// Create new host record
|
||||
sqlInsert := `
|
||||
|
|
@ -677,14 +676,7 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin
|
|||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "insert host")
|
||||
}
|
||||
|
||||
id, _ = result.LastInsertId()
|
||||
|
||||
_, err = d.writer.ExecContext(ctx, `INSERT INTO host_seen_times (host_id, seen_time) VALUES (?,?)`, id, time.Now().UTC())
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "new host seen time")
|
||||
}
|
||||
|
||||
hostID, _ = result.LastInsertId()
|
||||
default:
|
||||
// Prevent hosts from enrolling too often with the same identifier.
|
||||
// Prior to adding this we saw many hosts (probably VMs) with the
|
||||
|
|
@ -692,7 +684,7 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin
|
|||
if cooldown > 0 && time.Since(host.LastEnrolledAt) < cooldown {
|
||||
return backoff.Permanent(ctxerr.Errorf(ctx, "host identified by %s enrolling too often", osqueryHostID))
|
||||
}
|
||||
id = int64(host.ID)
|
||||
hostID = int64(host.ID)
|
||||
// Update existing host record
|
||||
sqlUpdate := `
|
||||
UPDATE hosts
|
||||
|
|
@ -706,20 +698,24 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin
|
|||
return ctxerr.Wrap(ctx, err, "update host")
|
||||
}
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
INSERT INTO host_seen_times (host_id, seen_time) VALUES (?,?)
|
||||
ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`,
|
||||
hostID, time.Now().UTC())
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "new host seen time")
|
||||
}
|
||||
sqlSelect := `
|
||||
SELECT * FROM hosts WHERE id = ? LIMIT 1
|
||||
`
|
||||
err = sqlx.GetContext(ctx, tx, &host, sqlSelect, id)
|
||||
err = sqlx.GetContext(ctx, tx, &host, sqlSelect, hostID)
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "getting the host to return")
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, id)
|
||||
_, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, hostID)
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "insert new host into all hosts label")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -745,34 +741,25 @@ func (d *Datastore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee
|
|||
return host, nil
|
||||
}
|
||||
|
||||
func (d *Datastore) MarkHostSeen(ctx context.Context, host *fleet.Host, t time.Time) error {
|
||||
sqlStatement := `UPDATE host_seen_times SET seen_time = ? WHERE host_id=?`
|
||||
|
||||
_, err := d.writer.ExecContext(ctx, sqlStatement, t, host.ID)
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "marking host seen")
|
||||
}
|
||||
|
||||
host.UpdatedAt = t
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error {
|
||||
if len(hostIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
|
||||
query := `UPDATE host_seen_times SET seen_time = ? WHERE host_id IN (?)`
|
||||
query, args, err := sqlx.In(query, t, hostIDs)
|
||||
if err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "sqlx in")
|
||||
var insertArgs []interface{}
|
||||
for _, hostID := range hostIDs {
|
||||
insertArgs = append(insertArgs, hostID, t)
|
||||
}
|
||||
query = tx.Rebind(query)
|
||||
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
|
||||
insertValues := strings.TrimSuffix(strings.Repeat("(?, ?),", len(hostIDs)), ",")
|
||||
query := fmt.Sprintf(`
|
||||
INSERT INTO host_seen_times (host_id, seen_time) VALUES %s
|
||||
ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`,
|
||||
insertValues,
|
||||
)
|
||||
if _, err := tx.ExecContext(ctx, query, insertArgs...); err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "exec update")
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "MarkHostsSeen transaction")
|
||||
|
|
@ -789,7 +776,12 @@ func (d *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Ti
|
|||
// - An optional list of IDs to omit from the search.
|
||||
func (d *Datastore) SearchHosts(ctx context.Context, filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error) {
|
||||
var sqlb strings.Builder
|
||||
sqlb.WriteString("SELECT h.*, hst.seen_time FROM hosts h LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) WHERE")
|
||||
sqlb.WriteString(`SELECT
|
||||
h.*,
|
||||
COALESCE(hst.seen_time, h.created_at) AS seen_time
|
||||
FROM hosts h
|
||||
LEFT JOIN host_seen_times hst
|
||||
ON (h.id=hst.host_id) WHERE`)
|
||||
|
||||
var args []interface{}
|
||||
if len(query) > 0 {
|
||||
|
|
@ -816,7 +808,7 @@ func (d *Datastore) SearchHosts(ctx context.Context, filter fleet.TeamFilter, qu
|
|||
args = append(args, in)
|
||||
sqlb.WriteString(" id NOT IN (?) AND ")
|
||||
sqlb.WriteString(d.whereFilterHostsByTeams(filter, "h"))
|
||||
sqlb.WriteString(` ORDER BY hst.seen_time DESC LIMIT 10`)
|
||||
sqlb.WriteString(` ORDER BY COALESCE(hst.seen_time, h.created_at) DESC LIMIT 10`)
|
||||
|
||||
sql, args, err := sqlx.In(sqlb.String(), args...)
|
||||
if err != nil {
|
||||
|
|
@ -969,22 +961,25 @@ func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host)
|
|||
return nil
|
||||
}
|
||||
|
||||
func (d *Datastore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (int, int, error) {
|
||||
var totalCount, unseenCount int
|
||||
err := sqlx.GetContext(ctx, d.reader, &totalCount, "SELECT count(*) FROM hosts")
|
||||
if err != nil {
|
||||
return 0, 0, ctxerr.Wrap(ctx, err, "getting total host count")
|
||||
func (d *Datastore) TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error) {
|
||||
var counts struct {
|
||||
Total int `db:"total"`
|
||||
Unseen int `db:"unseen"`
|
||||
}
|
||||
|
||||
err = sqlx.GetContext(ctx, d.reader, &unseenCount,
|
||||
"SELECT count(*) FROM host_seen_times WHERE DATEDIFF(CURRENT_DATE, seen_time) >= ?",
|
||||
err = sqlx.GetContext(ctx, d.reader, &counts,
|
||||
`SELECT
|
||||
COUNT(*) as total,
|
||||
SUM(IF(DATEDIFF(CURRENT_DATE, COALESCE(hst.seen_time, h.created_at)) >= ?, 1, 0)) as unseen
|
||||
FROM hosts h
|
||||
LEFT JOIN host_seen_times hst
|
||||
ON h.id = hst.host_id`,
|
||||
daysCount,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, 0, ctxerr.Wrap(ctx, err, "getting unseen host count")
|
||||
return 0, 0, ctxerr.Wrap(ctx, err, "getting total and unseen host counts")
|
||||
}
|
||||
|
||||
return totalCount, unseenCount, nil
|
||||
return counts.Total, counts.Unseen, nil
|
||||
}
|
||||
|
||||
func (d *Datastore) DeleteHosts(ctx context.Context, ids []uint) error {
|
||||
|
|
@ -1050,7 +1045,10 @@ func (d *Datastore) CleanupExpiredHosts(ctx context.Context) error {
|
|||
|
||||
rows, err := d.writer.QueryContext(
|
||||
ctx,
|
||||
`SELECT host_id FROM host_seen_times WHERE seen_time < DATE_SUB(NOW(), INTERVAL ? DAY)`,
|
||||
`SELECT h.id FROM hosts h
|
||||
LEFT JOIN host_seen_times hst
|
||||
ON h.id = hst.host_id
|
||||
WHERE COALESCE(hst.seen_time, h.created_at) < DATE_SUB(NOW(), INTERVAL ? DAY)`,
|
||||
ac.HostExpirySettings.HostExpiryWindow,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ func TestHosts(t *testing.T) {
|
|||
{"HostsPackStatsMultipleHosts", testHostsPackStatsMultipleHosts},
|
||||
{"HostsPackStatsForPlatform", testHostsPackStatsForPlatform},
|
||||
{"HostsReadsLessRows", testHostsReadsLessRows},
|
||||
{"HostsNoSeenTime", testHostsNoSeenTime},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
|
@ -1157,7 +1158,7 @@ func testHostsMarkSeen(t *testing.T, ds *Datastore) {
|
|||
assert.WithinDuration(t, aDayAgo, h1Verify.SeenTime, time.Second)
|
||||
}
|
||||
|
||||
err = ds.MarkHostSeen(context.Background(), h1, anHourAgo)
|
||||
err = ds.MarkHostsSeen(context.Background(), []uint{h1.ID}, anHourAgo)
|
||||
assert.Nil(t, err)
|
||||
|
||||
{
|
||||
|
|
@ -2941,3 +2942,187 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) {
|
|||
}
|
||||
require.ElementsMatch(t, packStats2[0].QueryStats, zeroStats)
|
||||
}
|
||||
|
||||
// testHostsNoSeenTime tests all changes around the seen_time issue #3095.
|
||||
func testHostsNoSeenTime(t *testing.T, ds *Datastore) {
|
||||
h1, err := ds.NewHost(context.Background(), &fleet.Host{
|
||||
ID: 1,
|
||||
OsqueryHostID: "1",
|
||||
NodeKey: "1",
|
||||
Platform: "linux",
|
||||
Hostname: "host1",
|
||||
DetailUpdatedAt: time.Now(),
|
||||
LabelUpdatedAt: time.Now(),
|
||||
PolicyUpdatedAt: time.Now(),
|
||||
SeenTime: time.Now(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
removeHostSeenTimes := func(hostID uint) {
|
||||
result, err := ds.writer.Exec("DELETE FROM host_seen_times WHERE host_id = ?", hostID)
|
||||
require.NoError(t, err)
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, rowsAffected)
|
||||
}
|
||||
removeHostSeenTimes(h1.ID)
|
||||
|
||||
h1, err = ds.Host(context.Background(), h1.ID, true)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h1.CreatedAt, h1.SeenTime)
|
||||
|
||||
teamFilter := fleet.TeamFilter{User: test.UserAdmin}
|
||||
hosts, err := ds.ListHosts(context.Background(), teamFilter, fleet.HostListOptions{})
|
||||
require.NoError(t, err)
|
||||
hostsLen := len(hosts)
|
||||
require.Equal(t, hostsLen, 1)
|
||||
var foundHost *fleet.Host
|
||||
for _, host := range hosts {
|
||||
if host.ID == h1.ID {
|
||||
foundHost = host
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, foundHost)
|
||||
require.Equal(t, foundHost.CreatedAt, foundHost.SeenTime)
|
||||
hostCount, err := ds.CountHosts(context.Background(), teamFilter, fleet.HostListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, hostsLen, hostCount)
|
||||
|
||||
labelID := uint(1)
|
||||
l1 := &fleet.LabelSpec{
|
||||
ID: labelID,
|
||||
Name: "label foo",
|
||||
Query: "query1",
|
||||
}
|
||||
err = ds.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{l1})
|
||||
require.NoError(t, err)
|
||||
err = ds.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false)
|
||||
require.NoError(t, err)
|
||||
listHostsInLabelCheckCount(t, ds, fleet.TeamFilter{
|
||||
User: test.UserAdmin,
|
||||
}, labelID, fleet.HostListOptions{}, 1)
|
||||
|
||||
mockClock := clock.NewMockClock()
|
||||
summary, err := ds.GenerateHostStatusStatistics(context.Background(), teamFilter, mockClock.Now())
|
||||
assert.NoError(t, err)
|
||||
assert.Nil(t, summary.TeamID)
|
||||
assert.Equal(t, uint(1), summary.TotalsHostsCount)
|
||||
assert.Equal(t, uint(1), summary.OnlineCount)
|
||||
assert.Equal(t, uint(0), summary.OfflineCount)
|
||||
assert.Equal(t, uint(0), summary.MIACount)
|
||||
assert.Equal(t, uint(1), summary.NewCount)
|
||||
|
||||
var count []int
|
||||
err = ds.writer.Select(&count, "SELECT COUNT(*) FROM host_seen_times")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, count, 1)
|
||||
require.Zero(t, count[0])
|
||||
|
||||
// Enroll existing host.
|
||||
_, err = ds.EnrollHost(context.Background(), "1", "1", nil, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
var seenTime1 []time.Time
|
||||
err = ds.writer.Select(&seenTime1, "SELECT seen_time FROM host_seen_times WHERE host_id = ?", h1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, seenTime1, 1)
|
||||
require.NotZero(t, seenTime1[0])
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// Enroll again to trigger an update of host_seen_times.
|
||||
_, err = ds.EnrollHost(context.Background(), "1", "1", nil, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
var seenTime2 []time.Time
|
||||
err = ds.writer.Select(&seenTime2, "SELECT seen_time FROM host_seen_times WHERE host_id = ?", h1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, seenTime2, 1)
|
||||
require.NotZero(t, seenTime2[0])
|
||||
|
||||
require.True(t, seenTime2[0].After(seenTime1[0]), "%s vs. %s", seenTime1[0], seenTime2[0])
|
||||
|
||||
removeHostSeenTimes(h1.ID)
|
||||
|
||||
h2, err := ds.NewHost(context.Background(), &fleet.Host{
|
||||
ID: 2,
|
||||
OsqueryHostID: "2",
|
||||
NodeKey: "2",
|
||||
Platform: "windows",
|
||||
Hostname: "host2",
|
||||
DetailUpdatedAt: time.Now(),
|
||||
LabelUpdatedAt: time.Now(),
|
||||
PolicyUpdatedAt: time.Now(),
|
||||
SeenTime: time.Now(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t1 := time.Now().UTC()
|
||||
// h1 has no host_seen_times entry, h2 does.
|
||||
err = ds.MarkHostsSeen(context.Background(), []uint{h1.ID, h2.ID}, t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Reload hosts.
|
||||
h1, err = ds.Host(context.Background(), h1.ID, true)
|
||||
require.NoError(t, err)
|
||||
h2, err = ds.Host(context.Background(), h2.ID, true)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Equal doesn't work, it looks like a time.Time scanned from
|
||||
// the database is different from the original in some fields
|
||||
// (wall and ext).
|
||||
require.WithinDuration(t, t1, h1.SeenTime, time.Second)
|
||||
require.WithinDuration(t, t1, h2.SeenTime, time.Second)
|
||||
|
||||
removeHostSeenTimes(h1.ID)
|
||||
|
||||
foundHosts, err := ds.SearchHosts(context.Background(), teamFilter, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, foundHosts, 2)
|
||||
// SearchHosts orders by seen time.
|
||||
require.Equal(t, h2.ID, foundHosts[0].ID)
|
||||
require.WithinDuration(t, t1, foundHosts[0].SeenTime, time.Second)
|
||||
require.Equal(t, h1.ID, foundHosts[1].ID)
|
||||
require.Equal(t, foundHosts[1].SeenTime, foundHosts[1].CreatedAt)
|
||||
|
||||
total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, total, 2)
|
||||
require.Equal(t, unseen, 0)
|
||||
|
||||
h3, err := ds.NewHost(context.Background(), &fleet.Host{
|
||||
ID: 3,
|
||||
OsqueryHostID: "3",
|
||||
NodeKey: "3",
|
||||
Platform: "darwin",
|
||||
Hostname: "host3",
|
||||
DetailUpdatedAt: time.Now(),
|
||||
LabelUpdatedAt: time.Now(),
|
||||
PolicyUpdatedAt: time.Now(),
|
||||
SeenTime: time.Now(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
removeHostSeenTimes(h3.ID)
|
||||
|
||||
err = ds.CleanupExpiredHosts(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
hosts, err = ds.ListHosts(context.Background(), teamFilter, fleet.HostListOptions{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hosts, 3)
|
||||
|
||||
err = ds.RecordLabelQueryExecutions(context.Background(), h2, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false)
|
||||
require.NoError(t, err)
|
||||
err = ds.RecordLabelQueryExecutions(context.Background(), h3, map[uint]*bool{l1.ID: ptr.Bool(true)}, time.Now(), false)
|
||||
require.NoError(t, err)
|
||||
metrics, err := ds.CountHostsInTargets(context.Background(), teamFilter, fleet.HostTargets{
|
||||
LabelIDs: []uint{l1.ID},
|
||||
}, mockClock.Now())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint(3), metrics.TotalHosts)
|
||||
assert.Equal(t, uint(0), metrics.OfflineHosts)
|
||||
assert.Equal(t, uint(3), metrics.OnlineHosts)
|
||||
assert.Equal(t, uint(0), metrics.MissingInActionHosts)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -424,7 +424,10 @@ func (d *Datastore) ListLabelsForHost(ctx context.Context, hid uint) ([]*fleet.L
|
|||
// with fleet.Label referened by Label ID
|
||||
func (d *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilter, lid uint, opt fleet.HostListOptions) ([]*fleet.Host, error) {
|
||||
query := `
|
||||
SELECT h.*, hst.seen_time, (SELECT name FROM teams t WHERE t.id = h.team_id) AS team_name
|
||||
SELECT
|
||||
h.*,
|
||||
COALESCE(hst.seen_time, h.created_at) as seen_time,
|
||||
(SELECT name FROM teams t WHERE t.id = h.team_id) AS team_name
|
||||
FROM label_membership lm
|
||||
JOIN hosts h ON (lm.host_id = h.id)
|
||||
LEFT JOIN host_seen_times hst ON (h.id=hst.host_id)
|
||||
|
|
|
|||
|
|
@ -23,12 +23,12 @@ func (d *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamFi
|
|||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
COUNT(*) total,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(hst.seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(hst.seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new
|
||||
FROM hosts h
|
||||
JOIN host_seen_times hst ON (h.id=hst.host_id)
|
||||
LEFT JOIN host_seen_times hst ON (h.id=hst.host_id)
|
||||
WHERE (id IN (?) OR (id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id IN (?))) OR team_id IN (?)) AND %s
|
||||
`, fleet.OnlineIntervalBuffer, fleet.OnlineIntervalBuffer, d.whereFilterHostsByTeams(filter, "h"))
|
||||
|
||||
|
|
|
|||
|
|
@ -54,8 +54,8 @@ func testTargetsCountHosts(t *testing.T, ds *Datastore) {
|
|||
ConfigTLSRefresh: configTLSRefresh,
|
||||
TeamID: teamID,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, ds.MarkHostSeen(context.Background(), h, seenTime))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, seenTime))
|
||||
return h
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +228,7 @@ func testTargetsHostStatus(t *testing.T, ds *Datastore) {
|
|||
expectOffline := fleet.TargetMetrics{TotalHosts: 1, OfflineHosts: 1}
|
||||
expectMIA := fleet.TargetMetrics{TotalHosts: 1, MissingInActionHosts: 1}
|
||||
|
||||
var testCases = []struct {
|
||||
testCases := []struct {
|
||||
seenTime time.Time
|
||||
distributedInterval uint
|
||||
configTLSRefresh uint
|
||||
|
|
@ -257,14 +257,14 @@ func testTargetsHostStatus(t *testing.T, ds *Datastore) {
|
|||
// Save interval values
|
||||
h.DistributedInterval = tt.distributedInterval
|
||||
h.ConfigTLSRefresh = tt.configTLSRefresh
|
||||
require.Nil(t, ds.SaveHost(context.Background(), h))
|
||||
require.NoError(t, ds.SaveHost(context.Background(), h))
|
||||
|
||||
// Mark seen
|
||||
require.Nil(t, ds.MarkHostSeen(context.Background(), h, tt.seenTime))
|
||||
require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, tt.seenTime))
|
||||
|
||||
// Verify status
|
||||
metrics, err := ds.CountHostsInTargets(context.Background(), filter, fleet.HostTargets{HostIDs: []uint{h.ID}}, mockClock.Now())
|
||||
require.Nil(t, err)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.metrics, metrics)
|
||||
})
|
||||
}
|
||||
|
|
@ -382,8 +382,8 @@ func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) {
|
|||
ConfigTLSRefresh: configTLSRefresh,
|
||||
TeamID: teamID,
|
||||
})
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, ds.MarkHostSeen(context.Background(), h, seenTime))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, seenTime))
|
||||
return h
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -188,7 +188,6 @@ type Datastore interface {
|
|||
// "additional" information as this is not typically necessary for the operations performed by the osquery
|
||||
// endpoints.
|
||||
AuthenticateHost(ctx context.Context, nodeKey string) (*Host, error)
|
||||
MarkHostSeen(ctx context.Context, host *Host, t time.Time) error
|
||||
MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error
|
||||
SearchHosts(ctx context.Context, filter TeamFilter, query string, omit ...uint) ([]*Host, error)
|
||||
// CleanupIncomingHosts deletes hosts that have enrolled but never updated their status details. This clears dead
|
||||
|
|
@ -206,7 +205,7 @@ type Datastore interface {
|
|||
// AddHostsToTeam adds hosts to an existing team, clearing their team settings if teamID is nil.
|
||||
AddHostsToTeam(ctx context.Context, teamID *uint, hostIDs []uint) error
|
||||
|
||||
TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (int, int, error)
|
||||
TotalAndUnseenHostsSince(ctx context.Context, daysCount int) (total int, unseen int, err error)
|
||||
|
||||
DeleteHosts(ctx context.Context, ids []uint) error
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import (
|
|||
func TestHostStatus(t *testing.T) {
|
||||
mockClock := clock.NewMockClock()
|
||||
|
||||
var testCases = []struct {
|
||||
testCases := []struct {
|
||||
seenTime time.Time
|
||||
distributedInterval uint
|
||||
configTLSRefresh uint
|
||||
|
|
@ -47,7 +47,6 @@ func TestHostStatus(t *testing.T) {
|
|||
assert.Equal(t, tt.status, h.Status(mockClock.Now()))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestHostIsNew(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -153,8 +153,6 @@ type ListHostsFunc func(ctx context.Context, filter fleet.TeamFilter, opt fleet.
|
|||
|
||||
type AuthenticateHostFunc func(ctx context.Context, nodeKey string) (*fleet.Host, error)
|
||||
|
||||
type MarkHostSeenFunc func(ctx context.Context, host *fleet.Host, t time.Time) error
|
||||
|
||||
type MarkHostsSeenFunc func(ctx context.Context, hostIDs []uint, t time.Time) error
|
||||
|
||||
type SearchHostsFunc func(ctx context.Context, filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error)
|
||||
|
|
@ -537,9 +535,6 @@ type DataStore struct {
|
|||
AuthenticateHostFunc AuthenticateHostFunc
|
||||
AuthenticateHostFuncInvoked bool
|
||||
|
||||
MarkHostSeenFunc MarkHostSeenFunc
|
||||
MarkHostSeenFuncInvoked bool
|
||||
|
||||
MarkHostsSeenFunc MarkHostsSeenFunc
|
||||
MarkHostsSeenFuncInvoked bool
|
||||
|
||||
|
|
@ -1148,11 +1143,6 @@ func (s *DataStore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee
|
|||
return s.AuthenticateHostFunc(ctx, nodeKey)
|
||||
}
|
||||
|
||||
func (s *DataStore) MarkHostSeen(ctx context.Context, host *fleet.Host, t time.Time) error {
|
||||
s.MarkHostSeenFuncInvoked = true
|
||||
return s.MarkHostSeenFunc(ctx, host, t)
|
||||
}
|
||||
|
||||
func (s *DataStore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.Time) error {
|
||||
s.MarkHostsSeenFuncInvoked = true
|
||||
return s.MarkHostsSeenFunc(ctx, hostIDs, t)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
|
||||
"github.com/fleetdm/fleet/v4/server/contexts/viewer"
|
||||
|
|
@ -147,7 +146,7 @@ func TestGetNodeKey(t *testing.T) {
|
|||
NodeKey int
|
||||
}
|
||||
|
||||
var getNodeKeyTests = []struct {
|
||||
getNodeKeyTests := []struct {
|
||||
i interface{}
|
||||
expectKey string
|
||||
shouldErr bool
|
||||
|
|
@ -206,9 +205,6 @@ func TestAuthenticatedHost(t *testing.T) {
|
|||
|
||||
}
|
||||
}
|
||||
ds.MarkHostSeenFunc = func(ctx context.Context, host *fleet.Host, t time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
endpoint := authenticatedHost(
|
||||
svc,
|
||||
|
|
@ -221,7 +217,7 @@ func TestAuthenticatedHost(t *testing.T) {
|
|||
},
|
||||
)
|
||||
|
||||
var authenticatedHostTests = []struct {
|
||||
authenticatedHostTests := []struct {
|
||||
nodeKey string
|
||||
shouldErr bool
|
||||
}{
|
||||
|
|
@ -241,7 +237,7 @@ func TestAuthenticatedHost(t *testing.T) {
|
|||
|
||||
for _, tt := range authenticatedHostTests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
var r = struct{ NodeKey string }{NodeKey: tt.nodeKey}
|
||||
r := struct{ NodeKey string }{NodeKey: tt.nodeKey}
|
||||
_, err := endpoint(context.Background(), r)
|
||||
if tt.shouldErr {
|
||||
assert.IsType(t, osqueryError{}, err)
|
||||
|
|
|
|||
|
|
@ -1566,9 +1566,6 @@ func (e notFoundError) IsNotFound() bool {
|
|||
|
||||
func TestAuthenticationErrors(t *testing.T) {
|
||||
ms := new(mock.Store)
|
||||
ms.MarkHostSeenFunc = func(context.Context, *fleet.Host, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
ms.AuthenticateHostFunc = func(ctx context.Context, nodeKey string) (*fleet.Host, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t
|
|||
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, h.ID)
|
||||
require.NoError(t, ds.MarkHostSeen(context.Background(), h, now))
|
||||
require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now))
|
||||
|
||||
return h
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue