mirror of
https://github.com/fleetdm/fleet
synced 2026-05-23 00:49:03 +00:00
15380 extend hosts api (#15421)
This commit is contained in:
Tim Lee
GitHub
parent
5c12b0a6ae
commit
f1acd30bcf
5 changed files with 263 additions and 78 deletions
|
|
@ -372,50 +372,54 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext,
|
|||
if teamID != nil {
|
||||
teamID_ = *teamID
|
||||
}
|
||||
ds := dialect.From(goqu.I("queries").As("q")).Select(
|
||||
goqu.I("q.id"),
|
||||
goqu.I("q.name"),
|
||||
goqu.I("q.description"),
|
||||
goqu.I("q.team_id"),
|
||||
goqu.I("q.schedule_interval").As("schedule_interval"),
|
||||
goqu.COALESCE(goqu.I("sqs.average_memory"), 0).As("average_memory"),
|
||||
goqu.COALESCE(goqu.I("sqs.denylisted"), false).As("denylisted"),
|
||||
goqu.COALESCE(goqu.I("sqs.executions"), 0).As("executions"),
|
||||
goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("timestamp(?)", pastDate)).As("last_executed"),
|
||||
goqu.COALESCE(goqu.I("sqs.output_size"), 0).As("output_size"),
|
||||
goqu.COALESCE(goqu.I("sqs.system_time"), 0).As("system_time"),
|
||||
goqu.COALESCE(goqu.I("sqs.user_time"), 0).As("user_time"),
|
||||
goqu.COALESCE(goqu.I("sqs.wall_time"), 0).As("wall_time"),
|
||||
).LeftJoin(
|
||||
dialect.From("scheduled_query_stats").As("sqs").Where(
|
||||
goqu.I("host_id").Eq(hid),
|
||||
),
|
||||
goqu.On(goqu.I("sqs.scheduled_query_id").Eq(goqu.I("q.id"))),
|
||||
).Where(
|
||||
goqu.And(
|
||||
goqu.Or(
|
||||
// sq.platform empty or NULL means the scheduled query is set to
|
||||
// run on all hosts.
|
||||
goqu.I("q.platform").Eq(""),
|
||||
goqu.I("q.platform").IsNull(),
|
||||
// scheduled_queries.platform can be a comma-separated list of
|
||||
// platforms, e.g. "darwin,windows".
|
||||
goqu.L("FIND_IN_SET(?, q.platform)", fleet.PlatformFromHost(hostPlatform)).Neq(0),
|
||||
),
|
||||
goqu.I("q.schedule_interval").Gt(0),
|
||||
goqu.I("q.automations_enabled").IsTrue(),
|
||||
goqu.Or(
|
||||
goqu.I("q.team_id").IsNull(),
|
||||
goqu.I("q.team_id").Eq(teamID_),
|
||||
),
|
||||
),
|
||||
)
|
||||
sql, args, err := ds.ToSQL()
|
||||
if err != nil {
|
||||
return nil, ctxerr.Wrap(ctx, err, "sql build")
|
||||
|
||||
sqlQuery := `
|
||||
SELECT
|
||||
q.id,
|
||||
q.name,
|
||||
q.description,
|
||||
q.team_id,
|
||||
q.schedule_interval AS schedule_interval,
|
||||
q.discard_data,
|
||||
q.automations_enabled,
|
||||
MAX(qr.last_fetched) as last_fetched,
|
||||
COALESCE(MAX(sqs.average_memory), 0) AS average_memory,
|
||||
COALESCE(MAX(sqs.denylisted), false) AS denylisted,
|
||||
COALESCE(MAX(sqs.executions), 0) AS executions,
|
||||
COALESCE(MAX(sqs.last_executed), TIMESTAMP(?)) AS last_executed,
|
||||
COALESCE(MAX(sqs.output_size), 0) AS output_size,
|
||||
COALESCE(MAX(sqs.system_time), 0) AS system_time,
|
||||
COALESCE(MAX(sqs.user_time), 0) AS user_time,
|
||||
COALESCE(MAX(sqs.wall_time), 0) AS wall_time
|
||||
FROM
|
||||
queries q
|
||||
LEFT JOIN scheduled_query_stats sqs ON (q.id = sqs.scheduled_query_id AND sqs.host_id = ?)
|
||||
LEFT JOIN query_results qr ON (q.id = qr.query_id AND qr.host_id = ?)
|
||||
WHERE
|
||||
(q.platform = '' OR q.platform IS NULL OR FIND_IN_SET(?, q.platform) != 0)
|
||||
AND q.schedule_interval > 0
|
||||
AND (q.automations_enabled IS TRUE OR (q.discard_data IS FALSE AND q.logging_type = ?))
|
||||
AND (q.team_id IS NULL OR q.team_id = ?)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM query_results
|
||||
WHERE query_results.query_id = q.id
|
||||
AND query_results.host_id = ?
|
||||
)
|
||||
GROUP BY q.id
|
||||
`
|
||||
|
||||
args := []interface{}{
|
||||
pastDate,
|
||||
hid,
|
||||
hid,
|
||||
fleet.PlatformFromHost(hostPlatform),
|
||||
fleet.LoggingSnapshot,
|
||||
teamID_,
|
||||
hid,
|
||||
}
|
||||
|
||||
var stats []fleet.QueryStats
|
||||
if err := sqlx.SelectContext(ctx, db, &stats, sql, args...); err != nil {
|
||||
if err := sqlx.SelectContext(ctx, db, &stats, sqlQuery, args...); err != nil {
|
||||
return nil, ctxerr.Wrap(ctx, err, "load query stats")
|
||||
}
|
||||
return stats, nil
|
||||
|
|
@ -690,6 +694,9 @@ func queryStatsToScheduledQueryStats(queriesStats []fleet.QueryStats, packName s
|
|||
Denylisted: queryStats.Denylisted,
|
||||
Executions: queryStats.Executions,
|
||||
Interval: queryStats.Interval,
|
||||
DiscardData: queryStats.DiscardData,
|
||||
AutomationsEnabled: queryStats.AutomationsEnabled,
|
||||
LastFetched: queryStats.LastFetched,
|
||||
LastExecuted: queryStats.LastExecuted,
|
||||
OutputSize: queryStats.OutputSize,
|
||||
SystemTime: queryStats.SystemTime,
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ func TestHosts(t *testing.T) {
|
|||
{"HostsListByDiskEncryptionStatus", testHostsListMacOSSettingsDiskEncryptionStatus},
|
||||
{"HostsListFailingPolicies", printReadsInTest(testHostsListFailingPolicies)},
|
||||
{"HostsExpiration", testHostsExpiration},
|
||||
{"HostsIncludesScheduledQueriesInPackStats", testHostsIncludesScheduledQueriesInPackStats},
|
||||
{"HostsAllPackStats", testHostsAllPackStats},
|
||||
{"HostsPackStatsMultipleHosts", testHostsPackStatsMultipleHosts},
|
||||
{"HostsPackStatsForPlatform", testHostsPackStatsForPlatform},
|
||||
|
|
@ -618,17 +619,20 @@ func testHostsWithTeamPackStats(t *testing.T, ds *Datastore) {
|
|||
PackName: pack1.Name,
|
||||
ScheduledQueryName: squery1.Name,
|
||||
|
||||
QueryName: query1.Name,
|
||||
PackID: pack1.ID,
|
||||
AverageMemory: 8000,
|
||||
Denylisted: false,
|
||||
Executions: 164,
|
||||
Interval: 30,
|
||||
LastExecuted: time.Unix(1620325191, 0).UTC(),
|
||||
OutputSize: 1337,
|
||||
SystemTime: 150,
|
||||
UserTime: 180,
|
||||
WallTime: 0,
|
||||
QueryName: query1.Name,
|
||||
PackID: pack1.ID,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: false,
|
||||
LastFetched: nil,
|
||||
AverageMemory: 8000,
|
||||
Denylisted: false,
|
||||
Executions: 164,
|
||||
Interval: 30,
|
||||
LastExecuted: time.Unix(1620325191, 0).UTC(),
|
||||
OutputSize: 1337,
|
||||
SystemTime: 150,
|
||||
UserTime: 180,
|
||||
WallTime: 0,
|
||||
},
|
||||
}
|
||||
stats2 := []fleet.ScheduledQueryStats{
|
||||
|
|
@ -636,17 +640,20 @@ func testHostsWithTeamPackStats(t *testing.T, ds *Datastore) {
|
|||
PackName: fmt.Sprintf("team-%d", team.ID),
|
||||
ScheduledQueryName: tpQuery.Name,
|
||||
|
||||
QueryName: tpQuery.Name,
|
||||
PackID: 0, // pack_id will be 0 for stats of queries not in packs.
|
||||
AverageMemory: 8000,
|
||||
Denylisted: false,
|
||||
Executions: 164,
|
||||
Interval: 30,
|
||||
LastExecuted: time.Unix(1620325191, 0).UTC(),
|
||||
OutputSize: 1337,
|
||||
SystemTime: 150,
|
||||
UserTime: 180,
|
||||
WallTime: 0,
|
||||
QueryName: tpQuery.Name,
|
||||
PackID: 0, // pack_id will be 0 for stats of queries not in packs.
|
||||
LastFetched: nil,
|
||||
DiscardData: tpQuery.DiscardData,
|
||||
AutomationsEnabled: tpQuery.AutomationsEnabled,
|
||||
AverageMemory: 8000,
|
||||
Denylisted: false,
|
||||
Executions: 164,
|
||||
Interval: 30,
|
||||
LastExecuted: time.Unix(1620325191, 0).UTC(),
|
||||
OutputSize: 1337,
|
||||
SystemTime: 150,
|
||||
UserTime: 180,
|
||||
WallTime: 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -3785,6 +3792,171 @@ func testHostsExpiration(t *testing.T, ds *Datastore) {
|
|||
require.Len(t, hosts, 5)
|
||||
}
|
||||
|
||||
func testHostsIncludesScheduledQueriesInPackStats(t *testing.T, ds *Datastore) {
|
||||
host, err := ds.NewHost(context.Background(), &fleet.Host{
|
||||
DetailUpdatedAt: time.Now(),
|
||||
LabelUpdatedAt: time.Now(),
|
||||
PolicyUpdatedAt: time.Now(),
|
||||
SeenTime: time.Now(),
|
||||
NodeKey: ptr.String("1"),
|
||||
UUID: "1",
|
||||
Hostname: "foo.local",
|
||||
PrimaryIP: "192.168.1.1",
|
||||
PrimaryMac: "30-65-EC-6F-C4-58",
|
||||
Platform: "darwin",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, host)
|
||||
|
||||
team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"})
|
||||
require.NoError(t, err)
|
||||
err = ds.AddHostsToTeam(context.Background(), &team.ID, []uint{host.ID})
|
||||
require.NoError(t, err)
|
||||
|
||||
query1 := &fleet.Query{
|
||||
Name: "Only Logged in Query Report",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: nil,
|
||||
Interval: 60,
|
||||
Logging: fleet.LoggingSnapshot,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: false,
|
||||
}
|
||||
|
||||
_, err = ds.NewQuery(context.Background(), query1)
|
||||
require.NoError(t, err)
|
||||
|
||||
query2 := &fleet.Query{
|
||||
Name: "Logged In Report and Log Destination",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: nil,
|
||||
Interval: 60,
|
||||
Logging: fleet.LoggingSnapshot,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: true,
|
||||
}
|
||||
_, err = ds.NewQuery(context.Background(), query2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// This query should not be included in the pack stats
|
||||
query3 := &fleet.Query{
|
||||
Name: "Not LoggingSnapshot",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: nil,
|
||||
Interval: 60,
|
||||
Logging: fleet.LoggingDifferential,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: false, // automations not on
|
||||
}
|
||||
_, err = ds.NewQuery(context.Background(), query3)
|
||||
require.NoError(t, err)
|
||||
|
||||
// This query should not be included in the pack stats
|
||||
query4 := &fleet.Query{
|
||||
Name: "Query Report No Interval",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: nil,
|
||||
Interval: 0,
|
||||
Logging: fleet.LoggingSnapshot,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: false,
|
||||
}
|
||||
_, err = ds.NewQuery(context.Background(), query4)
|
||||
require.NoError(t, err)
|
||||
|
||||
// this query should not be included in the pack stats
|
||||
query5 := &fleet.Query{
|
||||
Name: "Automations No Interval",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: nil,
|
||||
Interval: 0,
|
||||
Logging: fleet.LoggingSnapshot,
|
||||
DiscardData: true,
|
||||
AutomationsEnabled: true,
|
||||
}
|
||||
_, err = ds.NewQuery(context.Background(), query5)
|
||||
require.NoError(t, err)
|
||||
|
||||
query6 := &fleet.Query{
|
||||
Name: "Team Query",
|
||||
Query: "select * from time",
|
||||
AuthorID: nil,
|
||||
Platform: "darwin",
|
||||
Saved: true,
|
||||
TeamID: &team.ID,
|
||||
Interval: 60,
|
||||
Logging: fleet.LoggingSnapshot,
|
||||
DiscardData: false,
|
||||
AutomationsEnabled: true,
|
||||
}
|
||||
_, err = ds.NewQuery(context.Background(), query6)
|
||||
require.NoError(t, err)
|
||||
|
||||
hostResult, err := ds.Host(context.Background(), host.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
globalQueryStats := hostResult.PackStats[0].QueryStats
|
||||
require.NotNil(t, hostResult)
|
||||
require.Equal(t, 2, len(globalQueryStats))
|
||||
require.Equal(t, query1.Name, globalQueryStats[0].ScheduledQueryName)
|
||||
require.Equal(t, query2.Name, globalQueryStats[1].ScheduledQueryName)
|
||||
|
||||
teamQueryStats := hostResult.PackStats[1].QueryStats
|
||||
require.Equal(t, query6.Name, teamQueryStats[0].ScheduledQueryName)
|
||||
|
||||
// Queries with Query Results should be included in the pack stats
|
||||
// regardless of the query interval
|
||||
queryResultRow := []*fleet.ScheduledQueryResultRow{
|
||||
{
|
||||
QueryID: query4.ID, // no interval
|
||||
HostID: host.ID,
|
||||
Data: ptr.RawMessage(json.RawMessage(`{"foo": "bar"}`)),
|
||||
},
|
||||
{
|
||||
QueryID: query4.ID, // no interval
|
||||
HostID: host.ID,
|
||||
Data: ptr.RawMessage(json.RawMessage(`{"foo": "baz"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), queryResultRow)
|
||||
require.NoError(t, err)
|
||||
|
||||
hostResult, err = ds.Host(context.Background(), host.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, hostResult)
|
||||
|
||||
assertContains := func(stats []fleet.ScheduledQueryStats, name string) {
|
||||
t.Helper()
|
||||
for _, stat := range stats {
|
||||
if stat.ScheduledQueryName == name {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("expected to find %s in stats", name)
|
||||
}
|
||||
|
||||
globalQueryStats = hostResult.PackStats[0].QueryStats
|
||||
require.Equal(t, 3, len(globalQueryStats))
|
||||
assertContains(globalQueryStats, query1.Name)
|
||||
assertContains(globalQueryStats, query2.Name)
|
||||
assertContains(globalQueryStats, query4.Name) // no interval, but has a query result
|
||||
}
|
||||
|
||||
func testHostsAllPackStats(t *testing.T, ds *Datastore) {
|
||||
host, err := ds.NewHost(context.Background(), &fleet.Host{
|
||||
DetailUpdatedAt: time.Now(),
|
||||
|
|
|
|||
|
|
@ -414,12 +414,15 @@ type QueryStats struct {
|
|||
Denylisted bool `json:"denylisted" db:"denylisted"`
|
||||
Executions uint64 `json:"executions" db:"executions"`
|
||||
// Note schedule_interval is used for DB since "interval" is a reserved word in MySQL
|
||||
Interval int `json:"interval" db:"schedule_interval"`
|
||||
LastExecuted time.Time `json:"last_executed" db:"last_executed"`
|
||||
OutputSize uint64 `json:"output_size" db:"output_size"`
|
||||
SystemTime uint64 `json:"system_time" db:"system_time"`
|
||||
UserTime uint64 `json:"user_time" db:"user_time"`
|
||||
WallTime uint64 `json:"wall_time" db:"wall_time"`
|
||||
Interval int `json:"interval" db:"schedule_interval"`
|
||||
DiscardData bool `json:"discard_data" db:"discard_data"`
|
||||
LastFetched *time.Time `json:"last_fetched" db:"last_fetched"`
|
||||
AutomationsEnabled bool `json:"automations_enabled" db:"automations_enabled"`
|
||||
LastExecuted time.Time `json:"last_executed" db:"last_executed"`
|
||||
OutputSize uint64 `json:"output_size" db:"output_size"`
|
||||
SystemTime uint64 `json:"system_time" db:"system_time"`
|
||||
UserTime uint64 `json:"user_time" db:"user_time"`
|
||||
WallTime uint64 `json:"wall_time" db:"wall_time"`
|
||||
}
|
||||
|
||||
// MapQueryReportsResultsToRows converts the scheduled query results as stored in Fleet's database
|
||||
|
|
|
|||
|
|
@ -155,12 +155,15 @@ type ScheduledQueryStats struct {
|
|||
Denylisted bool `json:"denylisted" db:"denylisted"`
|
||||
Executions uint64 `json:"executions" db:"executions"`
|
||||
// Note schedule_interval is used for DB since "interval" is a reserved word in MySQL
|
||||
Interval int `json:"interval" db:"schedule_interval"`
|
||||
LastExecuted time.Time `json:"last_executed" db:"last_executed"`
|
||||
OutputSize uint64 `json:"output_size" db:"output_size"`
|
||||
SystemTime uint64 `json:"system_time" db:"system_time"`
|
||||
UserTime uint64 `json:"user_time" db:"user_time"`
|
||||
WallTime uint64 `json:"wall_time" db:"wall_time"`
|
||||
Interval int `json:"interval" db:"schedule_interval"`
|
||||
DiscardData bool `json:"discard_data" db:"discard_data"`
|
||||
LastFetched *time.Time `json:"last_fetched" db:"last_fetched"`
|
||||
AutomationsEnabled bool `json:"automations_enabled" db:"automations_enabled"`
|
||||
LastExecuted time.Time `json:"last_executed" db:"last_executed"`
|
||||
OutputSize uint64 `json:"output_size" db:"output_size"`
|
||||
SystemTime uint64 `json:"system_time" db:"system_time"`
|
||||
UserTime uint64 `json:"user_time" db:"user_time"`
|
||||
WallTime uint64 `json:"wall_time" db:"wall_time"`
|
||||
}
|
||||
|
||||
// TeamID returns the team id if the stat is for a team query stat result
|
||||
|
|
|
|||
|
|
@ -4356,7 +4356,7 @@ func (s *integrationEnterpriseTestSuite) TestRunHostScript() {
|
|||
// create a valid sync script execution request, fails because the
|
||||
// request will time-out waiting for a result.
|
||||
runSyncResp = runScriptSyncResponse{}
|
||||
s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "echo"}, http.StatusGatewayTimeout, &runSyncResp)
|
||||
s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "echo"}, http.StatusRequestTimeout, &runSyncResp)
|
||||
require.Equal(t, host.ID, runSyncResp.HostID)
|
||||
require.NotEmpty(t, runSyncResp.ExecutionID)
|
||||
require.True(t, runSyncResp.HostTimeout)
|
||||
|
|
@ -4574,7 +4574,7 @@ func (s *integrationEnterpriseTestSuite) TestRunHostSavedScript() {
|
|||
// create a valid sync script execution request, fails because the
|
||||
// request will time-out waiting for a result.
|
||||
var runSyncResp runScriptSyncResponse
|
||||
s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptID: &savedNoTmScript.ID}, http.StatusGatewayTimeout, &runSyncResp)
|
||||
s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptID: &savedNoTmScript.ID}, http.StatusRequestTimeout, &runSyncResp)
|
||||
require.Equal(t, host.ID, runSyncResp.HostID)
|
||||
require.NotEmpty(t, runSyncResp.ExecutionID)
|
||||
require.NotNil(t, runSyncResp.ScriptID)
|
||||
|
|
|
|||
Loading…
Reference in a new issue