diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index a724d42dfd..2b92311ab5 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -372,50 +372,54 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, if teamID != nil { teamID_ = *teamID } - ds := dialect.From(goqu.I("queries").As("q")).Select( - goqu.I("q.id"), - goqu.I("q.name"), - goqu.I("q.description"), - goqu.I("q.team_id"), - goqu.I("q.schedule_interval").As("schedule_interval"), - goqu.COALESCE(goqu.I("sqs.average_memory"), 0).As("average_memory"), - goqu.COALESCE(goqu.I("sqs.denylisted"), false).As("denylisted"), - goqu.COALESCE(goqu.I("sqs.executions"), 0).As("executions"), - goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("timestamp(?)", pastDate)).As("last_executed"), - goqu.COALESCE(goqu.I("sqs.output_size"), 0).As("output_size"), - goqu.COALESCE(goqu.I("sqs.system_time"), 0).As("system_time"), - goqu.COALESCE(goqu.I("sqs.user_time"), 0).As("user_time"), - goqu.COALESCE(goqu.I("sqs.wall_time"), 0).As("wall_time"), - ).LeftJoin( - dialect.From("scheduled_query_stats").As("sqs").Where( - goqu.I("host_id").Eq(hid), - ), - goqu.On(goqu.I("sqs.scheduled_query_id").Eq(goqu.I("q.id"))), - ).Where( - goqu.And( - goqu.Or( - // sq.platform empty or NULL means the scheduled query is set to - // run on all hosts. - goqu.I("q.platform").Eq(""), - goqu.I("q.platform").IsNull(), - // scheduled_queries.platform can be a comma-separated list of - // platforms, e.g. "darwin,windows". - goqu.L("FIND_IN_SET(?, q.platform)", fleet.PlatformFromHost(hostPlatform)).Neq(0), - ), - goqu.I("q.schedule_interval").Gt(0), - goqu.I("q.automations_enabled").IsTrue(), - goqu.Or( - goqu.I("q.team_id").IsNull(), - goqu.I("q.team_id").Eq(teamID_), - ), - ), - ) - sql, args, err := ds.ToSQL() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "sql build") + + sqlQuery := ` + SELECT + q.id, + q.name, + q.description, + q.team_id, + q.schedule_interval AS schedule_interval, + q.discard_data, + q.automations_enabled, + MAX(qr.last_fetched) as last_fetched, + COALESCE(MAX(sqs.average_memory), 0) AS average_memory, + COALESCE(MAX(sqs.denylisted), false) AS denylisted, + COALESCE(MAX(sqs.executions), 0) AS executions, + COALESCE(MAX(sqs.last_executed), TIMESTAMP(?)) AS last_executed, + COALESCE(MAX(sqs.output_size), 0) AS output_size, + COALESCE(MAX(sqs.system_time), 0) AS system_time, + COALESCE(MAX(sqs.user_time), 0) AS user_time, + COALESCE(MAX(sqs.wall_time), 0) AS wall_time + FROM + queries q + LEFT JOIN scheduled_query_stats sqs ON (q.id = sqs.scheduled_query_id AND sqs.host_id = ?) + LEFT JOIN query_results qr ON (q.id = qr.query_id AND qr.host_id = ?) + WHERE + (q.platform = '' OR q.platform IS NULL OR FIND_IN_SET(?, q.platform) != 0) + AND q.schedule_interval > 0 + AND (q.automations_enabled IS TRUE OR (q.discard_data IS FALSE AND q.logging_type = ?)) + AND (q.team_id IS NULL OR q.team_id = ?) + OR EXISTS ( + SELECT 1 FROM query_results + WHERE query_results.query_id = q.id + AND query_results.host_id = ? + ) + GROUP BY q.id + ` + + args := []interface{}{ + pastDate, + hid, + hid, + fleet.PlatformFromHost(hostPlatform), + fleet.LoggingSnapshot, + teamID_, + hid, } + var stats []fleet.QueryStats - if err := sqlx.SelectContext(ctx, db, &stats, sql, args...); err != nil { + if err := sqlx.SelectContext(ctx, db, &stats, sqlQuery, args...); err != nil { return nil, ctxerr.Wrap(ctx, err, "load query stats") } return stats, nil @@ -690,6 +694,9 @@ func queryStatsToScheduledQueryStats(queriesStats []fleet.QueryStats, packName s Denylisted: queryStats.Denylisted, Executions: queryStats.Executions, Interval: queryStats.Interval, + DiscardData: queryStats.DiscardData, + AutomationsEnabled: queryStats.AutomationsEnabled, + LastFetched: queryStats.LastFetched, LastExecuted: queryStats.LastExecuted, OutputSize: queryStats.OutputSize, SystemTime: queryStats.SystemTime, diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 260b8efa36..f41b62f3be 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -116,6 +116,7 @@ func TestHosts(t *testing.T) { {"HostsListByDiskEncryptionStatus", testHostsListMacOSSettingsDiskEncryptionStatus}, {"HostsListFailingPolicies", printReadsInTest(testHostsListFailingPolicies)}, {"HostsExpiration", testHostsExpiration}, + {"HostsIncludesScheduledQueriesInPackStats", testHostsIncludesScheduledQueriesInPackStats}, {"HostsAllPackStats", testHostsAllPackStats}, {"HostsPackStatsMultipleHosts", testHostsPackStatsMultipleHosts}, {"HostsPackStatsForPlatform", testHostsPackStatsForPlatform}, @@ -618,17 +619,20 @@ func testHostsWithTeamPackStats(t *testing.T, ds *Datastore) { PackName: pack1.Name, ScheduledQueryName: squery1.Name, - QueryName: query1.Name, - PackID: pack1.ID, - AverageMemory: 8000, - Denylisted: false, - Executions: 164, - Interval: 30, - LastExecuted: time.Unix(1620325191, 0).UTC(), - OutputSize: 1337, - SystemTime: 150, - UserTime: 180, - WallTime: 0, + QueryName: query1.Name, + PackID: pack1.ID, + DiscardData: false, + AutomationsEnabled: false, + LastFetched: nil, + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, }, } stats2 := []fleet.ScheduledQueryStats{ @@ -636,17 +640,20 @@ func testHostsWithTeamPackStats(t *testing.T, ds *Datastore) { PackName: fmt.Sprintf("team-%d", team.ID), ScheduledQueryName: tpQuery.Name, - QueryName: tpQuery.Name, - PackID: 0, // pack_id will be 0 for stats of queries not in packs. - AverageMemory: 8000, - Denylisted: false, - Executions: 164, - Interval: 30, - LastExecuted: time.Unix(1620325191, 0).UTC(), - OutputSize: 1337, - SystemTime: 150, - UserTime: 180, - WallTime: 0, + QueryName: tpQuery.Name, + PackID: 0, // pack_id will be 0 for stats of queries not in packs. + LastFetched: nil, + DiscardData: tpQuery.DiscardData, + AutomationsEnabled: tpQuery.AutomationsEnabled, + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, }, } @@ -3785,6 +3792,171 @@ func testHostsExpiration(t *testing.T, ds *Datastore) { require.Len(t, hosts, 5) } +func testHostsIncludesScheduledQueriesInPackStats(t *testing.T, ds *Datastore) { + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: ptr.String("1"), + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + Platform: "darwin", + }) + require.NoError(t, err) + require.NotNil(t, host) + + team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + err = ds.AddHostsToTeam(context.Background(), &team.ID, []uint{host.ID}) + require.NoError(t, err) + + query1 := &fleet.Query{ + Name: "Only Logged in Query Report", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: nil, + Interval: 60, + Logging: fleet.LoggingSnapshot, + DiscardData: false, + AutomationsEnabled: false, + } + + _, err = ds.NewQuery(context.Background(), query1) + require.NoError(t, err) + + query2 := &fleet.Query{ + Name: "Logged In Report and Log Destination", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: nil, + Interval: 60, + Logging: fleet.LoggingSnapshot, + DiscardData: false, + AutomationsEnabled: true, + } + _, err = ds.NewQuery(context.Background(), query2) + require.NoError(t, err) + + // This query should not be included in the pack stats + query3 := &fleet.Query{ + Name: "Not LoggingSnapshot", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: nil, + Interval: 60, + Logging: fleet.LoggingDifferential, + DiscardData: false, + AutomationsEnabled: false, // automations not on + } + _, err = ds.NewQuery(context.Background(), query3) + require.NoError(t, err) + + // This query should not be included in the pack stats + query4 := &fleet.Query{ + Name: "Query Report No Interval", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: nil, + Interval: 0, + Logging: fleet.LoggingSnapshot, + DiscardData: false, + AutomationsEnabled: false, + } + _, err = ds.NewQuery(context.Background(), query4) + require.NoError(t, err) + + // this query should not be included in the pack stats + query5 := &fleet.Query{ + Name: "Automations No Interval", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: nil, + Interval: 0, + Logging: fleet.LoggingSnapshot, + DiscardData: true, + AutomationsEnabled: true, + } + _, err = ds.NewQuery(context.Background(), query5) + require.NoError(t, err) + + query6 := &fleet.Query{ + Name: "Team Query", + Query: "select * from time", + AuthorID: nil, + Platform: "darwin", + Saved: true, + TeamID: &team.ID, + Interval: 60, + Logging: fleet.LoggingSnapshot, + DiscardData: false, + AutomationsEnabled: true, + } + _, err = ds.NewQuery(context.Background(), query6) + require.NoError(t, err) + + hostResult, err := ds.Host(context.Background(), host.ID) + require.NoError(t, err) + + globalQueryStats := hostResult.PackStats[0].QueryStats + require.NotNil(t, hostResult) + require.Equal(t, 2, len(globalQueryStats)) + require.Equal(t, query1.Name, globalQueryStats[0].ScheduledQueryName) + require.Equal(t, query2.Name, globalQueryStats[1].ScheduledQueryName) + + teamQueryStats := hostResult.PackStats[1].QueryStats + require.Equal(t, query6.Name, teamQueryStats[0].ScheduledQueryName) + + // Queries with Query Results should be included in the pack stats + // regardless of the query interval + queryResultRow := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query4.ID, // no interval + HostID: host.ID, + Data: ptr.RawMessage(json.RawMessage(`{"foo": "bar"}`)), + }, + { + QueryID: query4.ID, // no interval + HostID: host.ID, + Data: ptr.RawMessage(json.RawMessage(`{"foo": "baz"}`)), + }, + } + err = ds.OverwriteQueryResultRows(context.Background(), queryResultRow) + require.NoError(t, err) + + hostResult, err = ds.Host(context.Background(), host.ID) + require.NoError(t, err) + require.NotNil(t, hostResult) + + assertContains := func(stats []fleet.ScheduledQueryStats, name string) { + t.Helper() + for _, stat := range stats { + if stat.ScheduledQueryName == name { + return + } + } + t.Errorf("expected to find %s in stats", name) + } + + globalQueryStats = hostResult.PackStats[0].QueryStats + require.Equal(t, 3, len(globalQueryStats)) + assertContains(globalQueryStats, query1.Name) + assertContains(globalQueryStats, query2.Name) + assertContains(globalQueryStats, query4.Name) // no interval, but has a query result +} + func testHostsAllPackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), diff --git a/server/fleet/queries.go b/server/fleet/queries.go index 59472e17f9..0e1aa84f80 100644 --- a/server/fleet/queries.go +++ b/server/fleet/queries.go @@ -414,12 +414,15 @@ type QueryStats struct { Denylisted bool `json:"denylisted" db:"denylisted"` Executions uint64 `json:"executions" db:"executions"` // Note schedule_interval is used for DB since "interval" is a reserved word in MySQL - Interval int `json:"interval" db:"schedule_interval"` - LastExecuted time.Time `json:"last_executed" db:"last_executed"` - OutputSize uint64 `json:"output_size" db:"output_size"` - SystemTime uint64 `json:"system_time" db:"system_time"` - UserTime uint64 `json:"user_time" db:"user_time"` - WallTime uint64 `json:"wall_time" db:"wall_time"` + Interval int `json:"interval" db:"schedule_interval"` + DiscardData bool `json:"discard_data" db:"discard_data"` + LastFetched *time.Time `json:"last_fetched" db:"last_fetched"` + AutomationsEnabled bool `json:"automations_enabled" db:"automations_enabled"` + LastExecuted time.Time `json:"last_executed" db:"last_executed"` + OutputSize uint64 `json:"output_size" db:"output_size"` + SystemTime uint64 `json:"system_time" db:"system_time"` + UserTime uint64 `json:"user_time" db:"user_time"` + WallTime uint64 `json:"wall_time" db:"wall_time"` } // MapQueryReportsResultsToRows converts the scheduled query results as stored in Fleet's database diff --git a/server/fleet/scheduled_queries.go b/server/fleet/scheduled_queries.go index 0eaac54cb5..0c3af78c7f 100644 --- a/server/fleet/scheduled_queries.go +++ b/server/fleet/scheduled_queries.go @@ -155,12 +155,15 @@ type ScheduledQueryStats struct { Denylisted bool `json:"denylisted" db:"denylisted"` Executions uint64 `json:"executions" db:"executions"` // Note schedule_interval is used for DB since "interval" is a reserved word in MySQL - Interval int `json:"interval" db:"schedule_interval"` - LastExecuted time.Time `json:"last_executed" db:"last_executed"` - OutputSize uint64 `json:"output_size" db:"output_size"` - SystemTime uint64 `json:"system_time" db:"system_time"` - UserTime uint64 `json:"user_time" db:"user_time"` - WallTime uint64 `json:"wall_time" db:"wall_time"` + Interval int `json:"interval" db:"schedule_interval"` + DiscardData bool `json:"discard_data" db:"discard_data"` + LastFetched *time.Time `json:"last_fetched" db:"last_fetched"` + AutomationsEnabled bool `json:"automations_enabled" db:"automations_enabled"` + LastExecuted time.Time `json:"last_executed" db:"last_executed"` + OutputSize uint64 `json:"output_size" db:"output_size"` + SystemTime uint64 `json:"system_time" db:"system_time"` + UserTime uint64 `json:"user_time" db:"user_time"` + WallTime uint64 `json:"wall_time" db:"wall_time"` } // TeamID returns the team id if the stat is for a team query stat result diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 5d15d97125..9ee2883fb5 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -4356,7 +4356,7 @@ func (s *integrationEnterpriseTestSuite) TestRunHostScript() { // create a valid sync script execution request, fails because the // request will time-out waiting for a result. runSyncResp = runScriptSyncResponse{} - s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "echo"}, http.StatusGatewayTimeout, &runSyncResp) + s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptContents: "echo"}, http.StatusRequestTimeout, &runSyncResp) require.Equal(t, host.ID, runSyncResp.HostID) require.NotEmpty(t, runSyncResp.ExecutionID) require.True(t, runSyncResp.HostTimeout) @@ -4574,7 +4574,7 @@ func (s *integrationEnterpriseTestSuite) TestRunHostSavedScript() { // create a valid sync script execution request, fails because the // request will time-out waiting for a result. var runSyncResp runScriptSyncResponse - s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptID: &savedNoTmScript.ID}, http.StatusGatewayTimeout, &runSyncResp) + s.DoJSON("POST", "/api/latest/fleet/scripts/run/sync", fleet.HostScriptRequestPayload{HostID: host.ID, ScriptID: &savedNoTmScript.ID}, http.StatusRequestTimeout, &runSyncResp) require.Equal(t, host.ID, runSyncResp.HostID) require.NotEmpty(t, runSyncResp.ExecutionID) require.NotNil(t, runSyncResp.ScriptID)