From 8d559665530cf465fddfb3485f849b2ba7ee01bf Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Fri, 14 Jul 2023 13:37:09 -0400 Subject: [PATCH] Updated osquery/config endpoint to include scheduled queries (#12723) Updated GetClientConfig API endpoint --- ...clude-scheduled-queries-in-getclientconfig | 2 + server/datastore/mysql/hosts_test.go | 265 +++++++----------- server/datastore/mysql/packs.go | 17 +- server/datastore/mysql/packs_test.go | 41 +++ server/datastore/mysql/queries.go | 34 ++- server/datastore/mysql/queries_test.go | 55 ++++ server/fleet/app.go | 6 +- server/fleet/datastore.go | 2 +- server/fleet/queries.go | 42 +++ server/fleet/queries_test.go | 54 ++++ server/service/osquery.go | 54 +++- server/service/osquery_test.go | 72 +++++ 12 files changed, 457 insertions(+), 187 deletions(-) create mode 100644 changes/12644-include-scheduled-queries-in-getclientconfig diff --git a/changes/12644-include-scheduled-queries-in-getclientconfig b/changes/12644-include-scheduled-queries-in-getclientconfig new file mode 100644 index 0000000000..f572537098 --- /dev/null +++ b/changes/12644-include-scheduled-queries-in-getclientconfig @@ -0,0 +1,2 @@ +- The `osquery/config` endpoint should include scheduled queries for the host's team stored in the + `queries` table. diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index df9448f348..3f84b3300d 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -3621,29 +3621,6 @@ func testHostsAllPackStats(t *testing.T, ds *Datastore) { require.NoError(t, err) require.NotNil(t, host) - // Create global pack (and one scheduled query in it). - test.AddAllHostsLabel(t, ds) // the global pack needs the "All Hosts" label. - labels, err := ds.ListLabels(context.Background(), fleet.TeamFilter{}, fleet.ListOptions{}) - require.NoError(t, err) - require.Len(t, labels, 1) - globalPack, err := ds.EnsureGlobalPack(context.Background()) - require.NoError(t, err) - globalQuery := test.NewQuery(t, ds, nil, "global-time", "select * from time", 0, true) - globalSQuery := test.NewScheduledQuery(t, ds, globalPack.ID, globalQuery.ID, 30, true, true, "time-scheduled-global") - err = ds.AsyncBatchInsertLabelMembership(context.Background(), [][2]uint{{labels[0].ID, host.ID}}) - require.NoError(t, err) - - // Create a team and its pack (and one scheduled query in it). - team, err := ds.NewTeam(context.Background(), &fleet.Team{ - Name: "team1", - }) - require.NoError(t, err) - require.NoError(t, ds.AddHostsToTeam(context.Background(), &team.ID, []uint{host.ID})) - teamPack, err := ds.EnsureTeamPack(context.Background(), team.ID) - require.NoError(t, err) - teamQuery := test.NewQuery(t, ds, nil, "team-time", "select * from time", 0, true) - teamSQuery := test.NewScheduledQuery(t, ds, teamPack.ID, teamQuery.ID, 31, true, true, "time-scheduled-team") - // Create a "user created" pack (and one scheduled query in it). userPack, err := ds.NewPack(context.Background(), &fleet.Pack{ Name: "test1", @@ -3657,7 +3634,7 @@ func testHostsAllPackStats(t *testing.T, ds *Datastore) { host, err = ds.Host(context.Background(), host.ID) require.NoError(t, err) packStats := host.PackStats - require.Len(t, packStats, 3) + require.Len(t, packStats, 1) sort.Sort(packStatsSlice(packStats)) for _, tc := range []struct { expectedPack *fleet.Pack @@ -3665,23 +3642,11 @@ func testHostsAllPackStats(t *testing.T, ds *Datastore) { expectedSQuery *fleet.ScheduledQuery packStats fleet.PackStats }{ - { - expectedPack: globalPack, - expectedQuery: globalQuery, - expectedSQuery: globalSQuery, - packStats: packStats[0], - }, - { - expectedPack: teamPack, - expectedQuery: teamQuery, - expectedSQuery: teamSQuery, - packStats: packStats[1], - }, { expectedPack: userPack, expectedQuery: userQuery, expectedSQuery: userSQuery, - packStats: packStats[2], + packStats: packStats[0], }, } { require.Equal(t, tc.expectedPack.ID, tc.packStats.PackID) @@ -3705,38 +3670,6 @@ func testHostsAllPackStats(t *testing.T, ds *Datastore) { require.Zero(t, tc.packStats.QueryStats[0].WallTime) } - globalPackSQueryStats := []fleet.ScheduledQueryStats{{ - ScheduledQueryName: globalSQuery.Name, - ScheduledQueryID: globalSQuery.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, - AverageMemory: 8000, - Denylisted: false, - Executions: 164, - Interval: 30, - LastExecuted: time.Unix(1620325191, 0).UTC(), - OutputSize: 1337, - SystemTime: 150, - UserTime: 180, - WallTime: 0, - }} - teamPackSQueryStats := []fleet.ScheduledQueryStats{{ - ScheduledQueryName: teamSQuery.Name, - ScheduledQueryID: teamSQuery.ID, - QueryName: teamQuery.Name, - PackName: teamPack.Name, - PackID: teamPack.ID, - AverageMemory: 8001, - Denylisted: true, - Executions: 165, - Interval: 31, - LastExecuted: time.Unix(1620325190, 0).UTC(), - OutputSize: 1338, - SystemTime: 151, - UserTime: 181, - WallTime: 1, - }} userPackSQueryStats := []fleet.ScheduledQueryStats{{ ScheduledQueryName: userSQuery.Name, ScheduledQueryID: userSQuery.ID, @@ -3756,22 +3689,14 @@ func testHostsAllPackStats(t *testing.T, ds *Datastore) { // Reload the host and set the scheduled queries stats. host, err = ds.Host(context.Background(), host.ID) require.NoError(t, err) - hostPackStats := []fleet.PackStats{ - {PackID: globalPack.ID, PackName: globalPack.Name, QueryStats: globalPackSQueryStats}, - {PackID: teamPack.ID, PackName: teamPack.Name, QueryStats: teamPackSQueryStats}, - } - err = ds.SaveHostPackStats(context.Background(), host.ID, hostPackStats) - require.NoError(t, err) host, err = ds.Host(context.Background(), host.ID) require.NoError(t, err) packStats = host.PackStats - require.Len(t, packStats, 3) + require.Len(t, packStats, 1) sort.Sort(packStatsSlice(packStats)) - require.ElementsMatch(t, packStats[0].QueryStats, globalPackSQueryStats) - require.ElementsMatch(t, packStats[1].QueryStats, teamPackSQueryStats) - require.ElementsMatch(t, packStats[2].QueryStats, userPackSQueryStats) + require.ElementsMatch(t, packStats[0].QueryStats, userPackSQueryStats) } // See #2965. @@ -3792,6 +3717,7 @@ func testHostsPackStatsMultipleHosts(t *testing.T, ds *Datastore) { }) require.NoError(t, err) require.NotNil(t, host1) + osqueryHostID2, _ := server.GenerateRandomText(10) host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -3814,10 +3740,15 @@ func testHostsPackStatsMultipleHosts(t *testing.T, ds *Datastore) { labels, err := ds.ListLabels(context.Background(), fleet.TeamFilter{}, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, labels, 1) - globalPack, err := ds.EnsureGlobalPack(context.Background()) + + userPack, err := ds.NewPack(context.Background(), &fleet.Pack{ + Name: "test1", + HostIDs: []uint{host1.ID, host2.ID}, + }) require.NoError(t, err) - globalQuery := test.NewQuery(t, ds, nil, "global-time", "select * from time", 0, true) - globalSQuery := test.NewScheduledQuery(t, ds, globalPack.ID, globalQuery.ID, 30, true, true, "time-scheduled-global") + + userQuery := test.NewQuery(t, ds, nil, "global-time", "select * from time", 0, true) + userSQuery := test.NewScheduledQuery(t, ds, userPack.ID, userQuery.ID, 30, true, true, "time-scheduled-global") err = ds.AsyncBatchInsertLabelMembership(context.Background(), [][2]uint{ {labels[0].ID, host1.ID}, {labels[0].ID, host2.ID}, @@ -3825,11 +3756,11 @@ func testHostsPackStatsMultipleHosts(t *testing.T, ds *Datastore) { require.NoError(t, err) globalStatsHost1 := []fleet.ScheduledQueryStats{{ - ScheduledQueryName: globalSQuery.Name, - ScheduledQueryID: globalSQuery.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery.Name, + ScheduledQueryID: userSQuery.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8000, Denylisted: false, Executions: 164, @@ -3841,11 +3772,11 @@ func testHostsPackStatsMultipleHosts(t *testing.T, ds *Datastore) { WallTime: 0, }} globalStatsHost2 := []fleet.ScheduledQueryStats{{ - ScheduledQueryName: globalSQuery.Name, - ScheduledQueryID: globalSQuery.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery.Name, + ScheduledQueryID: userSQuery.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 9000, Denylisted: false, Executions: 165, @@ -3874,7 +3805,7 @@ func testHostsPackStatsMultipleHosts(t *testing.T, ds *Datastore) { host, err := ds.Host(context.Background(), tc.hostID) require.NoError(t, err) hostPackStats := []fleet.PackStats{ - {PackID: globalPack.ID, PackName: globalPack.Name, QueryStats: tc.globalStats}, + {PackID: userPack.ID, PackName: userPack.Name, QueryStats: tc.globalStats}, } err = ds.SaveHostPackStats(context.Background(), host.ID, hostPackStats) require.NoError(t, err) @@ -3921,6 +3852,7 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { }) require.NoError(t, err) require.NotNil(t, host1) + osqueryHostID2, _ := server.GenerateRandomText(10) host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -3938,69 +3870,76 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { require.NoError(t, err) require.NotNil(t, host2) - // Create global pack (and one scheduled query in it). - test.AddAllHostsLabel(t, ds) // the global pack needs the "All Hosts" label. + test.AddAllHostsLabel(t, ds) labels, err := ds.ListLabels(context.Background(), fleet.TeamFilter{}, fleet.ListOptions{}) require.NoError(t, err) require.Len(t, labels, 1) - globalPack, err := ds.EnsureGlobalPack(context.Background()) + + userPack, err := ds.NewPack(context.Background(), &fleet.Pack{ + Name: "test1", + HostIDs: []uint{host1.ID, host2.ID}, + }) require.NoError(t, err) - globalQuery := test.NewQuery(t, ds, nil, "global-time", "select * from time", 0, true) - globalSQuery1, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ + userQuery := test.NewQuery(t, ds, nil, "global-time", "select * from time", 0, true) + userSQuery1, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ Name: "Scheduled Query For Linux only", - PackID: globalPack.ID, - QueryID: globalQuery.ID, + PackID: userPack.ID, + QueryID: userQuery.ID, Interval: 30, Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String("linux"), }) require.NoError(t, err) - require.NotZero(t, globalSQuery1.ID) - globalSQuery2, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ + require.NotZero(t, userSQuery1.ID) + + userSQuery2, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ Name: "Scheduled Query For Darwin only", - PackID: globalPack.ID, - QueryID: globalQuery.ID, + PackID: userPack.ID, + QueryID: userQuery.ID, Interval: 30, Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String("darwin"), }) require.NoError(t, err) - require.NotZero(t, globalSQuery2.ID) - globalSQuery3, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ + require.NotZero(t, userSQuery2.ID) + + userSQuery3, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ Name: "Scheduled Query For Darwin and Linux", - PackID: globalPack.ID, - QueryID: globalQuery.ID, + PackID: userPack.ID, + QueryID: userQuery.ID, Interval: 30, Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String("darwin,linux"), }) require.NoError(t, err) - require.NotZero(t, globalSQuery3.ID) - globalSQuery4, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ + require.NotZero(t, userSQuery3.ID) + + userSQuery4, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ Name: "Scheduled Query For All Platforms", - PackID: globalPack.ID, - QueryID: globalQuery.ID, + PackID: userPack.ID, + QueryID: userQuery.ID, Interval: 30, Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: ptr.String(""), }) require.NoError(t, err) - require.NotZero(t, globalSQuery4.ID) - globalSQuery5, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ + require.NotZero(t, userSQuery4.ID) + + userSQuery5, err := ds.NewScheduledQuery(context.Background(), &fleet.ScheduledQuery{ Name: "Scheduled Query For All Platforms v2", - PackID: globalPack.ID, - QueryID: globalQuery.ID, + PackID: userPack.ID, + QueryID: userQuery.ID, Interval: 30, Snapshot: ptr.Bool(true), Removed: ptr.Bool(true), Platform: nil, }) require.NoError(t, err) - require.NotZero(t, globalSQuery5.ID) + require.NotZero(t, userSQuery5.ID) err = ds.AsyncBatchInsertLabelMembership(context.Background(), [][2]uint{ {labels[0].ID, host1.ID}, @@ -4010,11 +3949,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { globalStats := []fleet.ScheduledQueryStats{ { - ScheduledQueryName: globalSQuery2.Name, - ScheduledQueryID: globalSQuery2.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery2.Name, + ScheduledQueryID: userSQuery2.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8001, Denylisted: false, Executions: 165, @@ -4026,11 +3965,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 1, }, { - ScheduledQueryName: globalSQuery3.Name, - ScheduledQueryID: globalSQuery3.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery3.Name, + ScheduledQueryID: userSQuery3.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8002, Denylisted: false, Executions: 166, @@ -4042,11 +3981,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 2, }, { - ScheduledQueryName: globalSQuery4.Name, - ScheduledQueryID: globalSQuery4.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery4.Name, + ScheduledQueryID: userSQuery4.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8003, Denylisted: false, Executions: 167, @@ -4058,11 +3997,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 3, }, { - ScheduledQueryName: globalSQuery5.Name, - ScheduledQueryID: globalSQuery5.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery5.Name, + ScheduledQueryID: userSQuery5.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8003, Denylisted: false, Executions: 167, @@ -4083,11 +4022,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { stats[i] = globalStats[i] } stats = append(stats, fleet.ScheduledQueryStats{ - ScheduledQueryName: globalSQuery1.Name, - ScheduledQueryID: globalSQuery1.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery1.Name, + ScheduledQueryID: userSQuery1.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 8003, Denylisted: false, Executions: 167, @@ -4101,7 +4040,7 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { host, err := ds.Host(context.Background(), host1.ID) require.NoError(t, err) hostPackStats := []fleet.PackStats{ - {PackID: globalPack.ID, PackName: globalPack.Name, QueryStats: stats}, + {PackID: userPack.ID, PackName: userPack.Name, QueryStats: stats}, } err = ds.SaveHostPackStats(context.Background(), host.ID, hostPackStats) require.NoError(t, err) @@ -4130,11 +4069,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { require.Len(t, packStats2[0].QueryStats, 4) zeroStats := []fleet.ScheduledQueryStats{ { - ScheduledQueryName: globalSQuery1.Name, - ScheduledQueryID: globalSQuery1.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery1.Name, + ScheduledQueryID: userSQuery1.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 0, Denylisted: false, Executions: 0, @@ -4146,11 +4085,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 0, }, { - ScheduledQueryName: globalSQuery3.Name, - ScheduledQueryID: globalSQuery3.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery3.Name, + ScheduledQueryID: userSQuery3.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 0, Denylisted: false, Executions: 0, @@ -4162,11 +4101,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 0, }, { - ScheduledQueryName: globalSQuery4.Name, - ScheduledQueryID: globalSQuery4.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery4.Name, + ScheduledQueryID: userSQuery4.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 0, Denylisted: false, Executions: 0, @@ -4178,11 +4117,11 @@ func testHostsPackStatsForPlatform(t *testing.T, ds *Datastore) { WallTime: 0, }, { - ScheduledQueryName: globalSQuery5.Name, - ScheduledQueryID: globalSQuery5.ID, - QueryName: globalQuery.Name, - PackName: globalPack.Name, - PackID: globalPack.ID, + ScheduledQueryName: userSQuery5.Name, + ScheduledQueryID: userSQuery5.ID, + QueryName: userQuery.Name, + PackName: userPack.Name, + PackID: userPack.ID, AverageMemory: 0, Denylisted: false, Executions: 0, diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index b14854d971..fd17e09bb4 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -577,10 +577,10 @@ func (ds *Datastore) ListPacksForHost(ctx context.Context, hid uint) ([]*fleet.P return listPacksForHost(ctx, ds.reader(ctx), hid) } -// listPacksForHost returns all the packs that are configured to run on the given host. +// listPacksForHost returns all the "user packs" that are configured to run on the given host. func listPacksForHost(ctx context.Context, db sqlx.QueryerContext, hid uint) ([]*fleet.Pack, error) { query := ` -SELECT DISTINCT packs.* FROM ( + SELECT DISTINCT packs.* FROM ( ( SELECT p.* FROM packs p JOIN pack_targets pt @@ -590,26 +590,29 @@ SELECT DISTINCT packs.* FROM ( AND pt.target_id = lm.label_id AND pt.type = ? ) - WHERE lm.host_id = ? AND NOT p.disabled + WHERE lm.host_id = ? AND NOT p.disabled AND p.pack_type IS NULL ) UNION ALL ( SELECT p.* FROM packs p - JOIN pack_targets pt - ON (p.id = pt.pack_id AND pt.type = ? AND pt.target_id = ?) + JOIN pack_targets pt ON (p.id = pt.pack_id AND pt.type = ? AND pt.target_id = ?) + WHERE p.pack_type IS NULL ) UNION ALL ( SELECT p.* FROM packs p JOIN pack_targets pt - ON (p.id = pt.pack_id AND pt.type = ? AND pt.target_id = (SELECT team_id FROM hosts WHERE id = ?))) - ) packs` + ON (p.id = pt.pack_id AND pt.type = ? AND pt.target_id = (SELECT team_id FROM hosts WHERE id = ?)) + WHERE p.pack_type IS NULL + )) packs` + packs := []*fleet.Pack{} if err := sqlx.SelectContext(ctx, db, &packs, query, fleet.TargetLabel, hid, fleet.TargetHost, hid, fleet.TargetTeam, hid, ); err != nil && err != sql.ErrNoRows { return nil, ctxerr.Wrap(ctx, err, "listing hosts in pack") } + return packs, nil } diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 21c91af67d..a774e28269 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -38,6 +38,7 @@ func TestPacks(t *testing.T) { {"ApplySpecFailsOnTargetIDNull", testPacksApplySpecFailsOnTargetIDNull}, {"ApplyStatsNotLocking", testPacksApplyStatsNotLocking}, {"ApplyStatsNotLockingTryTwo", testPacksApplyStatsNotLockingTryTwo}, + {"ListForHostIncludesOnlyUserPacks", testListForHostIncludesOnlyUserPacks}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -683,3 +684,43 @@ func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { cancelFunc() } + +func testListForHostIncludesOnlyUserPacks(t *testing.T, ds *Datastore) { + mockClock := clock.NewMockClock() + h1 := test.NewHost(t, ds, "h1.local", "10.10.10.1", "1", "1", mockClock.Now()) + ctx := context.Background() + + label := &fleet.LabelSpec{ + ID: 1, + Name: "All Hosts", + } + require.NoError(t, ds.ApplyLabelSpecs(ctx, []*fleet.LabelSpec{label})) + + pack := &fleet.PackSpec{ + ID: 1, + Name: "foo_pack", + Targets: fleet.PackSpecTargets{ + Labels: []string{ + label.Name, + }, + }, + } + require.NoError(t, ds.ApplyPackSpecs(ctx, []*fleet.PackSpec{pack})) + require.NoError(t, ds.RecordLabelQueryExecutions(ctx, h1, map[uint]*bool{label.ID: ptr.Bool(true)}, mockClock.Now(), false)) + + _, err := ds.EnsureGlobalPack(ctx) + require.NoError(t, err) + + team, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + require.NoError(t, ds.AddHostsToTeam(ctx, &team.ID, []uint{h1.ID})) + _, err = ds.EnsureTeamPack(ctx, team.ID) + require.NoError(t, err) + + packs, err := ds.ListPacksForHost(ctx, h1.ID) + require.Nil(t, err) + if assert.Len(t, packs, 1) { + assert.Equal(t, "foo_pack", packs[0].Name) + } +} diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index 1816c13087..0a8af019b0 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -353,24 +353,34 @@ func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions FROM queries q LEFT JOIN users u ON (q.author_id = u.id) LEFT JOIN aggregated_stats ag ON (ag.id = q.id AND ag.global_stats = ? AND ag.type = ?) - WHERE saved = true - ` - if opt.OnlyObserverCanRun { - sql += " AND q.observer_can_run=true" - } + WHERE saved = true` args := []interface{}{false, aggregatedStatsTypeQuery} - whereClause := " AND team_id_char = ''" - if opt.TeamID != nil { - args = append(args, fmt.Sprint(*opt.TeamID)) - whereClause = " AND team_id_char = ?" - } - sql += whereClause + whereClauses := "" + if opt.OnlyObserverCanRun { + whereClauses += " AND q.observer_can_run=true" + } + + if opt.TeamID != nil { + args = append(args, *opt.TeamID) + whereClauses += " AND team_id = ?" + } else { + whereClauses += " AND team_id IS NULL" + } + + if opt.IsScheduled != nil { + if *opt.IsScheduled { + whereClauses += " AND (q.schedule_interval>0 AND q.automations_enabled=1)" + } else { + whereClauses += " AND (q.schedule_interval=0 OR q.automations_enabled=0)" + } + } + + sql += whereClauses sql = appendListOptionsToSQL(sql, &opt.ListOptions) results := []*fleet.Query{} - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, sql, args...); err != nil { return nil, ctxerr.Wrap(ctx, err, "listing queries") } diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index edb9d028fc..2d82321357 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +31,7 @@ func TestQueries(t *testing.T) { {"ListFiltersObservers", testQueriesListFiltersObservers}, {"ObserverCanRunQuery", testObserverCanRunQuery}, {"ListFiltersByTeamID", testQueriesListFiltersByTeamID}, + {"ListFiltersByIsScheduled", testQueriesListFiltersByIsScheduled}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -614,3 +616,56 @@ func testQueriesListFiltersByTeamID(t *testing.T, ds *Datastore) { require.NoError(t, err) test.QueryElementsMatch(t, queries, []*fleet.Query{teamQ1, teamQ2, teamQ3}) } + +func testQueriesListFiltersByIsScheduled(t *testing.T, ds *Datastore) { + q1, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Query: "select 1;", + Saved: true, + ScheduleInterval: 0, + }) + require.NoError(t, err) + q2, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Query: "select 1;", + Saved: true, + ScheduleInterval: 10, + AutomationsEnabled: false, + }) + require.NoError(t, err) + q3, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query3", + Query: "select 1;", + Saved: true, + ScheduleInterval: 20, + AutomationsEnabled: true, + }) + require.NoError(t, err) + + testCases := []struct { + opts fleet.ListQueryOptions + expected []*fleet.Query + }{ + { + opts: fleet.ListQueryOptions{}, + expected: []*fleet.Query{q1, q2, q3}, + }, + { + opts: fleet.ListQueryOptions{IsScheduled: ptr.Bool(true)}, + expected: []*fleet.Query{q3}, + }, + { + opts: fleet.ListQueryOptions{IsScheduled: ptr.Bool(false)}, + expected: []*fleet.Query{q1, q2}, + }, + } + + for i, tCase := range testCases { + queries, err := ds.ListQueries( + context.Background(), + tCase.opts, + ) + require.NoError(t, err) + test.QueryElementsMatch(t, queries, tCase.expected, i) + } +} diff --git a/server/fleet/app.go b/server/fleet/app.go index 87dcbddf36..04512b0de6 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -762,8 +762,10 @@ type ListQueryOptions struct { ListOptions // TeamID which team the queries belong to. If teamID is nil, then it is assumed the 'global' - // team - TeamID *uint + // team. + TeamID *uint + // IsScheduled filters queries that are meant to run at a set interval. + IsScheduled *bool OnlyObserverCanRun bool } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 28341a941f..9c0792d839 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -140,7 +140,7 @@ type Datastore interface { // PackByName fetches pack if it exists, if the pack exists the bool return value is true PackByName(ctx context.Context, name string, opts ...OptionalArg) (*Pack, bool, error) - // ListPacksForHost lists the packs that a host should execute. + // ListPacksForHost lists the "user packs" that a host should execute. ListPacksForHost(ctx context.Context, hid uint) (packs []*Pack, err error) // EnsureGlobalPack gets or inserts a pack with type global diff --git a/server/fleet/queries.go b/server/fleet/queries.go index f6548448e1..5d450789c2 100644 --- a/server/fleet/queries.go +++ b/server/fleet/queries.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/ghodss/yaml" ) @@ -69,6 +70,36 @@ func (q *Query) TeamIDStr() string { return fmt.Sprint(*q.TeamID) } +func (q *Query) GetSnapshot() *bool { + var loggingType string + if q != nil { + loggingType = q.LoggingType + } + + switch loggingType { + case "snapshot": + return ptr.Bool(true) + default: + return nil + } +} + +func (q *Query) GetRemoved() *bool { + var loggingType string + if q != nil { + loggingType = q.LoggingType + } + + switch loggingType { + case "differential": + return ptr.Bool(true) + case "differential_ignore_removals": + return ptr.Bool(false) + default: + return nil + } +} + // Verify verifies the query payload is valid. func (q *QueryPayload) Verify() error { if q.Name != nil { @@ -95,6 +126,17 @@ func (q *Query) Verify() error { return nil } +func (q *Query) ToQueryContent() QueryContent { + return QueryContent{ + Query: q.Query, + Interval: q.ScheduleInterval, + Platform: &q.Platform, + Version: &q.MinOsqueryVersion, + Removed: q.GetRemoved(), + Snapshot: q.GetSnapshot(), + } +} + type TargetedQuery struct { *Query HostTargets HostTargets `json:"host_targets"` diff --git a/server/fleet/queries_test.go b/server/fleet/queries_test.go index 24983d764b..9786ff5b12 100644 --- a/server/fleet/queries_test.go +++ b/server/fleet/queries_test.go @@ -8,6 +8,60 @@ import ( "github.com/stretchr/testify/require" ) +func TestGetSnapshot(t *testing.T) { + testCases := []struct { + query *Query + expected *bool + }{ + { + query: nil, + expected: nil, + }, + { + query: &Query{LoggingType: "snapshot"}, + expected: ptr.Bool(true), + }, + { + query: &Query{LoggingType: "differential"}, + expected: nil, + }, + { + query: &Query{LoggingType: "differential_ignore_removals"}, + expected: nil, + }, + } + for _, tCase := range testCases { + require.Equal(t, tCase.expected, tCase.query.GetSnapshot()) + } +} + +func TestGetRemoved(t *testing.T) { + testCases := []struct { + query *Query + expected *bool + }{ + { + query: nil, + expected: nil, + }, + { + query: &Query{LoggingType: "snapshot"}, + expected: nil, + }, + { + query: &Query{LoggingType: "differential"}, + expected: ptr.Bool(true), + }, + { + query: &Query{LoggingType: "differential_ignore_removals"}, + expected: ptr.Bool(false), + }, + } + for i, tCase := range testCases { + require.Equal(t, tCase.expected, tCase.query.GetRemoved(), i) + } +} + func TestTeamIDStr(t *testing.T) { testCases := []struct { query *Query diff --git a/server/service/osquery.go b/server/service/osquery.go index 64ba71342e..af9d8e5286 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -346,6 +346,25 @@ func getClientConfigEndpoint(ctx context.Context, request interface{}, svc fleet }, nil } +func (svc *Service) getScheduledQueries(ctx context.Context, teamID *uint) (fleet.Queries, error) { + opts := fleet.ListQueryOptions{IsScheduled: ptr.Bool(true), TeamID: teamID} + queries, err := svc.ds.ListQueries(ctx, opts) + if err != nil { + return nil, err + } + + if len(queries) == 0 { + return nil, nil + } + + config := make(fleet.Queries, len(queries)) + for _, query := range queries { + config[query.Name] = query.ToQueryContent() + } + + return config, nil +} + func (svc *Service) GetClientConfig(ctx context.Context) (map[string]interface{}, error) { // skipauth: Authorization is currently for user endpoints only. svc.authz.SkipAuthorization(ctx) @@ -368,12 +387,12 @@ func (svc *Service) GetClientConfig(ctx context.Context) (map[string]interface{} } } + packConfig := fleet.Packs{} + packs, err := svc.ds.ListPacksForHost(ctx, host.ID) if err != nil { return nil, newOsqueryError("database error: " + err.Error()) } - - packConfig := fleet.Packs{} for _, pack := range packs { // first, we must figure out what queries are in this pack queries, err := svc.ds.ListScheduledQueriesInPack(ctx, pack.ID) @@ -414,6 +433,37 @@ func (svc *Service) GetClientConfig(ctx context.Context) (map[string]interface{} } } + globalQueries, err := svc.getScheduledQueries(ctx, nil) + if err != nil { + return nil, newOsqueryError("database error: " + err.Error()) + } + if len(globalQueries) > 0 { + packConfig["Global"] = fleet.PackContent{ + Queries: globalQueries, + } + } + + if host.TeamID != nil { + team, err := svc.ds.Team(ctx, *host.TeamID) + if err != nil { + return nil, newOsqueryError("database error: " + err.Error()) + } + + if team != nil { + teamQueries, err := svc.getScheduledQueries(ctx, host.TeamID) + if err != nil { + return nil, newOsqueryError("database error: " + err.Error()) + } + if len(teamQueries) > 0 { + packName := fmt.Sprintf("Team: %s", team.Name) + packConfig[packName] = fleet.PackContent{ + Queries: teamQueries, + } + } + } + + } + if len(packConfig) > 0 { packJSON, err := json.Marshal(packConfig) if err != nil { diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index 625bb8761e..af76ee5aba 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -40,6 +40,16 @@ import ( func TestGetClientConfig(t *testing.T) { ds := new(mock.Store) + ds.TeamFunc = func(ctx context.Context, tid uint) (*fleet.Team, error) { + return &fleet.Team{ + Name: "Alamo", + ID: 1, + }, nil + } + + ds.TeamAgentOptionsFunc = func(ctx context.Context, teamID uint) (*json.RawMessage, error) { + return nil, nil + } ds.ListPacksForHostFunc = func(ctx context.Context, hid uint) ([]*fleet.Pack, error) { return []*fleet.Pack{}, nil } @@ -61,6 +71,30 @@ func TestGetClientConfig(t *testing.T) { return []*fleet.ScheduledQuery{}, nil } } + ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { + if opt.TeamID == nil { + return nil, nil + } + return []*fleet.Query{ + { + Query: "SELECT 1 FROM table_1", + Name: "Some strings carry more weight than others", + ScheduleInterval: 10, + Platform: "linux", + MinOsqueryVersion: "5.12.2", + LoggingType: "snapshot", + TeamID: ptr.Uint(1), + }, + { + Query: "SELECT 1 FROM table_2", + Name: "You shall not pass", + ScheduleInterval: 20, + Platform: "macos", + LoggingType: "differential", + TeamID: ptr.Uint(1), + }, + }, nil + } ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{AgentOptions: ptr.RawMessage(json.RawMessage(`{"config":{"options":{"baz":"bar"}}}`))}, nil } @@ -78,6 +112,7 @@ func TestGetClientConfig(t *testing.T) { ctx1 := hostctx.NewContext(ctx, &fleet.Host{ID: 1}) ctx2 := hostctx.NewContext(ctx, &fleet.Host{ID: 2}) + ctx3 := hostctx.NewContext(ctx, &fleet.Host{ID: 1, TeamID: ptr.Uint(1)}) expectedOptions := map[string]interface{}{ "baz": "bar", @@ -144,6 +179,43 @@ func TestGetClientConfig(t *testing.T) { }`, string(conf["packs"].(json.RawMessage)), ) + + // Check scheduled queries are loaded properly + conf, err = svc.GetClientConfig(ctx3) + require.NoError(t, err) + assert.JSONEq(t, `{ + "pack_by_label": { + "queries":{ + "time":{"query":"select * from time","interval":30,"removed":false} + } + }, + "pack_by_other_label": { + "queries": { + "foobar":{"query":"select 3","interval":20,"shard":42}, + "froobing":{"query":"select 'guacamole'","interval":60,"snapshot":true} + } + }, + "Team: Alamo": { + "queries": { + "Some strings carry more weight than others": { + "query": "SELECT 1 FROM table_1", + "interval": 10, + "platform": "linux", + "version": "5.12.2", + "snapshot": true + }, + "You shall not pass": { + "query": "SELECT 1 FROM table_2", + "interval": 20, + "platform": "macos", + "removed": true, + "version": "" + } + } + } + }`, + string(conf["packs"].(json.RawMessage)), + ) } func TestAgentOptionsForHost(t *testing.T) {