mirror of
https://github.com/fleetdm/fleet
synced 2026-04-21 13:37:30 +00:00
Added server_settings.query_report_cap (#19692)
#19600 - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [X] Added/updated tests - [X] Manual QA for all new/changed functionality
This commit is contained in:
Lucas Manuel Rodriguez
GitHub
parent
f62d5eda20
commit
904e8a6825
33 changed files with 178 additions and 91 deletions
1
changes/19600-add-config-to-set-query-report-cap
Normal file
1
changes/19600-add-config-to-set-query-report-cap
Normal file
|
|
@ -0,0 +1 @@
|
|||
* Added a server setting to configure the query repory cap size, `server_settings.query_report_cap` (default is 1000).
|
||||
|
|
@ -418,6 +418,7 @@ func TestFullGlobalGitOps(t *testing.T) {
|
|||
assert.Equal(t, orgName, savedAppConfig.OrgInfo.OrgName)
|
||||
assert.Equal(t, fleetServerURL, savedAppConfig.ServerSettings.ServerURL)
|
||||
assert.Contains(t, string(*savedAppConfig.AgentOptions), "distributed_denylist_duration")
|
||||
assert.Equal(t, 2000, savedAppConfig.ServerSettings.QueryReportCap)
|
||||
assert.Len(t, enrolledSecrets, 2)
|
||||
assert.True(t, policyDeleted)
|
||||
assert.Len(t, appliedPolicySpecs, 5)
|
||||
|
|
@ -923,7 +924,6 @@ team_settings:
|
|||
_ = runAppForTest(t, []string{"gitops", "-f", globalFile.Name(), "-f", teamFile.Name(), "--delete-other-teams"})
|
||||
assert.True(t, ds.ListTeamsFuncInvoked)
|
||||
assert.True(t, ds.DeleteTeamFuncInvoked)
|
||||
|
||||
}
|
||||
|
||||
func TestFullGlobalAndTeamGitOps(t *testing.T) {
|
||||
|
|
@ -1059,7 +1059,6 @@ func TestTeamSofwareInstallersGitOps(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, **fleet.Team) {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"server_settings": {
|
||||
"server_url": "",
|
||||
"live_query_disabled": false,
|
||||
"query_report_cap": 0,
|
||||
"query_reports_disabled": false,
|
||||
"enable_analytics": false,
|
||||
"deferred_save_host": false,
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ spec:
|
|||
deferred_save_host: false
|
||||
enable_analytics: false
|
||||
live_query_disabled: false
|
||||
query_report_cap: 0
|
||||
query_reports_disabled: false
|
||||
server_url: ""
|
||||
scripts_disabled: false
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"server_settings": {
|
||||
"server_url": "",
|
||||
"live_query_disabled": false,
|
||||
"query_report_cap": 0,
|
||||
"query_reports_disabled": false,
|
||||
"enable_analytics": false,
|
||||
"deferred_save_host": false,
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ spec:
|
|||
deferred_save_host: false
|
||||
enable_analytics: false
|
||||
live_query_disabled: false
|
||||
query_report_cap: 0
|
||||
query_reports_disabled: false
|
||||
server_url: ""
|
||||
scripts_disabled: false
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ org_settings:
|
|||
deferred_save_host: false
|
||||
enable_analytics: true
|
||||
live_query_disabled: false
|
||||
query_report_cap: 2000
|
||||
query_reports_disabled: false
|
||||
scripts_disabled: false
|
||||
server_url: $FLEET_SERVER_URL
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ spec:
|
|||
deferred_save_host: false
|
||||
enable_analytics: true
|
||||
live_query_disabled: false
|
||||
query_report_cap: 0
|
||||
query_reports_disabled: false
|
||||
server_url: https://example.org
|
||||
scripts_disabled: false
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ spec:
|
|||
deferred_save_host: false
|
||||
enable_analytics: true
|
||||
live_query_disabled: false
|
||||
query_report_cap: 0
|
||||
query_reports_disabled: false
|
||||
server_url: https://example.org
|
||||
scripts_disabled: false
|
||||
|
|
|
|||
|
|
@ -320,6 +320,7 @@ const DEFAULT_QUERY_REPORT_MOCK: IQueryReport = {
|
|||
},
|
||||
},
|
||||
],
|
||||
report_clipped: false,
|
||||
};
|
||||
|
||||
const createMockQueryReport = (
|
||||
|
|
|
|||
|
|
@ -9,4 +9,5 @@ export interface IQueryReportResultRow {
|
|||
export interface IQueryReport {
|
||||
query_id: number;
|
||||
results: IQueryReportResultRow[];
|
||||
report_clipped: boolean;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ import NoResults from "../components/NoResults/NoResults";
|
|||
import {
|
||||
DEFAULT_SORT_HEADER,
|
||||
DEFAULT_SORT_DIRECTION,
|
||||
QUERY_REPORT_RESULTS_LIMIT,
|
||||
} from "./QueryDetailsPageConfig";
|
||||
|
||||
interface IQueryDetailsPageProps {
|
||||
|
|
@ -199,8 +198,7 @@ const QueryDetailsPage = ({
|
|||
|
||||
const isLoading = isStoredQueryLoading || isQueryReportLoading;
|
||||
const isApiError = storedQueryError || queryReportError;
|
||||
const isClipped =
|
||||
(queryReport?.results?.length ?? 0) >= QUERY_REPORT_RESULTS_LIMIT;
|
||||
const isClipped = queryReport?.report_clipped;
|
||||
const disabledLiveQuery = config?.server_settings.live_query_disabled;
|
||||
|
||||
const renderHeader = () => {
|
||||
|
|
|
|||
|
|
@ -11,5 +11,3 @@ export type QueryDetailsPageQueryParams = Record<
|
|||
|
||||
export const DEFAULT_SORT_HEADER = "host_name";
|
||||
export const DEFAULT_SORT_DIRECTION = "asc";
|
||||
|
||||
export const QUERY_REPORT_RESULTS_LIMIT = 1000;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ describe("QueryReport", () => {
|
|||
columns: { col1: "value3", col2: "value4" },
|
||||
},
|
||||
],
|
||||
report_clipped: false,
|
||||
},
|
||||
];
|
||||
render(<QueryReport {...{ isClipped, queryReport }} />);
|
||||
|
|
@ -56,6 +57,7 @@ describe("QueryReport", () => {
|
|||
},
|
||||
},
|
||||
],
|
||||
report_clipped: false,
|
||||
},
|
||||
];
|
||||
render(<QueryReport {...{ isClipped, queryReport }} />);
|
||||
|
|
@ -83,6 +85,7 @@ describe("QueryReport", () => {
|
|||
columns: { col1: "value1", col2: "value2" },
|
||||
},
|
||||
],
|
||||
report_clipped: true,
|
||||
},
|
||||
];
|
||||
render(<QueryReport {...{ isClipped, queryReport }} />);
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ func TestValidGitOpsYaml(t *testing.T) {
|
|||
serverSettings, ok := gitops.OrgSettings["server_settings"]
|
||||
assert.True(t, ok, "server_settings not found")
|
||||
assert.Equal(t, "https://fleet.example.com", serverSettings.(map[string]interface{})["server_url"])
|
||||
assert.EqualValues(t, 2000, serverSettings.(map[string]interface{})["query_report_cap"])
|
||||
assert.Contains(t, gitops.OrgSettings, "org_info")
|
||||
orgInfo, ok := gitops.OrgSettings["org_info"].(map[string]interface{})
|
||||
assert.True(t, ok)
|
||||
|
|
|
|||
1
pkg/spec/testdata/global_config_no_paths.yml
vendored
1
pkg/spec/testdata/global_config_no_paths.yml
vendored
|
|
@ -101,6 +101,7 @@ org_settings:
|
|||
deferred_save_host: false
|
||||
enable_analytics: true
|
||||
live_query_disabled: false
|
||||
query_report_cap: 2000
|
||||
query_reports_disabled: false
|
||||
scripts_disabled: false
|
||||
server_url: https://fleet.example.com
|
||||
|
|
|
|||
1
pkg/spec/testdata/org-settings.yml
vendored
1
pkg/spec/testdata/org-settings.yml
vendored
|
|
@ -4,6 +4,7 @@ server_settings:
|
|||
deferred_save_host: false
|
||||
enable_analytics: true
|
||||
live_query_disabled: false
|
||||
query_report_cap: 2000
|
||||
query_reports_disabled: false
|
||||
scripts_disabled: false
|
||||
server_url: https://fleet.example.com
|
||||
|
|
|
|||
|
|
@ -4255,7 +4255,7 @@ func testHostsIncludesScheduledQueriesInPackStats(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage(json.RawMessage(`{"foo": "baz"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), queryResultRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), queryResultRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
hostResult, err = ds.Host(context.Background(), host.ID)
|
||||
|
|
@ -9067,7 +9067,7 @@ func testHostsAddToTeamCleansUpTeamQueryResults(t *testing.T, ds *Datastore) {
|
|||
h4Global0Results,
|
||||
h4Query1Results,
|
||||
} {
|
||||
err = ds.OverwriteQueryResultRows(ctx, results)
|
||||
err = ds.OverwriteQueryResultRows(ctx, results, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import (
|
|||
// OverwriteQueryResultRows overwrites the query result rows for a given query and host
|
||||
// in a single transaction, ensuring that the number of rows for the given query
|
||||
// does not exceed the maximum allowed
|
||||
func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) (err error) {
|
||||
func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) (err error) {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -31,7 +31,7 @@ func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet
|
|||
return ctxerr.Wrap(ctx, err, "counting existing query results")
|
||||
}
|
||||
|
||||
if countExisting >= fleet.MaxQueryReportRows {
|
||||
if countExisting >= maxQueryReportRows {
|
||||
// do not delete any rows if we are already at the limit
|
||||
return nil
|
||||
}
|
||||
|
|
@ -53,7 +53,7 @@ func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet
|
|||
|
||||
// Calculate how many new rows can be added given the maximum limit
|
||||
netRowsAfterDeletion := countExisting - int(countDeleted)
|
||||
allowedNewRows := fleet.MaxQueryReportRows - netRowsAfterDeletion
|
||||
allowedNewRows := maxQueryReportRows - netRowsAfterDeletion
|
||||
if allowedNewRows == 0 {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ func testGetQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), query1Rows)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), query1Rows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert Result Row for different Scheduled Query
|
||||
|
|
@ -76,7 +76,7 @@ func testGetQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
},
|
||||
}
|
||||
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), query2Rows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), query2Rows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
results, err := ds.QueryResultRows(context.Background(), query.ID, fleet.TeamFilter{User: test.UserAdmin})
|
||||
|
|
@ -125,7 +125,7 @@ func testGetQueryResultRowsForHost(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRows)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert 1 Result Row for Query1 Host2
|
||||
|
|
@ -137,7 +137,7 @@ func testGetQueryResultRowsForHost(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that Query1 returns 2 results for Host1
|
||||
|
|
@ -215,7 +215,7 @@ func testQueryResultRowsTeamFilter(t *testing.T, ds *Datastore) {
|
|||
},
|
||||
}
|
||||
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), globalRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), globalRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
teamRow := []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -229,7 +229,7 @@ func testQueryResultRowsTeamFilter(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), teamRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), teamRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
observerTeamRow := []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -243,7 +243,7 @@ func testQueryResultRowsTeamFilter(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), observerTeamRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), observerTeamRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
filter := fleet.TeamFilter{
|
||||
|
|
@ -286,7 +286,7 @@ func testCountResultsForQuery(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRow)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert Nil Result Row for Query1, nil data rows are not counted
|
||||
|
|
@ -298,7 +298,7 @@ func testCountResultsForQuery(t *testing.T, ds *Datastore) {
|
|||
Data: nil,
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert 5 Result Rows for Query2
|
||||
|
|
@ -317,7 +317,7 @@ func testCountResultsForQuery(t *testing.T, ds *Datastore) {
|
|||
resultRows = append(resultRows, resultRow2)
|
||||
}
|
||||
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), resultRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), resultRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that ResultCountForQuery returns 1
|
||||
|
|
@ -366,7 +366,7 @@ func testCountResultsForQueryAndHost(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRows)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), host1ResultRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
host1Query2 := []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -380,7 +380,7 @@ func testCountResultsForQueryAndHost(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host1Query2)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host1Query2, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
host2ResultRow := []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -394,7 +394,7 @@ func testCountResultsForQueryAndHost(t *testing.T, ds *Datastore) {
|
|||
}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host2ResultRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
host3ResultRow := []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -405,7 +405,7 @@ func testCountResultsForQueryAndHost(t *testing.T, ds *Datastore) {
|
|||
Data: nil,
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host3ResultRow)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), host3ResultRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that Query1 returns 2
|
||||
|
|
@ -451,7 +451,7 @@ func testOverwriteQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
},
|
||||
}
|
||||
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), initialRow)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), initialRow, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Overwrite Result Rows with new data
|
||||
|
|
@ -465,7 +465,7 @@ func testOverwriteQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
},
|
||||
}
|
||||
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that we get the overwritten data (1 result with USB Mouse data)
|
||||
|
|
@ -486,7 +486,7 @@ func testOverwriteQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that the data has not changed
|
||||
|
|
@ -511,7 +511,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
mockTime := time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// Generate max rows -1
|
||||
maxRows := fleet.MaxQueryReportRows - 1
|
||||
maxRows := fleet.DefaultMaxQueryReportRows - 1
|
||||
maxMinusOneRows := make([]*fleet.ScheduledQueryResultRow, maxRows)
|
||||
for i := 0; i < maxRows; i++ {
|
||||
maxMinusOneRows[i] = &fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -521,7 +521,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
}
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), maxMinusOneRows)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), maxMinusOneRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add an empty data rows which do not count towards the max
|
||||
|
|
@ -532,7 +532,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
LastFetched: mockTime,
|
||||
Data: nil,
|
||||
},
|
||||
})
|
||||
}, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Confirm that we can still add a row
|
||||
|
|
@ -543,13 +543,13 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
LastFetched: mockTime,
|
||||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
})
|
||||
}, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that we now have max rows
|
||||
count, err := ds.ResultCountForQuery(context.Background(), query.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fleet.MaxQueryReportRows, count)
|
||||
require.Equal(t, fleet.DefaultMaxQueryReportRows, count)
|
||||
|
||||
// Attempt to add another row
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), []*fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -559,7 +559,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
LastFetched: mockTime,
|
||||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
})
|
||||
}, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that the last row was not added
|
||||
|
|
@ -568,7 +568,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
require.Len(t, host4result, 0)
|
||||
|
||||
// Generate more than max rows in Query 2
|
||||
rows := fleet.MaxQueryReportRows + 50
|
||||
rows := fleet.DefaultMaxQueryReportRows + 50
|
||||
largeBatchRows := make([]*fleet.ScheduledQueryResultRow, rows)
|
||||
for i := 0; i < rows; i++ {
|
||||
largeBatchRows[i] = &fleet.ScheduledQueryResultRow{
|
||||
|
|
@ -578,13 +578,13 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
}
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), largeBatchRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), largeBatchRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Confirm only max rows are stored for the queryID
|
||||
allResults, err := ds.QueryResultRowsForHost(context.Background(), query2.ID, host1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, allResults, fleet.MaxQueryReportRows)
|
||||
require.Len(t, allResults, fleet.DefaultMaxQueryReportRows)
|
||||
|
||||
// Confirm that new rows are not added when the max is reached
|
||||
newMockTime := mockTime.Add(2 * time.Minute)
|
||||
|
|
@ -597,7 +597,7 @@ func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) {
|
|||
},
|
||||
}
|
||||
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
host2Results, err := ds.QueryResultRowsForHost(context.Background(), query2.ID, host2.ID)
|
||||
|
|
@ -619,7 +619,7 @@ func testQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "USB Mouse", "vendor": "Logitech"}`)),
|
||||
},
|
||||
}
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), overwriteRows)
|
||||
err := ds.OverwriteQueryResultRows(context.Background(), overwriteRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
filter := fleet.TeamFilter{User: user, IncludeObserver: true}
|
||||
|
|
@ -655,7 +655,7 @@ func testCleanupQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "Keyboard", "vendor": "Microsoft"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), rows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), rows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Call OverwriteQueryResultRows again with different rows
|
||||
|
|
@ -673,7 +673,7 @@ func testCleanupQueryResultRows(t *testing.T, ds *Datastore) {
|
|||
Data: ptr.RawMessage([]byte(`{"model": "Speakers", "vendor": "Bose"}`)),
|
||||
},
|
||||
}
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows)
|
||||
err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup query result rows
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ CREATE TABLE `app_config_json` (
|
|||
UNIQUE KEY `id` (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false, \"enable_release_device_manually\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"ai_features_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"activities_webhook\": {\"destination_url\": \"\", \"enable_activities_webhook\": false}, \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}, \"activity_expiry_settings\": {\"activity_expiry_window\": 0, \"activity_expiry_enabled\": false}}','2020-01-01 01:01:01','2020-01-01 01:01:01');
|
||||
INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false, \"enable_release_device_manually\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"query_report_cap\": 0, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"ai_features_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"activities_webhook\": {\"destination_url\": \"\", \"enable_activities_webhook\": false}, \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}, \"activity_expiry_settings\": {\"activity_expiry_window\": 0, \"activity_expiry_enabled\": false}}','2020-01-01 01:01:01','2020-01-01 01:01:01');
|
||||
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
||||
/*!40101 SET character_set_client = utf8 */;
|
||||
CREATE TABLE `calendar_events` (
|
||||
|
|
|
|||
|
|
@ -888,6 +888,16 @@ type ServerSettings struct {
|
|||
QueryReportsDisabled bool `json:"query_reports_disabled"`
|
||||
ScriptsDisabled bool `json:"scripts_disabled"`
|
||||
AIFeaturesDisabled bool `json:"ai_features_disabled"`
|
||||
QueryReportCap int `json:"query_report_cap"`
|
||||
}
|
||||
|
||||
const DefaultMaxQueryReportRows int = 1000
|
||||
|
||||
func (f *ServerSettings) GetQueryReportCap() int {
|
||||
if f.QueryReportCap <= 0 {
|
||||
return DefaultMaxQueryReportRows
|
||||
}
|
||||
return f.QueryReportCap
|
||||
}
|
||||
|
||||
// HostExpirySettings contains settings pertaining to automatic host expiry.
|
||||
|
|
|
|||
|
|
@ -457,7 +457,7 @@ type Datastore interface {
|
|||
QueryResultRowsForHost(ctx context.Context, queryID, hostID uint) ([]*ScheduledQueryResultRow, error)
|
||||
ResultCountForQuery(ctx context.Context, queryID uint) (int, error)
|
||||
ResultCountForQueryAndHost(ctx context.Context, queryID, hostID uint) (int, error)
|
||||
OverwriteQueryResultRows(ctx context.Context, rows []*ScheduledQueryResultRow) error
|
||||
OverwriteQueryResultRows(ctx context.Context, rows []*ScheduledQueryResultRow, maxQueryReportRows int) error
|
||||
// CleanupDiscardedQueryResults deletes all query results for queries with DiscardData enabled.
|
||||
// Used in cleanups_then_aggregation cron to cleanup rows that were inserted immediately
|
||||
// after DiscardData was set to true due to query caching.
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ type Stats struct {
|
|||
|
||||
const (
|
||||
// StatusOK is the success code returned by osquery
|
||||
StatusOK OsqueryStatus = 0
|
||||
MaxQueryReportRows int = 1000
|
||||
StatusOK OsqueryStatus = 0
|
||||
)
|
||||
|
||||
// QueryContent is the format of a query stanza in an osquery configuration.
|
||||
|
|
|
|||
|
|
@ -275,12 +275,13 @@ type Service interface {
|
|||
// included in the results.
|
||||
ListQueries(ctx context.Context, opt ListOptions, teamID *uint, scheduled *bool, mergeInherited bool) ([]*Query, error)
|
||||
GetQuery(ctx context.Context, id uint) (*Query, error)
|
||||
// GetQueryReportResults returns all the stored results of a query for hosts the requestor has access to
|
||||
GetQueryReportResults(ctx context.Context, id uint) ([]HostQueryResultRow, error)
|
||||
// GetQueryReportResults returns all the stored results of a query for hosts the requestor has access to.
|
||||
// Returns a boolean indicating whether the report is clipped.
|
||||
GetQueryReportResults(ctx context.Context, id uint) ([]HostQueryResultRow, bool, error)
|
||||
// GetHostQueryReportResults returns all stored results of a query for a specific host
|
||||
GetHostQueryReportResults(ctx context.Context, hid uint, queryID uint) (rows []HostQueryReportResult, lastFetched *time.Time, err error)
|
||||
// QueryReportIsClipped returns true if the number of query report rows exceeds the maximum
|
||||
QueryReportIsClipped(ctx context.Context, queryID uint) (bool, error)
|
||||
QueryReportIsClipped(ctx context.Context, queryID uint, maxQueryReportRows int) (bool, error)
|
||||
NewQuery(ctx context.Context, p QueryPayload) (*Query, error)
|
||||
ModifyQuery(ctx context.Context, id uint, p QueryPayload) (*Query, error)
|
||||
DeleteQuery(ctx context.Context, teamID *uint, name string) error
|
||||
|
|
|
|||
|
|
@ -339,7 +339,7 @@ type ResultCountForQueryFunc func(ctx context.Context, queryID uint) (int, error
|
|||
|
||||
type ResultCountForQueryAndHostFunc func(ctx context.Context, queryID uint, hostID uint) (int, error)
|
||||
|
||||
type OverwriteQueryResultRowsFunc func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error
|
||||
type OverwriteQueryResultRowsFunc func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error
|
||||
|
||||
type CleanupDiscardedQueryResultsFunc func(ctx context.Context) error
|
||||
|
||||
|
|
@ -3508,11 +3508,11 @@ func (s *DataStore) ResultCountForQueryAndHost(ctx context.Context, queryID uint
|
|||
return s.ResultCountForQueryAndHostFunc(ctx, queryID, hostID)
|
||||
}
|
||||
|
||||
func (s *DataStore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
func (s *DataStore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
s.mu.Lock()
|
||||
s.OverwriteQueryResultRowsFuncInvoked = true
|
||||
s.mu.Unlock()
|
||||
return s.OverwriteQueryResultRowsFunc(ctx, rows)
|
||||
return s.OverwriteQueryResultRowsFunc(ctx, rows, maxQueryReportRows)
|
||||
}
|
||||
|
||||
func (s *DataStore) CleanupDiscardedQueryResults(ctx context.Context) error {
|
||||
|
|
|
|||
|
|
@ -1231,7 +1231,12 @@ func getHostQueryReportEndpoint(ctx context.Context, request interface{}, svc fl
|
|||
return getHostQueryReportResponse{Err: err}, nil
|
||||
}
|
||||
|
||||
isClipped, err := svc.QueryReportIsClipped(ctx, req.QueryID)
|
||||
appConfig, err := svc.AppConfigObfuscated(ctx)
|
||||
if err != nil {
|
||||
return getHostQueryReportResponse{Err: err}, nil
|
||||
}
|
||||
|
||||
isClipped, err := svc.QueryReportIsClipped(ctx, req.QueryID, appConfig.ServerSettings.GetQueryReportCap())
|
||||
if err != nil {
|
||||
return getHostQueryReportResponse{Err: err}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10837,12 +10837,14 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
}, http.StatusOK, &applyResp)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 0)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// Re-add results to our query and check that they're actually there
|
||||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 1)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// don't change platform or min_osquery_version and results should not be deleted
|
||||
s.DoJSON("POST", "/api/latest/fleet/spec/queries", applyQuerySpecsRequest{
|
||||
|
|
@ -10850,6 +10852,7 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
}, http.StatusOK, &applyResp)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 1)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// now update the platform and results should be deleted.
|
||||
osqueryInfoQuerySpec.Platform = "darwin"
|
||||
|
|
@ -10858,30 +10861,35 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
}, http.StatusOK, &applyResp)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 0)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// Update logging type, which should cause results deletion
|
||||
s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/queries/%d", usbDevicesQuery.ID), modifyQueryRequest{ID: usbDevicesQuery.ID, QueryPayload: fleet.QueryPayload{Logging: &fleet.LoggingDifferential}}, http.StatusOK, &modifyQueryResp)
|
||||
require.Equal(t, fleet.LoggingDifferential, modifyQueryResp.Query.Logging)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", usbDevicesQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 0)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// Re-add results to our query and check that they're actually there
|
||||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 1)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
discardData := true
|
||||
s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/queries/%d", osqueryInfoQuery.ID), modifyQueryRequest{ID: osqueryInfoQuery.ID, QueryPayload: fleet.QueryPayload{DiscardData: &discardData}}, http.StatusOK, &modifyQueryResp)
|
||||
require.True(t, modifyQueryResp.Query.DiscardData)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 0)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// check that now that discardData is set, we don't add new results
|
||||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, 0)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
// Verify that we can't have more than 1k results
|
||||
|
||||
|
|
@ -10893,7 +10901,7 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
NodeKey: *host1Global.NodeKey,
|
||||
LogType: "result",
|
||||
Data: json.RawMessage(`[{
|
||||
"snapshot": [` + results(1000, host1Global.UUID) + `
|
||||
"snapshot": [` + results(fleet.DefaultMaxQueryReportRows, host1Global.UUID) + `
|
||||
],
|
||||
"action": "snapshot",
|
||||
"name": "pack/Global/` + osqueryInfoQuery.Name + `",
|
||||
|
|
@ -10916,13 +10924,14 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, fleet.MaxQueryReportRows)
|
||||
require.Len(t, gqrr.Results, fleet.DefaultMaxQueryReportRows)
|
||||
require.True(t, gqrr.ReportClipped)
|
||||
|
||||
ghqrr = getHostQueryReportResponse{}
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/queries/%d", host1Global.ID, osqueryInfoQuery.ID), getHostQueryReportRequest{}, http.StatusOK, &ghqrr)
|
||||
require.NoError(t, ghqrr.Err)
|
||||
require.Len(t, ghqrr.Results, fleet.DefaultMaxQueryReportRows)
|
||||
require.True(t, ghqrr.ReportClipped)
|
||||
require.Len(t, ghqrr.Results, fleet.MaxQueryReportRows)
|
||||
|
||||
slreq.Data = json.RawMessage(`[{
|
||||
"snapshot": [` + results(1, host1Global.UUID) + `
|
||||
|
|
@ -10944,7 +10953,41 @@ func (s *integrationTestSuite) TestQueryReports() {
|
|||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, fleet.MaxQueryReportRows)
|
||||
require.Len(t, gqrr.Results, fleet.DefaultMaxQueryReportRows)
|
||||
require.True(t, gqrr.ReportClipped)
|
||||
|
||||
appConfigSpec := map[string]map[string]int{
|
||||
"server_settings": {"query_report_cap": fleet.DefaultMaxQueryReportRows + 1},
|
||||
}
|
||||
s.Do("PATCH", "/api/latest/fleet/config", appConfigSpec, http.StatusOK)
|
||||
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, fleet.DefaultMaxQueryReportRows)
|
||||
require.False(t, gqrr.ReportClipped)
|
||||
|
||||
slreq.Data = json.RawMessage(`[{
|
||||
"snapshot": [` + results(1002, host1Global.UUID) + `
|
||||
],
|
||||
"action": "snapshot",
|
||||
"name": "pack/Global/` + osqueryInfoQuery.Name + `",
|
||||
"hostIdentifier": "` + *host1Global.OsqueryHostID + `",
|
||||
"calendarTime": "Fri Oct 6 18:13:04 2023 UTC",
|
||||
"unixTime": 1696615984,
|
||||
"epoch": 0,
|
||||
"counter": 0,
|
||||
"numerics": false,
|
||||
"decorations": {
|
||||
"host_uuid": "187c4d56-8e45-1a9d-8513-ac17efd2f0fd",
|
||||
"hostname": "` + host1Global.Hostname + `"
|
||||
}
|
||||
}]`)
|
||||
|
||||
s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres)
|
||||
require.NoError(t, slres.Err)
|
||||
|
||||
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr)
|
||||
require.Len(t, gqrr.Results, fleet.DefaultMaxQueryReportRows+1)
|
||||
require.True(t, gqrr.ReportClipped)
|
||||
|
||||
// TODO: Set global discard flag and verify that all data is gone.
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1799,7 +1799,8 @@ func (svc *Service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage
|
|||
|
||||
unmarshaledResults, queriesDBData := svc.preProcessOsqueryResults(ctx, logs, queryReportsDisabled)
|
||||
if !queryReportsDisabled {
|
||||
svc.saveResultLogsToQueryReports(ctx, unmarshaledResults, queriesDBData)
|
||||
maxQueryReportRows := appConfig.ServerSettings.GetQueryReportCap()
|
||||
svc.saveResultLogsToQueryReports(ctx, unmarshaledResults, queriesDBData, maxQueryReportRows)
|
||||
}
|
||||
|
||||
var filteredLogs []json.RawMessage
|
||||
|
|
@ -1861,7 +1862,12 @@ func (svc *Service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage
|
|||
// Query Reports
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func (svc *Service) saveResultLogsToQueryReports(ctx context.Context, unmarshaledResults []*fleet.ScheduledQueryResult, queriesDBData map[string]*fleet.Query) {
|
||||
func (svc *Service) saveResultLogsToQueryReports(
|
||||
ctx context.Context,
|
||||
unmarshaledResults []*fleet.ScheduledQueryResult,
|
||||
queriesDBData map[string]*fleet.Query,
|
||||
maxQueryReportRows int,
|
||||
) {
|
||||
// skipauth: Authorization is currently for user endpoints only.
|
||||
svc.authz.SkipAuthorization(ctx)
|
||||
|
||||
|
|
@ -1903,11 +1909,11 @@ func (svc *Service) saveResultLogsToQueryReports(ctx context.Context, unmarshale
|
|||
level.Error(svc.logger).Log("msg", "get result count for query", "err", err, "query_id", dbQuery.ID)
|
||||
continue
|
||||
}
|
||||
if count >= fleet.MaxQueryReportRows {
|
||||
if count >= maxQueryReportRows {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := svc.overwriteResultRows(ctx, result, dbQuery.ID, host.ID); err != nil {
|
||||
if err := svc.overwriteResultRows(ctx, result, dbQuery.ID, host.ID, maxQueryReportRows); err != nil {
|
||||
level.Error(svc.logger).Log("msg", "overwrite results", "err", err, "query_id", dbQuery.ID, "host_id", host.ID)
|
||||
continue
|
||||
}
|
||||
|
|
@ -1919,7 +1925,7 @@ func (svc *Service) saveResultLogsToQueryReports(ctx context.Context, unmarshale
|
|||
// The "snapshot" array in a ScheduledQueryResult can contain multiple rows.
|
||||
// Each row is saved as a separate ScheduledQueryResultRow, i.e. a result could contain
|
||||
// many USB Devices or a result could contain all user accounts on a host.
|
||||
func (svc *Service) overwriteResultRows(ctx context.Context, result *fleet.ScheduledQueryResult, queryID, hostID uint) error {
|
||||
func (svc *Service) overwriteResultRows(ctx context.Context, result *fleet.ScheduledQueryResult, queryID, hostID uint, maxQueryReportRows int) error {
|
||||
fetchTime := time.Now()
|
||||
|
||||
rows := make([]*fleet.ScheduledQueryResultRow, 0, len(result.Snapshot))
|
||||
|
|
@ -1945,7 +1951,7 @@ func (svc *Service) overwriteResultRows(ctx context.Context, result *fleet.Sched
|
|||
rows = append(rows, row)
|
||||
}
|
||||
|
||||
if err := svc.ds.OverwriteQueryResultRows(ctx, rows); err != nil {
|
||||
if err := svc.ds.OverwriteQueryResultRows(ctx, rows, maxQueryReportRows); err != nil {
|
||||
return ctxerr.Wrap(ctx, err, "overwriting query result rows")
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -614,7 +614,7 @@ func TestSubmitResultLogsToLogDestination(t *testing.T) {
|
|||
return 0, nil
|
||||
}
|
||||
teamQueryResultsStored := false
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -766,7 +766,7 @@ func TestSaveResultLogsToQueryReports(t *testing.T) {
|
|||
Logging: fleet.LoggingSnapshot,
|
||||
},
|
||||
}
|
||||
serv.saveResultLogsToQueryReports(ctx, results, discardDataFalse)
|
||||
serv.saveResultLogsToQueryReports(ctx, results, discardDataFalse, fleet.DefaultMaxQueryReportRows)
|
||||
assert.False(t, ds.OverwriteQueryResultRowsFuncInvoked)
|
||||
|
||||
// Happy Path: Results saved
|
||||
|
|
@ -777,13 +777,13 @@ func TestSaveResultLogsToQueryReports(t *testing.T) {
|
|||
Logging: fleet.LoggingSnapshot,
|
||||
},
|
||||
}
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
return nil
|
||||
}
|
||||
ds.ResultCountForQueryFunc = func(ctx context.Context, queryID uint) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
serv.saveResultLogsToQueryReports(ctx, results, discardDataTrue)
|
||||
serv.saveResultLogsToQueryReports(ctx, results, discardDataTrue, fleet.DefaultMaxQueryReportRows)
|
||||
require.True(t, ds.OverwriteQueryResultRowsFuncInvoked)
|
||||
}
|
||||
|
||||
|
|
@ -825,7 +825,7 @@ func TestSubmitResultLogsToQueryResultsWithEmptySnapShot(t *testing.T) {
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
require.Len(t, rows, 1)
|
||||
require.Equal(t, uint(999), rows[0].HostID)
|
||||
require.NotZero(t, rows[0].LastFetched)
|
||||
|
|
@ -876,7 +876,7 @@ func TestSubmitResultLogsToQueryResultsDoesNotCountNullDataRows(t *testing.T) {
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
require.Len(t, rows, 1)
|
||||
require.Equal(t, uint(999), rows[0].HostID)
|
||||
require.NotZero(t, rows[0].LastFetched)
|
||||
|
|
@ -933,7 +933,7 @@ func TestSubmitResultLogsFail(t *testing.T) {
|
|||
ds.ResultCountForQueryFunc = func(ctx context.Context, queryID uint) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error {
|
||||
ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow, maxQueryReportRows int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -121,16 +121,17 @@ type getQueryReportRequest struct {
|
|||
}
|
||||
|
||||
type getQueryReportResponse struct {
|
||||
QueryID uint `json:"query_id"`
|
||||
Results []fleet.HostQueryResultRow `json:"results"`
|
||||
Err error `json:"error,omitempty"`
|
||||
QueryID uint `json:"query_id"`
|
||||
Results []fleet.HostQueryResultRow `json:"results"`
|
||||
ReportClipped bool `json:"report_clipped"`
|
||||
Err error `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (r getQueryReportResponse) error() error { return r.Err }
|
||||
|
||||
func getQueryReportEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
|
||||
req := request.(*getQueryReportRequest)
|
||||
queryReportResults, err := svc.GetQueryReportResults(ctx, req.ID)
|
||||
queryReportResults, reportClipped, err := svc.GetQueryReportResults(ctx, req.ID)
|
||||
if err != nil {
|
||||
return listQueriesResponse{Err: err}, nil
|
||||
}
|
||||
|
|
@ -140,44 +141,53 @@ func getQueryReportEndpoint(ctx context.Context, request interface{}, svc fleet.
|
|||
results = queryReportResults
|
||||
}
|
||||
return getQueryReportResponse{
|
||||
QueryID: req.ID,
|
||||
Results: results,
|
||||
QueryID: req.ID,
|
||||
Results: results,
|
||||
ReportClipped: reportClipped,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (svc *Service) GetQueryReportResults(ctx context.Context, id uint) ([]fleet.HostQueryResultRow, error) {
|
||||
func (svc *Service) GetQueryReportResults(ctx context.Context, id uint) ([]fleet.HostQueryResultRow, bool, error) {
|
||||
// Load query first to get its teamID.
|
||||
query, err := svc.ds.Query(ctx, id)
|
||||
if err != nil {
|
||||
setAuthCheckedOnPreAuthErr(ctx)
|
||||
return nil, ctxerr.Wrap(ctx, err, "get query from datastore")
|
||||
return nil, false, ctxerr.Wrap(ctx, err, "get query from datastore")
|
||||
}
|
||||
if err := svc.authz.Authorize(ctx, query, fleet.ActionRead); err != nil {
|
||||
return nil, err
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
if query.DiscardData {
|
||||
return nil, nil
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
vc, ok := viewer.FromContext(ctx)
|
||||
if !ok {
|
||||
return nil, fleet.ErrNoContext
|
||||
return nil, false, fleet.ErrNoContext
|
||||
}
|
||||
filter := fleet.TeamFilter{User: vc.User, IncludeObserver: true}
|
||||
|
||||
queryReportResultRows, err := svc.ds.QueryResultRows(ctx, id, filter)
|
||||
if err != nil {
|
||||
return nil, ctxerr.Wrap(ctx, err, "get query report results")
|
||||
return nil, false, ctxerr.Wrap(ctx, err, "get query report results")
|
||||
}
|
||||
queryReportResults, err := fleet.MapQueryReportResultsToRows(queryReportResultRows)
|
||||
if err != nil {
|
||||
return nil, ctxerr.Wrap(ctx, err, "map db rows to results")
|
||||
return nil, false, ctxerr.Wrap(ctx, err, "map db rows to results")
|
||||
}
|
||||
return queryReportResults, nil
|
||||
appConfig, err := svc.ds.AppConfig(ctx)
|
||||
if err != nil {
|
||||
return nil, false, ctxerr.Wrap(ctx, err, "get app config")
|
||||
}
|
||||
reportClipped, err := svc.QueryReportIsClipped(ctx, id, appConfig.ServerSettings.GetQueryReportCap())
|
||||
if err != nil {
|
||||
return nil, false, ctxerr.Wrap(ctx, err, "check query report is clipped")
|
||||
}
|
||||
return queryReportResults, reportClipped, nil
|
||||
}
|
||||
|
||||
func (svc *Service) QueryReportIsClipped(ctx context.Context, queryID uint) (bool, error) {
|
||||
func (svc *Service) QueryReportIsClipped(ctx context.Context, queryID uint, maxQueryReportRows int) (bool, error) {
|
||||
query, err := svc.ds.Query(ctx, queryID)
|
||||
if err != nil {
|
||||
setAuthCheckedOnPreAuthErr(ctx)
|
||||
|
|
@ -191,7 +201,7 @@ func (svc *Service) QueryReportIsClipped(ctx context.Context, queryID uint) (boo
|
|||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count >= fleet.MaxQueryReportRows, nil
|
||||
return count >= maxQueryReportRows, nil
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
|
|
|||
|
|
@ -644,7 +644,7 @@ func TestQueryAuth(t *testing.T) {
|
|||
_, err = svc.GetQuery(ctx, tt.qid)
|
||||
checkAuthErr(t, tt.shouldFailRead, err)
|
||||
|
||||
_, err = svc.QueryReportIsClipped(ctx, tt.qid)
|
||||
_, err = svc.QueryReportIsClipped(ctx, tt.qid, fleet.DefaultMaxQueryReportRows)
|
||||
checkAuthErr(t, tt.shouldFailRead, err)
|
||||
|
||||
_, err = svc.ListQueries(ctx, fleet.ListOptions{}, query.TeamID, nil, false)
|
||||
|
|
@ -688,15 +688,15 @@ func TestQueryReportIsClipped(t *testing.T) {
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
isClipped, err := svc.QueryReportIsClipped(viewerCtx, 1)
|
||||
isClipped, err := svc.QueryReportIsClipped(viewerCtx, 1, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
require.False(t, isClipped)
|
||||
|
||||
ds.ResultCountForQueryFunc = func(ctx context.Context, queryID uint) (int, error) {
|
||||
return fleet.MaxQueryReportRows, nil
|
||||
return fleet.DefaultMaxQueryReportRows, nil
|
||||
}
|
||||
|
||||
isClipped, err = svc.QueryReportIsClipped(viewerCtx, 1)
|
||||
isClipped, err = svc.QueryReportIsClipped(viewerCtx, 1, fleet.DefaultMaxQueryReportRows)
|
||||
require.NoError(t, err)
|
||||
require.True(t, isClipped)
|
||||
}
|
||||
|
|
@ -725,9 +725,10 @@ func TestQueryReportReturnsNilIfDiscardDataIsTrue(t *testing.T) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
results, err := svc.GetQueryReportResults(viewerCtx, 1)
|
||||
results, reportClipped, err := svc.GetQueryReportResults(viewerCtx, 1)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, results)
|
||||
require.False(t, reportClipped)
|
||||
}
|
||||
|
||||
func TestComparePlatforms(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ github.com/fleetdm/fleet/v4/server/fleet/ServerSettings DeferredSaveHost bool
|
|||
github.com/fleetdm/fleet/v4/server/fleet/ServerSettings QueryReportsDisabled bool
|
||||
github.com/fleetdm/fleet/v4/server/fleet/ServerSettings ScriptsDisabled bool
|
||||
github.com/fleetdm/fleet/v4/server/fleet/ServerSettings AIFeaturesDisabled bool
|
||||
github.com/fleetdm/fleet/v4/server/fleet/ServerSettings QueryReportCap int
|
||||
github.com/fleetdm/fleet/v4/server/fleet/AppConfig SMTPSettings *fleet.SMTPSettings
|
||||
github.com/fleetdm/fleet/v4/server/fleet/SMTPSettings SMTPEnabled bool
|
||||
github.com/fleetdm/fleet/v4/server/fleet/SMTPSettings SMTPConfigured bool
|
||||
|
|
|
|||
Loading…
Reference in a new issue