From f89d78d065de18d3af62d223359924b35c6bdad9 Mon Sep 17 00:00:00 2001 From: Luke Heath Date: Tue, 5 Dec 2023 17:03:04 -0600 Subject: [PATCH 01/60] Update air guitar process (#15451) Now that Mike is not attending all design reviews, we are going to assign the finished air guitar issue to Noah, and he will bring to a review session with Mike. --- handbook/company/product-groups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handbook/company/product-groups.md b/handbook/company/product-groups.md index 0d37ae6698..5008d3086e 100644 --- a/handbook/company/product-groups.md +++ b/handbook/company/product-groups.md @@ -218,7 +218,7 @@ Anyone in the product group can initiate an air guitar session. 5. Document: Summarize the learnings, decisions, and next steps in the user story issue. -6. Decide: Bring the issue to a design review to determine an outcome: +6. Decide: Assign the issue to the Head of Product Design to determine an outcome: 1. Move forward with the formal drafting process leading to engineering. 2. Keep it open for future consideration. 3. Discard if it is invalidated through the process. From d40555e7cd9e41678f99e8167b1726b0f12b17db Mon Sep 17 00:00:00 2001 From: Sharon Katz <121527325+sharon-fdm@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:21:12 -0500 Subject: [PATCH 02/60] Script for comparing two CIS PDF files (#15307) --- tools/cis/CIS-Benchmark-diff.py | 130 ++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 tools/cis/CIS-Benchmark-diff.py diff --git a/tools/cis/CIS-Benchmark-diff.py b/tools/cis/CIS-Benchmark-diff.py new file mode 100755 index 0000000000..b12bfb40b3 --- /dev/null +++ b/tools/cis/CIS-Benchmark-diff.py @@ -0,0 +1,130 @@ +# This script takes two CIS Benchmark PDFs as input and diffs them +# For example: It will generate a diff of the Win10 & W11 benchmarks +# Requires installation of the PyMuPDF dep (pip3 install PyMuPDF). +# cmd line example: Python3 ./CIS-Benchmark-diff.py File1.pdf File2.pdf + +import fitz # PyMuPDF +import re +import difflib +import sys +from datetime import datetime + +def is_start_of_new_item(line): + """ + Check if a line starts with a number pattern like '1', '1.1', up to '100.7.32'. + """ + return bool(re.match(r'\d{1,3}(?:\.\d{1,2}){0,2}', line.strip())) + +def remove_trailing_whitespace(text): + """ + Remove trailing whitespace from each line in the text. + """ + return '\n'.join(line.rstrip() for line in text.split('\n')) + +def correct_word_wrapping(text): + """ + Correct word wrapping issues in the extracted text. + Each line should start with a number pattern from '1' to '100.7.32'. + """ + lines = text.split('\n') + corrected_lines = [] + for line in lines: + if corrected_lines and not is_start_of_new_item(line): + # Append this line to the previous one + corrected_lines[-1] += ' ' + line + else: + corrected_lines.append(line) + return '\n'.join(corrected_lines) + +def extract_recommendations_fitz(pdf_path, start_phrase, end_phrase): + """ + Extract a specific section from a PDF file. + """ + doc = fitz.open(pdf_path) + recommendations = "" + capture = False + + for page in doc: + text_blocks = page.get_text("blocks") + for block in text_blocks: + block_text = block[4].strip() # Extract text from the block + if block_text: + # Check for the start and end of the section + if start_phrase in block_text and not capture: + capture = True + elif end_phrase in block_text and capture: + capture = False + break + + if capture: + recommendations += block_text + "\n" + + # Cleanup process + recommendations_cleaned = re.sub(r'Page\s+\d{1,3}', '', recommendations) # Remove "Page " lines + recommendations_cleaned = re.sub(r'\.{2,}\s*\d+', '', recommendations_cleaned) # Remove periods followed by page numbers + recommendations_cleaned = re.sub(r'\s+\d{2,4}\s*$', '', recommendations_cleaned, flags=re.MULTILINE) # Remove 2 to 4 digit numbers at the end of lines + recommendations_corrected = correct_word_wrapping(recommendations_cleaned) # Correct word wrapping + final_recommendations = remove_trailing_whitespace(recommendations_corrected) # Remove trailing whitespace + + return final_recommendations + +def create_custom_diff(text1, text2): + """ + Create a custom diff of two texts with custom labels. + """ + text1_lines = text1.splitlines() + text2_lines = text2.splitlines() + + # Generate a diff without additional context lines + diff = difflib.unified_diff(text1_lines, text2_lines, lineterm='', + fromfile='file1', tofile='file2', + n=0) # 'n=0' for no context lines + + # Customizing diff output to replace '+' and '-' with 'file1' and 'file2' + custom_diff = [] + for line in diff: + if line.startswith('-'): + custom_diff.append('file1: ' + line[1:]) + elif line.startswith('+'): + custom_diff.append('file2: ' + line[1:]) + else: + custom_diff.append(line) + + return '\n'.join(custom_diff) + +def main(file1, file2): + # Start and end phrases for the extraction + start_phrase = "Recommendations ..." + end_phrase = "Appendix: Summary Table ..." + + # Extract recommendations from both PDFs + recommendations_file1 = extract_recommendations_fitz(file1, start_phrase, end_phrase) + recommendations_file2 = extract_recommendations_fitz(file2, start_phrase, end_phrase) + + # Write the cleaned and corrected data to a file + with open('cleaned.txt', 'w') as file: + file.write("Cleaned Data from file 1 PDF:\n\n") + file.write(recommendations_file1) + file.write("\n\nCleaned Data from file 2 PDF:\n\n") + file.write(recommendations_file2) + print("Cleaned data file created: cleaned.txt") + + # Perform the custom diff + diff_result = create_custom_diff(recommendations_file1, recommendations_file2) + + # Write the diff result to a file with a timestamp + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with open('cis_benchmarks_diff.txt', 'w') as file: + file.write(f"Diff generated on: {timestamp}\n\n") + file.write(diff_result) + print("Diff file created: cis_benchmarks_diff.txt") + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python script.py ") + sys.exit(1) + + file1 = sys.argv[1] + file2 = sys.argv[2] + main(file1, file2) + From f19dc8abe040084176352e573b5955f90a4d6d17 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 6 Dec 2023 08:30:49 -0600 Subject: [PATCH 03/60] Add `GET software/versions` and `GET software/versions/:id` endpoints (#15450) --- changes/15229-list-software-versions | 4 + cmd/fleetctl/get_test.go | 10 ++ server/datastore/mysql/software.go | 2 + server/fleet/software.go | 2 +- server/service/client_software.go | 4 +- server/service/handler.go | 6 + server/service/integration_core_test.go | 157 +++++++++++++++++- server/service/integration_enterprise_test.go | 28 ++++ server/service/software.go | 43 +++++ 9 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 changes/15229-list-software-versions diff --git a/changes/15229-list-software-versions b/changes/15229-list-software-versions new file mode 100644 index 0000000000..bec06e2f8e --- /dev/null +++ b/changes/15229-list-software-versions @@ -0,0 +1,4 @@ +- Added `GET software/versions` endpoint to list and filter software versions. +- Added `GET software/versions/{id}` endpoint to get a specific software version. +- Deprecated `GET software` endpoint. +- Deprecated `GET software/{id}` endpoint. diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index dbbe953bb3..88e4aa70d2 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -623,6 +623,10 @@ func TestGetSoftware(t *testing.T) { return []fleet.Software{foo001, foo002, foo003, bar003}, nil } + ds.CountSoftwareFunc = func(ctx context.Context, opt fleet.SoftwareListOptions) (int, error) { + return 4, nil + } + expected := `+------+---------+-------------------+--------------------------+-----------+ | NAME | VERSION | SOURCE | CPE | # OF CVES | +------+---------+-------------------+--------------------------+-----------+ @@ -644,6 +648,7 @@ spec: id: 0 name: foo source: chrome_extensions + browser: "" version: 0.0.1 vulnerabilities: - cve: cve-321-432-543 @@ -662,6 +667,7 @@ spec: id: 0 name: foo source: chrome_extensions + browser: "" version: 0.0.3 vulnerabilities: null - bundle_identifier: bundle @@ -669,6 +675,7 @@ spec: id: 0 name: bar source: deb_packages + browser: "" version: 0.0.3 vulnerabilities: null ` @@ -683,6 +690,7 @@ spec: "name": "foo", "version": "0.0.1", "source": "chrome_extensions", + "browser": "", "generated_cpe": "somecpe", "vulnerabilities": [ { @@ -710,6 +718,7 @@ spec: "name": "foo", "version": "0.0.3", "source": "chrome_extensions", + "browser": "", "generated_cpe": "someothercpewithoutvulns", "vulnerabilities": null }, @@ -719,6 +728,7 @@ spec: "version": "0.0.3", "bundle_identifier": "bundle", "source": "deb_packages", + "browser": "", "generated_cpe": "", "vulnerabilities": null } diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 18f67b0c5e..dd9af369fd 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -276,6 +276,7 @@ SELECT s.name, s.version, s.source, + s.browser, s.bundle_identifier, s.release, s.vendor, @@ -1066,6 +1067,7 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, includeCVEScores "s.name", "s.version", "s.source", + "s.browser", "s.bundle_identifier", "s.release", "s.vendor", diff --git a/server/fleet/software.go b/server/fleet/software.go index c7e345d886..701a958798 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -45,7 +45,7 @@ type Software struct { // ExtensionID is the browser extension id (from osquery chrome_extensions and firefox_addons) ExtensionID string `json:"extension_id,omitempty" db:"extension_id"` // Browser is the browser type (from osquery chrome_extensions) - Browser string `json:"browser,omitempty" db:"browser"` + Browser string `json:"browser" db:"browser"` // Release is the version of the OS this software was released on // (e.g. "30.el7" for a CentOS package). diff --git a/server/service/client_software.go b/server/service/client_software.go index e7a0fca8d9..0a2348a423 100644 --- a/server/service/client_software.go +++ b/server/service/client_software.go @@ -6,8 +6,8 @@ import ( // ListSoftware retrieves the software running across hosts. func (c *Client) ListSoftware(query string) ([]fleet.Software, error) { - verb, path := "GET", "/api/latest/fleet/software" - var responseBody listSoftwareResponse + verb, path := "GET", "/api/latest/fleet/software/versions" + var responseBody listSoftwareVersionsResponse err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query) if err != nil { return nil, err diff --git a/server/service/handler.go b/server/service/handler.go index fee67a10d3..4274c784d8 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -358,8 +358,14 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/spec/packs", getPackSpecsEndpoint, nil) ue.GET("/api/_version_/fleet/spec/packs/{name}", getPackSpecEndpoint, getGenericSpecRequest{}) + ue.GET("/api/_version_/fleet/software/versions", listSoftwareVersionsEndpoint, listSoftwareRequest{}) + ue.GET("/api/_version_/fleet/software/versions/{id:[0-9]+}", getSoftwareEndpoint, getSoftwareRequest{}) + + // DEPRECATED: use /api/_version_/fleet/software/versions instead ue.GET("/api/_version_/fleet/software", listSoftwareEndpoint, listSoftwareRequest{}) + // DEPRECATED: use /api/_version_/fleet/software/versions{id:[0-9]+} instead ue.GET("/api/_version_/fleet/software/{id:[0-9]+}", getSoftwareEndpoint, getSoftwareRequest{}) + // DEPRECATED: software version counts are now included directly in the software version list ue.GET("/api/_version_/fleet/software/count", countSoftwareEndpoint, countSoftwareRequest{}) ue.GET("/api/_version_/fleet/host_summary", getHostSummaryEndpoint, getHostSummaryRequest{}) diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 2a5e5803cf..fd51e11696 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -764,11 +764,20 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { bodyBytes, err = io.ReadAll(resp.Body) require.NoError(t, err) assert.Contains(t, string(bodyBytes), `"counts_updated_at": null`) - require.NoError(t, json.Unmarshal(bodyBytes, &lsResp)) require.Len(t, lsResp.Software, 0) assert.Nil(t, lsResp.CountsUpdatedAt) + var versionsResp listSoftwareVersionsResponse + resp = s.Do("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") + bodyBytes, err = io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(bodyBytes), `"counts_updated_at": null`) + require.NoError(t, json.Unmarshal(bodyBytes, &versionsResp)) + require.Len(t, versionsResp.Software, 0) + require.Equal(t, 0, versionsResp.Count) + assert.Nil(t, versionsResp.CountsUpdatedAt) + // calculate hosts counts hostsCountTs := time.Now().UTC() require.NoError(t, s.ds.SyncHostsSoftware(context.Background(), hostsCountTs)) @@ -794,6 +803,17 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { require.NotNil(t, lsResp.CountsUpdatedAt) assert.WithinDuration(t, hostsCountTs, *lsResp.CountsUpdatedAt, time.Second) + versionsResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versionsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") + require.Len(t, versionsResp.Software, 1) + require.Equal(t, 1, versionsResp.Count) + assert.Equal(t, soft1.ID, versionsResp.Software[0].ID) + assert.Equal(t, soft1.ExtensionID, versionsResp.Software[0].ExtensionID) + assert.Equal(t, soft1.Browser, versionsResp.Software[0].Browser) + assert.Len(t, versionsResp.Software[0].Vulnerabilities, 1) + require.NotNil(t, versionsResp.CountsUpdatedAt) + assert.WithinDuration(t, hostsCountTs, *versionsResp.CountsUpdatedAt, time.Second) + // the count endpoint still returns 1 countResp = countSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software/count", countReq, http.StatusOK, &countResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") @@ -806,6 +826,13 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { require.NotNil(t, lsResp.CountsUpdatedAt) assert.WithinDuration(t, hostsCountTs, *lsResp.CountsUpdatedAt, time.Second) + versionsResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versionsResp) + require.True(t, len(versionsResp.Software) >= len(software)) + require.True(t, versionsResp.Count >= len(software)) + require.NotNil(t, versionsResp.CountsUpdatedAt) + assert.WithinDuration(t, hostsCountTs, *versionsResp.CountsUpdatedAt, time.Second) + // request with a per_page limit (see #4058) lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "page", "0", "per_page", "2", "order_key", "hosts_count", "order_direction", "desc") @@ -813,6 +840,13 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { require.NotNil(t, lsResp.CountsUpdatedAt) assert.WithinDuration(t, hostsCountTs, *lsResp.CountsUpdatedAt, time.Second) + versionsResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versionsResp, "page", "0", "per_page", "2", "order_key", "hosts_count", "order_direction", "desc") + require.Len(t, versionsResp.Software, 2) + require.True(t, versionsResp.Count >= 2) + require.NotNil(t, versionsResp.CountsUpdatedAt) + assert.WithinDuration(t, hostsCountTs, *versionsResp.CountsUpdatedAt, time.Second) + // request next page, with per_page limit lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "1", "order_key", "hosts_count", "order_direction", "desc") @@ -820,13 +854,27 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { require.NotNil(t, lsResp.CountsUpdatedAt) assert.WithinDuration(t, hostsCountTs, *lsResp.CountsUpdatedAt, time.Second) + versionsResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versionsResp, "per_page", "2", "page", "1", "order_key", "hosts_count", "order_direction", "desc") + require.Len(t, versionsResp.Software, 1) + require.True(t, versionsResp.Count >= 2) + require.NotNil(t, versionsResp.CountsUpdatedAt) + assert.WithinDuration(t, hostsCountTs, *versionsResp.CountsUpdatedAt, time.Second) + // request one past the last page lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "2", "order_key", "hosts_count", "order_direction", "desc") require.Len(t, lsResp.Software, 0) require.Nil(t, lsResp.CountsUpdatedAt) + versionsResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versionsResp, "per_page", "2", "page", "2", "order_key", "hosts_count", "order_direction", "desc") + require.Len(t, versionsResp.Software, 0) + require.True(t, versionsResp.Count >= 2) + require.Nil(t, versionsResp.CountsUpdatedAt) // CONFIRM: legacy counts updated at is calculated by the server based on the software entries in the paginated response so how should we handle now? + s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusBadRequest, &lsResp, "per_page", "2", "page", "-10") + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusBadRequest, &lsResp, "per_page", "-2", "page", "2") s.DoJSON("GET", "/api/latest/fleet/software/count", nil, http.StatusBadRequest, &lsResp, "per_page", "-2", "page", "2") } @@ -5410,7 +5458,7 @@ func (s *integrationTestSuite) TestQuerySpecs() { assert.Equal(t, uint(3), delBatchResp.Deleted) } -func (s *integrationTestSuite) TestPaginateListSoftware() { +func (s *integrationTestSuite) TestListSoftwareAndSoftwareDetails() { t := s.T() // create a few hosts specific to this test @@ -5437,6 +5485,10 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { sws := make([]fleet.Software, 20) for i := range sws { sw := fleet.Software{Name: "sw" + strconv.Itoa(i), Version: "0.0." + strconv.Itoa(i), Source: "apps"} + if i%2 == 0 { + sw.Source = "chrome_extensions" + sw.Browser = "chrome" + } sws[i] = sw } @@ -5477,6 +5529,7 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { require.NoError(t, err) require.True(t, inserted) } + expectedVulnVersionsCount := 10 // create a team and make the last 3 hosts part of it (meaning 3 that use // sws[19], 2 for sws[18], and 1 for sws[17]) @@ -5485,6 +5538,29 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { }) require.NoError(t, err) require.NoError(t, s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{hosts[19].ID, hosts[18].ID, hosts[17].ID})) + expectedTeamVersionsCount := 3 + + assertSoftwareDetails := func(expectedSoftware []fleet.Software) { + // this is just a basic sanity check of the software details endpoints and doesn't test all of the + // fields that may be present in the response (e.g., vulnerabilities) + for _, sw := range expectedSoftware { + var detailsResp getSoftwareResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/%d", sw.ID), nil, http.StatusOK, &detailsResp) + assert.Equal(t, sw.ID, detailsResp.Software.ID) + assert.Equal(t, sw.Name, detailsResp.Software.Name) + assert.Equal(t, sw.Version, detailsResp.Software.Version) + assert.Equal(t, sw.Source, detailsResp.Software.Source) + assert.Equal(t, sw.Browser, detailsResp.Software.Browser) + + detailsResp = getSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/versions/%d", sw.ID), nil, http.StatusOK, &detailsResp) + assert.Equal(t, sw.ID, detailsResp.Software.ID) + assert.Equal(t, sw.Name, detailsResp.Software.Name) + assert.Equal(t, sw.Version, detailsResp.Software.Version) + assert.Equal(t, sw.Source, detailsResp.Software.Source) + assert.Equal(t, sw.Browser, detailsResp.Software.Browser) + } + } assertResp := func(resp listSoftwareResponse, want []fleet.Software, ts time.Time, counts ...int) { require.Len(t, resp.Software, len(want)) @@ -5493,6 +5569,14 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { assert.Equal(t, wantID, gotID) wantCount, gotCount := counts[i], resp.Software[i].HostsCount assert.Equal(t, wantCount, gotCount) + wantName, gotName := want[i].Name, resp.Software[i].Name + assert.Equal(t, wantName, gotName) + wantVersion, gotVersion := want[i].Version, resp.Software[i].Version + assert.Equal(t, wantVersion, gotVersion) + wantSource, gotSource := want[i].Source, resp.Software[i].Source + assert.Equal(t, wantSource, gotSource) + wantBrowser, gotBrowser := want[i].Browser, resp.Software[i].Browser + assert.Equal(t, wantBrowser, gotBrowser) } if ts.IsZero() { assert.Nil(t, resp.CountsUpdatedAt) @@ -5500,17 +5584,50 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { require.NotNil(t, resp.CountsUpdatedAt) assert.WithinDuration(t, ts, *resp.CountsUpdatedAt, time.Second) } + assertSoftwareDetails(resp.Software) + } + + assertVersionsResp := func(resp listSoftwareVersionsResponse, want []fleet.Software, ts time.Time, swCount int, hostCounts ...int) { + require.Equal(t, swCount, resp.Count) + require.Len(t, resp.Software, len(want)) + for i := range resp.Software { + wantID, gotID := want[i].ID, resp.Software[i].ID + assert.Equal(t, wantID, gotID) + wantCount, gotCount := hostCounts[i], resp.Software[i].HostsCount + assert.Equal(t, wantCount, gotCount) + wantName, gotName := want[i].Name, resp.Software[i].Name + assert.Equal(t, wantName, gotName) + wantVersion, gotVersion := want[i].Version, resp.Software[i].Version + assert.Equal(t, wantVersion, gotVersion) + wantSource, gotSource := want[i].Source, resp.Software[i].Source + assert.Equal(t, wantSource, gotSource) + wantBrowser, gotBrowser := want[i].Browser, resp.Software[i].Browser + assert.Equal(t, wantBrowser, gotBrowser) + } + if ts.IsZero() { + assert.Nil(t, resp.CountsUpdatedAt) + } else { + require.NotNil(t, resp.CountsUpdatedAt) + assert.WithinDuration(t, ts, *resp.CountsUpdatedAt, time.Second) + } + assertSoftwareDetails(resp.Software) } // no software host counts have been calculated yet, so this returns nothing var lsResp listSoftwareResponse s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, nil, time.Time{}) + var versResp listSoftwareVersionsResponse + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, nil, time.Time{}, 0) // same with a team filter lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) assertResp(lsResp, nil, time.Time{}) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertVersionsResp(versResp, nil, time.Time{}, 0) // calculate hosts counts hostsCountTs := time.Now().UTC() @@ -5520,61 +5637,97 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[19], sws[18], sws[17], sws[16], sws[15]}, hostsCountTs, 20, 19, 18, 17, 16) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[19], sws[18], sws[17], sws[16], sws[15]}, hostsCountTs, len(sws), 20, 19, 18, 17, 16) // second page (page=1) lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[14], sws[13], sws[12], sws[11], sws[10]}, hostsCountTs, 15, 14, 13, 12, 11) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[14], sws[13], sws[12], sws[11], sws[10]}, hostsCountTs, len(sws), 15, 14, 13, 12, 11) // third page (page=2) lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, 10, 9, 8, 7, 6) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, len(sws), 10, 9, 8, 7, 6) // last page (page=3) lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "3", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, 5, 4, 3, 2, 1) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "5", "page", "3", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, len(sws), 5, 4, 3, 2, 1) // past the end lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "4", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, nil, time.Time{}) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "5", "page", "4", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, nil, time.Time{}, len(sws)) // no explicit sort order, defaults to hosts_count DESC lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "0") assertResp(lsResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, 20, 19) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "2", "page", "0") + assertVersionsResp(versResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, len(sws), 20, 19) // hosts_count ascending lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "3", "page", "0", "order_key", "hosts_count", "order_direction", "asc") assertResp(lsResp, []fleet.Software{sws[0], sws[1], sws[2]}, hostsCountTs, 1, 2, 3) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "3", "page", "0", "order_key", "hosts_count", "order_direction", "asc") + assertVersionsResp(versResp, []fleet.Software{sws[0], sws[1], sws[2]}, hostsCountTs, len(sws), 1, 2, 3) // vulnerable software only lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, 10, 9, 8, 7, 6) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "vulnerable", "true", "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, expectedVulnVersionsCount, 10, 9, 8, 7, 6) // vulnerable software only, next page lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, 5, 4, 3, 2, 1) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "vulnerable", "true", "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, expectedVulnVersionsCount, 5, 4, 3, 2, 1) // vulnerable software only, past last page lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") assertResp(lsResp, nil, time.Time{}) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "vulnerable", "true", "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") + assertVersionsResp(versResp, nil, time.Time{}, expectedVulnVersionsCount) // filter by the team, 2 by page lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "0", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) assertResp(lsResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, 3, 2) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "2", "page", "0", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertVersionsResp(versResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, expectedTeamVersionsCount, 3, 2) // filter by the team, 2 by page, next page lsResp = listSoftwareResponse{} s.DoJSON("GET", "/api/latest/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "1", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) assertResp(lsResp, []fleet.Software{sws[17]}, hostsCountTs, 1) + versResp = listSoftwareVersionsResponse{} + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &versResp, "per_page", "2", "page", "1", "order_key", "hosts_count", "order_direction", "desc", "team_id", fmt.Sprintf("%d", tm.ID)) + assertVersionsResp(versResp, []fleet.Software{sws[17]}, hostsCountTs, expectedTeamVersionsCount, 1) } func (s *integrationTestSuite) TestChangeUserEmail() { diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 751a29ae85..61d60cbab9 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -3448,6 +3448,32 @@ func (s *integrationEnterpriseTestSuite) TestListSoftware() { require.Equal(t, barPayload.Vulnerabilities[0].CVEPublished, ptr.TimePtr(now)) require.Equal(t, barPayload.Vulnerabilities[0].Description, ptr.StringPtr("a long description of the cve")) require.Equal(t, barPayload.Vulnerabilities[0].ResolvedInVersion, ptr.StringPtr("1.2.3")) + + var respVersions listSoftwareVersionsResponse + s.DoJSON("GET", "/api/latest/fleet/software/versions", nil, http.StatusOK, &respVersions) + require.NotNil(t, resp) + + for _, s := range resp.Software { + switch s.Name { + case "foo": + fooPayload = s + case "bar": + barPayload = s + default: + require.Failf(t, "unrecognized software %s", s.Name) + + } + } + + require.Empty(t, fooPayload.Vulnerabilities) + require.Len(t, barPayload.Vulnerabilities, 1) + require.Equal(t, barPayload.Vulnerabilities[0].CVE, "cve-123") + require.NotNil(t, barPayload.Vulnerabilities[0].CVSSScore, ptr.Float64Ptr(5.4)) + require.NotNil(t, barPayload.Vulnerabilities[0].EPSSProbability, ptr.Float64Ptr(0.5)) + require.NotNil(t, barPayload.Vulnerabilities[0].CISAKnownExploit, ptr.BoolPtr(true)) + require.Equal(t, barPayload.Vulnerabilities[0].CVEPublished, ptr.TimePtr(now)) + require.Equal(t, barPayload.Vulnerabilities[0].Description, ptr.StringPtr("a long description of the cve")) + require.Equal(t, barPayload.Vulnerabilities[0].ResolvedInVersion, ptr.StringPtr("1.2.3")) } // TestGitOpsUserActions tests the permissions listed in ../../docs/Using-Fleet/Permissions.md. @@ -3636,11 +3662,13 @@ func (s *integrationEnterpriseTestSuite) TestGitOpsUserActions() { s.DoJSON("DELETE", "/api/latest/fleet/labels/foo2", deleteLabelRequest{}, http.StatusOK, &deleteLabelResponse{}) // Attempt to list all software, should fail. + s.DoJSON("GET", "/api/latest/fleet/software/versions", listSoftwareRequest{}, http.StatusForbidden, &listSoftwareVersionsResponse{}) s.DoJSON("GET", "/api/latest/fleet/software", listSoftwareRequest{}, http.StatusForbidden, &listSoftwareResponse{}) s.DoJSON("GET", "/api/latest/fleet/software/count", countSoftwareRequest{}, http.StatusForbidden, &countSoftwareResponse{}) // Attempt to list a software, should fail. s.DoJSON("GET", "/api/latest/fleet/software/1", getSoftwareRequest{}, http.StatusForbidden, &getSoftwareResponse{}) + s.DoJSON("GET", "/api/latest/fleet/software/versions/1", getSoftwareRequest{}, http.StatusForbidden, &getSoftwareResponse{}) // Attempt to read app config, should fail. s.DoJSON("GET", "/api/latest/fleet/config", nil, http.StatusForbidden, &appConfigResponse{}) diff --git a/server/service/software.go b/server/service/software.go index 5ef4837b19..06148dd222 100644 --- a/server/service/software.go +++ b/server/service/software.go @@ -15,6 +15,9 @@ type listSoftwareRequest struct { fleet.SoftwareListOptions } +// DEPRECATED: listSoftwareResponse is the response struct for the deprecated +// listSoftwareEndpoint. It differs from listSoftwareVersionsResponse in that +// the latter includes a count of the total number of software items. type listSoftwareResponse struct { CountsUpdatedAt *time.Time `json:"counts_updated_at"` Software []fleet.Software `json:"software,omitempty"` @@ -23,6 +26,7 @@ type listSoftwareResponse struct { func (r listSoftwareResponse) error() error { return r.Err } +// DEPRECATED: use listSoftwareVersionsEndpoint instead func listSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*listSoftwareRequest) resp, err := svc.ListSoftware(ctx, req.SoftwareListOptions) @@ -45,6 +49,43 @@ func listSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Se return listResp, nil } +type listSoftwareVersionsResponse struct { + Count int `json:"count"` + CountsUpdatedAt *time.Time `json:"counts_updated_at"` + Software []fleet.Software `json:"software,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r listSoftwareVersionsResponse) error() error { return r.Err } + +func listSoftwareVersionsEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*listSoftwareRequest) + resp, err := svc.ListSoftware(ctx, req.SoftwareListOptions) + if err != nil { + return listSoftwareVersionsResponse{Err: err}, nil + } + + // calculate the latest counts_updated_at + var latest time.Time + for _, sw := range resp { + if !sw.CountsUpdatedAt.IsZero() && sw.CountsUpdatedAt.After(latest) { + latest = sw.CountsUpdatedAt + } + } + listResp := listSoftwareVersionsResponse{Software: resp} + if !latest.IsZero() { + listResp.CountsUpdatedAt = &latest + } + + c, err := svc.CountSoftware(ctx, req.SoftwareListOptions) + if err != nil { + return listSoftwareVersionsResponse{Err: err}, nil + } + listResp.Count = c + + return listResp, nil +} + func (svc *Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) { if err := svc.authz.Authorize(ctx, &fleet.AuthzSoftwareInventory{ TeamID: opt.TeamID, @@ -121,6 +162,8 @@ type countSoftwareResponse struct { func (r countSoftwareResponse) error() error { return r.Err } +// DEPRECATED: counts are now included directly in the listSoftwareVersionsResponse. This +// endpoint is retained for backwards compatibility. func countSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*countSoftwareRequest) count, err := svc.CountSoftware(ctx, req.SoftwareListOptions) From d0cfc65786ff273c45ef31d5e9bcdf09a6c7c85a Mon Sep 17 00:00:00 2001 From: Noah Talerman <47070608+noahtalerman@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:52:28 -0500 Subject: [PATCH 04/60] Update Hosts page (#15465) - Use "hosts" instead of "devices" in copy on **Hosts** page --- frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx b/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx index 385138ad1b..480175995c 100644 --- a/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx +++ b/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx @@ -1367,9 +1367,9 @@ const ManageHostsPage = ({ const emptyState = () => { const emptyHosts: IEmptyTableProps = { graphicName: "empty-hosts", - header: "Devices will show up here once they’re added to Fleet.", + header: "Hosts will show up here once they’re added to Fleet.", info: - "Expecting to see devices? Try again in a few seconds as the system catches up.", + "Expecting to see hosts? Try again in a few seconds as the system catches up.", }; if (includesFilterQueryParam) { delete emptyHosts.graphicName; @@ -1377,8 +1377,8 @@ const ManageHostsPage = ({ emptyHosts.info = "Expecting to see new hosts? Try again in a few seconds as the system catches up."; } else if (canEnrollHosts) { - emptyHosts.header = "Add your devices to Fleet"; - emptyHosts.info = "Generate an installer to add your own devices."; + emptyHosts.header = "Add your hosts to Fleet"; + emptyHosts.info = "Generate an installer to add your own hosts."; emptyHosts.primaryButton = (