From d70500a6e98f8c49bf83f3fd0464f86d5fd40e5a Mon Sep 17 00:00:00 2001 From: Jonathan Katz <44128041+jkatz01@users.noreply.github.com> Date: Wed, 17 Sep 2025 11:30:49 -0400 Subject: [PATCH] Add sw_edition to cpe db generation and cpe translations (#32879) Fixes: #31989 # Adding sw_edition to CPE generation and translation This PR adds the ability to override sw_edition with cpe translations. This adds a new column to cpe.sqlite that is generated daily. Old versions of fleet will still work with the new cpe db and translations. Versions from this change forward will require the new cpe db for cpe translations to work. # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. ## Testing - [x] Added/updated automated tests - [ ] Where appropriate, [automated tests simulate multiple hosts and test for host isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing) (updates to one hosts's records do not affect another) - [x] QA'd all new/changed functionality manually ## Backwards Compatibility Testing with physical machines and for Firefox ESR fix | Fleet version | cpe db | translations | vuln. soft. # | Firefox ESR cpe | Firefox ESR vuln. # | | ------- | ------ | ------------ | ------------- | ---------------- | ------------------- | | Updated | old | old | 58 | `:*:macos:*:*` | 168 | | Updated | new | new | 58 | `:esr:macos:*:*` | 92 | | 4.71.1 | old | old | 58 | `:*:macos:*:*` | 168 | | 4.71.1 | new | new | 58 | `:*:macos:*:*` | 168 | Testing with osquery-perf hosts | Fleet version | cpe db | translations | vuln. soft. # | Vulnerabilities | | ------- | ------ | ------------ | ------------- | --------------- | | Updated | old | old | 156/161 | 3136 | | Updated | new | new | 156/161 | 3136 | | 4.71.1 | old | old | 156/161 | 3951 | | 4.71.1 | new | new | 156/161 | 3951 | --------- Co-authored-by: Ian Littman --- .../31989-firefox-esr-sw_edition-translation | 2 ++ cmd/cpe/testdata/test1.golden | 2 +- cmd/cpe/testdata/test2.golden | 2 +- server/vulnerabilities/nvd/cpe.go | 8 ++++++ server/vulnerabilities/nvd/cpe_test.go | 26 +++++++++++++++++++ .../vulnerabilities/nvd/cpe_translations.go | 9 ++++--- .../vulnerabilities/nvd/cpe_translations.json | 14 +++++++++- server/vulnerabilities/nvd/db.go | 8 ++++-- .../vulnerabilities/nvd/indexed_cpe_item.go | 2 ++ 9 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 changes/31989-firefox-esr-sw_edition-translation diff --git a/changes/31989-firefox-esr-sw_edition-translation b/changes/31989-firefox-esr-sw_edition-translation new file mode 100644 index 0000000000..fb53371990 --- /dev/null +++ b/changes/31989-firefox-esr-sw_edition-translation @@ -0,0 +1,2 @@ +* Added support for vulnerabilities feed CPE translation JSON to override sw_edition field +* Fixed cases where Firefox ESR installations would have false-positive vulnerabilities reported that were backported to the ESR diff --git a/cmd/cpe/testdata/test1.golden b/cmd/cpe/testdata/test1.golden index 7661509716..48be1e36dc 100644 --- a/cmd/cpe/testdata/test1.golden +++ b/cmd/cpe/testdata/test1.golden @@ -1 +1 @@ -[[cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:* HP Radia Notify Daemon hp radia_notify_daemon - false] [cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:* HP SANworks hp sanworks - false] [cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:* HP Scanjet Utilities hp scanjet_utilities - false] [cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:* HP Secure Web Console hp secure_web_console - false] [cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:* HP sendmail hp sendmail - false] [cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:* Linux Kernel 2.6.2 linux linux_kernel 2.6.2 true]] \ No newline at end of file +[[cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:* HP Radia Notify Daemon hp radia_notify_daemon - false] [cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:* HP SANworks hp sanworks - false] [cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:* HP Scanjet Utilities hp scanjet_utilities - false] [cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:* HP Secure Web Console hp secure_web_console - false] [cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:* HP sendmail hp sendmail - false] [cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:* Linux Kernel 2.6.2 linux linux_kernel 2.6.2 true]] \ No newline at end of file diff --git a/cmd/cpe/testdata/test2.golden b/cmd/cpe/testdata/test2.golden index 01088adc87..f1d8b7e90b 100644 --- a/cmd/cpe/testdata/test2.golden +++ b/cmd/cpe/testdata/test2.golden @@ -1 +1 @@ -[[cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:* Denkgroot Spina 2.3.5 denkgroot spina 2.3.5 false] [cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:* Denkgroot Spina 2.3.4 denkgroot spina 2.3.4 false]] \ No newline at end of file +[[cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:* Denkgroot Spina 2.3.5 denkgroot spina 2.3.5 false] [cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:* Denkgroot Spina 2.3.4 denkgroot spina 2.3.4 false]] \ No newline at end of file diff --git a/server/vulnerabilities/nvd/cpe.go b/server/vulnerabilities/nvd/cpe.go index 3ae0317ec3..3d64c5f94b 100644 --- a/server/vulnerabilities/nvd/cpe.go +++ b/server/vulnerabilities/nvd/cpe.go @@ -439,6 +439,7 @@ func CPEFromSoftware(logger log.Logger, db *sqlx.DB, software *fleet.Software, t "c.rowid", "c.product", "c.vendor", + "c.sw_edition", "c.deprecated", goqu.L("1 as weight"), ).Limit(1) @@ -464,6 +465,13 @@ func CPEFromSoftware(logger log.Logger, db *sqlx.DB, software *fleet.Software, t } ds = ds.Where(goqu.Or(exps...)) } + if len(translation.SWEdition) > 0 { + var exps []goqu.Expression + for _, SWEdition := range translation.SWEdition { + exps = append(exps, goqu.I("c.sw_edition").Eq(SWEdition)) + } + ds = ds.Where(goqu.Or(exps...)) + } stm, args, _ := ds.ToSQL() diff --git a/server/vulnerabilities/nvd/cpe_test.go b/server/vulnerabilities/nvd/cpe_test.go index 3146c8c5fc..00931ab431 100644 --- a/server/vulnerabilities/nvd/cpe_test.go +++ b/server/vulnerabilities/nvd/cpe_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "regexp" "strings" "testing" "time" @@ -1809,6 +1810,22 @@ func TestCPEFromSoftwareIntegration(t *testing.T) { }, cpe: "cpe:2.3:a:minio:minio:2020-03-10T00-00-00Z:*:*:*:*:macos:*:*", }, + { + software: fleet.Software{ + Name: "Firefox.app", + Source: "apps", + Version: "137.0.2", + }, + cpe: "cpe:2.3:a:mozilla:firefox:137.0.2:*:*:*:*:macos:*:*", + }, + { + software: fleet.Software{ + Name: "Firefox ESR.app", + Source: "apps", + Version: "128.14.0", + }, + cpe: "cpe:2.3:a:mozilla:firefox:128.14.0:*:*:*:esr:macos:*:*", + }, } // NVD_TEST_CPEDB_PATH can be used to speed up development (sync cpe.sqlite only once). @@ -1836,6 +1853,15 @@ func TestCPEFromSoftwareIntegration(t *testing.T) { for _, tt := range testCases { tt := tt cpe, err := CPEFromSoftware(log.NewNopLogger(), db, &tt.software, cpeTranslations, reCache) + + translation, okT, _ := cpeTranslations.Translate(reCache, &tt.software) + if okT { + if len(translation.SWEdition) == 0 || translation.SWEdition[0] == "" { + re := regexp.MustCompile(`\*:[^*]+:[^*]+:\*:\*$`) + assert.False(t, re.MatchString(cpe), "did not expect sw_edition for:"+cpe) + } + } + require.NoError(t, err) assert.Equal(t, tt.cpe, cpe, tt.software.Name) } diff --git a/server/vulnerabilities/nvd/cpe_translations.go b/server/vulnerabilities/nvd/cpe_translations.go index a631a7fbd0..cdc4b85116 100644 --- a/server/vulnerabilities/nvd/cpe_translations.go +++ b/server/vulnerabilities/nvd/cpe_translations.go @@ -217,10 +217,11 @@ func (c CPETranslationSoftware) Matches(reCache *regexpCache, s *fleet.Software) } type CPETranslation struct { - Product []string `json:"product"` - Vendor []string `json:"vendor"` - TargetSW []string `json:"target_sw"` - Part string `json:"part"` + Product []string `json:"product"` + Vendor []string `json:"vendor"` + TargetSW []string `json:"target_sw"` + SWEdition []string `json:"sw_edition"` + Part string `json:"part"` // If Skip is set, no NVD vulnerabilities will be reported for the matching software. Skip bool `json:"skip"` } diff --git a/server/vulnerabilities/nvd/cpe_translations.json b/server/vulnerabilities/nvd/cpe_translations.json index 8f0c1f2033..1f2afd38be 100644 --- a/server/vulnerabilities/nvd/cpe_translations.json +++ b/server/vulnerabilities/nvd/cpe_translations.json @@ -119,7 +119,8 @@ }, "filter": { "product": ["desktop"], - "vendor": ["docker"] + "vendor": ["docker"], + "sw_edition": [""] } }, { @@ -613,5 +614,16 @@ "product": ["iterm2"], "vendor": ["iterm2"] } + }, + { + "software": { + "name": ["Firefox ESR.app"], + "source": ["apps"] + }, + "filter": { + "product": ["firefox"], + "vendor": ["mozilla"], + "sw_edition": ["esr"] + } } ] diff --git a/server/vulnerabilities/nvd/db.go b/server/vulnerabilities/nvd/db.go index b0e0c33b9e..8ca990e600 100644 --- a/server/vulnerabilities/nvd/db.go +++ b/server/vulnerabilities/nvd/db.go @@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS cpe_2 ( product TEXT, version TEXT, target_sw TEXT, + sw_edition TEST, deprecated BOOLEAN DEFAULT FALSE ); CREATE VIEW IF NOT EXISTS cpe AS @@ -54,6 +55,7 @@ CREATE INDEX IF NOT EXISTS idx_cpe_2_vendor ON cpe_2 (vendor); CREATE INDEX IF NOT EXISTS idx_cpe_2_product ON cpe_2 (product); CREATE INDEX IF NOT EXISTS idx_cpe_2_version ON cpe_2 (version); CREATE INDEX IF NOT EXISTS idx_cpe_2_target_sw ON cpe_2 (target_sw); +CREATE INDEX IF NOT EXISTS idx_cpe_2_sw_edition ON cpe_2 (sw_edition); CREATE INDEX IF NOT EXISTS idx_deprecated_by ON deprecated_by (cpe23); `) return err @@ -69,8 +71,9 @@ func generateCPEItem(item cpedict.CPEItem) ([]interface{}, map[string]string, er product := wfn.StripSlashes(item.CPE23.Name.Product) version := wfn.StripSlashes(item.CPE23.Name.Version) targetSW := wfn.StripSlashes(item.CPE23.Name.TargetSW) + SWEdition := wfn.StripSlashes(item.CPE23.Name.SWEdition) - cpes = append(cpes, cpe23, title, vendor, product, version, targetSW, item.Deprecated) + cpes = append(cpes, cpe23, title, vendor, product, version, targetSW, SWEdition, item.Deprecated) if item.CPE23.Deprecation != nil { for _, deprecatedBy := range item.CPE23.Deprecation.DeprecatedBy { @@ -165,7 +168,7 @@ func bulkInsertDeprecations(deprecationsCount int, db *sqlx.DB, allDeprecations } func bulkInsertCPEs(cpesCount int, db *sqlx.DB, allCPEs []interface{}) error { - values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?, ?, ?), ", cpesCount), ", ") + values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?, ?, ?, ?), ", cpesCount), ", ") _, err := db.Exec( fmt.Sprintf(` INSERT INTO cpe_2 ( @@ -175,6 +178,7 @@ INSERT INTO cpe_2 ( product, version, target_sw, + sw_edition, deprecated ) VALUES %s`, values), diff --git a/server/vulnerabilities/nvd/indexed_cpe_item.go b/server/vulnerabilities/nvd/indexed_cpe_item.go index ef4b56bb89..c42d939b09 100644 --- a/server/vulnerabilities/nvd/indexed_cpe_item.go +++ b/server/vulnerabilities/nvd/indexed_cpe_item.go @@ -13,6 +13,7 @@ type IndexedCPEItem struct { Part string Product string `json:"product" db:"product"` Vendor string `json:"vendor" db:"vendor"` + SWEdition string `json:"sw_edition" db:"sw_edition"` Deprecated bool `json:"deprecated" db:"deprecated"` Weight int `db:"weight"` } @@ -23,6 +24,7 @@ func (i *IndexedCPEItem) FmtStr(s *fleet.Software) string { cpe.Vendor = i.Vendor cpe.Product = i.Product cpe.TargetSW = targetSW(s) + cpe.SWEdition = i.SWEdition // Some version strings (e.g. Python pre-releases) contain a part that should be placed in the // CPE's update field. Parse that out (if it exists).