Add sw_edition to cpe db generation and cpe translations (#32879)

Fixes: #31989 
# Adding sw_edition to CPE generation and translation
This PR adds the ability to override sw_edition with cpe translations.
This adds a new column to cpe.sqlite that is generated daily.
Old versions of fleet will still work with the new cpe db and
translations.
Versions from this change forward will require the new cpe db for cpe
translations to work.

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

## Testing

- [x] Added/updated automated tests
- [ ] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)

- [x] QA'd all new/changed functionality manually

## Backwards Compatibility
Testing with physical machines and for Firefox ESR fix
| Fleet version | cpe db | translations | vuln. soft. # | Firefox ESR
cpe | Firefox ESR vuln. # |
| ------- | ------ | ------------ | ------------- | ---------------- |
------------------- |
| Updated | old | old | 58 | `:*:macos:*:*` | 168 |
| Updated | new | new | 58 | `:esr:macos:*:*` | 92 |
| 4.71.1 | old | old | 58 | `:*:macos:*:*` | 168 |
| 4.71.1 | new | new | 58 | `:*:macos:*:*` | 168 |

Testing with osquery-perf hosts
| Fleet version | cpe db | translations | vuln. soft. # |
Vulnerabilities |
| ------- | ------ | ------------ | ------------- | --------------- |
| Updated | old    | old          | 156/161       | 3136            |
| Updated | new    | new          | 156/161       | 3136            |
| 4.71.1  | old    | old          | 156/161       | 3951            |
| 4.71.1  | new    | new          | 156/161       | 3951            |

---------

Co-authored-by: Ian Littman <iansltx@gmail.com>

changes/31989-firefox-esr-sw_edition-translation

@@ -0,0 +1,2 @@
+* Added support for vulnerabilities feed CPE translation JSON to override sw_edition field
+* Fixed cases where Firefox ESR installations would have false-positive vulnerabilities reported that were backported to the ESR

cmd/cpe/testdata/test1.golden

@@ -1 +1 @@
-[[cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:* HP Radia Notify Daemon hp radia_notify_daemon -  false] [cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:* HP SANworks hp sanworks -  false] [cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:* HP Scanjet Utilities hp scanjet_utilities -  false] [cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:* HP Secure Web Console hp secure_web_console -  false] [cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:* HP sendmail hp sendmail -  false] [cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:* Linux Kernel 2.6.2 linux linux_kernel 2.6.2  true]]
+[[cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:* HP Radia Notify Daemon hp radia_notify_daemon -   false] [cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:* HP SANworks hp sanworks -   false] [cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:* HP Scanjet Utilities hp scanjet_utilities -   false] [cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:* HP Secure Web Console hp secure_web_console -   false] [cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:* HP sendmail hp sendmail -   false] [cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:* Linux Kernel 2.6.2 linux linux_kernel 2.6.2   true]]

cmd/cpe/testdata/test2.golden

@@ -1 +1 @@
-[[cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:* Denkgroot Spina 2.3.5 denkgroot spina 2.3.5  false] [cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:* Denkgroot Spina 2.3.4 denkgroot spina 2.3.4  false]]
+[[cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:* Denkgroot Spina 2.3.5 denkgroot spina 2.3.5   false] [cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:* Denkgroot Spina 2.3.4 denkgroot spina 2.3.4   false]]

server/vulnerabilities/nvd/cpe.go

@@ -439,6 +439,7 @@ func CPEFromSoftware(logger log.Logger, db *sqlx.DB, software *fleet.Software, t
 				"c.rowid",
 				"c.product",
 				"c.vendor",
+				"c.sw_edition",
 				"c.deprecated",
 				goqu.L("1 as weight"),
 			).Limit(1)
@@ -464,6 +465,13 @@ func CPEFromSoftware(logger log.Logger, db *sqlx.DB, software *fleet.Software, t
 			}
 			ds = ds.Where(goqu.Or(exps...))
 		}
+		if len(translation.SWEdition) > 0 {
+			var exps []goqu.Expression
+			for _, SWEdition := range translation.SWEdition {
+				exps = append(exps, goqu.I("c.sw_edition").Eq(SWEdition))
+			}
+			ds = ds.Where(goqu.Or(exps...))
+		}
 
 		stm, args, _ := ds.ToSQL()
 

server/vulnerabilities/nvd/cpe_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -1809,6 +1810,22 @@ func TestCPEFromSoftwareIntegration(t *testing.T) {
 			},
 			cpe: "cpe:2.3:a:minio:minio:2020-03-10T00-00-00Z:*:*:*:*:macos:*:*",
 		},
+		{
+			software: fleet.Software{
+				Name:    "Firefox.app",
+				Source:  "apps",
+				Version: "137.0.2",
+			},
+			cpe: "cpe:2.3:a:mozilla:firefox:137.0.2:*:*:*:*:macos:*:*",
+		},
+		{
+			software: fleet.Software{
+				Name:    "Firefox ESR.app",
+				Source:  "apps",
+				Version: "128.14.0",
+			},
+			cpe: "cpe:2.3:a:mozilla:firefox:128.14.0:*:*:*:esr:macos:*:*",
+		},
 	}
 
 	// NVD_TEST_CPEDB_PATH can be used to speed up development (sync cpe.sqlite only once).
@@ -1836,6 +1853,15 @@ func TestCPEFromSoftwareIntegration(t *testing.T) {
 	for _, tt := range testCases {
 		tt := tt
 		cpe, err := CPEFromSoftware(log.NewNopLogger(), db, &tt.software, cpeTranslations, reCache)
+
+		translation, okT, _ := cpeTranslations.Translate(reCache, &tt.software)
+		if okT {
+			if len(translation.SWEdition) == 0 || translation.SWEdition[0] == "" {
+				re := regexp.MustCompile(`\*:[^*]+:[^*]+:\*:\*$`)
+				assert.False(t, re.MatchString(cpe), "did not expect sw_edition for:"+cpe)
+			}
+		}
+
 		require.NoError(t, err)
 		assert.Equal(t, tt.cpe, cpe, tt.software.Name)
 	}

server/vulnerabilities/nvd/cpe_translations.go

@@ -217,10 +217,11 @@ func (c CPETranslationSoftware) Matches(reCache *regexpCache, s *fleet.Software)
 }
 
 type CPETranslation struct {
-	Product  []string `json:"product"`
-	Vendor   []string `json:"vendor"`
-	TargetSW []string `json:"target_sw"`
-	Part     string   `json:"part"`
+	Product   []string `json:"product"`
+	Vendor    []string `json:"vendor"`
+	TargetSW  []string `json:"target_sw"`
+	SWEdition []string `json:"sw_edition"`
+	Part      string   `json:"part"`
 	// If Skip is set, no NVD vulnerabilities will be reported for the matching software.
 	Skip bool `json:"skip"`
 }

server/vulnerabilities/nvd/cpe_translations.json

@@ -119,7 +119,8 @@
     },
     "filter": {
       "product": ["desktop"],
-      "vendor": ["docker"]
+      "vendor": ["docker"],
+      "sw_edition": [""]
     }
   },
   {
@@ -613,5 +614,16 @@
       "product": ["iterm2"],
       "vendor": ["iterm2"]
     }
+  },
+  {
+    "software": {
+      "name": ["Firefox ESR.app"],
+      "source": ["apps"]
+    },
+    "filter": {
+      "product": ["firefox"],
+      "vendor": ["mozilla"],
+      "sw_edition": ["esr"]
+    }
   }
 ]

server/vulnerabilities/nvd/db.go

@@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS cpe_2 (
     product TEXT,
     version TEXT,
     target_sw TEXT,
+    sw_edition TEST,
     deprecated BOOLEAN DEFAULT FALSE
 );
 CREATE VIEW IF NOT EXISTS cpe AS
@@ -54,6 +55,7 @@ CREATE INDEX IF NOT EXISTS idx_cpe_2_vendor ON cpe_2 (vendor);
 CREATE INDEX IF NOT EXISTS idx_cpe_2_product ON cpe_2 (product);
 CREATE INDEX IF NOT EXISTS idx_cpe_2_version ON cpe_2 (version);
 CREATE INDEX IF NOT EXISTS idx_cpe_2_target_sw ON cpe_2 (target_sw);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_sw_edition ON cpe_2 (sw_edition);
 CREATE INDEX IF NOT EXISTS idx_deprecated_by ON deprecated_by (cpe23);
 `)
 	return err
@@ -69,8 +71,9 @@ func generateCPEItem(item cpedict.CPEItem) ([]interface{}, map[string]string, er
 	product := wfn.StripSlashes(item.CPE23.Name.Product)
 	version := wfn.StripSlashes(item.CPE23.Name.Version)
 	targetSW := wfn.StripSlashes(item.CPE23.Name.TargetSW)
+	SWEdition := wfn.StripSlashes(item.CPE23.Name.SWEdition)
 
-	cpes = append(cpes, cpe23, title, vendor, product, version, targetSW, item.Deprecated)
+	cpes = append(cpes, cpe23, title, vendor, product, version, targetSW, SWEdition, item.Deprecated)
 
 	if item.CPE23.Deprecation != nil {
 		for _, deprecatedBy := range item.CPE23.Deprecation.DeprecatedBy {
@@ -165,7 +168,7 @@ func bulkInsertDeprecations(deprecationsCount int, db *sqlx.DB, allDeprecations
 }
 
 func bulkInsertCPEs(cpesCount int, db *sqlx.DB, allCPEs []interface{}) error {
-	values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?, ?, ?), ", cpesCount), ", ")
+	values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?, ?, ?, ?), ", cpesCount), ", ")
 	_, err := db.Exec(
 		fmt.Sprintf(`
 INSERT INTO cpe_2 (
@@ -175,6 +178,7 @@ INSERT INTO cpe_2 (
 	product,
 	version,
 	target_sw,
+	sw_edition,
 	deprecated
 )
 VALUES %s`, values),

server/vulnerabilities/nvd/indexed_cpe_item.go

@@ -13,6 +13,7 @@ type IndexedCPEItem struct {
 	Part       string
 	Product    string `json:"product" db:"product"`
 	Vendor     string `json:"vendor" db:"vendor"`
+	SWEdition  string `json:"sw_edition" db:"sw_edition"`
 	Deprecated bool   `json:"deprecated" db:"deprecated"`
 	Weight     int    `db:"weight"`
 }
@@ -23,6 +24,7 @@ func (i *IndexedCPEItem) FmtStr(s *fleet.Software) string {
 	cpe.Vendor = i.Vendor
 	cpe.Product = i.Product
 	cpe.TargetSW = targetSW(s)
+	cpe.SWEdition = i.SWEdition
 
 	// Some version strings (e.g. Python pre-releases) contain a part that should be placed in the
 	// CPE's update field. Parse that out (if it exists).