fleet/cmd/cpe/validate/main.go
Scott Gress 4bed761f77
Add validator for NVD feed items (#29282)
for #21304 

# Checklist for submitter

- [X] Manual QA for all new/changed functionality

## Details

This PR adds a new validator for NVD feed files to be run as part of the
nvd repo workflow. The intention is for that workflow to fail if any of
the files it creates are not valid (i.e. they would not be parseable by
the Fleet server) so that we don't publish and tag a release with bad
files in it.

This follows the pattern from
https://github.com/fleetdm/fleet/issues/21300 as suggested by @iansltx.

## Testing

I downloaded all of the latest release files to my local system using
```bash
gh release download 202505190037 -D ~/Downloads/nvd
```
and then ran the validator on them with
```bash
go run cmd/cpe/validate/main.go --db_dir ~/Downloads/nvd
```
To simulate file issues, I modified one section of each file to change a
value into the wrong type, and validated that this caused the validator
to panic. Examples:
```
panic: failed to load CPE translations: decode json: json: cannot unmarshal string into Go struct field CPETranslation.filter.vendor of type []string

goroutine 1 [running]:
main.checkCPETranslations({0x16dc975f9?, 0x14000192190?})
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:34 +0xa8
main.main()
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:24 +0xb0
exit status 2
```
---
```
panic: failed to parse MacOffice release notes fleet_macoffice_release_notes_macoffice-2025_05_19.json: parsing time "xyz" as "2006-01-02T15:04:05Z07:00": cannot parse "xyz" as "2006"

goroutine 1 [running]:
main.checkMacOfficeNotes({0x16f7af5f9, 0x1a})
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:56 +0x1f0
main.main()
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:25 +0xbc
exit status 2
```
---
```
panic: failed to parse MSRC feed fleet_msrc_Windows_Server_2012_R2-2025_05_19.json: json: cannot unmarshal array into Go struct field Vulnerability.Vulnerabities.RemediatedBy of type bool

goroutine 1 [running]:
main.checkMSRCVulnerabilities({0x16f49b5f9, 0x1a})
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:74 +0x1ac
main.main()
	/Users/scott/Development/fleet/cmd/cpe/validate/main.go:26 +0xc8
exit status 2
```

Additionally I tried the validator in [a run of the NVD
workflow](https://github.com/fleetdm/nvd/actions/runs/15121687898/job/42505283781)
and it executed successfully.
2025-05-22 14:51:52 -05:00

130 lines
3.8 KiB
Go

package main
import (
"compress/gzip"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/macoffice"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd"
"github.com/jmoiron/sqlx"
)
func main() {
dbDir := flag.String("db_dir", "/tmp/vulndbs", "Path to the vulnerability database")
flag.Parse()
vulnPath := *dbDir
checkCPETranslations(vulnPath)
checkMacOfficeNotes(vulnPath)
checkMSRCVulnerabilities(vulnPath)
checkSqliteDb(vulnPath)
}
func checkCPETranslations(vulnPath string) {
// Check that the CPE translations file is parseable into an array of CPETranslationItem
_, err := nvd.LoadCPETranslations(filepath.Join(vulnPath, "cpe_translations.json"))
if err != nil {
panic(fmt.Sprintf("failed to load CPE translations: %v", err))
}
}
func checkMacOfficeNotes(vulnPath string) {
// Iterate over each file in the vulnPath directory that starts with `fleet_macoffice`
files, err := os.ReadDir(vulnPath)
if err != nil {
panic(fmt.Sprintf("failed to read directory: %v", err))
}
for _, file := range files {
if strings.HasPrefix(file.Name(), "fleet_macoffice") && strings.HasSuffix(file.Name(), ".json") {
filePath := filepath.Join(vulnPath, file.Name())
payload, err := os.ReadFile(filePath)
if err != nil {
panic(fmt.Sprintf("failed to read MacOffice release notes file %s: %v", file.Name(), err))
}
// Attempt to parse the file as a MacOffice release notes.
relNotes := macoffice.ReleaseNotes{}
err = json.Unmarshal(payload, &relNotes)
if err != nil {
panic(fmt.Sprintf("failed to parse MacOffice release notes %s: %v", file.Name(), err))
}
}
}
}
func checkMSRCVulnerabilities(vulnPath string) {
// Iterate over each file in the vulnPath directory that starts with `fleet_msrc`
files, err := os.ReadDir(vulnPath)
if err != nil {
panic(fmt.Sprintf("failed to read directory: %v", err))
}
for _, file := range files {
if strings.HasPrefix(file.Name(), "fleet_msrc") && strings.HasSuffix(file.Name(), ".json") {
filePath := filepath.Join(vulnPath, file.Name())
// Attempt to parse the file as a MSRC feed.
_, err := parsed.UnmarshalBulletin(filePath)
if err != nil {
panic(fmt.Sprintf("failed to parse MSRC feed %s: %v", file.Name(), err))
}
}
}
}
func checkSqliteDb(vulnPath string) {
// Iterate over each file in the vulnPath directory to find the sqlite.gz file
files, err := os.ReadDir(vulnPath)
if err != nil {
panic(fmt.Sprintf("failed to read directory: %v", err))
}
var sqliteFilename string
for _, file := range files {
if strings.HasSuffix(file.Name(), ".sqlite.gz") {
sqliteFilename = file.Name()
break
}
}
if sqliteFilename == "" {
panic(fmt.Sprintf("no sqlite.gz file found: %v", err))
}
// Unzip the sqlite.gz file and create a new sqlite.db file
gzFile, err := os.Open(filepath.Join(vulnPath, sqliteFilename))
if err != nil {
panic(fmt.Sprintf("error opening sqlite.gz file: %v", err))
}
defer gzFile.Close()
sqliteFile, err := os.Create(filepath.Join(vulnPath, "sqlite.db"))
if err != nil {
panic(fmt.Sprintf("error creating test sqlite.db file: %v", err))
}
defer sqliteFile.Close()
gzReader, err := gzip.NewReader(gzFile)
if err != nil {
panic(fmt.Sprintf("error creating new gzip reader: %v", err))
}
defer gzReader.Close()
for {
_, err := io.CopyN(sqliteFile, gzReader, 100*1024*1024)
if err != nil {
if err == io.EOF {
break
}
panic(fmt.Sprintf("error unzipping sqlite file: %v", err))
}
}
db, err := sqlx.Open("sqlite3", filepath.Join(vulnPath, "sqlite.db"))
if err != nil {
panic(fmt.Sprintf("error opening sqlite db: %v", err))
}
// Check that the database is valid
_, err = db.Exec(`SELECT * FROM cpe_2 LIMIT 1`)
if err != nil {
panic(fmt.Sprintf("error executing query on sqlite db: %v", err))
}
}