mirror of
https://github.com/fleetdm/fleet
synced 2026-05-06 06:48:54 +00:00
for #21304 # Checklist for submitter - [X] Manual QA for all new/changed functionality ## Details This PR adds a new validator for NVD feed files to be run as part of the nvd repo workflow. The intention is for that workflow to fail if any of the files it creates are not valid (i.e. they would not be parseable by the Fleet server) so that we don't publish and tag a release with bad files in it. This follows the pattern from https://github.com/fleetdm/fleet/issues/21300 as suggested by @iansltx. ## Testing I downloaded all of the latest release files to my local system using ```bash gh release download 202505190037 -D ~/Downloads/nvd ``` and then ran the validator on them with ```bash go run cmd/cpe/validate/main.go --db_dir ~/Downloads/nvd ``` To simulate file issues, I modified one section of each file to change a value into the wrong type, and validated that this caused the validator to panic. Examples: ``` panic: failed to load CPE translations: decode json: json: cannot unmarshal string into Go struct field CPETranslation.filter.vendor of type []string goroutine 1 [running]: main.checkCPETranslations({0x16dc975f9?, 0x14000192190?}) /Users/scott/Development/fleet/cmd/cpe/validate/main.go:34 +0xa8 main.main() /Users/scott/Development/fleet/cmd/cpe/validate/main.go:24 +0xb0 exit status 2 ``` --- ``` panic: failed to parse MacOffice release notes fleet_macoffice_release_notes_macoffice-2025_05_19.json: parsing time "xyz" as "2006-01-02T15:04:05Z07:00": cannot parse "xyz" as "2006" goroutine 1 [running]: main.checkMacOfficeNotes({0x16f7af5f9, 0x1a}) /Users/scott/Development/fleet/cmd/cpe/validate/main.go:56 +0x1f0 main.main() /Users/scott/Development/fleet/cmd/cpe/validate/main.go:25 +0xbc exit status 2 ``` --- ``` panic: failed to parse MSRC feed fleet_msrc_Windows_Server_2012_R2-2025_05_19.json: json: cannot unmarshal array into Go struct field Vulnerability.Vulnerabities.RemediatedBy of type bool goroutine 1 [running]: main.checkMSRCVulnerabilities({0x16f49b5f9, 0x1a}) /Users/scott/Development/fleet/cmd/cpe/validate/main.go:74 +0x1ac main.main() /Users/scott/Development/fleet/cmd/cpe/validate/main.go:26 +0xc8 exit status 2 ``` Additionally I tried the validator in [a run of the NVD workflow](https://github.com/fleetdm/nvd/actions/runs/15121687898/job/42505283781) and it executed successfully.
130 lines
3.8 KiB
Go
130 lines
3.8 KiB
Go
package main
|
|
|
|
import (
|
|
"compress/gzip"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/fleetdm/fleet/v4/server/vulnerabilities/macoffice"
|
|
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed"
|
|
"github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd"
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
func main() {
|
|
dbDir := flag.String("db_dir", "/tmp/vulndbs", "Path to the vulnerability database")
|
|
flag.Parse()
|
|
|
|
vulnPath := *dbDir
|
|
checkCPETranslations(vulnPath)
|
|
checkMacOfficeNotes(vulnPath)
|
|
checkMSRCVulnerabilities(vulnPath)
|
|
checkSqliteDb(vulnPath)
|
|
}
|
|
|
|
func checkCPETranslations(vulnPath string) {
|
|
// Check that the CPE translations file is parseable into an array of CPETranslationItem
|
|
_, err := nvd.LoadCPETranslations(filepath.Join(vulnPath, "cpe_translations.json"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to load CPE translations: %v", err))
|
|
}
|
|
}
|
|
|
|
func checkMacOfficeNotes(vulnPath string) {
|
|
// Iterate over each file in the vulnPath directory that starts with `fleet_macoffice`
|
|
files, err := os.ReadDir(vulnPath)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to read directory: %v", err))
|
|
}
|
|
for _, file := range files {
|
|
if strings.HasPrefix(file.Name(), "fleet_macoffice") && strings.HasSuffix(file.Name(), ".json") {
|
|
filePath := filepath.Join(vulnPath, file.Name())
|
|
|
|
payload, err := os.ReadFile(filePath)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to read MacOffice release notes file %s: %v", file.Name(), err))
|
|
}
|
|
// Attempt to parse the file as a MacOffice release notes.
|
|
relNotes := macoffice.ReleaseNotes{}
|
|
err = json.Unmarshal(payload, &relNotes)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to parse MacOffice release notes %s: %v", file.Name(), err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func checkMSRCVulnerabilities(vulnPath string) {
|
|
// Iterate over each file in the vulnPath directory that starts with `fleet_msrc`
|
|
files, err := os.ReadDir(vulnPath)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to read directory: %v", err))
|
|
}
|
|
for _, file := range files {
|
|
if strings.HasPrefix(file.Name(), "fleet_msrc") && strings.HasSuffix(file.Name(), ".json") {
|
|
filePath := filepath.Join(vulnPath, file.Name())
|
|
// Attempt to parse the file as a MSRC feed.
|
|
_, err := parsed.UnmarshalBulletin(filePath)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to parse MSRC feed %s: %v", file.Name(), err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func checkSqliteDb(vulnPath string) {
|
|
// Iterate over each file in the vulnPath directory to find the sqlite.gz file
|
|
files, err := os.ReadDir(vulnPath)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("failed to read directory: %v", err))
|
|
}
|
|
var sqliteFilename string
|
|
for _, file := range files {
|
|
if strings.HasSuffix(file.Name(), ".sqlite.gz") {
|
|
sqliteFilename = file.Name()
|
|
break
|
|
}
|
|
}
|
|
if sqliteFilename == "" {
|
|
panic(fmt.Sprintf("no sqlite.gz file found: %v", err))
|
|
}
|
|
// Unzip the sqlite.gz file and create a new sqlite.db file
|
|
gzFile, err := os.Open(filepath.Join(vulnPath, sqliteFilename))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("error opening sqlite.gz file: %v", err))
|
|
}
|
|
defer gzFile.Close()
|
|
sqliteFile, err := os.Create(filepath.Join(vulnPath, "sqlite.db"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("error creating test sqlite.db file: %v", err))
|
|
}
|
|
defer sqliteFile.Close()
|
|
gzReader, err := gzip.NewReader(gzFile)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("error creating new gzip reader: %v", err))
|
|
}
|
|
defer gzReader.Close()
|
|
for {
|
|
_, err := io.CopyN(sqliteFile, gzReader, 100*1024*1024)
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
panic(fmt.Sprintf("error unzipping sqlite file: %v", err))
|
|
}
|
|
}
|
|
db, err := sqlx.Open("sqlite3", filepath.Join(vulnPath, "sqlite.db"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("error opening sqlite db: %v", err))
|
|
}
|
|
// Check that the database is valid
|
|
_, err = db.Exec(`SELECT * FROM cpe_2 LIMIT 1`)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("error executing query on sqlite db: %v", err))
|
|
}
|
|
}
|