mirror of
https://github.com/fleetdm/fleet
synced 2026-05-24 09:28:54 +00:00
18439 Vulncheck data processing bug (#18440)
This commit is contained in:
Tim Lee
GitHub
parent
9c04f45818
commit
9e1878e01c
7 changed files with 214 additions and 53 deletions
|
|
@ -600,8 +600,6 @@ func (s *CVE) downloadVulnCheckArchive(ctx context.Context, downloadURL, outFile
|
|||
}
|
||||
|
||||
func (s *CVE) processVulnCheckFile(fileName string) error {
|
||||
cvesByYear := make(map[int][]VulnCheckCVE)
|
||||
|
||||
sanitizedPath, err := sanitizeArchivePath(s.dbDir, fileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error sanitizing archive path: %w", err)
|
||||
|
|
@ -617,15 +615,14 @@ func (s *CVE) processVulnCheckFile(fileName string) error {
|
|||
return zipReader.File[i].Name > zipReader.File[j].Name
|
||||
})
|
||||
|
||||
var data VulnCheckBackupDataFile
|
||||
var stopProcessing bool
|
||||
|
||||
// files are in reverse chronological order by modification date
|
||||
// so we can stop processing files once we find one that is older
|
||||
// than the configured vulnCheckStartDate
|
||||
var addCount int
|
||||
var modCount int
|
||||
for _, file := range zipReader.File {
|
||||
cvesByYear := make(map[int][]VulnCheckCVE)
|
||||
var stopProcessing bool
|
||||
|
||||
gzFile, err := file.Open()
|
||||
if err != nil {
|
||||
|
|
@ -637,6 +634,7 @@ func (s *CVE) processVulnCheckFile(fileName string) error {
|
|||
return fmt.Errorf("error creating gzip reader for file %s: %w", file.Name, err)
|
||||
}
|
||||
|
||||
var data VulnCheckBackupDataFile
|
||||
if err := json.NewDecoder(gReader).Decode(&data); err != nil {
|
||||
return fmt.Errorf("error decoding JSON from file %s: %w", file.Name, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -150,8 +150,10 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) {
|
|||
// gzip the vulncheck data
|
||||
testDataPath := filepath.Join("testdata", "cve", "vulncheck_test_data")
|
||||
nvdFile := filepath.Join(testDataPath, "nvdcve-1.1-2024.json")
|
||||
vulncheckFile := filepath.Join(testDataPath, "nvdcve-2.0-122.json")
|
||||
gzipFile := filepath.Join(testDataPath, "nvdcve-2.0-122.json.gz")
|
||||
vulncheckFile1 := filepath.Join(testDataPath, "nvdcve-2.0-122.json")
|
||||
vulncheckFile2 := filepath.Join(testDataPath, "nvdcve-2.0-121.json")
|
||||
gzipFile1 := filepath.Join(testDataPath, "nvdcve-2.0-122.json.gz")
|
||||
gzipFile2 := filepath.Join(testDataPath, "nvdcve-2.0-121.json.gz")
|
||||
zFile := filepath.Join(testDataPath, "vulncheck.zip")
|
||||
|
||||
// backup the original data to new directory
|
||||
|
|
@ -162,14 +164,20 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) {
|
|||
err = copyFile(nvdFile, filepath.Join(backupPath, "nvdcve-1.1-2024.json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = copyFile(vulncheckFile, filepath.Join(backupPath, "nvdcve-2.0-122.json"))
|
||||
err = copyFile(vulncheckFile1, filepath.Join(backupPath, "nvdcve-2.0-122.json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = copyFile(vulncheckFile2, filepath.Join(backupPath, "nvdcve-2.0-121.json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
// compress the vulncheck file to mimic the real data
|
||||
err = CompressFile(vulncheckFile, gzipFile)
|
||||
err = CompressFile(vulncheckFile1, gzipFile1)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = zipFile(gzipFile, zFile)
|
||||
err = CompressFile(vulncheckFile2, gzipFile2)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = zipFiles([]string{gzipFile1, gzipFile2}, zFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
|
|
@ -180,10 +188,16 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) {
|
|||
err = copyFile(filepath.Join(backupPath, "nvdcve-2.0-122.json"), filepath.Join(testDataPath, "nvdcve-2.0-122.json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = copyFile(filepath.Join(backupPath, "nvdcve-2.0-121.json"), filepath.Join(testDataPath, "nvdcve-2.0-121.json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = os.RemoveAll(backupPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = os.Remove(gzipFile)
|
||||
err = os.Remove(gzipFile1)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = os.Remove(gzipFile2)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = os.Remove(zFile)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"CVE_data_format": "MITRE",
|
||||
"CVE_data_numberOfCVEs": "5",
|
||||
"CVE_data_numberOfCVEs": "6",
|
||||
"CVE_data_timestamp": "2024-04-08T12:01:42Z",
|
||||
"CVE_data_type": "CVE",
|
||||
"CVE_data_version": "4.0",
|
||||
|
|
@ -90,7 +90,7 @@
|
|||
{
|
||||
"cpe_match": [
|
||||
{
|
||||
"cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"cpe23Uri": "cpe:2.3:a:0002foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"vulnerable": true
|
||||
}
|
||||
],
|
||||
|
|
@ -138,7 +138,7 @@
|
|||
{
|
||||
"cpe_match": [
|
||||
{
|
||||
"cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"cpe23Uri": "cpe:2.3:a:0003foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"vulnerable": true
|
||||
}
|
||||
],
|
||||
|
|
@ -223,7 +223,7 @@
|
|||
{
|
||||
"cpe_match": [
|
||||
{
|
||||
"cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"cpe23Uri": "cpe:2.3:a:0005foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"vulnerable": true
|
||||
},
|
||||
{
|
||||
|
|
@ -238,6 +238,43 @@
|
|||
"impact": {},
|
||||
"lastModifiedDate": "2024-04-03T17:24Z",
|
||||
"publishedDate": "2024-04-03T17:15Z"
|
||||
},
|
||||
{
|
||||
"cve": {
|
||||
"affects": null,
|
||||
"CVE_data_meta": {
|
||||
"ASSIGNER": "psirt@paloaltonetworks.com",
|
||||
"ID": "CVE-2024-0006"
|
||||
},
|
||||
"data_format": "MITRE",
|
||||
"data_type": "CVE",
|
||||
"data_version": "4.0",
|
||||
"description": {
|
||||
"description_data": [
|
||||
{
|
||||
"lang": "en",
|
||||
"value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes"
|
||||
}
|
||||
]
|
||||
},
|
||||
"problemtype": {
|
||||
"problemtype_data": []
|
||||
},
|
||||
"references": {
|
||||
"reference_data": [
|
||||
{
|
||||
"name": "https://foo.com/CVE-2024-0006",
|
||||
"url": "https://foo.com/CVE-2024-0006"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"configurations": {
|
||||
"CVE_data_version": "4.0"
|
||||
},
|
||||
"impact": {},
|
||||
"lastModifiedDate": "2024-02-15T06:23Z",
|
||||
"publishedDate": "2024-02-14T18:15Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,6 +87,43 @@
|
|||
"impact": {},
|
||||
"lastModifiedDate": "2024-02-15T06:23Z",
|
||||
"publishedDate": "2024-02-14T18:15Z"
|
||||
},
|
||||
{
|
||||
"cve": {
|
||||
"affects": null,
|
||||
"CVE_data_meta": {
|
||||
"ASSIGNER": "psirt@paloaltonetworks.com",
|
||||
"ID": "CVE-2024-0006"
|
||||
},
|
||||
"data_format": "MITRE",
|
||||
"data_type": "CVE",
|
||||
"data_version": "4.0",
|
||||
"description": {
|
||||
"description_data": [
|
||||
{
|
||||
"lang": "en",
|
||||
"value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes"
|
||||
}
|
||||
]
|
||||
},
|
||||
"problemtype": {
|
||||
"problemtype_data": []
|
||||
},
|
||||
"references": {
|
||||
"reference_data": [
|
||||
{
|
||||
"name": "https://foo.com/CVE-2024-0006",
|
||||
"url": "https://foo.com/CVE-2024-0006"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"configurations": {
|
||||
"CVE_data_version": "4.0"
|
||||
},
|
||||
"impact": {},
|
||||
"lastModifiedDate": "2024-02-15T06:23Z",
|
||||
"publishedDate": "2024-02-14T18:15Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
71
server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-121.json
vendored
Normal file
71
server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-121.json
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"resultsPerPage": 147,
|
||||
"startIndex": 244000,
|
||||
"totalResults": 244147,
|
||||
"format": "NVD_CVE",
|
||||
"version": "2.0",
|
||||
"timestamp": "2024-04-04T20:32:52.097",
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"cve": {
|
||||
"id": "CVE-2024-0006",
|
||||
"sourceIdentifier": "ff5b8ace-8b95-4078-9743-eac1ca5451de",
|
||||
"vulnStatus": "Awaiting Analysis",
|
||||
"published": "2024-04-03T19:15:44.387",
|
||||
"lastModified": "2024-04-04T12:48:41.700",
|
||||
"descriptions": [
|
||||
{
|
||||
"lang": "en",
|
||||
"value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes"
|
||||
}
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"url": "https://foo.com/CVE-2024-0006",
|
||||
"source": "ff5b8ace-8b95-4078-9743-eac1ca5451de"
|
||||
},
|
||||
{
|
||||
"url": "https://foo.com/CVE-2024-0006",
|
||||
"source": "ff5b8ace-8b95-4078-9743-eac1ca5451de"
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"cvssMetricV31": [
|
||||
{
|
||||
"source": "ff5b8ace-8b95-4078-9743-eac1ca5451de",
|
||||
"type": "Secondary",
|
||||
"cvssData": {
|
||||
"version": "3.1",
|
||||
"vectorString": "CVSS:3.1/AV:N/AC:H/PR:H/UI:R/S:U/C:N/I:L/A:L",
|
||||
"attackVector": "NETWORK",
|
||||
"attackComplexity": "HIGH",
|
||||
"privilegesRequired": "HIGH",
|
||||
"userInteraction": "REQUIRED",
|
||||
"scope": "UNCHANGED",
|
||||
"confidentialityImpact": "NONE",
|
||||
"integrityImpact": "LOW",
|
||||
"availabilityImpact": "LOW",
|
||||
"baseScore": 3.1,
|
||||
"baseSeverity": "LOW"
|
||||
},
|
||||
"exploitabilityScore": 0.5,
|
||||
"impactScore": 2.5
|
||||
}
|
||||
]
|
||||
},
|
||||
"weaknesses": [
|
||||
{
|
||||
"source": "ff5b8ace-8b95-4078-9743-eac1ca5451de",
|
||||
"type": "Secondary",
|
||||
"description": [
|
||||
{
|
||||
"lang": "en",
|
||||
"value": "CWE-20"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -73,14 +73,14 @@
|
|||
"cpeMatch": [
|
||||
{
|
||||
"vulnerable": true,
|
||||
"criteria": "cpe:2.3:a:concretecms:concrete_cms:*:*:*:*:*:*:*:*",
|
||||
"criteria": "cpe:2.3:a:0001foo:bar:*:*:*:*:*:*:*:*",
|
||||
"versionStartIncluding": "5.0.0",
|
||||
"versionEndExcluding": "8.5.16",
|
||||
"matchCriteriaId": ""
|
||||
},
|
||||
{
|
||||
"vulnerable": true,
|
||||
"criteria": "cpe:2.3:a:concretecms:concrete_cms:*:*:*:*:*:*:*:*",
|
||||
"criteria": "cpe:2.3:a:0001foo:bar:*:*:*:*:*:*:*:*",
|
||||
"versionStartIncluding": "9.0.0",
|
||||
"versionEndExcluding": "9.2.8",
|
||||
"matchCriteriaId": ""
|
||||
|
|
@ -154,7 +154,7 @@
|
|||
"cpeMatch": [
|
||||
{
|
||||
"vulnerable": true,
|
||||
"criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"criteria": "cpe:2.3:a:0002foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"matchCriteriaId": ""
|
||||
}
|
||||
]
|
||||
|
|
@ -229,7 +229,7 @@
|
|||
"cpeMatch": [
|
||||
{
|
||||
"vulnerable": true,
|
||||
"criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"criteria": "cpe:2.3:a:0003foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"matchCriteriaId": ""
|
||||
}
|
||||
]
|
||||
|
|
@ -360,7 +360,7 @@
|
|||
"cpeMatch": [
|
||||
{
|
||||
"vulnerable": true,
|
||||
"criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"criteria": "cpe:2.3:a:0005foo:bar:2023.2.0.21408:*:*:*:*:*:*:*",
|
||||
"matchCriteriaId": ""
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ func CompressFile(fileName string, newFileName string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func zipFile(source, target string) error {
|
||||
func zipFiles(sources []string, target string) error {
|
||||
// Create a new zip archive.
|
||||
zipFile, err := os.Create(target)
|
||||
if err != nil {
|
||||
|
|
@ -56,37 +56,41 @@ func zipFile(source, target string) error {
|
|||
zipWriter := zip.NewWriter(zipFile)
|
||||
defer zipWriter.Close()
|
||||
|
||||
// Add a file to the archive.
|
||||
fileToZip, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
for _, source := range sources {
|
||||
// Add a file to the archive.
|
||||
fileToZip, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fileToZip.Close()
|
||||
|
||||
// Get the file information.
|
||||
info, err := fileToZip.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Using FileInfoHeader() above only uses the basename of the file. If you want
|
||||
// to preserve the folder structure (for example, if you're zipping files from
|
||||
// a directory), you would need to set header.Name to the full path.
|
||||
header.Name = source
|
||||
|
||||
// Change to deflate to reduce file size but keep it compatible with unzip.
|
||||
header.Method = zip.Deflate
|
||||
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = io.Copy(writer, fileToZip); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer fileToZip.Close()
|
||||
|
||||
// Get the file information.
|
||||
info, err := fileToZip.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Using FileInfoHeader() above only uses the basename of the file. If you want
|
||||
// to preserve the folder structure (for example, if you're zipping files from
|
||||
// a directory), you would need to set header.Name to the full path.
|
||||
header.Name = source
|
||||
|
||||
// Change to deflate to reduce file size but keep it compatible with unzip.
|
||||
header.Method = zip.Deflate
|
||||
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, fileToZip)
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue