diff --git a/server/vulnerabilities/nvd/sync/cve_syncer.go b/server/vulnerabilities/nvd/sync/cve_syncer.go index c693463979..065d8faec8 100644 --- a/server/vulnerabilities/nvd/sync/cve_syncer.go +++ b/server/vulnerabilities/nvd/sync/cve_syncer.go @@ -600,8 +600,6 @@ func (s *CVE) downloadVulnCheckArchive(ctx context.Context, downloadURL, outFile } func (s *CVE) processVulnCheckFile(fileName string) error { - cvesByYear := make(map[int][]VulnCheckCVE) - sanitizedPath, err := sanitizeArchivePath(s.dbDir, fileName) if err != nil { return fmt.Errorf("error sanitizing archive path: %w", err) @@ -617,15 +615,14 @@ func (s *CVE) processVulnCheckFile(fileName string) error { return zipReader.File[i].Name > zipReader.File[j].Name }) - var data VulnCheckBackupDataFile - var stopProcessing bool - // files are in reverse chronological order by modification date // so we can stop processing files once we find one that is older // than the configured vulnCheckStartDate var addCount int var modCount int for _, file := range zipReader.File { + cvesByYear := make(map[int][]VulnCheckCVE) + var stopProcessing bool gzFile, err := file.Open() if err != nil { @@ -637,6 +634,7 @@ func (s *CVE) processVulnCheckFile(fileName string) error { return fmt.Errorf("error creating gzip reader for file %s: %w", file.Name, err) } + var data VulnCheckBackupDataFile if err := json.NewDecoder(gReader).Decode(&data); err != nil { return fmt.Errorf("error decoding JSON from file %s: %w", file.Name, err) } diff --git a/server/vulnerabilities/nvd/sync/cve_syncer_test.go b/server/vulnerabilities/nvd/sync/cve_syncer_test.go index 2c05860043..d58e7449c4 100644 --- a/server/vulnerabilities/nvd/sync/cve_syncer_test.go +++ b/server/vulnerabilities/nvd/sync/cve_syncer_test.go @@ -150,8 +150,10 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) { // gzip the vulncheck data testDataPath := filepath.Join("testdata", "cve", "vulncheck_test_data") nvdFile := filepath.Join(testDataPath, "nvdcve-1.1-2024.json") - vulncheckFile := filepath.Join(testDataPath, "nvdcve-2.0-122.json") - gzipFile := filepath.Join(testDataPath, "nvdcve-2.0-122.json.gz") + vulncheckFile1 := filepath.Join(testDataPath, "nvdcve-2.0-122.json") + vulncheckFile2 := filepath.Join(testDataPath, "nvdcve-2.0-121.json") + gzipFile1 := filepath.Join(testDataPath, "nvdcve-2.0-122.json.gz") + gzipFile2 := filepath.Join(testDataPath, "nvdcve-2.0-121.json.gz") zFile := filepath.Join(testDataPath, "vulncheck.zip") // backup the original data to new directory @@ -162,14 +164,20 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) { err = copyFile(nvdFile, filepath.Join(backupPath, "nvdcve-1.1-2024.json")) require.NoError(t, err) - err = copyFile(vulncheckFile, filepath.Join(backupPath, "nvdcve-2.0-122.json")) + err = copyFile(vulncheckFile1, filepath.Join(backupPath, "nvdcve-2.0-122.json")) + require.NoError(t, err) + + err = copyFile(vulncheckFile2, filepath.Join(backupPath, "nvdcve-2.0-121.json")) require.NoError(t, err) // compress the vulncheck file to mimic the real data - err = CompressFile(vulncheckFile, gzipFile) + err = CompressFile(vulncheckFile1, gzipFile1) require.NoError(t, err) - err = zipFile(gzipFile, zFile) + err = CompressFile(vulncheckFile2, gzipFile2) + require.NoError(t, err) + + err = zipFiles([]string{gzipFile1, gzipFile2}, zFile) require.NoError(t, err) defer func() { @@ -180,10 +188,16 @@ func TestEnhanceNVDwithVulncheck(t *testing.T) { err = copyFile(filepath.Join(backupPath, "nvdcve-2.0-122.json"), filepath.Join(testDataPath, "nvdcve-2.0-122.json")) require.NoError(t, err) + err = copyFile(filepath.Join(backupPath, "nvdcve-2.0-121.json"), filepath.Join(testDataPath, "nvdcve-2.0-121.json")) + require.NoError(t, err) + err = os.RemoveAll(backupPath) require.NoError(t, err) - err = os.Remove(gzipFile) + err = os.Remove(gzipFile1) + require.NoError(t, err) + + err = os.Remove(gzipFile2) require.NoError(t, err) err = os.Remove(zFile) diff --git a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024-expected.json b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024-expected.json index c7a1750f97..afc2ea80f5 100644 --- a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024-expected.json +++ b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024-expected.json @@ -1,6 +1,6 @@ { "CVE_data_format": "MITRE", - "CVE_data_numberOfCVEs": "5", + "CVE_data_numberOfCVEs": "6", "CVE_data_timestamp": "2024-04-08T12:01:42Z", "CVE_data_type": "CVE", "CVE_data_version": "4.0", @@ -90,7 +90,7 @@ { "cpe_match": [ { - "cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "cpe23Uri": "cpe:2.3:a:0002foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "vulnerable": true } ], @@ -138,7 +138,7 @@ { "cpe_match": [ { - "cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "cpe23Uri": "cpe:2.3:a:0003foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "vulnerable": true } ], @@ -223,7 +223,7 @@ { "cpe_match": [ { - "cpe23Uri": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "cpe23Uri": "cpe:2.3:a:0005foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "vulnerable": true }, { @@ -238,6 +238,43 @@ "impact": {}, "lastModifiedDate": "2024-04-03T17:24Z", "publishedDate": "2024-04-03T17:15Z" + }, + { + "cve": { + "affects": null, + "CVE_data_meta": { + "ASSIGNER": "psirt@paloaltonetworks.com", + "ID": "CVE-2024-0006" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "en", + "value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes" + } + ] + }, + "problemtype": { + "problemtype_data": [] + }, + "references": { + "reference_data": [ + { + "name": "https://foo.com/CVE-2024-0006", + "url": "https://foo.com/CVE-2024-0006" + } + ] + } + }, + "configurations": { + "CVE_data_version": "4.0" + }, + "impact": {}, + "lastModifiedDate": "2024-02-15T06:23Z", + "publishedDate": "2024-02-14T18:15Z" } ] } diff --git a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024.json b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024.json index c43698ef81..a77b2d176f 100644 --- a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024.json +++ b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-1.1-2024.json @@ -87,6 +87,43 @@ "impact": {}, "lastModifiedDate": "2024-02-15T06:23Z", "publishedDate": "2024-02-14T18:15Z" + }, + { + "cve": { + "affects": null, + "CVE_data_meta": { + "ASSIGNER": "psirt@paloaltonetworks.com", + "ID": "CVE-2024-0006" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "en", + "value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes" + } + ] + }, + "problemtype": { + "problemtype_data": [] + }, + "references": { + "reference_data": [ + { + "name": "https://foo.com/CVE-2024-0006", + "url": "https://foo.com/CVE-2024-0006" + } + ] + } + }, + "configurations": { + "CVE_data_version": "4.0" + }, + "impact": {}, + "lastModifiedDate": "2024-02-15T06:23Z", + "publishedDate": "2024-02-14T18:15Z" } ] } diff --git a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-121.json b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-121.json new file mode 100644 index 0000000000..41b7651349 --- /dev/null +++ b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-121.json @@ -0,0 +1,71 @@ +{ + "resultsPerPage": 147, + "startIndex": 244000, + "totalResults": 244147, + "format": "NVD_CVE", + "version": "2.0", + "timestamp": "2024-04-04T20:32:52.097", + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0006", + "sourceIdentifier": "ff5b8ace-8b95-4078-9743-eac1ca5451de", + "vulnStatus": "Awaiting Analysis", + "published": "2024-04-03T19:15:44.387", + "lastModified": "2024-04-04T12:48:41.700", + "descriptions": [ + { + "lang": "en", + "value": "A CVE in file nvd-2.0-121.json that shouldn't populate configuration nodes" + } + ], + "references": [ + { + "url": "https://foo.com/CVE-2024-0006", + "source": "ff5b8ace-8b95-4078-9743-eac1ca5451de" + }, + { + "url": "https://foo.com/CVE-2024-0006", + "source": "ff5b8ace-8b95-4078-9743-eac1ca5451de" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "source": "ff5b8ace-8b95-4078-9743-eac1ca5451de", + "type": "Secondary", + "cvssData": { + "version": "3.1", + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:H/UI:R/S:U/C:N/I:L/A:L", + "attackVector": "NETWORK", + "attackComplexity": "HIGH", + "privilegesRequired": "HIGH", + "userInteraction": "REQUIRED", + "scope": "UNCHANGED", + "confidentialityImpact": "NONE", + "integrityImpact": "LOW", + "availabilityImpact": "LOW", + "baseScore": 3.1, + "baseSeverity": "LOW" + }, + "exploitabilityScore": 0.5, + "impactScore": 2.5 + } + ] + }, + "weaknesses": [ + { + "source": "ff5b8ace-8b95-4078-9743-eac1ca5451de", + "type": "Secondary", + "description": [ + { + "lang": "en", + "value": "CWE-20" + } + ] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-122.json b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-122.json index 604b4baa95..0662813cdd 100644 --- a/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-122.json +++ b/server/vulnerabilities/nvd/sync/testdata/cve/vulncheck_test_data/nvdcve-2.0-122.json @@ -73,14 +73,14 @@ "cpeMatch": [ { "vulnerable": true, - "criteria": "cpe:2.3:a:concretecms:concrete_cms:*:*:*:*:*:*:*:*", + "criteria": "cpe:2.3:a:0001foo:bar:*:*:*:*:*:*:*:*", "versionStartIncluding": "5.0.0", "versionEndExcluding": "8.5.16", "matchCriteriaId": "" }, { "vulnerable": true, - "criteria": "cpe:2.3:a:concretecms:concrete_cms:*:*:*:*:*:*:*:*", + "criteria": "cpe:2.3:a:0001foo:bar:*:*:*:*:*:*:*:*", "versionStartIncluding": "9.0.0", "versionEndExcluding": "9.2.8", "matchCriteriaId": "" @@ -154,7 +154,7 @@ "cpeMatch": [ { "vulnerable": true, - "criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "criteria": "cpe:2.3:a:0002foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "matchCriteriaId": "" } ] @@ -229,7 +229,7 @@ "cpeMatch": [ { "vulnerable": true, - "criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "criteria": "cpe:2.3:a:0003foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "matchCriteriaId": "" } ] @@ -360,7 +360,7 @@ "cpeMatch": [ { "vulnerable": true, - "criteria": "cpe:2.3:a:foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", + "criteria": "cpe:2.3:a:0005foo:bar:2023.2.0.21408:*:*:*:*:*:*:*", "matchCriteriaId": "" }, { diff --git a/server/vulnerabilities/nvd/sync/utils.go b/server/vulnerabilities/nvd/sync/utils.go index cb0545ce73..e2b127e415 100644 --- a/server/vulnerabilities/nvd/sync/utils.go +++ b/server/vulnerabilities/nvd/sync/utils.go @@ -45,7 +45,7 @@ func CompressFile(fileName string, newFileName string) error { return nil } -func zipFile(source, target string) error { +func zipFiles(sources []string, target string) error { // Create a new zip archive. zipFile, err := os.Create(target) if err != nil { @@ -56,37 +56,41 @@ func zipFile(source, target string) error { zipWriter := zip.NewWriter(zipFile) defer zipWriter.Close() - // Add a file to the archive. - fileToZip, err := os.Open(source) - if err != nil { - return err + for _, source := range sources { + // Add a file to the archive. + fileToZip, err := os.Open(source) + if err != nil { + return err + } + defer fileToZip.Close() + + // Get the file information. + info, err := fileToZip.Stat() + if err != nil { + return err + } + + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + // Using FileInfoHeader() above only uses the basename of the file. If you want + // to preserve the folder structure (for example, if you're zipping files from + // a directory), you would need to set header.Name to the full path. + header.Name = source + + // Change to deflate to reduce file size but keep it compatible with unzip. + header.Method = zip.Deflate + + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + + if _, err = io.Copy(writer, fileToZip); err != nil { + return err + } } - defer fileToZip.Close() - - // Get the file information. - info, err := fileToZip.Stat() - if err != nil { - return err - } - - header, err := zip.FileInfoHeader(info) - if err != nil { - return err - } - - // Using FileInfoHeader() above only uses the basename of the file. If you want - // to preserve the folder structure (for example, if you're zipping files from - // a directory), you would need to set header.Name to the full path. - header.Name = source - - // Change to deflate to reduce file size but keep it compatible with unzip. - header.Method = zip.Deflate - - writer, err := zipWriter.CreateHeader(header) - if err != nil { - return err - } - - _, err = io.Copy(writer, fileToZip) - return err + return nil }