diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 1b2e7eabcf..9d3a896b9b 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -416,7 +416,7 @@ func checkNVDVulnerabilities( } } - if err := vulnerabilities.LoadCVEMeta(vulnPath, ds); err != nil { + if err := vulnerabilities.LoadCVEMeta(logger, vulnPath, ds); err != nil { err = fmt.Errorf("load cve meta: %w", err) level.Error(logger).Log("err", err) sentry.CaptureException(err) diff --git a/server/vulnerabilities/sync.go b/server/vulnerabilities/sync.go index f40ab998d6..5431cea7d6 100644 --- a/server/vulnerabilities/sync.go +++ b/server/vulnerabilities/sync.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/go-kit/kit/log/level" + "github.com/go-kit/kit/log" "github.com/facebookincubator/nvdtools/cvefeed" feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd" "github.com/fleetdm/fleet/v4/pkg/download" @@ -161,36 +163,47 @@ func DownloadCISAKnownExploitsFeed(vulnPath string, client *http.Client) error { // LoadCVEMeta loads the cvss scores, epss scores, and known exploits from the previously downloaded feeds and saves // them to the database. -func LoadCVEMeta(vulnPath string, ds fleet.Datastore) error { +func LoadCVEMeta(logger log.Logger, vulnPath string, ds fleet.Datastore) error { // load cvss scores files, err := getNVDCVEFeedFiles(vulnPath) if err != nil { return fmt.Errorf("get nvd cve feeds: %w", err) } - dict, err := cvefeed.LoadJSONDictionary(files...) - if err != nil { - return err - } - metaMap := make(map[string]fleet.CVEMeta) - for cve := range dict { - schema := dict[cve].(*feednvd.Vuln).Schema() - if schema.Impact.BaseMetricV3 == nil { - continue - } - baseScore := schema.Impact.BaseMetricV3.CVSSV3.BaseScore - published, err := time.Parse(publishedDateFmt, schema.PublishedDate) + + for _, file := range files { + + // Load json files one at a time. Attempting to load them all uses too much memory, > 1 GB. + dict, err := cvefeed.LoadJSONDictionary(file) if err != nil { - return fmt.Errorf("parse published_date: %w", err) + return err } - meta := fleet.CVEMeta{ - CVE: cve, - CVSSScore: &baseScore, - Published: &published, + for cve := range dict { + vuln, ok := dict[cve].(*feednvd.Vuln) + if !ok { + level.Error(logger).Log("msg", "unexpected type for Vuln interface", "cve", cve, "type", fmt.Sprintf("%T", dict[cve])) + continue + } + schema := vuln.Schema() + + meta := fleet.CVEMeta{ + CVE: cve, + } + + if schema.Impact.BaseMetricV3 != nil { + meta.CVSSScore = &schema.Impact.BaseMetricV3.CVSSV3.BaseScore + } + + if published, err := time.Parse(publishedDateFmt, schema.PublishedDate); err != nil { + level.Error(logger). Log("msg", "failed to parse published data", "cve", cve, "published_date", schema.PublishedDate, "err", err) + } else { + meta.Published = &published + } + + metaMap[cve] = meta } - metaMap[cve] = meta } // load epss scores diff --git a/server/vulnerabilities/sync_test.go b/server/vulnerabilities/sync_test.go index 549ebd0b75..4e569108dc 100644 --- a/server/vulnerabilities/sync_test.go +++ b/server/vulnerabilities/sync_test.go @@ -2,7 +2,6 @@ package vulnerabilities import ( "context" - "fmt" "path/filepath" "strings" "testing" @@ -11,6 +10,7 @@ import ( "github.com/fleetdm/fleet/v4/pkg/nettest" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" + "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" "github.com/tj/assert" ) @@ -50,7 +50,8 @@ func TestLoadCVEMeta(t *testing.T) { return nil } - err := LoadCVEMeta("testdata", ds) + logger := log.NewNopLogger() + err := LoadCVEMeta(logger, "testdata", ds) require.NoError(t, err) require.True(t, ds.InsertCVEMetaFuncInvoked) @@ -58,9 +59,6 @@ func TestLoadCVEMeta(t *testing.T) { metaMap := make(map[string]fleet.CVEMeta) for _, meta := range cveMeta { metaMap[meta.CVE] = meta - if meta.CVSSScore != nil { - fmt.Println(meta.CVE, *meta.CVSSScore) - } } meta := metaMap["CVE-2022-29676"]