fix mem usage cve sync (#6131)

cmd/fleet/cron.go

@@ -416,7 +416,7 @@ func checkNVDVulnerabilities(
 		}
 	}
 
-	if err := vulnerabilities.LoadCVEMeta(vulnPath, ds); err != nil {
+	if err := vulnerabilities.LoadCVEMeta(logger, vulnPath, ds); err != nil {
 		err = fmt.Errorf("load cve meta: %w", err)
 		level.Error(logger).Log("err", err)
 		sentry.CaptureException(err)

server/vulnerabilities/sync.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/kit/log"
 	"github.com/facebookincubator/nvdtools/cvefeed"
 	feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
 	"github.com/fleetdm/fleet/v4/pkg/download"
@@ -161,36 +163,47 @@ func DownloadCISAKnownExploitsFeed(vulnPath string, client *http.Client) error {
 
 // LoadCVEMeta loads the cvss scores, epss scores, and known exploits from the previously downloaded feeds and saves
 // them to the database.
-func LoadCVEMeta(vulnPath string, ds fleet.Datastore) error {
+func LoadCVEMeta(logger log.Logger, vulnPath string, ds fleet.Datastore) error {
 	// load cvss scores
 	files, err := getNVDCVEFeedFiles(vulnPath)
 	if err != nil {
 		return fmt.Errorf("get nvd cve feeds: %w", err)
 	}
 
-	dict, err := cvefeed.LoadJSONDictionary(files...)
-	if err != nil {
-		return err
-	}
-
 	metaMap := make(map[string]fleet.CVEMeta)
-	for cve := range dict {
-		schema := dict[cve].(*feednvd.Vuln).Schema()
-		if schema.Impact.BaseMetricV3 == nil {
-			continue
-		}
-		baseScore := schema.Impact.BaseMetricV3.CVSSV3.BaseScore
-		published, err := time.Parse(publishedDateFmt, schema.PublishedDate)
+
+	for _, file := range files {
+
+		// Load json files one at a time. Attempting to load them all uses too much memory, > 1 GB.
+		dict, err := cvefeed.LoadJSONDictionary(file)
 		if err != nil {
-			return fmt.Errorf("parse published_date: %w", err)
+			return err
 		}
 
-		meta := fleet.CVEMeta{
-			CVE:       cve,
-			CVSSScore: &baseScore,
-			Published: &published,
+		for cve := range dict {
+			vuln, ok := dict[cve].(*feednvd.Vuln)
+			if !ok {
+				level.Error(logger).Log("msg", "unexpected type for Vuln interface", "cve", cve, "type", fmt.Sprintf("%T", dict[cve]))
+				continue
+			}
+			schema := vuln.Schema()
+
+			meta := fleet.CVEMeta{
+				CVE:       cve,
+			}
+
+			if schema.Impact.BaseMetricV3 != nil {
+				meta.CVSSScore = &schema.Impact.BaseMetricV3.CVSSV3.BaseScore
+			}
+
+			if published, err := time.Parse(publishedDateFmt, schema.PublishedDate); err != nil {
+				level.Error(logger). Log("msg", "failed to parse published data", "cve", cve, "published_date", schema.PublishedDate, "err", err)
+			} else {
+				meta.Published = &published
+			}
+
+			metaMap[cve] = meta
 		}
-		metaMap[cve] = meta
 	}
 
 	// load epss scores

server/vulnerabilities/sync_test.go

@@ -2,7 +2,6 @@ package vulnerabilities
 
 import (
 	"context"
-	"fmt"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -11,6 +10,7 @@ import (
 	"github.com/fleetdm/fleet/v4/pkg/nettest"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/mock"
+	"github.com/go-kit/kit/log"
 	"github.com/stretchr/testify/require"
 	"github.com/tj/assert"
 )
@@ -50,7 +50,8 @@ func TestLoadCVEMeta(t *testing.T) {
 		return nil
 	}
 
-	err := LoadCVEMeta("testdata", ds)
+	logger := log.NewNopLogger()
+	err := LoadCVEMeta(logger, "testdata", ds)
 	require.NoError(t, err)
 	require.True(t, ds.InsertCVEMetaFuncInvoked)
 
@@ -58,9 +59,6 @@ func TestLoadCVEMeta(t *testing.T) {
 	metaMap := make(map[string]fleet.CVEMeta)
 	for _, meta := range cveMeta {
 		metaMap[meta.CVE] = meta
-		if meta.CVSSScore != nil {
-			fmt.Println(meta.CVE, *meta.CVSSScore)
-		}
 	}
 
 	meta := metaMap["CVE-2022-29676"]