fleet/cmd/cve/generate.go

package main

import (
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
	"github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd"
	nvdsync "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/sync"
	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
)

const emptyData = `{
  "CVE_data_type" : "CVE",
  "CVE_data_format" : "MITRE",
  "CVE_data_version" : "4.0",
  "CVE_data_numberOfCVEs" : "859",
  "CVE_data_timestamp" : "2023-11-17T19:00Z",
  "CVE_Items" : [ ]
}`

var cleanEnvVar = "VULNERABILITIES_CLEAN"

func main() {
	dbDir := flag.String("db_dir", "/tmp/vulndbs", "Path to the vulnerability database")
	debug := flag.Bool("debug", false, "Sets debug mode")
	flag.Parse()

	logger := log.NewJSONLogger(os.Stdout)
	if *debug {
		logger = level.NewFilter(logger, level.AllowDebug())
	} else {
		logger = level.NewFilter(logger, level.AllowInfo())
	}

	if err := os.MkdirAll(*dbDir, os.ModePerm); err != nil {
		panic(err)
	}

	if os.Getenv(cleanEnvVar) == "false" {
		logger.Log("msg", "Downloading latest release")
		maxRetries := 3
		for i := 0; i < maxRetries; i++ {
			err := downloadLatestRelease(*dbDir, *debug, logger)
			if err == nil {
				break
			}

			if i == maxRetries-1 {
				logger.Log("msg", "Failed to download latest release. Continuing with full NVD Sync", "err", err)
				break
			}

			logger.Log("msg", "Failed to download latest release. Retrying in 30 seconds", "err", err)
			time.Sleep(30 * time.Second)
		}
	}

	// Sync the CVE files
	if err := nvd.GenerateCVEFeeds(*dbDir, *debug, logger); err != nil {
		panic(err)
	}

	// Remove Vulncheck archive
	if err := os.RemoveAll(filepath.Join(*dbDir, "vulncheck.zip")); err != nil {
		logger.Log("msg", "Failed to remove vulncheck.zip", "err", err)
	}

	// Read in every cpe file and create a corresponding metadata file
	// nvd data feeds start in 2002
	logger.Log("msg", "Generating metadata files ...")
	const startingYear = 2002
	currentYear := time.Now().Year()
	if currentYear < startingYear {
		panic("system date is in the past, cannot continue")
	}
	entries := (currentYear - startingYear) + 1
	for i := 0; i < entries; i++ {
		year := startingYear + i
		suffix := strconv.Itoa(year)
		fileNameRaw := filepath.Join(*dbDir, fileFmt(suffix, "json", ""))
		fileName := filepath.Join(*dbDir, fileFmt(suffix, "json", "gz"))
		metaName := filepath.Join(*dbDir, fileFmt(suffix, "meta", ""))
		// skip if file does not exist
		if _, err := os.Stat(fileNameRaw); os.IsNotExist(err) {
			logger.Log("msg", "Skipping metadata generation for missing file", "file", fileNameRaw)
			continue
		}
		err := nvdsync.CompressFile(fileNameRaw, fileName)
		if err != nil {
			panic(err)
		}
		createMetadata(fileName, metaName)
	}

	// Create modified and recent files
	createEmptyFiles(*dbDir, "modified")
	createEmptyFiles(*dbDir, "recent")
}

func downloadLatestRelease(dbDir string, debug bool, logger log.Logger) error {
	// Download the latest release
	err := nvd.DownloadCVEFeed(dbDir, "", debug, logger)
	if err != nil {
		return fmt.Errorf("download cve feed: %w", err)
	}

	// gunzip json files
	files, err := filepath.Glob(filepath.Join(dbDir, "nvdcve-1.1-*.json.gz"))
	if err != nil {
		return fmt.Errorf("glob json files: %w", err)
	}
	for _, file := range files {
		err = gunzipFileToDisk(file, dbDir)
		if err != nil {
			return fmt.Errorf("gunzip file %s to disk: %w", file, err)
		}
	}

	// Download the last mod start date
	err = downloadLatestGitHubAsset(dbDir, "last_mod_start_date.txt")
	if err != nil {
		return fmt.Errorf("downloading last_mod_start_date asset: %w", err)
	}

	return nil
}

// downloadAsset downloads the asset from the latest release and writes it to a file
func downloadLatestGitHubAsset(dbDir, fileName string) error {
	assetPath, err := nvd.GetGitHubCVEAssetPath()
	if err != nil {
		return fmt.Errorf("get github cve asset path: %w", err)
	}

	client := fleethttp.NewClient()
	resp, err := client.Get(assetPath + fileName)
	if err != nil {
		return fmt.Errorf("get last mod start date: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("get last mod start date: %w", fmt.Errorf("unexpected status code %d", resp.StatusCode))
	}

	lastModStartDate, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("read last mod start date: %w", err)
	}

	// Write the last mod start date to a file
	lastModStartDateFile := filepath.Join(dbDir, fileName)
	err = os.WriteFile(lastModStartDateFile, lastModStartDate, 0o644)
	if err != nil {
		return fmt.Errorf("write last mod start date: %w", err)
	}

	return nil
}

func createMetadata(fileName string, metaName string) {
	fileInfo, err := os.Stat(fileName)
	if err != nil {
		panic(err)
	}
	hash, err := gunzipFileAndComputeSHA256(fileName)
	if err != nil {
		panic(err)
	}
	metaFile, err := os.Create(metaName)
	if err != nil {
		panic(err)
	}
	defer metaFile.Close()
	if _, err = metaFile.WriteString(fmt.Sprintf("gzSize:%v\r\n", fileInfo.Size())); err != nil {
		panic(err)
	}
	if _, err = metaFile.WriteString(fmt.Sprintf("sha256:%v\r\n", hash)); err != nil {
		panic(err)
	}
}

func createEmptyFiles(baseDir, suffix string) {
	fileName := filepath.Join(baseDir, fileFmt(suffix, "json", "gz"))
	metaName := filepath.Join(baseDir, fileFmt(suffix, "meta", ""))
	dataFile, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	writer := gzip.NewWriter(dataFile)
	if _, err = writer.Write([]byte(emptyData)); err != nil {
		panic(err)
	}
	if err = writer.Close(); err != nil {
		panic(err)
	}
	dataFile.Close()
	createMetadata(fileName, metaName)
}

func fileFmt(suffix, encoding, compression string) string {
	const version = "1.1"
	s := fmt.Sprintf("nvdcve-%s-%s.%s", version, suffix, encoding)
	if compression != "" {
		s += "." + compression
	}
	return s
}

func computeSHA256(r io.Reader) (string, error) {
	hashImpl := sha256.New()
	_, err := io.Copy(hashImpl, r)
	if err != nil {
		return "", err
	}
	hash := hashImpl.Sum(nil)
	return strings.ToUpper(hex.EncodeToString(hash)), nil
}

func gunzipAndComputeSHA256(r io.Reader) (string, error) {
	f, err := gzip.NewReader(r)
	if err != nil {
		return "", err
	}
	defer f.Close()
	return computeSHA256(f)
}

func gunzipFileAndComputeSHA256(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()
	return gunzipAndComputeSHA256(f)
}

func gunzipFileToDisk(filename, dbpath string) error {
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("open file: %w", err)
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return fmt.Errorf("new gzip reader: %w", err)
	}
	defer gz.Close()

	filepath := filepath.Join(dbpath, strings.TrimSuffix(filepath.Base(filename), ".gz"))

	out, err := os.Create(filepath)
	if err != nil {
		return fmt.Errorf("create file: %w", err)
	}
	defer out.Close()

	// Using a maxBytes limit to prevent decompression bombs: gosec G110
	maxBytes := 200 * 1024 * 1024 // 200MB
	_, err = io.CopyN(out, gz, int64(maxBytes))
	if err != nil && err != io.EOF {
		msg := fmt.Sprintf("error copying file %s: %v", f.Name(), err)
		panic(msg)
	}

	return nil
}