Process stored CPEs and store found CVEs (#1533)

* WIP

* WIP

* Make path optional and fix tests

* Add first generate

* Move to nvd package

* remove replace

* Re-add replace

* It's path, not file name

* Change how db path is set and use etag

* Fix typos

* Make db generation faster

* Remove quotes

* Doesn't like comments

* Samitize etag and save to file

* Refactor some things and improve writing of etagenv

* Compress file and truncate amount of items for faster testing

* Remove quotes

* Try to improve performance

* Ignore truncate error if not exists

* Minor cleanup and make sqlite have cpe prefix

* Simplify code and test sync

* Add VCR for sync test

* Check for nvdRelease nil

* Add test for the actual translation

* Address review comments

* Rename generate command because we'll have a cve one too

* Move to its own dir

* Add first cve db generation

* WIP but with final strategy, preparring to merge main

* Fix merge conflicts

* WIP

* wip

* Insert CVEs to the db

* Remove unused code

* Use wg instead of counting

* Call cancelFunc to avoid ctx leak

* Fix logs for better readability

* Point code to fleetdm instead of my repo
This commit is contained in:
Tomas Touceda 2021-08-04 18:01:39 -03:00 committed by GitHub
parent d5974aad97
commit f8b7a83cc6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 458 additions and 237 deletions

View file

@ -0,0 +1 @@
* Converts all stored CPEs into their corresponding CVEs if any are present. Fixes issue 1436

View file

@ -491,16 +491,18 @@ func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
}
func cronVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, locker Locker, identifier string) {
config, err := ds.AppConfig()
appConfig, err := ds.AppConfig()
if err != nil {
level.Error(logger).Log("config", "couldn't read app config", "err", err)
return
}
if config.VulnerabilityDatabasesPath == nil {
if appConfig.VulnerabilityDatabasesPath == nil {
level.Info(logger).Log("vulnerability scanning", "not configured")
return
}
vulnPath := *appConfig.VulnerabilityDatabasesPath
ticker := time.NewTicker(1 * time.Hour)
for {
level.Debug(logger).Log("waiting", "on ticker")
@ -516,10 +518,16 @@ func cronVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.
continue
}
err := vulnerabilities.TranslateSoftwareToCPE(ds)
err := vulnerabilities.TranslateSoftwareToCPE(ds, vulnPath)
if err != nil {
level.Error(logger).Log("err", "analyzing vulnerable software", "details", err)
level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err)
}
err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger)
if err != nil {
level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err)
}
level.Debug(logger).Log("loop", "done")
}
}

4
go.mod
View file

@ -21,6 +21,7 @@ require (
github.com/dnaeon/go-vcr/v2 v2.0.1
github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20
github.com/elazarl/go-bindata-assetfs v1.0.0
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c // indirect
github.com/facebookincubator/nvdtools v0.1.4
github.com/fatih/color v1.10.0 // indirect
github.com/fleetdm/goose v0.0.0-20210209032905-c3c01484bacb
@ -35,6 +36,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.4.2
github.com/gosuri/uilive v0.0.4
github.com/groob/mockimpl v0.0.0-20170306012045-dfa944a2a940 // indirect
github.com/igm/sockjs-go/v3 v3.0.0
github.com/jmoiron/sqlx v1.2.0
github.com/jonboulle/clockwork v0.2.2 // indirect
@ -66,7 +68,7 @@ require (
github.com/urfave/cli/v2 v2.3.0
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/tools v0.1.4 // indirect
golang.org/x/tools v0.1.5 // indirect
google.golang.org/grpc v1.38.0
gopkg.in/guregu/null.v3 v3.4.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3

8
go.sum
View file

@ -154,6 +154,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:Q
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c h1:KqlxcP2nuOcMjudCvK0qME2K/aFBDH+xcvYv7HYQaYc=
github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk=
github.com/facebookincubator/nvdtools v0.1.4 h1:x1Ucw9+bSkMd8DJJN4jNQ1Lk4PSFlJarGOxp9D6WUMo=
github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0iUceDhdSHPlIwIX44Y=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@ -296,6 +298,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
github.com/groob/mockimpl v0.0.0-20170306012045-dfa944a2a940 h1:7qYt+uqKEGE5yHfRFOsgG6b8sW0qMSsNU50GbfskYAI=
github.com/groob/mockimpl v0.0.0-20170306012045-dfa944a2a940/go.mod h1:KeaEsoeCyhGRrPvJFbO+SMJfKLRqNGzL22LObUSCY38=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA=
@ -540,6 +544,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sasha-s/goimpl v0.0.0-20170811183550-a4e1c77df6c3 h1:nq8L+fiTb/lfzCyNpyaOGUBuW3h7Cq/9N0L+DjXTaSg=
github.com/sasha-s/goimpl v0.0.0-20170811183550-a4e1c77df6c3/go.mod h1:ILqB0kExSd5VozPhLcQclmxqKCd/HOsC/ocRk0HWLr8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
@ -881,6 +887,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View file

@ -0,0 +1,34 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20210802135933, Down_20210802135933)
}
func Up_20210802135933(tx *sql.Tx) error {
sql := `
CREATE TABLE IF NOT EXISTS software_cve (
id int(10) unsigned NOT NULL AUTO_INCREMENT,
cpe_id int(10) unsigned,
cve varchar(255) NOT NULL,
created_at timestamp DEFAULT CURRENT_TIMESTAMP,
updated_at timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY fk_software_cve_cpe_id (cpe_id) REFERENCES software_cpe(id) ON DELETE CASCADE,
UNIQUE KEY unique_cpe_cve(cpe_id, cve)
)
`
if _, err := tx.Exec(sql); err != nil {
return errors.Wrap(err, "create cve")
}
return nil
}
func Down_20210802135933(tx *sql.Tx) error {
return nil
}

View file

@ -273,3 +273,27 @@ func (d *Datastore) AddCPEForSoftware(software fleet.Software, cpe string) error
}
return nil
}
func (d *Datastore) AllCPEs() ([]string, error) {
sql := `SELECT cpe FROM software_cpe`
var cpes []string
err := d.db.Select(cpes, sql)
if err != nil {
return nil, errors.Wrap(err, "loads cpes")
}
return cpes, nil
}
func (d *Datastore) InsertCVEForCPE(cve string, cpes []string) error {
values := strings.TrimSuffix(strings.Repeat("((SELECT id FROM software_cpe WHERE cpe=?),?),", len(cpes)), ",")
sql := fmt.Sprintf(`INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES %s`, values)
var args []interface{}
for _, cpe := range cpes {
args = append(args, cpe, cve)
}
_, err := d.db.Exec(sql, args...)
if err != nil {
return errors.Wrap(err, "insert software cve")
}
return nil
}

View file

@ -168,6 +168,27 @@ func TestSoftwareCPE(t *testing.T) {
assert.Equal(t, len(host1.Software)-1, loops)
}
func TestInsertCVEs(t *testing.T) {
ds := CreateMySQLDS(t)
defer ds.Close()
host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())
soft := fleet.HostSoftware{
Modified: true,
Software: []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
},
}
host.HostSoftware = soft
require.NoError(t, ds.SaveHostSoftware(host))
require.NoError(t, ds.LoadHostSoftware(host))
require.NoError(t, ds.AddCPEForSoftware(host.Software[0], "somecpe"))
require.NoError(t, ds.InsertCVEForCPE("cve-123-123-132", []string{"somecpe"}))
}
func TestHostSoftwareDuplicates(t *testing.T) {
ds := CreateMySQLDS(t)
defer ds.Close()

View file

@ -5,6 +5,8 @@ type SoftwareStore interface {
LoadHostSoftware(host *Host) error
AllSoftwareWithoutCPEIterator() (SoftwareIterator, error)
AddCPEForSoftware(software Software, cpe string) error
AllCPEs() ([]string, error)
InsertCVEForCPE(cve string, cpes []string) error
}
// Software is a named and versioned piece of software installed on a device.

View file

@ -7,7 +7,7 @@ import (
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/health"
"github.com/fleetdm/fleet/v4/server/mock"
"github.com/fleetdm/fleet/v4/server/service/mock"
"github.com/go-kit/kit/log"
"github.com/kolide/osquery-go/plugin/distributed"
"github.com/stretchr/testify/assert"

View file

@ -2,21 +2,23 @@ package mock
import "github.com/fleetdm/fleet/v4/server/fleet"
//go:generate mockimpl -o datastore_users.go "s *UserStore" "fleet.UserStore"
//go:generate mockimpl -o datastore_invites.go "s *InviteStore" "fleet.InviteStore"
//go:generate mockimpl -o datastore_activities.go "s *ActivitiesStore" "fleet.ActivitiesStore"
//go:generate mockimpl -o datastore_appconfig.go "s *AppConfigStore" "fleet.AppConfigStore"
//go:generate mockimpl -o datastore_labels.go "s *LabelStore" "fleet.LabelStore"
//go:generate mockimpl -o datastore_options.go "s *OptionStore" "fleet.OptionStore"
//go:generate mockimpl -o datastore_packs.go "s *PackStore" "fleet.PackStore"
//go:generate mockimpl -o datastore_campaigns.go "s *CampaignStore" "fleet.CampaignStore"
//go:generate mockimpl -o datastore_carves.go "s *CarveStore" "fleet.CarveStore"
//go:generate mockimpl -o datastore_hosts.go "s *HostStore" "fleet.HostStore"
//go:generate mockimpl -o datastore_fim.go "s *FileIntegrityMonitoringStore" "fleet.FileIntegrityMonitoringStore"
//go:generate mockimpl -o datastore_osquery_options.go "s *OsqueryOptionsStore" "fleet.OsqueryOptionsStore"
//go:generate mockimpl -o datastore_scheduled_queries.go "s *ScheduledQueryStore" "fleet.ScheduledQueryStore"
//go:generate mockimpl -o datastore_invites.go "s *InviteStore" "fleet.InviteStore"
//go:generate mockimpl -o datastore_labels.go "s *LabelStore" "fleet.LabelStore"
//go:generate mockimpl -o datastore_packs.go "s *PackStore" "fleet.PackStore"
//go:generate mockimpl -o datastore_queries.go "s *QueryStore" "fleet.QueryStore"
//go:generate mockimpl -o datastore_query_results.go "s *QueryResultStore" "fleet.QueryResultStore"
//go:generate mockimpl -o datastore_campaigns.go "s *CampaignStore" "fleet.CampaignStore"
//go:generate mockimpl -o datastore_scheduled_queries.go "s *ScheduledQueryStore" "fleet.ScheduledQueryStore"
//go:generate mockimpl -o datastore_sessions.go "s *SessionStore" "fleet.SessionStore"
//go:generate mockimpl -o datastore_activities.go "s *ActivitiesStore" "fleet.ActivitiesStore"
//go:generate mockimpl -o datastore_software.go "s *SoftwareStore" "fleet.SoftwareStore"
//go:generate mockimpl -o datastore_statistics.go "s *StatisticsStore" "fleet.StatisticsStore"
//go:generate mockimpl -o datastore_targets.go "s *TargetStore" "fleet.TargetStore"
//go:generate mockimpl -o datastore_teams.go "s *TeamStore" "fleet.TeamStore"
//go:generate mockimpl -o datastore_users.go "s *UserStore" "fleet.UserStore"
var _ fleet.Datastore = (*Store)(nil)

View file

@ -7,6 +7,7 @@ import "github.com/fleetdm/fleet/v4/server/fleet"
var _ fleet.ActivitiesStore = (*ActivitiesStore)(nil)
type NewActivityFunc func(user *fleet.User, activityType string, details *map[string]interface{}) error
type ListActivitiesFunc func(opt fleet.ListOptions) ([]*fleet.Activity, error)
type ActivitiesStore struct {

View file

@ -14,10 +14,10 @@ type SaveAppConfigFunc func(info *fleet.AppConfig) error
type VerifyEnrollSecretFunc func(secret string) (*fleet.EnrollSecret, error)
type ApplyEnrollSecretsFunc func(treamID *uint, secrets []*fleet.EnrollSecret) error
type GetEnrollSecretsFunc func(teamID *uint) ([]*fleet.EnrollSecret, error)
type ApplyEnrollSecretsFunc func(teamID *uint, secrets []*fleet.EnrollSecret) error
type AppConfigStore struct {
NewAppConfigFunc NewAppConfigFunc
NewAppConfigFuncInvoked bool
@ -31,11 +31,11 @@ type AppConfigStore struct {
VerifyEnrollSecretFunc VerifyEnrollSecretFunc
VerifyEnrollSecretFuncInvoked bool
ApplyEnrollSecretsFunc ApplyEnrollSecretsFunc
ApplyEnrollSecretsFuncInvoked bool
GetEnrollSecretsFunc GetEnrollSecretsFunc
GetEnrollSecretsFuncInvoked bool
ApplyEnrollSecretsFunc ApplyEnrollSecretsFunc
ApplyEnrollSecretsFuncInvoked bool
}
func (s *AppConfigStore) NewAppConfig(info *fleet.AppConfig) (*fleet.AppConfig, error) {
@ -58,12 +58,12 @@ func (s *AppConfigStore) VerifyEnrollSecret(secret string) (*fleet.EnrollSecret,
return s.VerifyEnrollSecretFunc(secret)
}
func (s *AppConfigStore) ApplyEnrollSecrets(teamID *uint, secrets []*fleet.EnrollSecret) error {
s.ApplyEnrollSecretsFuncInvoked = true
return s.ApplyEnrollSecretsFunc(teamID, secrets)
}
func (s *AppConfigStore) GetEnrollSecrets(teamID *uint) ([]*fleet.EnrollSecret, error) {
s.GetEnrollSecretsFuncInvoked = true
return s.GetEnrollSecretsFunc(teamID)
}
func (s *AppConfigStore) ApplyEnrollSecrets(teamID *uint, secrets []*fleet.EnrollSecret) error {
s.ApplyEnrollSecretsFuncInvoked = true
return s.ApplyEnrollSecretsFunc(teamID, secrets)
}

View file

@ -10,18 +10,18 @@ import (
var _ fleet.CarveStore = (*CarveStore)(nil)
type NewCarveFunc func(c *fleet.CarveMetadata) (*fleet.CarveMetadata, error)
type NewCarveFunc func(metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error)
type UpdateCarveFunc func(c *fleet.CarveMetadata) error
type UpdateCarveFunc func(metadata *fleet.CarveMetadata) error
type CarveFunc func(carveId int64) (*fleet.CarveMetadata, error)
type ListCarvesFunc func(opt fleet.CarveListOptions) ([]*fleet.CarveMetadata, error)
type CarveBySessionIdFunc func(sessionId string) (*fleet.CarveMetadata, error)
type CarveByNameFunc func(name string) (*fleet.CarveMetadata, error)
type ListCarvesFunc func(opt fleet.CarveListOptions) ([]*fleet.CarveMetadata, error)
type NewBlockFunc func(metadata *fleet.CarveMetadata, blockId int64, data []byte) error
type GetBlockFunc func(metadata *fleet.CarveMetadata, blockId int64) ([]byte, error)
@ -38,15 +38,15 @@ type CarveStore struct {
CarveFunc CarveFunc
CarveFuncInvoked bool
ListCarvesFunc ListCarvesFunc
ListCarvesFuncInvoked bool
CarveBySessionIdFunc CarveBySessionIdFunc
CarveBySessionIdFuncInvoked bool
CarveByNameFunc CarveByNameFunc
CarveByNameFuncInvoked bool
ListCarvesFunc ListCarvesFunc
ListCarvesFuncInvoked bool
NewBlockFunc NewBlockFunc
NewBlockFuncInvoked bool
@ -57,14 +57,14 @@ type CarveStore struct {
CleanupCarvesFuncInvoked bool
}
func (s *CarveStore) NewCarve(c *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
func (s *CarveStore) NewCarve(metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
s.NewCarveFuncInvoked = true
return s.NewCarveFunc(c)
return s.NewCarveFunc(metadata)
}
func (s *CarveStore) UpdateCarve(c *fleet.CarveMetadata) error {
func (s *CarveStore) UpdateCarve(metadata *fleet.CarveMetadata) error {
s.UpdateCarveFuncInvoked = true
return s.UpdateCarveFunc(c)
return s.UpdateCarveFunc(metadata)
}
func (s *CarveStore) Carve(carveId int64) (*fleet.CarveMetadata, error) {
@ -72,11 +72,6 @@ func (s *CarveStore) Carve(carveId int64) (*fleet.CarveMetadata, error) {
return s.CarveFunc(carveId)
}
func (s *CarveStore) ListCarves(opt fleet.CarveListOptions) ([]*fleet.CarveMetadata, error) {
s.ListCarvesFuncInvoked = true
return s.ListCarvesFunc(opt)
}
func (s *CarveStore) CarveBySessionId(sessionId string) (*fleet.CarveMetadata, error) {
s.CarveBySessionIdFuncInvoked = true
return s.CarveBySessionIdFunc(sessionId)
@ -87,6 +82,11 @@ func (s *CarveStore) CarveByName(name string) (*fleet.CarveMetadata, error) {
return s.CarveByNameFunc(name)
}
func (s *CarveStore) ListCarves(opt fleet.CarveListOptions) ([]*fleet.CarveMetadata, error) {
s.ListCarvesFuncInvoked = true
return s.ListCarvesFunc(opt)
}
func (s *CarveStore) NewBlock(metadata *fleet.CarveMetadata, blockId int64, data []byte) error {
s.NewBlockFuncInvoked = true
return s.NewBlockFunc(metadata, blockId, data)

View file

@ -18,28 +18,26 @@ type DeleteHostFunc func(hid uint) error
type HostFunc func(id uint) (*fleet.Host, error)
type HostByIdentifierFunc func(identifier string) (*fleet.Host, error)
type EnrollHostFunc func(osqueryHostId string, nodeKey string, teamID *uint, cooldown time.Duration) (*fleet.Host, error)
type ListHostsFunc func(filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error)
type EnrollHostFunc func(osqueryHostId, nodeKey string, teamID *uint, cooldown time.Duration) (*fleet.Host, error)
type AuthenticateHostFunc func(nodeKey string) (*fleet.Host, error)
type MarkHostSeenFunc func(host *fleet.Host, t time.Time) error
type MarkHostsSeenFunc func(hostIDs []uint, t time.Time) error
type CleanupIncomingHostsFunc func(t time.Time) error
type SearchHostsFunc func(filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error)
type CleanupIncomingHostsFunc func(now time.Time) error
type GenerateHostStatusStatisticsFunc func(filter fleet.TeamFilter, now time.Time) (online uint, offline uint, mia uint, new uint, err error)
type DistributedQueriesForHostFunc func(host *fleet.Host) (map[uint]string, error)
type HostIDsByNameFunc func(filter fleet.TeamFilter, hostnames []string) ([]uint, error)
type HostByIdentifierFunc func(identifier string) (*fleet.Host, error)
type AddHostsToTeamFunc func(teamID *uint, hostIDs []uint) error
type SaveHostAdditionalFunc func(host *fleet.Host) error
@ -57,15 +55,12 @@ type HostStore struct {
HostFunc HostFunc
HostFuncInvoked bool
HostByIdentifierFunc HostByIdentifierFunc
HostByIdentifierFuncInvoked bool
EnrollHostFunc EnrollHostFunc
EnrollHostFuncInvoked bool
ListHostsFunc ListHostsFunc
ListHostsFuncInvoked bool
EnrollHostFunc EnrollHostFunc
EnrollHostFuncInvoked bool
AuthenticateHostFunc AuthenticateHostFunc
AuthenticateHostFuncInvoked bool
@ -75,21 +70,21 @@ type HostStore struct {
MarkHostsSeenFunc MarkHostsSeenFunc
MarkHostsSeenFuncInvoked bool
CleanupIncomingHostsFunc CleanupIncomingHostsFunc
CleanupIncomingHostsFuncInvoked bool
SearchHostsFunc SearchHostsFunc
SearchHostsFuncInvoked bool
CleanupIncomingHostsFunc CleanupIncomingHostsFunc
CleanupIncomingHostsFuncInvoked bool
GenerateHostStatusStatisticsFunc GenerateHostStatusStatisticsFunc
GenerateHostStatusStatisticsFuncInvoked bool
DistributedQueriesForHostFunc DistributedQueriesForHostFunc
DistributedQueriesForHostFuncInvoked bool
HostIDsByNameFunc HostIDsByNameFunc
HostIDsByNameFuncInvoked bool
HostByIdentifierFunc HostByIdentifierFunc
HostByIdentifierFuncInvoked bool
AddHostsToTeamFunc AddHostsToTeamFunc
AddHostsToTeamFuncInvoked bool
@ -117,9 +112,9 @@ func (s *HostStore) Host(id uint) (*fleet.Host, error) {
return s.HostFunc(id)
}
func (s *HostStore) HostByIdentifier(identifier string) (*fleet.Host, error) {
s.HostByIdentifierFuncInvoked = true
return s.HostByIdentifierFunc(identifier)
func (s *HostStore) EnrollHost(osqueryHostId string, nodeKey string, teamID *uint, cooldown time.Duration) (*fleet.Host, error) {
s.EnrollHostFuncInvoked = true
return s.EnrollHostFunc(osqueryHostId, nodeKey, teamID, cooldown)
}
func (s *HostStore) ListHosts(filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) {
@ -127,11 +122,6 @@ func (s *HostStore) ListHosts(filter fleet.TeamFilter, opt fleet.HostListOptions
return s.ListHostsFunc(filter, opt)
}
func (s *HostStore) EnrollHost(osqueryHostId, nodeKey string, teamID *uint, cooldown time.Duration) (*fleet.Host, error) {
s.EnrollHostFuncInvoked = true
return s.EnrollHostFunc(osqueryHostId, nodeKey, teamID, cooldown)
}
func (s *HostStore) AuthenticateHost(nodeKey string) (*fleet.Host, error) {
s.AuthenticateHostFuncInvoked = true
return s.AuthenticateHostFunc(nodeKey)
@ -147,31 +137,31 @@ func (s *HostStore) MarkHostsSeen(hostIDs []uint, t time.Time) error {
return s.MarkHostsSeenFunc(hostIDs, t)
}
func (s *HostStore) CleanupIncomingHosts(t time.Time) error {
s.CleanupIncomingHostsFuncInvoked = true
return s.CleanupIncomingHostsFunc(t)
}
func (s *HostStore) SearchHosts(filter fleet.TeamFilter, query string, omit ...uint) ([]*fleet.Host, error) {
s.SearchHostsFuncInvoked = true
return s.SearchHostsFunc(filter, query, omit...)
}
func (s *HostStore) CleanupIncomingHosts(now time.Time) error {
s.CleanupIncomingHostsFuncInvoked = true
return s.CleanupIncomingHostsFunc(now)
}
func (s *HostStore) GenerateHostStatusStatistics(filter fleet.TeamFilter, now time.Time) (online uint, offline uint, mia uint, new uint, err error) {
s.GenerateHostStatusStatisticsFuncInvoked = true
return s.GenerateHostStatusStatisticsFunc(filter, now)
}
func (s *HostStore) DistributedQueriesForHost(host *fleet.Host) (map[uint]string, error) {
s.DistributedQueriesForHostFuncInvoked = true
return s.DistributedQueriesForHostFunc(host)
}
func (s *HostStore) HostIDsByName(filter fleet.TeamFilter, hostnames []string) ([]uint, error) {
s.HostIDsByNameFuncInvoked = true
return s.HostIDsByNameFunc(filter, hostnames)
}
func (s *HostStore) HostByIdentifier(identifier string) (*fleet.Host, error) {
s.HostByIdentifierFuncInvoked = true
return s.HostByIdentifierFunc(identifier)
}
func (s *HostStore) AddHostsToTeam(teamID *uint, hostIDs []uint) error {
s.AddHostsToTeamFuncInvoked = true
return s.AddHostsToTeamFunc(teamID, hostIDs)

View file

@ -24,22 +24,8 @@ type ListPacksFunc func(opt fleet.ListOptions) ([]*fleet.Pack, error)
type PackByNameFunc func(name string, opts ...fleet.OptionalArg) (*fleet.Pack, bool, error)
type AddLabelToPackFunc func(lid uint, pid uint, opts ...fleet.OptionalArg) error
type RemoveLabelFromPackFunc func(lid uint, pid uint) error
type ListLabelsForPackFunc func(pid uint) ([]*fleet.Label, error)
type AddHostToPackFunc func(hid uint, pid uint) error
type RemoveHostFromPackFunc func(hid uint, pid uint) error
type ListPacksForHostFunc func(hid uint) (packs []*fleet.Pack, err error)
type ListHostsInPackFunc func(pid uint, opt fleet.ListOptions) ([]uint, error)
type ListExplicitHostsInPackFunc func(pid uint, opt fleet.ListOptions) ([]uint, error)
type EnsureGlobalPackFunc func() (*fleet.Pack, error)
type EnsureTeamPackFunc func(teamID uint) (*fleet.Pack, error)
@ -72,30 +58,9 @@ type PackStore struct {
PackByNameFunc PackByNameFunc
PackByNameFuncInvoked bool
AddLabelToPackFunc AddLabelToPackFunc
AddLabelToPackFuncInvoked bool
RemoveLabelFromPackFunc RemoveLabelFromPackFunc
RemoveLabelFromPackFuncInvoked bool
ListLabelsForPackFunc ListLabelsForPackFunc
ListLabelsForPackFuncInvoked bool
AddHostToPackFunc AddHostToPackFunc
AddHostToPackFuncInvoked bool
RemoveHostFromPackFunc RemoveHostFromPackFunc
RemoveHostFromPackFuncInvoked bool
ListPacksForHostFunc ListPacksForHostFunc
ListPacksForHostFuncInvoked bool
ListHostsInPackFunc ListHostsInPackFunc
ListHostsInPackFuncInvoked bool
ListExplicitHostsInPackFunc ListExplicitHostsInPackFunc
ListExplicitHostsInPackFuncInvoked bool
EnsureGlobalPackFunc EnsureGlobalPackFunc
EnsureGlobalPackFuncInvoked bool
@ -158,42 +123,7 @@ func (s *PackStore) PackByName(name string, opts ...fleet.OptionalArg) (*fleet.P
return s.PackByNameFunc(name, opts...)
}
func (s *PackStore) AddLabelToPack(lid uint, pid uint, opts ...fleet.OptionalArg) error {
s.AddLabelToPackFuncInvoked = true
return s.AddLabelToPackFunc(lid, pid, opts...)
}
func (s *PackStore) RemoveLabelFromPack(lid uint, pid uint) error {
s.RemoveLabelFromPackFuncInvoked = true
return s.RemoveLabelFromPackFunc(lid, pid)
}
func (s *PackStore) ListLabelsForPack(pid uint) ([]*fleet.Label, error) {
s.ListLabelsForPackFuncInvoked = true
return s.ListLabelsForPackFunc(pid)
}
func (s *PackStore) AddHostToPack(hid uint, pid uint) error {
s.AddHostToPackFuncInvoked = true
return s.AddHostToPackFunc(hid, pid)
}
func (s *PackStore) RemoveHostFromPack(hid uint, pid uint) error {
s.RemoveHostFromPackFuncInvoked = true
return s.RemoveHostFromPackFunc(hid, pid)
}
func (s *PackStore) ListPacksForHost(hid uint) (packs []*fleet.Pack, err error) {
s.ListPacksForHostFuncInvoked = true
return s.ListPacksForHostFunc(hid)
}
func (s *PackStore) ListHostsInPack(pid uint, opt fleet.ListOptions) ([]uint, error) {
s.ListHostsInPackFuncInvoked = true
return s.ListHostsInPackFunc(pid, opt)
}
func (s *PackStore) ListExplicitHostsInPack(pid uint, opt fleet.ListOptions) ([]uint, error) {
s.ListExplicitHostsInPackFuncInvoked = true
return s.ListExplicitHostsInPackFunc(pid, opt)
}

View file

@ -16,8 +16,6 @@ type DeleteScheduledQueryFunc func(id uint) error
type ScheduledQueryFunc func(id uint) (*fleet.ScheduledQuery, error)
type SaveScheduledQueriesFunc func(sqs []*fleet.ScheduledQuery) ([]*fleet.ScheduledQuery, error)
type ScheduledQueryStore struct {
ListScheduledQueriesInPackFunc ListScheduledQueriesInPackFunc
ListScheduledQueriesInPackFuncInvoked bool
@ -33,9 +31,6 @@ type ScheduledQueryStore struct {
ScheduledQueryFunc ScheduledQueryFunc
ScheduledQueryFuncInvoked bool
SaveScheduledQueriesFunc SaveScheduledQueriesFunc
SaveScheduledQueriesFuncInvoked bool
}
func (s *ScheduledQueryStore) ListScheduledQueriesInPack(id uint, opts fleet.ListOptions) ([]*fleet.ScheduledQuery, error) {

View file

@ -10,10 +10,14 @@ type SaveHostSoftwareFunc func(host *fleet.Host) error
type LoadHostSoftwareFunc func(host *fleet.Host) error
type AllSoftwareIteratorFunc func() (fleet.SoftwareIterator, error)
type AllSoftwareWithoutCPEIteratorFunc func() (fleet.SoftwareIterator, error)
type AddCPEForSoftwareFunc func(software fleet.Software, cpe string) error
type AllCPEsFunc func() ([]string, error)
type InsertCVEForCPEFunc func(cve string, cpes []string) error
type SoftwareStore struct {
SaveHostSoftwareFunc SaveHostSoftwareFunc
SaveHostSoftwareFuncInvoked bool
@ -21,16 +25,17 @@ type SoftwareStore struct {
LoadHostSoftwareFunc LoadHostSoftwareFunc
LoadHostSoftwareFuncInvoked bool
AllSoftwareIteratorFunc AllSoftwareIteratorFunc
AllSoftwareIteratorFuncInvoked bool
AllSoftwareWithoutCPEIteratorFunc AllSoftwareWithoutCPEIteratorFunc
AllSoftwareWithoutCPEIteratorFuncInvoked bool
AddCPEForSoftwareFunc AddCPEForSoftwareFunc
AddCPEForSoftwareFuncInvoked bool
}
func (s *SoftwareStore) AllSoftwareWithoutCPEIterator() (fleet.SoftwareIterator, error) {
s.AllSoftwareIteratorFuncInvoked = true
return s.AllSoftwareIteratorFunc()
AllCPEsFunc AllCPEsFunc
AllCPEsFuncInvoked bool
InsertCVEForCPEFunc InsertCVEForCPEFunc
InsertCVEForCPEFuncInvoked bool
}
func (s *SoftwareStore) SaveHostSoftware(host *fleet.Host) error {
@ -43,7 +48,22 @@ func (s *SoftwareStore) LoadHostSoftware(host *fleet.Host) error {
return s.LoadHostSoftwareFunc(host)
}
func (s *SoftwareStore) AllSoftwareWithoutCPEIterator() (fleet.SoftwareIterator, error) {
s.AllSoftwareWithoutCPEIteratorFuncInvoked = true
return s.AllSoftwareWithoutCPEIteratorFunc()
}
func (s *SoftwareStore) AddCPEForSoftware(software fleet.Software, cpe string) error {
s.AddCPEForSoftwareFuncInvoked = true
return s.AddCPEForSoftwareFunc(software, cpe)
}
func (s *SoftwareStore) AllCPEs() ([]string, error) {
s.AllCPEsFuncInvoked = true
return s.AllCPEsFunc()
}
func (s *SoftwareStore) InsertCVEForCPE(cve string, cpes []string) error {
s.InsertCVEForCPEFuncInvoked = true
return s.InsertCVEForCPEFunc(cve, cpes)
}

View file

@ -11,6 +11,7 @@ import (
var _ fleet.StatisticsStore = (*StatisticsStore)(nil)
type ShouldSendStatisticsFunc func(frequency time.Duration) (fleet.StatisticsPayload, bool, error)
type RecordStatisticsSentFunc func() error
type StatisticsStore struct {

View file

@ -2,9 +2,7 @@
package mock
import (
"github.com/fleetdm/fleet/v4/server/fleet"
)
import "github.com/fleetdm/fleet/v4/server/fleet"
var _ fleet.TeamStore = (*TeamStore)(nil)
@ -12,9 +10,9 @@ type NewTeamFunc func(team *fleet.Team) (*fleet.Team, error)
type SaveTeamFunc func(team *fleet.Team) (*fleet.Team, error)
type DeleteTeamFunc func(tid uint) error
type TeamFunc func(tid uint) (*fleet.Team, error)
type TeamFunc func(id uint) (*fleet.Team, error)
type DeleteTeamFunc func(tid uint) error
type TeamByNameFunc func(name string) (*fleet.Team, error)
@ -31,12 +29,12 @@ type TeamStore struct {
SaveTeamFunc SaveTeamFunc
SaveTeamFuncInvoked bool
DeleteTeamFunc DeleteTeamFunc
DeleteTeamFuncInvoked bool
TeamFunc TeamFunc
TeamFuncInvoked bool
DeleteTeamFunc DeleteTeamFunc
DeleteTeamFuncInvoked bool
TeamByNameFunc TeamByNameFunc
TeamByNameFuncInvoked bool
@ -60,19 +58,19 @@ func (s *TeamStore) SaveTeam(team *fleet.Team) (*fleet.Team, error) {
return s.SaveTeamFunc(team)
}
func (s *TeamStore) Team(tid uint) (*fleet.Team, error) {
s.TeamFuncInvoked = true
return s.TeamFunc(tid)
}
func (s *TeamStore) DeleteTeam(tid uint) error {
s.DeleteTeamFuncInvoked = true
return s.DeleteTeamFunc(tid)
}
func (s *TeamStore) Team(id uint) (*fleet.Team, error) {
s.TeamFuncInvoked = true
return s.TeamFunc(id)
}
func (s *TeamStore) TeamByName(identifier string) (*fleet.Team, error) {
func (s *TeamStore) TeamByName(name string) (*fleet.Team, error) {
s.TeamByNameFuncInvoked = true
return s.TeamByNameFunc(identifier)
return s.TeamByNameFunc(name)
}
func (s *TeamStore) ListTeams(filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) {

View file

@ -16,7 +16,7 @@ type UserByIDFunc func(id uint) (*fleet.User, error)
type SaveUserFunc func(user *fleet.User) error
type SaveUsersFunc func(user []*fleet.User) error
type SaveUsersFunc func(users []*fleet.User) error
type DeleteUserFunc func(id uint) error

View file

@ -13,7 +13,7 @@ var _ fleet.OsqueryService = (*TLSService)(nil)
type EnrollAgentFunc func(ctx context.Context, enrollSecret string, hostIdentifier string, hostDetails map[string](map[string]string)) (nodeKey string, err error)
type AuthenticateHostFuncI func(ctx context.Context, nodeKey string) (host *fleet.Host, err error)
type AuthenticateHostFunc func(ctx context.Context, nodeKey string) (host *fleet.Host, err error)
type GetClientConfigFunc func(ctx context.Context) (config map[string]interface{}, err error)
@ -29,7 +29,7 @@ type TLSService struct {
EnrollAgentFunc EnrollAgentFunc
EnrollAgentFuncInvoked bool
AuthenticateHostFunc AuthenticateHostFuncI
AuthenticateHostFunc AuthenticateHostFunc
AuthenticateHostFuncInvoked bool
GetClientConfigFunc GetClientConfigFunc

View file

@ -317,9 +317,6 @@ func TestLabelQueries(t *testing.T) {
ds.LabelQueriesForHostFunc = func(host *fleet.Host, cutoff time.Time) (map[string]string, error) {
return map[string]string{}, nil
}
ds.DistributedQueriesForHostFunc = func(host *fleet.Host) (map[uint]string, error) {
return map[uint]string{}, nil
}
ds.HostFunc = func(id uint) (*fleet.Host, error) {
return host, nil
}
@ -1163,9 +1160,6 @@ func TestNewDistributedQueryCampaign(t *testing.T) {
ds.LabelQueriesForHostFunc = func(host *fleet.Host, cutoff time.Time) (map[string]string, error) {
return map[string]string{}, nil
}
ds.DistributedQueriesForHostFunc = func(host *fleet.Host) (map[uint]string, error) {
return map[uint]string{}, nil
}
ds.SaveHostFunc = func(host *fleet.Host) error {
return nil
}
@ -1237,9 +1231,6 @@ func TestDistributedQueryResults(t *testing.T) {
ds.SaveHostFunc = func(host *fleet.Host) error {
return nil
}
ds.DistributedQueriesForHostFunc = func(host *fleet.Host) (map[uint]string, error) {
return map[uint]string{campaign.ID: "select * from time"}, nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
}

View file

@ -14,11 +14,12 @@ import (
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/google/go-github/v37/github"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
)
const (
owner = "chiiph"
owner = "fleetdm"
repo = "nvd"
)
@ -44,6 +45,8 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) {
cpeURL := ""
// TODO: get not draft release
for _, asset := range releases[0].Assets {
if asset != nil {
matched := cpeSqliteRegex.MatchString(asset.GetName())
@ -120,7 +123,7 @@ func cleanAppName(appName string) string {
return strings.TrimSuffix(appName, ".app")
}
func CPEFromSoftware(dbPath string, software *fleet.Software) (string, error) {
func CPEFromSoftware(db *sqlx.DB, software *fleet.Software) (string, error) {
targetSW := ""
switch software.Source {
case "apps":
@ -145,11 +148,6 @@ func CPEFromSoftware(dbPath string, software *fleet.Software) (string, error) {
case "chocolatey_packages":
}
db, err := CPEDB(dbPath)
if err != nil {
return "", errors.Wrap(err, "opening the cpe db")
}
checkTargetSW := ""
args := []interface{}{cleanAppName(software.Name)}
if targetSW != "" {
@ -165,7 +163,7 @@ func CPEFromSoftware(dbPath string, software *fleet.Software) (string, error) {
checkTargetSW,
)
var indexedCPEs []IndexedCPEItem
err = db.Select(&indexedCPEs, query, args...)
err := db.Select(&indexedCPEs, query, args...)
if err != nil {
return "", errors.Wrap(err, "getting cpes")
}
@ -201,17 +199,8 @@ func CPEFromSoftware(dbPath string, software *fleet.Software) (string, error) {
return "", nil
}
func TranslateSoftwareToCPE(ds fleet.Datastore) error {
config, err := ds.AppConfig()
if err != nil {
return err
}
if config.VulnerabilityDatabasesPath == nil {
return errors.New(
"Can't translate CPE without a database. vulnerability_databases_path is not configured.")
}
dbPath := path.Join(*config.VulnerabilityDatabasesPath, "cpe.sqlite")
func TranslateSoftwareToCPE(ds fleet.Datastore, vulnPath string) error {
dbPath := path.Join(vulnPath, "cpe.sqlite")
client := &http.Client{}
if err := syncCPEDatabase(client, dbPath); err != nil {
@ -224,12 +213,18 @@ func TranslateSoftwareToCPE(ds fleet.Datastore) error {
}
defer iterator.Close()
db, err := sqliteDB(dbPath)
if err != nil {
return errors.Wrap(err, "opening the cpe db")
}
defer db.Close()
for iterator.Next() {
software, err := iterator.Value()
if err != nil {
return err
}
cpe, err := CPEFromSoftware(dbPath, software)
cpe, err := CPEFromSoftware(db, software)
if err != nil {
return err
}

View file

@ -20,11 +20,6 @@ import (
func TestCpeFromSoftware(t *testing.T) {
tempDir := os.TempDir()
ds := new(mock.Store)
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{VulnerabilityDatabasesPath: &tempDir}, nil
}
items, err := cpedict.Decode(strings.NewReader(XmlCPETestDict))
require.NoError(t, err)
@ -32,18 +27,21 @@ func TestCpeFromSoftware(t *testing.T) {
err = GenerateCPEDB(dbPath, items)
require.NoError(t, err)
db, err := sqliteDB(dbPath)
require.NoError(t, err)
// checking an non existent version returns empty
cpe, err := CPEFromSoftware(dbPath, &fleet.Software{Name: "Vendor Product.app", Version: "2.3.4", Source: "apps"})
cpe, err := CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product.app", Version: "2.3.4", Source: "apps"})
require.NoError(t, err)
require.Equal(t, "", cpe)
// checking a version that exists works
cpe, err = CPEFromSoftware(dbPath, &fleet.Software{Name: "Vendor Product.app", Version: "1.2.3", Source: "apps"})
cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product.app", Version: "1.2.3", Source: "apps"})
require.NoError(t, err)
require.Equal(t, "cpe:2.3:a:vendor:product:1.2.3:*:*:*:*:macos:*:*", cpe)
// follows many deprecations
cpe, err = CPEFromSoftware(dbPath, &fleet.Software{Name: "Vendor2 Product2.app", Version: "0.3", Source: "apps"})
cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor2 Product2.app", Version: "0.3", Source: "apps"})
require.NoError(t, err)
require.Equal(t, "cpe:2.3:a:vendor2:product4:999:*:*:*:*:macos:*:*", cpe)
}
@ -70,16 +68,19 @@ func TestSyncCPEDatabase(t *testing.T) {
err = syncCPEDatabase(client, dbPath)
require.NoError(t, err)
db, err := sqliteDB(dbPath)
require.NoError(t, err)
// and this works afterwards
software := &fleet.Software{Name: "1Password.app", Version: "7.2.3", Source: "apps"}
cpe, err := CPEFromSoftware(dbPath, software)
cpe, err := CPEFromSoftware(db, software)
require.NoError(t, err)
require.Equal(t, "cpe:2.3:a:1password:1password:7.2.3:beta0:*:*:*:macos:*:*", cpe)
// but now we truncate to make sure searching for cpe fails
err = os.Truncate(dbPath, 0)
require.NoError(t, err)
_, err = CPEFromSoftware(dbPath, software)
_, err = CPEFromSoftware(db, software)
require.Error(t, err)
// and we make the db older than the release
@ -96,7 +97,12 @@ func TestSyncCPEDatabase(t *testing.T) {
require.NoError(t, err)
mtime := stat.ModTime()
cpe, err = CPEFromSoftware(dbPath, software)
db.Close()
db, err = sqliteDB(dbPath)
require.NoError(t, err)
defer db.Close()
cpe, err = CPEFromSoftware(db, software)
require.NoError(t, err)
require.Equal(t, "cpe:2.3:a:1password:1password:7.2.3:beta0:*:*:*:macos:*:*", cpe)
@ -132,12 +138,11 @@ func (f *fakeSoftwareIterator) Err() error { return nil }
func (f *fakeSoftwareIterator) Close() error { f.closed = true; return nil }
func TestTranslateSoftwareToCPE(t *testing.T) {
tempDir := os.TempDir()
tempDir, err := os.MkdirTemp(os.TempDir(), "TestTranslateSoftwareToCPE-*")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
ds := new(mock.Store)
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{VulnerabilityDatabasesPath: &tempDir}, nil
}
var cpes []string
@ -163,7 +168,7 @@ func TestTranslateSoftwareToCPE(t *testing.T) {
},
}
ds.AllSoftwareIteratorFunc = func() (fleet.SoftwareIterator, error) {
ds.AllSoftwareWithoutCPEIteratorFunc = func() (fleet.SoftwareIterator, error) {
return iterator, nil
}
@ -174,7 +179,7 @@ func TestTranslateSoftwareToCPE(t *testing.T) {
err = GenerateCPEDB(dbPath, items)
require.NoError(t, err)
err = TranslateSoftwareToCPE(ds)
err = TranslateSoftwareToCPE(ds, tempDir)
require.NoError(t, err)
assert.Equal(t, []string{
"cpe:2.3:a:vendor:product:1.2.3:*:*:*:*:macos:*:*",

View file

@ -0,0 +1,143 @@
package vulnerabilities
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"sync"
"time"
"github.com/facebookincubator/nvdtools/cvefeed"
"github.com/facebookincubator/nvdtools/providers/nvd"
"github.com/facebookincubator/nvdtools/wfn"
"github.com/fleetdm/fleet/v4/server/fleet"
kitlog "github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
)
func syncCVEData(vulnPath string) error {
cve := nvd.SupportedCVE["cve-1.1.json.gz"]
source := nvd.NewSourceConfig()
dfs := nvd.Sync{
Feeds: []nvd.Syncer{cve},
Source: source,
LocalDir: vulnPath,
}
ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancelFunc()
return dfs.Do(ctx)
}
func TranslateCPEToCVE(ctx context.Context, ds fleet.Datastore, vulnPath string, logger kitlog.Logger) error {
err := syncCVEData(vulnPath)
if err != nil {
return err
}
cpeList, err := ds.AllCPEs()
if err != nil {
return err
}
cpes := make([]*wfn.Attributes, 0, len(cpeList))
for _, uri := range cpeList {
attr, err := wfn.Parse(uri)
if err != nil {
return err
}
cpes = append(cpes, attr)
}
if len(cpes) == 0 {
return nil
}
var files []string
err = filepath.Walk(vulnPath, func(path string, info os.FileInfo, err error) error {
if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil {
return nil
}
files = append(files, path)
return nil
})
if err != nil {
return err
}
dict, err := cvefeed.LoadJSONDictionary(files...)
if err != nil {
return err
}
cache := cvefeed.NewCache(dict).SetRequireVersion(true).SetMaxSize(0)
cache.Idx = cvefeed.NewIndex(dict)
cpeCh := make(chan *wfn.Attributes)
var wg sync.WaitGroup
for i := 0; i < runtime.NumCPU(); i++ {
wg.Add(1)
goRoutineKey := i
go func() {
defer wg.Done()
logKey := fmt.Sprintf("cpe-processing-%d", goRoutineKey)
level.Debug(logger).Log(logKey, "start")
for {
select {
case cpe, more := <-cpeCh:
if !more {
level.Debug(logger).Log(logKey, "done")
return
}
cacheHits := cache.Get([]*wfn.Attributes{cpe})
for _, matches := range cacheHits {
ml := len(matches.CPEs)
if ml == 0 {
continue
}
matchingCPEs := make([]string, 0, ml)
for _, attr := range matches.CPEs {
if attr == nil {
level.Error(logger).Log("matches nil CPE", matches.CVE.ID())
continue
}
cpe := attr.BindToFmtString()
if len(cpe) == 0 {
continue
}
matchingCPEs = append(matchingCPEs, cpe)
}
err = ds.InsertCVEForCPE(matches.CVE.ID(), matchingCPEs)
if err != nil {
level.Error(logger).Log("cpe processing", "error", "err", err)
}
}
case <-ctx.Done():
level.Debug(logger).Log(logKey, "quitting")
return
}
}
}()
}
level.Debug(logger).Log("pushing cpes", "start")
for _, cpe := range cpes {
cpeCh <- cpe
}
close(cpeCh)
level.Debug(logger).Log("pushing cpes", "done")
wg.Wait()
return nil
}

View file

@ -0,0 +1,54 @@
package vulnerabilities
import (
"context"
"os"
"sync"
"testing"
"github.com/fleetdm/fleet/v4/server/mock"
kitlog "github.com/go-kit/kit/log"
"github.com/stretchr/testify/require"
)
var cvetests = []struct {
cpe, cve string
}{
{"cpe:2.3:a:1password:1password:3.9.9:*:*:*:*:macos:*:*", "CVE-2012-6369"},
{"cpe:2.3:a:1password:1password:3.9.9:*:*:*:*:*:*:*", "CVE-2012-6369"},
}
func TestTranslateCPEToCVE(t *testing.T) {
tempDir, err := os.MkdirTemp(os.TempDir(), "TestTranslateCPEToCVE-*")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
ds := new(mock.Store)
ctx := context.Background()
for _, tt := range cvetests {
t.Run(tt.cpe, func(t *testing.T) {
ds.AllCPEsFunc = func() ([]string, error) {
return []string{tt.cpe}, nil
}
cveLock := &sync.Mutex{}
cveToCPEs := make(map[string][]string)
var cvesFound []string
ds.InsertCVEForCPEFunc = func(cve string, cpes []string) error {
cveLock.Lock()
defer cveLock.Unlock()
cveToCPEs[cve] = cpes
cvesFound = append(cvesFound, cve)
return nil
}
err = TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout))
require.NoError(t, err)
require.Equal(t, []string{tt.cve}, cvesFound)
require.Equal(t, []string{tt.cpe}, cveToCPEs[tt.cve])
})
}
}

View file

@ -3,6 +3,7 @@ package vulnerabilities
import (
"errors"
"fmt"
"os"
"strings"
@ -12,7 +13,7 @@ import (
_ "github.com/mattn/go-sqlite3"
)
func CPEDB(dbPath string) (*sqlx.DB, error) {
func sqliteDB(dbPath string) (*sqlx.DB, error) {
db, err := sqlx.Open("sqlite3", dbPath)
if err != nil {
return nil, err
@ -20,12 +21,8 @@ func CPEDB(dbPath string) (*sqlx.DB, error) {
return db, nil
}
func applyCPEDatabaseSchema(dbPath string) error {
db, err := CPEDB(dbPath)
if err != nil {
return err
}
_, err = db.Exec(`
func applyCPEDatabaseSchema(db *sqlx.DB) error {
_, err := db.Exec(`
CREATE TABLE IF NOT EXISTS cpe (
cpe23 TEXT NOT NULL,
title TEXT NOT NULL,
@ -44,10 +41,7 @@ func applyCPEDatabaseSchema(dbPath string) error {
CREATE INDEX IF NOT EXISTS idx_target_sw ON cpe (target_sw);
CREATE INDEX IF NOT EXISTS idx_deprecated_by ON deprecated_by (cpe23);
`)
if err != nil {
return err
}
return nil
return err
}
func generateCPEItem(item cpedict.CPEItem) ([]interface{}, map[string]string, error) {
@ -77,11 +71,13 @@ func GenerateCPEDB(path string, items *cpedict.CPEList) error {
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
db, err := CPEDB(path)
db, err := sqliteDB(path)
if err != nil {
return err
}
err = applyCPEDatabaseSchema(path)
defer db.Close()
err = applyCPEDatabaseSchema(db)
if err != nil {
return err
}