Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-04-21 13:37:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

Added support for atomic imports and dry run imports (#1510)

Browse source 
Closes issue #1475

The command line tool that uses this endpoint -> https://github.com/kolide/configimporter

* Added support for atomic imports and dry run imports

* Added code so that imports are idempotent
This commit is contained in:
John Murphy 2017-05-30 14:42:00 -05:00 committed by GitHub
parent 151ce35e8c
commit 7a8f418d0f
45 changed files with 468 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -1,4 +1,6 @@
* Added support for Osquery decorators.
* Added feature that allows users to import existing Osquery configuration files using the [configimporter](https://github.com/kolide/configimporter) utility.
* Added support for Osquery decorators.
* Added SAML single sign on support.

9
server/datastore/datastore_options_test.go
View file

@ -62,7 +62,6 @@ func testOptions(t *testing.T, ds kolide.Datastore) {
require.False(t, opt.GetValue().(bool))
opt, _ = ds.OptionByName("aws_profile_name")
oldValue := opt.GetValue()
assert.False(t, opt.OptionSet())
opt.SetValue("zip")
opt2, _ = ds.OptionByName("disable_distributed")
@ -71,15 +70,19 @@ func testOptions(t *testing.T, ds kolide.Datastore) {
modList := []kolide.Option{*opt, *opt2}
// The aws access key option can be saved but because the disable_events can't
// be we want to verify that the whole transaction is rolled back
err = ds.SaveOptions(modList)
tx, err := ds.Begin()
require.Nil(t, err)
err = ds.SaveOptions(modList, kolide.HasTransaction(tx))
assert.NotNil(t, err)
err = tx.Rollback()
require.Nil(t, err)
opt2, err = ds.OptionByName("disable_distributed")
require.Nil(t, err)
assert.Equal(t, false, opt2.GetValue())
opt, err = ds.OptionByName("aws_profile_name")
require.Nil(t, err)
assert.Equal(t, oldValue, opt.GetValue())
assert.False(t, opt.OptionSet())
}

54
server/datastore/datastore_yara_test.go
View file

@ -51,3 +51,57 @@ func testYARAStore(t *testing.T, ds kolide.Datastore) {
require.Nil(t, err)
assert.Len(t, yaraSection.Signatures["sig2"], 1)
}
func testYARATransactions(t *testing.T, ds kolide.Datastore) {
if ds.Name() == "inmem" {
t.Skip("not implemented for inmem")
}
ysg := &kolide.YARASignatureGroup{
SignatureName: "sig1",
Paths: []string{
"path1",
"path2",
},
}
ysg, err := ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
require.True(t, ysg.ID > 0)
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err = ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
tx, err := ds.Begin()
require.Nil(t, err)
err = ds.NewYARAFilePath("fp1", "sig1", kolide.HasTransaction(tx))
require.Nil(t, err)
err = tx.Rollback()
require.Nil(t, err)
yaraSection, err := ds.YARASection()
require.Nil(t, err)
require.NotNil(t, yaraSection)
// there shouldn't be any file paths because we rolled back the transaciton
require.Len(t, yaraSection.FilePaths, 0)
// try it again
tx, err = ds.Begin()
require.Nil(t, err)
err = ds.NewYARAFilePath("fp1", "sig1", kolide.HasTransaction(tx))
require.Nil(t, err)
err = tx.Commit()
require.Nil(t, err)
yaraSection, err = ds.YARASection()
require.Nil(t, err)
require.NotNil(t, yaraSection)
// file path should exist because we committed
require.Len(t, yaraSection.FilePaths, 1)
}

6
server/datastore/inmem/decorators.go
View file

@ -2,7 +2,7 @@ package inmem
import "github.com/kolide/kolide/server/kolide"
func (d *Datastore) SaveDecorator(dec *kolide.Decorator) error {
func (d *Datastore) SaveDecorator(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.decorators[dec.ID]; !ok {
@ -12,7 +12,7 @@ func (d *Datastore) SaveDecorator(dec *kolide.Decorator) error {
return nil
}
func (d *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) {
func (d *Datastore) NewDecorator(decorator *kolide.Decorator, opts ...kolide.OptionalArg) (*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
decorator.ID = d.nextID(decorator)
@ -39,7 +39,7 @@ func (d *Datastore) Decorator(id uint) (*kolide.Decorator, error) {
return nil, notFound("Decorator").WithID(id)
}
func (d *Datastore) ListDecorators() ([]*kolide.Decorator, error) {
func (d *Datastore) ListDecorators(opts ...kolide.OptionalArg) ([]*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
var result []*kolide.Decorator

2
server/datastore/inmem/file_integrity_monitoring.go
View file

@ -4,7 +4,7 @@ import (
"github.com/kolide/kolide/server/kolide"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (*kolide.FIMSection, error) {
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
fp.ID = d.nextID(fp)

9
server/datastore/inmem/inmem.go
View file

@ -55,6 +55,15 @@ func New(config config.KolideConfig) (*Datastore, error) {
return ds, nil
}
type imemTransactionStub struct{}
func (im *imemTransactionStub) Rollback() error { return nil }
func (im *imemTransactionStub) Commit() error { return nil }
func (d *Datastore) Begin() (kolide.Transaction, error) {
return &imemTransactionStub{}, nil
}
func (d *Datastore) Name() string {
return "inmem"
}

2
server/datastore/inmem/labels.go
View file

@ -11,7 +11,7 @@ import (
"github.com/patrickmn/sortutil"
)
func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
func (d *Datastore) NewLabel(label *kolide.Label, opts ...kolide.OptionalArg) (*kolide.Label, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
newLabel := *label

4
server/datastore/inmem/options.go
View file

@ -11,7 +11,7 @@ func (d *Datastore) ResetOptions() ([]kolide.Option, error) {
panic("inmem is being deprecated")
}
func (d *Datastore) OptionByName(name string) (*kolide.Option, error) {
func (d *Datastore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, opt := range d.options {
@ -28,7 +28,7 @@ type optPair struct {
existingOpt *kolide.Option
}
func (d *Datastore) SaveOptions(opts []kolide.Option) error {
func (d *Datastore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
var validPairs []optPair

6
server/datastore/inmem/packs.go
View file

@ -6,7 +6,7 @@ import (
"github.com/kolide/kolide/server/kolide"
)
func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
func (d *Datastore) PackByName(name string, opts ...kolide.OptionalArg) (*kolide.Pack, bool, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, p := range d.packs {
@ -17,7 +17,7 @@ func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
return nil, false, nil
}
func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
func (d *Datastore) NewPack(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error) {
newPack := *pack
for _, q := range d.packs {
@ -111,7 +111,7 @@ func (d *Datastore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) {
return packs, nil
}
func (d *Datastore) AddLabelToPack(lid, pid uint) error {
func (d *Datastore) AddLabelToPack(lid, pid uint, opts ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()

4
server/datastore/inmem/queries.go
View file

@ -7,7 +7,7 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
func (d *Datastore) NewQuery(query *kolide.Query, opts ...kolide.OptionalArg) (*kolide.Query, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
@ -26,7 +26,7 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
return &newQuery, nil
}
func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) {
func (d *Datastore) QueryByName(name string, opts ...kolide.OptionalArg) (*kolide.Query, bool, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, q := range d.queries {

2
server/datastore/inmem/scheduled_queries.go
View file

@ -6,7 +6,7 @@ import (
"github.com/kolide/kolide/server/kolide"
)
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery, opts ...kolide.OptionalArg) (*kolide.ScheduledQuery, error) {
d.mtx.Lock()
defer d.mtx.Unlock()

4
server/datastore/inmem/yara.go
View file

@ -2,7 +2,7 @@ package inmem
import "github.com/kolide/kolide/server/kolide"
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (*kolide.YARASignatureGroup, error) {
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup, opts ...kolide.OptionalArg) (*kolide.YARASignatureGroup, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
ysg.ID = d.nextID(ysg)
@ -10,7 +10,7 @@ func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (*koli
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error {
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string, opts ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
d.yaraFilePaths[fileSectionName] = append(d.yaraFilePaths[fileSectionName], sigGroupName)

16
server/datastore/mysql/decorators.go
View file

@ -8,7 +8,8 @@ import (
"github.com/kolide/kolide/server/kolide"
)
func (ds *Datastore) SaveDecorator(dec *kolide.Decorator) error {
func (ds *Datastore) SaveDecorator(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
db := ds.getTransaction(opts)
sqlStatement :=
"UPDATE decorators SET " +
"`name` = ?, " +
@ -16,7 +17,7 @@ func (ds *Datastore) SaveDecorator(dec *kolide.Decorator) error {
"`type` = ?, " +
"`interval` = ? " +
"WHERE id = ?"
_, err := ds.db.Exec(
_, err := db.Exec(
sqlStatement,
dec.Name,
dec.Query,
@ -30,7 +31,8 @@ func (ds *Datastore) SaveDecorator(dec *kolide.Decorator) error {
return nil
}
func (ds *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) {
func (ds *Datastore) NewDecorator(decorator *kolide.Decorator, opts ...kolide.OptionalArg) (*kolide.Decorator, error) {
db := ds.getTransaction(opts)
sqlStatement :=
"INSERT INTO decorators (" +
"`name`," +
@ -38,7 +40,8 @@ func (ds *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorato
"`type`," +
"`interval` ) " +
"VALUES (?, ?, ?, ?)"
result, err := ds.db.Exec(sqlStatement, decorator.Name, decorator.Query, decorator.Type, decorator.Interval)
result, err := db.Exec(sqlStatement, decorator.Name, decorator.Query, decorator.Type, decorator.Interval)
if err != nil {
return nil, errors.Wrap(err, "creating decorator")
}
@ -80,14 +83,15 @@ func (ds *Datastore) Decorator(id uint) (*kolide.Decorator, error) {
return &result, nil
}
func (ds *Datastore) ListDecorators() ([]*kolide.Decorator, error) {
func (ds *Datastore) ListDecorators(opts ...kolide.OptionalArg) ([]*kolide.Decorator, error) {
db := ds.getTransaction(opts)
sqlStatement := `
SELECT *
FROM decorators
ORDER by built_in DESC, name ASC
`
var results []*kolide.Decorator
err := ds.db.Select(&results, sqlStatement)
err := db.Select(&results, sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "listing decorators")
}

24
server/datastore/mysql/file_integrity_monitoring.go
View file

@ -7,20 +7,8 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (result *kolide.FIMSection, err error) {
txn, err := d.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "update options begin transaction")
}
var success bool
defer func() {
if success {
if err = txn.Commit(); err == nil {
return
}
}
txn.Rollback()
}()
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection, opts ...kolide.OptionalArg) (result *kolide.FIMSection, err error) {
db := d.getTransaction(opts)
sqlStatement := `
INSERT INTO file_integrity_monitorings (
@ -29,7 +17,10 @@ func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (result *kolide.FIMSect
) VALUES( ?, ?)
`
var resp sql.Result
resp, err = txn.Exec(sqlStatement, fp.SectionName, fp.Description)
resp, err = db.Exec(sqlStatement, fp.SectionName, fp.Description)
if isDuplicate(err) {
return nil, alreadyExists("fim_section", 0)
}
if err != nil {
return nil, errors.Wrap(err, "creating fim section")
}
@ -42,12 +33,11 @@ func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (result *kolide.FIMSect
) VALUES( ?, ? )
`
for _, fileName := range fp.Paths {
_, err = txn.Exec(sqlStatement, fileName, fp.ID)
_, err = db.Exec(sqlStatement, fileName, fp.ID)
if err != nil {
return nil, errors.Wrap(err, "adding path to fim section")
}
}
success = true
return fp, nil
}

7
server/datastore/mysql/labels.go
View file

@ -10,12 +10,13 @@ import (
)
// NewLabel creates a new kolide.Label
func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
func (d *Datastore) NewLabel(label *kolide.Label, opts ...kolide.OptionalArg) (*kolide.Label, error) {
db := d.getTransaction(opts)
var (
deletedLabel kolide.Label
query string
)
err := d.db.Get(&deletedLabel,
err := db.Get(&deletedLabel,
"SELECT * FROM labels WHERE name = ? AND deleted", label.Name)
switch err {
case nil:
@ -41,7 +42,7 @@ func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
default:
return nil, errors.Wrap(err, "check for existing label")
}
result, err := d.db.Exec(query, label.Name, label.Description, label.Query, label.Platform, label.LabelType)
result, err := db.Exec(query, label.Name, label.Description, label.Query, label.Platform, label.LabelType)
if err != nil {
return nil, errors.Wrap(err, "inserting label")
}

23
server/datastore/mysql/mysql.go
View file

@ -3,6 +3,7 @@ package mysql
import (
"crypto/tls"
"crypto/x509"
"database/sql"
"fmt"
"io/ioutil"
"time"
@ -31,6 +32,24 @@ type Datastore struct {
config config.MysqlConfig
}
type dbfunctions interface {
Exec(query string, args ...interface{}) (sql.Result, error)
Get(dest interface{}, query string, args ...interface{}) error
Select(dest interface{}, query string, args ...interface{}) error
}
func (d *Datastore) getTransaction(opts []kolide.OptionalArg) dbfunctions {
var result dbfunctions
result = d.db
for _, opt := range opts {
switch t := opt().(type) {
case dbfunctions:
result = t
}
}
return result
}
// New creates an MySQL datastore.
func New(config config.MysqlConfig, c clock.Clock, opts ...DBOption) (*Datastore, error) {
options := &dbOptions{
@ -83,6 +102,10 @@ func New(config config.MysqlConfig, c clock.Clock, opts ...DBOption) (*Datastore
}
func (d *Datastore) Begin() (kolide.Transaction, error) {
return d.db.Beginx()
}
func (d *Datastore) Name() string {
return "mysql"
}

28
server/datastore/mysql/options.go
View file

@ -78,14 +78,15 @@ func (d *Datastore) ResetOptions() (opts []kolide.Option, err error) {
return opts, nil
}
func (d *Datastore) OptionByName(name string) (*kolide.Option, error) {
func (d *Datastore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
db := d.getTransaction(args)
sqlStatement := `
SELECT *
FROM options
WHERE name = ?
`
var option kolide.Option
if err := d.db.Get(&option, sqlStatement, name); err != nil {
if err := db.Get(&option, sqlStatement, name); err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Option")
}
@ -94,28 +95,17 @@ func (d *Datastore) OptionByName(name string) (*kolide.Option, error) {
return &option, nil
}
func (d *Datastore) SaveOptions(opts []kolide.Option) (err error) {
func (d *Datastore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) (err error) {
db := d.getTransaction(args)
sqlStatement := `
UPDATE options
SET value = ?
WHERE id = ? AND type = ? AND NOT read_only
`
txn, err := d.db.Begin()
if err != nil {
return errors.Wrap(err, "update options begin transaction")
}
var success bool
defer func() {
if success {
if err = txn.Commit(); err == nil {
return
}
}
txn.Rollback()
}()
for _, opt := range opts {
resultInfo, err := txn.Exec(sqlStatement, opt.Value, opt.ID, opt.Type)
resultInfo, err := db.Exec(sqlStatement, opt.Value, opt.ID, opt.Type)
if err != nil {
return errors.Wrap(err, "update options")
}
@ -127,10 +117,6 @@ func (d *Datastore) SaveOptions(opts []kolide.Option) (err error) {
return notFound("Option").WithID(opt.ID)
}
}
// If all the updates succeed, set the success flag, this will cause the
// function we defined in defer to commit the transaction. Otherwise, all
// changes will be rolled back
success = true
return err
}

18
server/datastore/mysql/packs.go
View file

@ -8,14 +8,15 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
func (d *Datastore) PackByName(name string, opts ...kolide.OptionalArg) (*kolide.Pack, bool, error) {
db := d.getTransaction(opts)
sqlStatement := `
SELECT *
FROM packs
WHERE name = ? AND NOT deleted
`
var pack kolide.Pack
err := d.db.Get(&pack, sqlStatement, name)
err := db.Get(&pack, sqlStatement, name)
if err != nil {
if err == sql.ErrNoRows {
return nil, false, nil
@ -27,12 +28,13 @@ func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
}
// NewPack creates a new Pack
func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
func (d *Datastore) NewPack(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error) {
db := d.getTransaction(opts)
var (
deletedPack kolide.Pack
query string
)
err := d.db.Get(&deletedPack,
err := db.Get(&deletedPack,
"SELECT * FROM packs WHERE name = ? AND deleted", pack.Name)
switch err {
case nil:
@ -52,7 +54,7 @@ func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
}
deleted := false
result, err := d.db.Exec(query, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled, deleted)
result, err := db.Exec(query, pack.Name, pack.Description, pack.Platform, pack.CreatedBy, pack.Disabled, deleted)
if err != nil && isDuplicate(err) {
return nil, alreadyExists("Pack", deletedPack.ID)
} else if err != nil {
@ -117,13 +119,15 @@ func (d *Datastore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) {
}
// AddLabelToPack associates a kolide.Label with a kolide.Pack
func (d *Datastore) AddLabelToPack(lid uint, pid uint) error {
func (d *Datastore) AddLabelToPack(lid uint, pid uint, opts ...kolide.OptionalArg) error {
db := d.getTransaction(opts)
query := `
INSERT INTO pack_targets ( pack_id, type, target_id )
VALUES ( ?, ?, ? )
ON DUPLICATE KEY UPDATE id=id
`
_, err := d.db.Exec(query, pid, kolide.TargetLabel, lid)
_, err := db.Exec(query, pid, kolide.TargetLabel, lid)
if err != nil {
return errors.Wrap(err, "adding label to pack")
}

12
server/datastore/mysql/queries.go
View file

@ -8,14 +8,15 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) {
func (d *Datastore) QueryByName(name string, opts ...kolide.OptionalArg) (*kolide.Query, bool, error) {
db := d.getTransaction(opts)
sqlStatement := `
SELECT *
FROM queries
WHERE name = ? AND NOT deleted
`
var query kolide.Query
err := d.db.Get(&query, sqlStatement, name)
err := db.Get(&query, sqlStatement, name)
if err != nil {
if err == sql.ErrNoRows {
return nil, false, nil
@ -27,12 +28,13 @@ func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) {
// NewQuery creates a New Query. If a query with the same name was soft-deleted,
// NewQuery will replace the old one.
func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
func (d *Datastore) NewQuery(query *kolide.Query, opts ...kolide.OptionalArg) (*kolide.Query, error) {
db := d.getTransaction(opts)
var (
deletedQuery kolide.Query
sqlStatement string
)
err := d.db.Get(&deletedQuery,
err := db.Get(&deletedQuery,
"SELECT * FROM queries WHERE name = ? AND deleted", query.Name)
switch err {
case nil:
@ -61,7 +63,7 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
return nil, errors.Wrap(err, "check for existing Query")
}
deleted := false
result, err := d.db.Exec(sqlStatement, query.Name, query.Description, query.Query, query.Saved, query.AuthorID, deleted)
result, err := db.Exec(sqlStatement, query.Name, query.Description, query.Query, query.Saved, query.AuthorID, deleted)
if err != nil && isDuplicate(err) {
return nil, alreadyExists("Query", deletedQuery.ID)
} else if err != nil {

7
server/datastore/mysql/scheduled_queries.go
View file

@ -7,7 +7,8 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.ScheduledQuery, error) {
func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery, opts ...kolide.OptionalArg) (*kolide.ScheduledQuery, error) {
db := d.getTransaction(opts)
query := `
INSERT INTO scheduled_queries (
pack_id,
@ -20,7 +21,7 @@ func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.Schedu
shard
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`
result, err := d.db.Exec(query, sq.PackID, sq.QueryID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard)
result, err := db.Exec(query, sq.PackID, sq.QueryID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard)
if err != nil {
return nil, errors.Wrap(err, "inserting scheduled query")
}
@ -34,7 +35,7 @@ func (d *Datastore) NewScheduledQuery(sq *kolide.ScheduledQuery) (*kolide.Schedu
Name string
}{}
err = d.db.Select(&metadata, query, sq.QueryID)
err = db.Select(&metadata, query, sq.QueryID)
if err != nil && err == sql.ErrNoRows {
return nil, notFound("Query").WithID(sq.QueryID)
} else if err != nil {

34
server/datastore/mysql/yara.go
View file

@ -7,27 +7,19 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (sg *kolide.YARASignatureGroup, err error) {
var success bool
txn, err := d.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "new yara signature group begin transaction")
}
defer func() {
if success {
if err = txn.Commit(); err == nil {
return
}
}
txn.Rollback()
}()
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup, opts ...kolide.OptionalArg) (sg *kolide.YARASignatureGroup, err error) {
dbTx := d.getTransaction(opts)
sqlStatement := `
INSERT INTO yara_signatures (
signature_name
) VALUES( ? )
`
var result sql.Result
result, err = txn.Exec(sqlStatement, ysg.SignatureName)
result, err = dbTx.Exec(sqlStatement, ysg.SignatureName)
if isDuplicate(err) {
return nil, alreadyExists("yara signature", 0)
}
if err != nil {
return nil, errors.Wrap(err, "inserting new yara signature group")
}
@ -41,16 +33,17 @@ func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (sg *k
`
for _, path := range ysg.Paths {
_, err = txn.Exec(sqlStatement, path, ysg.ID)
_, err = dbTx.Exec(sqlStatement, path, ysg.ID)
if err != nil {
return nil, errors.Wrap(err, "inserting new signature path")
}
}
success = true
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error {
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string, opts ...kolide.OptionalArg) error {
dbTx := d.getTransaction(opts)
sqlStatement := `
INSERT INTO yara_file_paths (
file_integrity_monitoring_id,
@ -70,7 +63,10 @@ func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error
)
)
`
_, err := d.db.Exec(sqlStatement, fileSectionName, sigGroupName)
_, err := dbTx.Exec(sqlStatement, fileSectionName, sigGroupName)
if isDuplicate(err) {
return alreadyExists("yara file path", 0)
}
if err != nil {
return errors.Wrap(err, "inserting yara file path")
}

3
server/kolide/datastore.go
View file

@ -28,6 +28,7 @@ type Datastore interface {
// MigrationStatus returns nil if migrations are complete, and an error
// if migrations need to be run.
MigrationStatus() (MigrationStatus, error)
Begin() (Transaction, error)
}
type MigrationStatus int
@ -50,3 +51,5 @@ type AlreadyExistsError interface {
error
IsExists() bool
}
type OptionalArg func() interface{}

6
server/kolide/decorators.go
View file

@ -10,15 +10,15 @@ import (
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type DecoratorStore interface {
// NewDecorator creates a decorator query.
NewDecorator(decorator *Decorator) (*Decorator, error)
NewDecorator(decorator *Decorator, opts ...OptionalArg) (*Decorator, error)
// DeleteDecorator removes a decorator query.
DeleteDecorator(id uint) error
// Decorator retrieves a decorator query with supplied ID.
Decorator(id uint) (*Decorator, error)
// ListDecorators returns all decorator queries.
ListDecorators() ([]*Decorator, error)
ListDecorators(opts ...OptionalArg) ([]*Decorator, error)
// SaveDecorator updates an existing decorator
SaveDecorator(dec *Decorator) error
SaveDecorator(dec *Decorator, opts ...OptionalArg) error
}
// DecoratorService exposes decorators data so it can be manipulated by

2
server/kolide/file_integrity_monitoring.go
View file

@ -4,7 +4,7 @@ type FIMSections map[string][]string
type FileIntegrityMonitoringStore interface {
// NewFIMSection creates a named group of file paths
NewFIMSection(path *FIMSection) (*FIMSection, error)
NewFIMSection(path *FIMSection, opts ...OptionalArg) (*FIMSection, error)
// FIMSections returns all named file sections
FIMSections() (FIMSections, error)
}

4
server/kolide/import_config.go
View file

@ -40,6 +40,8 @@ const (
OptionReadonly = "option_readonly"
OptionUnknown = "option_unknown"
QueryDuplicate = "duplicate_query"
FIMDuplicate = "duplicate_fim"
YARADuplicate = "duplicate_yara"
Unsupported = "unsupported"
)
@ -156,6 +158,8 @@ type PackNameToPackDetails map[string]PackDetails
// documentation has further details.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type ImportConfig struct {
// DryRun if true an import will be attempted, and if successful will be completely rolled back
DryRun bool
// Options is a map of option name to a value which can be an int,
// bool, or string.
Options OptionNameToValueMap `json:"options"`

2
server/kolide/labels.go
View file

@ -7,7 +7,7 @@ import (
type LabelStore interface {
// Label methods
NewLabel(Label *Label) (*Label, error)
NewLabel(Label *Label, opts ...OptionalArg) (*Label, error)
DeleteLabel(lid uint) error
Label(lid uint) (*Label, error)
ListLabels(opt ListOptions) ([]*Label, error)

4
server/kolide/options.go
View file

@ -14,13 +14,13 @@ type OptionStore interface {
// values fails validation none of the writes will succeed. Note only option
// values are written. Other option fields are created in migration and do
// not change. Attempting to write ReadOnly options will cause an error.
SaveOptions(opts []Option) error
SaveOptions(opts []Option, args ...OptionalArg) error
// Options returns all options
ListOptions() ([]Option, error)
// Option return an option by ID
Option(id uint) (*Option, error)
// OptionByName returns an option uniquely identified by name
OptionByName(name string) (*Option, error)
OptionByName(name string, args ...OptionalArg) (*Option, error)
// GetOsqueryConfigOptions returns options in a format that will be the options
// section of osquery configuration
GetOsqueryConfigOptions() (map[string]interface{}, error)

6
server/kolide/packs.go
View file

@ -7,7 +7,7 @@ import (
// PackStore is the datastore interface for managing query packs.
type PackStore interface {
// NewPack creates a new pack in the datastore.
NewPack(pack *Pack) (*Pack, error)
NewPack(pack *Pack, opts ...OptionalArg) (*Pack, error)
// SavePack updates an existing pack in the datastore.
SavePack(pack *Pack) error
@ -22,10 +22,10 @@ type PackStore interface {
ListPacks(opt ListOptions) ([]*Pack, error)
// PackByName fetches pack if it exists, if the pack
// exists the bool return value is true
PackByName(name string) (*Pack, bool, error)
PackByName(name string, opts ...OptionalArg) (*Pack, bool, error)
// AddLabelToPack adds an existing label to an existing pack, both by ID.
AddLabelToPack(lid, pid uint) error
AddLabelToPack(lid, pid uint, opts ...OptionalArg) error
// RemoveLabelFromPack removes an existing label from it's association with
// an existing pack, both by ID.

4
server/kolide/queries.go
View file

@ -7,7 +7,7 @@ import (
type QueryStore interface {
// NewQuery creates a new query object in thie datastore. The returned
// query should have the ID updated.
NewQuery(query *Query) (*Query, error)
NewQuery(query *Query, opts ...OptionalArg) (*Query, error)
// SaveQuery saves changes to an existing query object.
SaveQuery(query *Query) error
// DeleteQuery (soft) deletes an existing query object.
@ -24,7 +24,7 @@ type QueryStore interface {
ListQueries(opt ListOptions) ([]*Query, error)
// QueryByName looks up a query by name, the second bool is true if a query
// by the name exists.
QueryByName(name string) (*Query, bool, error)
QueryByName(name string, opts ...OptionalArg) (*Query, bool, error)
}
type QueryService interface {

2
server/kolide/scheduled_queries.go
View file

@ -5,7 +5,7 @@ import (
)
type ScheduledQueryStore interface {
NewScheduledQuery(sq *ScheduledQuery) (*ScheduledQuery, error)
NewScheduledQuery(sq *ScheduledQuery, opts ...OptionalArg) (*ScheduledQuery, error)
SaveScheduledQuery(sq *ScheduledQuery) (*ScheduledQuery, error)
DeleteScheduledQuery(id uint) error
ScheduledQuery(id uint) (*ScheduledQuery, error)

16
server/kolide/transactions.go Normal file
View file

@ -0,0 +1,16 @@
package kolide
type Transactions interface {
Begin() (Transaction, error)
}
type Transaction interface {
Commit() error
Rollback() error
}
func HasTransaction(tx Transaction) OptionalArg {
return func() interface{} {
return tx
}
}

4
server/kolide/yara.go
View file

@ -8,10 +8,10 @@ type YARAFilePaths map[string][]string
type YARAStore interface {
// NewYARASignatureGroup creates a new mapping of a name to
// a group of YARA signatures
NewYARASignatureGroup(*YARASignatureGroup) (*YARASignatureGroup, error)
NewYARASignatureGroup(ysg *YARASignatureGroup, opts ...OptionalArg) (*YARASignatureGroup, error)
// NewYARAFilePath maps a named set of files to one or more
// groups of YARA signatures
NewYARAFilePath(fileSectionName, sigGroupName string) error
NewYARAFilePath(fileSectionName, sigGroupName string, opts ...OptionalArg) error
// YARASection creates the osquery configuration YARA section
YARASection() (*YARASection, error)
}

9
server/mock/datastore.go
View file

@ -49,3 +49,12 @@ func (m *Store) MigrationStatus() (kolide.MigrationStatus, error) {
func (m *Store) Name() string {
return "mock"
}
type mockTransaction struct{}
func (m *mockTransaction) Commit() error { return nil }
func (m *mockTransaction) Rollback() error { return nil }
func (m *Store) Begin() (kolide.Transaction, error) {
return &mockTransaction{}, nil
}

18
server/mock/datastore_decorators.go
View file

@ -6,15 +6,15 @@ import "github.com/kolide/kolide/server/kolide"
var _ kolide.DecoratorStore = (*DecoratorStore)(nil)
type NewDecoratorFunc func(decorator *kolide.Decorator) (*kolide.Decorator, error)
type NewDecoratorFunc func(decorator *kolide.Decorator, opts ...kolide.OptionalArg) (*kolide.Decorator, error)
type DeleteDecoratorFunc func(id uint) error
type DecoratorFunc func(id uint) (*kolide.Decorator, error)
type ListDecoratorsFunc func() ([]*kolide.Decorator, error)
type ListDecoratorsFunc func(opts ...kolide.OptionalArg) ([]*kolide.Decorator, error)
type SaveDecoratorFunc func(dec *kolide.Decorator) error
type SaveDecoratorFunc func(dec *kolide.Decorator, opts ...kolide.OptionalArg) error
type DecoratorStore struct {
NewDecoratorFunc NewDecoratorFunc
@ -33,9 +33,9 @@ type DecoratorStore struct {
SaveDecoratorFuncInvoked bool
}
func (s *DecoratorStore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) {
func (s *DecoratorStore) NewDecorator(decorator *kolide.Decorator, opts ...kolide.OptionalArg) (*kolide.Decorator, error) {
s.NewDecoratorFuncInvoked = true
return s.NewDecoratorFunc(decorator)
return s.NewDecoratorFunc(decorator, opts...)
}
func (s *DecoratorStore) DeleteDecorator(id uint) error {
@ -48,12 +48,12 @@ func (s *DecoratorStore) Decorator(id uint) (*kolide.Decorator, error) {
return s.DecoratorFunc(id)
}
func (s *DecoratorStore) ListDecorators() ([]*kolide.Decorator, error) {
func (s *DecoratorStore) ListDecorators(opts ...kolide.OptionalArg) ([]*kolide.Decorator, error) {
s.ListDecoratorsFuncInvoked = true
return s.ListDecoratorsFunc()
return s.ListDecoratorsFunc(opts...)
}
func (s *DecoratorStore) SaveDecorator(dec *kolide.Decorator) error {
func (s *DecoratorStore) SaveDecorator(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
s.SaveDecoratorFuncInvoked = true
return s.SaveDecoratorFunc(dec)
return s.SaveDecoratorFunc(dec, opts...)
}

6
server/mock/datastore_labels.go
View file

@ -10,7 +10,7 @@ import (
var _ kolide.LabelStore = (*LabelStore)(nil)
type NewLabelFunc func(Label *kolide.Label) (*kolide.Label, error)
type NewLabelFunc func(Label *kolide.Label, opts ...kolide.OptionalArg) (*kolide.Label, error)
type DeleteLabelFunc func(lid uint) error
@ -67,9 +67,9 @@ type LabelStore struct {
SaveLabelFuncInvoked bool
}
func (s *LabelStore) NewLabel(Label *kolide.Label) (*kolide.Label, error) {
func (s *LabelStore) NewLabel(Label *kolide.Label, opts ...kolide.OptionalArg) (*kolide.Label, error) {
s.NewLabelFuncInvoked = true
return s.NewLabelFunc(Label)
return s.NewLabelFunc(Label, opts...)
}
func (s *LabelStore) DeleteLabel(lid uint) error {

12
server/mock/datastore_options.go
View file

@ -6,13 +6,13 @@ import "github.com/kolide/kolide/server/kolide"
var _ kolide.OptionStore = (*OptionStore)(nil)
type SaveOptionsFunc func(opts []kolide.Option) error
type SaveOptionsFunc func(opts []kolide.Option, args ...kolide.OptionalArg) error
type ListOptionsFunc func() ([]kolide.Option, error)
type OptionFunc func(id uint) (*kolide.Option, error)
type OptionByNameFunc func(name string) (*kolide.Option, error)
type OptionByNameFunc func(name string, args ...kolide.OptionalArg) (*kolide.Option, error)
type GetOsqueryConfigOptionsFunc func() (map[string]interface{}, error)
@ -38,9 +38,9 @@ type OptionStore struct {
ResetOptionsFuncInvoked bool
}
func (s *OptionStore) SaveOptions(opts []kolide.Option) error {
func (s *OptionStore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) error {
s.SaveOptionsFuncInvoked = true
return s.SaveOptionsFunc(opts)
return s.SaveOptionsFunc(opts, args...)
}
func (s *OptionStore) ListOptions() ([]kolide.Option, error) {
@ -53,9 +53,9 @@ func (s *OptionStore) Option(id uint) (*kolide.Option, error) {
return s.OptionFunc(id)
}
func (s *OptionStore) OptionByName(name string) (*kolide.Option, error) {
func (s *OptionStore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
s.OptionByNameFuncInvoked = true
return s.OptionByNameFunc(name)
return s.OptionByNameFunc(name, args...)
}
func (s *OptionStore) GetOsqueryConfigOptions() (map[string]interface{}, error) {

18
server/mock/datastore_packs.go
View file

@ -6,7 +6,7 @@ import "github.com/kolide/kolide/server/kolide"
var _ kolide.PackStore = (*PackStore)(nil)
type NewPackFunc func(pack *kolide.Pack) (*kolide.Pack, error)
type NewPackFunc func(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error)
type SavePackFunc func(pack *kolide.Pack) error
@ -16,9 +16,9 @@ type PackFunc func(pid uint) (*kolide.Pack, error)
type ListPacksFunc func(opt kolide.ListOptions) ([]*kolide.Pack, error)
type PackByNameFunc func(name string) (*kolide.Pack, bool, error)
type PackByNameFunc func(name string, opts ...kolide.OptionalArg) (*kolide.Pack, bool, error)
type AddLabelToPackFunc func(lid uint, pid uint) error
type AddLabelToPackFunc func(lid uint, pid uint, opts ...kolide.OptionalArg) error
type RemoveLabelFromPackFunc func(lid uint, pid uint) error
@ -73,9 +73,9 @@ type PackStore struct {
ListExplicitHostsInPackFuncInvoked bool
}
func (s *PackStore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
func (s *PackStore) NewPack(pack *kolide.Pack, opts ...kolide.OptionalArg) (*kolide.Pack, error) {
s.NewPackFuncInvoked = true
return s.NewPackFunc(pack)
return s.NewPackFunc(pack, opts...)
}
func (s *PackStore) SavePack(pack *kolide.Pack) error {
@ -98,14 +98,14 @@ func (s *PackStore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) {
return s.ListPacksFunc(opt)
}
func (s *PackStore) PackByName(name string) (*kolide.Pack, bool, error) {
func (s *PackStore) PackByName(name string, opts ...kolide.OptionalArg) (*kolide.Pack, bool, error) {
s.PackByNameFuncInvoked = true
return s.PackByNameFunc(name)
return s.PackByNameFunc(name, opts...)
}
func (s *PackStore) AddLabelToPack(lid uint, pid uint) error {
func (s *PackStore) AddLabelToPack(lid uint, pid uint, opts ...kolide.OptionalArg) error {
s.AddLabelToPackFuncInvoked = true
return s.AddLabelToPackFunc(lid, pid)
return s.AddLabelToPackFunc(lid, pid, opts...)
}
func (s *PackStore) RemoveLabelFromPack(lid uint, pid uint) error {

1
server/service/endpoint_import_config.go
View file

@ -8,6 +8,7 @@ import (
)
type importRequest struct {
DryRun bool `json:"dry_run"`
// Config contains a JSON osquery config supplied by the end user
Config string `json:"config"`
// ExternalPackConfigs contains a map of external Pack configs keyed by

27
server/service/logging_import_config.go Normal file
View file

@ -0,0 +1,27 @@
package service
import (
"context"
"time"
"github.com/kolide/kolide/server/kolide"
)
func (mw loggingMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var (
resp *kolide.ImportConfigResponse
err error
)
defer func(begin time.Time) {
_ = mw.logger.Log(
"method", "ImportConfig",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
resp, err = mw.Service.ImportConfig(ctx, cfg)
return resp, err
}

23
server/service/metrics_import_config.go Normal file
View file

@ -0,0 +1,23 @@
package service
import (
"context"
"fmt"
"time"
"github.com/kolide/kolide/server/kolide"
)
func (mw metricsMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var (
resp *kolide.ImportConfigResponse
err error
)
defer func(begin time.Time) {
lvs := []string{"method", "ImportConfig", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
resp, err = mw.Service.ImportConfig(ctx, cfg)
return resp, err
}

196
server/service/service_import_config.go
View file

@ -2,12 +2,14 @@ package service
import (
"context"
"errors"
"crypto/md5"
"fmt"
"strconv"
"strings"
"github.com/kolide/kolide/server/contexts/viewer"
"github.com/kolide/kolide/server/kolide"
"github.com/pkg/errors"
)
func (svc service) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
@ -18,32 +20,65 @@ func (svc service) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (
if !ok {
return nil, errors.New("internal error, unable to fetch user")
}
if err := svc.importOptions(cfg.Options, resp); err != nil {
tx, err := svc.ds.Begin()
if err != nil {
return nil, err
}
if err := svc.importPacks(vc.UserID(), cfg, resp); err != nil {
return nil, err
if err := svc.importOptions(cfg.Options, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importOptions")
return nil, errors.Wrap(err, "importOptions failed")
}
if err := svc.importScheduledQueries(vc.UserID(), cfg, resp); err != nil {
return nil, err
if err := svc.importPacks(vc.UserID(), cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importPacks")
return nil, errors.Wrap(err, "importPacks failed")
}
if err := svc.importDecorators(cfg, resp); err != nil {
return nil, err
if err := svc.importScheduledQueries(vc.UserID(), cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importScheduledQueries")
return nil, errors.Wrap(err, "importScheduledQueries failed")
}
if err := svc.importFIMSections(cfg, resp); err != nil {
return nil, err
if err := svc.importDecorators(cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importDecorators")
return nil, errors.Wrap(err, "importDecorators")
}
if err := svc.importFIMSections(cfg, resp, tx); err != nil {
svc.rollbackImportConfig(tx, "importFIMSections")
return nil, errors.Wrap(err, "importFIMSections")
}
if cfg.DryRun {
if err := tx.Rollback(); err != nil {
return nil, errors.Wrap(err, "dry run rollback failed")
}
return resp, nil
}
if err := tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit failed")
}
return resp, nil
}
func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
func (svc service) rollbackImportConfig(tx kolide.Transaction, method string) {
if err := tx.Rollback(); err != nil {
svc.logger.Log(
"method", method,
"err", errors.Wrap(err, fmt.Sprintf("db rollback failed in %s", method)),
)
}
}
func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.YARA != nil {
for sig, paths := range cfg.YARA.Signatures {
ysg := &kolide.YARASignatureGroup{
SignatureName: sig,
Paths: paths,
}
_, err := svc.ds.NewYARASignatureGroup(ysg)
_, err := svc.ds.NewYARASignatureGroup(ysg, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.YARAFileSection).SkipCount++
resp.Status(kolide.YARAFileSection).Warning(kolide.YARADuplicate, "skipped '%s', already exists", sig)
continue
}
if err != nil {
return err
}
@ -52,19 +87,28 @@ func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfi
}
for section, sigs := range cfg.YARA.FilePaths {
for _, sig := range sigs {
err := svc.ds.NewYARAFilePath(section, sig)
err := svc.ds.NewYARAFilePath(section, sig, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.YARAFileSection).SkipCount++
resp.Status(kolide.YARAFileSection).Warning(kolide.YARADuplicate, "skipped '%s', already exists", section)
continue
}
if err != nil {
return err
}
resp.Status(kolide.YARAFileSection).ImportCount++
resp.Status(kolide.YARAFileSection).Message("imported '%s'", section)
}
resp.Status(kolide.YARAFileSection).ImportCount++
resp.Status(kolide.YARAFileSection).Message("imported '%s'", section)
}
}
return nil
}
func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
type dbDuplicateError interface {
IsExists() bool
}
func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.FileIntegrityMonitoring != nil {
for sectionName, paths := range cfg.FileIntegrityMonitoring {
fp := &kolide.FIMSection{
@ -72,7 +116,12 @@ func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.Impo
Description: "imported",
Paths: paths,
}
_, err := svc.ds.NewFIMSection(fp)
_, err := svc.ds.NewFIMSection(fp, kolide.HasTransaction(tx))
if _, ok := err.(dbDuplicateError); ok {
resp.Status(kolide.FilePathsSection).SkipCount++
resp.Status(kolide.FilePathsSection).Warning(kolide.FIMDuplicate, "skipped '%s', already exists", sectionName)
continue
}
if err != nil {
return err
}
@ -81,47 +130,102 @@ func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.Impo
}
}
// this has to happen AFTER fim section, because it requires file paths
return svc.importYARA(cfg, resp)
return svc.importYARA(cfg, resp, tx)
}
func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
func (svc service) getExistingDecoratorQueries(tx kolide.Transaction) (map[string]int, error) {
decs, err := svc.ds.ListDecorators(kolide.HasTransaction(tx))
if err != nil {
return nil, err
}
queryHashes := map[string]int{}
for _, dec := range decs {
hash := fmt.Sprintf("%x", md5.Sum([]byte(dec.Query)))
queryHashes[hash] = 0
}
return queryHashes, nil
}
func decoratorExists(query string, queryHashes map[string]int) bool {
hash := fmt.Sprintf("%x", md5.Sum([]byte(query)))
_, exists := queryHashes[hash]
return exists
}
func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
if cfg.Decorators != nil {
queryHashes, err := svc.getExistingDecoratorQueries(tx)
if err != nil {
return errors.Wrap(err, "getting existing queries")
}
for _, query := range cfg.Decorators.Load {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped load '%s'", query)
continue
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorLoad,
}
_, err := svc.ds.NewDecorator(decorator)
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported load '%s'", query)
resp.Status(kolide.DecoratorsSection).Warning("imported load '%s'", query)
}
for _, query := range cfg.Decorators.Always {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped always '%s'", query)
continue
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorAlways,
}
_, err := svc.ds.NewDecorator(decorator)
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported always '%s'", query)
}
for key, queries := range cfg.Decorators.Interval {
for _, query := range queries {
if decoratorExists(query, queryHashes) {
resp.Status(kolide.DecoratorsSection).SkipCount++
resp.Status(kolide.DecoratorsSection).Warning(kolide.QueryDuplicate, "skipped interval '%s'", query)
continue
}
interval, err := strconv.ParseInt(key, 10, 32)
if err != nil {
return err
}
decName, err := uniqueImportName()
if err != nil {
return err
}
decorator := &kolide.Decorator{
Name: decName,
Query: query,
Type: kolide.DecoratorInterval,
Interval: uint(interval),
}
_, err = svc.ds.NewDecorator(decorator)
_, err = svc.ds.NewDecorator(decorator, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -134,8 +238,8 @@ func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.Impor
return nil
}
func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
_, ok, err := svc.ds.PackByName(kolide.ImportPackName)
func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
_, ok, err := svc.ds.PackByName(kolide.ImportPackName, kolide.HasTransaction(tx))
if ok {
resp.Status(kolide.PacksSection).Warning(
kolide.PackDuplicate, "skipped '%s' already exists", kolide.ImportPackName,
@ -150,7 +254,7 @@ func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, re
CreatedBy: uid,
Disabled: false,
}
pack, err = svc.ds.NewPack(pack)
pack, err = svc.ds.NewPack(pack, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -159,7 +263,7 @@ func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, re
for queryName, queryDetails := range cfg.Schedule {
var query *kolide.Query
query, ok, err = svc.ds.QueryByName(queryName)
query, ok, err = svc.ds.QueryByName(queryName, kolide.HasTransaction(tx))
// if we find the query check to see if the import query matches the
// query we have, if it doesn't skip it
if ok {
@ -185,7 +289,7 @@ func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, re
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query)
query, err = svc.ds.NewQuery(query, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -204,7 +308,7 @@ func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, re
Version: queryDetails.Version,
Shard: configInt2Ptr(queryDetails.Shard),
}
_, err = svc.ds.NewScheduledQuery(sq)
_, err = svc.ds.NewScheduledQuery(sq, kolide.HasTransaction(tx))
if err != nil {
return nil
}
@ -215,14 +319,14 @@ func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, re
return nil
}
func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
labelCache := map[string]*kolide.Label{}
packs, err := cfg.CollectPacks()
if err != nil {
return err
}
for packName, packDetails := range packs {
_, ok, err := svc.ds.PackByName(packName)
_, ok, err := svc.ds.PackByName(packName, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -253,15 +357,15 @@ func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.
Description: "Imported pack",
Platform: packDetails.Platform,
}
pack, err = svc.ds.NewPack(pack)
pack, err = svc.ds.NewPack(pack, kolide.HasTransaction(tx))
if err != nil {
return err
}
err = svc.createLabelsForPack(pack, &packDetails, labelCache, resp)
err = svc.createLabelsForPack(pack, &packDetails, labelCache, resp, tx)
if err != nil {
return err
}
err = svc.createQueriesForPack(uid, pack, &packDetails, resp)
err = svc.createQueriesForPack(uid, pack, &packDetails, resp, tx)
if err != nil {
return err
}
@ -281,7 +385,7 @@ func hashQuery(platform, query string) string {
}
func uniqueImportName() (string, error) {
random, err := kolide.RandomText(12)
random, err := kolide.RandomText(6)
if err != nil {
return "", err
}
@ -289,9 +393,9 @@ func uniqueImportName() (string, error) {
}
func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *kolide.PackDetails,
resp *kolide.ImportConfigResponse) error {
resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
for queryName, queryDetails := range details.Queries {
query, ok, err := svc.ds.QueryByName(queryName)
query, ok, err := svc.ds.QueryByName(queryName, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -304,7 +408,7 @@ func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *ko
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query)
query, err = svc.ds.NewQuery(query, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -324,7 +428,7 @@ func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *ko
Version: queryDetails.Version,
Shard: configInt2Ptr(queryDetails.Shard),
}
_, err = svc.ds.NewScheduledQuery(scheduledQuery)
_, err = svc.ds.NewScheduledQuery(scheduledQuery, kolide.HasTransaction(tx))
if err != nil {
return nil
}
@ -338,13 +442,13 @@ func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *ko
// each query and assigns it to the pack passed as an argument. Once a Label is created we cache
// it for reuse.
func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDetails,
cache map[string]*kolide.Label, resp *kolide.ImportConfigResponse) error {
cache map[string]*kolide.Label, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
for _, query := range details.Discovery {
hash := hashQuery(details.Platform, query)
label, ok := cache[hash]
// add existing label to pack
if ok {
err := svc.ds.AddLabelToPack(label.ID, pack.ID)
err := svc.ds.AddLabelToPack(label.ID, pack.ID, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -365,13 +469,13 @@ func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDe
LabelType: kolide.LabelTypeRegular,
Platform: details.Platform,
}
label, err = svc.ds.NewLabel(label)
label, err = svc.ds.NewLabel(label, kolide.HasTransaction(tx))
if err != nil {
return err
}
// hang on to label so we can reuse it for other packs if needed
cache[hash] = label
err = svc.ds.AddLabelToPack(label.ID, pack.ID)
err = svc.ds.AddLabelToPack(label.ID, pack.ID, kolide.HasTransaction(tx))
if err != nil {
return err
}
@ -382,10 +486,10 @@ func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDe
return nil
}
func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.ImportConfigResponse) error {
func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.ImportConfigResponse, tx kolide.Transaction) error {
var updateOptions []kolide.Option
for optName, optValue := range opts {
opt, err := svc.ds.OptionByName(optName)
opt, err := svc.ds.OptionByName(optName, kolide.HasTransaction(tx))
if err != nil {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionUnknown, "skipped '%s' can't find option", optName,
@ -413,7 +517,7 @@ func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.
updateOptions = append(updateOptions, *opt)
}
if len(updateOptions) > 0 {
if err := svc.ds.SaveOptions(updateOptions); err != nil {
if err := svc.ds.SaveOptions(updateOptions, kolide.HasTransaction(tx)); err != nil {
return err
}
}

19
server/service/service_import_config_test.go
View file

@ -69,7 +69,8 @@ func TestImportFilePaths(t *testing.T) {
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importFIMSections(cfg, resp)
tx, _ := svc.ds.Begin()
err := svc.importFIMSections(cfg, resp, tx)
require.Nil(t, err)
assert.Equal(t, 2, resp.Status(kolide.FilePathsSection).ImportCount)
sections, err := svc.ds.FIMSections()
@ -105,7 +106,8 @@ func TestImportDecorators(t *testing.T) {
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importDecorators(cfg, resp)
tx, _ := svc.ds.Begin()
err := svc.importDecorators(cfg, resp, tx)
require.Nil(t, err)
assert.Equal(t, 5, resp.Status(kolide.DecoratorsSection).ImportCount)
dec, err := svc.ds.ListDecorators()
@ -165,8 +167,8 @@ func TestImportScheduledQueries(t *testing.T) {
}
_, err = svc.ds.NewQuery(noskipQuery)
require.Nil(t, err)
err = svc.importScheduledQueries(user.ID, cfg, resp)
tx, _ := svc.ds.Begin()
err = svc.importScheduledQueries(user.ID, cfg, resp, tx)
require.Nil(t, err)
_, ok, err := svc.ds.QueryByName("q1")
require.Nil(t, err)
@ -188,7 +190,8 @@ func TestOptionsImportConfig(t *testing.T) {
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importOptions(opts, resp)
tx, _ := svc.ds.Begin()
err := svc.importOptions(opts, resp, tx)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
@ -216,13 +219,14 @@ func TestOptionsImportConfigWithSkips(t *testing.T) {
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
// set option val, it should be skipped
opt, err := svc.ds.OptionByName("aws_firehose_period")
require.Nil(t, err)
opt.SetValue(23)
err = svc.ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
err = svc.importOptions(opts, resp)
err = svc.importOptions(opts, resp, tx)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
@ -236,6 +240,7 @@ func TestOptionsImportConfigWithSkips(t *testing.T) {
func TestPacksImportConfig(t *testing.T) {
svc := createServiceMockForImport(t)
tx, _ := svc.ds.Begin()
p := &kolide.Pack{
Name: "dup",
@ -320,7 +325,7 @@ func TestPacksImportConfig(t *testing.T) {
packs, err := importConfig.CollectPacks()
require.Nil(t, err)
assert.Len(t, packs, 4)
err = svc.importPacks(user.ID, &importConfig, resp)
err = svc.importPacks(user.ID, &importConfig, resp, tx)
require.Nil(t, err)
queries, err := svc.ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)

2
server/service/service_osquery_test.go
View file

@ -795,7 +795,7 @@ func TestUpdateHostIntervals(t *testing.T) {
svc, err := newTestService(ds, nil)
require.Nil(t, err)
ds.ListDecoratorsFunc = func() ([]*kolide.Decorator, error) {
ds.ListDecoratorsFunc = func(opt ...kolide.OptionalArg) ([]*kolide.Decorator, error) {
return []*kolide.Decorator{}, nil
}
ds.ListPacksFunc = func(opt kolide.ListOptions) ([]*kolide.Pack, error) {

1
server/service/transport_import_config.go
View file

@ -15,6 +15,7 @@ func decodeImportConfigRequest(ctx context.Context, r *http.Request) (interface{
}
// Unmarshal main config
conf := kolide.ImportConfig{
DryRun: req.DryRun,
Packs: make(kolide.PackNameMap),
ExternalPacks: make(kolide.PackNameToPackDetails),
}

8
server/service/validation_decorators_test.go
View file

@ -21,7 +21,7 @@ func TestDecoratorValidation(t *testing.T) {
Type: kolide.DecoratorAlways,
}, nil
}
ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error {
ds.SaveDecoratorFunc = func(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
return nil
}
svc := &service{
@ -53,7 +53,7 @@ func TestDecoratorValidationIntervalMissing(t *testing.T) {
Type: kolide.DecoratorAlways,
}, nil
}
ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error {
ds.SaveDecoratorFunc = func(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
return nil
}
svc := &service{
@ -86,7 +86,7 @@ func TestDecoratorValidationIntervalSameType(t *testing.T) {
Interval: 600,
}, nil
}
ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error {
ds.SaveDecoratorFunc = func(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
return nil
}
svc := &service{
@ -118,7 +118,7 @@ func TestDecoratorValidationIntervalInvalid(t *testing.T) {
Interval: 600,
}, nil
}
ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error {
ds.SaveDecoratorFunc = func(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
return nil
}
svc := &service{