From 6f4dcdd082a9c630e48f869c7c5734a152d7acf7 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Fri, 13 Jan 2017 12:35:25 -0600 Subject: [PATCH] Import Config - /config/import #366 (#764) --- server/datastore/datastore_decorators_test.go | 30 ++ server/datastore/datastore_options_test.go | 3 - server/datastore/datastore_packs_test.go | 21 + server/datastore/datastore_queries_test.go | 14 + server/datastore/datastore_test.go | 5 + server/datastore/datastore_yara_test.go | 53 +++ .../file_integrity_monitoring_test.go | 38 ++ server/datastore/inmem/decorators.go | 40 ++ .../inmem/file_integrity_monitoring.go | 23 + server/datastore/inmem/inmem.go | 12 +- server/datastore/inmem/labels.go | 4 +- server/datastore/inmem/packs.go | 11 + server/datastore/inmem/queries.go | 17 +- server/datastore/inmem/yara.go | 35 ++ server/datastore/inmem_test.go | 1 + server/datastore/mysql/app_configs.go | 6 +- server/datastore/mysql/campaigns.go | 22 +- server/datastore/mysql/decorators.go | 71 +++ .../mysql/file_integrity_monitoring.go | 78 ++++ server/datastore/mysql/labels.go | 28 +- ...20170105151732_AddNameIndexToQueryTable.go | 20 + .../20170108191242_CreateDecoratorTable.go | 27 ++ .../tables/20170109094020_CreateFIMTables.go | 51 +++ .../tables/20170109130438_CreateYARATables.go | 66 +++ server/datastore/mysql/packs.go | 19 + server/datastore/mysql/password_reset.go | 18 +- server/datastore/mysql/queries.go | 44 +- server/datastore/mysql/sessions.go | 16 +- server/datastore/mysql/yara.go | 126 ++++++ server/datastore/mysql_test.go | 1 - server/errors/errors.go | 5 - server/errors/errors_test.go | 13 - server/kolide/datastore.go | 3 + server/kolide/decorators.go | 34 ++ server/kolide/file_integrity_monitoring.go | 20 + server/kolide/import_config.go | 218 +++++++++ server/kolide/import_config_test.go | 78 ++++ server/kolide/import_config_unmarshaler.go | 102 +++++ server/kolide/options.go | 13 + server/kolide/packs.go | 3 + server/kolide/queries.go | 26 +- server/kolide/service.go | 1 + server/kolide/yara.go | 31 ++ server/mock/datastore.go | 3 + server/service/endpoint_import_config.go | 41 ++ server/service/endpoint_import_config_test.go | 108 +++++ server/service/endpoint_test.go | 4 + server/service/handler.go | 6 + server/service/service.go | 2 +- server/service/service_import_config.go | 421 ++++++++++++++++++ server/service/service_import_config_test.go | 340 ++++++++++++++ server/service/service_invites_test.go | 4 +- server/service/transport_import_config.go | 34 ++ server/service/util_test.go | 4 + server/service/validation_import_config.go | 113 +++++ server/service/validation_users.go | 7 + 56 files changed, 2420 insertions(+), 114 deletions(-) create mode 100644 server/datastore/datastore_decorators_test.go create mode 100644 server/datastore/datastore_yara_test.go create mode 100644 server/datastore/file_integrity_monitoring_test.go create mode 100644 server/datastore/inmem/decorators.go create mode 100644 server/datastore/inmem/file_integrity_monitoring.go create mode 100644 server/datastore/inmem/yara.go create mode 100644 server/datastore/mysql/decorators.go create mode 100644 server/datastore/mysql/file_integrity_monitoring.go create mode 100644 server/datastore/mysql/migrations/tables/20170105151732_AddNameIndexToQueryTable.go create mode 100644 server/datastore/mysql/migrations/tables/20170108191242_CreateDecoratorTable.go create mode 100644 server/datastore/mysql/migrations/tables/20170109094020_CreateFIMTables.go create mode 100644 server/datastore/mysql/migrations/tables/20170109130438_CreateYARATables.go create mode 100644 server/datastore/mysql/yara.go create mode 100644 server/kolide/decorators.go create mode 100644 server/kolide/file_integrity_monitoring.go create mode 100644 server/kolide/import_config.go create mode 100644 server/kolide/import_config_test.go create mode 100644 server/kolide/import_config_unmarshaler.go create mode 100644 server/kolide/yara.go create mode 100644 server/service/endpoint_import_config.go create mode 100644 server/service/endpoint_import_config_test.go create mode 100644 server/service/service_import_config.go create mode 100644 server/service/service_import_config_test.go create mode 100644 server/service/transport_import_config.go create mode 100644 server/service/validation_import_config.go diff --git a/server/datastore/datastore_decorators_test.go b/server/datastore/datastore_decorators_test.go new file mode 100644 index 0000000000..6ced6e60ec --- /dev/null +++ b/server/datastore/datastore_decorators_test.go @@ -0,0 +1,30 @@ +package datastore + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testDecorators(t *testing.T, ds kolide.Datastore) { + decorator := &kolide.Decorator{ + Query: "select from something", + Type: kolide.DecoratorInterval, + Interval: 60, + } + decorator, err := ds.NewDecorator(decorator) + require.Nil(t, err) + require.True(t, decorator.ID > 0) + result, err := ds.Decorator(decorator.ID) + require.Nil(t, err) + assert.Equal(t, decorator.Query, result.Query) + results, err := ds.ListDecorators() + require.Nil(t, err) + assert.Len(t, results, 1) + err = ds.DeleteDecorator(decorator.ID) + require.Nil(t, err) + result, err = ds.Decorator(decorator.ID) + assert.NotNil(t, err) +} diff --git a/server/datastore/datastore_options_test.go b/server/datastore/datastore_options_test.go index 7dd6c01388..f4ef2147bd 100644 --- a/server/datastore/datastore_options_test.go +++ b/server/datastore/datastore_options_test.go @@ -12,7 +12,6 @@ import ( func testOptions(t *testing.T, ds kolide.Datastore) { require.Nil(t, ds.MigrateData()) - // were options pre-loaded? opts, err := ds.ListOptions() require.Nil(t, err) @@ -56,12 +55,10 @@ func testOptions(t *testing.T, ds kolide.Datastore) { opt2, err = ds.OptionByName("disable_distributed") require.Nil(t, err) assert.Equal(t, false, opt2.GetValue()) - } func testOptionsToConfig(t *testing.T, ds kolide.Datastore) { require.Nil(t, ds.MigrateData()) - resp, err := ds.GetOsqueryConfigOptions() require.Nil(t, err) assert.Len(t, resp, 10) diff --git a/server/datastore/datastore_packs_test.go b/server/datastore/datastore_packs_test.go index 25b700b14a..6f3a1b265e 100644 --- a/server/datastore/datastore_packs_test.go +++ b/server/datastore/datastore_packs_test.go @@ -30,6 +30,27 @@ func testDeletePack(t *testing.T, ds kolide.Datastore) { assert.NotNil(t, err) } +func testGetPackByName(t *testing.T, ds kolide.Datastore) { + pack := &kolide.Pack{ + Name: "foo", + } + _, err := ds.NewPack(pack) + assert.Nil(t, err) + assert.NotEqual(t, uint(0), pack.ID) + + pack, ok, err := ds.PackByName(pack.Name) + require.Nil(t, err) + assert.True(t, ok) + assert.NotNil(t, pack) + assert.Equal(t, "foo", pack.Name) + + pack, ok, err = ds.PackByName("bar") + require.Nil(t, err) + assert.False(t, ok) + assert.Nil(t, pack) + +} + func testGetHostsInPack(t *testing.T, ds kolide.Datastore) { user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true) diff --git a/server/datastore/datastore_queries_test.go b/server/datastore/datastore_queries_test.go index 889e9617c8..8b8d1fc1c9 100644 --- a/server/datastore/datastore_queries_test.go +++ b/server/datastore/datastore_queries_test.go @@ -32,6 +32,20 @@ func testDeleteQuery(t *testing.T, ds kolide.Datastore) { assert.NotNil(t, err) } +func testGetQueryByName(t *testing.T, ds kolide.Datastore) { + user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true) + test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) + actual, ok, err := ds.QueryByName("q1") + require.Nil(t, err) + assert.True(t, ok) + assert.Equal(t, "q1", actual.Name) + + actual, ok, err = ds.QueryByName("xxx") + assert.Nil(t, err) + assert.False(t, ok) + +} + func testDeleteQueries(t *testing.T, ds kolide.Datastore) { user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true) diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index 86767afc87..d15bc692f0 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -59,6 +59,11 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testOptions, testNewScheduledQuery, testOptionsToConfig, + testGetPackByName, + testGetQueryByName, + testDecorators, + testFileIntegrityMonitoring, + testYARAStore, testAddLabelToPackTwice, testGenerateHostStatusStatistics, testMarkHostSeen, diff --git a/server/datastore/datastore_yara_test.go b/server/datastore/datastore_yara_test.go new file mode 100644 index 0000000000..454efdf5af --- /dev/null +++ b/server/datastore/datastore_yara_test.go @@ -0,0 +1,53 @@ +package datastore + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testYARAStore(t *testing.T, ds kolide.Datastore) { + ysg := &kolide.YARASignatureGroup{ + SignatureName: "sig1", + Paths: []string{ + "path1", + "path2", + }, + } + ysg, err := ds.NewYARASignatureGroup(ysg) + require.Nil(t, err) + require.True(t, ysg.ID > 0) + fp := &kolide.FIMSection{ + SectionName: "fp1", + Paths: []string{ + "path1", + "path2", + "path3", + }, + } + fp, err = ds.NewFIMSection(fp) + require.Nil(t, err) + assert.True(t, fp.ID > 0) + + err = ds.NewYARAFilePath("fp1", "sig1") + require.Nil(t, err) + yaraSection, err := ds.YARASection() + require.Nil(t, err) + require.Len(t, yaraSection.FilePaths, 1) + assert.Len(t, yaraSection.FilePaths["fp1"], 1) + require.Len(t, yaraSection.Signatures, 1) + assert.Len(t, yaraSection.Signatures["sig1"], 2) + ysg = &kolide.YARASignatureGroup{ + SignatureName: "sig2", + Paths: []string{ + "path3", + }, + } + ysg, err = ds.NewYARASignatureGroup(ysg) + require.Nil(t, err) + yaraSection, err = ds.YARASection() + require.Nil(t, err) + assert.Len(t, yaraSection.Signatures["sig2"], 1) +} diff --git a/server/datastore/file_integrity_monitoring_test.go b/server/datastore/file_integrity_monitoring_test.go new file mode 100644 index 0000000000..5680ea40a6 --- /dev/null +++ b/server/datastore/file_integrity_monitoring_test.go @@ -0,0 +1,38 @@ +package datastore + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testFileIntegrityMonitoring(t *testing.T, ds kolide.Datastore) { + fp := &kolide.FIMSection{ + SectionName: "fp1", + Paths: []string{ + "path1", + "path2", + "path3", + }, + } + fp, err := ds.NewFIMSection(fp) + require.Nil(t, err) + assert.True(t, fp.ID > 0) + fp = &kolide.FIMSection{ + SectionName: "fp2", + Paths: []string{ + "path4", + "path5", + }, + } + _, err = ds.NewFIMSection(fp) + require.Nil(t, err) + + actual, err := ds.FIMSections() + require.Nil(t, err) + assert.Len(t, actual, 2) + assert.Len(t, actual["fp1"], 3) + assert.Len(t, actual["fp2"], 2) +} diff --git a/server/datastore/inmem/decorators.go b/server/datastore/inmem/decorators.go new file mode 100644 index 0000000000..c22058fa09 --- /dev/null +++ b/server/datastore/inmem/decorators.go @@ -0,0 +1,40 @@ +package inmem + +import "github.com/kolide/kolide-ose/server/kolide" + +func (d *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + decorator.ID = d.nextID(decorator) + d.decorators[decorator.ID] = decorator + return decorator, nil +} + +func (d *Datastore) DeleteDecorator(id uint) error { + d.mtx.Lock() + defer d.mtx.Unlock() + if _, ok := d.decorators[id]; !ok { + return notFound("Decorator").WithID(id) + } + delete(d.decorators, id) + return nil +} + +func (d *Datastore) Decorator(id uint) (*kolide.Decorator, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + if result, ok := d.decorators[id]; ok { + return result, nil + } + return nil, notFound("Decorator").WithID(id) +} + +func (d *Datastore) ListDecorators() ([]*kolide.Decorator, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + var result []*kolide.Decorator + for _, dec := range d.decorators { + result = append(result, dec) + } + return result, nil +} diff --git a/server/datastore/inmem/file_integrity_monitoring.go b/server/datastore/inmem/file_integrity_monitoring.go new file mode 100644 index 0000000000..a37eb3f2b6 --- /dev/null +++ b/server/datastore/inmem/file_integrity_monitoring.go @@ -0,0 +1,23 @@ +package inmem + +import ( + "github.com/kolide/kolide-ose/server/kolide" +) + +func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (*kolide.FIMSection, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + fp.ID = d.nextID(fp) + d.filePaths[fp.ID] = fp + return fp, nil +} + +func (d *Datastore) FIMSections() (kolide.FIMSections, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + result := make(kolide.FIMSections) + for _, filePath := range d.filePaths { + result[filePath.SectionName] = append(result[filePath.SectionName], filePath.Paths...) + } + return result, nil +} diff --git a/server/datastore/inmem/inmem.go b/server/datastore/inmem/inmem.go index e545e898f3..40d31ad58e 100644 --- a/server/datastore/inmem/inmem.go +++ b/server/datastore/inmem/inmem.go @@ -33,6 +33,10 @@ type Datastore struct { distributedQueryCampaigns map[uint]kolide.DistributedQueryCampaign distributedQueryCampaignTargets map[uint]kolide.DistributedQueryCampaignTarget options map[uint]*kolide.Option + decorators map[uint]*kolide.Decorator + filePaths map[uint]*kolide.FIMSection + yaraFilePaths kolide.YARAFilePaths + yaraSignatureGroups map[uint]*kolide.YARASignatureGroup appConfig *kolide.AppConfig config *config.KolideConfig } @@ -89,13 +93,15 @@ func (d *Datastore) MigrateTables() error { d.distributedQueryCampaigns = make(map[uint]kolide.DistributedQueryCampaign) d.distributedQueryCampaignTargets = make(map[uint]kolide.DistributedQueryCampaignTarget) d.options = make(map[uint]*kolide.Option) + d.decorators = make(map[uint]*kolide.Decorator) + d.filePaths = make(map[uint]*kolide.FIMSection) + d.yaraFilePaths = make(kolide.YARAFilePaths) + d.yaraSignatureGroups = make(map[uint]*kolide.YARASignatureGroup) return nil } func (d *Datastore) MigrateData() error { - d.mtx.Lock() - for _, initData := range appstate.Options() { opt := kolide.Option{ Name: initData.Name, @@ -115,8 +121,6 @@ func (d *Datastore) MigrateData() error { SMTPVerifySSLCerts: true, } - d.mtx.Unlock() - if err := d.createBuiltinLabels(); err != nil { return err } diff --git a/server/datastore/inmem/labels.go b/server/datastore/inmem/labels.go index 6f9c3b985c..1dfa1c50e6 100644 --- a/server/datastore/inmem/labels.go +++ b/server/datastore/inmem/labels.go @@ -12,9 +12,10 @@ import ( ) func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) { + d.mtx.Lock() + defer d.mtx.Unlock() newLabel := *label - d.mtx.Lock() for _, l := range d.labels { if l.Name == label.Name { return nil, alreadyExists("Label", l.ID) @@ -23,7 +24,6 @@ func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) { newLabel.ID = d.nextID(label) d.labels[newLabel.ID] = &newLabel - d.mtx.Unlock() return &newLabel, nil } diff --git a/server/datastore/inmem/packs.go b/server/datastore/inmem/packs.go index 1041311950..935d2a70c3 100644 --- a/server/datastore/inmem/packs.go +++ b/server/datastore/inmem/packs.go @@ -6,6 +6,17 @@ import ( "github.com/kolide/kolide-ose/server/kolide" ) +func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + for _, p := range d.packs { + if p.Name == name { + return p, true, nil + } + } + return nil, false, nil +} + func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { newPack := *pack diff --git a/server/datastore/inmem/queries.go b/server/datastore/inmem/queries.go index 086fa396c8..0696197026 100644 --- a/server/datastore/inmem/queries.go +++ b/server/datastore/inmem/queries.go @@ -3,8 +3,8 @@ package inmem import ( "sort" - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { @@ -26,6 +26,17 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { return &newQuery, nil } +func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + for _, q := range d.queries { + if name == q.Name { + return q, true, nil + } + } + return nil, false, nil +} + func (d *Datastore) SaveQuery(query *kolide.Query) error { d.mtx.Lock() defer d.mtx.Unlock() @@ -86,7 +97,7 @@ func (d *Datastore) Query(id uint) (*kolide.Query, error) { query.AuthorName = d.getUserNameByID(query.AuthorID) if err := d.loadPacksForQueries([]*kolide.Query{query}); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "error fetching query by id") } return query, nil @@ -136,7 +147,7 @@ func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) queries = queries[low:high] if err := d.loadPacksForQueries(queries); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "error listing queries") } return queries, nil diff --git a/server/datastore/inmem/yara.go b/server/datastore/inmem/yara.go new file mode 100644 index 0000000000..356929a5f4 --- /dev/null +++ b/server/datastore/inmem/yara.go @@ -0,0 +1,35 @@ +package inmem + +import "github.com/kolide/kolide-ose/server/kolide" + +func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (*kolide.YARASignatureGroup, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + ysg.ID = d.nextID(ysg) + d.yaraSignatureGroups[ysg.ID] = ysg + return ysg, nil +} + +func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error { + d.mtx.Lock() + defer d.mtx.Unlock() + d.yaraFilePaths[fileSectionName] = append(d.yaraFilePaths[fileSectionName], sigGroupName) + return nil +} + +func (d *Datastore) YARASection() (*kolide.YARASection, error) { + d.mtx.Lock() + defer d.mtx.Unlock() + result := &kolide.YARASection{ + Signatures: make(map[string][]string), + FilePaths: make(map[string][]string), + } + for _, ysg := range d.yaraSignatureGroups { + result.Signatures[ysg.SignatureName] = append(result.Signatures[ysg.SignatureName], ysg.Paths...) + } + for fileSection, sigSection := range d.yaraFilePaths { + result.FilePaths[fileSection] = append(result.FilePaths[fileSection], sigSection...) + } + + return result, nil +} diff --git a/server/datastore/inmem_test.go b/server/datastore/inmem_test.go index a362bef702..8fa6efd188 100644 --- a/server/datastore/inmem_test.go +++ b/server/datastore/inmem_test.go @@ -13,6 +13,7 @@ func TestInmem(t *testing.T) { for _, f := range testFunctions { t.Run(functionName(f), func(t *testing.T) { ds, err := inmem.New(config.TestConfig()) + require.Nil(t, err) defer func() { require.Nil(t, ds.Drop()) }() require.Nil(t, err) f(t, ds) diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go index 35d8ab7320..f1ede2224e 100644 --- a/server/datastore/mysql/app_configs.go +++ b/server/datastore/mysql/app_configs.go @@ -1,13 +1,13 @@ package mysql import ( - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) func (d *Datastore) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) { if err := d.SaveAppConfig(info); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "new app config") } return info, nil @@ -18,7 +18,7 @@ func (d *Datastore) AppConfig() (*kolide.AppConfig, error) { info := &kolide.AppConfig{} err := d.db.Get(info, "SELECT * FROM app_configs LIMIT 1") if err != nil { - return nil, err + return nil, errors.Wrap(err, "selecting app config") } return info, nil } diff --git a/server/datastore/mysql/campaigns.go b/server/datastore/mysql/campaigns.go index df80a581cb..569f6535b8 100644 --- a/server/datastore/mysql/campaigns.go +++ b/server/datastore/mysql/campaigns.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) func (d *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) (*kolide.DistributedQueryCampaign, error) { @@ -20,7 +20,7 @@ func (d *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCam ` result, err := d.db.Exec(sqlStatement, camp.QueryID, camp.Status, camp.UserID) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting distributed query campaign") } id, _ := result.LastInsertId() @@ -34,7 +34,7 @@ func (d *Datastore) DistributedQueryCampaign(id uint) (*kolide.DistributedQueryC ` campaign := &kolide.DistributedQueryCampaign{} if err := d.db.Get(campaign, sql, id); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting distributed query campaign") } return campaign, nil @@ -51,7 +51,7 @@ func (d *Datastore) SaveDistributedQueryCampaign(camp *kolide.DistributedQueryCa ` _, err := d.db.Exec(sqlStatement, camp.QueryID, camp.Status, camp.UserID, camp.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "updating distributed query campaign") } return nil @@ -64,7 +64,7 @@ func (d *Datastore) DistributedQueryCampaignTargetIDs(id uint) (hostIDs []uint, targets := []kolide.DistributedQueryCampaignTarget{} if err = d.db.Select(&targets, sqlStatement, id); err != nil { - return nil, nil, errors.DatabaseError(err) + return nil, nil, errors.Wrap(err, "selecting distributed campaign target") } hostIDs = []uint{} @@ -93,7 +93,7 @@ func (d *Datastore) NewDistributedQueryCampaignTarget(target *kolide.Distributed ` result, err := d.db.Exec(sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "insert distributed campaign target") } id, _ := result.LastInsertId() @@ -114,7 +114,7 @@ func (d *Datastore) NewDistributedQueryExecution(exec *kolide.DistributedQueryEx result, err := d.db.Exec(sqlStatement, exec.HostID, exec.DistributedQueryCampaignID, exec.Status, exec.Error, exec.ExecutionDuration) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "insert distributed campaign target") } id, _ := result.LastInsertId() @@ -135,12 +135,12 @@ func (d *Datastore) CleanupDistributedQueryCampaigns(now time.Time) (expired uin kolide.QueryWaiting, now.Add(-1*time.Minute), kolide.QueryRunning, now.Add(-24*time.Hour)) if err != nil { - return expired, deleted, errors.DatabaseError(err) + return expired, deleted, errors.Wrap(err, "updating distributed query campaign") } exp, err := result.RowsAffected() if err != nil { - return expired, deleted, errors.DatabaseError(err) + return expired, deleted, errors.Wrap(err, "rows effected updating distributed query campaign") } expired = uint(exp) @@ -154,12 +154,12 @@ func (d *Datastore) CleanupDistributedQueryCampaigns(now time.Time) (expired uin ` result, err = d.db.Exec(sqlStatement, kolide.QueryComplete) if err != nil { - return expired, deleted, errors.DatabaseError(err) + return expired, deleted, errors.Wrap(err, "deleting distributed campaign executions") } del, err := result.RowsAffected() if err != nil { - return expired, deleted, errors.DatabaseError(err) + return expired, deleted, errors.Wrap(err, "rows effected deleting distributed campaign") } deleted = uint(del) diff --git a/server/datastore/mysql/decorators.go b/server/datastore/mysql/decorators.go new file mode 100644 index 0000000000..17649509f3 --- /dev/null +++ b/server/datastore/mysql/decorators.go @@ -0,0 +1,71 @@ +package mysql + +import ( + "database/sql" + + "github.com/pkg/errors" + + "github.com/kolide/kolide-ose/server/kolide" +) + +func (ds *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) { + sqlStatement := + "INSERT INTO decorators (" + + "`query`," + + "`type`," + + "`interval` ) " + + "VALUES (?, ?, ?)" + result, err := ds.db.Exec(sqlStatement, decorator.Query, decorator.Type, decorator.Interval) + if err != nil { + return nil, errors.Wrap(err, "creating decorator") + } + id, _ := result.LastInsertId() + decorator.ID = uint(id) + return decorator, nil +} + +func (ds *Datastore) DeleteDecorator(id uint) error { + sqlStatement := ` + DELETE FROM decorators + WHERE id = ? + ` + res, err := ds.db.Exec(sqlStatement, id) + if err != nil { + return errors.Wrap(err, "deleting decorator") + } + deleted, _ := res.RowsAffected() + if deleted < 1 { + return notFound("Decorator").WithID(id) + } + return nil +} + +func (ds *Datastore) Decorator(id uint) (*kolide.Decorator, error) { + sqlStatement := ` + SELECT * + FROM decorators + WHERE id = ? + ` + var result kolide.Decorator + err := ds.db.Get(&result, sqlStatement, id) + if err != nil { + if err == sql.ErrNoRows { + return nil, notFound("Decorator").WithID(id) + } + return nil, errors.Wrap(err, "retrieving decorator") + } + return &result, nil +} + +func (ds *Datastore) ListDecorators() ([]*kolide.Decorator, error) { + sqlStatement := ` + SELECT * + FROM decorators + ` + var results []*kolide.Decorator + err := ds.db.Select(&results, sqlStatement) + if err != nil { + return nil, errors.Wrap(err, "listing decorators") + } + return results, nil +} diff --git a/server/datastore/mysql/file_integrity_monitoring.go b/server/datastore/mysql/file_integrity_monitoring.go new file mode 100644 index 0000000000..4fcdb1bd45 --- /dev/null +++ b/server/datastore/mysql/file_integrity_monitoring.go @@ -0,0 +1,78 @@ +package mysql + +import ( + "database/sql" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" +) + +func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (result *kolide.FIMSection, err error) { + txn, err := d.db.Begin() + if err != nil { + return nil, errors.Wrap(err, "update options begin transaction") + } + var success bool + defer func() { + if success { + if err = txn.Commit(); err == nil { + return + } + } + txn.Rollback() + }() + + sqlStatement := ` + INSERT INTO file_integrity_monitorings ( + section_name, + description + ) VALUES( ?, ?) + ` + var resp sql.Result + resp, err = txn.Exec(sqlStatement, fp.SectionName, fp.Description) + if err != nil { + return nil, errors.Wrap(err, "creating fim section") + } + id, _ := resp.LastInsertId() + fp.ID = uint(id) + sqlStatement = ` + INSERT INTO file_integrity_monitoring_files ( + file, + file_integrity_monitoring_id + ) VALUES( ?, ? ) + ` + for _, fileName := range fp.Paths { + _, err = txn.Exec(sqlStatement, fileName, fp.ID) + if err != nil { + return nil, errors.Wrap(err, "adding path to fim section") + } + } + success = true + return fp, nil +} + +func (d *Datastore) FIMSections() (kolide.FIMSections, error) { + sqlStatement := ` + SELECT fim.section_name, mf.file FROM + file_integrity_monitorings AS fim + INNER JOIN file_integrity_monitoring_files AS mf + ON (fim.id = mf.file_integrity_monitoring_id) + ` + rows, err := d.db.Query(sqlStatement) + if err != nil { + if err == sql.ErrNoRows { + return nil, notFound("FilePath") + } + return nil, errors.Wrap(err, "retrieving fim sections") + } + result := make(kolide.FIMSections) + for rows.Next() { + var sectionName, fileName string + err = rows.Scan(§ionName, &fileName) + if err != nil { + return nil, errors.Wrap(err, "retrieving path for fim section") + } + result[sectionName] = append(result[sectionName], fileName) + } + return result, nil +} diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index b12d628f3d..994f874b7b 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -5,8 +5,8 @@ import ( "time" "github.com/jmoiron/sqlx" - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) // NewLabel creates a new kolide.Label @@ -23,7 +23,7 @@ func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) { ` result, err := d.db.Exec(sql, label.Name, label.Description, label.Query, label.Platform, label.LabelType) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting label") } id, _ := result.LastInsertId() @@ -46,7 +46,7 @@ func (d *Datastore) Label(lid uint) (*kolide.Label, error) { label := &kolide.Label{} if err := d.db.Get(label, sql, lid); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting label") } return label, nil @@ -61,7 +61,7 @@ func (d *Datastore) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) labels := []*kolide.Label{} if err := d.db.Select(&labels, sql); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting labels") } return labels, nil @@ -84,7 +84,7 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma ` rows, err := d.db.Query(sqlStatment, host.Platform, host.ID, cutoff) if err != nil && err != sql.ErrNoRows { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting label queries for host") } defer rows.Close() @@ -94,7 +94,7 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma var id, query string if err = rows.Scan(&id, &query); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "scanning label queries for host") } results[id] = query @@ -129,7 +129,7 @@ func (d *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[st _, err := d.db.Exec(sqlStatement, vals...) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "inserting label query execution") } return nil @@ -148,7 +148,7 @@ func (d *Datastore) ListLabelsForHost(hid uint) ([]kolide.Label, error) { labels := []kolide.Label{} err := d.db.Select(&labels, sqlStatement, hid) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting host labels") } return labels, nil @@ -169,7 +169,7 @@ func (d *Datastore) ListHostsInLabel(lid uint) ([]kolide.Host, error) { hosts := []kolide.Host{} err := d.db.Select(&hosts, sqlStatement, lid) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting label query executions") } return hosts, nil } @@ -191,14 +191,14 @@ func (d *Datastore) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error ` query, args, err := sqlx.In(sqlStatement, labels) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "building query listing unique hosts in labels") } query = d.db.Rebind(query) hosts := []kolide.Host{} err = d.db.Select(&hosts, query, args...) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "listing unique hosts in labels") } return hosts, nil @@ -229,7 +229,7 @@ func (d *Datastore) searchLabelsWithOmits(query string, omit ...uint) ([]kolide. sql, args, err := sqlx.In(sqlStatement, query, kolide.LabelTypeBuiltIn, omit) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "building query for labels with omits") } sql = d.db.Rebind(sql) @@ -237,7 +237,7 @@ func (d *Datastore) searchLabelsWithOmits(query string, omit ...uint) ([]kolide. matches := []kolide.Label{} err = d.db.Select(&matches, sql, args...) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting labels with omits") } return matches, nil @@ -272,7 +272,7 @@ func (d *Datastore) SearchLabels(query string, omit ...uint) ([]kolide.Label, er matches := []kolide.Label{} err := d.db.Select(&matches, sqlStatement, query, kolide.LabelTypeBuiltIn) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting labels for search") } return matches, nil diff --git a/server/datastore/mysql/migrations/tables/20170105151732_AddNameIndexToQueryTable.go b/server/datastore/mysql/migrations/tables/20170105151732_AddNameIndexToQueryTable.go new file mode 100644 index 0000000000..28e3d4a29d --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20170105151732_AddNameIndexToQueryTable.go @@ -0,0 +1,20 @@ +package tables + +import "database/sql" + +func init() { + MigrationClient.AddMigration(Up_20170105151732, Down_20170105151732) +} + +func Up_20170105151732(tx *sql.Tx) error { + sqlStatement := "CREATE UNIQUE INDEX idx_query_unique_name " + + " ON `queries` (`name` ASC);" + _, err := tx.Exec(sqlStatement) + return err +} + +func Down_20170105151732(tx *sql.Tx) error { + sqlStatement := "DROP INDEX idx_query_unique_name ON `queries`;" + _, err := tx.Exec(sqlStatement) + return err +} diff --git a/server/datastore/mysql/migrations/tables/20170108191242_CreateDecoratorTable.go b/server/datastore/mysql/migrations/tables/20170108191242_CreateDecoratorTable.go new file mode 100644 index 0000000000..fa585de2e4 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20170108191242_CreateDecoratorTable.go @@ -0,0 +1,27 @@ +package tables + +import "database/sql" + +func init() { + MigrationClient.AddMigration(Up_20170108191242, Down_20170108191242) +} + +func Up_20170108191242(tx *sql.Tx) error { + _, err := tx.Exec( + "CREATE TABLE `decorators` ( " + + "`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, " + + "`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " + + "`updated_at` TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, " + + "`query` VARCHAR(255) NOT NULL, " + + "`type` INT UNSIGNED NOT NULL, " + + "`interval` INT UNSIGNED NOT NULL, " + + "PRIMARY KEY (`id`) " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", + ) + return err +} + +func Down_20170108191242(tx *sql.Tx) error { + _, err := tx.Exec("DROP TABLE IF EXISTS decorators;") + return err +} diff --git a/server/datastore/mysql/migrations/tables/20170109094020_CreateFIMTables.go b/server/datastore/mysql/migrations/tables/20170109094020_CreateFIMTables.go new file mode 100644 index 0000000000..e3480dd987 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20170109094020_CreateFIMTables.go @@ -0,0 +1,51 @@ +package tables + +import "database/sql" + +func init() { + MigrationClient.AddMigration(Up_20170109094020, Down_20170109094020) +} + +func Up_20170109094020(tx *sql.Tx) error { + sqlStatement := + "CREATE TABLE `file_integrity_monitorings` ( " + + " `id` int(10) NOT NULL AUTO_INCREMENT, " + + " `section_name` varchar(255) NOT NULL DEFAULT '', " + + " `description` varchar(255) NOT NULL DEFAULT ''," + + " PRIMARY KEY (`id`)," + + " UNIQUE KEY `idx_unique_section_name` (`section_name`) USING BTREE" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;" + _, err := tx.Exec(sqlStatement) + if err != nil { + return err + } + sqlStatement = + "CREATE TABLE `file_integrity_monitoring_files` (" + + " `id` int(10) NOT NULL AUTO_INCREMENT," + + " `file` varchar(255) NOT NULL DEFAULT ''," + + " `file_integrity_monitoring_id` int(10) NOT NULL DEFAULT '0'," + + " PRIMARY KEY (`id`)," + + " UNIQUE KEY `idx_fim_unique_file_name` (`file`) USING BTREE," + + " KEY `fk_file_integrity_monitoring` (`file_integrity_monitoring_id`)," + + " CONSTRAINT `fk_file_integrity_monitoring` FOREIGN KEY (`file_integrity_monitoring_id`) REFERENCES `file_integrity_monitorings` (`id`) ON DELETE CASCADE" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;" + _, err = tx.Exec(sqlStatement) + if err != nil { + return err + } + return nil +} + +func Down_20170109094020(tx *sql.Tx) error { + sqlStatement := "DROP TABLE IF EXISTS `file_integrity_monitoring_files`; " + _, err := tx.Exec(sqlStatement) + if err != nil { + return err + } + sqlStatement = "DROP TABLE IF EXISTS `file_integrity_monitorings`;" + _, err = tx.Exec(sqlStatement) + if err != nil { + return err + } + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20170109130438_CreateYARATables.go b/server/datastore/mysql/migrations/tables/20170109130438_CreateYARATables.go new file mode 100644 index 0000000000..02fbf2b0d8 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20170109130438_CreateYARATables.go @@ -0,0 +1,66 @@ +package tables + +import "database/sql" + +func init() { + MigrationClient.AddMigration(Up_20170109130438, Down_20170109130438) +} + +func Up_20170109130438(tx *sql.Tx) error { + sqlStatement := + "CREATE TABLE `yara_signatures` ( " + + " `id` int(11) NOT NULL AUTO_INCREMENT, " + + " `signature_name` varchar(128) NOT NULL DEFAULT '', " + + " PRIMARY KEY (`id`), " + + " UNIQUE KEY `idx_yara_signatures_unique_name` (`signature_name`) USING BTREE " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8; " + if _, err := tx.Exec(sqlStatement); err != nil { + return err + } + + sqlStatement = + "CREATE TABLE `yara_file_paths` ( " + + " `file_integrity_monitoring_id` int(11) NOT NULL DEFAULT '0', " + + " `yara_signature_id` int(11) NOT NULL DEFAULT '0', " + + " PRIMARY KEY (`file_integrity_monitoring_id`,`yara_signature_id`), " + + " KEY `fk_yara_signature_id` (`yara_signature_id`), " + + " CONSTRAINT `fk_file_integrity_monitoring_id` FOREIGN KEY (`file_integrity_monitoring_id`) REFERENCES `file_integrity_monitorings` (`id`), " + + " CONSTRAINT `fk_yara_signature_id` FOREIGN KEY (`yara_signature_id`) REFERENCES `yara_signatures` (`id`) " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8; " + if _, err := tx.Exec(sqlStatement); err != nil { + return err + } + sqlStatement = + "CREATE TABLE `yara_signature_paths` ( " + + " `id` int(11) NOT NULL AUTO_INCREMENT, " + + " `file_path` varchar(255) NOT NULL DEFAULT '', " + + " `yara_signature_id` int(11) NOT NULL DEFAULT '0', " + + " PRIMARY KEY (`id`), " + + " KEY `fk_yara_signature` (`yara_signature_id`), " + + " CONSTRAINT `fk_yara_signature` FOREIGN KEY (`yara_signature_id`) REFERENCES `yara_signatures` (`id`) ON DELETE CASCADE " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8; " + if _, err := tx.Exec(sqlStatement); err != nil { + return err + } + + return nil +} + +func Down_20170109130438(tx *sql.Tx) error { + sqlStatement := "DROP TABLE IF EXISTS `yara_signature_paths`;" + _, err := tx.Exec(sqlStatement) + if err != nil { + return err + } + sqlStatement = "DROP TABLE IF EXISTS `yara_file_paths`;" + _, err = tx.Exec(sqlStatement) + if err != nil { + return err + } + sqlStatement = "DROP TABLE IF EXISTS `yara_signatures`;" + _, err = tx.Exec(sqlStatement) + if err != nil { + return err + } + return nil +} diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index b1f64dff69..2c5cc35d0e 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -8,6 +8,24 @@ import ( "github.com/pkg/errors" ) +func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) { + sqlStatement := ` + SELECT * + FROM packs + WHERE name = ? AND NOT deleted + ` + var pack kolide.Pack + err := d.db.Get(&pack, sqlStatement, name) + if err != nil { + if err == sql.ErrNoRows { + return nil, false, nil + } + return nil, false, errors.Wrap(err, "fetching packs by name") + } + + return &pack, true, nil +} + // NewPack creates a new Pack func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { @@ -192,6 +210,7 @@ func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide ) WHERE pt.pack_id = ? ` + hosts := []*kolide.Host{} if err := d.db.Select(&hosts, appendListOptionsToSQL(query, opt), kolide.TargetLabel, kolide.TargetHost, pid); err != nil && err != sql.ErrNoRows { return nil, errors.Wrap(err, "listing hosts in pack") diff --git a/server/datastore/mysql/password_reset.go b/server/datastore/mysql/password_reset.go index 81fb10f64c..2bfbb9546b 100644 --- a/server/datastore/mysql/password_reset.go +++ b/server/datastore/mysql/password_reset.go @@ -1,8 +1,8 @@ package mysql import ( - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) func (d *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) { @@ -13,7 +13,7 @@ func (d *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (* ` response, err := d.db.Exec(sqlStatement, req.UserID, req.Token) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting password reset requests") } id, _ := response.LastInsertId() @@ -32,7 +32,7 @@ func (d *Datastore) SavePasswordResetRequest(req *kolide.PasswordResetRequest) e ` _, err := d.db.Exec(sqlStatement, req.ExpiresAt, req.UserID, req.Token, req.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "updating password reset requests") } return nil @@ -45,7 +45,7 @@ func (d *Datastore) DeletePasswordResetRequest(req *kolide.PasswordResetRequest) ` _, err := d.db.Exec(sqlStatement, req.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "deleting from password reset request") } return nil @@ -57,7 +57,7 @@ func (d *Datastore) DeletePasswordResetRequestsForUser(userID uint) error { ` _, err := d.db.Exec(sqlStatement, userID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "deleting password reset request by user") } return nil @@ -71,7 +71,7 @@ func (d *Datastore) FindPassswordResetByID(id uint) (*kolide.PasswordResetReques passwordResetRequest := &kolide.PasswordResetRequest{} err := d.db.Get(&passwordResetRequest, sqlStatement, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting password reset by id") } return passwordResetRequest, nil @@ -86,7 +86,7 @@ func (d *Datastore) FindPassswordResetsByUserID(id uint) ([]*kolide.PasswordRese passwordResetRequests := []*kolide.PasswordResetRequest{} err := d.db.Select(&passwordResetRequests, sqlStatement, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "finding password resets by user id") } return passwordResetRequests, nil @@ -101,7 +101,7 @@ func (d *Datastore) FindPassswordResetByToken(token string) (*kolide.PasswordRes passwordResetRequest := &kolide.PasswordResetRequest{} err := d.db.Get(passwordResetRequest, sqlStatement, token) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting password reset requests") } return passwordResetRequest, nil @@ -117,7 +117,7 @@ func (d *Datastore) FindPassswordResetByTokenAndUserID(token string, id uint) (* passwordResetRequest := &kolide.PasswordResetRequest{} err := d.db.Get(passwordResetRequest, sqlStatement, id, token) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting password reset by token and user id") } return passwordResetRequest, nil diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index f6367c8061..c84082ed59 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -1,14 +1,32 @@ package mysql import ( + "database/sql" + "github.com/jmoiron/sqlx" - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) +func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) { + sqlStatement := ` + SELECT * + FROM queries + WHERE name = ? AND NOT deleted + ` + var query kolide.Query + err := d.db.Get(&query, sqlStatement, name) + if err != nil { + if err == sql.ErrNoRows { + return nil, false, nil + } + return nil, false, errors.Wrap(err, "selecting query by name") + } + return &query, true, nil +} + // NewQuery creates a Query func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { - sql := ` INSERT INTO queries ( name, @@ -20,7 +38,7 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { ` result, err := d.db.Exec(sql, query.Name, query.Description, query.Query, query.Saved, query.AuthorID) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting new query") } id, _ := result.LastInsertId() @@ -38,7 +56,7 @@ func (d *Datastore) SaveQuery(q *kolide.Query) error { ` _, err := d.db.Exec(sql, q.Name, q.Description, q.Query, q.AuthorID, q.Saved, q.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "updating query") } return nil @@ -59,17 +77,17 @@ func (d *Datastore) DeleteQueries(ids []uint) (uint, error) { ` query, args, err := sqlx.In(sql, ids) if err != nil { - return 0, errors.DatabaseError(err) + return 0, errors.Wrap(err, "building delete query query") } result, err := d.db.Exec(query, args...) if err != nil { - return 0, errors.DatabaseError(err) + return 0, errors.Wrap(err, "updating delete query") } deleted, err := result.RowsAffected() if err != nil { - return 0, errors.DatabaseError(err) + return 0, errors.Wrap(err, "fetching delete query rows effected") } return uint(deleted), nil @@ -88,11 +106,11 @@ func (d *Datastore) Query(id uint) (*kolide.Query, error) { ` query := &kolide.Query{} if err := d.db.Get(query, sql, id); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting query") } if err := d.loadPacksForQueries([]*kolide.Query{query}); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "loading packs for queries") } return query, nil @@ -113,11 +131,11 @@ func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) results := []*kolide.Query{} if err := d.db.Select(&results, sql); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "listing queries") } if err := d.loadPacksForQueries(results); err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "loading packs for queries") } return results, nil @@ -150,7 +168,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error { query, args, err := sqlx.In(sql, ids) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "building query in load packs for queries") } rows := []struct { @@ -160,7 +178,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error { err = d.db.Select(&rows, query, args...) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "selecting load packs for queries") } for _, row := range rows { diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index e1363762f7..02376ee88c 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -1,8 +1,8 @@ package mysql import ( - "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" ) func (d *Datastore) SessionByKey(key string) (*kolide.Session, error) { @@ -13,7 +13,7 @@ func (d *Datastore) SessionByKey(key string) (*kolide.Session, error) { session := &kolide.Session{} err := d.db.Get(session, sqlStatement, key) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting sessions") } return session, nil @@ -28,7 +28,7 @@ func (d *Datastore) SessionByID(id uint) (*kolide.Session, error) { session := &kolide.Session{} err := d.db.Get(session, sqlStatement, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting session by id") } return session, nil @@ -42,7 +42,7 @@ func (d *Datastore) ListSessionsForUser(id uint) ([]*kolide.Session, error) { sessions := []*kolide.Session{} err := d.db.Select(&sessions, sqlStatement, id) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "selecting sessions for user") } return sessions, nil @@ -59,7 +59,7 @@ func (d *Datastore) NewSession(session *kolide.Session) (*kolide.Session, error) ` result, err := d.db.Exec(sqlStatement, session.UserID, session.Key) if err != nil { - return nil, errors.DatabaseError(err) + return nil, errors.Wrap(err, "inserting session") } id, _ := result.LastInsertId() @@ -73,7 +73,7 @@ func (d *Datastore) DestroySession(session *kolide.Session) error { ` _, err := d.db.Exec(sqlStatement, session.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "deleting session") } return nil @@ -85,7 +85,7 @@ func (d *Datastore) DestroyAllSessionsForUser(id uint) error { ` _, err := d.db.Exec(sqlStatement, id) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "deleting sessions for user") } return nil @@ -99,7 +99,7 @@ func (d *Datastore) MarkSessionAccessed(session *kolide.Session) error { ` _, err := d.db.Exec(sqlStatement, d.clock.Now(), session.ID) if err != nil { - return errors.DatabaseError(err) + return errors.Wrap(err, "updating mark session as accessed") } return nil diff --git a/server/datastore/mysql/yara.go b/server/datastore/mysql/yara.go new file mode 100644 index 0000000000..2fa39dc61d --- /dev/null +++ b/server/datastore/mysql/yara.go @@ -0,0 +1,126 @@ +package mysql + +import ( + "database/sql" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/pkg/errors" +) + +func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (sg *kolide.YARASignatureGroup, err error) { + var success bool + txn, err := d.db.Begin() + if err != nil { + return nil, errors.Wrap(err, "new yara signature group begin transaction") + } + defer func() { + if success { + if err = txn.Commit(); err == nil { + return + } + } + txn.Rollback() + }() + sqlStatement := ` + INSERT INTO yara_signatures ( + signature_name + ) VALUES( ? ) + ` + var result sql.Result + result, err = txn.Exec(sqlStatement, ysg.SignatureName) + if err != nil { + return nil, errors.Wrap(err, "inserting new yara signature group") + } + id, _ := result.LastInsertId() + ysg.ID = uint(id) + sqlStatement = ` + INSERT INTO yara_signature_paths ( + file_path, + yara_signature_id + ) VALUES( ?, ? ) + ` + + for _, path := range ysg.Paths { + _, err = txn.Exec(sqlStatement, path, ysg.ID) + if err != nil { + return nil, errors.Wrap(err, "inserting new signature path") + } + } + success = true + return ysg, nil +} + +func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error { + sqlStatement := ` + INSERT INTO yara_file_paths ( + file_integrity_monitoring_id, + yara_signature_id + ) VALUES ( + ( + SELECT fim.id + FROM file_integrity_monitorings AS fim + WHERE fim.section_name = ? + LIMIT 1 + ), + ( + SELECT ys.id AS ys + FROM yara_signatures AS ys + WHERE ys.signature_name = ? + LIMIT 1 + ) + ) + ` + _, err := d.db.Exec(sqlStatement, fileSectionName, sigGroupName) + if err != nil { + return errors.Wrap(err, "inserting yara file path") + } + return nil +} + +func (d *Datastore) YARASection() (*kolide.YARASection, error) { + result := &kolide.YARASection{ + Signatures: make(map[string][]string), + FilePaths: make(map[string][]string), + } + sqlStatement := ` + SELECT s.signature_name, p.file_path + FROM yara_signatures AS s + INNER JOIN yara_signature_paths AS p + ON ( s.id = p.yara_signature_id ) + ` + rows, err := d.db.Query(sqlStatement) + if err != nil { + return nil, errors.Wrap(err, "selecting yara information") + } + for rows.Next() { + var sigName, sigPath string + err = rows.Scan(&sigName, &sigPath) + if err != nil { + return nil, errors.Wrap(err, "scanning yara information") + } + result.Signatures[sigName] = append(result.Signatures[sigName], sigPath) + } + + sqlStatement = ` + SELECT f.section_name, y.signature_name + FROM file_integrity_monitorings AS f + INNER JOIN yara_file_paths AS yfp + ON (f.id = yfp.file_integrity_monitoring_id) + INNER JOIN yara_signatures AS y + ON (y.id = yfp.yara_signature_id ) + ` + rows, err = d.db.Query(sqlStatement) + if err != nil { + return nil, errors.Wrap(err, "selecting yara signatures") + } + for rows.Next() { + var sectionName, signatureName string + err = rows.Scan(§ionName, &signatureName) + if err != nil { + return nil, errors.Wrap(err, "scanning yara signature values") + } + result.FilePaths[sectionName] = append(result.FilePaths[sectionName], signatureName) + } + + return result, nil +} diff --git a/server/datastore/mysql_test.go b/server/datastore/mysql_test.go index a6e69a4def..83928ece0e 100644 --- a/server/datastore/mysql_test.go +++ b/server/datastore/mysql_test.go @@ -49,7 +49,6 @@ func TestMySQL(t *testing.T) { t.Run(functionName(f), func(t *testing.T) { defer func() { require.Nil(t, ds.Drop()) }() require.Nil(t, ds.MigrateTables()) - f(t, ds) }) } diff --git a/server/errors/errors.go b/server/errors/errors.go index f9e3d3bf03..8c424a4421 100644 --- a/server/errors/errors.go +++ b/server/errors/errors.go @@ -53,11 +53,6 @@ func NewFromError(err error, status int, publicMessage string) *KolideError { } } -// Wrap a DB error with the extra KolideError decorations -func DatabaseError(err error) *KolideError { - return NewFromError(err, http.StatusInternalServerError, "Database error: "+err.Error()) -} - // Wrap a server error with the extra KolideError decorations func InternalServerError(err error) *KolideError { return NewFromError(err, http.StatusInternalServerError, "Internal server error") diff --git a/server/errors/errors_test.go b/server/errors/errors_test.go index 2c7fd4f253..43e2907696 100644 --- a/server/errors/errors_test.go +++ b/server/errors/errors_test.go @@ -47,19 +47,6 @@ func TestNewFromError(t *testing.T) { assert.Equal(t, expect, kolideErr) } -func TestDatabaseError(t *testing.T) { - err := errors.New("Foo error") - kolideErr := DatabaseError(err) - - expect := &KolideError{ - Err: err, - StatusCode: http.StatusInternalServerError, - PublicMessage: "Database error: " + err.Error(), - PrivateMessage: "Foo error", - } - assert.Equal(t, expect, kolideErr) -} - // These types and functions for performing an unordered comparison on a // []map[string]string] as parsed from the error JSON type errorField map[string]string diff --git a/server/kolide/datastore.go b/server/kolide/datastore.go index bee7ce7dbf..0e195f31f7 100644 --- a/server/kolide/datastore.go +++ b/server/kolide/datastore.go @@ -14,6 +14,9 @@ type Datastore interface { InviteStore ScheduledQueryStore OptionStore + DecoratorStore + FileIntegrityMonitoringStore + YARAStore Name() string Drop() error // MigrateTables creates and migrates the table schemas diff --git a/server/kolide/decorators.go b/server/kolide/decorators.go new file mode 100644 index 0000000000..8fb1317adb --- /dev/null +++ b/server/kolide/decorators.go @@ -0,0 +1,34 @@ +package kolide + +// DecoratorStore methods to manipulate decorator queries. +// See https://osquery.readthedocs.io/en/stable/deployment/configuration/ +type DecoratorStore interface { + // NewDecorator creates a decorator query. + NewDecorator(decorator *Decorator) (*Decorator, error) + // DeleteDecorator removes a decorator query. + DeleteDecorator(id uint) error + // Decorator retrieves a decorator query with supplied ID. + Decorator(id uint) (*Decorator, error) + // ListDecorators returns all decorator queries. + ListDecorators() ([]*Decorator, error) +} + +// DecoratorType refers to the allowable types of decorator queries. +// See https://osquery.readthedocs.io/en/stable/deployment/configuration/ +type DecoratorType int + +const ( + DecoratorLoad DecoratorType = iota + DecoratorAlways + DecoratorInterval +) + +// Decorator contains information about a decorator query. +type Decorator struct { + UpdateCreateTimestamps + ID uint + Type DecoratorType + // Interval note this is only pertainent for DecoratorInterval type. + Interval uint + Query string +} diff --git a/server/kolide/file_integrity_monitoring.go b/server/kolide/file_integrity_monitoring.go new file mode 100644 index 0000000000..64ad56e7e3 --- /dev/null +++ b/server/kolide/file_integrity_monitoring.go @@ -0,0 +1,20 @@ +package kolide + +type FIMSections map[string][]string + +type FileIntegrityMonitoringStore interface { + // NewFIMSection creates a named group of file paths + NewFIMSection(path *FIMSection) (*FIMSection, error) + // FIMSections returns all named file sections + FIMSections() (FIMSections, error) +} + +// FilePath maps a name to a group of files for the osquery file_paths +// section. +// See https://osquery.readthedocs.io/en/stable/deployment/configuration/ +type FIMSection struct { + ID uint + SectionName string `db:"section_name"` + Description string + Paths []string `db:"-"` +} diff --git a/server/kolide/import_config.go b/server/kolide/import_config.go new file mode 100644 index 0000000000..3b551cb8c0 --- /dev/null +++ b/server/kolide/import_config.go @@ -0,0 +1,218 @@ +package kolide + +import ( + "errors" + "fmt" + "strings" + + "golang.org/x/net/context" +) + +type ImportConfigService interface { + // ImportConfig create packs, queries, options etc based on imported + // osquery configuration. + ImportConfig(ctx context.Context, cfg *ImportConfig) (*ImportConfigResponse, error) +} + +// ImportSection is used to categorize information associated with the import +// of a particular section of an imported osquery configuration file. +type ImportSection string + +const ( + OptionsSection ImportSection = "options" + PacksSection = "packs" + QueriesSection = "queries" + DecoratorsSection = "decorators" + FilePathsSection = "file_paths" + YARASigSection = "yara_signature_group" + YARAFileSection = "yara_file_group" +) + +// WarningType is used to group associated warnings for options, packs etc +// when importing on osquery configuration file. +type WarningType string + +const ( + PackDuplicate WarningType = "duplicate_pack" + DifferentQuerySameName = "different_query_same_name" + OptionAlreadySet = "option_already_set" + OptionReadonly = "option_readonly" + OptionUnknown = "option_unknown" + QueryDuplicate = "duplicate_query" + Unsupported = "unsupported" +) + +// ImportStatus contains information pertaining to the import of a section +// of an osquery configuration file. +type ImportStatus struct { + // Title human readable name of the section of the import file that this + // status pertains to. + Title string `json:"title"` + // ImportCount count of items successfully imported. + ImportCount int `json:"import_count"` + // SkipCount count of items that are skipped. The reasons for the omissions + // can be found in Warnings. + SkipCount int `json:"skip_count"` + // Warnings groups catagories of warnings with one or more detail messages. + Warnings map[WarningType][]string `json:"warnings"` + // Messages contains an entry for each import attempt. + Messages []string `json:"messages"` +} + +// Warning is used to add a warning message to ImportStatus. +func (is *ImportStatus) Warning(warnType WarningType, fmtMsg string, fmtArgs ...interface{}) { + is.Warnings[warnType] = append(is.Warnings[warnType], fmt.Sprintf(fmtMsg, fmtArgs...)) +} + +// Message is used to add a general message to ImportStatus, usually indicating +// what was changed in a successful import. +func (is *ImportStatus) Message(fmtMsg string, args ...interface{}) { + is.Messages = append(is.Messages, fmt.Sprintf(fmtMsg, args...)) +} + +// ImportConfigResponse contains information about the import of an osquery +// configuration file. +type ImportConfigResponse struct { + ImportStatusBySection map[ImportSection]*ImportStatus `json:"import_status"` +} + +// Status returns a structure that contains information about the import +// of a particular section of an osquery configuration file. +func (ic *ImportConfigResponse) Status(section ImportSection) (status *ImportStatus) { + var ok bool + if status, ok = ic.ImportStatusBySection[section]; !ok { + status = new(ImportStatus) + status.Title = strings.Title(string(section)) + status.Warnings = make(map[WarningType][]string) + ic.ImportStatusBySection[section] = status + } + return status +} + +const ( + GlobPacks = "*" + // ImportPackName is a custom pack name used for a pack we create to + // hold imported scheduled queries. + ImportPackName = "imported" +) + +// QueryDetails represents the query objects used in the packs and the +// schedule section of an osquery configuration. +type QueryDetails struct { + Query string `json:"query"` + Interval uint `json:"interval"` + // Optional fields + Removed *bool `json:"removed"` + Platform *string `json:"platform"` + Version *string `json:"version"` + Shard *uint `json:"shard"` + Snapshot *bool `json:"snapshot"` +} + +// PackDetails represents the "packs" section of an osquery configuration +// file. +type PackDetails struct { + Queries QueryNameToQueryDetailsMap `json:"queries"` + Shard *uint `json:"shard"` + Version *string `json:"version"` + Platform string `json:"platform"` + Discovery []string `json:"discovery"` +} + +// YARAConfig yara configuration maps keys to lists of files. +// See https://osquery.readthedocs.io/en/stable/deployment/yara/ +type YARAConfig struct { + Signatures map[string][]string `json:"signatures"` + FilePaths map[string][]string `json:"file_paths"` +} + +// Decorator section of osquery config each section contains rows of decorator +// queries. +type DecoratorConfig struct { + Load []string `json:"load"` + Always []string `json:"always"` + /* + Interval maps a string representation of a numeric interval to a set + of decorator queries. + { + "interval": { + "3600": [ + "SELECT total_seconds FROM uptime;" + ] + } + } + */ + Interval map[string][]string `json:"interval"` +} + +type OptionNameToValueMap map[string]interface{} +type QueryNameToQueryDetailsMap map[string]QueryDetails +type PackNameMap map[string]interface{} +type FIMCategoryToPaths map[string][]string +type PackNameToPackDetails map[string]PackDetails + +// ImportConfig is a representation of an Osquery configuration. Osquery +// documentation has further details. +// See https://osquery.readthedocs.io/en/stable/deployment/configuration/ +type ImportConfig struct { + // Options is a map of option name to a value which can be an int, + // bool, or string. + Options OptionNameToValueMap `json:"options"` + // Schedule is a map of query names to details + Schedule QueryNameToQueryDetailsMap `json:"schedule"` + // Packs is a map of pack names to either PackDetails, or a string + // containing a file path with a pack config. If a string, we expect + // PackDetails to be stored in ExternalPacks. + Packs PackNameMap `json:"packs"` + // FileIntegrityMonitoring file integrity monitoring information. + // See https://osquery.readthedocs.io/en/stable/deployment/file-integrity-monitoring/ + FileIntegrityMonitoring FIMCategoryToPaths `json:"file_paths"` + // YARA configuration + YARA *YARAConfig `json:"yara"` + Decorators *DecoratorConfig `json:"decorators"` + // ExternalPacks are packs referenced when an item in the Packs map references + // an external file. The PackName here must match the PackName in the Packs map. + ExternalPacks PackNameToPackDetails `json:"-"` + // GlobPackNames lists pack names that are globbed. + GlobPackNames []string `json:"glob"` +} + +func (ic *ImportConfig) fetchGlobPacks(packs *PackNameToPackDetails) error { + for _, packName := range ic.GlobPackNames { + pack, ok := ic.ExternalPacks[packName] + if !ok { + return fmt.Errorf("glob pack '%s' details not found", packName) + } + (*packs)[packName] = pack + } + return nil +} + +// CollectPacks consolidates packs, globbed packs and external packs. +func (ic *ImportConfig) CollectPacks() (PackNameToPackDetails, error) { + result := make(PackNameToPackDetails) + for packName, packContent := range ic.Packs { + // special case handling for Globbed packs + if packName == GlobPacks { + if err := ic.fetchGlobPacks(&result); err != nil { + return nil, err + } + continue + } + // content can either be a file path, in which case we expect to find + // pack in ExternalPacks, or pack details + switch content := packContent.(type) { + case string: + pack, ok := ic.ExternalPacks[packName] + if !ok { + return nil, fmt.Errorf("external pack '%s' details not found", packName) + } + result[packName] = pack + case PackDetails: + result[packName] = content + default: + return nil, errors.New("unexpected pack content") + } + } + return result, nil +} diff --git a/server/kolide/import_config_test.go b/server/kolide/import_config_test.go new file mode 100644 index 0000000000..4e6bd8d90f --- /dev/null +++ b/server/kolide/import_config_test.go @@ -0,0 +1,78 @@ +package kolide + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPackNameMapUnmarshal(t *testing.T) { + pnm := PackNameMap{ + "path": "/this/is/a/path", + "details": PackDetails{ + Queries: QueryNameToQueryDetailsMap{ + "q1": QueryDetails{ + Query: "select from foo", + Interval: 100, + Removed: new(bool), + Platform: strptr("linux"), + Shard: new(uint), + Snapshot: new(bool), + }, + }, + Discovery: []string{ + "select from something", + }, + }, + } + b, _ := json.Marshal(pnm) + actual := make(PackNameMap) + err := json.Unmarshal(b, &actual) + require.Nil(t, err) + assert.Len(t, actual, 2) + + pnm = PackNameMap{ + "path": "/this/is/a/path", + "details": PackDetails{ + Queries: QueryNameToQueryDetailsMap{ + "q1": QueryDetails{ + Query: "select from foo", + Interval: 100, + Removed: new(bool), + Platform: strptr("linux"), + Shard: new(uint), + Snapshot: new(bool), + }, + }, + Shard: uintptr(float64(10)), + Version: strptr("1.0"), + Platform: "linux", + Discovery: []string{ + "select from something", + }, + }, + "details2": PackDetails{ + Queries: QueryNameToQueryDetailsMap{ + "q1": QueryDetails{ + Query: "select from bar", + Interval: 100, + Removed: new(bool), + Platform: strptr("linux"), + Shard: new(uint), + Snapshot: new(bool), + }, + }, + Shard: uintptr(float64(10)), + Version: strptr("1.0"), + Platform: "linux", + }, + } + + b, _ = json.Marshal(pnm) + actual = make(PackNameMap) + err = json.Unmarshal(b, &actual) + require.Nil(t, err) + assert.Len(t, actual, 3) +} diff --git a/server/kolide/import_config_unmarshaler.go b/server/kolide/import_config_unmarshaler.go new file mode 100644 index 0000000000..c0c9488e73 --- /dev/null +++ b/server/kolide/import_config_unmarshaler.go @@ -0,0 +1,102 @@ +package kolide + +import ( + "encoding/json" + "errors" +) + +// UnmarshalJSON custom unmarshaling for PackNameMap will determine whether +// the pack section of an osquery config file refers to a file path, or +// pack details. Pack details are unmarshalled into into PackDetails structure +// as oppossed to nested map[string]interface{} +func (pnm PackNameMap) UnmarshalJSON(b []byte) error { + var temp map[string]interface{} + err := json.Unmarshal(b, &temp) + if err != nil { + return err + } + for key, val := range temp { + switch t := val.(type) { + case string: + pnm[key] = t + case map[string]interface{}: + pnm[key] = unmarshalPackDetails(t) + default: + return errors.New("can't unmarshal json") + } + } + return nil +} + +func strptr(v interface{}) *string { + if v == nil { + return nil + } + s := new(string) + *s = v.(string) + return s +} + +func boolptr(v interface{}) *bool { + if v == nil { + return nil + } + b := new(bool) + *b = v.(bool) + return b +} + +func uintptr(v interface{}) *uint { + if v == nil { + return nil + } + i := new(uint) + *i = uint(v.(float64)) + return i +} + +func unmarshalPackDetails(v map[string]interface{}) PackDetails { + return PackDetails{ + Queries: unmarshalQueryDetails(v["queries"]), + Shard: uintptr(v["shard"]), + Version: strptr(v["version"]), + Platform: v["platform"].(string), + Discovery: unmarshalDiscovery(v["discovery"]), + } +} + +func unmarshalDiscovery(val interface{}) []string { + var result []string + if val == nil { + return result + } + v := val.([]interface{}) + for _, val := range v { + result = append(result, val.(string)) + } + return result +} + +func unmarshalQueryDetails(v interface{}) QueryNameToQueryDetailsMap { + result := make(QueryNameToQueryDetailsMap) + if v == nil { + return result + } + for qn, details := range v.(map[string]interface{}) { + result[qn] = unmarshalQueryDetail(details) + } + return result +} + +func unmarshalQueryDetail(val interface{}) QueryDetails { + v := val.(map[string]interface{}) + return QueryDetails{ + Query: v["query"].(string), + Interval: uint(v["interval"].(float64)), + Removed: boolptr(v["removed"]), + Platform: strptr(v["platform"]), + Version: strptr(v["version"]), + Shard: uintptr(v["shard"]), + Snapshot: boolptr(v["snapshot"]), + } +} diff --git a/server/kolide/options.go b/server/kolide/options.go index 2668e1463f..46f936c60b 100644 --- a/server/kolide/options.go +++ b/server/kolide/options.go @@ -140,6 +140,19 @@ func (opt *Option) SetValue(v interface{}) { opt.Value.Val = v } +func (opt *Option) SameType(compare interface{}) bool { + switch compare.(type) { + case float64: + return opt.Type == OptionTypeInt + case string: + return opt.Type == OptionTypeString + case bool: + return opt.Type == OptionTypeBool + default: + return false + } +} + // OptionSet returns true if the option has a value assigned to it func (opt *Option) OptionSet() bool { return opt.Value.Val != nil diff --git a/server/kolide/packs.go b/server/kolide/packs.go index 9f25097cdc..9450e764eb 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -20,6 +20,9 @@ type PackStore interface { // ListPacks lists all packs in the datastore. ListPacks(opt ListOptions) ([]*Pack, error) + // PackByName fetches pack if it exists, if the pack + // exists the bool return value is true + PackByName(name string) (*Pack, bool, error) // AddLabelToPack adds an existing label to an existing pack, both by ID. AddLabelToPack(lid, pid uint) error diff --git a/server/kolide/queries.go b/server/kolide/queries.go index 7c7a798f26..cd3082a259 100644 --- a/server/kolide/queries.go +++ b/server/kolide/queries.go @@ -1,10 +1,6 @@ package kolide -import ( - "time" - - "golang.org/x/net/context" -) +import "golang.org/x/net/context" type QueryStore interface { // NewQuery creates a new query object in thie datastore. The returned @@ -24,6 +20,9 @@ type QueryStore interface { // ListQueries returns a list of queries with the provided sorting and // paging options. Associated packs should also be loaded. ListQueries(opt ListOptions) ([]*Query, error) + // QueryByName looks up a query by name, the second bool is true if a query + // by the name exists. + QueryByName(name string) (*Query, bool, error) } type QueryService interface { @@ -63,20 +62,3 @@ type Query struct { // table in the MySQL backend. Packs []Pack `json:"packs" db:"-"` } - -type DecoratorType int - -const ( - DecoratorLoad DecoratorType = iota - DecoratorAlways - DecoratorInterval -) - -type Decorator struct { - ID uint - CreatedAt time.Time - UpdatedAt time.Time - Type DecoratorType - Interval int - Query string -} diff --git a/server/kolide/service.go b/server/kolide/service.go index 7d2b16fd49..3313c7104f 100644 --- a/server/kolide/service.go +++ b/server/kolide/service.go @@ -15,4 +15,5 @@ type Service interface { TargetService ScheduledQueryService OptionService + ImportConfigService } diff --git a/server/kolide/yara.go b/server/kolide/yara.go new file mode 100644 index 0000000000..b8c259ae6b --- /dev/null +++ b/server/kolide/yara.go @@ -0,0 +1,31 @@ +package kolide + +// YARAFilePaths represents the files_path section of an osquery config. The +// key maps to file_paths section_name and maps to one or more YARA signature +// group names +type YARAFilePaths map[string][]string + +type YARAStore interface { + // NewYARASignatureGroup creates a new mapping of a name to + // a group of YARA signatures + NewYARASignatureGroup(*YARASignatureGroup) (*YARASignatureGroup, error) + // NewYARAFilePath maps a named set of files to one or more + // groups of YARA signatures + NewYARAFilePath(fileSectionName, sigGroupName string) error + // YARASection creates the osquery configuration YARA section + YARASection() (*YARASection, error) +} + +// YARASignatureGroup maps a name to a group of YARA Signatures +// See https://osquery.readthedocs.io/en/stable/deployment/yara/ +type YARASignatureGroup struct { + ID uint + SignatureName string `db:"signature_name"` + Paths []string `db:"-"` +} + +// YARASection represents the osquery config for YARA +type YARASection struct { + Signatures map[string][]string `json:"signatures"` + FilePaths map[string][]string `json:"file_paths"` +} diff --git a/server/mock/datastore.go b/server/mock/datastore.go index e1b3b52531..7610f4d3ad 100644 --- a/server/mock/datastore.go +++ b/server/mock/datastore.go @@ -18,6 +18,9 @@ type Store struct { kolide.QueryStore kolide.OptionStore kolide.ScheduledQueryStore + kolide.DecoratorStore + kolide.FileIntegrityMonitoringStore + kolide.YARAStore InviteStore UserStore diff --git a/server/service/endpoint_import_config.go b/server/service/endpoint_import_config.go new file mode 100644 index 0000000000..e1c25bb8bf --- /dev/null +++ b/server/service/endpoint_import_config.go @@ -0,0 +1,41 @@ +package service + +import ( + "github.com/go-kit/kit/endpoint" + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +type importRequest struct { + // Config contains a JSON osquery config supplied by the end user + Config string `json:"config"` + // ExternalPackConfigs contains a map of external Pack configs keyed by + // Pack name, this includes external packs referenced by the globbing + // feature. Not in the case of globbed packs, we expect the user to + // generate unique pack names since we don't know what they are, these + // names must be included in the GlobPackNames field so that we can + // validate that they've been accounted for. + ExternalPackConfigs map[string]string `json:"external_pack_configs"` + // GlobPackNames list of user generated names for external packs + // referenced by the glob feature, the JSON for the globbed packs + // is stored in ExternalPackConfigs keyed by the GlobPackName + GlobPackNames []string `json:"glob_pack_names"` +} + +type importResponse struct { + Response *kolide.ImportConfigResponse `json:"response,omitempty"` + Err error `json:"error,omitempty"` +} + +func (ir importResponse) error() error { return ir.Err } + +func makeImportConfigEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + config := request.(kolide.ImportConfig) + resp, err := svc.ImportConfig(ctx, &config) + if err != nil { + return importResponse{Err: err}, nil + } + return importResponse{Response: resp}, nil + } +} diff --git a/server/service/endpoint_import_config_test.go b/server/service/endpoint_import_config_test.go new file mode 100644 index 0000000000..12f83e047e --- /dev/null +++ b/server/service/endpoint_import_config_test.go @@ -0,0 +1,108 @@ +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testImportConfigWithGlob(t *testing.T, r *testResource) { + testJSON := ` +{ + "config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"glob\":[\"globpack\"],\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}", + "external_pack_configs": { + "external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}", + "globpack": "{\"discovery\":[\"select pid from processes where name = 'zip';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from other;\",\"interval\":1200,\"description\":\"Check other.\"}}}" + }, + "glob_pack_names": ["globpack"] +} +` + buff := bytes.NewBufferString(testJSON) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + var impResponse importResponse + err = json.NewDecoder(resp.Body).Decode(&impResponse) + require.Nil(t, err) + assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.PacksSection].ImportCount) +} + +func testImportConfigWithMissingGlob(t *testing.T, r *testResource) { + testJSON := ` + { + "config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}", + "external_pack_configs": { + "external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}" + } + } + ` + buff := bytes.NewBufferString(testJSON) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + var v mockValidationError + err = json.NewDecoder(resp.Body).Decode(&v) + require.Nil(t, err) + require.Len(t, v.Errors, 1) + assert.Equal(t, "missing glob packs", v.Errors[0].Reason) + +} + +func testImportConfig(t *testing.T, r *testResource) { + + testJSON := ` + { + "config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}", + "external_pack_configs": { + "external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}" + } + } + ` + buff := bytes.NewBufferString(testJSON) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + var impResponse importResponse + err = json.NewDecoder(resp.Body).Decode(&impResponse) + require.Nil(t, err) + assert.Equal(t, 2, impResponse.Response.ImportStatusBySection[kolide.YARASigSection].ImportCount) + assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.DecoratorsSection].ImportCount) +} + +func testImportConfigMissingExternal(t *testing.T, r *testResource) { + testJSON := ` + { + "config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3603\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}" + } + ` + buff := bytes.NewBufferString(testJSON) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + + var v mockValidationError + err = json.NewDecoder(resp.Body).Decode(&v) + require.Nil(t, err) + require.Len(t, v.Errors, 2) + assert.Equal(t, "missing content for 'external_pack'", v.Errors[0].Reason) + assert.Equal(t, "interval '3603' must be divisible by 60", v.Errors[1].Reason) + +} diff --git a/server/service/endpoint_test.go b/server/service/endpoint_test.go index 4f3e8176bb..0cf7c21553 100644 --- a/server/service/endpoint_test.go +++ b/server/service/endpoint_test.go @@ -97,6 +97,10 @@ var testFunctions = [...]func(*testing.T, *testResource){ testGetOptions, testModifyOptions, testModifyOptionsValidationFail, + testImportConfig, + testImportConfigMissingExternal, + testImportConfigWithMissingGlob, + testImportConfigWithGlob, } func TestEndpoints(t *testing.T) { diff --git a/server/service/handler.go b/server/service/handler.go index 5d8cea4a4f..1742dfe97d 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -70,6 +70,7 @@ type KolideEndpoints struct { SearchTargets endpoint.Endpoint GetOptions endpoint.Endpoint ModifyOptions endpoint.Endpoint + ImportConfig endpoint.Endpoint } // MakeKolideServerEndpoints creates the Kolide API endpoints. @@ -135,6 +136,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint SearchTargets: authenticatedUser(jwtKey, svc, makeSearchTargetsEndpoint(svc)), GetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeGetOptionsEndpoint(svc))), ModifyOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeModifyOptionsEndpoint(svc))), + ImportConfig: authenticatedUser(jwtKey, svc, makeImportConfigEndpoint(svc)), // Osquery endpoints EnrollAgent: makeEnrollAgentEndpoint(svc), @@ -201,6 +203,7 @@ type kolideHandlers struct { SearchTargets http.Handler GetOptions http.Handler ModifyOptions http.Handler + ImportConfig http.Handler } func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithttp.ServerOption) *kolideHandlers { @@ -263,6 +266,7 @@ func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithtt SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest), GetOptions: newServer(e.GetOptions, decodeNoParamsRequest), ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest), + ImportConfig: newServer(e.ImportConfig, decodeImportConfigRequest), } } @@ -363,6 +367,8 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) { r.Handle("/api/v1/kolide/targets", h.SearchTargets).Methods("POST").Name("search_targets") + r.Handle("/api/v1/kolide/osquery/config/import", h.ImportConfig).Methods("POST").Name("import_config") + r.Handle("/api/v1/osquery/enroll", h.EnrollAgent).Methods("POST").Name("enroll_agent") r.Handle("/api/v1/osquery/config", h.GetClientConfig).Methods("POST").Name("get_client_config") r.Handle("/api/v1/osquery/distributed/read", h.GetDistributedQueries).Methods("POST").Name("get_distributed_queries") diff --git a/server/service/service.go b/server/service/service.go index 0bca0bf0f7..4af61745e0 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -36,7 +36,7 @@ func NewService(ds kolide.Datastore, resultStore kolide.QueryResultStore, logger osqueryResultLogWriter: logFile(kolideConfig.Osquery.ResultLogFile), mailService: mailService, } - svc = validationMiddleware{svc} + svc = validationMiddleware{svc, ds} return svc, nil } diff --git a/server/service/service_import_config.go b/server/service/service_import_config.go new file mode 100644 index 0000000000..120acab787 --- /dev/null +++ b/server/service/service_import_config.go @@ -0,0 +1,421 @@ +package service + +import ( + "errors" + "strconv" + "strings" + + "github.com/kolide/kolide-ose/server/contexts/viewer" + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +func (svc service) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) { + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + vc, ok := viewer.FromContext(ctx) + if !ok { + return nil, errors.New("internal error, unable to fetch user") + } + if err := svc.importOptions(cfg.Options, resp); err != nil { + return nil, err + } + if err := svc.importPacks(vc.UserID(), cfg, resp); err != nil { + return nil, err + } + if err := svc.importScheduledQueries(vc.UserID(), cfg, resp); err != nil { + return nil, err + } + if err := svc.importDecorators(cfg, resp); err != nil { + return nil, err + } + if err := svc.importFIMSections(cfg, resp); err != nil { + return nil, err + } + return resp, nil +} + +func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error { + if cfg.YARA != nil { + for sig, paths := range cfg.YARA.Signatures { + ysg := &kolide.YARASignatureGroup{ + SignatureName: sig, + Paths: paths, + } + _, err := svc.ds.NewYARASignatureGroup(ysg) + if err != nil { + return err + } + resp.Status(kolide.YARASigSection).ImportCount++ + resp.Status(kolide.YARASigSection).Message("imported '%s'", sig) + } + for section, sigs := range cfg.YARA.FilePaths { + for _, sig := range sigs { + err := svc.ds.NewYARAFilePath(section, sig) + if err != nil { + return err + } + } + resp.Status(kolide.YARAFileSection).ImportCount++ + resp.Status(kolide.YARAFileSection).Message("imported '%s'", section) + } + } + return nil +} + +func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error { + if cfg.FileIntegrityMonitoring != nil { + for sectionName, paths := range cfg.FileIntegrityMonitoring { + fp := &kolide.FIMSection{ + SectionName: sectionName, + Description: "imported", + Paths: paths, + } + _, err := svc.ds.NewFIMSection(fp) + if err != nil { + return err + } + resp.Status(kolide.FilePathsSection).ImportCount++ + resp.Status(kolide.FilePathsSection).Message("imported '%s'", sectionName) + } + } + // this has to happen AFTER fim section, because it requires file paths + return svc.importYARA(cfg, resp) +} + +func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error { + if cfg.Decorators != nil { + for _, query := range cfg.Decorators.Load { + decorator := &kolide.Decorator{ + Query: query, + Type: kolide.DecoratorLoad, + } + _, err := svc.ds.NewDecorator(decorator) + if err != nil { + return err + } + resp.Status(kolide.DecoratorsSection).ImportCount++ + resp.Status(kolide.DecoratorsSection).Message("imported load '%s'", query) + } + for _, query := range cfg.Decorators.Always { + decorator := &kolide.Decorator{ + Query: query, + Type: kolide.DecoratorAlways, + } + _, err := svc.ds.NewDecorator(decorator) + if err != nil { + return err + } + resp.Status(kolide.DecoratorsSection).ImportCount++ + resp.Status(kolide.DecoratorsSection).Message("imported always '%s'", query) + } + for key, queries := range cfg.Decorators.Interval { + for _, query := range queries { + interval, err := strconv.ParseInt(key, 10, 32) + if err != nil { + return err + } + decorator := &kolide.Decorator{ + Query: query, + Type: kolide.DecoratorInterval, + Interval: uint(interval), + } + _, err = svc.ds.NewDecorator(decorator) + if err != nil { + return err + } + resp.Status(kolide.DecoratorsSection).ImportCount++ + resp.Status(kolide.DecoratorsSection).Message("imported interval %d '%s'", interval, query) + } + } + + } + return nil +} + +func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error { + _, ok, err := svc.ds.PackByName(kolide.ImportPackName) + if ok { + resp.Status(kolide.PacksSection).Warning( + kolide.PackDuplicate, "skipped '%s' already exists", kolide.ImportPackName, + ) + resp.Status(kolide.PacksSection).SkipCount++ + return nil + } + // create import pack to hold imported scheduled queries + pack := &kolide.Pack{ + Name: kolide.ImportPackName, + Description: "holds imported scheduled queries", + CreatedBy: uid, + Disabled: false, + } + pack, err = svc.ds.NewPack(pack) + if err != nil { + return err + } + resp.Status(kolide.PacksSection).ImportCount++ + resp.Status(kolide.PacksSection).Message("created import pack") + + for queryName, queryDetails := range cfg.Schedule { + var query *kolide.Query + query, ok, err = svc.ds.QueryByName(queryName) + // if we find the query check to see if the import query matches the + // query we have, if it doesn't skip it + if ok { + if hashQuery("", query.Query) != hashQuery("", queryDetails.Query) { + resp.Status(kolide.PacksSection).Warning( + kolide.DifferentQuerySameName, + "queries named '%s' have different statements and won't be added to '%s'", + queryName, + pack.Name, + ) + continue + } + resp.Status(kolide.QueriesSection).Warning( + kolide.QueryDuplicate, "skipped '%s' different query of same name already exists", queryName, + ) + resp.Status(kolide.QueriesSection).SkipCount++ + } else { + // if query doesn't exist, create it + query = &kolide.Query{ + Name: queryName, + Description: "imported", + Query: queryDetails.Query, + Saved: true, + AuthorID: uid, + } + query, err = svc.ds.NewQuery(query) + if err != nil { + return err + } + resp.Status(kolide.QueriesSection).ImportCount++ + resp.Status(kolide.QueriesSection).Message( + "imported scheduled query '%s'", query.Name, + ) + } + sq := &kolide.ScheduledQuery{ + PackID: pack.ID, + QueryID: query.ID, + Interval: queryDetails.Interval, + Snapshot: queryDetails.Snapshot, + Removed: queryDetails.Removed, + Platform: queryDetails.Platform, + Version: queryDetails.Version, + Shard: queryDetails.Shard, + } + _, err = svc.ds.NewScheduledQuery(sq) + if err != nil { + return nil + } + resp.Status(kolide.PacksSection).Message( + "added query '%s' to '%s'", query.Name, pack.Name, + ) + } + return nil +} + +func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error { + labelCache := map[string]*kolide.Label{} + packs, err := cfg.CollectPacks() + if err != nil { + return err + } + for packName, packDetails := range packs { + _, ok, err := svc.ds.PackByName(packName) + if err != nil { + return err + } + if ok { + resp.Status(kolide.PacksSection).Warning( + kolide.PackDuplicate, "skipped '%s' already exists", packName, + ) + resp.Status(kolide.PacksSection).SkipCount++ + continue + } + // import new pack + if packDetails.Shard != nil { + resp.Status(kolide.PacksSection).Warning( + kolide.Unsupported, + "shard for pack '%s'", + packName, + ) + } + if packDetails.Version != nil { + resp.Status(kolide.PacksSection).Warning( + kolide.Unsupported, + "version for pack '%s'", + packName, + ) + } + pack := &kolide.Pack{ + Name: packName, + Description: "Imported pack", + Platform: packDetails.Platform, + } + pack, err = svc.ds.NewPack(pack) + if err != nil { + return err + } + err = svc.createLabelsForPack(pack, &packDetails, labelCache, resp) + if err != nil { + return err + } + err = svc.createQueriesForPack(uid, pack, &packDetails, resp) + if err != nil { + return err + } + resp.Status(kolide.PacksSection).ImportCount++ + resp.Status(kolide.PacksSection).Message("imported '%s'", packName) + } + return nil +} + +func hashQuery(platform, query string) string { + s := strings.Replace(query, " ", "", -1) + s = strings.Replace(s, "\t", "", -1) + s = strings.Replace(s, "\n", "", -1) + s = strings.Trim(s, ";") + s = platform + s + return strings.ToLower(s) +} + +func uniqueImportName() (string, error) { + random, err := kolide.RandomText(12) + if err != nil { + return "", err + } + return "import_" + random, nil +} + +func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *kolide.PackDetails, + resp *kolide.ImportConfigResponse) error { + for queryName, queryDetails := range details.Queries { + query, ok, err := svc.ds.QueryByName(queryName) + if err != nil { + return err + } + // if the query isn't already in the database, create it + if !ok { + query = &kolide.Query{ + Name: queryName, + Description: "imported", + Query: queryDetails.Query, + Saved: true, + AuthorID: uid, + } + query, err = svc.ds.NewQuery(query) + if err != nil { + return err + } + resp.Status(kolide.QueriesSection).Message( + "created '%s' as part of pack '%s'", queryName, pack.Name, + ) + resp.Status(kolide.QueriesSection).ImportCount++ + } + // associate query with pack + scheduledQuery := &kolide.ScheduledQuery{ + PackID: pack.ID, + QueryID: query.ID, + Interval: queryDetails.Interval, + Platform: queryDetails.Platform, + Snapshot: queryDetails.Snapshot, + Removed: queryDetails.Removed, + Version: queryDetails.Version, + Shard: queryDetails.Shard, + } + _, err = svc.ds.NewScheduledQuery(scheduledQuery) + if err != nil { + return nil + } + resp.Status(kolide.PacksSection).Message("added query '%s'", query.Name) + + } + return nil +} + +// createLabelsForPack Iterates through discover queries, creates a label for +// each query and assigns it to the pack passed as an argument. Once a Label is created we cache +// it for reuse. +func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDetails, + cache map[string]*kolide.Label, resp *kolide.ImportConfigResponse) error { + for _, query := range details.Discovery { + hash := hashQuery(details.Platform, query) + label, ok := cache[hash] + // add existing label to pack + if ok { + err := svc.ds.AddLabelToPack(label.ID, pack.ID) + if err != nil { + return err + } + resp.Status(kolide.PacksSection).Message( + "added label '%s' to pack '%s'", label.Name, pack.Name, + ) + continue + } + // create new label and add it to pack + labelName, err := uniqueImportName() + if err != nil { + return err + } + label = &kolide.Label{ + Name: labelName, + Query: query, + Description: "imported", + LabelType: kolide.LabelTypeRegular, + Platform: details.Platform, + } + label, err = svc.ds.NewLabel(label) + if err != nil { + return err + } + // hang on to label so we can reuse it for other packs if needed + cache[hash] = label + err = svc.ds.AddLabelToPack(label.ID, pack.ID) + if err != nil { + return err + } + resp.Status(kolide.PacksSection).Message( + "added label '%s' to '%s'", label.Name, pack.Name, + ) + } + return nil +} + +func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.ImportConfigResponse) error { + var updateOptions []kolide.Option + for optName, optValue := range opts { + opt, err := svc.ds.OptionByName(optName) + if err != nil { + resp.Status(kolide.OptionsSection).Warning( + kolide.OptionUnknown, "skipped '%s' can't find option", optName, + ) + resp.Status(kolide.OptionsSection).SkipCount++ + continue + } + if opt.ReadOnly { + resp.Status(kolide.OptionsSection).Warning( + kolide.OptionReadonly, "skipped '%s' can't change read only option", optName, + ) + resp.Status(kolide.OptionsSection).SkipCount++ + continue + } + if opt.OptionSet() { + resp.Status(kolide.OptionsSection).Warning( + kolide.OptionAlreadySet, "skipped '%s' can't change option that is already set", optName, + ) + resp.Status(kolide.OptionsSection).SkipCount++ + continue + } + opt.SetValue(optValue) + resp.Status(kolide.OptionsSection).Message("set %s value to %v", optName, optValue) + resp.Status(kolide.OptionsSection).ImportCount++ + updateOptions = append(updateOptions, *opt) + } + if len(updateOptions) > 0 { + if err := svc.ds.SaveOptions(updateOptions); err != nil { + return err + } + } + return nil +} diff --git a/server/service/service_import_config_test.go b/server/service/service_import_config_test.go new file mode 100644 index 0000000000..6b9fdb6d4a --- /dev/null +++ b/server/service/service_import_config_test.go @@ -0,0 +1,340 @@ +package service + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/config" + "github.com/kolide/kolide-ose/server/datastore/inmem" + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createServiceMockForImport(t *testing.T) *service { + ds, err := inmem.New(config.TestConfig()) + require.Nil(t, err) + err = ds.MigrateData() + require.Nil(t, err) + return &service{ + ds: ds, + } +} + +func TestHashQuery(t *testing.T) { + q1 := `SELECT * FROM t1 INNER JOIN ON + t1.id = t2.t1id + WHERE t1.name = 'foo' ` + q2 := "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t1.name = 'foo';" + h1 := hashQuery("platform", q1) + h2 := hashQuery("platform", q2) + assert.Equal(t, h1, h2) + q2 = "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t2.name = 'foo';" + h2 = hashQuery("platform", q2) + assert.NotEqual(t, h1, h2) + +} +func TestImportFilePaths(t *testing.T) { + cfg := &kolide.ImportConfig{ + FileIntegrityMonitoring: kolide.FIMCategoryToPaths{ + "files1": []string{ + "path1", + "path2", + }, + "files2": []string{ + "path3", + }, + }, + YARA: &kolide.YARAConfig{ + Signatures: map[string][]string{ + "sig1": []string{ + "path4", + "path5", + }, + "sig2": []string{ + "path6", + }, + }, + FilePaths: map[string][]string{ + "files1": []string{ + "sig1", + "sig2", + }, + "files2": []string{ + "sig1", + }, + }, + }, + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + svc := createServiceMockForImport(t) + err := svc.importFIMSections(cfg, resp) + require.Nil(t, err) + assert.Equal(t, 2, resp.Status(kolide.FilePathsSection).ImportCount) + sections, err := svc.ds.FIMSections() + require.Nil(t, err) + assert.Len(t, sections, 2) + yara, err := svc.ds.YARASection() + require.Nil(t, err) + assert.Len(t, yara.Signatures, 2) + assert.Len(t, yara.FilePaths, 2) +} + +func TestImportDecorators(t *testing.T) { + cfg := &kolide.ImportConfig{ + Decorators: &kolide.DecoratorConfig{ + Load: []string{ + "select from foo", + "select from bar", + }, + Always: []string{ + "select from always", + }, + Interval: map[string][]string{ + "100": []string{ + "select from 100", + }, + "200": []string{ + "select from 200", + }, + }, + }, + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + svc := createServiceMockForImport(t) + err := svc.importDecorators(cfg, resp) + require.Nil(t, err) + assert.Equal(t, 5, resp.Status(kolide.DecoratorsSection).ImportCount) + dec, err := svc.ds.ListDecorators() + require.Nil(t, err) + assert.Len(t, dec, 5) +} + +func TestImportScheduledQueries(t *testing.T) { + cfg := &kolide.ImportConfig{ + Schedule: kolide.QueryNameToQueryDetailsMap{ + "q1": kolide.QueryDetails{ + Query: "select pid from processes", + Interval: 60, + Platform: stringPtr("linux"), + }, + "q2": kolide.QueryDetails{ + Query: "select uid from users", + Interval: 120, + Platform: stringPtr("linux"), + Version: stringPtr("1.0"), + }, + "q3": kolide.QueryDetails{ + Query: "select name from os", + Interval: 240, + Platform: stringPtr("linux"), + Snapshot: boolPtr(true), + }, + }, + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + svc := createServiceMockForImport(t) + user := &kolide.User{ + Username: "bob", + Password: []byte("secret"), + Email: "bob@something.com", + Admin: false, + AdminForcedPasswordReset: false, + } + user, err := svc.ds.NewUser(user) + require.Nil(t, err) + skipQuery := &kolide.Query{ + Name: "q3", + Query: "select version from os", + Description: "should be skipped", + Saved: true, + AuthorID: user.ID, + } + _, err = svc.ds.NewQuery(skipQuery) + require.Nil(t, err) + noskipQuery := &kolide.Query{ + Name: "q2", + Query: "select uid from users", + Saved: true, + AuthorID: user.ID, + } + _, err = svc.ds.NewQuery(noskipQuery) + require.Nil(t, err) + + err = svc.importScheduledQueries(user.ID, cfg, resp) + require.Nil(t, err) + _, ok, err := svc.ds.QueryByName("q1") + require.Nil(t, err) + require.True(t, ok) + _, ok, err = svc.ds.QueryByName("q2") + require.Nil(t, err) + require.True(t, ok) + _, ok, err = svc.ds.QueryByName("q3") + require.Nil(t, err) + require.True(t, ok) + +} + +func TestOptionsImportConfig(t *testing.T) { + opts := kolide.OptionNameToValueMap{ + "aws_access_key_id": "foo", + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + svc := createServiceMockForImport(t) + err := svc.importOptions(opts, resp) + require.Nil(t, err) + status := resp.Status(kolide.OptionsSection) + require.NotNil(t, status) + assert.Equal(t, 1, status.ImportCount) + opt, err := svc.ds.OptionByName("aws_access_key_id") + require.Nil(t, err) + assert.Equal(t, "foo", opt.GetValue()) + require.Len(t, status.Messages, 1) + assert.Equal(t, "set aws_access_key_id value to foo", status.Messages[0]) +} + +func TestOptionsImportConfigWithSkips(t *testing.T) { + opts := kolide.OptionNameToValueMap{ + "aws_access_key_id": "foo", + "aws_secret_access_key": "secret", + // this should be skipped because it's already set + "aws_firehose_period": 100, + // these should be skipped because it's read only + "disable_distributed": false, + "pack_delimiter": "x", + // this should be skipped because it's not an option we know about + "wombat": "not venomous", + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + svc := createServiceMockForImport(t) + // set option val, it should be skipped + opt, err := svc.ds.OptionByName("aws_firehose_period") + require.Nil(t, err) + opt.SetValue(23) + err = svc.ds.SaveOptions([]kolide.Option{*opt}) + require.Nil(t, err) + err = svc.importOptions(opts, resp) + require.Nil(t, err) + status := resp.Status(kolide.OptionsSection) + require.NotNil(t, status) + assert.Equal(t, 2, status.ImportCount) + assert.Equal(t, 4, status.SkipCount) + assert.Len(t, status.Warnings[kolide.OptionAlreadySet], 1) + assert.Len(t, status.Warnings[kolide.OptionReadonly], 2) + assert.Len(t, status.Warnings[kolide.OptionUnknown], 1) + assert.Len(t, status.Messages, 2) +} + +func TestPacksImportConfig(t *testing.T) { + svc := createServiceMockForImport(t) + + p := &kolide.Pack{ + Name: "dup", + } + _, err := svc.ds.NewPack(p) + require.Nil(t, err) + + q1 := kolide.QueryDetails{ + Query: "select * from foo", + Interval: 100, + Removed: boolPtr(false), + Platform: stringPtr("linux"), + Version: stringPtr("1.0"), + } + q2 := kolide.QueryDetails{ + Query: "select * from bar", + Interval: 50, + Removed: boolPtr(false), + Platform: stringPtr("linux"), + Version: stringPtr("1.0"), + } + q3 := kolide.QueryDetails{ + Query: "select * from baz", + Interval: 500, + Removed: boolPtr(false), + Platform: stringPtr("linux"), + Version: stringPtr("1.0"), + } + + importConfig := kolide.ImportConfig{ + Packs: kolide.PackNameMap{ + "ext1": "/home/usr/ext1.json", + "pack1": kolide.PackDetails{ + Queries: kolide.QueryNameToQueryDetailsMap{ + "q1": q1, + "q2": q2, + }, + Discovery: []string{ + "select * from zz", + "select id, xx from yy", + }, + }, + "dup": kolide.PackDetails{ + Queries: kolide.QueryNameToQueryDetailsMap{ + "q1": q1, + "q2": q2, + }, + }, + "*": "/home/usr/packs/*", + }, + ExternalPacks: kolide.PackNameToPackDetails{ + "ext1": kolide.PackDetails{ + Queries: kolide.QueryNameToQueryDetailsMap{ + "q1": q1, + }, + Discovery: []string{ + "select * from zz", + "select a, b, c from processes", + }, + }, + "ext2": kolide.PackDetails{ + Queries: kolide.QueryNameToQueryDetailsMap{ + "q3": q3, + }, + }, + }, + GlobPackNames: []string{"ext2"}, + } + resp := &kolide.ImportConfigResponse{ + ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus), + } + user := &kolide.User{ + Username: "bob", + Password: []byte("secret"), + Email: "bob@something.com", + Admin: false, + AdminForcedPasswordReset: false, + } + user, err = svc.ds.NewUser(user) + require.Nil(t, err) + + packs, err := importConfig.CollectPacks() + require.Nil(t, err) + assert.Len(t, packs, 4) + err = svc.importPacks(user.ID, &importConfig, resp) + require.Nil(t, err) + queries, err := svc.ds.ListQueries(kolide.ListOptions{}) + require.Nil(t, err) + assert.Len(t, queries, 3) + pack, ok, err := svc.ds.PackByName("pack1") + require.Nil(t, err) + require.True(t, ok) + sqs, err := svc.ds.ListScheduledQueriesInPack(pack.ID, kolide.ListOptions{}) + require.Nil(t, err) + assert.Len(t, sqs, 2) + labels, err := svc.ds.ListLabels(kolide.ListOptions{}) + require.Nil(t, err) + assert.Len(t, labels, 8) + assert.Equal(t, 3, resp.Status(kolide.PacksSection).ImportCount) + assert.Equal(t, 1, resp.Status(kolide.PacksSection).SkipCount) + assert.Equal(t, 3, resp.Status(kolide.QueriesSection).ImportCount) +} diff --git a/server/service/service_invites_test.go b/server/service/service_invites_test.go index 7d6d77a1b3..7ef1a62c16 100644 --- a/server/service/service_invites_test.go +++ b/server/service/service_invites_test.go @@ -80,6 +80,7 @@ func TestListInvites(t *testing.T) { } func setupInviteTest(t *testing.T) (kolide.Service, *mock.Store, *mockMailService) { + ms := new(mock.Store) ms.UserByEmailFunc = mock.UserWithEmailNotFound() ms.UserByIDFunc = mock.UserWithID(adminUser) @@ -88,13 +89,12 @@ func setupInviteTest(t *testing.T) (kolide.Service, *mock.Store, *mockMailServic KolideServerURL: "https://acme.co", }) mailer := &mockMailService{SendEmailFn: func(e kolide.Email) error { return nil }} - svc := validationMiddleware{service{ ds: ms, config: config.TestConfig(), mailService: mailer, clock: clock.NewMockClock(), - }} + }, ms} return svc, ms, mailer } diff --git a/server/service/transport_import_config.go b/server/service/transport_import_config.go new file mode 100644 index 0000000000..bcfb042163 --- /dev/null +++ b/server/service/transport_import_config.go @@ -0,0 +1,34 @@ +package service + +import ( + "encoding/json" + "net/http" + + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +func decodeImportConfigRequest(ctx context.Context, r *http.Request) (interface{}, error) { + var req importRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return nil, err + } + // Unmarshal main config + conf := kolide.ImportConfig{ + Packs: make(kolide.PackNameMap), + ExternalPacks: make(kolide.PackNameToPackDetails), + } + if err := json.Unmarshal([]byte(req.Config), &conf); err != nil { + return nil, err + } + // Unmarshal external packs + for packName, packConfig := range req.ExternalPackConfigs { + var pack kolide.PackDetails + if err := json.Unmarshal([]byte(packConfig), &pack); err != nil { + return nil, err + } + conf.ExternalPacks[packName] = pack + } + conf.GlobPackNames = req.GlobPackNames + return conf, nil +} diff --git a/server/service/util_test.go b/server/service/util_test.go index 76ec7c7e6f..ba3467b755 100644 --- a/server/service/util_test.go +++ b/server/service/util_test.go @@ -111,3 +111,7 @@ func stringPtr(s string) *string { func boolPtr(b bool) *bool { return &b } + +func uintPtr(n uint) *uint { + return &n +} diff --git a/server/service/validation_import_config.go b/server/service/validation_import_config.go new file mode 100644 index 0000000000..fffc050fec --- /dev/null +++ b/server/service/validation_import_config.go @@ -0,0 +1,113 @@ +package service + +import ( + "strconv" + + "github.com/kolide/kolide-ose/server/kolide" + "golang.org/x/net/context" +) + +func (vm validationMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) { + var invalid invalidArgumentError + vm.validateConfigOptions(cfg, &invalid) + vm.validatePacks(cfg, &invalid) + vm.validateDecorator(cfg, &invalid) + vm.validateYARA(cfg, &invalid) + if invalid.HasErrors() { + return nil, invalid + } + return vm.Service.ImportConfig(ctx, cfg) +} + +func (vm validationMiddleware) validateYARA(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) { + if cfg.YARA != nil { + if cfg.YARA.FilePaths == nil { + argErrs.Append("yara", "missing file_paths") + return + } + if cfg.YARA.Signatures == nil { + argErrs.Append("yara", "missing signatures") + } + for fileSection, sigs := range cfg.YARA.FilePaths { + if cfg.FileIntegrityMonitoring == nil { + argErrs.Append("yara", "missing file paths section") + return + } + if _, ok := cfg.FileIntegrityMonitoring[fileSection]; !ok { + argErrs.Appendf("yara", "missing referenced file_paths section '%s'", fileSection) + } + for _, sig := range sigs { + if _, ok := cfg.YARA.Signatures[sig]; !ok { + argErrs.Appendf( + "yara", + "missing signature '%s' referenced in '%s'", + sig, + fileSection, + ) + } + } + } + } +} + +func (vm validationMiddleware) validateDecorator(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) { + if cfg.Decorators != nil { + for str := range cfg.Decorators.Interval { + val, err := strconv.ParseInt(str, 10, 32) + if err != nil { + argErrs.Appendf("decorators", "interval '%s' must be an integer", str) + continue + } + if val%60 != 0 { + + argErrs.Appendf("decorators", "interval '%d' must be divisible by 60", val) + } + } + } +} + +func (vm validationMiddleware) validateConfigOptions(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) { + if cfg.Options != nil { + for optName, optValue := range cfg.Options { + opt, err := vm.ds.OptionByName(string(optName)) + if err != nil { + // skip validation for an option we don't know about, this will generate + // a warning in the service layer + continue + } + if !opt.SameType(optValue) { + argErrs.Appendf("options", "invalid type for '%s'", optName) + } + } + } +} + +func (vm validationMiddleware) validatePacks(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) { + if cfg.Packs != nil { + for packName, pack := range cfg.Packs { + // if glob packs is defined we expect at least one external pack + if packName == kolide.GlobPacks { + if len(cfg.GlobPackNames) == 0 { + argErrs.Append("external_packs", "missing glob packs") + continue + } + // make sure that each glob pack has JSON content + for _, p := range cfg.GlobPackNames { + if _, ok := cfg.ExternalPacks[p]; !ok { + argErrs.Appendf("external_packs", "missing content for '%s'", p) + } + } + continue + } + // if value is a string we expect a file path, in this case, the user has to supply the + // contents of said file which we store in ExternalPacks, if it's not there we need to + // raise an error + switch pack.(type) { + case string: + if _, ok := cfg.ExternalPacks[packName]; !ok { + argErrs.Appendf("external_packs", "missing content for '%s'", packName) + } + } + } + } +} diff --git a/server/service/validation_users.go b/server/service/validation_users.go index 856994cc7e..2817670aa2 100644 --- a/server/service/validation_users.go +++ b/server/service/validation_users.go @@ -10,6 +10,7 @@ import ( type validationMiddleware struct { kolide.Service + ds kolide.Datastore } func (mw validationMiddleware) NewUser(ctx context.Context, p kolide.UserPayload) (*kolide.User, error) { @@ -137,6 +138,12 @@ func (e *invalidArgumentError) Append(name, reason string) { reason: reason, }) } +func (e *invalidArgumentError) Appendf(name, reasonFmt string, args ...interface{}) { + *e = append(*e, invalidArgument{ + name: name, + reason: fmt.Sprintf(reasonFmt, args...), + }) +} func (e *invalidArgumentError) HasErrors() bool { return len(*e) != 0