Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-24 01:18:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Import Config - /config/import #366 (#764)

Browse source
This commit is contained in:
John Murphy 2017-01-13 12:35:25 -06:00 committed by GitHub
parent de3794b17b
commit 6f4dcdd082
56 changed files with 2420 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
server/datastore/datastore_decorators_test.go Normal file
View file

@ -0,0 +1,30 @@
package datastore
import (
"testing"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testDecorators(t *testing.T, ds kolide.Datastore) {
decorator := &kolide.Decorator{
Query: "select from something",
Type: kolide.DecoratorInterval,
Interval: 60,
}
decorator, err := ds.NewDecorator(decorator)
require.Nil(t, err)
require.True(t, decorator.ID > 0)
result, err := ds.Decorator(decorator.ID)
require.Nil(t, err)
assert.Equal(t, decorator.Query, result.Query)
results, err := ds.ListDecorators()
require.Nil(t, err)
assert.Len(t, results, 1)
err = ds.DeleteDecorator(decorator.ID)
require.Nil(t, err)
result, err = ds.Decorator(decorator.ID)
assert.NotNil(t, err)
}

3
server/datastore/datastore_options_test.go
View file

@ -12,7 +12,6 @@ import (
func testOptions(t *testing.T, ds kolide.Datastore) {
require.Nil(t, ds.MigrateData())
// were options pre-loaded?
opts, err := ds.ListOptions()
require.Nil(t, err)
@ -56,12 +55,10 @@ func testOptions(t *testing.T, ds kolide.Datastore) {
opt2, err = ds.OptionByName("disable_distributed")
require.Nil(t, err)
assert.Equal(t, false, opt2.GetValue())
}
func testOptionsToConfig(t *testing.T, ds kolide.Datastore) {
require.Nil(t, ds.MigrateData())
resp, err := ds.GetOsqueryConfigOptions()
require.Nil(t, err)
assert.Len(t, resp, 10)

21
server/datastore/datastore_packs_test.go
View file

@ -30,6 +30,27 @@ func testDeletePack(t *testing.T, ds kolide.Datastore) {
assert.NotNil(t, err)
}
func testGetPackByName(t *testing.T, ds kolide.Datastore) {
pack := &kolide.Pack{
Name: "foo",
}
_, err := ds.NewPack(pack)
assert.Nil(t, err)
assert.NotEqual(t, uint(0), pack.ID)
pack, ok, err := ds.PackByName(pack.Name)
require.Nil(t, err)
assert.True(t, ok)
assert.NotNil(t, pack)
assert.Equal(t, "foo", pack.Name)
pack, ok, err = ds.PackByName("bar")
require.Nil(t, err)
assert.False(t, ok)
assert.Nil(t, pack)
}
func testGetHostsInPack(t *testing.T, ds kolide.Datastore) {
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)

14
server/datastore/datastore_queries_test.go
View file

@ -32,6 +32,20 @@ func testDeleteQuery(t *testing.T, ds kolide.Datastore) {
assert.NotNil(t, err)
}
func testGetQueryByName(t *testing.T, ds kolide.Datastore) {
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)
test.NewQuery(t, ds, "q1", "select * from time", user.ID, true)
actual, ok, err := ds.QueryByName("q1")
require.Nil(t, err)
assert.True(t, ok)
assert.Equal(t, "q1", actual.Name)
actual, ok, err = ds.QueryByName("xxx")
assert.Nil(t, err)
assert.False(t, ok)
}
func testDeleteQueries(t *testing.T, ds kolide.Datastore) {
user := test.NewUser(t, ds, "Zach", "zwass", "zwass@kolide.co", true)

5
server/datastore/datastore_test.go
View file

@ -59,6 +59,11 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testOptions,
testNewScheduledQuery,
testOptionsToConfig,
testGetPackByName,
testGetQueryByName,
testDecorators,
testFileIntegrityMonitoring,
testYARAStore,
testAddLabelToPackTwice,
testGenerateHostStatusStatistics,
testMarkHostSeen,

53
server/datastore/datastore_yara_test.go Normal file
View file

@ -0,0 +1,53 @@
package datastore
import (
"testing"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testYARAStore(t *testing.T, ds kolide.Datastore) {
ysg := &kolide.YARASignatureGroup{
SignatureName: "sig1",
Paths: []string{
"path1",
"path2",
},
}
ysg, err := ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
require.True(t, ysg.ID > 0)
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err = ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
err = ds.NewYARAFilePath("fp1", "sig1")
require.Nil(t, err)
yaraSection, err := ds.YARASection()
require.Nil(t, err)
require.Len(t, yaraSection.FilePaths, 1)
assert.Len(t, yaraSection.FilePaths["fp1"], 1)
require.Len(t, yaraSection.Signatures, 1)
assert.Len(t, yaraSection.Signatures["sig1"], 2)
ysg = &kolide.YARASignatureGroup{
SignatureName: "sig2",
Paths: []string{
"path3",
},
}
ysg, err = ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
yaraSection, err = ds.YARASection()
require.Nil(t, err)
assert.Len(t, yaraSection.Signatures["sig2"], 1)
}

38
server/datastore/file_integrity_monitoring_test.go Normal file
View file

@ -0,0 +1,38 @@
package datastore
import (
"testing"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testFileIntegrityMonitoring(t *testing.T, ds kolide.Datastore) {
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err := ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
fp = &kolide.FIMSection{
SectionName: "fp2",
Paths: []string{
"path4",
"path5",
},
}
_, err = ds.NewFIMSection(fp)
require.Nil(t, err)
actual, err := ds.FIMSections()
require.Nil(t, err)
assert.Len(t, actual, 2)
assert.Len(t, actual["fp1"], 3)
assert.Len(t, actual["fp2"], 2)
}

40
server/datastore/inmem/decorators.go Normal file
View file

@ -0,0 +1,40 @@
package inmem
import "github.com/kolide/kolide-ose/server/kolide"
func (d *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
decorator.ID = d.nextID(decorator)
d.decorators[decorator.ID] = decorator
return decorator, nil
}
func (d *Datastore) DeleteDecorator(id uint) error {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.decorators[id]; !ok {
return notFound("Decorator").WithID(id)
}
delete(d.decorators, id)
return nil
}
func (d *Datastore) Decorator(id uint) (*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
if result, ok := d.decorators[id]; ok {
return result, nil
}
return nil, notFound("Decorator").WithID(id)
}
func (d *Datastore) ListDecorators() ([]*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
var result []*kolide.Decorator
for _, dec := range d.decorators {
result = append(result, dec)
}
return result, nil
}

23
server/datastore/inmem/file_integrity_monitoring.go Normal file
View file

@ -0,0 +1,23 @@
package inmem
import (
"github.com/kolide/kolide-ose/server/kolide"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (*kolide.FIMSection, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
fp.ID = d.nextID(fp)
d.filePaths[fp.ID] = fp
return fp, nil
}
func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
result := make(kolide.FIMSections)
for _, filePath := range d.filePaths {
result[filePath.SectionName] = append(result[filePath.SectionName], filePath.Paths...)
}
return result, nil
}

12
server/datastore/inmem/inmem.go
View file

@ -33,6 +33,10 @@ type Datastore struct {
distributedQueryCampaigns map[uint]kolide.DistributedQueryCampaign
distributedQueryCampaignTargets map[uint]kolide.DistributedQueryCampaignTarget
options map[uint]*kolide.Option
decorators map[uint]*kolide.Decorator
filePaths map[uint]*kolide.FIMSection
yaraFilePaths kolide.YARAFilePaths
yaraSignatureGroups map[uint]*kolide.YARASignatureGroup
appConfig *kolide.AppConfig
config *config.KolideConfig
}
@ -89,13 +93,15 @@ func (d *Datastore) MigrateTables() error {
d.distributedQueryCampaigns = make(map[uint]kolide.DistributedQueryCampaign)
d.distributedQueryCampaignTargets = make(map[uint]kolide.DistributedQueryCampaignTarget)
d.options = make(map[uint]*kolide.Option)
d.decorators = make(map[uint]*kolide.Decorator)
d.filePaths = make(map[uint]*kolide.FIMSection)
d.yaraFilePaths = make(kolide.YARAFilePaths)
d.yaraSignatureGroups = make(map[uint]*kolide.YARASignatureGroup)
return nil
}
func (d *Datastore) MigrateData() error {
d.mtx.Lock()
for _, initData := range appstate.Options() {
opt := kolide.Option{
Name: initData.Name,
@ -115,8 +121,6 @@ func (d *Datastore) MigrateData() error {
SMTPVerifySSLCerts: true,
}
d.mtx.Unlock()
if err := d.createBuiltinLabels(); err != nil {
return err
}

4
server/datastore/inmem/labels.go
View file

@ -12,9 +12,10 @@ import (
)
func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
newLabel := *label
d.mtx.Lock()
for _, l := range d.labels {
if l.Name == label.Name {
return nil, alreadyExists("Label", l.ID)
@ -23,7 +24,6 @@ func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
newLabel.ID = d.nextID(label)
d.labels[newLabel.ID] = &newLabel
d.mtx.Unlock()
return &newLabel, nil
}

11
server/datastore/inmem/packs.go
View file

@ -6,6 +6,17 @@ import (
"github.com/kolide/kolide-ose/server/kolide"
)
func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, p := range d.packs {
if p.Name == name {
return p, true, nil
}
}
return nil, false, nil
}
func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
newPack := *pack

17
server/datastore/inmem/queries.go
View file

@ -3,8 +3,8 @@ package inmem
import (
"sort"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
@ -26,6 +26,17 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
return &newQuery, nil
}
func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, q := range d.queries {
if name == q.Name {
return q, true, nil
}
}
return nil, false, nil
}
func (d *Datastore) SaveQuery(query *kolide.Query) error {
d.mtx.Lock()
defer d.mtx.Unlock()
@ -86,7 +97,7 @@ func (d *Datastore) Query(id uint) (*kolide.Query, error) {
query.AuthorName = d.getUserNameByID(query.AuthorID)
if err := d.loadPacksForQueries([]*kolide.Query{query}); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "error fetching query by id")
}
return query, nil
@ -136,7 +147,7 @@ func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error)
queries = queries[low:high]
if err := d.loadPacksForQueries(queries); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "error listing queries")
}
return queries, nil

35
server/datastore/inmem/yara.go Normal file
View file

@ -0,0 +1,35 @@
package inmem
import "github.com/kolide/kolide-ose/server/kolide"
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (*kolide.YARASignatureGroup, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
ysg.ID = d.nextID(ysg)
d.yaraSignatureGroups[ysg.ID] = ysg
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error {
d.mtx.Lock()
defer d.mtx.Unlock()
d.yaraFilePaths[fileSectionName] = append(d.yaraFilePaths[fileSectionName], sigGroupName)
return nil
}
func (d *Datastore) YARASection() (*kolide.YARASection, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
result := &kolide.YARASection{
Signatures: make(map[string][]string),
FilePaths: make(map[string][]string),
}
for _, ysg := range d.yaraSignatureGroups {
result.Signatures[ysg.SignatureName] = append(result.Signatures[ysg.SignatureName], ysg.Paths...)
}
for fileSection, sigSection := range d.yaraFilePaths {
result.FilePaths[fileSection] = append(result.FilePaths[fileSection], sigSection...)
}
return result, nil
}

1
server/datastore/inmem_test.go
View file

@ -13,6 +13,7 @@ func TestInmem(t *testing.T) {
for _, f := range testFunctions {
t.Run(functionName(f), func(t *testing.T) {
ds, err := inmem.New(config.TestConfig())
require.Nil(t, err)
defer func() { require.Nil(t, ds.Drop()) }()
require.Nil(t, err)
f(t, ds)

6
server/datastore/mysql/app_configs.go
View file

@ -1,13 +1,13 @@
package mysql
import (
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) {
if err := d.SaveAppConfig(info); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "new app config")
}
return info, nil
@ -18,7 +18,7 @@ func (d *Datastore) AppConfig() (*kolide.AppConfig, error) {
info := &kolide.AppConfig{}
err := d.db.Get(info, "SELECT * FROM app_configs LIMIT 1")
if err != nil {
return nil, err
return nil, errors.Wrap(err, "selecting app config")
}
return info, nil
}

22
server/datastore/mysql/campaigns.go
View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) (*kolide.DistributedQueryCampaign, error) {
@ -20,7 +20,7 @@ func (d *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCam
`
result, err := d.db.Exec(sqlStatement, camp.QueryID, camp.Status, camp.UserID)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting distributed query campaign")
}
id, _ := result.LastInsertId()
@ -34,7 +34,7 @@ func (d *Datastore) DistributedQueryCampaign(id uint) (*kolide.DistributedQueryC
`
campaign := &kolide.DistributedQueryCampaign{}
if err := d.db.Get(campaign, sql, id); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting distributed query campaign")
}
return campaign, nil
@ -51,7 +51,7 @@ func (d *Datastore) SaveDistributedQueryCampaign(camp *kolide.DistributedQueryCa
`
_, err := d.db.Exec(sqlStatement, camp.QueryID, camp.Status, camp.UserID, camp.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "updating distributed query campaign")
}
return nil
@ -64,7 +64,7 @@ func (d *Datastore) DistributedQueryCampaignTargetIDs(id uint) (hostIDs []uint,
targets := []kolide.DistributedQueryCampaignTarget{}
if err = d.db.Select(&targets, sqlStatement, id); err != nil {
return nil, nil, errors.DatabaseError(err)
return nil, nil, errors.Wrap(err, "selecting distributed campaign target")
}
hostIDs = []uint{}
@ -93,7 +93,7 @@ func (d *Datastore) NewDistributedQueryCampaignTarget(target *kolide.Distributed
`
result, err := d.db.Exec(sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "insert distributed campaign target")
}
id, _ := result.LastInsertId()
@ -114,7 +114,7 @@ func (d *Datastore) NewDistributedQueryExecution(exec *kolide.DistributedQueryEx
result, err := d.db.Exec(sqlStatement, exec.HostID, exec.DistributedQueryCampaignID,
exec.Status, exec.Error, exec.ExecutionDuration)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "insert distributed campaign target")
}
id, _ := result.LastInsertId()
@ -135,12 +135,12 @@ func (d *Datastore) CleanupDistributedQueryCampaigns(now time.Time) (expired uin
kolide.QueryWaiting, now.Add(-1*time.Minute),
kolide.QueryRunning, now.Add(-24*time.Hour))
if err != nil {
return expired, deleted, errors.DatabaseError(err)
return expired, deleted, errors.Wrap(err, "updating distributed query campaign")
}
exp, err := result.RowsAffected()
if err != nil {
return expired, deleted, errors.DatabaseError(err)
return expired, deleted, errors.Wrap(err, "rows effected updating distributed query campaign")
}
expired = uint(exp)
@ -154,12 +154,12 @@ func (d *Datastore) CleanupDistributedQueryCampaigns(now time.Time) (expired uin
`
result, err = d.db.Exec(sqlStatement, kolide.QueryComplete)
if err != nil {
return expired, deleted, errors.DatabaseError(err)
return expired, deleted, errors.Wrap(err, "deleting distributed campaign executions")
}
del, err := result.RowsAffected()
if err != nil {
return expired, deleted, errors.DatabaseError(err)
return expired, deleted, errors.Wrap(err, "rows effected deleting distributed campaign")
}
deleted = uint(del)

71
server/datastore/mysql/decorators.go Normal file
View file

@ -0,0 +1,71 @@
package mysql
import (
"database/sql"
"github.com/pkg/errors"
"github.com/kolide/kolide-ose/server/kolide"
)
func (ds *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) {
sqlStatement :=
"INSERT INTO decorators (" +
"`query`," +
"`type`," +
"`interval` ) " +
"VALUES (?, ?, ?)"
result, err := ds.db.Exec(sqlStatement, decorator.Query, decorator.Type, decorator.Interval)
if err != nil {
return nil, errors.Wrap(err, "creating decorator")
}
id, _ := result.LastInsertId()
decorator.ID = uint(id)
return decorator, nil
}
func (ds *Datastore) DeleteDecorator(id uint) error {
sqlStatement := `
DELETE FROM decorators
WHERE id = ?
`
res, err := ds.db.Exec(sqlStatement, id)
if err != nil {
return errors.Wrap(err, "deleting decorator")
}
deleted, _ := res.RowsAffected()
if deleted < 1 {
return notFound("Decorator").WithID(id)
}
return nil
}
func (ds *Datastore) Decorator(id uint) (*kolide.Decorator, error) {
sqlStatement := `
SELECT *
FROM decorators
WHERE id = ?
`
var result kolide.Decorator
err := ds.db.Get(&result, sqlStatement, id)
if err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Decorator").WithID(id)
}
return nil, errors.Wrap(err, "retrieving decorator")
}
return &result, nil
}
func (ds *Datastore) ListDecorators() ([]*kolide.Decorator, error) {
sqlStatement := `
SELECT *
FROM decorators
`
var results []*kolide.Decorator
err := ds.db.Select(&results, sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "listing decorators")
}
return results, nil
}

78
server/datastore/mysql/file_integrity_monitoring.go Normal file
View file

@ -0,0 +1,78 @@
package mysql
import (
"database/sql"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection) (result *kolide.FIMSection, err error) {
txn, err := d.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "update options begin transaction")
}
var success bool
defer func() {
if success {
if err = txn.Commit(); err == nil {
return
}
}
txn.Rollback()
}()
sqlStatement := `
INSERT INTO file_integrity_monitorings (
section_name,
description
) VALUES( ?, ?)
`
var resp sql.Result
resp, err = txn.Exec(sqlStatement, fp.SectionName, fp.Description)
if err != nil {
return nil, errors.Wrap(err, "creating fim section")
}
id, _ := resp.LastInsertId()
fp.ID = uint(id)
sqlStatement = `
INSERT INTO file_integrity_monitoring_files (
file,
file_integrity_monitoring_id
) VALUES( ?, ? )
`
for _, fileName := range fp.Paths {
_, err = txn.Exec(sqlStatement, fileName, fp.ID)
if err != nil {
return nil, errors.Wrap(err, "adding path to fim section")
}
}
success = true
return fp, nil
}
func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
sqlStatement := `
SELECT fim.section_name, mf.file FROM
file_integrity_monitorings AS fim
INNER JOIN file_integrity_monitoring_files AS mf
ON (fim.id = mf.file_integrity_monitoring_id)
`
rows, err := d.db.Query(sqlStatement)
if err != nil {
if err == sql.ErrNoRows {
return nil, notFound("FilePath")
}
return nil, errors.Wrap(err, "retrieving fim sections")
}
result := make(kolide.FIMSections)
for rows.Next() {
var sectionName, fileName string
err = rows.Scan(&sectionName, &fileName)
if err != nil {
return nil, errors.Wrap(err, "retrieving path for fim section")
}
result[sectionName] = append(result[sectionName], fileName)
}
return result, nil
}

28
server/datastore/mysql/labels.go
View file

@ -5,8 +5,8 @@ import (
"time"
"github.com/jmoiron/sqlx"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
// NewLabel creates a new kolide.Label
@ -23,7 +23,7 @@ func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) {
`
result, err := d.db.Exec(sql, label.Name, label.Description, label.Query, label.Platform, label.LabelType)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting label")
}
id, _ := result.LastInsertId()
@ -46,7 +46,7 @@ func (d *Datastore) Label(lid uint) (*kolide.Label, error) {
label := &kolide.Label{}
if err := d.db.Get(label, sql, lid); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting label")
}
return label, nil
@ -61,7 +61,7 @@ func (d *Datastore) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error)
labels := []*kolide.Label{}
if err := d.db.Select(&labels, sql); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting labels")
}
return labels, nil
@ -84,7 +84,7 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma
`
rows, err := d.db.Query(sqlStatment, host.Platform, host.ID, cutoff)
if err != nil && err != sql.ErrNoRows {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting label queries for host")
}
defer rows.Close()
@ -94,7 +94,7 @@ func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (ma
var id, query string
if err = rows.Scan(&id, &query); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "scanning label queries for host")
}
results[id] = query
@ -129,7 +129,7 @@ func (d *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[st
_, err := d.db.Exec(sqlStatement, vals...)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "inserting label query execution")
}
return nil
@ -148,7 +148,7 @@ func (d *Datastore) ListLabelsForHost(hid uint) ([]kolide.Label, error) {
labels := []kolide.Label{}
err := d.db.Select(&labels, sqlStatement, hid)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting host labels")
}
return labels, nil
@ -169,7 +169,7 @@ func (d *Datastore) ListHostsInLabel(lid uint) ([]kolide.Host, error) {
hosts := []kolide.Host{}
err := d.db.Select(&hosts, sqlStatement, lid)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting label query executions")
}
return hosts, nil
}
@ -191,14 +191,14 @@ func (d *Datastore) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error
`
query, args, err := sqlx.In(sqlStatement, labels)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "building query listing unique hosts in labels")
}
query = d.db.Rebind(query)
hosts := []kolide.Host{}
err = d.db.Select(&hosts, query, args...)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "listing unique hosts in labels")
}
return hosts, nil
@ -229,7 +229,7 @@ func (d *Datastore) searchLabelsWithOmits(query string, omit ...uint) ([]kolide.
sql, args, err := sqlx.In(sqlStatement, query, kolide.LabelTypeBuiltIn, omit)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "building query for labels with omits")
}
sql = d.db.Rebind(sql)
@ -237,7 +237,7 @@ func (d *Datastore) searchLabelsWithOmits(query string, omit ...uint) ([]kolide.
matches := []kolide.Label{}
err = d.db.Select(&matches, sql, args...)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting labels with omits")
}
return matches, nil
@ -272,7 +272,7 @@ func (d *Datastore) SearchLabels(query string, omit ...uint) ([]kolide.Label, er
matches := []kolide.Label{}
err := d.db.Select(&matches, sqlStatement, query, kolide.LabelTypeBuiltIn)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting labels for search")
}
return matches, nil

20
server/datastore/mysql/migrations/tables/20170105151732_AddNameIndexToQueryTable.go Normal file
View file

@ -0,0 +1,20 @@
package tables
import "database/sql"
func init() {
MigrationClient.AddMigration(Up_20170105151732, Down_20170105151732)
}
func Up_20170105151732(tx *sql.Tx) error {
sqlStatement := "CREATE UNIQUE INDEX idx_query_unique_name " +
" ON `queries` (`name` ASC);"
_, err := tx.Exec(sqlStatement)
return err
}
func Down_20170105151732(tx *sql.Tx) error {
sqlStatement := "DROP INDEX idx_query_unique_name ON `queries`;"
_, err := tx.Exec(sqlStatement)
return err
}

27
server/datastore/mysql/migrations/tables/20170108191242_CreateDecoratorTable.go Normal file
View file

@ -0,0 +1,27 @@
package tables
import "database/sql"
func init() {
MigrationClient.AddMigration(Up_20170108191242, Down_20170108191242)
}
func Up_20170108191242(tx *sql.Tx) error {
_, err := tx.Exec(
"CREATE TABLE `decorators` ( " +
"`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, " +
"`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
"`updated_at` TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, " +
"`query` VARCHAR(255) NOT NULL, " +
"`type` INT UNSIGNED NOT NULL, " +
"`interval` INT UNSIGNED NOT NULL, " +
"PRIMARY KEY (`id`) " +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;",
)
return err
}
func Down_20170108191242(tx *sql.Tx) error {
_, err := tx.Exec("DROP TABLE IF EXISTS decorators;")
return err
}

51
server/datastore/mysql/migrations/tables/20170109094020_CreateFIMTables.go Normal file
View file

@ -0,0 +1,51 @@
package tables
import "database/sql"
func init() {
MigrationClient.AddMigration(Up_20170109094020, Down_20170109094020)
}
func Up_20170109094020(tx *sql.Tx) error {
sqlStatement :=
"CREATE TABLE `file_integrity_monitorings` ( " +
" `id` int(10) NOT NULL AUTO_INCREMENT, " +
" `section_name` varchar(255) NOT NULL DEFAULT '', " +
" `description` varchar(255) NOT NULL DEFAULT ''," +
" PRIMARY KEY (`id`)," +
" UNIQUE KEY `idx_unique_section_name` (`section_name`) USING BTREE" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;"
_, err := tx.Exec(sqlStatement)
if err != nil {
return err
}
sqlStatement =
"CREATE TABLE `file_integrity_monitoring_files` (" +
" `id` int(10) NOT NULL AUTO_INCREMENT," +
" `file` varchar(255) NOT NULL DEFAULT ''," +
" `file_integrity_monitoring_id` int(10) NOT NULL DEFAULT '0'," +
" PRIMARY KEY (`id`)," +
" UNIQUE KEY `idx_fim_unique_file_name` (`file`) USING BTREE," +
" KEY `fk_file_integrity_monitoring` (`file_integrity_monitoring_id`)," +
" CONSTRAINT `fk_file_integrity_monitoring` FOREIGN KEY (`file_integrity_monitoring_id`) REFERENCES `file_integrity_monitorings` (`id`) ON DELETE CASCADE" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;"
_, err = tx.Exec(sqlStatement)
if err != nil {
return err
}
return nil
}
func Down_20170109094020(tx *sql.Tx) error {
sqlStatement := "DROP TABLE IF EXISTS `file_integrity_monitoring_files`; "
_, err := tx.Exec(sqlStatement)
if err != nil {
return err
}
sqlStatement = "DROP TABLE IF EXISTS `file_integrity_monitorings`;"
_, err = tx.Exec(sqlStatement)
if err != nil {
return err
}
return nil
}

66
server/datastore/mysql/migrations/tables/20170109130438_CreateYARATables.go Normal file
View file

@ -0,0 +1,66 @@
package tables
import "database/sql"
func init() {
MigrationClient.AddMigration(Up_20170109130438, Down_20170109130438)
}
func Up_20170109130438(tx *sql.Tx) error {
sqlStatement :=
"CREATE TABLE `yara_signatures` ( " +
" `id` int(11) NOT NULL AUTO_INCREMENT, " +
" `signature_name` varchar(128) NOT NULL DEFAULT '', " +
" PRIMARY KEY (`id`), " +
" UNIQUE KEY `idx_yara_signatures_unique_name` (`signature_name`) USING BTREE " +
") ENGINE=InnoDB DEFAULT CHARSET=utf8; "
if _, err := tx.Exec(sqlStatement); err != nil {
return err
}
sqlStatement =
"CREATE TABLE `yara_file_paths` ( " +
" `file_integrity_monitoring_id` int(11) NOT NULL DEFAULT '0', " +
" `yara_signature_id` int(11) NOT NULL DEFAULT '0', " +
" PRIMARY KEY (`file_integrity_monitoring_id`,`yara_signature_id`), " +
" KEY `fk_yara_signature_id` (`yara_signature_id`), " +
" CONSTRAINT `fk_file_integrity_monitoring_id` FOREIGN KEY (`file_integrity_monitoring_id`) REFERENCES `file_integrity_monitorings` (`id`), " +
" CONSTRAINT `fk_yara_signature_id` FOREIGN KEY (`yara_signature_id`) REFERENCES `yara_signatures` (`id`) " +
") ENGINE=InnoDB DEFAULT CHARSET=utf8; "
if _, err := tx.Exec(sqlStatement); err != nil {
return err
}
sqlStatement =
"CREATE TABLE `yara_signature_paths` ( " +
" `id` int(11) NOT NULL AUTO_INCREMENT, " +
" `file_path` varchar(255) NOT NULL DEFAULT '', " +
" `yara_signature_id` int(11) NOT NULL DEFAULT '0', " +
" PRIMARY KEY (`id`), " +
" KEY `fk_yara_signature` (`yara_signature_id`), " +
" CONSTRAINT `fk_yara_signature` FOREIGN KEY (`yara_signature_id`) REFERENCES `yara_signatures` (`id`) ON DELETE CASCADE " +
") ENGINE=InnoDB DEFAULT CHARSET=utf8; "
if _, err := tx.Exec(sqlStatement); err != nil {
return err
}
return nil
}
func Down_20170109130438(tx *sql.Tx) error {
sqlStatement := "DROP TABLE IF EXISTS `yara_signature_paths`;"
_, err := tx.Exec(sqlStatement)
if err != nil {
return err
}
sqlStatement = "DROP TABLE IF EXISTS `yara_file_paths`;"
_, err = tx.Exec(sqlStatement)
if err != nil {
return err
}
sqlStatement = "DROP TABLE IF EXISTS `yara_signatures`;"
_, err = tx.Exec(sqlStatement)
if err != nil {
return err
}
return nil
}

19
server/datastore/mysql/packs.go
View file

@ -8,6 +8,24 @@ import (
"github.com/pkg/errors"
)
func (d *Datastore) PackByName(name string) (*kolide.Pack, bool, error) {
sqlStatement := `
SELECT *
FROM packs
WHERE name = ? AND NOT deleted
`
var pack kolide.Pack
err := d.db.Get(&pack, sqlStatement, name)
if err != nil {
if err == sql.ErrNoRows {
return nil, false, nil
}
return nil, false, errors.Wrap(err, "fetching packs by name")
}
return &pack, true, nil
}
// NewPack creates a new Pack
func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) {
@ -192,6 +210,7 @@ func (d *Datastore) ListHostsInPack(pid uint, opt kolide.ListOptions) ([]*kolide
)
WHERE pt.pack_id = ?
`
hosts := []*kolide.Host{}
if err := d.db.Select(&hosts, appendListOptionsToSQL(query, opt), kolide.TargetLabel, kolide.TargetHost, pid); err != nil && err != sql.ErrNoRows {
return nil, errors.Wrap(err, "listing hosts in pack")

18
server/datastore/mysql/password_reset.go
View file

@ -1,8 +1,8 @@
package mysql
import (
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) {
@ -13,7 +13,7 @@ func (d *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*
`
response, err := d.db.Exec(sqlStatement, req.UserID, req.Token)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting password reset requests")
}
id, _ := response.LastInsertId()
@ -32,7 +32,7 @@ func (d *Datastore) SavePasswordResetRequest(req *kolide.PasswordResetRequest) e
`
_, err := d.db.Exec(sqlStatement, req.ExpiresAt, req.UserID, req.Token, req.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "updating password reset requests")
}
return nil
@ -45,7 +45,7 @@ func (d *Datastore) DeletePasswordResetRequest(req *kolide.PasswordResetRequest)
`
_, err := d.db.Exec(sqlStatement, req.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "deleting from password reset request")
}
return nil
@ -57,7 +57,7 @@ func (d *Datastore) DeletePasswordResetRequestsForUser(userID uint) error {
`
_, err := d.db.Exec(sqlStatement, userID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "deleting password reset request by user")
}
return nil
@ -71,7 +71,7 @@ func (d *Datastore) FindPassswordResetByID(id uint) (*kolide.PasswordResetReques
passwordResetRequest := &kolide.PasswordResetRequest{}
err := d.db.Get(&passwordResetRequest, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting password reset by id")
}
return passwordResetRequest, nil
@ -86,7 +86,7 @@ func (d *Datastore) FindPassswordResetsByUserID(id uint) ([]*kolide.PasswordRese
passwordResetRequests := []*kolide.PasswordResetRequest{}
err := d.db.Select(&passwordResetRequests, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "finding password resets by user id")
}
return passwordResetRequests, nil
@ -101,7 +101,7 @@ func (d *Datastore) FindPassswordResetByToken(token string) (*kolide.PasswordRes
passwordResetRequest := &kolide.PasswordResetRequest{}
err := d.db.Get(passwordResetRequest, sqlStatement, token)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting password reset requests")
}
return passwordResetRequest, nil
@ -117,7 +117,7 @@ func (d *Datastore) FindPassswordResetByTokenAndUserID(token string, id uint) (*
passwordResetRequest := &kolide.PasswordResetRequest{}
err := d.db.Get(passwordResetRequest, sqlStatement, id, token)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting password reset by token and user id")
}
return passwordResetRequest, nil

44
server/datastore/mysql/queries.go
View file

@ -1,14 +1,32 @@
package mysql
import (
"database/sql"
"github.com/jmoiron/sqlx"
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) QueryByName(name string) (*kolide.Query, bool, error) {
sqlStatement := `
SELECT *
FROM queries
WHERE name = ? AND NOT deleted
`
var query kolide.Query
err := d.db.Get(&query, sqlStatement, name)
if err != nil {
if err == sql.ErrNoRows {
return nil, false, nil
}
return nil, false, errors.Wrap(err, "selecting query by name")
}
return &query, true, nil
}
// NewQuery creates a Query
func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
sql := `
INSERT INTO queries (
name,
@ -20,7 +38,7 @@ func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) {
`
result, err := d.db.Exec(sql, query.Name, query.Description, query.Query, query.Saved, query.AuthorID)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting new query")
}
id, _ := result.LastInsertId()
@ -38,7 +56,7 @@ func (d *Datastore) SaveQuery(q *kolide.Query) error {
`
_, err := d.db.Exec(sql, q.Name, q.Description, q.Query, q.AuthorID, q.Saved, q.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "updating query")
}
return nil
@ -59,17 +77,17 @@ func (d *Datastore) DeleteQueries(ids []uint) (uint, error) {
`
query, args, err := sqlx.In(sql, ids)
if err != nil {
return 0, errors.DatabaseError(err)
return 0, errors.Wrap(err, "building delete query query")
}
result, err := d.db.Exec(query, args...)
if err != nil {
return 0, errors.DatabaseError(err)
return 0, errors.Wrap(err, "updating delete query")
}
deleted, err := result.RowsAffected()
if err != nil {
return 0, errors.DatabaseError(err)
return 0, errors.Wrap(err, "fetching delete query rows effected")
}
return uint(deleted), nil
@ -88,11 +106,11 @@ func (d *Datastore) Query(id uint) (*kolide.Query, error) {
`
query := &kolide.Query{}
if err := d.db.Get(query, sql, id); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting query")
}
if err := d.loadPacksForQueries([]*kolide.Query{query}); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "loading packs for queries")
}
return query, nil
@ -113,11 +131,11 @@ func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error)
results := []*kolide.Query{}
if err := d.db.Select(&results, sql); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "listing queries")
}
if err := d.loadPacksForQueries(results); err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "loading packs for queries")
}
return results, nil
@ -150,7 +168,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error {
query, args, err := sqlx.In(sql, ids)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "building query in load packs for queries")
}
rows := []struct {
@ -160,7 +178,7 @@ func (d *Datastore) loadPacksForQueries(queries []*kolide.Query) error {
err = d.db.Select(&rows, query, args...)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "selecting load packs for queries")
}
for _, row := range rows {

16
server/datastore/mysql/sessions.go
View file

@ -1,8 +1,8 @@
package mysql
import (
"github.com/kolide/kolide-ose/server/errors"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) SessionByKey(key string) (*kolide.Session, error) {
@ -13,7 +13,7 @@ func (d *Datastore) SessionByKey(key string) (*kolide.Session, error) {
session := &kolide.Session{}
err := d.db.Get(session, sqlStatement, key)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting sessions")
}
return session, nil
@ -28,7 +28,7 @@ func (d *Datastore) SessionByID(id uint) (*kolide.Session, error) {
session := &kolide.Session{}
err := d.db.Get(session, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting session by id")
}
return session, nil
@ -42,7 +42,7 @@ func (d *Datastore) ListSessionsForUser(id uint) ([]*kolide.Session, error) {
sessions := []*kolide.Session{}
err := d.db.Select(&sessions, sqlStatement, id)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "selecting sessions for user")
}
return sessions, nil
@ -59,7 +59,7 @@ func (d *Datastore) NewSession(session *kolide.Session) (*kolide.Session, error)
`
result, err := d.db.Exec(sqlStatement, session.UserID, session.Key)
if err != nil {
return nil, errors.DatabaseError(err)
return nil, errors.Wrap(err, "inserting session")
}
id, _ := result.LastInsertId()
@ -73,7 +73,7 @@ func (d *Datastore) DestroySession(session *kolide.Session) error {
`
_, err := d.db.Exec(sqlStatement, session.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "deleting session")
}
return nil
@ -85,7 +85,7 @@ func (d *Datastore) DestroyAllSessionsForUser(id uint) error {
`
_, err := d.db.Exec(sqlStatement, id)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "deleting sessions for user")
}
return nil
@ -99,7 +99,7 @@ func (d *Datastore) MarkSessionAccessed(session *kolide.Session) error {
`
_, err := d.db.Exec(sqlStatement, d.clock.Now(), session.ID)
if err != nil {
return errors.DatabaseError(err)
return errors.Wrap(err, "updating mark session as accessed")
}
return nil

126
server/datastore/mysql/yara.go Normal file
View file

@ -0,0 +1,126 @@
package mysql
import (
"database/sql"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup) (sg *kolide.YARASignatureGroup, err error) {
var success bool
txn, err := d.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "new yara signature group begin transaction")
}
defer func() {
if success {
if err = txn.Commit(); err == nil {
return
}
}
txn.Rollback()
}()
sqlStatement := `
INSERT INTO yara_signatures (
signature_name
) VALUES( ? )
`
var result sql.Result
result, err = txn.Exec(sqlStatement, ysg.SignatureName)
if err != nil {
return nil, errors.Wrap(err, "inserting new yara signature group")
}
id, _ := result.LastInsertId()
ysg.ID = uint(id)
sqlStatement = `
INSERT INTO yara_signature_paths (
file_path,
yara_signature_id
) VALUES( ?, ? )
`
for _, path := range ysg.Paths {
_, err = txn.Exec(sqlStatement, path, ysg.ID)
if err != nil {
return nil, errors.Wrap(err, "inserting new signature path")
}
}
success = true
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string) error {
sqlStatement := `
INSERT INTO yara_file_paths (
file_integrity_monitoring_id,
yara_signature_id
) VALUES (
(
SELECT fim.id
FROM file_integrity_monitorings AS fim
WHERE fim.section_name = ?
LIMIT 1
),
(
SELECT ys.id AS ys
FROM yara_signatures AS ys
WHERE ys.signature_name = ?
LIMIT 1
)
)
`
_, err := d.db.Exec(sqlStatement, fileSectionName, sigGroupName)
if err != nil {
return errors.Wrap(err, "inserting yara file path")
}
return nil
}
func (d *Datastore) YARASection() (*kolide.YARASection, error) {
result := &kolide.YARASection{
Signatures: make(map[string][]string),
FilePaths: make(map[string][]string),
}
sqlStatement := `
SELECT s.signature_name, p.file_path
FROM yara_signatures AS s
INNER JOIN yara_signature_paths AS p
ON ( s.id = p.yara_signature_id )
`
rows, err := d.db.Query(sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "selecting yara information")
}
for rows.Next() {
var sigName, sigPath string
err = rows.Scan(&sigName, &sigPath)
if err != nil {
return nil, errors.Wrap(err, "scanning yara information")
}
result.Signatures[sigName] = append(result.Signatures[sigName], sigPath)
}
sqlStatement = `
SELECT f.section_name, y.signature_name
FROM file_integrity_monitorings AS f
INNER JOIN yara_file_paths AS yfp
ON (f.id = yfp.file_integrity_monitoring_id)
INNER JOIN yara_signatures AS y
ON (y.id = yfp.yara_signature_id )
`
rows, err = d.db.Query(sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "selecting yara signatures")
}
for rows.Next() {
var sectionName, signatureName string
err = rows.Scan(&sectionName, &signatureName)
if err != nil {
return nil, errors.Wrap(err, "scanning yara signature values")
}
result.FilePaths[sectionName] = append(result.FilePaths[sectionName], signatureName)
}
return result, nil
}

1
server/datastore/mysql_test.go
View file

@ -49,7 +49,6 @@ func TestMySQL(t *testing.T) {
t.Run(functionName(f), func(t *testing.T) {
defer func() { require.Nil(t, ds.Drop()) }()
require.Nil(t, ds.MigrateTables())
f(t, ds)
})
}

5
server/errors/errors.go
View file

@ -53,11 +53,6 @@ func NewFromError(err error, status int, publicMessage string) *KolideError {
}
}
// Wrap a DB error with the extra KolideError decorations
func DatabaseError(err error) *KolideError {
return NewFromError(err, http.StatusInternalServerError, "Database error: "+err.Error())
}
// Wrap a server error with the extra KolideError decorations
func InternalServerError(err error) *KolideError {
return NewFromError(err, http.StatusInternalServerError, "Internal server error")

13
server/errors/errors_test.go
View file

@ -47,19 +47,6 @@ func TestNewFromError(t *testing.T) {
assert.Equal(t, expect, kolideErr)
}
func TestDatabaseError(t *testing.T) {
err := errors.New("Foo error")
kolideErr := DatabaseError(err)
expect := &KolideError{
Err: err,
StatusCode: http.StatusInternalServerError,
PublicMessage: "Database error: " + err.Error(),
PrivateMessage: "Foo error",
}
assert.Equal(t, expect, kolideErr)
}
// These types and functions for performing an unordered comparison on a
// []map[string]string] as parsed from the error JSON
type errorField map[string]string

3
server/kolide/datastore.go
View file

@ -14,6 +14,9 @@ type Datastore interface {
InviteStore
ScheduledQueryStore
OptionStore
DecoratorStore
FileIntegrityMonitoringStore
YARAStore
Name() string
Drop() error
// MigrateTables creates and migrates the table schemas

34
server/kolide/decorators.go Normal file
View file

@ -0,0 +1,34 @@
package kolide
// DecoratorStore methods to manipulate decorator queries.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type DecoratorStore interface {
// NewDecorator creates a decorator query.
NewDecorator(decorator *Decorator) (*Decorator, error)
// DeleteDecorator removes a decorator query.
DeleteDecorator(id uint) error
// Decorator retrieves a decorator query with supplied ID.
Decorator(id uint) (*Decorator, error)
// ListDecorators returns all decorator queries.
ListDecorators() ([]*Decorator, error)
}
// DecoratorType refers to the allowable types of decorator queries.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type DecoratorType int
const (
DecoratorLoad DecoratorType = iota
DecoratorAlways
DecoratorInterval
)
// Decorator contains information about a decorator query.
type Decorator struct {
UpdateCreateTimestamps
ID uint
Type DecoratorType
// Interval note this is only pertainent for DecoratorInterval type.
Interval uint
Query string
}

20
server/kolide/file_integrity_monitoring.go Normal file
View file

@ -0,0 +1,20 @@
package kolide
type FIMSections map[string][]string
type FileIntegrityMonitoringStore interface {
// NewFIMSection creates a named group of file paths
NewFIMSection(path *FIMSection) (*FIMSection, error)
// FIMSections returns all named file sections
FIMSections() (FIMSections, error)
}
// FilePath maps a name to a group of files for the osquery file_paths
// section.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type FIMSection struct {
ID uint
SectionName string `db:"section_name"`
Description string
Paths []string `db:"-"`
}

218
server/kolide/import_config.go Normal file
View file

@ -0,0 +1,218 @@
package kolide
import (
"errors"
"fmt"
"strings"
"golang.org/x/net/context"
)
type ImportConfigService interface {
// ImportConfig create packs, queries, options etc based on imported
// osquery configuration.
ImportConfig(ctx context.Context, cfg *ImportConfig) (*ImportConfigResponse, error)
}
// ImportSection is used to categorize information associated with the import
// of a particular section of an imported osquery configuration file.
type ImportSection string
const (
OptionsSection ImportSection = "options"
PacksSection = "packs"
QueriesSection = "queries"
DecoratorsSection = "decorators"
FilePathsSection = "file_paths"
YARASigSection = "yara_signature_group"
YARAFileSection = "yara_file_group"
)
// WarningType is used to group associated warnings for options, packs etc
// when importing on osquery configuration file.
type WarningType string
const (
PackDuplicate WarningType = "duplicate_pack"
DifferentQuerySameName = "different_query_same_name"
OptionAlreadySet = "option_already_set"
OptionReadonly = "option_readonly"
OptionUnknown = "option_unknown"
QueryDuplicate = "duplicate_query"
Unsupported = "unsupported"
)
// ImportStatus contains information pertaining to the import of a section
// of an osquery configuration file.
type ImportStatus struct {
// Title human readable name of the section of the import file that this
// status pertains to.
Title string `json:"title"`
// ImportCount count of items successfully imported.
ImportCount int `json:"import_count"`
// SkipCount count of items that are skipped. The reasons for the omissions
// can be found in Warnings.
SkipCount int `json:"skip_count"`
// Warnings groups catagories of warnings with one or more detail messages.
Warnings map[WarningType][]string `json:"warnings"`
// Messages contains an entry for each import attempt.
Messages []string `json:"messages"`
}
// Warning is used to add a warning message to ImportStatus.
func (is *ImportStatus) Warning(warnType WarningType, fmtMsg string, fmtArgs ...interface{}) {
is.Warnings[warnType] = append(is.Warnings[warnType], fmt.Sprintf(fmtMsg, fmtArgs...))
}
// Message is used to add a general message to ImportStatus, usually indicating
// what was changed in a successful import.
func (is *ImportStatus) Message(fmtMsg string, args ...interface{}) {
is.Messages = append(is.Messages, fmt.Sprintf(fmtMsg, args...))
}
// ImportConfigResponse contains information about the import of an osquery
// configuration file.
type ImportConfigResponse struct {
ImportStatusBySection map[ImportSection]*ImportStatus `json:"import_status"`
}
// Status returns a structure that contains information about the import
// of a particular section of an osquery configuration file.
func (ic *ImportConfigResponse) Status(section ImportSection) (status *ImportStatus) {
var ok bool
if status, ok = ic.ImportStatusBySection[section]; !ok {
status = new(ImportStatus)
status.Title = strings.Title(string(section))
status.Warnings = make(map[WarningType][]string)
ic.ImportStatusBySection[section] = status
}
return status
}
const (
GlobPacks = "*"
// ImportPackName is a custom pack name used for a pack we create to
// hold imported scheduled queries.
ImportPackName = "imported"
)
// QueryDetails represents the query objects used in the packs and the
// schedule section of an osquery configuration.
type QueryDetails struct {
Query string `json:"query"`
Interval uint `json:"interval"`
// Optional fields
Removed *bool `json:"removed"`
Platform *string `json:"platform"`
Version *string `json:"version"`
Shard *uint `json:"shard"`
Snapshot *bool `json:"snapshot"`
}
// PackDetails represents the "packs" section of an osquery configuration
// file.
type PackDetails struct {
Queries QueryNameToQueryDetailsMap `json:"queries"`
Shard *uint `json:"shard"`
Version *string `json:"version"`
Platform string `json:"platform"`
Discovery []string `json:"discovery"`
}
// YARAConfig yara configuration maps keys to lists of files.
// See https://osquery.readthedocs.io/en/stable/deployment/yara/
type YARAConfig struct {
Signatures map[string][]string `json:"signatures"`
FilePaths map[string][]string `json:"file_paths"`
}
// Decorator section of osquery config each section contains rows of decorator
// queries.
type DecoratorConfig struct {
Load []string `json:"load"`
Always []string `json:"always"`
/*
Interval maps a string representation of a numeric interval to a set
of decorator queries.
{
"interval": {
"3600": [
"SELECT total_seconds FROM uptime;"
]
}
}
*/
Interval map[string][]string `json:"interval"`
}
type OptionNameToValueMap map[string]interface{}
type QueryNameToQueryDetailsMap map[string]QueryDetails
type PackNameMap map[string]interface{}
type FIMCategoryToPaths map[string][]string
type PackNameToPackDetails map[string]PackDetails
// ImportConfig is a representation of an Osquery configuration. Osquery
// documentation has further details.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type ImportConfig struct {
// Options is a map of option name to a value which can be an int,
// bool, or string.
Options OptionNameToValueMap `json:"options"`
// Schedule is a map of query names to details
Schedule QueryNameToQueryDetailsMap `json:"schedule"`
// Packs is a map of pack names to either PackDetails, or a string
// containing a file path with a pack config. If a string, we expect
// PackDetails to be stored in ExternalPacks.
Packs PackNameMap `json:"packs"`
// FileIntegrityMonitoring file integrity monitoring information.
// See https://osquery.readthedocs.io/en/stable/deployment/file-integrity-monitoring/
FileIntegrityMonitoring FIMCategoryToPaths `json:"file_paths"`
// YARA configuration
YARA *YARAConfig `json:"yara"`
Decorators *DecoratorConfig `json:"decorators"`
// ExternalPacks are packs referenced when an item in the Packs map references
// an external file. The PackName here must match the PackName in the Packs map.
ExternalPacks PackNameToPackDetails `json:"-"`
// GlobPackNames lists pack names that are globbed.
GlobPackNames []string `json:"glob"`
}
func (ic *ImportConfig) fetchGlobPacks(packs *PackNameToPackDetails) error {
for _, packName := range ic.GlobPackNames {
pack, ok := ic.ExternalPacks[packName]
if !ok {
return fmt.Errorf("glob pack '%s' details not found", packName)
}
(*packs)[packName] = pack
}
return nil
}
// CollectPacks consolidates packs, globbed packs and external packs.
func (ic *ImportConfig) CollectPacks() (PackNameToPackDetails, error) {
result := make(PackNameToPackDetails)
for packName, packContent := range ic.Packs {
// special case handling for Globbed packs
if packName == GlobPacks {
if err := ic.fetchGlobPacks(&result); err != nil {
return nil, err
}
continue
}
// content can either be a file path, in which case we expect to find
// pack in ExternalPacks, or pack details
switch content := packContent.(type) {
case string:
pack, ok := ic.ExternalPacks[packName]
if !ok {
return nil, fmt.Errorf("external pack '%s' details not found", packName)
}
result[packName] = pack
case PackDetails:
result[packName] = content
default:
return nil, errors.New("unexpected pack content")
}
}
return result, nil
}

78
server/kolide/import_config_test.go Normal file
View file

@ -0,0 +1,78 @@
package kolide
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPackNameMapUnmarshal(t *testing.T) {
pnm := PackNameMap{
"path": "/this/is/a/path",
"details": PackDetails{
Queries: QueryNameToQueryDetailsMap{
"q1": QueryDetails{
Query: "select from foo",
Interval: 100,
Removed: new(bool),
Platform: strptr("linux"),
Shard: new(uint),
Snapshot: new(bool),
},
},
Discovery: []string{
"select from something",
},
},
}
b, _ := json.Marshal(pnm)
actual := make(PackNameMap)
err := json.Unmarshal(b, &actual)
require.Nil(t, err)
assert.Len(t, actual, 2)
pnm = PackNameMap{
"path": "/this/is/a/path",
"details": PackDetails{
Queries: QueryNameToQueryDetailsMap{
"q1": QueryDetails{
Query: "select from foo",
Interval: 100,
Removed: new(bool),
Platform: strptr("linux"),
Shard: new(uint),
Snapshot: new(bool),
},
},
Shard: uintptr(float64(10)),
Version: strptr("1.0"),
Platform: "linux",
Discovery: []string{
"select from something",
},
},
"details2": PackDetails{
Queries: QueryNameToQueryDetailsMap{
"q1": QueryDetails{
Query: "select from bar",
Interval: 100,
Removed: new(bool),
Platform: strptr("linux"),
Shard: new(uint),
Snapshot: new(bool),
},
},
Shard: uintptr(float64(10)),
Version: strptr("1.0"),
Platform: "linux",
},
}
b, _ = json.Marshal(pnm)
actual = make(PackNameMap)
err = json.Unmarshal(b, &actual)
require.Nil(t, err)
assert.Len(t, actual, 3)
}

102
server/kolide/import_config_unmarshaler.go Normal file
View file

@ -0,0 +1,102 @@
package kolide
import (
"encoding/json"
"errors"
)
// UnmarshalJSON custom unmarshaling for PackNameMap will determine whether
// the pack section of an osquery config file refers to a file path, or
// pack details. Pack details are unmarshalled into into PackDetails structure
// as oppossed to nested map[string]interface{}
func (pnm PackNameMap) UnmarshalJSON(b []byte) error {
var temp map[string]interface{}
err := json.Unmarshal(b, &temp)
if err != nil {
return err
}
for key, val := range temp {
switch t := val.(type) {
case string:
pnm[key] = t
case map[string]interface{}:
pnm[key] = unmarshalPackDetails(t)
default:
return errors.New("can't unmarshal json")
}
}
return nil
}
func strptr(v interface{}) *string {
if v == nil {
return nil
}
s := new(string)
*s = v.(string)
return s
}
func boolptr(v interface{}) *bool {
if v == nil {
return nil
}
b := new(bool)
*b = v.(bool)
return b
}
func uintptr(v interface{}) *uint {
if v == nil {
return nil
}
i := new(uint)
*i = uint(v.(float64))
return i
}
func unmarshalPackDetails(v map[string]interface{}) PackDetails {
return PackDetails{
Queries: unmarshalQueryDetails(v["queries"]),
Shard: uintptr(v["shard"]),
Version: strptr(v["version"]),
Platform: v["platform"].(string),
Discovery: unmarshalDiscovery(v["discovery"]),
}
}
func unmarshalDiscovery(val interface{}) []string {
var result []string
if val == nil {
return result
}
v := val.([]interface{})
for _, val := range v {
result = append(result, val.(string))
}
return result
}
func unmarshalQueryDetails(v interface{}) QueryNameToQueryDetailsMap {
result := make(QueryNameToQueryDetailsMap)
if v == nil {
return result
}
for qn, details := range v.(map[string]interface{}) {
result[qn] = unmarshalQueryDetail(details)
}
return result
}
func unmarshalQueryDetail(val interface{}) QueryDetails {
v := val.(map[string]interface{})
return QueryDetails{
Query: v["query"].(string),
Interval: uint(v["interval"].(float64)),
Removed: boolptr(v["removed"]),
Platform: strptr(v["platform"]),
Version: strptr(v["version"]),
Shard: uintptr(v["shard"]),
Snapshot: boolptr(v["snapshot"]),
}
}

13
server/kolide/options.go
View file

@ -140,6 +140,19 @@ func (opt *Option) SetValue(v interface{}) {
opt.Value.Val = v
}
func (opt *Option) SameType(compare interface{}) bool {
switch compare.(type) {
case float64:
return opt.Type == OptionTypeInt
case string:
return opt.Type == OptionTypeString
case bool:
return opt.Type == OptionTypeBool
default:
return false
}
}
// OptionSet returns true if the option has a value assigned to it
func (opt *Option) OptionSet() bool {
return opt.Value.Val != nil

3
server/kolide/packs.go
View file

@ -20,6 +20,9 @@ type PackStore interface {
// ListPacks lists all packs in the datastore.
ListPacks(opt ListOptions) ([]*Pack, error)
// PackByName fetches pack if it exists, if the pack
// exists the bool return value is true
PackByName(name string) (*Pack, bool, error)
// AddLabelToPack adds an existing label to an existing pack, both by ID.
AddLabelToPack(lid, pid uint) error

26
server/kolide/queries.go
View file

@ -1,10 +1,6 @@
package kolide
import (
"time"
"golang.org/x/net/context"
)
import "golang.org/x/net/context"
type QueryStore interface {
// NewQuery creates a new query object in thie datastore. The returned
@ -24,6 +20,9 @@ type QueryStore interface {
// ListQueries returns a list of queries with the provided sorting and
// paging options. Associated packs should also be loaded.
ListQueries(opt ListOptions) ([]*Query, error)
// QueryByName looks up a query by name, the second bool is true if a query
// by the name exists.
QueryByName(name string) (*Query, bool, error)
}
type QueryService interface {
@ -63,20 +62,3 @@ type Query struct {
// table in the MySQL backend.
Packs []Pack `json:"packs" db:"-"`
}
type DecoratorType int
const (
DecoratorLoad DecoratorType = iota
DecoratorAlways
DecoratorInterval
)
type Decorator struct {
ID uint
CreatedAt time.Time
UpdatedAt time.Time
Type DecoratorType
Interval int
Query string
}

1
server/kolide/service.go
View file

@ -15,4 +15,5 @@ type Service interface {
TargetService
ScheduledQueryService
OptionService
ImportConfigService
}

31
server/kolide/yara.go Normal file
View file

@ -0,0 +1,31 @@
package kolide
// YARAFilePaths represents the files_path section of an osquery config. The
// key maps to file_paths section_name and maps to one or more YARA signature
// group names
type YARAFilePaths map[string][]string
type YARAStore interface {
// NewYARASignatureGroup creates a new mapping of a name to
// a group of YARA signatures
NewYARASignatureGroup(*YARASignatureGroup) (*YARASignatureGroup, error)
// NewYARAFilePath maps a named set of files to one or more
// groups of YARA signatures
NewYARAFilePath(fileSectionName, sigGroupName string) error
// YARASection creates the osquery configuration YARA section
YARASection() (*YARASection, error)
}
// YARASignatureGroup maps a name to a group of YARA Signatures
// See https://osquery.readthedocs.io/en/stable/deployment/yara/
type YARASignatureGroup struct {
ID uint
SignatureName string `db:"signature_name"`
Paths []string `db:"-"`
}
// YARASection represents the osquery config for YARA
type YARASection struct {
Signatures map[string][]string `json:"signatures"`
FilePaths map[string][]string `json:"file_paths"`
}

3
server/mock/datastore.go
View file

@ -18,6 +18,9 @@ type Store struct {
kolide.QueryStore
kolide.OptionStore
kolide.ScheduledQueryStore
kolide.DecoratorStore
kolide.FileIntegrityMonitoringStore
kolide.YARAStore
InviteStore
UserStore

41
server/service/endpoint_import_config.go Normal file
View file

@ -0,0 +1,41 @@
package service
import (
"github.com/go-kit/kit/endpoint"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
type importRequest struct {
// Config contains a JSON osquery config supplied by the end user
Config string `json:"config"`
// ExternalPackConfigs contains a map of external Pack configs keyed by
// Pack name, this includes external packs referenced by the globbing
// feature. Not in the case of globbed packs, we expect the user to
// generate unique pack names since we don't know what they are, these
// names must be included in the GlobPackNames field so that we can
// validate that they've been accounted for.
ExternalPackConfigs map[string]string `json:"external_pack_configs"`
// GlobPackNames list of user generated names for external packs
// referenced by the glob feature, the JSON for the globbed packs
// is stored in ExternalPackConfigs keyed by the GlobPackName
GlobPackNames []string `json:"glob_pack_names"`
}
type importResponse struct {
Response *kolide.ImportConfigResponse `json:"response,omitempty"`
Err error `json:"error,omitempty"`
}
func (ir importResponse) error() error { return ir.Err }
func makeImportConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
config := request.(kolide.ImportConfig)
resp, err := svc.ImportConfig(ctx, &config)
if err != nil {
return importResponse{Err: err}, nil
}
return importResponse{Response: resp}, nil
}
}

108
server/service/endpoint_import_config_test.go Normal file
View file

@ -0,0 +1,108 @@
package service
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"testing"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testImportConfigWithGlob(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"glob\":[\"globpack\"],\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}",
"globpack": "{\"discovery\":[\"select pid from processes where name = 'zip';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from other;\",\"interval\":1200,\"description\":\"Check other.\"}}}"
},
"glob_pack_names": ["globpack"]
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var impResponse importResponse
err = json.NewDecoder(resp.Body).Decode(&impResponse)
require.Nil(t, err)
assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.PacksSection].ImportCount)
}
func testImportConfigWithMissingGlob(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"*\":\"/path/to/glob/*\",\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}"
}
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var v mockValidationError
err = json.NewDecoder(resp.Body).Decode(&v)
require.Nil(t, err)
require.Len(t, v.Errors, 1)
assert.Equal(t, "missing glob packs", v.Errors[0].Reason)
}
func testImportConfig(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3600\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}",
"external_pack_configs": {
"external_pack": "{\"discovery\":[\"select pid from processes where name = 'baz';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"something\":{\"query\":\"select * from something;\",\"interval\":1200,\"description\":\"Check something.\"}}}"
}
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var impResponse importResponse
err = json.NewDecoder(resp.Body).Decode(&impResponse)
require.Nil(t, err)
assert.Equal(t, 2, impResponse.Response.ImportStatusBySection[kolide.YARASigSection].ImportCount)
assert.Equal(t, 4, impResponse.Response.ImportStatusBySection[kolide.DecoratorsSection].ImportCount)
}
func testImportConfigMissingExternal(t *testing.T, r *testResource) {
testJSON := `
{
"config": "{\"options\":{\"host_identifier\":\"hostname\",\"schedule_splay_percent\":10},\"schedule\":{\"macosx_kextstat\":{\"query\":\"SELECT * FROM kernel_extensions;\",\"interval\":10},\"foobar\":{\"query\":\"SELECT foo, bar, pid FROM foobar_table;\",\"interval\":600}},\"packs\":{\"external_pack\":\"/path/to/external_pack.conf\",\"internal_pack\":{\"discovery\":[\"select pid from processes where name = 'foobar';\",\"select count(*) from users where username like 'www%';\"],\"platform\":\"linux\",\"version\":\"1.5.2\",\"queries\":{\"active_directory\":{\"query\":\"select * from ad_config;\",\"interval\":1200,\"description\":\"Check each user's active directory cached settings.\"}}}},\"decorators\":{\"load\":[\"SELECT version FROM osquery_info\",\"SELECT uuid AS host_uuid FROM system_info\"],\"always\":[\"SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1;\"],\"interval\":{\"3603\":[\"SELECT total_seconds AS uptime FROM uptime;\"]}},\"yara\":{\"signatures\":{\"sig_group_1\":[\"/Users/wxs/sigs/foo.sig\",\"/Users/wxs/sigs/bar.sig\"],\"sig_group_2\":[\"/Users/wxs/sigs/baz.sig\"]},\"file_paths\":{\"system_binaries\":[\"sig_group_1\"],\"tmp\":[\"sig_group_1\",\"sig_group_2\"]}},\"file_paths\":{\"system_binaries\":[\"/usr/bin/%\",\"/usr/sbin/%\"],\"tmp\":[\"/Users/%/tmp/%%\",\"/tmp/%\"]}}"
}
`
buff := bytes.NewBufferString(testJSON)
req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/osquery/config/import", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.userToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var v mockValidationError
err = json.NewDecoder(resp.Body).Decode(&v)
require.Nil(t, err)
require.Len(t, v.Errors, 2)
assert.Equal(t, "missing content for 'external_pack'", v.Errors[0].Reason)
assert.Equal(t, "interval '3603' must be divisible by 60", v.Errors[1].Reason)
}

4
server/service/endpoint_test.go
View file

@ -97,6 +97,10 @@ var testFunctions = [...]func(*testing.T, *testResource){
testGetOptions,
testModifyOptions,
testModifyOptionsValidationFail,
testImportConfig,
testImportConfigMissingExternal,
testImportConfigWithMissingGlob,
testImportConfigWithGlob,
}
func TestEndpoints(t *testing.T) {

6
server/service/handler.go
View file

@ -70,6 +70,7 @@ type KolideEndpoints struct {
SearchTargets endpoint.Endpoint
GetOptions endpoint.Endpoint
ModifyOptions endpoint.Endpoint
ImportConfig endpoint.Endpoint
}
// MakeKolideServerEndpoints creates the Kolide API endpoints.
@ -135,6 +136,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint
SearchTargets: authenticatedUser(jwtKey, svc, makeSearchTargetsEndpoint(svc)),
GetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeGetOptionsEndpoint(svc))),
ModifyOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeModifyOptionsEndpoint(svc))),
ImportConfig: authenticatedUser(jwtKey, svc, makeImportConfigEndpoint(svc)),
// Osquery endpoints
EnrollAgent: makeEnrollAgentEndpoint(svc),
@ -201,6 +203,7 @@ type kolideHandlers struct {
SearchTargets http.Handler
GetOptions http.Handler
ModifyOptions http.Handler
ImportConfig http.Handler
}
func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithttp.ServerOption) *kolideHandlers {
@ -263,6 +266,7 @@ func makeKolideKitHandlers(ctx context.Context, e KolideEndpoints, opts []kithtt
SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest),
GetOptions: newServer(e.GetOptions, decodeNoParamsRequest),
ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest),
ImportConfig: newServer(e.ImportConfig, decodeImportConfigRequest),
}
}
@ -363,6 +367,8 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/targets", h.SearchTargets).Methods("POST").Name("search_targets")
r.Handle("/api/v1/kolide/osquery/config/import", h.ImportConfig).Methods("POST").Name("import_config")
r.Handle("/api/v1/osquery/enroll", h.EnrollAgent).Methods("POST").Name("enroll_agent")
r.Handle("/api/v1/osquery/config", h.GetClientConfig).Methods("POST").Name("get_client_config")
r.Handle("/api/v1/osquery/distributed/read", h.GetDistributedQueries).Methods("POST").Name("get_distributed_queries")

2
server/service/service.go
View file

@ -36,7 +36,7 @@ func NewService(ds kolide.Datastore, resultStore kolide.QueryResultStore, logger
osqueryResultLogWriter: logFile(kolideConfig.Osquery.ResultLogFile),
mailService: mailService,
}
svc = validationMiddleware{svc}
svc = validationMiddleware{svc, ds}
return svc, nil
}

421
server/service/service_import_config.go Normal file
View file

@ -0,0 +1,421 @@
package service
import (
"errors"
"strconv"
"strings"
"github.com/kolide/kolide-ose/server/contexts/viewer"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
func (svc service) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
vc, ok := viewer.FromContext(ctx)
if !ok {
return nil, errors.New("internal error, unable to fetch user")
}
if err := svc.importOptions(cfg.Options, resp); err != nil {
return nil, err
}
if err := svc.importPacks(vc.UserID(), cfg, resp); err != nil {
return nil, err
}
if err := svc.importScheduledQueries(vc.UserID(), cfg, resp); err != nil {
return nil, err
}
if err := svc.importDecorators(cfg, resp); err != nil {
return nil, err
}
if err := svc.importFIMSections(cfg, resp); err != nil {
return nil, err
}
return resp, nil
}
func (svc service) importYARA(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
if cfg.YARA != nil {
for sig, paths := range cfg.YARA.Signatures {
ysg := &kolide.YARASignatureGroup{
SignatureName: sig,
Paths: paths,
}
_, err := svc.ds.NewYARASignatureGroup(ysg)
if err != nil {
return err
}
resp.Status(kolide.YARASigSection).ImportCount++
resp.Status(kolide.YARASigSection).Message("imported '%s'", sig)
}
for section, sigs := range cfg.YARA.FilePaths {
for _, sig := range sigs {
err := svc.ds.NewYARAFilePath(section, sig)
if err != nil {
return err
}
}
resp.Status(kolide.YARAFileSection).ImportCount++
resp.Status(kolide.YARAFileSection).Message("imported '%s'", section)
}
}
return nil
}
func (svc service) importFIMSections(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
if cfg.FileIntegrityMonitoring != nil {
for sectionName, paths := range cfg.FileIntegrityMonitoring {
fp := &kolide.FIMSection{
SectionName: sectionName,
Description: "imported",
Paths: paths,
}
_, err := svc.ds.NewFIMSection(fp)
if err != nil {
return err
}
resp.Status(kolide.FilePathsSection).ImportCount++
resp.Status(kolide.FilePathsSection).Message("imported '%s'", sectionName)
}
}
// this has to happen AFTER fim section, because it requires file paths
return svc.importYARA(cfg, resp)
}
func (svc service) importDecorators(cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
if cfg.Decorators != nil {
for _, query := range cfg.Decorators.Load {
decorator := &kolide.Decorator{
Query: query,
Type: kolide.DecoratorLoad,
}
_, err := svc.ds.NewDecorator(decorator)
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported load '%s'", query)
}
for _, query := range cfg.Decorators.Always {
decorator := &kolide.Decorator{
Query: query,
Type: kolide.DecoratorAlways,
}
_, err := svc.ds.NewDecorator(decorator)
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported always '%s'", query)
}
for key, queries := range cfg.Decorators.Interval {
for _, query := range queries {
interval, err := strconv.ParseInt(key, 10, 32)
if err != nil {
return err
}
decorator := &kolide.Decorator{
Query: query,
Type: kolide.DecoratorInterval,
Interval: uint(interval),
}
_, err = svc.ds.NewDecorator(decorator)
if err != nil {
return err
}
resp.Status(kolide.DecoratorsSection).ImportCount++
resp.Status(kolide.DecoratorsSection).Message("imported interval %d '%s'", interval, query)
}
}
}
return nil
}
func (svc service) importScheduledQueries(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
_, ok, err := svc.ds.PackByName(kolide.ImportPackName)
if ok {
resp.Status(kolide.PacksSection).Warning(
kolide.PackDuplicate, "skipped '%s' already exists", kolide.ImportPackName,
)
resp.Status(kolide.PacksSection).SkipCount++
return nil
}
// create import pack to hold imported scheduled queries
pack := &kolide.Pack{
Name: kolide.ImportPackName,
Description: "holds imported scheduled queries",
CreatedBy: uid,
Disabled: false,
}
pack, err = svc.ds.NewPack(pack)
if err != nil {
return err
}
resp.Status(kolide.PacksSection).ImportCount++
resp.Status(kolide.PacksSection).Message("created import pack")
for queryName, queryDetails := range cfg.Schedule {
var query *kolide.Query
query, ok, err = svc.ds.QueryByName(queryName)
// if we find the query check to see if the import query matches the
// query we have, if it doesn't skip it
if ok {
if hashQuery("", query.Query) != hashQuery("", queryDetails.Query) {
resp.Status(kolide.PacksSection).Warning(
kolide.DifferentQuerySameName,
"queries named '%s' have different statements and won't be added to '%s'",
queryName,
pack.Name,
)
continue
}
resp.Status(kolide.QueriesSection).Warning(
kolide.QueryDuplicate, "skipped '%s' different query of same name already exists", queryName,
)
resp.Status(kolide.QueriesSection).SkipCount++
} else {
// if query doesn't exist, create it
query = &kolide.Query{
Name: queryName,
Description: "imported",
Query: queryDetails.Query,
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query)
if err != nil {
return err
}
resp.Status(kolide.QueriesSection).ImportCount++
resp.Status(kolide.QueriesSection).Message(
"imported scheduled query '%s'", query.Name,
)
}
sq := &kolide.ScheduledQuery{
PackID: pack.ID,
QueryID: query.ID,
Interval: queryDetails.Interval,
Snapshot: queryDetails.Snapshot,
Removed: queryDetails.Removed,
Platform: queryDetails.Platform,
Version: queryDetails.Version,
Shard: queryDetails.Shard,
}
_, err = svc.ds.NewScheduledQuery(sq)
if err != nil {
return nil
}
resp.Status(kolide.PacksSection).Message(
"added query '%s' to '%s'", query.Name, pack.Name,
)
}
return nil
}
func (svc service) importPacks(uid uint, cfg *kolide.ImportConfig, resp *kolide.ImportConfigResponse) error {
labelCache := map[string]*kolide.Label{}
packs, err := cfg.CollectPacks()
if err != nil {
return err
}
for packName, packDetails := range packs {
_, ok, err := svc.ds.PackByName(packName)
if err != nil {
return err
}
if ok {
resp.Status(kolide.PacksSection).Warning(
kolide.PackDuplicate, "skipped '%s' already exists", packName,
)
resp.Status(kolide.PacksSection).SkipCount++
continue
}
// import new pack
if packDetails.Shard != nil {
resp.Status(kolide.PacksSection).Warning(
kolide.Unsupported,
"shard for pack '%s'",
packName,
)
}
if packDetails.Version != nil {
resp.Status(kolide.PacksSection).Warning(
kolide.Unsupported,
"version for pack '%s'",
packName,
)
}
pack := &kolide.Pack{
Name: packName,
Description: "Imported pack",
Platform: packDetails.Platform,
}
pack, err = svc.ds.NewPack(pack)
if err != nil {
return err
}
err = svc.createLabelsForPack(pack, &packDetails, labelCache, resp)
if err != nil {
return err
}
err = svc.createQueriesForPack(uid, pack, &packDetails, resp)
if err != nil {
return err
}
resp.Status(kolide.PacksSection).ImportCount++
resp.Status(kolide.PacksSection).Message("imported '%s'", packName)
}
return nil
}
func hashQuery(platform, query string) string {
s := strings.Replace(query, " ", "", -1)
s = strings.Replace(s, "\t", "", -1)
s = strings.Replace(s, "\n", "", -1)
s = strings.Trim(s, ";")
s = platform + s
return strings.ToLower(s)
}
func uniqueImportName() (string, error) {
random, err := kolide.RandomText(12)
if err != nil {
return "", err
}
return "import_" + random, nil
}
func (svc service) createQueriesForPack(uid uint, pack *kolide.Pack, details *kolide.PackDetails,
resp *kolide.ImportConfigResponse) error {
for queryName, queryDetails := range details.Queries {
query, ok, err := svc.ds.QueryByName(queryName)
if err != nil {
return err
}
// if the query isn't already in the database, create it
if !ok {
query = &kolide.Query{
Name: queryName,
Description: "imported",
Query: queryDetails.Query,
Saved: true,
AuthorID: uid,
}
query, err = svc.ds.NewQuery(query)
if err != nil {
return err
}
resp.Status(kolide.QueriesSection).Message(
"created '%s' as part of pack '%s'", queryName, pack.Name,
)
resp.Status(kolide.QueriesSection).ImportCount++
}
// associate query with pack
scheduledQuery := &kolide.ScheduledQuery{
PackID: pack.ID,
QueryID: query.ID,
Interval: queryDetails.Interval,
Platform: queryDetails.Platform,
Snapshot: queryDetails.Snapshot,
Removed: queryDetails.Removed,
Version: queryDetails.Version,
Shard: queryDetails.Shard,
}
_, err = svc.ds.NewScheduledQuery(scheduledQuery)
if err != nil {
return nil
}
resp.Status(kolide.PacksSection).Message("added query '%s'", query.Name)
}
return nil
}
// createLabelsForPack Iterates through discover queries, creates a label for
// each query and assigns it to the pack passed as an argument. Once a Label is created we cache
// it for reuse.
func (svc service) createLabelsForPack(pack *kolide.Pack, details *kolide.PackDetails,
cache map[string]*kolide.Label, resp *kolide.ImportConfigResponse) error {
for _, query := range details.Discovery {
hash := hashQuery(details.Platform, query)
label, ok := cache[hash]
// add existing label to pack
if ok {
err := svc.ds.AddLabelToPack(label.ID, pack.ID)
if err != nil {
return err
}
resp.Status(kolide.PacksSection).Message(
"added label '%s' to pack '%s'", label.Name, pack.Name,
)
continue
}
// create new label and add it to pack
labelName, err := uniqueImportName()
if err != nil {
return err
}
label = &kolide.Label{
Name: labelName,
Query: query,
Description: "imported",
LabelType: kolide.LabelTypeRegular,
Platform: details.Platform,
}
label, err = svc.ds.NewLabel(label)
if err != nil {
return err
}
// hang on to label so we can reuse it for other packs if needed
cache[hash] = label
err = svc.ds.AddLabelToPack(label.ID, pack.ID)
if err != nil {
return err
}
resp.Status(kolide.PacksSection).Message(
"added label '%s' to '%s'", label.Name, pack.Name,
)
}
return nil
}
func (svc service) importOptions(opts kolide.OptionNameToValueMap, resp *kolide.ImportConfigResponse) error {
var updateOptions []kolide.Option
for optName, optValue := range opts {
opt, err := svc.ds.OptionByName(optName)
if err != nil {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionUnknown, "skipped '%s' can't find option", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
if opt.ReadOnly {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionReadonly, "skipped '%s' can't change read only option", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
if opt.OptionSet() {
resp.Status(kolide.OptionsSection).Warning(
kolide.OptionAlreadySet, "skipped '%s' can't change option that is already set", optName,
)
resp.Status(kolide.OptionsSection).SkipCount++
continue
}
opt.SetValue(optValue)
resp.Status(kolide.OptionsSection).Message("set %s value to %v", optName, optValue)
resp.Status(kolide.OptionsSection).ImportCount++
updateOptions = append(updateOptions, *opt)
}
if len(updateOptions) > 0 {
if err := svc.ds.SaveOptions(updateOptions); err != nil {
return err
}
}
return nil
}

340
server/service/service_import_config_test.go Normal file
View file

@ -0,0 +1,340 @@
package service
import (
"testing"
"github.com/kolide/kolide-ose/server/config"
"github.com/kolide/kolide-ose/server/datastore/inmem"
"github.com/kolide/kolide-ose/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func createServiceMockForImport(t *testing.T) *service {
ds, err := inmem.New(config.TestConfig())
require.Nil(t, err)
err = ds.MigrateData()
require.Nil(t, err)
return &service{
ds: ds,
}
}
func TestHashQuery(t *testing.T) {
q1 := `SELECT * FROM t1 INNER JOIN ON
t1.id = t2.t1id
WHERE t1.name = 'foo' `
q2 := "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t1.name = 'foo';"
h1 := hashQuery("platform", q1)
h2 := hashQuery("platform", q2)
assert.Equal(t, h1, h2)
q2 = "SELECT * from t1 INNER JOIN\tON t1.id = t2.t1id WHERE t2.name = 'foo';"
h2 = hashQuery("platform", q2)
assert.NotEqual(t, h1, h2)
}
func TestImportFilePaths(t *testing.T) {
cfg := &kolide.ImportConfig{
FileIntegrityMonitoring: kolide.FIMCategoryToPaths{
"files1": []string{
"path1",
"path2",
},
"files2": []string{
"path3",
},
},
YARA: &kolide.YARAConfig{
Signatures: map[string][]string{
"sig1": []string{
"path4",
"path5",
},
"sig2": []string{
"path6",
},
},
FilePaths: map[string][]string{
"files1": []string{
"sig1",
"sig2",
},
"files2": []string{
"sig1",
},
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importFIMSections(cfg, resp)
require.Nil(t, err)
assert.Equal(t, 2, resp.Status(kolide.FilePathsSection).ImportCount)
sections, err := svc.ds.FIMSections()
require.Nil(t, err)
assert.Len(t, sections, 2)
yara, err := svc.ds.YARASection()
require.Nil(t, err)
assert.Len(t, yara.Signatures, 2)
assert.Len(t, yara.FilePaths, 2)
}
func TestImportDecorators(t *testing.T) {
cfg := &kolide.ImportConfig{
Decorators: &kolide.DecoratorConfig{
Load: []string{
"select from foo",
"select from bar",
},
Always: []string{
"select from always",
},
Interval: map[string][]string{
"100": []string{
"select from 100",
},
"200": []string{
"select from 200",
},
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importDecorators(cfg, resp)
require.Nil(t, err)
assert.Equal(t, 5, resp.Status(kolide.DecoratorsSection).ImportCount)
dec, err := svc.ds.ListDecorators()
require.Nil(t, err)
assert.Len(t, dec, 5)
}
func TestImportScheduledQueries(t *testing.T) {
cfg := &kolide.ImportConfig{
Schedule: kolide.QueryNameToQueryDetailsMap{
"q1": kolide.QueryDetails{
Query: "select pid from processes",
Interval: 60,
Platform: stringPtr("linux"),
},
"q2": kolide.QueryDetails{
Query: "select uid from users",
Interval: 120,
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
},
"q3": kolide.QueryDetails{
Query: "select name from os",
Interval: 240,
Platform: stringPtr("linux"),
Snapshot: boolPtr(true),
},
},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
user := &kolide.User{
Username: "bob",
Password: []byte("secret"),
Email: "bob@something.com",
Admin: false,
AdminForcedPasswordReset: false,
}
user, err := svc.ds.NewUser(user)
require.Nil(t, err)
skipQuery := &kolide.Query{
Name: "q3",
Query: "select version from os",
Description: "should be skipped",
Saved: true,
AuthorID: user.ID,
}
_, err = svc.ds.NewQuery(skipQuery)
require.Nil(t, err)
noskipQuery := &kolide.Query{
Name: "q2",
Query: "select uid from users",
Saved: true,
AuthorID: user.ID,
}
_, err = svc.ds.NewQuery(noskipQuery)
require.Nil(t, err)
err = svc.importScheduledQueries(user.ID, cfg, resp)
require.Nil(t, err)
_, ok, err := svc.ds.QueryByName("q1")
require.Nil(t, err)
require.True(t, ok)
_, ok, err = svc.ds.QueryByName("q2")
require.Nil(t, err)
require.True(t, ok)
_, ok, err = svc.ds.QueryByName("q3")
require.Nil(t, err)
require.True(t, ok)
}
func TestOptionsImportConfig(t *testing.T) {
opts := kolide.OptionNameToValueMap{
"aws_access_key_id": "foo",
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
err := svc.importOptions(opts, resp)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
assert.Equal(t, 1, status.ImportCount)
opt, err := svc.ds.OptionByName("aws_access_key_id")
require.Nil(t, err)
assert.Equal(t, "foo", opt.GetValue())
require.Len(t, status.Messages, 1)
assert.Equal(t, "set aws_access_key_id value to foo", status.Messages[0])
}
func TestOptionsImportConfigWithSkips(t *testing.T) {
opts := kolide.OptionNameToValueMap{
"aws_access_key_id": "foo",
"aws_secret_access_key": "secret",
// this should be skipped because it's already set
"aws_firehose_period": 100,
// these should be skipped because it's read only
"disable_distributed": false,
"pack_delimiter": "x",
// this should be skipped because it's not an option we know about
"wombat": "not venomous",
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
svc := createServiceMockForImport(t)
// set option val, it should be skipped
opt, err := svc.ds.OptionByName("aws_firehose_period")
require.Nil(t, err)
opt.SetValue(23)
err = svc.ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
err = svc.importOptions(opts, resp)
require.Nil(t, err)
status := resp.Status(kolide.OptionsSection)
require.NotNil(t, status)
assert.Equal(t, 2, status.ImportCount)
assert.Equal(t, 4, status.SkipCount)
assert.Len(t, status.Warnings[kolide.OptionAlreadySet], 1)
assert.Len(t, status.Warnings[kolide.OptionReadonly], 2)
assert.Len(t, status.Warnings[kolide.OptionUnknown], 1)
assert.Len(t, status.Messages, 2)
}
func TestPacksImportConfig(t *testing.T) {
svc := createServiceMockForImport(t)
p := &kolide.Pack{
Name: "dup",
}
_, err := svc.ds.NewPack(p)
require.Nil(t, err)
q1 := kolide.QueryDetails{
Query: "select * from foo",
Interval: 100,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
q2 := kolide.QueryDetails{
Query: "select * from bar",
Interval: 50,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
q3 := kolide.QueryDetails{
Query: "select * from baz",
Interval: 500,
Removed: boolPtr(false),
Platform: stringPtr("linux"),
Version: stringPtr("1.0"),
}
importConfig := kolide.ImportConfig{
Packs: kolide.PackNameMap{
"ext1": "/home/usr/ext1.json",
"pack1": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
"q2": q2,
},
Discovery: []string{
"select * from zz",
"select id, xx from yy",
},
},
"dup": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
"q2": q2,
},
},
"*": "/home/usr/packs/*",
},
ExternalPacks: kolide.PackNameToPackDetails{
"ext1": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q1": q1,
},
Discovery: []string{
"select * from zz",
"select a, b, c from processes",
},
},
"ext2": kolide.PackDetails{
Queries: kolide.QueryNameToQueryDetailsMap{
"q3": q3,
},
},
},
GlobPackNames: []string{"ext2"},
}
resp := &kolide.ImportConfigResponse{
ImportStatusBySection: make(map[kolide.ImportSection]*kolide.ImportStatus),
}
user := &kolide.User{
Username: "bob",
Password: []byte("secret"),
Email: "bob@something.com",
Admin: false,
AdminForcedPasswordReset: false,
}
user, err = svc.ds.NewUser(user)
require.Nil(t, err)
packs, err := importConfig.CollectPacks()
require.Nil(t, err)
assert.Len(t, packs, 4)
err = svc.importPacks(user.ID, &importConfig, resp)
require.Nil(t, err)
queries, err := svc.ds.ListQueries(kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, queries, 3)
pack, ok, err := svc.ds.PackByName("pack1")
require.Nil(t, err)
require.True(t, ok)
sqs, err := svc.ds.ListScheduledQueriesInPack(pack.ID, kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, sqs, 2)
labels, err := svc.ds.ListLabels(kolide.ListOptions{})
require.Nil(t, err)
assert.Len(t, labels, 8)
assert.Equal(t, 3, resp.Status(kolide.PacksSection).ImportCount)
assert.Equal(t, 1, resp.Status(kolide.PacksSection).SkipCount)
assert.Equal(t, 3, resp.Status(kolide.QueriesSection).ImportCount)
}

4
server/service/service_invites_test.go
View file

@ -80,6 +80,7 @@ func TestListInvites(t *testing.T) {
}
func setupInviteTest(t *testing.T) (kolide.Service, *mock.Store, *mockMailService) {
ms := new(mock.Store)
ms.UserByEmailFunc = mock.UserWithEmailNotFound()
ms.UserByIDFunc = mock.UserWithID(adminUser)
@ -88,13 +89,12 @@ func setupInviteTest(t *testing.T) (kolide.Service, *mock.Store, *mockMailServic
KolideServerURL: "https://acme.co",
})
mailer := &mockMailService{SendEmailFn: func(e kolide.Email) error { return nil }}
svc := validationMiddleware{service{
ds: ms,
config: config.TestConfig(),
mailService: mailer,
clock: clock.NewMockClock(),
}}
}, ms}
return svc, ms, mailer
}

34
server/service/transport_import_config.go Normal file
View file

@ -0,0 +1,34 @@
package service
import (
"encoding/json"
"net/http"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
func decodeImportConfigRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var req importRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
return nil, err
}
// Unmarshal main config
conf := kolide.ImportConfig{
Packs: make(kolide.PackNameMap),
ExternalPacks: make(kolide.PackNameToPackDetails),
}
if err := json.Unmarshal([]byte(req.Config), &conf); err != nil {
return nil, err
}
// Unmarshal external packs
for packName, packConfig := range req.ExternalPackConfigs {
var pack kolide.PackDetails
if err := json.Unmarshal([]byte(packConfig), &pack); err != nil {
return nil, err
}
conf.ExternalPacks[packName] = pack
}
conf.GlobPackNames = req.GlobPackNames
return conf, nil
}

4
server/service/util_test.go
View file

@ -111,3 +111,7 @@ func stringPtr(s string) *string {
func boolPtr(b bool) *bool {
return &b
}
func uintPtr(n uint) *uint {
return &n
}

113
server/service/validation_import_config.go Normal file
View file

@ -0,0 +1,113 @@
package service
import (
"strconv"
"github.com/kolide/kolide-ose/server/kolide"
"golang.org/x/net/context"
)
func (vm validationMiddleware) ImportConfig(ctx context.Context, cfg *kolide.ImportConfig) (*kolide.ImportConfigResponse, error) {
var invalid invalidArgumentError
vm.validateConfigOptions(cfg, &invalid)
vm.validatePacks(cfg, &invalid)
vm.validateDecorator(cfg, &invalid)
vm.validateYARA(cfg, &invalid)
if invalid.HasErrors() {
return nil, invalid
}
return vm.Service.ImportConfig(ctx, cfg)
}
func (vm validationMiddleware) validateYARA(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.YARA != nil {
if cfg.YARA.FilePaths == nil {
argErrs.Append("yara", "missing file_paths")
return
}
if cfg.YARA.Signatures == nil {
argErrs.Append("yara", "missing signatures")
}
for fileSection, sigs := range cfg.YARA.FilePaths {
if cfg.FileIntegrityMonitoring == nil {
argErrs.Append("yara", "missing file paths section")
return
}
if _, ok := cfg.FileIntegrityMonitoring[fileSection]; !ok {
argErrs.Appendf("yara", "missing referenced file_paths section '%s'", fileSection)
}
for _, sig := range sigs {
if _, ok := cfg.YARA.Signatures[sig]; !ok {
argErrs.Appendf(
"yara",
"missing signature '%s' referenced in '%s'",
sig,
fileSection,
)
}
}
}
}
}
func (vm validationMiddleware) validateDecorator(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Decorators != nil {
for str := range cfg.Decorators.Interval {
val, err := strconv.ParseInt(str, 10, 32)
if err != nil {
argErrs.Appendf("decorators", "interval '%s' must be an integer", str)
continue
}
if val%60 != 0 {
argErrs.Appendf("decorators", "interval '%d' must be divisible by 60", val)
}
}
}
}
func (vm validationMiddleware) validateConfigOptions(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Options != nil {
for optName, optValue := range cfg.Options {
opt, err := vm.ds.OptionByName(string(optName))
if err != nil {
// skip validation for an option we don't know about, this will generate
// a warning in the service layer
continue
}
if !opt.SameType(optValue) {
argErrs.Appendf("options", "invalid type for '%s'", optName)
}
}
}
}
func (vm validationMiddleware) validatePacks(cfg *kolide.ImportConfig, argErrs *invalidArgumentError) {
if cfg.Packs != nil {
for packName, pack := range cfg.Packs {
// if glob packs is defined we expect at least one external pack
if packName == kolide.GlobPacks {
if len(cfg.GlobPackNames) == 0 {
argErrs.Append("external_packs", "missing glob packs")
continue
}
// make sure that each glob pack has JSON content
for _, p := range cfg.GlobPackNames {
if _, ok := cfg.ExternalPacks[p]; !ok {
argErrs.Appendf("external_packs", "missing content for '%s'", p)
}
}
continue
}
// if value is a string we expect a file path, in this case, the user has to supply the
// contents of said file which we store in ExternalPacks, if it's not there we need to
// raise an error
switch pack.(type) {
case string:
if _, ok := cfg.ExternalPacks[packName]; !ok {
argErrs.Appendf("external_packs", "missing content for '%s'", packName)
}
}
}
}
}

7
server/service/validation_users.go
View file

@ -10,6 +10,7 @@ import (
type validationMiddleware struct {
kolide.Service
ds kolide.Datastore
}
func (mw validationMiddleware) NewUser(ctx context.Context, p kolide.UserPayload) (*kolide.User, error) {
@ -137,6 +138,12 @@ func (e *invalidArgumentError) Append(name, reason string) {
reason: reason,
})
}
func (e *invalidArgumentError) Appendf(name, reasonFmt string, args ...interface{}) {
*e = append(*e, invalidArgument{
name: name,
reason: fmt.Sprintf(reasonFmt, args...),
})
}
func (e *invalidArgumentError) HasErrors() bool {
return len(*e) != 0