Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-21 07:58:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 110

Clean up unused DB tables, migrations, and code (#2258)

Browse source 
This PR removes unused types, code, DB tables, and associated migrations that are unused since Fleet 2.0.

An existing migration was refactored, and should remain compatible with both existing and new Fleet installations.
This commit is contained in:
Zachary Wasserman 2020-07-08 13:02:18 -07:00 committed by GitHub
parent def51ae930
commit fd61dcab67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 378 additions and 2406 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
server/datastore/datastore_options_test.go
View file

@ -1,135 +0,0 @@
package datastore
import (
"reflect"
"sort"
"testing"
"github.com/kolide/fleet/server/datastore/internal/appstate"
"github.com/kolide/fleet/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testOptions(t *testing.T, ds kolide.Datastore) {
if ds.Name() == "inmem" {
t.Skip("inmem is being depracated")
}
require.Nil(t, ds.MigrateData())
// were options pre-loaded?
opts, err := ds.ListOptions()
require.Nil(t, err)
assert.Len(t, opts, len(appstate.Options()))
opt, err := ds.OptionByName("aws_profile_name")
require.Nil(t, err)
assert.False(t, opt.OptionSet())
// try to save non-readonly list of options with same values, it should not err out
var writableOpts []kolide.Option
for _, o := range opts {
if !o.ReadOnly {
writableOpts = append(writableOpts, o)
}
}
err = ds.SaveOptions(writableOpts)
assert.Nil(t, err)
opt, err = ds.OptionByName("aws_access_key_id")
require.Nil(t, err)
require.NotNil(t, opt)
opt2, err := ds.Option(opt.ID)
require.Nil(t, err)
require.NotNil(t, opt2)
assert.True(t, reflect.DeepEqual(opt, opt2))
opt.SetValue("somekey")
err = ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
opt, err = ds.Option(opt.ID)
require.Nil(t, err)
assert.Equal(t, "somekey", opt.GetValue())
// can't change a read only option
opt, err = ds.OptionByName("disable_distributed")
require.Nil(t, err)
opt.SetValue(true)
err = ds.SaveOptions([]kolide.Option{*opt})
assert.NotNil(t, err)
// check that it didn't change
opt, err = ds.OptionByName("disable_distributed")
require.Nil(t, err)
require.False(t, opt.GetValue().(bool))
opt, _ = ds.OptionByName("aws_profile_name")
assert.False(t, opt.OptionSet())
opt.SetValue("zip")
opt2, _ = ds.OptionByName("disable_distributed")
assert.Equal(t, false, opt2.GetValue())
opt2.SetValue(true)
modList := []kolide.Option{*opt, *opt2}
// The aws access key option can be saved but because the disable_events can't
// be we want to verify that the whole transaction is rolled back
tx, err := ds.Begin()
require.Nil(t, err)
err = ds.SaveOptions(modList, kolide.HasTransaction(tx))
assert.NotNil(t, err)
err = tx.Rollback()
require.Nil(t, err)
opt2, err = ds.OptionByName("disable_distributed")
require.Nil(t, err)
assert.Equal(t, false, opt2.GetValue())
opt, err = ds.OptionByName("aws_profile_name")
require.Nil(t, err)
assert.False(t, opt.OptionSet())
}
func testOptionsToConfig(t *testing.T, ds kolide.Datastore) {
require.Nil(t, ds.MigrateData())
resp, err := ds.GetOsqueryConfigOptions()
require.Nil(t, err)
assert.Len(t, resp, 8)
opt, _ := ds.OptionByName("aws_profile_name")
assert.False(t, opt.OptionSet())
opt.SetValue("zip")
err = ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
resp, err = ds.GetOsqueryConfigOptions()
require.Nil(t, err)
assert.Len(t, resp, 9)
assert.Equal(t, "zip", resp["aws_profile_name"])
}
func testResetOptions(t *testing.T, ds kolide.Datastore) {
if ds.Name() == "inmem" {
t.Skip("inmem is being deprecated, test skipped")
}
require.Nil(t, ds.MigrateData())
// get originals
originals, err := ds.ListOptions()
require.Nil(t, err)
sort.SliceStable(originals, func(i, j int) bool { return originals[i].ID < originals[j].ID })
// grab and options, change it, save it, verify that saved
opt, err := ds.OptionByName("aws_profile_name")
require.Nil(t, err)
assert.False(t, opt.OptionSet())
opt.SetValue("zip")
err = ds.SaveOptions([]kolide.Option{*opt})
require.Nil(t, err)
opt, _ = ds.OptionByName("aws_profile_name")
assert.Equal(t, "zip", opt.GetValue())
resetOptions, err := ds.ResetOptions()
require.Nil(t, err)
sort.SliceStable(resetOptions, func(i, j int) bool { return resetOptions[i].ID < resetOptions[j].ID })
require.Equal(t, len(originals), len(resetOptions))
for i, _ := range originals {
require.Equal(t, originals[i].ID, resetOptions[i].ID)
require.Equal(t, originals[i].GetValue(), resetOptions[i].GetValue())
require.Equal(t, originals[i].Name, resetOptions[i].Name)
}
}

5
server/datastore/datastore_test.go
View file

@ -64,12 +64,8 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testNewScheduledQuery,
testListScheduledQueriesInPack,
testCascadingDeletionOfQueries,
testOptions,
testOptionsToConfig,
testGetPackByName,
testGetQueryByName,
testFileIntegrityMonitoring,
testYARAStore,
testAddLabelToPackTwice,
testGenerateHostStatusStatistics,
testMarkHostSeen,
@ -83,7 +79,6 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
testUnicode,
testCountHostsInTargets,
testHostStatus,
testResetOptions,
testApplyOsqueryOptions,
testApplyOsqueryOptionsNoOverrides,
testOsqueryOptionsForHost,

107
server/datastore/datastore_yara_test.go
View file

@ -1,107 +0,0 @@
package datastore
import (
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testYARAStore(t *testing.T, ds kolide.Datastore) {
ysg := &kolide.YARASignatureGroup{
SignatureName: "sig1",
Paths: []string{
"path1",
"path2",
},
}
ysg, err := ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
require.True(t, ysg.ID > 0)
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err = ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
err = ds.NewYARAFilePath("fp1", "sig1")
require.Nil(t, err)
yaraSection, err := ds.YARASection()
require.Nil(t, err)
require.Len(t, yaraSection.FilePaths, 1)
assert.Len(t, yaraSection.FilePaths["fp1"], 1)
require.Len(t, yaraSection.Signatures, 1)
assert.Len(t, yaraSection.Signatures["sig1"], 2)
ysg = &kolide.YARASignatureGroup{
SignatureName: "sig2",
Paths: []string{
"path3",
},
}
ysg, err = ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
yaraSection, err = ds.YARASection()
require.Nil(t, err)
assert.Len(t, yaraSection.Signatures["sig2"], 1)
}
func testYARATransactions(t *testing.T, ds kolide.Datastore) {
if ds.Name() == "inmem" {
t.Skip("not implemented for inmem")
}
ysg := &kolide.YARASignatureGroup{
SignatureName: "sig1",
Paths: []string{
"path1",
"path2",
},
}
ysg, err := ds.NewYARASignatureGroup(ysg)
require.Nil(t, err)
require.True(t, ysg.ID > 0)
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err = ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
tx, err := ds.Begin()
require.Nil(t, err)
err = ds.NewYARAFilePath("fp1", "sig1", kolide.HasTransaction(tx))
require.Nil(t, err)
err = tx.Rollback()
require.Nil(t, err)
yaraSection, err := ds.YARASection()
require.Nil(t, err)
require.NotNil(t, yaraSection)
// there shouldn't be any file paths because we rolled back the transaction
require.Len(t, yaraSection.FilePaths, 0)
// try it again
tx, err = ds.Begin()
require.Nil(t, err)
err = ds.NewYARAFilePath("fp1", "sig1", kolide.HasTransaction(tx))
require.Nil(t, err)
err = tx.Commit()
require.Nil(t, err)
yaraSection, err = ds.YARASection()
require.Nil(t, err)
require.NotNil(t, yaraSection)
// file path should exist because we committed
require.Len(t, yaraSection.FilePaths, 1)
}

45
server/datastore/file_integrity_monitoring_test.go
View file

@ -1,45 +0,0 @@
package datastore
import (
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testFileIntegrityMonitoring(t *testing.T, ds kolide.Datastore) {
fp := &kolide.FIMSection{
SectionName: "fp1",
Paths: []string{
"path1",
"path2",
"path3",
},
}
fp, err := ds.NewFIMSection(fp)
require.Nil(t, err)
assert.True(t, fp.ID > 0)
fp = &kolide.FIMSection{
SectionName: "fp2",
Paths: []string{
"path4",
"path5",
},
}
_, err = ds.NewFIMSection(fp)
require.Nil(t, err)
actual, err := ds.FIMSections()
require.Nil(t, err)
assert.Len(t, actual, 2)
assert.Len(t, actual["fp1"], 3)
assert.Len(t, actual["fp2"], 2)
err = ds.ClearFIMSections()
require.Nil(t, err)
fs, err := ds.FIMSections()
assert.Nil(t, err)
assert.Len(t, fs, 0)
}

50
server/datastore/inmem/decorators.go
View file

@ -1,50 +0,0 @@
package inmem
import "github.com/kolide/fleet/server/kolide"
func (d *Datastore) SaveDecorator(dec *kolide.Decorator, opts ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.decorators[dec.ID]; !ok {
return notFound("Decorator")
}
d.decorators[dec.ID] = dec
return nil
}
func (d *Datastore) NewDecorator(decorator *kolide.Decorator, opts ...kolide.OptionalArg) (*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
decorator.ID = d.nextID(decorator)
d.decorators[decorator.ID] = decorator
return decorator, nil
}
func (d *Datastore) DeleteDecorator(id uint) error {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.decorators[id]; !ok {
return notFound("Decorator").WithID(id)
}
delete(d.decorators, id)
return nil
}
func (d *Datastore) Decorator(id uint) (*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
if result, ok := d.decorators[id]; ok {
return result, nil
}
return nil, notFound("Decorator").WithID(id)
}
func (d *Datastore) ListDecorators(opts ...kolide.OptionalArg) ([]*kolide.Decorator, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
var result []*kolide.Decorator
for _, dec := range d.decorators {
result = append(result, dec)
}
return result, nil
}

27
server/datastore/inmem/file_integrity_monitoring.go
View file

@ -1,27 +0,0 @@
package inmem
import (
"github.com/kolide/fleet/server/kolide"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
fp.ID = d.nextID(fp)
d.filePaths[fp.ID] = fp
return fp, nil
}
func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
result := make(kolide.FIMSections)
for _, filePath := range d.filePaths {
result[filePath.SectionName] = append(result[filePath.SectionName], filePath.Paths...)
}
return result, nil
}
func (d *Datastore) ClearFIMSections() error {
panic("inmem is being deprecated")
}

22
server/datastore/inmem/inmem.go
View file

@ -8,7 +8,6 @@ import (
"time"
"github.com/kolide/fleet/server/config"
"github.com/kolide/fleet/server/datastore/internal/appstate"
"github.com/kolide/fleet/server/kolide"
"github.com/patrickmn/sortutil"
)
@ -32,11 +31,6 @@ type Datastore struct {
distributedQueryExecutions map[uint]kolide.DistributedQueryExecution
distributedQueryCampaigns map[uint]kolide.DistributedQueryCampaign
distributedQueryCampaignTargets map[uint]kolide.DistributedQueryCampaignTarget
options map[uint]*kolide.Option
decorators map[uint]*kolide.Decorator
filePaths map[uint]*kolide.FIMSection
yaraFilePaths kolide.YARAFilePaths
yaraSignatureGroups map[uint]*kolide.YARASignatureGroup
appConfig *kolide.AppConfig
config *config.KolideConfig
@ -105,27 +99,11 @@ func (d *Datastore) MigrateTables() error {
d.distributedQueryExecutions = make(map[uint]kolide.DistributedQueryExecution)
d.distributedQueryCampaigns = make(map[uint]kolide.DistributedQueryCampaign)
d.distributedQueryCampaignTargets = make(map[uint]kolide.DistributedQueryCampaignTarget)
d.options = make(map[uint]*kolide.Option)
d.decorators = make(map[uint]*kolide.Decorator)
d.filePaths = make(map[uint]*kolide.FIMSection)
d.yaraFilePaths = make(kolide.YARAFilePaths)
d.yaraSignatureGroups = make(map[uint]*kolide.YARASignatureGroup)
return nil
}
func (d *Datastore) MigrateData() error {
for _, initData := range appstate.Options() {
opt := kolide.Option{
Name: initData.Name,
Value: kolide.OptionValue{Val: initData.Value},
Type: initData.Type,
ReadOnly: initData.ReadOnly,
}
opt.ID = d.nextID(opt)
d.options[opt.ID] = &opt
}
d.appConfig = &kolide.AppConfig{
ID: 1,
SMTPEnableTLS: true,

90
server/datastore/inmem/options.go
View file

@ -1,90 +0,0 @@
package inmem
import (
"fmt"
"github.com/kolide/fleet/server/kolide"
"github.com/patrickmn/sortutil"
)
func (d *Datastore) ResetOptions() ([]kolide.Option, error) {
panic("inmem is being deprecated")
}
func (d *Datastore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
for _, opt := range d.options {
if opt.Name == name {
result := *opt
return &result, nil
}
}
return nil, notFound("options")
}
type optPair struct {
newOpt kolide.Option
existingOpt *kolide.Option
}
func (d *Datastore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
var validPairs []optPair
for _, opt := range opts {
if opt.ReadOnly {
return fmt.Errorf("readonly option can't be changed")
}
existing, ok := d.options[opt.ID]
if !ok {
return notFound("option")
}
if existing.Type != opt.Type {
return fmt.Errorf("type mismatch for option")
}
validPairs = append(validPairs, optPair{opt, existing})
}
// if all the options to be modified pass validation copy values over to
// existing options
if len(validPairs) == len(opts) {
for _, pair := range validPairs {
pair.existingOpt.Value.Val = pair.newOpt.Value.Val
}
}
return nil
}
func (d *Datastore) Option(id uint) (*kolide.Option, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
saved, ok := d.options[id]
if !ok {
return nil, notFound("Option").WithID(id)
}
result := *saved
return &result, nil
}
func (d *Datastore) ListOptions() ([]kolide.Option, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
result := []kolide.Option{}
for _, opt := range d.options {
result = append(result, *opt)
}
sortutil.AscByField(result, "Name")
return result, nil
}
func (d *Datastore) GetOsqueryConfigOptions() (map[string]interface{}, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
optConfig := map[string]interface{}{}
for _, opt := range d.options {
if opt.OptionSet() {
optConfig[opt.Name] = opt.GetValue()
}
}
return optConfig, nil
}

35
server/datastore/inmem/yara.go
View file

@ -1,35 +0,0 @@
package inmem
import "github.com/kolide/fleet/server/kolide"
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup, opts ...kolide.OptionalArg) (*kolide.YARASignatureGroup, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
ysg.ID = d.nextID(ysg)
d.yaraSignatureGroups[ysg.ID] = ysg
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string, opts ...kolide.OptionalArg) error {
d.mtx.Lock()
defer d.mtx.Unlock()
d.yaraFilePaths[fileSectionName] = append(d.yaraFilePaths[fileSectionName], sigGroupName)
return nil
}
func (d *Datastore) YARASection() (*kolide.YARASection, error) {
d.mtx.Lock()
defer d.mtx.Unlock()
result := &kolide.YARASection{
Signatures: make(map[string][]string),
FilePaths: make(map[string][]string),
}
for _, ysg := range d.yaraSignatureGroups {
result.Signatures[ysg.SignatureName] = append(result.Signatures[ysg.SignatureName], ysg.Paths...)
}
for fileSection, sigSection := range d.yaraFilePaths {
result.FilePaths[fileSection] = append(result.FilePaths[fileSection], sigSection...)
}
return result, nil
}

78
server/datastore/internal/appstate/options.go
View file

@ -1,78 +0,0 @@
package appstate
import "github.com/kolide/fleet/server/kolide"
// Options is the set of builtin osquery options that should be populated in
// the datastore
func Options() []struct {
Name string
Value interface{}
Type kolide.OptionType
ReadOnly bool
} {
return []struct {
Name string
Value interface{}
Type kolide.OptionType
ReadOnly bool
}{
// These options are read only, attempting to modify one of these will
// raise an error
{"disable_distributed", false, kolide.OptionTypeBool, kolide.ReadOnly},
{"distributed_plugin", "tls", kolide.OptionTypeString, kolide.ReadOnly},
{"pack_delimiter", "/", kolide.OptionTypeString, kolide.ReadOnly},
// These options may be modified by an admin user
{"aws_access_key_id", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_firehose_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"aws_firehose_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_kinesis_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"aws_kinesis_random_partition_key", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"aws_kinesis_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_profile_name", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_region", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_secret_access_key", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_sts_arn_role", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_sts_region", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_sts_session_name", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"aws_sts_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"buffered_log_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"decorations_top_level", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_caching", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_database", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_decorators", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_events", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_kernel", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_logging", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"disable_tables", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"distributed_interval", 10, kolide.OptionTypeInt, kolide.NotReadOnly},
{"distributed_tls_max_attempts", 3, kolide.OptionTypeInt, kolide.NotReadOnly},
{"enable_foreign", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"enable_monitor", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"ephemeral", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"events_expiry", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"events_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"events_optimize", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"host_identifier", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"logger_event_type", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"logger_mode", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"logger_path", nil, kolide.OptionTypeString, kolide.NotReadOnly},
{"logger_plugin", "tls", kolide.OptionTypeString, kolide.NotReadOnly},
{"logger_secondary_status_only", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"logger_syslog_facility", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"logger_tls_compress", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"logger_tls_endpoint", "/api/v1/osquery/log", kolide.OptionTypeString, kolide.NotReadOnly},
{"logger_tls_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"logger_tls_period", 10, kolide.OptionTypeInt, kolide.NotReadOnly},
{"pack_refresh_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"read_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"read_user_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"schedule_default_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"schedule_splay_percent", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"schedule_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"utc", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"value_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
{"verbose", nil, kolide.OptionTypeBool, kolide.NotReadOnly},
{"worker_threads", nil, kolide.OptionTypeInt, kolide.NotReadOnly},
}
}

74
server/datastore/mysql/file_integrity_monitoring.go
View file

@ -1,74 +0,0 @@
package mysql
import (
"database/sql"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewFIMSection(fp *kolide.FIMSection, opts ...kolide.OptionalArg) (result *kolide.FIMSection, err error) {
db := d.getTransaction(opts)
sqlStatement := `
INSERT INTO file_integrity_monitorings (
section_name,
description
) VALUES( ?, ?)
`
var resp sql.Result
resp, err = db.Exec(sqlStatement, fp.SectionName, fp.Description)
if isDuplicate(err) {
return nil, alreadyExists("fim_section", 0)
}
if err != nil {
return nil, errors.Wrap(err, "creating fim section")
}
id, _ := resp.LastInsertId()
fp.ID = uint(id)
sqlStatement = `
INSERT INTO file_integrity_monitoring_files (
file,
file_integrity_monitoring_id
) VALUES( ?, ? )
`
for _, fileName := range fp.Paths {
_, err = db.Exec(sqlStatement, fileName, fp.ID)
if err != nil {
return nil, errors.Wrap(err, "adding path to fim section")
}
}
return fp, nil
}
func (d *Datastore) ClearFIMSections() error {
sqlStatement := "DELETE FROM file_integrity_monitorings"
_, err := d.db.Exec(sqlStatement)
return err
}
func (d *Datastore) FIMSections() (kolide.FIMSections, error) {
sqlStatement := `
SELECT fim.section_name, mf.file FROM
file_integrity_monitorings AS fim
INNER JOIN file_integrity_monitoring_files AS mf
ON (fim.id = mf.file_integrity_monitoring_id)
`
rows, err := d.db.Query(sqlStatement)
if err != nil {
if err == sql.ErrNoRows {
return nil, notFound("FilePath")
}
return nil, errors.Wrap(err, "retrieving fim sections")
}
result := make(kolide.FIMSections)
for rows.Next() {
var sectionName, fileName string
err = rows.Scan(&sectionName, &fileName)
if err != nil {
return nil, errors.Wrap(err, "retrieving path for fim section")
}
result[sectionName] = append(result[sectionName], fileName)
}
return result, nil
}

54
server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go
View file

@ -1,54 +0,0 @@
package data
import (
"database/sql"
"github.com/kolide/fleet/server/datastore/internal/appstate"
"github.com/kolide/fleet/server/kolide"
)
func init() {
MigrationClient.AddMigration(Up_20161223115449, Down_20161223115449)
}
func Up_20161223115449(tx *sql.Tx) error {
sqlStatement := `
INSERT INTO options (
name,
type,
value,
read_only
) VALUES (?, ?, ?, ?)
`
for _, opt := range appstate.Options() {
ov := kolide.Option{
Name: opt.Name,
ReadOnly: opt.ReadOnly,
Type: opt.Type,
Value: kolide.OptionValue{
Val: opt.Value,
},
}
_, err := tx.Exec(sqlStatement, ov.Name, ov.Type, ov.Value, ov.ReadOnly)
if err != nil {
return err
}
}
return nil
}
func Down_20161223115449(tx *sql.Tx) error {
sqlStatement := `
DELETE FROM options
WHERE name = ?
`
for _, opt := range appstate.Options() {
_, err := tx.Exec(sqlStatement, opt.Name)
if err != nil {
return err
}
}
return nil
}

40
server/datastore/mysql/migrations/data/20170309091824_AddBuiltInDecorators.go
View file

@ -1,40 +0,0 @@
package data
import (
"database/sql"
"github.com/kolide/fleet/server/kolide"
)
func init() {
MigrationClient.AddMigration(Up_20170309091824, Down_20170309091824)
}
func Up_20170309091824(tx *sql.Tx) error {
sql := "INSERT INTO decorators (" +
"`type`, " +
"`query`, " +
"`built_in`, " +
"`interval`" +
") VALUES ( ?, ?, TRUE, 0 )"
rows := []struct {
t kolide.DecoratorType
q string
}{
{kolide.DecoratorLoad, "SELECT uuid AS host_uuid FROM system_info;"},
{kolide.DecoratorLoad, "SELECT hostname AS hostname FROM system_info;"},
}
for _, row := range rows {
_, err := tx.Exec(sql, row.t, row.q)
if err != nil {
return err
}
}
return nil
}
func Down_20170309091824(tx *sql.Tx) error {
_, err := tx.Exec("DELETE FROM decorators WHERE built_in = TRUE")
return err
}

33
server/datastore/mysql/migrations/data/20171027173700_AddTitlesToDecorators.go
View file

@ -1,33 +0,0 @@
package data
import (
"database/sql"
)
func init() {
MigrationClient.AddMigration(Up_20171027173700, Down_20171027173700)
}
func Up_20171027173700(tx *sql.Tx) error {
sql := "UPDATE decorators SET name=? where query=?"
rows := []struct {
name string
query string
}{
{"Host UUID", "SELECT uuid AS host_uuid FROM system_info;"},
{"Hostname", "SELECT hostname AS hostname FROM system_info;"},
}
for _, row := range rows {
_, err := tx.Exec(sql, row.name, row.query)
if err != nil {
return err
}
}
return nil
}
func Down_20171027173700(tx *sql.Tx) error {
_, err := tx.Exec("UPDATE decorators SET name='' WHERE built_in = TRUE")
return err
}

123
server/datastore/mysql/migrations/data/20171212182458_MigrateOsqueryOptions.go
View file

@ -1,123 +0,0 @@
package data
import (
"database/sql"
"encoding/json"
"fmt"
"strconv"
"github.com/jmoiron/sqlx"
"github.com/jmoiron/sqlx/reflectx"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20171212182458, Down_20171212182458)
}
type configForExport struct {
Options map[string]interface{} `json:"options"`
FilePaths map[string][]string `json:"file_paths,omitempty"`
Decorators kolide.Decorators `json:"decorators"`
}
func Up_20171212182458(tx *sql.Tx) error {
// Migrate pre fleetctl osquery options to the new osquery options
// formats.
txx := sqlx.Tx{Tx: tx, Mapper: reflectx.NewMapperFunc("db", sqlx.NameMapper)}
// Get basic osquery options
query := `
SELECT *
FROM options
WHERE value != "null"
`
// Intentionally initialize empty instead of nil so that we generate a
// config with empty options rather than a null value.
var opts []kolide.Option
if err := txx.Select(&opts, query); err != nil && err != sql.ErrNoRows {
return errors.Wrap(err, "getting options")
}
optConfig := map[string]interface{}{}
for _, opt := range opts {
optConfig[opt.Name] = opt.GetValue()
}
// Get FIM paths from fim table
query = `
SELECT fim.section_name, mf.file
FROM file_integrity_monitorings AS fim
INNER JOIN file_integrity_monitoring_files AS mf
ON (fim.id = mf.file_integrity_monitoring_id)
`
rows, err := txx.Query(query)
if err != nil && err != sql.ErrNoRows {
return errors.Wrap(err, "retrieving fim paths")
}
fimConfig := kolide.FIMSections{}
for rows.Next() {
var sectionName, fileName string
err = rows.Scan(&sectionName, &fileName)
if err != nil {
return errors.Wrap(err, "retrieving path for fim section")
}
fimConfig[sectionName] = append(fimConfig[sectionName], fileName)
}
query = `
SELECT *
FROM decorators
ORDER by built_in DESC, name ASC
`
var decorators []*kolide.Decorator
err = txx.Select(&decorators, query)
if err != nil {
return errors.Wrap(err, "retrieving decorators")
}
decConfig := kolide.Decorators{
Interval: make(map[string][]string),
}
for _, dec := range decorators {
switch dec.Type {
case kolide.DecoratorLoad:
decConfig.Load = append(decConfig.Load, dec.Query)
case kolide.DecoratorAlways:
decConfig.Always = append(decConfig.Always, dec.Query)
case kolide.DecoratorInterval:
key := strconv.Itoa(int(dec.Interval))
decConfig.Interval[key] = append(decConfig.Interval[key], dec.Query)
default:
fmt.Printf("Unable to migrate decorator. Please migrate manually: '%s'\n", dec.Query)
}
}
// Create config JSON
config := configForExport{
Options: optConfig,
FilePaths: fimConfig,
Decorators: decConfig,
}
confJSON, err := json.Marshal(config)
if err != nil {
return errors.Wrap(err, "marshal config JSON")
}
// Save config JSON
query = `
INSERT INTO osquery_options (
override_type, override_identifier, options
) VALUES (?, ?, ?)
`
if _, err = txx.Exec(query, kolide.OptionOverrideTypeDefault, "", string(confJSON)); err != nil {
return errors.Wrap(err, "saving converted options")
}
return nil
}
func Down_20171212182458(tx *sql.Tx) error {
// No down migration
return nil
}

146
server/datastore/mysql/migrations/tables/20171116163618_CreateTableOsqueryOptions.go
View file

@ -1,6 +1,16 @@
package tables
import "database/sql"
import (
"database/sql"
"encoding/json"
"fmt"
"strconv"
"github.com/jmoiron/sqlx"
"github.com/jmoiron/sqlx/reflectx"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20171116163618, Down_20171116163618)
@ -15,7 +25,139 @@ func Up_20171116163618(tx *sql.Tx) error {
"PRIMARY KEY (`id`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8;"
_, err := tx.Exec(sqlStatement)
return err
if err != nil {
return errors.Wrap(err, "create table osquery_options")
}
// Check whether there are existing options to migrate, or we should insert
// a new default set of options
var count int
err = tx.QueryRow("SELECT count(1) FROM options").Scan(&count)
if err != nil {
return errors.Wrap(err, "get options count")
}
if count > 0 {
// Migrate existing options
err = migrateOptions(tx)
if err != nil {
return errors.Wrap(err, "migrate options")
}
} else {
// Insert default options
_, err = tx.Exec("INSERT INTO `osquery_options`" +
"(override_type, override_identifier, options)" +
`VALUES (0, "", '{"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}')`,
)
if err != nil {
return errors.Wrap(err, "insert options")
}
}
return nil
}
func migrateOptions(tx *sql.Tx) error {
// This migration uses the deprecated types in deprecated_types.go
type configForExport struct {
Options map[string]interface{} `json:"options"`
FilePaths map[string][]string `json:"file_paths,omitempty"`
Decorators decorators `json:"decorators"`
}
// Migrate pre fleetctl osquery options to the new osquery options
// formats.
txx := sqlx.Tx{Tx: tx, Mapper: reflectx.NewMapperFunc("db", sqlx.NameMapper)}
// Get basic osquery options
query := `
SELECT *
FROM options
WHERE value != "null"
`
// Intentionally initialize empty instead of nil so that we generate a
// config with empty options rather than a null value.
var opts []option
if err := txx.Select(&opts, query); err != nil && err != sql.ErrNoRows {
return errors.Wrap(err, "getting options")
}
optConfig := map[string]interface{}{}
for _, opt := range opts {
optConfig[opt.Name] = opt.GetValue()
}
// Get FIM paths from fim table
query = `
SELECT fim.section_name, mf.file
FROM file_integrity_monitorings AS fim
INNER JOIN file_integrity_monitoring_files AS mf
ON (fim.id = mf.file_integrity_monitoring_id)
`
rows, err := txx.Query(query)
if err != nil && err != sql.ErrNoRows {
return errors.Wrap(err, "retrieving fim paths")
}
fimConfig := map[string][]string{}
for rows.Next() {
var sectionName, fileName string
err = rows.Scan(&sectionName, &fileName)
if err != nil {
return errors.Wrap(err, "retrieving path for fim section")
}
fimConfig[sectionName] = append(fimConfig[sectionName], fileName)
}
query = `
SELECT *
FROM decorators
ORDER by built_in DESC, name ASC
`
var decs []*decorator
err = txx.Select(&decs, query)
if err != nil {
return errors.Wrap(err, "retrieving decorators")
}
decConfig := decorators{
Interval: make(map[string][]string),
}
for _, dec := range decs {
switch dec.Type {
case decoratorLoad:
decConfig.Load = append(decConfig.Load, dec.Query)
case decoratorAlways:
decConfig.Always = append(decConfig.Always, dec.Query)
case decoratorInterval:
key := strconv.Itoa(int(dec.Interval))
decConfig.Interval[key] = append(decConfig.Interval[key], dec.Query)
default:
fmt.Printf("Unable to migrate decorator. Please migrate manually: '%s'\n", dec.Query)
}
}
// Create config JSON
config := configForExport{
Options: optConfig,
FilePaths: fimConfig,
Decorators: decConfig,
}
confJSON, err := json.Marshal(config)
if err != nil {
return errors.Wrap(err, "marshal config JSON")
}
// Save config JSON
query = `
INSERT INTO osquery_options (
override_type, override_identifier, options
) VALUES (?, ?, ?)
`
if _, err = txx.Exec(query, kolide.OptionOverrideTypeDefault, "", string(confJSON)); err != nil {
return errors.Wrap(err, "saving converted options")
}
return nil
}
func Down_20171116163618(tx *sql.Tx) error {

54
server/datastore/mysql/migrations/tables/20200707120000_DropUnusedTables.go Normal file
View file

@ -0,0 +1,54 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20200707120000, Down_20200707120000)
}
func Up_20200707120000(tx *sql.Tx) error {
_, err := tx.Exec("DROP TABLE `decorators`")
if err != nil {
return errors.Wrap(err, "drop decorators table")
}
_, err = tx.Exec("DROP TABLE `yara_file_paths`")
if err != nil {
return errors.Wrap(err, "drop yara_file_paths table")
}
_, err = tx.Exec("DROP TABLE `yara_signature_paths`")
if err != nil {
return errors.Wrap(err, "drop yara_signature_paths table")
}
_, err = tx.Exec("DROP TABLE `yara_signatures`")
if err != nil {
return errors.Wrap(err, "drop yara_signatures table")
}
_, err = tx.Exec("DROP TABLE `file_integrity_monitoring_files`")
if err != nil {
return errors.Wrap(err, "drop file_integrity_monitoring_files table")
}
_, err = tx.Exec("DROP TABLE `file_integrity_monitorings`")
if err != nil {
return errors.Wrap(err, "drop file_integrity_monitorings table")
}
_, err = tx.Exec("DROP TABLE `options`")
if err != nil {
return errors.Wrap(err, "drop file_integrity_monitorings table")
}
return nil
}
func Down_20200707120000(tx *sql.Tx) error {
return nil
}

180
server/datastore/mysql/migrations/tables/deprecated_types.go Normal file
View file

@ -0,0 +1,180 @@
package tables
import (
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"strings"
)
// Include a number of types here that were previously used but not needed
// at global scope any more
type decorator struct {
ID uint `json:"id"`
// Name is an optional human friendly name for the decorator
Name string `json:"name"`
Type decoratorType `json:"type"`
// Interval note this is only pertinent for decoratorInterval type.
Interval uint `json:"interval"`
Query string `json:"query"`
// BuiltIn decorators are loaded in migrations and may not be changed
BuiltIn bool `json:"built_in" db:"built_in"`
}
type decorators struct {
Load []string `json:"load,omitempty"`
Always []string `json:"always,omitempty"`
Interval map[string][]string `json:"interval,omitempty"`
}
// optionType defines the type of the value assigned to an option
type optionType int
const (
optionTypeString optionType = iota
optionTypeInt
optionTypeBool
)
// MarshalJSON marshals option type to strings
func (ot optionType) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%s"`, ot)), nil
}
// UnmarshalJSON converts json to optionType
func (ot *optionType) UnmarshalJSON(b []byte) error {
switch typ := string(b); strings.Trim(typ, `"`) {
case "string":
*ot = optionTypeString
case "bool":
*ot = optionTypeBool
case "int":
*ot = optionTypeInt
default:
return fmt.Errorf("unsupported option type '%s'", typ)
}
return nil
}
// String is used to marshal optionType to human readable strings used in JSON payloads
func (ot optionType) String() string {
switch ot {
case optionTypeString:
return "string"
case optionTypeInt:
return "int"
case optionTypeBool:
return "bool"
default:
panic("stringer not implemented for optionType")
}
}
// optionValue supports Valuer and Scan interfaces for reading and writing
// to the database, and also JSON marshaling
type optionValue struct {
Val interface{}
}
// Value is called by the DB driver. Note that we store data as JSON
// types, so we can use the JSON marshaller to assign the appropriate
// type when we fetch it from the db
func (ov optionValue) Value() (dv driver.Value, err error) {
return json.Marshal(ov.Val)
}
// Scan takes db string and turns it into an option type
func (ov *optionValue) Scan(src interface{}) error {
if err := json.Unmarshal(src.([]byte), &ov.Val); err != nil {
return err
}
switch v := ov.Val.(type) {
case float64:
ov.Val = int(v)
}
return nil
}
// MarshalJSON implements the json.Marshaler interface
func (ov optionValue) MarshalJSON() ([]byte, error) {
return json.Marshal(ov.Val)
}
// UnmarshalJSON implements the json.Unmarshaler interface
func (ov *optionValue) UnmarshalJSON(b []byte) error {
return json.Unmarshal(b, &ov.Val)
}
// option represents a possible osquery confguration option
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type option struct {
// ID unique identifier for option assigned by the dbms
ID uint `json:"id"`
// Name of the option which must be unique
Name string `json:"name"`
// Type of value for the option
Type optionType `json:"type"`
// Value of the option which may be nil, bool, int, or string.
Value optionValue `json:"value"`
// ReadOnly if true, this option is required for Fleet to function
// properly and cannot be modified by the end user
ReadOnly bool `json:"read_only" db:"read_only"`
}
// GetValue returns the value associated with the option
func (opt option) GetValue() interface{} {
return opt.Value.Val
}
// decoratorType refers to the allowable types of decorator queries.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type decoratorType int
const (
decoratorLoad decoratorType = iota
decoratorAlways
decoratorInterval
decoratorUndefined
decoratorLoadName = "load"
decoratorAlwaysName = "always"
decoratorIntervalName = "interval"
)
func (dt decoratorType) String() string {
switch dt {
case decoratorLoad:
return decoratorLoadName
case decoratorAlways:
return decoratorAlwaysName
case decoratorInterval:
return decoratorIntervalName
default:
return ""
}
}
func (dt *decoratorType) MarshalJSON() ([]byte, error) {
name := dt.String()
if name == "" {
return nil, errors.New("Invalid decorator type")
}
return []byte(`"` + name + `"`), nil
}
func (dt *decoratorType) UnmarshalJSON(data []byte) error {
name := strings.Trim(string(data), `"`)
switch name {
case decoratorLoadName:
*dt = decoratorLoad
case decoratorAlwaysName:
*dt = decoratorAlways
case decoratorIntervalName:
*dt = decoratorInterval
default:
*dt = decoratorUndefined
}
return nil
}

174
server/datastore/mysql/options.go
View file

@ -1,174 +0,0 @@
package mysql
import (
"database/sql"
"github.com/kolide/fleet/server/datastore/internal/appstate"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
// ResetOptions note we use named return values so we can preserve and return
// errors in our defer function
func (d *Datastore) ResetOptions() (opts []kolide.Option, err error) {
// Atomically remove all existing options, reset auto increment so id's will be the
// same as original defaults, and re-insert defaults in option table.
var txn *sql.Tx
txn, err = d.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "reset options begin transaction")
}
defer func() {
if err != nil {
if txErr := txn.Rollback(); txErr != nil {
err = errors.Wrapf(err, "reset options failed, transaction rollback failed with error: %s", txErr)
}
}
}()
_, err = txn.Exec("DELETE FROM options")
if err != nil {
return nil, errors.Wrap(err, "deleting options in reset options")
}
// Reset auto increment
_, err = txn.Exec("ALTER TABLE `options` AUTO_INCREMENT = 1")
if err != nil {
return nil, errors.Wrap(err, "resetting auto increment counter in reset options")
}
sqlStatement := `
INSERT INTO options (
name,
type,
value,
read_only
) VALUES (?, ?, ?, ?)
`
for _, defaultOpt := range appstate.Options() {
opt := kolide.Option{
Name: defaultOpt.Name,
ReadOnly: defaultOpt.ReadOnly,
Type: defaultOpt.Type,
Value: kolide.OptionValue{
Val: defaultOpt.Value,
},
}
dbResponse, err := txn.Exec(
sqlStatement,
opt.Name,
opt.Type,
opt.Value,
opt.ReadOnly,
)
if err != nil {
return nil, errors.Wrap(err, "inserting default option in reset options")
}
id, err := dbResponse.LastInsertId()
if err != nil {
return nil, errors.Wrap(err, "fetching id in reset options")
}
opt.ID = uint(id)
opts = append(opts, opt)
}
err = txn.Commit()
if err != nil {
return nil, errors.Wrap(err, "committing reset options")
}
return opts, nil
}
func (d *Datastore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
db := d.getTransaction(args)
sqlStatement := `
SELECT *
FROM options
WHERE name = ?
`
var option kolide.Option
if err := db.Get(&option, sqlStatement, name); err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Option")
}
return nil, errors.Wrap(err, sqlStatement)
}
return &option, nil
}
func (d *Datastore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) (err error) {
db := d.getTransaction(args)
sqlStatement := `
UPDATE options
SET value = ?
WHERE id = ? AND type = ? AND NOT read_only
`
for _, opt := range opts {
resultInfo, err := db.Exec(sqlStatement, opt.Value, opt.ID, opt.Type)
if err != nil {
return errors.Wrap(err, "update options")
}
rowsMatched, err := resultInfo.RowsAffected()
if err != nil {
return errors.Wrap(err, "update options reading rows matched")
}
if rowsMatched == 0 {
return notFound("Option").WithID(opt.ID)
}
}
return err
}
func (d *Datastore) Option(id uint) (*kolide.Option, error) {
sqlStatement := `
SELECT *
FROM options
WHERE id = ?
`
var opt kolide.Option
if err := d.db.Get(&opt, sqlStatement, id); err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Option").WithID(id)
}
return nil, errors.Wrap(err, "select option by ID")
}
return &opt, nil
}
func (d *Datastore) ListOptions() ([]kolide.Option, error) {
sqlStatement := `
SELECT *
FROM options
ORDER BY name ASC
`
var opts []kolide.Option
if err := d.db.Select(&opts, sqlStatement); err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Option")
}
return nil, errors.Wrap(err, "select from options")
}
return opts, nil
}
func (d *Datastore) GetOsqueryConfigOptions() (map[string]interface{}, error) {
// Retrieve all the options that are set. The value field is JSON formatted so
// to retrieve options that are set, we check JSON null keyword
sqlStatement := `
SELECT *
FROM options
WHERE value != "null"
`
var opts []kolide.Option
if err := d.db.Select(&opts, sqlStatement); err != nil {
if err == sql.ErrNoRows {
return nil, notFound("Option")
}
return nil, errors.Wrap(err, "select from options")
}
optConfig := map[string]interface{}{}
for _, opt := range opts {
optConfig[opt.Name] = opt.GetValue()
}
return optConfig, nil
}

122
server/datastore/mysql/yara.go
View file

@ -1,122 +0,0 @@
package mysql
import (
"database/sql"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (d *Datastore) NewYARASignatureGroup(ysg *kolide.YARASignatureGroup, opts ...kolide.OptionalArg) (sg *kolide.YARASignatureGroup, err error) {
dbTx := d.getTransaction(opts)
sqlStatement := `
INSERT INTO yara_signatures (
signature_name
) VALUES( ? )
`
var result sql.Result
result, err = dbTx.Exec(sqlStatement, ysg.SignatureName)
if isDuplicate(err) {
return nil, alreadyExists("yara signature", 0)
}
if err != nil {
return nil, errors.Wrap(err, "inserting new yara signature group")
}
id, _ := result.LastInsertId()
ysg.ID = uint(id)
sqlStatement = `
INSERT INTO yara_signature_paths (
file_path,
yara_signature_id
) VALUES( ?, ? )
`
for _, path := range ysg.Paths {
_, err = dbTx.Exec(sqlStatement, path, ysg.ID)
if err != nil {
return nil, errors.Wrap(err, "inserting new signature path")
}
}
return ysg, nil
}
func (d *Datastore) NewYARAFilePath(fileSectionName, sigGroupName string, opts ...kolide.OptionalArg) error {
dbTx := d.getTransaction(opts)
sqlStatement := `
INSERT INTO yara_file_paths (
file_integrity_monitoring_id,
yara_signature_id
) VALUES (
(
SELECT fim.id
FROM file_integrity_monitorings AS fim
WHERE fim.section_name = ?
LIMIT 1
),
(
SELECT ys.id AS ys
FROM yara_signatures AS ys
WHERE ys.signature_name = ?
LIMIT 1
)
)
`
_, err := dbTx.Exec(sqlStatement, fileSectionName, sigGroupName)
if isDuplicate(err) {
return alreadyExists("yara file path", 0)
}
if err != nil {
return errors.Wrap(err, "inserting yara file path")
}
return nil
}
func (d *Datastore) YARASection() (*kolide.YARASection, error) {
result := &kolide.YARASection{
Signatures: make(map[string][]string),
FilePaths: make(map[string][]string),
}
sqlStatement := `
SELECT s.signature_name, p.file_path
FROM yara_signatures AS s
INNER JOIN yara_signature_paths AS p
ON ( s.id = p.yara_signature_id )
`
rows, err := d.db.Query(sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "selecting yara information")
}
for rows.Next() {
var sigName, sigPath string
err = rows.Scan(&sigName, &sigPath)
if err != nil {
return nil, errors.Wrap(err, "scanning yara information")
}
result.Signatures[sigName] = append(result.Signatures[sigName], sigPath)
}
sqlStatement = `
SELECT f.section_name, y.signature_name
FROM file_integrity_monitorings AS f
INNER JOIN yara_file_paths AS yfp
ON (f.id = yfp.file_integrity_monitoring_id)
INNER JOIN yara_signatures AS y
ON (y.id = yfp.yara_signature_id )
`
rows, err = d.db.Query(sqlStatement)
if err != nil {
return nil, errors.Wrap(err, "selecting yara signatures")
}
for rows.Next() {
var sectionName, signatureName string
err = rows.Scan(&sectionName, &signatureName)
if err != nil {
return nil, errors.Wrap(err, "scanning yara signature values")
}
result.FilePaths[sectionName] = append(result.FilePaths[sectionName], signatureName)
}
return result, nil
}

3
server/kolide/datastore.go
View file

@ -14,9 +14,6 @@ type Datastore interface {
AppConfigStore
InviteStore
ScheduledQueryStore
OptionStore
FileIntegrityMonitoringStore
YARAStore
OsqueryOptionsStore
Name() string
Drop() error

89
server/kolide/decorators.go
View file

@ -1,89 +0,0 @@
package kolide
import (
"errors"
"strings"
)
// DEPRECATED
// Decorators are now stored as JSON in the config, so these types are only
// useful for migrating existing Fleet installations.
// DecoratorType refers to the allowable types of decorator queries.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type DecoratorType int
const (
DecoratorLoad DecoratorType = iota
DecoratorAlways
DecoratorInterval
DecoratorUndefined
DecoratorLoadName = "load"
DecoratorAlwaysName = "always"
DecoratorIntervalName = "interval"
)
func (dt DecoratorType) String() string {
switch dt {
case DecoratorLoad:
return DecoratorLoadName
case DecoratorAlways:
return DecoratorAlwaysName
case DecoratorInterval:
return DecoratorIntervalName
default:
return ""
}
}
func (dt *DecoratorType) MarshalJSON() ([]byte, error) {
name := dt.String()
if name == "" {
return nil, errors.New("Invalid decorator type")
}
return []byte(`"` + name + `"`), nil
}
var decNameToType = map[string]DecoratorType{
DecoratorLoadName: DecoratorLoad,
DecoratorAlwaysName: DecoratorAlways,
DecoratorIntervalName: DecoratorInterval,
}
func (dt *DecoratorType) UnmarshalJSON(data []byte) error {
name := strings.Trim(string(data), `"`)
switch name {
case DecoratorLoadName:
*dt = DecoratorLoad
case DecoratorAlwaysName:
*dt = DecoratorAlways
case DecoratorIntervalName:
*dt = DecoratorInterval
default:
*dt = DecoratorUndefined
}
return nil
}
// Decorator contains information about a decorator query.
type Decorator struct {
UpdateCreateTimestamps
ID uint `json:"id"`
// Name is an optional human friendly name for the decorator
Name string `json:"name"`
Type DecoratorType `json:"type"`
// Interval note this is only pertinent for DecoratorInterval type.
Interval uint `json:"interval"`
Query string `json:"query"`
// BuiltIn decorators are loaded in migrations and may not be changed
BuiltIn bool `json:"built_in" db:"built_in"`
}
type DecoratorPayload struct {
ID uint `json:"id"`
Name *string `json:"name"`
DecoratorType *DecoratorType `json:"type"`
Interval *uint `json:"interval"`
Query *string `json:"query"`
}

45
server/kolide/file_integrity_monitoring.go
View file

@ -1,45 +0,0 @@
package kolide
import "context"
type FIMSections map[string][]string
type FileIntegrityMonitoringStore interface {
// NewFIMSection creates a named group of file paths
NewFIMSection(path *FIMSection, opts ...OptionalArg) (*FIMSection, error)
// FIMSections returns all named file sections
FIMSections() (FIMSections, error)
// ClearFIMSections removes all the FIM information
ClearFIMSections() error
}
// FileIntegrityMonitoringService methods to update
type FileIntegrityMonitoringService interface {
// GetFIM returns the FIM config
GetFIM(ctx context.Context) (*FIMConfig, error)
// ModifyFIM replaces existing FIM. To disable FIM send FIMConfig with
// empty FilePaths
ModifyFIM(ctx context.Context, fim FIMConfig) error
}
// FIMSection maps a name to a group of files for the osquery file_paths
// section.
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type FIMSection struct {
ID uint
SectionName string `db:"section_name"`
Description string
Paths []string `db:"-"`
}
// FIMConfig information to set up File Integrity Monitoring
type FIMConfig struct {
// Interval defines the frequency when the file monitor will run.
Interval uint `json:"interval"`
// FilePaths contains named groups of files to monitor. The hash key is the
// name, the array of strings contains paths to be monitored.
// See https://osquery.readthedocs.io/en/stable/deployment/file-integrity-monitoring/
FilePaths FIMSections `json:"file_paths,omitempty"`
// FileAccesses defines those name groups of FIMSections which will be monitored for file accesses
FileAccesses []string `json:"file_accesses,omitempty"`
}

181
server/kolide/options.go
View file

@ -1,181 +0,0 @@
package kolide
import (
"context"
"database/sql/driver"
"encoding/json"
"fmt"
"strings"
)
// OptionStore interface describes methods to access datastore
type OptionStore interface {
// SaveOptions transactional write of options to storage. If one or more
// values fails validation none of the writes will succeed. Note only option
// values are written. Other option fields are created in migration and do
// not change. Attempting to write ReadOnly options will cause an error.
SaveOptions(opts []Option, args ...OptionalArg) error
// Options returns all options
ListOptions() ([]Option, error)
// Option return an option by ID
Option(id uint) (*Option, error)
// OptionByName returns an option uniquely identified by name
OptionByName(name string, args ...OptionalArg) (*Option, error)
// GetOsqueryConfigOptions returns options in a format that will be the options
// section of osquery configuration
GetOsqueryConfigOptions() (map[string]interface{}, error)
// ResetOptions will reset options to their initial values. This should be used
// with caution as it will remove any options or changes to defaults made by
// the user. Returns a list of default options.
ResetOptions() ([]Option, error)
}
// OptionService interface describes methods that operate on osquery options
type OptionService interface {
// GetOptions retrieves all options
GetOptions(ctx context.Context) ([]Option, error)
// ModifyOptions will change values of the options in OptionRequest. Note
// passing ReadOnly options will cause an error.
ModifyOptions(ctx context.Context, req OptionRequest) ([]Option, error)
// ResetOptions resets all options to their default values
ResetOptions(ctx context.Context) ([]Option, error)
}
const (
ReadOnly = true
NotReadOnly = !ReadOnly
)
// OptionType defines the type of the value assigned to an option
type OptionType int
const (
OptionTypeString OptionType = iota
OptionTypeInt
OptionTypeBool
)
// String values that map from JSON to OptionType
const (
optionTypeString = "string"
optionTypeInt = "int"
optionTypeBool = "bool"
)
// MarshalJSON marshals option type to strings
func (ot OptionType) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%s"`, ot)), nil
}
// UnmarshalJSON converts json to OptionType
func (ot *OptionType) UnmarshalJSON(b []byte) error {
switch typ := string(b); strings.Trim(typ, `"`) {
case optionTypeString:
*ot = OptionTypeString
case optionTypeBool:
*ot = OptionTypeBool
case optionTypeInt:
*ot = OptionTypeInt
default:
return fmt.Errorf("unsupported option type '%s'", typ)
}
return nil
}
// String is used to marshal OptionType to human readable strings used in JSON payloads
func (ot OptionType) String() string {
switch ot {
case OptionTypeString:
return optionTypeString
case OptionTypeInt:
return optionTypeInt
case OptionTypeBool:
return optionTypeBool
default:
panic("stringer not implemented for OptionType")
}
}
// OptionValue supports Valuer and Scan interfaces for reading and writing
// to the database, and also JSON marshaling
type OptionValue struct {
Val interface{}
}
// Value is called by the DB driver. Note that we store data as JSON
// types, so we can use the JSON marshaller to assign the appropriate
// type when we fetch it from the db
func (ov OptionValue) Value() (dv driver.Value, err error) {
return json.Marshal(ov.Val)
}
// Scan takes db string and turns it into an option type
func (ov *OptionValue) Scan(src interface{}) error {
if err := json.Unmarshal(src.([]byte), &ov.Val); err != nil {
return err
}
switch v := ov.Val.(type) {
case float64:
ov.Val = int(v)
}
return nil
}
// MarshalJSON implements the json.Marshaler interface
func (ov OptionValue) MarshalJSON() ([]byte, error) {
return json.Marshal(ov.Val)
}
// UnmarshalJSON implements the json.Unmarshaler interface
func (ov *OptionValue) UnmarshalJSON(b []byte) error {
return json.Unmarshal(b, &ov.Val)
}
// Option represents a possible osquery confguration option
// See https://osquery.readthedocs.io/en/stable/deployment/configuration/
type Option struct {
// ID unique identifier for option assigned by the dbms
ID uint `json:"id"`
// Name of the option which must be unique
Name string `json:"name"`
// Type of value for the option
Type OptionType `json:"type"`
// Value of the option which may be nil, bool, int, or string.
Value OptionValue `json:"value"`
// ReadOnly if true, this option is required for Fleet to function
// properly and cannot be modified by the end user
ReadOnly bool `json:"read_only" db:"read_only"`
}
// SetValue sets the value associated with the option
func (opt *Option) SetValue(v interface{}) {
opt.Value.Val = v
}
func (opt *Option) SameType(compare interface{}) bool {
switch compare.(type) {
case float64:
return opt.Type == OptionTypeInt
case string:
return opt.Type == OptionTypeString
case bool:
return opt.Type == OptionTypeBool
default:
return false
}
}
// OptionSet returns true if the option has a value assigned to it
func (opt *Option) OptionSet() bool {
return opt.Value.Val != nil
}
// GetValue returns the value associated with the option
func (opt Option) GetValue() interface{} {
return opt.Value.Val
}
// OptionRequest contains options that are passed to modify options requests.
type OptionRequest struct {
Options []Option `json:"options"`
}

47
server/kolide/options_test.go
View file

@ -1,47 +0,0 @@
package kolide
import (
"encoding/json"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestOptionMarshaller(t *testing.T) {
tests := []struct {
value interface{}
typ OptionType
expected interface{}
}{
{23, OptionTypeInt, float64(23)},
{true, OptionTypeBool, true},
{"foobar", OptionTypeString, "foobar"},
}
for _, test := range tests {
optIn := &Option{1, "foo", test.typ, OptionValue{test.value}, true}
buff, err := json.Marshal(optIn)
require.Nil(t, err)
optOut := &Option{}
err = json.Unmarshal(buff, optOut)
require.Nil(t, err)
assert.Equal(t, optIn.ID, optOut.ID)
assert.Equal(t, optIn.Name, optOut.Name)
assert.Equal(t, optIn.ReadOnly, optOut.ReadOnly)
assert.Equal(t, optIn.Type, optOut.Type)
assert.Equal(t, test.expected, optOut.Value.Val)
}
// test nil
optIn := &Option{1, "bar", OptionTypeString, OptionValue{nil}, true}
buff, err := json.Marshal(optIn)
require.Nil(t, err)
optOut := &Option{}
err = json.Unmarshal(buff, optOut)
require.Nil(t, err)
assert.True(t, reflect.DeepEqual(optIn, optOut))
}

24
server/kolide/osquery.go
View file

@ -78,27 +78,3 @@ type PermissivePackContent struct {
type Packs map[string]PackContent
type PermissivePacks map[string]PermissivePackContent
// Decorators is the format of the decorator configuration in an osquery config.
type Decorators struct {
Load []string `json:"load,omitempty"`
Always []string `json:"always,omitempty"`
Interval map[string][]string `json:"interval,omitempty"`
}
// OsqueryConfig is a struct that can be serialized into a valid osquery config
// using Go's JSON tooling.
type OsqueryConfig struct {
Schedule map[string]QueryContent `json:"schedule,omitempty"`
Options map[string]interface{} `json:"options"`
Decorators Decorators `json:"decorators,omitempty"`
Packs Packs `json:"packs,omitempty"`
// FilePaths contains named collections of file paths used for
// FIM (File Integrity Monitoring)
FilePaths FIMSections `json:"file_paths,omitempty"`
}
type PermissiveOsqueryConfig struct {
OsqueryConfig
Packs PermissivePacks `jsoon:"packs,omitempty"`
}

2
server/kolide/service.go
View file

@ -15,7 +15,5 @@ type Service interface {
InviteService
TargetService
ScheduledQueryService
OptionService
FileIntegrityMonitoringService
StatusService
}

31
server/kolide/yara.go
View file

@ -1,31 +0,0 @@
package kolide
// YARAFilePaths represents the files_path section of an osquery config. The
// key maps to file_paths section_name and maps to one or more YARA signature
// group names
type YARAFilePaths map[string][]string
type YARAStore interface {
// NewYARASignatureGroup creates a new mapping of a name to
// a group of YARA signatures
NewYARASignatureGroup(ysg *YARASignatureGroup, opts ...OptionalArg) (*YARASignatureGroup, error)
// NewYARAFilePath maps a named set of files to one or more
// groups of YARA signatures
NewYARAFilePath(fileSectionName, sigGroupName string, opts ...OptionalArg) error
// YARASection creates the osquery configuration YARA section
YARASection() (*YARASection, error)
}
// YARASignatureGroup maps a name to a group of YARA Signatures
// See https://osquery.readthedocs.io/en/stable/deployment/yara/
type YARASignatureGroup struct {
ID uint
SignatureName string `db:"signature_name"`
Paths []string `db:"-"`
}
// YARASection represents the osquery config for YARA
type YARASection struct {
Signatures map[string][]string `json:"signatures"`
FilePaths map[string][]string `json:"file_paths"`
}

3
server/mock/datastore.go
View file

@ -21,18 +21,15 @@ var _ kolide.Datastore = (*Store)(nil)
type Store struct {
kolide.PasswordResetStore
kolide.YARAStore
TargetStore
SessionStore
CampaignStore
ScheduledQueryStore
OsqueryOptionsStore
FileIntegrityMonitoringStore
AppConfigStore
HostStore
InviteStore
LabelStore
OptionStore
PackStore
UserStore
QueryStore

39
server/mock/datastore_fim.go
View file

@ -1,39 +0,0 @@
// Automatically generated by mockimpl. DO NOT EDIT!
package mock
import "github.com/kolide/fleet/server/kolide"
var _ kolide.FileIntegrityMonitoringStore = (*FileIntegrityMonitoringStore)(nil)
type NewFIMSectionFunc func(path *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error)
type FIMSectionsFunc func() (kolide.FIMSections, error)
type ClearFIMSectionsFunc func() error
type FileIntegrityMonitoringStore struct {
NewFIMSectionFunc NewFIMSectionFunc
NewFIMSectionFuncInvoked bool
FIMSectionsFunc FIMSectionsFunc
FIMSectionsFuncInvoked bool
ClearFIMSectionsFunc ClearFIMSectionsFunc
ClearFIMSectionsFuncInvoked bool
}
func (s *FileIntegrityMonitoringStore) NewFIMSection(path *kolide.FIMSection, opts ...kolide.OptionalArg) (*kolide.FIMSection, error) {
s.NewFIMSectionFuncInvoked = true
return s.NewFIMSectionFunc(path, opts...)
}
func (s *FileIntegrityMonitoringStore) FIMSections() (kolide.FIMSections, error) {
s.FIMSectionsFuncInvoked = true
return s.FIMSectionsFunc()
}
func (s *FileIntegrityMonitoringStore) ClearFIMSections() error {
s.ClearFIMSectionsFuncInvoked = true
return s.ClearFIMSectionsFunc()
}

69
server/mock/datastore_options.go
View file

@ -1,69 +0,0 @@
// Automatically generated by mockimpl. DO NOT EDIT!
package mock
import "github.com/kolide/fleet/server/kolide"
var _ kolide.OptionStore = (*OptionStore)(nil)
type SaveOptionsFunc func(opts []kolide.Option, args ...kolide.OptionalArg) error
type ListOptionsFunc func() ([]kolide.Option, error)
type OptionFunc func(id uint) (*kolide.Option, error)
type OptionByNameFunc func(name string, args ...kolide.OptionalArg) (*kolide.Option, error)
type GetOsqueryConfigOptionsFunc func() (map[string]interface{}, error)
type ResetOptionsFunc func() ([]kolide.Option, error)
type OptionStore struct {
SaveOptionsFunc SaveOptionsFunc
SaveOptionsFuncInvoked bool
ListOptionsFunc ListOptionsFunc
ListOptionsFuncInvoked bool
OptionFunc OptionFunc
OptionFuncInvoked bool
OptionByNameFunc OptionByNameFunc
OptionByNameFuncInvoked bool
GetOsqueryConfigOptionsFunc GetOsqueryConfigOptionsFunc
GetOsqueryConfigOptionsFuncInvoked bool
ResetOptionsFunc ResetOptionsFunc
ResetOptionsFuncInvoked bool
}
func (s *OptionStore) SaveOptions(opts []kolide.Option, args ...kolide.OptionalArg) error {
s.SaveOptionsFuncInvoked = true
return s.SaveOptionsFunc(opts, args...)
}
func (s *OptionStore) ListOptions() ([]kolide.Option, error) {
s.ListOptionsFuncInvoked = true
return s.ListOptionsFunc()
}
func (s *OptionStore) Option(id uint) (*kolide.Option, error) {
s.OptionFuncInvoked = true
return s.OptionFunc(id)
}
func (s *OptionStore) OptionByName(name string, args ...kolide.OptionalArg) (*kolide.Option, error) {
s.OptionByNameFuncInvoked = true
return s.OptionByNameFunc(name, args...)
}
func (s *OptionStore) GetOsqueryConfigOptions() (map[string]interface{}, error) {
s.GetOsqueryConfigOptionsFuncInvoked = true
return s.GetOsqueryConfigOptionsFunc()
}
func (s *OptionStore) ResetOptions() ([]kolide.Option, error) {
s.ResetOptionsFuncInvoked = true
return s.ResetOptionsFunc()
}

40
server/service/endpoint_fim.go
View file

@ -1,40 +0,0 @@
package service
import (
"context"
"github.com/go-kit/kit/endpoint"
"github.com/kolide/fleet/server/kolide"
)
type modifyFIMResponse struct {
Err error `json:"error,omitempty"`
}
func (m modifyFIMResponse) error() error { return m.Err }
func makeModifyFIMEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, req interface{}) (interface{}, error) {
fimConfig := req.(kolide.FIMConfig)
var resp modifyFIMResponse
if err := svc.ModifyFIM(ctx, fimConfig); err != nil {
resp.Err = err
}
return resp, nil
}
}
type getFIMResponse struct {
Err error `json:"error,omitempty"`
Payload *kolide.FIMConfig `json:"payload,omitempty"`
}
func makeGetFIMEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, _ interface{}) (interface{}, error) {
fimConfig, err := svc.GetFIM(ctx)
if err != nil {
return getFIMResponse{Err: err}, nil
}
return getFIMResponse{Payload: fimConfig}, nil
}
}

46
server/service/endpoint_options.go
View file

@ -1,46 +0,0 @@
package service
import (
"context"
"github.com/go-kit/kit/endpoint"
"github.com/kolide/fleet/server/kolide"
)
type optionsResponse struct {
Options []kolide.Option `json:"options,omitempty"`
Err error `json:"error,omitempty"`
}
func (or optionsResponse) error() error { return or.Err }
func makeGetOptionsEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
options, err := svc.GetOptions(ctx)
if err != nil {
return optionsResponse{Err: err}, nil
}
return optionsResponse{Options: options}, nil
}
}
func makeModifyOptionsEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
payload := request.(kolide.OptionRequest)
opts, err := svc.ModifyOptions(ctx, payload)
if err != nil {
return optionsResponse{Err: err}, nil
}
return optionsResponse{Options: opts}, nil
}
}
func makeResetOptionsEndpoint(svc kolide.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
options, err := svc.ResetOptions(ctx)
if err != nil {
return optionsResponse{Err: err}, nil
}
return optionsResponse{Options: options}, nil
}
}

83
server/service/endpoint_options_test.go
View file

@ -1,83 +0,0 @@
package service
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testOptionNotFound(t *testing.T, r *testResource) {
// id 6000 is not an actual option
inJson := `{"options":[
{"id":6000,"name":"aws_access_key_id","type":"string","value":"foo","read_only":false},
{"id":7,"name":"aws_firehose_period","type":"int","value":23,"read_only":false}]}`
buff := bytes.NewBufferString(inJson)
req, err := http.NewRequest("PATCH", r.server.URL+"/api/v1/kolide/options", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
}
func testGetOptions(t *testing.T, r *testResource) {
req, err := http.NewRequest("GET", r.server.URL+"/api/v1/kolide/options", nil)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
var optsResp optionsResponse
err = json.NewDecoder(resp.Body).Decode(&optsResp)
require.Nil(t, err)
require.NotNil(t, optsResp.Options)
assert.Equal(t, "aws_access_key_id", optsResp.Options[0].Name)
}
func testModifyOptions(t *testing.T, r *testResource) {
inJson := `{"options":[
{"id":6,"name":"aws_access_key_id","type":"string","value":"foo","read_only":false},
{"id":7,"name":"aws_firehose_period","type":"int","value":23,"read_only":false}]}`
buff := bytes.NewBufferString(inJson)
req, err := http.NewRequest("PATCH", r.server.URL+"/api/v1/kolide/options", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
var optsResp optionsResponse
err = json.NewDecoder(resp.Body).Decode(&optsResp)
require.Nil(t, err)
require.NotNil(t, optsResp.Options)
require.Len(t, optsResp.Options, 2)
assert.Equal(t, "foo", optsResp.Options[0].GetValue())
assert.Equal(t, float64(23), optsResp.Options[1].GetValue())
}
func testModifyOptionsValidationFail(t *testing.T, r *testResource) {
inJson := `{"options":[
{"id":6,"name":"aws_access_key_id","type":"string","value":"foo","read_only":false},
{"id":7,"name":"aws_firehose_period","type":"int","value":"xxs","read_only":false}]}`
buff := bytes.NewBufferString(inJson)
req, err := http.NewRequest("PATCH", r.server.URL+"/api/v1/kolide/options", buff)
require.Nil(t, err)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken))
client := &http.Client{}
resp, err := client.Do(req)
require.Nil(t, err)
assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
var errStruct mockValidationError
err = json.NewDecoder(resp.Body).Decode(&errStruct)
require.Nil(t, err)
require.Len(t, errStruct.Errors, 1)
assert.Equal(t, "aws_firehose_period", errStruct.Errors[0].Name)
assert.Equal(t, "type mismatch", errStruct.Errors[0].Reason)
}

4
server/service/endpoint_test.go
View file

@ -103,10 +103,6 @@ var testFunctions = [...]func(*testing.T, *testResource){
testGetAppConfig,
testModifyAppConfig,
testModifyAppConfigWithValidationFail,
testGetOptions,
testModifyOptions,
testModifyOptionsValidationFail,
testOptionNotFound,
testAdminUserSetAdmin,
testNonAdminUserSetAdmin,
testAdminUserSetEnabled,

26
server/service/handler.go
View file

@ -87,9 +87,6 @@ type KolideEndpoints struct {
ListHosts endpoint.Endpoint
GetHostSummary endpoint.Endpoint
SearchTargets endpoint.Endpoint
GetOptions endpoint.Endpoint
ModifyOptions endpoint.Endpoint
ResetOptions endpoint.Endpoint
ApplyOsqueryOptionsSpec endpoint.Endpoint
GetOsqueryOptionsSpec endpoint.Endpoint
GetCertificate endpoint.Endpoint
@ -97,8 +94,6 @@ type KolideEndpoints struct {
InitiateSSO endpoint.Endpoint
CallbackSSO endpoint.Endpoint
SSOSettings endpoint.Endpoint
GetFIM endpoint.Endpoint
ModifyFIM endpoint.Endpoint
StatusResultStore endpoint.Endpoint
StatusLiveQuery endpoint.Endpoint
}
@ -185,15 +180,10 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey, urlPrefix string) Kol
GetLabelSpecs: authenticatedUser(jwtKey, svc, makeGetLabelSpecsEndpoint(svc)),
GetLabelSpec: authenticatedUser(jwtKey, svc, makeGetLabelSpecEndpoint(svc)),
SearchTargets: authenticatedUser(jwtKey, svc, makeSearchTargetsEndpoint(svc)),
GetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeGetOptionsEndpoint(svc))),
ModifyOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeModifyOptionsEndpoint(svc))),
ResetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeResetOptionsEndpoint(svc))),
ApplyOsqueryOptionsSpec: authenticatedUser(jwtKey, svc, makeApplyOsqueryOptionsSpecEndpoint(svc)),
GetOsqueryOptionsSpec: authenticatedUser(jwtKey, svc, makeGetOsqueryOptionsSpecEndpoint(svc)),
GetCertificate: authenticatedUser(jwtKey, svc, makeCertificateEndpoint(svc)),
ChangeEmail: authenticatedUser(jwtKey, svc, makeChangeEmailEndpoint(svc)),
GetFIM: authenticatedUser(jwtKey, svc, makeGetFIMEndpoint(svc)),
ModifyFIM: authenticatedUser(jwtKey, svc, makeModifyFIMEndpoint(svc)),
// Authenticated status endpoints
StatusResultStore: authenticatedUser(jwtKey, svc, makeStatusResultStoreEndpoint(svc)),
@ -280,9 +270,6 @@ type kolideHandlers struct {
ListHosts http.Handler
GetHostSummary http.Handler
SearchTargets http.Handler
GetOptions http.Handler
ModifyOptions http.Handler
ResetOptions http.Handler
ApplyOsqueryOptionsSpec http.Handler
GetOsqueryOptionsSpec http.Handler
GetCertificate http.Handler
@ -290,8 +277,6 @@ type kolideHandlers struct {
InitiateSSO http.Handler
CallbackSSO http.Handler
SettingsSSO http.Handler
ModifyFIM http.Handler
GetFIM http.Handler
StatusResultStore http.Handler
StatusLiveQuery http.Handler
}
@ -372,9 +357,6 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
ListHosts: newServer(e.ListHosts, decodeListHostsRequest),
GetHostSummary: newServer(e.GetHostSummary, decodeNoParamsRequest),
SearchTargets: newServer(e.SearchTargets, decodeSearchTargetsRequest),
GetOptions: newServer(e.GetOptions, decodeNoParamsRequest),
ModifyOptions: newServer(e.ModifyOptions, decodeModifyOptionsRequest),
ResetOptions: newServer(e.ResetOptions, decodeNoParamsRequest),
ApplyOsqueryOptionsSpec: newServer(e.ApplyOsqueryOptionsSpec, decodeApplyOsqueryOptionsSpecRequest),
GetOsqueryOptionsSpec: newServer(e.GetOsqueryOptionsSpec, decodeNoParamsRequest),
GetCertificate: newServer(e.GetCertificate, decodeNoParamsRequest),
@ -382,8 +364,6 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
InitiateSSO: newServer(e.InitiateSSO, decodeInitiateSSORequest),
CallbackSSO: newServer(e.CallbackSSO, decodeCallbackSSORequest),
SettingsSSO: newServer(e.SSOSettings, decodeNoParamsRequest),
ModifyFIM: newServer(e.ModifyFIM, decodeModifyFIMRequest),
GetFIM: newServer(e.GetFIM, decodeNoParamsRequest),
StatusResultStore: newServer(e.StatusResultStore, decodeNoParamsRequest),
StatusLiveQuery: newServer(e.StatusLiveQuery, decodeNoParamsRequest),
}
@ -506,12 +486,6 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET").Name("get_host")
r.Handle("/api/v1/kolide/hosts/{id}", h.DeleteHost).Methods("DELETE").Name("delete_host")
r.Handle("/api/v1/kolide/fim", h.GetFIM).Methods("GET").Name("get_fim")
r.Handle("/api/v1/kolide/fim", h.ModifyFIM).Methods("PATCH").Name("post_fim")
r.Handle("/api/v1/kolide/options", h.GetOptions).Methods("GET").Name("get_options")
r.Handle("/api/v1/kolide/options", h.ModifyOptions).Methods("PATCH").Name("modify_options")
r.Handle("/api/v1/kolide/options/reset", h.ResetOptions).Methods("GET").Name("reset_options")
r.Handle("/api/v1/kolide/spec/osquery_options", h.ApplyOsqueryOptionsSpec).Methods("POST").Name("apply_osquery_options_spec")
r.Handle("/api/v1/kolide/spec/osquery_options", h.GetOsqueryOptionsSpec).Methods("GET").Name("get_osquery_options_spec")

32
server/service/logging_fim.go
View file

@ -1,32 +0,0 @@
package service
import (
"context"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw loggingMiddleware) GetFIM(ctx context.Context) (cfg *kolide.FIMConfig, err error) {
defer func(begin time.Time) {
_ = mw.loggerDebug(err).Log(
"method", "GetFIM",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
cfg, err = mw.Service.GetFIM(ctx)
return cfg, err
}
func (mw loggingMiddleware) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) (err error) {
defer func(begin time.Time) {
_ = mw.loggerInfo(err).Log(
"method", "ModifyFIM",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
err = mw.Service.ModifyFIM(ctx, fim)
return err
}

66
server/service/logging_options.go
View file

@ -1,66 +0,0 @@
package service
import (
"context"
"time"
"github.com/kolide/fleet/server/contexts/viewer"
"github.com/kolide/fleet/server/kolide"
)
func (mw loggingMiddleware) GetOptions(ctx context.Context) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
)
defer func(begin time.Time) {
_ = mw.loggerDebug(err).Log(
"method", "GetOptions",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
options, err = mw.Service.GetOptions(ctx)
return options, err
}
func (mw loggingMiddleware) ModifyOptions(ctx context.Context, req kolide.OptionRequest) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
loggedInUser = "unauthenticated"
)
if vc, ok := viewer.FromContext(ctx); ok {
loggedInUser = vc.Username()
}
defer func(begin time.Time) {
_ = mw.loggerInfo(err).Log(
"method", "ModifyOptions",
"err", err,
"user", loggedInUser,
"took", time.Since(begin),
)
}(time.Now())
options, err = mw.Service.ModifyOptions(ctx, req)
return options, err
}
func (mw loggingMiddleware) ResetOptions(ctx context.Context) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
)
defer func(begin time.Time) {
_ = mw.loggerInfo(err).Log(
"method", "ResetOptions",
"err", err,
"took", time.Since(begin),
)
}(time.Now())
options, err = mw.Service.ResetOptions(ctx)
return options, err
}

29
server/service/metrics_fim.go
View file

@ -1,29 +0,0 @@
package service
import (
"context"
"fmt"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw metricsMiddleware) GetFIM(ctx context.Context) (cfg *kolide.FIMConfig, err error) {
defer func(begin time.Time) {
lvs := []string{"method", "GetFIM", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
cfg, err = mw.Service.GetFIM(ctx)
return cfg, err
}
func (mw metricsMiddleware) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) (err error) {
defer func(begin time.Time) {
lvs := []string{"method", "ModifyFIM", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
err = mw.Service.ModifyFIM(ctx, fim)
return err
}

51
server/service/metrics_options.go
View file

@ -1,51 +0,0 @@
package service
import (
"context"
"fmt"
"time"
"github.com/kolide/fleet/server/kolide"
)
func (mw metricsMiddleware) GetOptions(ctx context.Context) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
)
defer func(begin time.Time) {
lvs := []string{"method", "GetOptions", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
options, err = mw.Service.GetOptions(ctx)
return options, err
}
func (mw metricsMiddleware) ModifyOptions(ctx context.Context, or kolide.OptionRequest) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
)
defer func(begin time.Time) {
lvs := []string{"method", "ModifyOptions", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
options, err = mw.Service.ModifyOptions(ctx, or)
return options, err
}
func (mw metricsMiddleware) ResetOptions(ctx context.Context) ([]kolide.Option, error) {
var (
options []kolide.Option
err error
)
defer func(begin time.Time) {
lvs := []string{"method", "ResetOptions", "error", fmt.Sprint(err != nil)}
mw.requestCount.With(lvs...).Add(1)
mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())
options, err = mw.Service.ResetOptions(ctx)
return options, err
}

66
server/service/service_fim.go
View file

@ -1,66 +0,0 @@
package service
import (
"context"
"encoding/json"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
func (svc service) GetFIM(ctx context.Context) (*kolide.FIMConfig, error) {
config, err := svc.ds.AppConfig()
if err != nil {
return nil, errors.Wrap(err, "getting fim config")
}
paths, err := svc.ds.FIMSections()
if err != nil {
return nil, errors.Wrap(err, "getting fim paths")
}
var arr []string
if len(config.FIMFileAccesses) > 0 {
if err = json.Unmarshal([]byte(config.FIMFileAccesses), &arr); err != nil {
return nil, errors.Wrap(err, "Error reading fim section, fileaccesses must be formatted as an array [\"cassandra\",\"etc\",\"homes\"]")
}
}
result := &kolide.FIMConfig{
Interval: uint(config.FIMInterval),
FilePaths: paths,
FileAccesses: arr,
}
return result, nil
}
// ModifyFIM will remove existing FIM settings and replace it
func (svc service) ModifyFIM(ctx context.Context, fim kolide.FIMConfig) error {
if err := svc.ds.ClearFIMSections(); err != nil {
return errors.Wrap(err, "updating fim")
}
config, err := svc.ds.AppConfig()
if err != nil {
return errors.Wrap(err, "updating fim")
}
config.FIMInterval = int(fim.Interval)
if len(fim.FileAccesses) > 0 {
fileAccesses, err := json.Marshal(fim.FileAccesses)
if err != nil {
return errors.Wrap(err, "Error creating fim section, fileaccesses must be formatted as an array [\"cassandra\",\"etc\",\"homes\"]")
}
config.FIMFileAccesses = string(fileAccesses)
}
for sectionName, paths := range fim.FilePaths {
section := kolide.FIMSection{
SectionName: sectionName,
Paths: paths,
}
if _, err := svc.ds.NewFIMSection(&section); err != nil {
return errors.Wrap(err, "creating fim section")
}
}
return svc.ds.SaveAppConfig(config)
}

101
server/service/service_fim_test.go
View file

@ -1,101 +0,0 @@
package service
import (
"context"
"testing"
"github.com/kolide/fleet/server/kolide"
"github.com/kolide/fleet/server/mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetFIMService(t *testing.T) {
fileAccessesString := "[\"etc\", \"home\", \"cassandra\"]"
fileAccessStringValue := []string{"etc", "home", "cassandra"}
fimIntervalTestValue := 500 //300 is the default value
ds := &mock.Store{
AppConfigStore: mock.AppConfigStore{
AppConfigFunc: func() (*kolide.AppConfig, error) {
config := &kolide.AppConfig{
FIMInterval: fimIntervalTestValue,
FIMFileAccesses: fileAccessesString,
}
return config, nil
},
},
FileIntegrityMonitoringStore: mock.FileIntegrityMonitoringStore{
FIMSectionsFunc: func() (kolide.FIMSections, error) {
result := kolide.FIMSections{
"etc": []string{
"/etc/config/%%",
"/etc/zipp",
},
}
return result, nil
},
},
}
svc := service{
ds: ds,
}
resp, err := svc.GetFIM(context.Background())
require.Nil(t, err)
require.NotNil(t, resp)
assert.Equal(t, resp.Interval, uint(fimIntervalTestValue))
assert.Equal(t, resp.FileAccesses, fileAccessStringValue)
paths, ok := resp.FilePaths["etc"]
require.True(t, ok)
assert.Len(t, paths, 2)
}
func TestUpdateFIM(t *testing.T) {
fileAccessesString := "[\"etc\", \"home\", \"cassandra\"]"
fileAccessStringValue := []string{"etc", "home", "cassandra"}
fimIntervalTestValue := 500 //300 is the default value
ds := &mock.Store{
AppConfigStore: mock.AppConfigStore{
AppConfigFunc: func() (*kolide.AppConfig, error) {
config := &kolide.AppConfig{
FIMInterval: fimIntervalTestValue,
FIMFileAccesses: fileAccessesString,
}
return config, nil
},
SaveAppConfigFunc: func(_ *kolide.AppConfig) error {
return nil
},
},
FileIntegrityMonitoringStore: mock.FileIntegrityMonitoringStore{
ClearFIMSectionsFunc: func() error {
return nil
},
NewFIMSectionFunc: func(fs *kolide.FIMSection, _ ...kolide.OptionalArg) (*kolide.FIMSection, error) {
fs.ID = 1
return fs, nil
},
},
}
svc := service{
ds: ds,
}
fim := kolide.FIMConfig{
Interval: uint(fimIntervalTestValue),
FileAccesses: fileAccessStringValue,
FilePaths: kolide.FIMSections{
"etc": []string{
"/etc/config/%%",
"/etc/zipp",
},
},
}
err := svc.ModifyFIM(context.Background(), fim)
require.Nil(t, err)
assert.True(t, ds.NewFIMSectionFuncInvoked)
assert.True(t, ds.ClearFIMSectionsFuncInvoked)
assert.True(t, ds.AppConfigFuncInvoked)
assert.True(t, ds.SaveAppConfigFuncInvoked)
}

26
server/service/service_options.go
View file

@ -1,26 +0,0 @@
package service
import (
"context"
"github.com/kolide/fleet/server/kolide"
)
func (svc service) ResetOptions(ctx context.Context) ([]kolide.Option, error) {
return svc.ds.ResetOptions()
}
func (svc service) GetOptions(ctx context.Context) ([]kolide.Option, error) {
opts, err := svc.ds.ListOptions()
if err != nil {
return nil, err
}
return opts, nil
}
func (svc service) ModifyOptions(ctx context.Context, req kolide.OptionRequest) ([]kolide.Option, error) {
if err := svc.ds.SaveOptions(req.Options); err != nil {
return nil, err
}
return req.Options, nil
}

17
server/service/transport_fim.go
View file

@ -1,17 +0,0 @@
package service
import (
"context"
"encoding/json"
"net/http"
"github.com/kolide/fleet/server/kolide"
)
func decodeModifyFIMRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var fimConfig kolide.FIMConfig
if err := json.NewDecoder(r.Body).Decode(&fimConfig); err != nil {
return nil, err
}
return fimConfig, nil
}

17
server/service/transport_options.go
View file

@ -1,17 +0,0 @@
package service
import (
"context"
"encoding/json"
"net/http"
"github.com/kolide/fleet/server/kolide"
)
func decodeModifyOptionsRequest(ctx context.Context, r *http.Request) (interface{}, error) {
var req kolide.OptionRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
return nil, err
}
return req, nil
}

53
server/service/validation_options.go
View file

@ -1,53 +0,0 @@
package service
import (
"context"
"errors"
"github.com/kolide/fleet/server/kolide"
)
func (mw validationMiddleware) ModifyOptions(ctx context.Context, req kolide.OptionRequest) ([]kolide.Option, error) {
invalid := &invalidArgumentError{}
for _, opt := range req.Options {
if opt.ReadOnly {
invalid.Append(opt.Name, "readonly option")
continue
}
if err := validateValueMapsToOptionType(opt); err != nil {
invalid.Append(opt.Name, err.Error())
}
}
if invalid.HasErrors() {
return nil, invalid
}
return mw.Service.ModifyOptions(ctx, req)
}
var (
errTypeMismatch = errors.New("type mismatch")
errInvalidType = errors.New("invalid option type")
)
func validateValueMapsToOptionType(opt kolide.Option) error {
if !opt.OptionSet() {
return nil
}
switch opt.GetValue().(type) {
case int, uint, uint64, float64:
if opt.Type != kolide.OptionTypeInt {
return errTypeMismatch
}
case string:
if opt.Type != kolide.OptionTypeString {
return errTypeMismatch
}
case bool:
if opt.Type != kolide.OptionTypeBool {
return errTypeMismatch
}
default:
return errInvalidType
}
return nil
}