From 693600ba2bc8f838be17c8be0a56dd8442d829a4 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 28 Mar 2017 16:45:18 -0500 Subject: [PATCH] Decorator support (#1430) * Added migrations * Added handler for decorators * Added logging and metrics for decorators * WIP decorators * Wip added decorator service * Added service implementation * Added mock decorator * Added modify decorator * Added testing * Addressed code review issues raised by @zwass * Added logging for missing type per @zwass --- server/datastore/datastore_decorators_test.go | 9 + server/datastore/inmem/decorators.go | 10 + server/datastore/mysql/decorators.go | 20 ++ .../20170309091824_AddBuiltInDecorators.go | 40 ++++ ...0170309100733_AddBuiltInColToDecorators.go | 21 ++ server/kolide/decorators.go | 88 +++++++- server/kolide/service.go | 1 + server/mock/datastore.go | 3 +- server/mock/dateastore_decorators.go | 59 ++++++ server/service/endpoint_decorators.go | 78 +++++++ server/service/endpoint_decorators_test.go | 199 ++++++++++++++++++ server/service/endpoint_test.go | 7 + server/service/handler.go | 21 ++ server/service/logging_decorators.go | 69 ++++++ server/service/metrics_decorators.go | 62 ++++++ server/service/service_decorators.go | 45 ++++ server/service/service_osquery.go | 33 ++- server/service/transport_decorators.go | 39 ++++ server/service/validation_decorators.go | 103 +++++++++ server/service/validation_decorators_test.go | 143 +++++++++++++ 20 files changed, 1036 insertions(+), 14 deletions(-) create mode 100644 server/datastore/mysql/migrations/data/20170309091824_AddBuiltInDecorators.go create mode 100644 server/datastore/mysql/migrations/tables/20170309100733_AddBuiltInColToDecorators.go create mode 100644 server/mock/dateastore_decorators.go create mode 100644 server/service/endpoint_decorators.go create mode 100644 server/service/endpoint_decorators_test.go create mode 100644 server/service/logging_decorators.go create mode 100644 server/service/metrics_decorators.go create mode 100644 server/service/service_decorators.go create mode 100644 server/service/transport_decorators.go create mode 100644 server/service/validation_decorators.go create mode 100644 server/service/validation_decorators_test.go diff --git a/server/datastore/datastore_decorators_test.go b/server/datastore/datastore_decorators_test.go index d80ffac82e..216bf71914 100644 --- a/server/datastore/datastore_decorators_test.go +++ b/server/datastore/datastore_decorators_test.go @@ -23,8 +23,17 @@ func testDecorators(t *testing.T, ds kolide.Datastore) { results, err := ds.ListDecorators() require.Nil(t, err) assert.Len(t, results, 1) + + decorator.Query = "select foo from bar;" + err = ds.SaveDecorator(decorator) + require.Nil(t, err) + result, err = ds.Decorator(decorator.ID) + require.Nil(t, err) + assert.Equal(t, "select foo from bar;", result.Query) + err = ds.DeleteDecorator(decorator.ID) require.Nil(t, err) result, err = ds.Decorator(decorator.ID) assert.NotNil(t, err) + } diff --git a/server/datastore/inmem/decorators.go b/server/datastore/inmem/decorators.go index 4c2e19b735..333457862b 100644 --- a/server/datastore/inmem/decorators.go +++ b/server/datastore/inmem/decorators.go @@ -2,6 +2,16 @@ package inmem import "github.com/kolide/kolide/server/kolide" +func (d *Datastore) SaveDecorator(dec *kolide.Decorator) error { + d.mtx.Lock() + defer d.mtx.Unlock() + if _, ok := d.decorators[dec.ID]; !ok { + return notFound("Decorator") + } + d.decorators[dec.ID] = dec + return nil +} + func (d *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) { d.mtx.Lock() defer d.mtx.Unlock() diff --git a/server/datastore/mysql/decorators.go b/server/datastore/mysql/decorators.go index 4eeaf5ec69..977b61b4d3 100644 --- a/server/datastore/mysql/decorators.go +++ b/server/datastore/mysql/decorators.go @@ -8,6 +8,26 @@ import ( "github.com/kolide/kolide/server/kolide" ) +func (ds *Datastore) SaveDecorator(dec *kolide.Decorator) error { + sqlStatement := + "UPDATE decorators SET " + + "`query` = ?, " + + "`type` = ?, " + + "`interval` = ? " + + "WHERE id = ?" + _, err := ds.db.Exec( + sqlStatement, + dec.Query, + dec.Type, + dec.Interval, + dec.ID, + ) + if err != nil { + return errors.Wrap(err, "saving decorator") + } + return nil +} + func (ds *Datastore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) { sqlStatement := "INSERT INTO decorators (" + diff --git a/server/datastore/mysql/migrations/data/20170309091824_AddBuiltInDecorators.go b/server/datastore/mysql/migrations/data/20170309091824_AddBuiltInDecorators.go new file mode 100644 index 0000000000..c10bc29280 --- /dev/null +++ b/server/datastore/mysql/migrations/data/20170309091824_AddBuiltInDecorators.go @@ -0,0 +1,40 @@ +package data + +import ( + "database/sql" + + "github.com/kolide/kolide/server/kolide" +) + +func init() { + MigrationClient.AddMigration(Up_20170309091824, Down_20170309091824) +} + +func Up_20170309091824(tx *sql.Tx) error { + sql := "INSERT INTO decorators (" + + "`type`, " + + "`query`, " + + "`built_in`, " + + "`interval`" + + ") VALUES ( ?, ?, TRUE, 0 )" + + rows := []struct { + t kolide.DecoratorType + q string + }{ + {kolide.DecoratorLoad, "SELECT uuid AS host_uuid FROM system_info;"}, + {kolide.DecoratorLoad, "SELECT hostname AS hostname FROM system_info;"}, + } + for _, row := range rows { + _, err := tx.Exec(sql, row.t, row.q) + if err != nil { + return err + } + } + return nil +} + +func Down_20170309091824(tx *sql.Tx) error { + _, err := tx.Exec("DELETE FROM decorators WHERE built_in = TRUE") + return err +} diff --git a/server/datastore/mysql/migrations/tables/20170309100733_AddBuiltInColToDecorators.go b/server/datastore/mysql/migrations/tables/20170309100733_AddBuiltInColToDecorators.go new file mode 100644 index 0000000000..36b9eefd04 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20170309100733_AddBuiltInColToDecorators.go @@ -0,0 +1,21 @@ +package tables + +import ( + "database/sql" +) + +func init() { + MigrationClient.AddMigration(Up_20170309100733, Down_20170309100733) +} + +func Up_20170309100733(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE `decorators` " + + "ADD COLUMN built_in TINYINT(1) NOT NULL DEFAULT FALSE;") + return err +} + +func Down_20170309100733(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE `decorators` " + + "DROP COLUMN `built_in`;") + return err +} diff --git a/server/kolide/decorators.go b/server/kolide/decorators.go index 8fb1317adb..74041546d0 100644 --- a/server/kolide/decorators.go +++ b/server/kolide/decorators.go @@ -1,5 +1,12 @@ package kolide +import ( + "errors" + "strings" + + "golang.org/x/net/context" +) + // DecoratorStore methods to manipulate decorator queries. // See https://osquery.readthedocs.io/en/stable/deployment/configuration/ type DecoratorStore interface { @@ -11,6 +18,21 @@ type DecoratorStore interface { Decorator(id uint) (*Decorator, error) // ListDecorators returns all decorator queries. ListDecorators() ([]*Decorator, error) + // SaveDecorator updates an existing decorator + SaveDecorator(dec *Decorator) error +} + +// DecoratorService exposes decorators data so it can be manipulated by +// end users +type DecoratorService interface { + // ListDecorators returns decorators + ListDecorators(ctx context.Context) ([]*Decorator, error) + // DeleteDecorator removes an existing decorator if it is not built-in + DeleteDecorator(ctx context.Context, id uint) error + // NewDecorator creates a new decorator + NewDecorator(ctx context.Context, payload DecoratorPayload) (*Decorator, error) + // ModifyDecorator updates an existing decorator + ModifyDecorator(ctx context.Context, payload DecoratorPayload) (*Decorator, error) } // DecoratorType refers to the allowable types of decorator queries. @@ -21,14 +43,70 @@ const ( DecoratorLoad DecoratorType = iota DecoratorAlways DecoratorInterval + DecoratorUndefined + + DecoratorLoadName = "load" + DecoratorAlwaysName = "always" + DecoratorIntervalName = "interval" ) +func (dt DecoratorType) String() string { + switch dt { + case DecoratorLoad: + return DecoratorLoadName + case DecoratorAlways: + return DecoratorAlwaysName + case DecoratorInterval: + return DecoratorIntervalName + default: + return "" + } +} + +func (dt *DecoratorType) MarshalJSON() ([]byte, error) { + name := dt.String() + if name == "" { + return nil, errors.New("Invalid decorator type") + } + return []byte(`"` + name + `"`), nil +} + +var decNameToType = map[string]DecoratorType{ + DecoratorLoadName: DecoratorLoad, + DecoratorAlwaysName: DecoratorAlways, + DecoratorIntervalName: DecoratorInterval, +} + +func (dt *DecoratorType) UnmarshalJSON(data []byte) error { + name := strings.Trim(string(data), `"`) + switch name { + case DecoratorLoadName: + *dt = DecoratorLoad + case DecoratorAlwaysName: + *dt = DecoratorAlways + case DecoratorIntervalName: + *dt = DecoratorInterval + default: + *dt = DecoratorUndefined + } + return nil +} + // Decorator contains information about a decorator query. type Decorator struct { UpdateCreateTimestamps - ID uint - Type DecoratorType - // Interval note this is only pertainent for DecoratorInterval type. - Interval uint - Query string + ID uint `json:"id"` + Type DecoratorType `json:"type"` + // Interval note this is only pertinent for DecoratorInterval type. + Interval uint `json:"interval"` + Query string `json:"query"` + // BuiltIn decorators are loaded in migrations and may not be changed + BuiltIn bool `json:"built_in" db:"built_in"` +} + +type DecoratorPayload struct { + ID uint `json:"id"` + DecoratorType *DecoratorType `json:"decorator_type"` + Interval *uint `json:"interval"` + Query *string `json:"query"` } diff --git a/server/kolide/service.go b/server/kolide/service.go index 8f5bd11a21..9e0c9042ca 100644 --- a/server/kolide/service.go +++ b/server/kolide/service.go @@ -17,4 +17,5 @@ type Service interface { OptionService ImportConfigService LicenseService + DecoratorService } diff --git a/server/mock/datastore.go b/server/mock/datastore.go index 976fd2cbb6..ad07190ac5 100644 --- a/server/mock/datastore.go +++ b/server/mock/datastore.go @@ -5,6 +5,7 @@ package mock //go:generate mockimpl -o datastore_appconfig.go "s *AppConfigStore" "kolide.AppConfigStore" //go:generate mockimpl -o datastore_licenses.go "s *LicenseStore" "kolide.LicenseStore" //go:generate mockimpl -o datastore_labels.go "s *LabelStore" "kolide.LabelStore" +//go:generate mockimpl -o dateastore_decorators.go "s *DecoratorStore" "kolide.DecoratorStore" import "github.com/kolide/kolide/server/kolide" @@ -19,7 +20,6 @@ type Store struct { kolide.QueryStore kolide.OptionStore kolide.ScheduledQueryStore - kolide.DecoratorStore kolide.FileIntegrityMonitoringStore kolide.YARAStore LicenseStore @@ -27,6 +27,7 @@ type Store struct { UserStore AppConfigStore LabelStore + DecoratorStore } func (m *Store) Drop() error { diff --git a/server/mock/dateastore_decorators.go b/server/mock/dateastore_decorators.go new file mode 100644 index 0000000000..8771269599 --- /dev/null +++ b/server/mock/dateastore_decorators.go @@ -0,0 +1,59 @@ +// Automatically generated by mockimpl. DO NOT EDIT! + +package mock + +import "github.com/kolide/kolide/server/kolide" + +var _ kolide.DecoratorStore = (*DecoratorStore)(nil) + +type NewDecoratorFunc func(decorator *kolide.Decorator) (*kolide.Decorator, error) + +type DeleteDecoratorFunc func(id uint) error + +type DecoratorFunc func(id uint) (*kolide.Decorator, error) + +type ListDecoratorsFunc func() ([]*kolide.Decorator, error) + +type SaveDecoratorFunc func(dec *kolide.Decorator) error + +type DecoratorStore struct { + NewDecoratorFunc NewDecoratorFunc + NewDecoratorFuncInvoked bool + + DeleteDecoratorFunc DeleteDecoratorFunc + DeleteDecoratorFuncInvoked bool + + DecoratorFunc DecoratorFunc + DecoratorFuncInvoked bool + + ListDecoratorsFunc ListDecoratorsFunc + ListDecoratorsFuncInvoked bool + + SaveDecoratorFunc SaveDecoratorFunc + SaveDecoratorFuncInvoked bool +} + +func (s *DecoratorStore) NewDecorator(decorator *kolide.Decorator) (*kolide.Decorator, error) { + s.NewDecoratorFuncInvoked = true + return s.NewDecoratorFunc(decorator) +} + +func (s *DecoratorStore) DeleteDecorator(id uint) error { + s.DeleteDecoratorFuncInvoked = true + return s.DeleteDecoratorFunc(id) +} + +func (s *DecoratorStore) Decorator(id uint) (*kolide.Decorator, error) { + s.DecoratorFuncInvoked = true + return s.DecoratorFunc(id) +} + +func (s *DecoratorStore) ListDecorators() ([]*kolide.Decorator, error) { + s.ListDecoratorsFuncInvoked = true + return s.ListDecoratorsFunc() +} + +func (s *DecoratorStore) SaveDecorator(dec *kolide.Decorator) error { + s.SaveDecoratorFuncInvoked = true + return s.SaveDecoratorFunc(dec) +} diff --git a/server/service/endpoint_decorators.go b/server/service/endpoint_decorators.go new file mode 100644 index 0000000000..3628350d62 --- /dev/null +++ b/server/service/endpoint_decorators.go @@ -0,0 +1,78 @@ +package service + +import ( + "github.com/go-kit/kit/endpoint" + "github.com/kolide/kolide/server/kolide" + "golang.org/x/net/context" +) + +type listDecoratorResponse struct { + Decorators []*kolide.Decorator `json:"decorators,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r listDecoratorResponse) error() error { return r.Err } + +func makeListDecoratorsEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + decs, err := svc.ListDecorators(ctx) + if err != nil { + return listDecoratorResponse{Err: err}, nil + } + return listDecoratorResponse{Decorators: decs}, nil + } +} + +type newDecoratorRequest struct { + Payload kolide.DecoratorPayload `json:"payload"` +} + +type decoratorResponse struct { + Decorator *kolide.Decorator `json:"decorator,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r decoratorResponse) error() error { return r.Err } + +func makeNewDecoratorEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + r := request.(newDecoratorRequest) + dec, err := svc.NewDecorator(ctx, r.Payload) + if err != nil { + return decoratorResponse{Err: err}, nil + } + return decoratorResponse{Decorator: dec}, nil + } +} + +type deleteDecoratorRequest struct { + ID uint +} + +type deleteDecoratorResponse struct { + Err error `json:"error,omitempty"` +} + +func (r deleteDecoratorResponse) error() error { return r.Err } + +func makeDeleteDecoratorEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + r := request.(deleteDecoratorRequest) + err := svc.DeleteDecorator(ctx, r.ID) + if err != nil { + return deleteDecoratorResponse{Err: err}, nil + } + return deleteDecoratorResponse{}, nil + } +} + +func makeModifyDecoratorEndpoint(svc kolide.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + r := request.(newDecoratorRequest) + dec, err := svc.ModifyDecorator(ctx, r.Payload) + if err != nil { + return decoratorResponse{Err: err}, nil + } + return decoratorResponse{Decorator: dec}, nil + } +} diff --git a/server/service/endpoint_decorators_test.go b/server/service/endpoint_decorators_test.go new file mode 100644 index 0000000000..f8b14a42a7 --- /dev/null +++ b/server/service/endpoint_decorators_test.go @@ -0,0 +1,199 @@ +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/kolide/kolide/server/kolide" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupDecoratorTest(r *testResource) { + decs := []kolide.Decorator{ + kolide.Decorator{ + Type: kolide.DecoratorLoad, + Query: "select something from foo;", + }, + kolide.Decorator{ + Type: kolide.DecoratorLoad, + Query: "select bar from foo;", + }, + kolide.Decorator{ + Type: kolide.DecoratorAlways, + Query: "select x from y;", + }, + kolide.Decorator{ + Type: kolide.DecoratorInterval, + Query: "select name from system_info;", + Interval: 3600, + }, + } + for _, d := range decs { + r.ds.NewDecorator(&d) + } +} + +func testModifyDecorator(t *testing.T, r *testResource) { + dec := &kolide.Decorator{ + Type: kolide.DecoratorLoad, + Query: "select foo from bar;", + } + dec, err := r.ds.NewDecorator(dec) + require.Nil(t, err) + buffer := bytes.NewBufferString(`{ + "payload": { + "query": "select baz from boom;" + } + }`) + req, err := http.NewRequest("PATCH", r.server.URL+fmt.Sprintf("/api/v1/kolide/decorators/%d", dec.ID), buffer) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + + var decResp decoratorResponse + err = json.NewDecoder(resp.Body).Decode(&decResp) + require.Nil(t, err) + require.NotNil(t, decResp.Decorator) + assert.Equal(t, "select baz from boom;", decResp.Decorator.Query) +} + +// This test verifies that we can submit the same payload twice without +// raising an error +func testModifyDecoratorNoChanges(t *testing.T, r *testResource) { + dec := &kolide.Decorator{ + Type: kolide.DecoratorLoad, + Query: "select foo from bar;", + } + dec, err := r.ds.NewDecorator(dec) + require.Nil(t, err) + buffer := bytes.NewBufferString(`{ + "payload": { + "decorator_type": "load", + "query": "select foo from bar;" + } + }`) + req, err := http.NewRequest("PATCH", r.server.URL+fmt.Sprintf("/api/v1/kolide/decorators/%d", dec.ID), buffer) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + + var decResp decoratorResponse + err = json.NewDecoder(resp.Body).Decode(&decResp) + require.Nil(t, err) + require.NotNil(t, decResp.Decorator) + assert.Equal(t, "select foo from bar;", decResp.Decorator.Query) + assert.Equal(t, kolide.DecoratorLoad, decResp.Decorator.Type) +} + +func testListDecorator(t *testing.T, r *testResource) { + setupDecoratorTest(r) + req, err := http.NewRequest("GET", r.server.URL+"/api/v1/kolide/decorators", nil) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + var decs listDecoratorResponse + err = json.NewDecoder(resp.Body).Decode(&decs) + require.Nil(t, err) + + assert.Len(t, decs.Decorators, 4) +} + +func testNewDecorator(t *testing.T, r *testResource) { + buffer := bytes.NewBufferString( + `{ + "payload": { + "decorator_type": "load", + "query": "select x from y;" + } + }`) + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/decorators", buffer) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + var dec decoratorResponse + err = json.NewDecoder(resp.Body).Decode(&dec) + require.Nil(t, err) + require.NotNil(t, dec.Decorator) + assert.Equal(t, kolide.DecoratorLoad, dec.Decorator.Type) + assert.Equal(t, "select x from y;", dec.Decorator.Query) +} + +// invalid json +func testNewDecoratorFailType(t *testing.T, r *testResource) { + buffer := bytes.NewBufferString( + `{ + "payload": { + "decorator_type": "zip", + "query": "select x from y;" + } + }`) + + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/decorators", buffer) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + + var errStruct mockValidationError + err = json.NewDecoder(resp.Body).Decode(&errStruct) + require.Nil(t, err) + require.Len(t, errStruct.Errors, 1) + assert.Equal(t, "invalid value, must be load, always, or interval", errStruct.Errors[0].Reason) + +} + +func testNewDecoratorFailValidation(t *testing.T, r *testResource) { + buffer := bytes.NewBufferString( + `{ + "payload": { + "decorator_type": "interval", + "query": "select x from y;", + "interval": 3601 + } + }`) + + req, err := http.NewRequest("POST", r.server.URL+"/api/v1/kolide/decorators", buffer) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + + var errStruct mockValidationError + err = json.NewDecoder(resp.Body).Decode(&errStruct) + require.Nil(t, err) + require.Len(t, errStruct.Errors, 1) + assert.Equal(t, "must be divisible by 60", errStruct.Errors[0].Reason) +} + +func testDeleteDecorator(t *testing.T, r *testResource) { + setupDecoratorTest(r) + req, err := http.NewRequest("DELETE", r.server.URL+"/api/v1/kolide/decorators/1", nil) + require.Nil(t, err) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.adminToken)) + client := &http.Client{} + resp, err := client.Do(req) + require.Nil(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + decs, _ := r.ds.ListDecorators() + assert.Len(t, decs, 3) +} diff --git a/server/service/endpoint_test.go b/server/service/endpoint_test.go index da592bffd7..7dfe1a57b3 100644 --- a/server/service/endpoint_test.go +++ b/server/service/endpoint_test.go @@ -115,6 +115,13 @@ var testFunctions = [...]func(*testing.T, *testResource){ testNonAdminUserSetAdmin, testAdminUserSetEnabled, testNonAdminUserSetEnabled, + testModifyDecorator, + testListDecorator, + testNewDecorator, + testNewDecoratorFailType, + testNewDecoratorFailValidation, + testDeleteDecorator, + testModifyDecoratorNoChanges, } func TestEndpoints(t *testing.T) { diff --git a/server/service/handler.go b/server/service/handler.go index 2c95bac2f9..b2f5058d94 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -66,6 +66,10 @@ type KolideEndpoints struct { CreateLabel endpoint.Endpoint DeleteLabel endpoint.Endpoint ModifyLabel endpoint.Endpoint + ListDecorators endpoint.Endpoint + NewDecorator endpoint.Endpoint + ModifyDecorator endpoint.Endpoint + DeleteDecorator endpoint.Endpoint GetHost endpoint.Endpoint DeleteHost endpoint.Endpoint ListHosts endpoint.Endpoint @@ -143,6 +147,10 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey string) KolideEndpoint CreateLabel: authenticatedUser(jwtKey, svc, makeCreateLabelEndpoint(svc)), DeleteLabel: authenticatedUser(jwtKey, svc, makeDeleteLabelEndpoint(svc)), ModifyLabel: authenticatedUser(jwtKey, svc, makeModifyLabelEndpoint(svc)), + ListDecorators: authenticatedUser(jwtKey, svc, makeListDecoratorsEndpoint(svc)), + NewDecorator: authenticatedUser(jwtKey, svc, makeNewDecoratorEndpoint(svc)), + ModifyDecorator: authenticatedUser(jwtKey, svc, makeModifyDecoratorEndpoint(svc)), + DeleteDecorator: authenticatedUser(jwtKey, svc, makeDeleteDecoratorEndpoint(svc)), SearchTargets: authenticatedUser(jwtKey, svc, makeSearchTargetsEndpoint(svc)), GetOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeGetOptionsEndpoint(svc))), ModifyOptions: authenticatedUser(jwtKey, svc, mustBeAdmin(makeModifyOptionsEndpoint(svc))), @@ -213,6 +221,10 @@ type kolideHandlers struct { CreateLabel http.Handler DeleteLabel http.Handler ModifyLabel http.Handler + ListDecorators http.Handler + NewDecorator http.Handler + ModifyDecorator http.Handler + DeleteDecorator http.Handler GetHost http.Handler DeleteHost http.Handler ListHosts http.Handler @@ -283,6 +295,10 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli CreateLabel: newServer(e.CreateLabel, decodeCreateLabelRequest), DeleteLabel: newServer(e.DeleteLabel, decodeDeleteLabelRequest), ModifyLabel: newServer(e.ModifyLabel, decodeModifyLabelRequest), + ListDecorators: newServer(e.ListDecorators, decodeNoParamsRequest), + NewDecorator: newServer(e.NewDecorator, decodeNewDecoratorRequest), + ModifyDecorator: newServer(e.ModifyDecorator, decodeModifyDecoratorRequest), + DeleteDecorator: newServer(e.DeleteDecorator, decodeDeleteDecoratorRequest), GetHost: newServer(e.GetHost, decodeGetHostRequest), DeleteHost: newServer(e.DeleteHost, decodeDeleteHostRequest), ListHosts: newServer(e.ListHosts, decodeListHostsRequest), @@ -391,6 +407,11 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) { r.Handle("/api/v1/kolide/labels/{id}", h.DeleteLabel).Methods("DELETE").Name("delete_label") r.Handle("/api/v1/kolide/labels/{id}", h.ModifyLabel).Methods("PATCH").Name("modify_label") + r.Handle("/api/v1/kolide/decorators", h.ListDecorators).Methods("GET").Name("list_decorators") + r.Handle("/api/v1/kolide/decorators", h.NewDecorator).Methods("POST").Name("create_decorator") + r.Handle("/api/v1/kolide/decorators/{id}", h.ModifyDecorator).Methods("PATCH").Name("modify_decorator") + r.Handle("/api/v1/kolide/decorators/{id}", h.DeleteDecorator).Methods("DELETE").Name("delete_decorator") + r.Handle("/api/v1/kolide/hosts", h.ListHosts).Methods("GET").Name("list_hosts") r.Handle("/api/v1/kolide/host_summary", h.GetHostSummary).Methods("GET").Name("get_host_summary") r.Handle("/api/v1/kolide/hosts/{id}", h.GetHost).Methods("GET").Name("get_host") diff --git a/server/service/logging_decorators.go b/server/service/logging_decorators.go new file mode 100644 index 0000000000..013f687fa5 --- /dev/null +++ b/server/service/logging_decorators.go @@ -0,0 +1,69 @@ +package service + +import ( + "time" + + "github.com/kolide/kolide/server/kolide" + "golang.org/x/net/context" +) + +func (mw loggingMiddleware) ListDecorators(ctx context.Context) ([]*kolide.Decorator, error) { + var ( + decs []*kolide.Decorator + err error + ) + defer func(begin time.Time) { + mw.logger.Log( + "method", "ListDecorators", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + decs, err = mw.Service.ListDecorators(ctx) + return decs, err +} + +func (mw loggingMiddleware) NewDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + var ( + dec *kolide.Decorator + err error + ) + defer func(begin time.Time) { + mw.logger.Log( + "method", "NewDecorator", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + dec, err = mw.Service.NewDecorator(ctx, payload) + return dec, err +} + +func (mw loggingMiddleware) ModifyDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + var ( + dec *kolide.Decorator + err error + ) + defer func(begin time.Time) { + mw.logger.Log( + "method", "ModifyDecorator", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + dec, err = mw.Service.ModifyDecorator(ctx, payload) + return dec, err +} + +func (mw loggingMiddleware) DeleteDecorator(ctx context.Context, id uint) error { + var err error + defer func(begin time.Time) { + mw.logger.Log( + "method", "DeleteDecorator", + "err", err, + "took", time.Since(begin), + ) + }(time.Now()) + err = mw.Service.DeleteDecorator(ctx, id) + return err +} diff --git a/server/service/metrics_decorators.go b/server/service/metrics_decorators.go new file mode 100644 index 0000000000..842ad44f48 --- /dev/null +++ b/server/service/metrics_decorators.go @@ -0,0 +1,62 @@ +package service + +import ( + "fmt" + "time" + + "github.com/kolide/kolide/server/kolide" + "golang.org/x/net/context" +) + +func (mw metricsMiddleware) ListDecorators(ctx context.Context) ([]*kolide.Decorator, error) { + var ( + decs []*kolide.Decorator + err error + ) + defer func(begin time.Time) { + lvs := []string{"method", "ListDecorators", "error", fmt.Sprint(err != nil)} + mw.requestCount.With(lvs...).Add(1) + mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + decs, err = mw.Service.ListDecorators(ctx) + return decs, err +} + +func (mw metricsMiddleware) NewDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + var ( + dec *kolide.Decorator + err error + ) + defer func(begin time.Time) { + lvs := []string{"method", "NewDecorator", "error", fmt.Sprint(err != nil)} + mw.requestCount.With(lvs...).Add(1) + mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + dec, err = mw.Service.NewDecorator(ctx, payload) + return dec, err +} + +func (mw metricsMiddleware) ModifyDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + var ( + dec *kolide.Decorator + err error + ) + defer func(begin time.Time) { + lvs := []string{"method", "ModifyDecorator", "error", fmt.Sprint(err != nil)} + mw.requestCount.With(lvs...).Add(1) + mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + dec, err = mw.Service.ModifyDecorator(ctx, payload) + return dec, err +} + +func (mw metricsMiddleware) DeleteDecorator(ctx context.Context, id uint) error { + var err error + defer func(begin time.Time) { + lvs := []string{"method", "DeleteDecorator", "error", fmt.Sprint(err != nil)} + mw.requestCount.With(lvs...).Add(1) + mw.requestLatency.With(lvs...).Observe(time.Since(begin).Seconds()) + }(time.Now()) + err = mw.Service.DeleteDecorator(ctx, id) + return err +} diff --git a/server/service/service_decorators.go b/server/service/service_decorators.go new file mode 100644 index 0000000000..62cc045eaa --- /dev/null +++ b/server/service/service_decorators.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/kolide/kolide/server/kolide" + "golang.org/x/net/context" +) + +func (svc service) ListDecorators(ctx context.Context) ([]*kolide.Decorator, error) { + return svc.ds.ListDecorators() +} + +func (svc service) DeleteDecorator(ctx context.Context, uid uint) error { + return svc.ds.DeleteDecorator(uid) +} + +func (svc service) NewDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + var dec kolide.Decorator + dec.Query = *payload.Query + dec.Type = *payload.DecoratorType + if payload.Interval != nil { + dec.Interval = *payload.Interval + } + return svc.ds.NewDecorator(&dec) +} + +func (svc service) ModifyDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + dec, err := svc.ds.Decorator(payload.ID) + if err != nil { + return nil, err + } + if payload.DecoratorType != nil { + dec.Type = *payload.DecoratorType + } + if payload.Query != nil { + dec.Query = *payload.Query + } + if payload.Interval != nil { + dec.Interval = *payload.Interval + } + err = svc.ds.SaveDecorator(dec) + if err != nil { + return nil, err + } + return dec, nil +} diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index be68fb0e96..4c303828d0 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -78,15 +78,32 @@ func (svc service) GetClientConfig(ctx context.Context) (*kolide.OsqueryConfig, return nil, osqueryError{message: "internal error: unable to fetch configuration options"} } + decorators, err := svc.ds.ListDecorators() + if err != nil { + return nil, osqueryError{message: "internal error: unable to fetch decorators"} + } + decConfig := kolide.Decorators{ + Interval: make(map[string][]string), + } + for _, dec := range decorators { + switch dec.Type { + case kolide.DecoratorLoad: + decConfig.Load = append(decConfig.Load, dec.Query) + case kolide.DecoratorAlways: + decConfig.Always = append(decConfig.Always, dec.Query) + case kolide.DecoratorInterval: + key := strconv.Itoa(int(dec.Interval)) + decConfig.Interval[key] = append(decConfig.Interval[key], dec.Query) + default: + svc.logger.Log("component", "service", "method", "GetClientConfig", "err", + "unknown decorator type") + } + } + config := &kolide.OsqueryConfig{ - Options: options, - Decorators: kolide.Decorators{ - Load: []string{ - "SELECT uuid AS host_uuid FROM system_info;", - "SELECT hostname AS hostname FROM system_info;", - }, - }, - Packs: kolide.Packs{}, + Options: options, + Decorators: decConfig, + Packs: kolide.Packs{}, } packs, err := svc.ListPacksForHost(ctx, host.ID) diff --git a/server/service/transport_decorators.go b/server/service/transport_decorators.go new file mode 100644 index 0000000000..c97a9ecdf6 --- /dev/null +++ b/server/service/transport_decorators.go @@ -0,0 +1,39 @@ +package service + +import ( + "encoding/json" + "net/http" + + "golang.org/x/net/context" +) + +func decodeNewDecoratorRequest(ctx context.Context, req *http.Request) (interface{}, error) { + var dec newDecoratorRequest + err := json.NewDecoder(req.Body).Decode(&dec) + if err != nil { + return nil, err + } + return dec, nil +} + +func decodeDeleteDecoratorRequest(ctx context.Context, req *http.Request) (interface{}, error) { + id, err := idFromRequest(req, "id") + if err != nil { + return nil, err + } + return deleteDecoratorRequest{ID: id}, nil +} + +func decodeModifyDecoratorRequest(ctx context.Context, req *http.Request) (interface{}, error) { + var request newDecoratorRequest + id, err := idFromRequest(req, "id") + if err != nil { + return nil, err + } + err = json.NewDecoder(req.Body).Decode(&request) + if err != nil { + return nil, err + } + request.Payload.ID = id + return request, nil +} diff --git a/server/service/validation_decorators.go b/server/service/validation_decorators.go new file mode 100644 index 0000000000..f77ce8cf59 --- /dev/null +++ b/server/service/validation_decorators.go @@ -0,0 +1,103 @@ +package service + +import ( + "github.com/kolide/kolide/server/kolide" + "golang.org/x/net/context" +) + +func validateNewDecoratorType(payload kolide.DecoratorPayload, invalid *invalidArgumentError) { + if payload.DecoratorType == nil { + invalid.Append("decorator_type", "missing required argument") + return + } + if *payload.DecoratorType == kolide.DecoratorUndefined { + invalid.Append("decorator_type", "invalid value, must be load, always, or interval") + return + } + if *payload.DecoratorType == kolide.DecoratorInterval { + if payload.Interval == nil { + invalid.Append("interval", "missing required argument") + return + } + if *payload.Interval%60 != 0 { + invalid.Append("interval", "must be divisible by 60") + return + } + } +} + +// NewDecorator validator checks to make sure that a valid decorator type exists and +// if the decorator is of an interval type, an interval value is present and is +// divisable by 60 +// See https://osquery.readthedocs.io/en/stable/deployment/configuration/ +func (mw validationMiddleware) NewDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + invalid := &invalidArgumentError{} + validateNewDecoratorType(payload, invalid) + + if payload.Query == nil { + invalid.Append("query", "missing required argument") + } + + if invalid.HasErrors() { + return nil, invalid + } + return mw.Service.NewDecorator(ctx, payload) +} + +func (mw validationMiddleware) validateModifyDecoratorType(payload kolide.DecoratorPayload, invalid *invalidArgumentError) error { + if payload.DecoratorType != nil { + + if *payload.DecoratorType == kolide.DecoratorUndefined { + invalid.Append("decorator_type", "invalid value, must be load, always, or interval") + return nil + } + if *payload.DecoratorType == kolide.DecoratorInterval { + // special processing for interval type + existingDec, err := mw.ds.Decorator(payload.ID) + if err != nil { + // if decorator is not present we want to return a 404 to the client + return err + } + // if the type has changed from always or load to interval we need to + // check suitability of interval value + if existingDec.Type != kolide.DecoratorInterval { + if payload.Interval == nil { + invalid.Append("interval", "missing required argument") + return nil + } + } + } + + if payload.Interval != nil { + if *payload.Interval%60 != 0 { + invalid.Append("interval", "value must be divisible by 60") + } + } + } + return nil +} + +func (mw validationMiddleware) ModifyDecorator(ctx context.Context, payload kolide.DecoratorPayload) (*kolide.Decorator, error) { + invalid := &invalidArgumentError{} + err := mw.validateModifyDecoratorType(payload, invalid) + if err != nil { + return nil, err + } + if invalid.HasErrors() { + return nil, invalid + } + return mw.Service.ModifyDecorator(ctx, payload) +} + +func (mw validationMiddleware) DeleteDecorator(ctx context.Context, id uint) error { + invalid := &invalidArgumentError{} + dec, err := mw.ds.Decorator(id) + if err != nil { + return err + } + if dec.BuiltIn { + invalid.Append("decorator", "read only") + return invalid + } + return mw.Service.DeleteDecorator(ctx, id) +} diff --git a/server/service/validation_decorators_test.go b/server/service/validation_decorators_test.go new file mode 100644 index 0000000000..241dcfc283 --- /dev/null +++ b/server/service/validation_decorators_test.go @@ -0,0 +1,143 @@ +package service + +import ( + "testing" + + "github.com/kolide/kolide/server/kolide" + "github.com/kolide/kolide/server/mock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +var dtPtr = func(t kolide.DecoratorType) *kolide.DecoratorType { return &t } + +func TestDecoratorValidation(t *testing.T) { + ds := mock.Store{} + ds.DecoratorFunc = func(id uint) (*kolide.Decorator, error) { + return &kolide.Decorator{ + ID: 1, + Query: "select x from y;", + Type: kolide.DecoratorAlways, + }, nil + } + ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error { + return nil + } + svc := &service{ + ds: &ds, + } + validator := validationMiddleware{ + Service: svc, + ds: &ds, + } + + payload := kolide.DecoratorPayload{ + ID: uint(1), + DecoratorType: dtPtr(kolide.DecoratorInterval), + Interval: uintPtr(3600), + } + + dec, err := validator.ModifyDecorator(context.Background(), payload) + require.Nil(t, err) + assert.Equal(t, kolide.DecoratorInterval, dec.Type) + assert.Equal(t, uint(3600), dec.Interval) +} + +func TestDecoratorValidationIntervalMissing(t *testing.T) { + ds := mock.Store{} + ds.DecoratorFunc = func(id uint) (*kolide.Decorator, error) { + return &kolide.Decorator{ + ID: 1, + Query: "select x from y;", + Type: kolide.DecoratorAlways, + }, nil + } + ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error { + return nil + } + svc := &service{ + ds: &ds, + } + validator := validationMiddleware{ + Service: svc, + ds: &ds, + } + + payload := kolide.DecoratorPayload{ + ID: uint(1), + DecoratorType: dtPtr(kolide.DecoratorInterval), + } + + _, err := validator.ModifyDecorator(context.Background(), payload) + require.NotNil(t, err) + r, ok := err.(*invalidArgumentError) + require.True(t, ok) + assert.Equal(t, "missing required argument", (*r)[0].reason) +} + +func TestDecoratorValidationIntervalSameType(t *testing.T) { + ds := mock.Store{} + ds.DecoratorFunc = func(id uint) (*kolide.Decorator, error) { + return &kolide.Decorator{ + ID: 1, + Query: "select x from y;", + Type: kolide.DecoratorInterval, + Interval: 600, + }, nil + } + ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error { + return nil + } + svc := &service{ + ds: &ds, + } + validator := validationMiddleware{ + Service: svc, + ds: &ds, + } + + payload := kolide.DecoratorPayload{ + ID: uint(1), + DecoratorType: dtPtr(kolide.DecoratorInterval), + Interval: uintPtr(1200), + } + + dec, err := validator.ModifyDecorator(context.Background(), payload) + require.Nil(t, err) + assert.Equal(t, uint(1200), dec.Interval) +} + +func TestDecoratorValidationIntervalInvalid(t *testing.T) { + ds := mock.Store{} + ds.DecoratorFunc = func(id uint) (*kolide.Decorator, error) { + return &kolide.Decorator{ + ID: 1, + Query: "select x from y;", + Type: kolide.DecoratorInterval, + Interval: 600, + }, nil + } + ds.SaveDecoratorFunc = func(dec *kolide.Decorator) error { + return nil + } + svc := &service{ + ds: &ds, + } + validator := validationMiddleware{ + Service: svc, + ds: &ds, + } + + payload := kolide.DecoratorPayload{ + ID: uint(1), + DecoratorType: dtPtr(kolide.DecoratorInterval), + Interval: uintPtr(1203), + } + + _, err := validator.ModifyDecorator(context.Background(), payload) + require.NotNil(t, err) + r, ok := err.(*invalidArgumentError) + require.True(t, ok) + assert.Equal(t, "value must be divisible by 60", (*r)[0].reason) +}