From 619e36755c1225cb865023569a89546db03ca117 Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Thu, 21 May 2020 08:36:00 -0700 Subject: [PATCH] Add capability to collect "additional" information from hosts (#2236) Additional information is collected when host details are updated using the queries specified in the Fleet configuration. This additional information is then available in the host API responses. --- docs/cli/file-format.md | 13 +++- server/datastore/datastore_app_test.go | 22 ++++++ server/datastore/datastore_hosts_test.go | 76 +++++++++++++++++++ server/datastore/datastore_test.go | 2 + server/datastore/mysql/app_configs.go | 13 ++-- server/datastore/mysql/hosts.go | 42 +++++++++- .../20200504120000_AddAdditionalToHosts.go | 35 +++++++++ server/kolide/app.go | 16 +++- server/kolide/hosts.go | 6 ++ server/service/endpoint_appconfig.go | 4 + server/service/service_appconfig.go | 6 ++ server/service/service_osquery.go | 48 ++++++++++-- server/service/service_osquery_test.go | 29 +++++-- 13 files changed, 287 insertions(+), 25 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20200504120000_AddAdditionalToHosts.go diff --git a/docs/cli/file-format.md b/docs/cli/file-format.md index 382806695b..90916435e7 100644 --- a/docs/cli/file-format.md +++ b/docs/cli/file-format.md @@ -157,7 +157,7 @@ spec: ## Osquery Configuration Options -The following file describes configuration options passed to the osquery instance. All other configuration data will be over-written by the application of this file. +The following file describes options returned to osqueryd when it checks for configuration. See the [osquery documentation](https://osquery.readthedocs.io/en/stable/deployment/configuration/#options) for the available options. Existing options will be over-written by the application of this file. ```yaml apiVersion: v1 @@ -232,7 +232,7 @@ spec: 3600: "SELECT total_seconds AS uptime FROM uptime" ``` ## Fleet Configuration Options -The following file describes configuration options applied to the Fleet instance. +The following file describes configuration options applied to the Fleet server. ```yaml apiVersion: v1 @@ -241,6 +241,15 @@ spec: host_expiry_settings: host_expiry_enabled: true host_expiry_window: 10 + host_settings: + # "additional" information to collect from hosts along with the host + # details. This information will be updated at the same time as other host + # details and is returned by the API when host objects are returned. Users + # must take care to keep the data returned by these queries small in + # order to mitigate potential performance impacts on the Fleet server. + additional_queries: + time: select * from time + macs: select mac from interface_details org_info: org_logo_url: "https://example.org/logo.png" org_name: Example Org diff --git a/server/datastore/datastore_app_test.go b/server/datastore/datastore_app_test.go index 639363e34d..de50f1023e 100644 --- a/server/datastore/datastore_app_test.go +++ b/server/datastore/datastore_app_test.go @@ -1,6 +1,7 @@ package datastore import ( + "encoding/json" "testing" "github.com/kolide/fleet/server/kolide" @@ -53,3 +54,24 @@ func testOrgInfo(t *testing.T, ds kolide.Datastore) { assert.Nil(t, err) assert.Equal(t, info3, info4) } + +func testAdditionalQueries(t *testing.T, ds kolide.Datastore) { + additional := json.RawMessage("not valid json") + info := &kolide.AppConfig{ + OrgName: "Kolide", + OrgLogoURL: "localhost:8080/logo.png", + AdditionalQueries: &additional, + } + + _, err := ds.NewAppConfig(info) + assert.NotNil(t, err) + + additional = json.RawMessage(`{}`) + info, err = ds.NewAppConfig(info) + assert.Nil(t, err) + + additional = json.RawMessage(`{"foo": "bar"}`) + info, err = ds.NewAppConfig(info) + assert.Nil(t, err) + assert.JSONEq(t, `{"foo":"bar"}`, string(*info.AdditionalQueries)) +} diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index 8a6a1e3513..7d730c00e8 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -1,6 +1,7 @@ package datastore import ( + "encoding/json" "fmt" "sort" "strconv" @@ -72,12 +73,17 @@ func testSaveHosts(t *testing.T, ds kolide.Datastore) { }, } + additionalJSON := json.RawMessage(`{"foobar": "bim"}`) + host.Additional = &additionalJSON + err = ds.SaveHost(host) require.Nil(t, err) host, err = ds.Host(host.ID) require.Nil(t, err) require.NotNil(t, host) + require.NotNil(t, host.Additional) + assert.Equal(t, additionalJSON, *host.Additional) require.Equal(t, 2, len(host.NetworkInterfaces)) primaryNicID := host.NetworkInterfaces[0].ID host.PrimaryNetworkInterfaceID = &primaryNicID @@ -767,3 +773,73 @@ func testHostIDsByName(t *testing.T, ds kolide.Datastore) { sort.Slice(hosts, func(i, j int) bool { return hosts[i] < hosts[j] }) assert.Equal(t, hosts, []uint{2, 3, 6}) } + +func testHostAdditional(t *testing.T, ds kolide.Datastore) { + _, err := ds.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + SeenTime: time.Now(), + OsqueryHostID: "foobar", + NodeKey: "nodekey", + UUID: "uuid", + HostName: "foobar.local", + }) + require.Nil(t, err) + + h, err := ds.AuthenticateHost("nodekey") + require.Nil(t, err) + assert.Equal(t, "foobar.local", h.HostName) + assert.Nil(t, h.Additional) + + // Additional not yet set + h, err = ds.Host(h.ID) + require.Nil(t, err) + assert.Nil(t, h.Additional) + + // Add additional + additional := json.RawMessage(`{"additional": "result"}`) + h.Additional = &additional + err = ds.SaveHost(h) + require.Nil(t, err) + + h, err = ds.AuthenticateHost("nodekey") + require.Nil(t, err) + assert.Equal(t, "foobar.local", h.HostName) + assert.Nil(t, h.Additional) + + h, err = ds.Host(h.ID) + require.Nil(t, err) + assert.Equal(t, additional, *h.Additional) + + // Update besides additional. Additional should be unchanged. + h, err = ds.AuthenticateHost("nodekey") + require.Nil(t, err) + h.HostName = "baz.local" + err = ds.SaveHost(h) + require.Nil(t, err) + + h, err = ds.AuthenticateHost("nodekey") + require.Nil(t, err) + assert.Equal(t, "baz.local", h.HostName) + assert.Nil(t, h.Additional) + + h, err = ds.Host(h.ID) + require.Nil(t, err) + assert.Equal(t, additional, *h.Additional) + + // Update additional + additional = json.RawMessage(`{"other": "additional"}`) + h, err = ds.AuthenticateHost("nodekey") + require.Nil(t, err) + h.Additional = &additional + err = ds.SaveHost(h) + require.Nil(t, err) + + h, err = ds.AuthenticateHost("nodekey") + require.Nil(t, err) + assert.Equal(t, "baz.local", h.HostName) + assert.Nil(t, h.Additional) + + h, err = ds.Host(h.ID) + require.Nil(t, err) + assert.Equal(t, additional, *h.Additional) +} diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index 16536bdc4e..9b8ac32edc 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -17,6 +17,7 @@ func functionName(f func(*testing.T, kolide.Datastore)) string { var testFunctions = [...]func(*testing.T, kolide.Datastore){ testOrgInfo, + testAdditionalQueries, testCreateInvite, testInviteByEmail, testInviteByToken, @@ -93,4 +94,5 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testGetLabelSpec, testLabelIDsByName, testListLabelsForPack, + testHostAdditional, } diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go index 7d06b2c5ce..1960b06106 100644 --- a/server/datastore/mysql/app_configs.go +++ b/server/datastore/mysql/app_configs.go @@ -117,10 +117,11 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error { fim_interval, fim_file_accesses, host_expiry_enabled, - host_expiry_window, - live_query_disabled + host_expiry_window, + live_query_disabled, + additional_queries ) - VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) + VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE org_name = VALUES(org_name), org_logo_url = VALUES(org_logo_url), @@ -148,8 +149,9 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error { fim_interval = VALUES(fim_interval), fim_file_accesses = VALUES(fim_file_accesses), host_expiry_enabled = VALUES(host_expiry_enabled), - host_expiry_window = VALUES(host_expiry_window), - live_query_disabled = VALUES(live_query_disabled) + host_expiry_window = VALUES(host_expiry_window), + live_query_disabled = VALUES(live_query_disabled), + additional_queries = VALUES(additional_queries) ` _, err = d.db.Exec(insertStatement, @@ -181,6 +183,7 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error { info.HostExpiryEnabled, info.HostExpiryWindow, info.LiveQueryDisabled, + info.AdditionalQueries, ) return err diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 30e6fefaf0..2868103f12 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -171,7 +171,8 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { seen_time = ?, distributed_interval = ?, config_tls_refresh = ?, - logger_tls_period = ? + logger_tls_period = ?, + additional = COALESCE(?, additional) WHERE id = ? ` err := d.withRetryTxx(func(tx *sqlx.Tx) error { @@ -203,7 +204,9 @@ func (d *Datastore) SaveHost(host *kolide.Host) error { host.DistributedInterval, host.ConfigTLSRefresh, host.LoggerTLSPeriod, - host.ID) + host.Additional, + host.ID, + ) if err != nil { return errors.Wrap(err, "executing main SQL statement") } @@ -486,7 +489,40 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { sqlStatement := ` - SELECT * + SELECT + id, + osquery_host_id, + created_at, + updated_at, + deleted_at, + deleted, + detail_update_time, + node_key, + host_name, + uuid, + platform, + osquery_version, + os_version, + build, + platform_like, + code_name, + uptime, + physical_memory, + cpu_type, + cpu_subtype, + cpu_brand, + cpu_physical_cores, + cpu_logical_cores, + hardware_vendor, + hardware_model, + hardware_version, + hardware_serial, + computer_name, + primary_ip_id, + seen_time, + distributed_interval, + logger_tls_period, + config_tls_refresh FROM hosts WHERE node_key = ? AND NOT deleted LIMIT 1 diff --git a/server/datastore/mysql/migrations/tables/20200504120000_AddAdditionalToHosts.go b/server/datastore/mysql/migrations/tables/20200504120000_AddAdditionalToHosts.go new file mode 100644 index 0000000000..6a7c0524c4 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20200504120000_AddAdditionalToHosts.go @@ -0,0 +1,35 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20200504120000, Down_20200504120000) +} + +func Up_20200504120000(tx *sql.Tx) error { + _, err := tx.Exec( + "ALTER TABLE `hosts` " + + "ADD COLUMN `additional` JSON DEFAULT NULL;", + ) + if err != nil { + errors.Wrap(err, "add additional column") + } + + _, err = tx.Exec( + "ALTER TABLE `app_configs` " + + "ADD COLUMN `additional_queries` JSON DEFAULT NULL;", + ) + if err != nil { + errors.Wrap(err, "add additional_queries column") + } + + return nil +} + +func Down_20200504120000(tx *sql.Tx) error { + return nil +} diff --git a/server/kolide/app.go b/server/kolide/app.go index 8531afc9a2..e23ef273cc 100644 --- a/server/kolide/app.go +++ b/server/kolide/app.go @@ -2,6 +2,7 @@ package kolide import ( "context" + "encoding/json" ) // AppConfigStore contains method for saving and retrieving @@ -143,6 +144,10 @@ type AppConfig struct { // LiveQueryDisabled defines whether live queries are disabled. LiveQueryDisabled bool `db:"live_query_disabled"` + + // AdditionalQueries is the set of additional queries that should be run + // when collecting details from hosts. + AdditionalQueries *json.RawMessage `db:"additional_queries"` } // ModifyAppConfigRequest contains application configuration information @@ -217,6 +222,7 @@ type AppConfigPayload struct { ServerSettings *ServerSettings `json:"server_settings"` SMTPSettings *SMTPSettingsPayload `json:"smtp_settings"` HostExpirySettings *HostExpirySettings `json:"host_expiry_settings"` + HostSettings *HostSettings `json:"host_settings"` // SMTPTest is a flag that if set will cause the server to test email configuration SMTPTest *bool `json:"smtp_test,omitempty"` // SSOSettings single sign settings @@ -231,9 +237,9 @@ type OrgInfo struct { // ServerSettings contains general settings about the kolide App. type ServerSettings struct { - KolideServerURL *string `json:"kolide_server_url,omitempty"` - EnrollSecret *string `json:"osquery_enroll_secret,omitempty"` - LiveQueryDisabled *bool `json:"live_query_disabled,omitempty"` + KolideServerURL *string `json:"kolide_server_url,omitempty"` + EnrollSecret *string `json:"osquery_enroll_secret,omitempty"` + LiveQueryDisabled *bool `json:"live_query_disabled,omitempty"` } // HostExpirySettings contains settings pertaining to automatic host expiry. @@ -242,6 +248,10 @@ type HostExpirySettings struct { HostExpiryWindow *int `json:"host_expiry_window,omitempty"` } +type HostSettings struct { + AdditionalQueries *json.RawMessage `json:"additional_queries"` +} + type OrderDirection int const ( diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index c644093afc..ba3498c596 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/base64" + "encoding/json" "net" "time" ) @@ -39,6 +40,10 @@ type HostStore interface { Host(id uint) (*Host, error) ListHosts(opt ListOptions) ([]*Host, error) EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error) + // AuthenticateHost authenticates and returns host metadata by node key. + // This method should not return the host "additional" information as this + // is not typically necessary for the operations performed by the osquery + // endpoints. AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error SearchHosts(query string, omit ...uint) ([]*Host, error) @@ -107,6 +112,7 @@ type Host struct { DistributedInterval uint `json:"distributed_interval" db:"distributed_interval"` ConfigTLSRefresh uint `json:"config_tls_refresh" db:"config_tls_refresh"` LoggerTLSPeriod uint `json:"logger_tls_period" db:"logger_tls_period"` + Additional *json.RawMessage `json:"additional,omitempty" db:"additional"` } // HostSummary is a structure which represents a data summary about the total diff --git a/server/service/endpoint_appconfig.go b/server/service/endpoint_appconfig.go index 389c412470..41b4366fae 100644 --- a/server/service/endpoint_appconfig.go +++ b/server/service/endpoint_appconfig.go @@ -19,6 +19,7 @@ type appConfigResponse struct { SMTPSettings *kolide.SMTPSettingsPayload `json:"smtp_settings,omitempty"` SSOSettings *kolide.SSOSettingsPayload `json:"sso_settings,omitempty"` HostExpirySettings *kolide.HostExpirySettings `json:"host_expiry_settings,omitempty"` + HostSettings *kolide.HostSettings `json:"host_settings,omitempty"` Err error `json:"error,omitempty"` } @@ -70,6 +71,9 @@ func makeGetAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint { SMTPSettings: smtpSettings, SSOSettings: ssoSettings, HostExpirySettings: hostExpirySettings, + HostSettings: &kolide.HostSettings{ + AdditionalQueries: config.AdditionalQueries, + }, } return response, nil } diff --git a/server/service/service_appconfig.go b/server/service/service_appconfig.go index baec3b37d4..45da027935 100644 --- a/server/service/service_appconfig.go +++ b/server/service/service_appconfig.go @@ -157,6 +157,12 @@ func appConfigFromAppConfigPayload(p kolide.AppConfigPayload, config kolide.AppC } } + if settings := p.HostSettings; settings != nil { + if settings.AdditionalQueries != nil { + config.AdditionalQueries = settings.AdditionalQueries + } + } + populateSMTP := func(p *kolide.SMTPSettingsPayload) { if p.SMTPAuthenticationMethod != nil { switch *p.SMTPAuthenticationMethod { diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index 07202c82e9..57067e5e41 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -233,6 +233,10 @@ const hostLabelQueryPrefix = "kolide_label_query_" // provided as a detail query. const hostDetailQueryPrefix = "kolide_detail_query_" +// hostAdditionalQueryPrefix is appended before the query name when a query is +// provided as an additional query (additional info for hosts to retrieve). +const hostAdditionalQueryPrefix = "kolide_additional_query_" + // hostDistributedQueryPrefix is appended before the query name when a query is // run from a distributed query campaign const hostDistributedQueryPrefix = "kolide_distributed_query_" @@ -474,17 +478,37 @@ var detailQueries = map[string]struct { // hostDetailQueries returns the map of queries that should be executed by // osqueryd to fill in the host details -func (svc service) hostDetailQueries(host kolide.Host) map[string]string { +func (svc service) hostDetailQueries(host kolide.Host) (map[string]string, error) { queries := make(map[string]string) if host.DetailUpdateTime.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) { // No need to update already fresh details - return queries + return queries, nil } for name, query := range detailQueries { queries[hostDetailQueryPrefix+name] = query.Query } - return queries + + // Get additional queries + config, err := svc.ds.AppConfig() + if err != nil { + return nil, osqueryError{message: "get additional queries: " + err.Error()} + } + if config.AdditionalQueries == nil { + // No additional queries set + return queries, nil + } + + var additionalQueries map[string]string + if err := json.Unmarshal(*config.AdditionalQueries, &additionalQueries); err != nil { + return nil, osqueryError{message: "unmarshal additional queries: " + err.Error()} + } + + for name, query := range additionalQueries { + queries[hostAdditionalQueryPrefix+name] = query + } + + return queries, nil } func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) { @@ -493,7 +517,10 @@ func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string return nil, 0, osqueryError{message: "internal error: missing host from request context"} } - queries := svc.hostDetailQueries(host) + queries, err := svc.hostDetailQueries(host) + if err != nil { + return nil, 0, err + } // Retrieve the label queries that should be updated cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval) @@ -629,13 +656,18 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko } var err error - detailUpdated := false + detailUpdated := false // Whether detail or additional was updated + additionalResults := make(kolide.OsqueryDistributedQueryResults) labelResults := map[uint]bool{} for query, rows := range results { switch { case strings.HasPrefix(query, hostDetailQueryPrefix): err = svc.ingestDetailQuery(&host, query, rows) detailUpdated = true + case strings.HasPrefix(query, hostAdditionalQueryPrefix): + name := strings.TrimPrefix(query, hostAdditionalQueryPrefix) + additionalResults[name] = rows + detailUpdated = true case strings.HasPrefix(query, hostLabelQueryPrefix): err = svc.ingestLabelQuery(host, query, rows, labelResults) case strings.HasPrefix(query, hostDistributedQueryPrefix): @@ -663,6 +695,12 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko if detailUpdated { host.DetailUpdateTime = svc.clock.Now() + additionalJSON, err := json.Marshal(additionalResults) + if err != nil { + return osqueryError{message: "failed to marshal additional: " + err.Error()} + } + additional := json.RawMessage(additionalJSON) + host.Additional = &additional } if len(labelResults) > 0 || detailUpdated { diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index ee8e96bf2e..389a01d39a 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -201,6 +201,12 @@ func TestSubmitResultLogs(t *testing.T) { } func TestHostDetailQueries(t *testing.T) { + ds := new(mock.Store) + additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`) + ds.AppConfigFunc = func() (*kolide.AppConfig, error) { + return &kolide.AppConfig{AdditionalQueries: &additional}, nil + } + mockClock := clock.NewMockClock() host := kolide.Host{ ID: 1, @@ -219,22 +225,25 @@ func TestHostDetailQueries(t *testing.T) { UUID: "test_uuid", } - svc := service{clock: mockClock, config: config.TestConfig()} + svc := service{clock: mockClock, config: config.TestConfig(), ds: ds} - queries := svc.hostDetailQueries(host) - assert.Empty(t, queries) + queries, err := svc.hostDetailQueries(host) + assert.Nil(t, err) + assert.Empty(t, queries, 0) // Advance the time mockClock.AddTime(1*time.Hour + 1*time.Minute) - queries = svc.hostDetailQueries(host) - assert.Len(t, queries, len(detailQueries)) + queries, err = svc.hostDetailQueries(host) + assert.Nil(t, err) + assert.Len(t, queries, len(detailQueries)+2) for name, _ := range queries { assert.True(t, - strings.HasPrefix(name, hostDetailQueryPrefix), - fmt.Sprintf("%s not prefixed with %s", name, hostDetailQueryPrefix), + strings.HasPrefix(name, hostDetailQueryPrefix) || strings.HasPrefix(name, hostAdditionalQueryPrefix), ) } + assert.Equal(t, "bam", queries[hostAdditionalQueryPrefix+"bim"]) + assert.Equal(t, "select foo", queries[hostAdditionalQueryPrefix+"foobar"]) } func TestGetDistributedQueriesMissingHost(t *testing.T) { @@ -261,6 +270,9 @@ func TestLabelQueries(t *testing.T) { ds.SaveHostFunc = func(host *kolide.Host) error { return nil } + ds.AppConfigFunc = func() (*kolide.AppConfig, error) { + return &kolide.AppConfig{}, nil + } host := &kolide.Host{} ctx := hostctx.NewContext(context.Background(), *host) @@ -910,6 +922,9 @@ func TestDistributedQueryResults(t *testing.T) { gotExecution = exec return exec, nil } + ds.AppConfigFunc = func() (*kolide.AppConfig, error) { + return &kolide.AppConfig{}, nil + } host := &kolide.Host{ID: 1} hostCtx := hostctx.NewContext(context.Background(), *host)