Add capability to collect "additional" information from hosts (#2236)

Additional information is collected when host details are updated using
the queries specified in the Fleet configuration. This additional
information is then available in the host API responses.

docs/cli/file-format.md

@@ -157,7 +157,7 @@ spec:
 
 ## Osquery Configuration Options
 
-The following file describes configuration options passed to the osquery instance. All other configuration data will be over-written by the application of this file.
+The following file describes options returned to osqueryd when it checks for configuration. See the [osquery documentation](https://osquery.readthedocs.io/en/stable/deployment/configuration/#options) for the available options. Existing options will be over-written by the application of this file.
 
 ```yaml
 apiVersion: v1
@@ -232,7 +232,7 @@ spec:
             3600: "SELECT total_seconds AS uptime FROM uptime"
 ```
 ## Fleet Configuration Options
-The following file describes configuration options applied to the Fleet instance.
+The following file describes configuration options applied to the Fleet server.
 
 ```yaml
 apiVersion: v1
@@ -241,6 +241,15 @@ spec:
   host_expiry_settings:
     host_expiry_enabled: true
     host_expiry_window: 10
+  host_settings:
+    # "additional" information to collect from hosts along with the host
+    # details. This information will be updated at the same time as other host
+    # details and is returned by the API when host objects are returned. Users
+    # must take care to keep the data returned by these queries small in
+    # order to mitigate potential performance impacts on the Fleet server.
+    additional_queries:
+      time: select * from time
+      macs: select mac from interface_details
   org_info:
     org_logo_url: "https://example.org/logo.png"
     org_name: Example Org

server/datastore/datastore_app_test.go

@@ -1,6 +1,7 @@
 package datastore
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/kolide/fleet/server/kolide"
@@ -53,3 +54,24 @@ func testOrgInfo(t *testing.T, ds kolide.Datastore) {
 	assert.Nil(t, err)
 	assert.Equal(t, info3, info4)
 }
+
+func testAdditionalQueries(t *testing.T, ds kolide.Datastore) {
+	additional := json.RawMessage("not valid json")
+	info := &kolide.AppConfig{
+		OrgName:           "Kolide",
+		OrgLogoURL:        "localhost:8080/logo.png",
+		AdditionalQueries: &additional,
+	}
+
+	_, err := ds.NewAppConfig(info)
+	assert.NotNil(t, err)
+
+	additional = json.RawMessage(`{}`)
+	info, err = ds.NewAppConfig(info)
+	assert.Nil(t, err)
+
+	additional = json.RawMessage(`{"foo": "bar"}`)
+	info, err = ds.NewAppConfig(info)
+	assert.Nil(t, err)
+	assert.JSONEq(t, `{"foo":"bar"}`, string(*info.AdditionalQueries))
+}

server/datastore/datastore_hosts_test.go

@@ -1,6 +1,7 @@
 package datastore
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strconv"
@@ -72,12 +73,17 @@ func testSaveHosts(t *testing.T, ds kolide.Datastore) {
 		},
 	}
 
+	additionalJSON := json.RawMessage(`{"foobar": "bim"}`)
+	host.Additional = &additionalJSON
+
 	err = ds.SaveHost(host)
 	require.Nil(t, err)
 
 	host, err = ds.Host(host.ID)
 	require.Nil(t, err)
 	require.NotNil(t, host)
+	require.NotNil(t, host.Additional)
+	assert.Equal(t, additionalJSON, *host.Additional)
 	require.Equal(t, 2, len(host.NetworkInterfaces))
 	primaryNicID := host.NetworkInterfaces[0].ID
 	host.PrimaryNetworkInterfaceID = &primaryNicID
@@ -767,3 +773,73 @@ func testHostIDsByName(t *testing.T, ds kolide.Datastore) {
 	sort.Slice(hosts, func(i, j int) bool { return hosts[i] < hosts[j] })
 	assert.Equal(t, hosts, []uint{2, 3, 6})
 }
+
+func testHostAdditional(t *testing.T, ds kolide.Datastore) {
+	_, err := ds.NewHost(&kolide.Host{
+		DetailUpdateTime: time.Now(),
+		SeenTime:         time.Now(),
+		OsqueryHostID:    "foobar",
+		NodeKey:          "nodekey",
+		UUID:             "uuid",
+		HostName:         "foobar.local",
+	})
+	require.Nil(t, err)
+
+	h, err := ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	assert.Equal(t, "foobar.local", h.HostName)
+	assert.Nil(t, h.Additional)
+
+	// Additional not yet set
+	h, err = ds.Host(h.ID)
+	require.Nil(t, err)
+	assert.Nil(t, h.Additional)
+
+	// Add additional
+	additional := json.RawMessage(`{"additional": "result"}`)
+	h.Additional = &additional
+	err = ds.SaveHost(h)
+	require.Nil(t, err)
+
+	h, err = ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	assert.Equal(t, "foobar.local", h.HostName)
+	assert.Nil(t, h.Additional)
+
+	h, err = ds.Host(h.ID)
+	require.Nil(t, err)
+	assert.Equal(t, additional, *h.Additional)
+
+	// Update besides additional. Additional should be unchanged.
+	h, err = ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	h.HostName = "baz.local"
+	err = ds.SaveHost(h)
+	require.Nil(t, err)
+
+	h, err = ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	assert.Equal(t, "baz.local", h.HostName)
+	assert.Nil(t, h.Additional)
+
+	h, err = ds.Host(h.ID)
+	require.Nil(t, err)
+	assert.Equal(t, additional, *h.Additional)
+
+	// Update additional
+	additional = json.RawMessage(`{"other": "additional"}`)
+	h, err = ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	h.Additional = &additional
+	err = ds.SaveHost(h)
+	require.Nil(t, err)
+
+	h, err = ds.AuthenticateHost("nodekey")
+	require.Nil(t, err)
+	assert.Equal(t, "baz.local", h.HostName)
+	assert.Nil(t, h.Additional)
+
+	h, err = ds.Host(h.ID)
+	require.Nil(t, err)
+	assert.Equal(t, additional, *h.Additional)
+}

server/datastore/datastore_test.go

@@ -17,6 +17,7 @@ func functionName(f func(*testing.T, kolide.Datastore)) string {
 
 var testFunctions = [...]func(*testing.T, kolide.Datastore){
 	testOrgInfo,
+	testAdditionalQueries,
 	testCreateInvite,
 	testInviteByEmail,
 	testInviteByToken,
@@ -93,4 +94,5 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){
 	testGetLabelSpec,
 	testLabelIDsByName,
 	testListLabelsForPack,
+	testHostAdditional,
 }

server/datastore/mysql/app_configs.go

@@ -117,10 +117,11 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error {
       fim_interval,
       fim_file_accesses,
       host_expiry_enabled,
-	  host_expiry_window,
-	  live_query_disabled
+      host_expiry_window,
+      live_query_disabled,
+      additional_queries
     )
-    VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )
+    VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )
     ON DUPLICATE KEY UPDATE
       org_name = VALUES(org_name),
       org_logo_url = VALUES(org_logo_url),
@@ -148,8 +149,9 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error {
       fim_interval = VALUES(fim_interval),
       fim_file_accesses = VALUES(fim_file_accesses),
       host_expiry_enabled = VALUES(host_expiry_enabled),
-	  host_expiry_window = VALUES(host_expiry_window),
-	  live_query_disabled = VALUES(live_query_disabled)
+      host_expiry_window = VALUES(host_expiry_window),
+      live_query_disabled = VALUES(live_query_disabled),
+      additional_queries = VALUES(additional_queries)
     `
 
 	_, err = d.db.Exec(insertStatement,
@@ -181,6 +183,7 @@ func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error {
 		info.HostExpiryEnabled,
 		info.HostExpiryWindow,
 		info.LiveQueryDisabled,
+		info.AdditionalQueries,
 	)
 
 	return err

server/datastore/mysql/hosts.go

@@ -171,7 +171,8 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
 			seen_time = ?,
 			distributed_interval = ?,
 			config_tls_refresh = ?,
-			logger_tls_period = ?
+			logger_tls_period = ?,
+			additional = COALESCE(?, additional)
 		WHERE id = ?
 	`
 	err := d.withRetryTxx(func(tx *sqlx.Tx) error {
@@ -203,7 +204,9 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
 			host.DistributedInterval,
 			host.ConfigTLSRefresh,
 			host.LoggerTLSPeriod,
-			host.ID)
+			host.Additional,
+			host.ID,
+		)
 		if err != nil {
 			return errors.Wrap(err, "executing main SQL statement")
 		}
@@ -486,7 +489,40 @@ func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.H
 
 func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
 	sqlStatement := `
-		SELECT *
+		SELECT
+			id,
+			osquery_host_id,
+			created_at,
+			updated_at,
+			deleted_at,
+			deleted,
+			detail_update_time,
+			node_key,
+			host_name,
+			uuid,
+			platform,
+			osquery_version,
+			os_version,
+			build,
+			platform_like,
+			code_name,
+			uptime,
+			physical_memory,
+			cpu_type,
+			cpu_subtype,
+			cpu_brand,
+			cpu_physical_cores,
+			cpu_logical_cores,
+			hardware_vendor,
+			hardware_model,
+			hardware_version,
+			hardware_serial,
+			computer_name,
+			primary_ip_id,
+			seen_time,
+			distributed_interval,
+			logger_tls_period,
+			config_tls_refresh
 		FROM hosts
 		WHERE node_key = ? AND NOT deleted
 		LIMIT 1

server/datastore/mysql/migrations/tables/20200504120000_AddAdditionalToHosts.go

@@ -0,0 +1,35 @@
+package tables
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20200504120000, Down_20200504120000)
+}
+
+func Up_20200504120000(tx *sql.Tx) error {
+	_, err := tx.Exec(
+		"ALTER TABLE `hosts` " +
+			"ADD COLUMN `additional` JSON DEFAULT NULL;",
+	)
+	if err != nil {
+		errors.Wrap(err, "add additional column")
+	}
+
+	_, err = tx.Exec(
+		"ALTER TABLE `app_configs` " +
+			"ADD COLUMN `additional_queries` JSON DEFAULT NULL;",
+	)
+	if err != nil {
+		errors.Wrap(err, "add additional_queries column")
+	}
+	
+	return nil
+}
+
+func Down_20200504120000(tx *sql.Tx) error {
+	return nil
+}

server/kolide/app.go

@@ -2,6 +2,7 @@ package kolide
 
 import (
 	"context"
+	"encoding/json"
 )
 
 // AppConfigStore contains method for saving and retrieving
@@ -143,6 +144,10 @@ type AppConfig struct {
 
 	// LiveQueryDisabled defines whether live queries are disabled.
 	LiveQueryDisabled bool `db:"live_query_disabled"`
+
+	// AdditionalQueries is the set of additional queries that should be run
+	// when collecting details from hosts.
+	AdditionalQueries *json.RawMessage `db:"additional_queries"`
 }
 
 // ModifyAppConfigRequest contains application configuration information
@@ -217,6 +222,7 @@ type AppConfigPayload struct {
 	ServerSettings     *ServerSettings      `json:"server_settings"`
 	SMTPSettings       *SMTPSettingsPayload `json:"smtp_settings"`
 	HostExpirySettings *HostExpirySettings  `json:"host_expiry_settings"`
+	HostSettings       *HostSettings        `json:"host_settings"`
 	// SMTPTest is a flag that if set will cause the server to test email configuration
 	SMTPTest *bool `json:"smtp_test,omitempty"`
 	// SSOSettings single sign settings
@@ -231,9 +237,9 @@ type OrgInfo struct {
 
 // ServerSettings contains general settings about the kolide App.
 type ServerSettings struct {
-	KolideServerURL    *string `json:"kolide_server_url,omitempty"`
-	EnrollSecret       *string `json:"osquery_enroll_secret,omitempty"`
-	LiveQueryDisabled  *bool   `json:"live_query_disabled,omitempty"`
+	KolideServerURL   *string `json:"kolide_server_url,omitempty"`
+	EnrollSecret      *string `json:"osquery_enroll_secret,omitempty"`
+	LiveQueryDisabled *bool   `json:"live_query_disabled,omitempty"`
 }
 
 // HostExpirySettings contains settings pertaining to automatic host expiry.
@@ -242,6 +248,10 @@ type HostExpirySettings struct {
 	HostExpiryWindow  *int  `json:"host_expiry_window,omitempty"`
 }
 
+type HostSettings struct {
+	AdditionalQueries *json.RawMessage `json:"additional_queries"`
+}
+
 type OrderDirection int
 
 const (

server/kolide/hosts.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"encoding/json"
 	"net"
 	"time"
 )
@@ -39,6 +40,10 @@ type HostStore interface {
 	Host(id uint) (*Host, error)
 	ListHosts(opt ListOptions) ([]*Host, error)
 	EnrollHost(osqueryHostId string, nodeKeySize int) (*Host, error)
+	// AuthenticateHost authenticates and returns host metadata by node key.
+	// This method should not return the host "additional" information as this
+	// is not typically necessary for the operations performed by the osquery
+	// endpoints.
 	AuthenticateHost(nodeKey string) (*Host, error)
 	MarkHostSeen(host *Host, t time.Time) error
 	SearchHosts(query string, omit ...uint) ([]*Host, error)
@@ -107,6 +112,7 @@ type Host struct {
 	DistributedInterval       uint                `json:"distributed_interval" db:"distributed_interval"`
 	ConfigTLSRefresh          uint                `json:"config_tls_refresh" db:"config_tls_refresh"`
 	LoggerTLSPeriod           uint                `json:"logger_tls_period" db:"logger_tls_period"`
+	Additional                *json.RawMessage    `json:"additional,omitempty" db:"additional"`
 }
 
 // HostSummary is a structure which represents a data summary about the total

server/service/endpoint_appconfig.go

@@ -19,6 +19,7 @@ type appConfigResponse struct {
 	SMTPSettings       *kolide.SMTPSettingsPayload `json:"smtp_settings,omitempty"`
 	SSOSettings        *kolide.SSOSettingsPayload  `json:"sso_settings,omitempty"`
 	HostExpirySettings *kolide.HostExpirySettings  `json:"host_expiry_settings,omitempty"`
+	HostSettings       *kolide.HostSettings        `json:"host_settings,omitempty"`
 	Err                error                       `json:"error,omitempty"`
 }
 
@@ -70,6 +71,9 @@ func makeGetAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
 			SMTPSettings:       smtpSettings,
 			SSOSettings:        ssoSettings,
 			HostExpirySettings: hostExpirySettings,
+			HostSettings: &kolide.HostSettings{
+				AdditionalQueries: config.AdditionalQueries,
+			},
 		}
 		return response, nil
 	}

server/service/service_appconfig.go

@@ -157,6 +157,12 @@ func appConfigFromAppConfigPayload(p kolide.AppConfigPayload, config kolide.AppC
 		}
 	}
 
+	if settings := p.HostSettings; settings != nil {
+		if settings.AdditionalQueries != nil {
+			config.AdditionalQueries = settings.AdditionalQueries
+		}
+	}
+
 	populateSMTP := func(p *kolide.SMTPSettingsPayload) {
 		if p.SMTPAuthenticationMethod != nil {
 			switch *p.SMTPAuthenticationMethod {

server/service/service_osquery.go

@@ -233,6 +233,10 @@ const hostLabelQueryPrefix = "kolide_label_query_"
 // provided as a detail query.
 const hostDetailQueryPrefix = "kolide_detail_query_"
 
+// hostAdditionalQueryPrefix is appended before the query name when a query is
+// provided as an additional query (additional info for hosts to retrieve).
+const hostAdditionalQueryPrefix = "kolide_additional_query_"
+
 // hostDistributedQueryPrefix is appended before the query name when a query is
 // run from a distributed query campaign
 const hostDistributedQueryPrefix = "kolide_distributed_query_"
@@ -474,17 +478,37 @@ var detailQueries = map[string]struct {
 
 // hostDetailQueries returns the map of queries that should be executed by
 // osqueryd to fill in the host details
-func (svc service) hostDetailQueries(host kolide.Host) map[string]string {
+func (svc service) hostDetailQueries(host kolide.Host) (map[string]string, error) {
 	queries := make(map[string]string)
 	if host.DetailUpdateTime.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) {
 		// No need to update already fresh details
-		return queries
+		return queries, nil
 	}
 
 	for name, query := range detailQueries {
 		queries[hostDetailQueryPrefix+name] = query.Query
 	}
-	return queries
+
+	// Get additional queries
+	config, err := svc.ds.AppConfig()
+	if err != nil {
+		return nil, osqueryError{message: "get additional queries: " + err.Error()}
+	}
+	if config.AdditionalQueries == nil {
+		// No additional queries set
+		return queries, nil
+	}
+
+	var additionalQueries map[string]string
+	if err := json.Unmarshal(*config.AdditionalQueries, &additionalQueries); err != nil {
+		return nil, osqueryError{message: "unmarshal additional queries: " + err.Error()}
+	}
+
+	for name, query := range additionalQueries {
+		queries[hostAdditionalQueryPrefix+name] = query
+	}
+
+	return queries, nil
 }
 
 func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) {
@@ -493,7 +517,10 @@ func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string
 		return nil, 0, osqueryError{message: "internal error: missing host from request context"}
 	}
 
-	queries := svc.hostDetailQueries(host)
+	queries, err := svc.hostDetailQueries(host)
+	if err != nil {
+		return nil, 0, err
+	}
 
 	// Retrieve the label queries that should be updated
 	cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval)
@@ -629,13 +656,18 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko
 	}
 
 	var err error
-	detailUpdated := false
+	detailUpdated := false // Whether detail or additional was updated
+	additionalResults := make(kolide.OsqueryDistributedQueryResults)
 	labelResults := map[uint]bool{}
 	for query, rows := range results {
 		switch {
 		case strings.HasPrefix(query, hostDetailQueryPrefix):
 			err = svc.ingestDetailQuery(&host, query, rows)
 			detailUpdated = true
+		case strings.HasPrefix(query, hostAdditionalQueryPrefix):
+			name := strings.TrimPrefix(query, hostAdditionalQueryPrefix)
+			additionalResults[name] = rows
+			detailUpdated = true
 		case strings.HasPrefix(query, hostLabelQueryPrefix):
 			err = svc.ingestLabelQuery(host, query, rows, labelResults)
 		case strings.HasPrefix(query, hostDistributedQueryPrefix):
@@ -663,6 +695,12 @@ func (svc service) SubmitDistributedQueryResults(ctx context.Context, results ko
 
 	if detailUpdated {
 		host.DetailUpdateTime = svc.clock.Now()
+		additionalJSON, err := json.Marshal(additionalResults)
+		if err != nil {
+			return osqueryError{message: "failed to marshal additional: " + err.Error()}
+		}
+		additional := json.RawMessage(additionalJSON)
+		host.Additional = &additional
 	}
 
 	if len(labelResults) > 0 || detailUpdated {

server/service/service_osquery_test.go

@@ -201,6 +201,12 @@ func TestSubmitResultLogs(t *testing.T) {
 }
 
 func TestHostDetailQueries(t *testing.T) {
+	ds := new(mock.Store)
+	additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`)
+	ds.AppConfigFunc = func() (*kolide.AppConfig, error) {
+		return &kolide.AppConfig{AdditionalQueries: &additional}, nil
+	}
+
 	mockClock := clock.NewMockClock()
 	host := kolide.Host{
 		ID: 1,
@@ -219,22 +225,25 @@ func TestHostDetailQueries(t *testing.T) {
 		UUID:             "test_uuid",
 	}
 
-	svc := service{clock: mockClock, config: config.TestConfig()}
+	svc := service{clock: mockClock, config: config.TestConfig(), ds: ds}
 
-	queries := svc.hostDetailQueries(host)
-	assert.Empty(t, queries)
+	queries, err := svc.hostDetailQueries(host)
+	assert.Nil(t, err)
+	assert.Empty(t, queries, 0)
 
 	// Advance the time
 	mockClock.AddTime(1*time.Hour + 1*time.Minute)
 
-	queries = svc.hostDetailQueries(host)
-	assert.Len(t, queries, len(detailQueries))
+	queries, err = svc.hostDetailQueries(host)
+	assert.Nil(t, err)
+	assert.Len(t, queries, len(detailQueries)+2)
 	for name, _ := range queries {
 		assert.True(t,
-			strings.HasPrefix(name, hostDetailQueryPrefix),
-			fmt.Sprintf("%s not prefixed with %s", name, hostDetailQueryPrefix),
+			strings.HasPrefix(name, hostDetailQueryPrefix) || strings.HasPrefix(name, hostAdditionalQueryPrefix),
 		)
 	}
+	assert.Equal(t, "bam", queries[hostAdditionalQueryPrefix+"bim"])
+	assert.Equal(t, "select foo", queries[hostAdditionalQueryPrefix+"foobar"])
 }
 
 func TestGetDistributedQueriesMissingHost(t *testing.T) {
@@ -261,6 +270,9 @@ func TestLabelQueries(t *testing.T) {
 	ds.SaveHostFunc = func(host *kolide.Host) error {
 		return nil
 	}
+	ds.AppConfigFunc = func() (*kolide.AppConfig, error) {
+		return &kolide.AppConfig{}, nil
+	}
 
 	host := &kolide.Host{}
 	ctx := hostctx.NewContext(context.Background(), *host)
@@ -910,6 +922,9 @@ func TestDistributedQueryResults(t *testing.T) {
 		gotExecution = exec
 		return exec, nil
 	}
+	ds.AppConfigFunc = func() (*kolide.AppConfig, error) {
+		return &kolide.AppConfig{}, nil
+	}
 
 	host := &kolide.Host{ID: 1}
 	hostCtx := hostctx.NewContext(context.Background(), *host)