From 5371647d2c0539e54e550f78d8276d61b765631e Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Fri, 6 Jan 2017 12:30:43 -0800 Subject: [PATCH] Add builtin labels via migration (#728) Changing from the existing method of adding built in labels at server startup. This new method should be friendlier to long term changes, and falls in line with the new pattern established for osquery options. Fixes #702 --- server/datastore/datastore_labels_test.go | 11 +- server/datastore/datastore_options_test.go | 2 +- server/datastore/inmem/inmem.go | 87 ++---------- server/datastore/internal/appstate/labels.go | 39 ++++++ server/datastore/internal/appstate/options.go | 132 ++++++++++-------- .../20161223115449_InsertOsqueryOptions.go | 4 +- .../20161229171615_InsertBuiltinLabels.go | 48 +++++++ server/datastore/mysql/mysql.go | 58 -------- 8 files changed, 170 insertions(+), 211 deletions(-) create mode 100644 server/datastore/internal/appstate/labels.go create mode 100644 server/datastore/mysql/migrations/data/20161229171615_InsertBuiltinLabels.go diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index cfba572b63..e1702ed0f2 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "github.com/kolide/kolide-ose/server/datastore/inmem" - "github.com/kolide/kolide-ose/server/datastore/mysql" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -298,14 +296,7 @@ func testListHostsInLabel(t *testing.T, db kolide.Datastore) { } func testBuiltInLabels(t *testing.T, db kolide.Datastore) { - if i, ok := db.(*mysql.Datastore); ok { - err := i.Initialize() - require.Nil(t, err) - } - if i, ok := db.(*inmem.Datastore); ok { - err := i.Initialize() - require.Nil(t, err) - } + require.Nil(t, db.MigrateData()) hits, err := db.SearchLabels("Mac OS X") require.Nil(t, err) diff --git a/server/datastore/datastore_options_test.go b/server/datastore/datastore_options_test.go index 813cfbb3a3..7dd6c01388 100644 --- a/server/datastore/datastore_options_test.go +++ b/server/datastore/datastore_options_test.go @@ -16,7 +16,7 @@ func testOptions(t *testing.T, ds kolide.Datastore) { // were options pre-loaded? opts, err := ds.ListOptions() require.Nil(t, err) - assert.Len(t, opts, len(appstate.Options)) + assert.Len(t, opts, len(appstate.Options())) opt, err := ds.OptionByName("aws_access_key_id") require.Nil(t, err) diff --git a/server/datastore/inmem/inmem.go b/server/datastore/inmem/inmem.go index fd14cb04b9..e545e898f3 100644 --- a/server/datastore/inmem/inmem.go +++ b/server/datastore/inmem/inmem.go @@ -95,9 +95,8 @@ func (d *Datastore) MigrateTables() error { func (d *Datastore) MigrateData() error { d.mtx.Lock() - defer d.mtx.Unlock() - for _, initData := range appstate.Options { + for _, initData := range appstate.Options() { opt := kolide.Option{ Name: initData.Name, Value: kolide.OptionValue{Val: initData.Value}, @@ -116,6 +115,12 @@ func (d *Datastore) MigrateData() error { SMTPVerifySSLCerts: true, } + d.mtx.Unlock() + + if err := d.createBuiltinLabels(); err != nil { + return err + } + return nil } @@ -124,10 +129,6 @@ func (d *Datastore) Drop() error { } func (d *Datastore) Initialize() error { - if err := d.createBuiltinLabels(); err != nil { - return err - } - if err := d.createDevUsers(); err != nil { return err } @@ -260,79 +261,7 @@ func (d *Datastore) createDevPacksAndQueries() error { } func (d *Datastore) createBuiltinLabels() error { - labels := []kolide.Label{ - { - UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ - CreateTimestamp: kolide.CreateTimestamp{ - CreatedAt: time.Now().UTC(), - }, - UpdateTimestamp: kolide.UpdateTimestamp{ - UpdatedAt: time.Now().UTC(), - }, - }, - Platform: "darwin", - Name: "Mac OS X", - Query: "select 1 from osquery_info where build_platform = 'darwin';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ - CreateTimestamp: kolide.CreateTimestamp{ - CreatedAt: time.Now().UTC(), - }, - UpdateTimestamp: kolide.UpdateTimestamp{ - UpdatedAt: time.Now().UTC(), - }, - }, - Platform: "ubuntu", - Name: "Ubuntu Linux", - Query: "select 1 from osquery_info where build_platform = 'ubuntu';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ - CreateTimestamp: kolide.CreateTimestamp{ - CreatedAt: time.Now().UTC(), - }, - UpdateTimestamp: kolide.UpdateTimestamp{ - UpdatedAt: time.Now().UTC(), - }, - }, - Platform: "centos", - Name: "CentOS Linux", - Query: "select 1 from osquery_info where build_platform = 'centos';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ - CreateTimestamp: kolide.CreateTimestamp{ - CreatedAt: time.Now().UTC(), - }, - UpdateTimestamp: kolide.UpdateTimestamp{ - UpdatedAt: time.Now().UTC(), - }, - }, - Platform: "windows", - Name: "MS Windows", - Query: "select 1 from osquery_info where build_platform = 'windows';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ - CreateTimestamp: kolide.CreateTimestamp{ - CreatedAt: time.Now().UTC(), - }, - UpdateTimestamp: kolide.UpdateTimestamp{ - UpdatedAt: time.Now().UTC(), - }, - }, - Name: "All Hosts", - Query: "select 1;", - LabelType: kolide.LabelTypeBuiltIn, - }, - } - - for _, label := range labels { + for _, label := range appstate.Labels() { label := label _, err := d.NewLabel(&label) if err != nil { diff --git a/server/datastore/internal/appstate/labels.go b/server/datastore/internal/appstate/labels.go new file mode 100644 index 0000000000..4039cbd531 --- /dev/null +++ b/server/datastore/internal/appstate/labels.go @@ -0,0 +1,39 @@ +package appstate + +import "github.com/kolide/kolide-ose/server/kolide" + +// Labels is the set of builtin labels that should be populated in the +// datastore +func Labels() []kolide.Label { + return []kolide.Label{ + { + Name: "All Hosts", + Query: "select 1;", + LabelType: kolide.LabelTypeBuiltIn, + }, + { + Platform: "darwin", + Name: "Mac OS X", + Query: "select 1 from osquery_info where build_platform = 'darwin';", + LabelType: kolide.LabelTypeBuiltIn, + }, + { + Platform: "ubuntu", + Name: "Ubuntu Linux", + Query: "select 1 from osquery_info where build_platform = 'ubuntu';", + LabelType: kolide.LabelTypeBuiltIn, + }, + { + Platform: "centos", + Name: "CentOS Linux", + Query: "select 1 from osquery_info where build_platform = 'centos';", + LabelType: kolide.LabelTypeBuiltIn, + }, + { + Platform: "windows", + Name: "MS Windows", + Query: "select 1 from osquery_info where build_platform = 'windows';", + LabelType: kolide.LabelTypeBuiltIn, + }, + } +} diff --git a/server/datastore/internal/appstate/options.go b/server/datastore/internal/appstate/options.go index 9686cf9a65..0f10f5bcaa 100644 --- a/server/datastore/internal/appstate/options.go +++ b/server/datastore/internal/appstate/options.go @@ -2,69 +2,79 @@ package appstate import "github.com/kolide/kolide-ose/server/kolide" -var Options = []struct { +// Options is the set of builtin osquery options that should be populated in +// the datastore +func Options() []struct { Name string Value interface{} Type kolide.OptionType ReadOnly bool -}{ - // These options are read only, attempting to modify one of these will - // raise an error - {"disable_distributed", false, kolide.OptionTypeBool, kolide.ReadOnly}, - {"distributed_plugin", "tls", kolide.OptionTypeString, kolide.ReadOnly}, - {"distributed_tls_read_endpoint", "/api/v1/osquery/distributed/read", kolide.OptionTypeString, kolide.ReadOnly}, - {"distributed_tls_write_endpoint", "/api/v1/osquery/distributed/write", kolide.OptionTypeString, kolide.ReadOnly}, - {"pack_delimiter", "/", kolide.OptionTypeString, kolide.ReadOnly}, - // These options may be modified by an admin user - {"aws_access_key_id", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_firehose_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"aws_firehose_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_kinesis_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"aws_kinesis_random_partition_key", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"aws_kinesis_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_profile_name", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_region", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_secret_access_key", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_sts_arn_role", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_sts_region", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_sts_session_name", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"aws_sts_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"buffered_log_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"decorations_top_level", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_caching", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_database", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_decorators", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_events", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_kernel", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_logging", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"disable_tables", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"distributed_interval", 10, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"distributed_tls_max_attempts", 3, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"enable_foreign", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"enable_monitor", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"ephemeral", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"events_expiry", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"events_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"events_optimize", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"host_identifier", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"logger_event_type", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"logger_mode", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"logger_path", nil, kolide.OptionTypeString, kolide.NotReadOnly}, - {"logger_plugin", "tls", kolide.OptionTypeString, kolide.NotReadOnly}, - {"logger_secondary_status_only", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"logger_syslog_facility", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"logger_tls_compress", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"logger_tls_endpoint", "/api/v1/osquery/log", kolide.OptionTypeString, kolide.NotReadOnly}, - {"logger_tls_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"logger_tls_period", 10, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"pack_refresh_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"read_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"read_user_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"schedule_default_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"schedule_splay_percent", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"schedule_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"utc", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"value_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, - {"verbose", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, - {"worker_threads", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, +} { + + return []struct { + Name string + Value interface{} + Type kolide.OptionType + ReadOnly bool + }{ + // These options are read only, attempting to modify one of these will + // raise an error + {"disable_distributed", false, kolide.OptionTypeBool, kolide.ReadOnly}, + {"distributed_plugin", "tls", kolide.OptionTypeString, kolide.ReadOnly}, + {"distributed_tls_read_endpoint", "/api/v1/osquery/distributed/read", kolide.OptionTypeString, kolide.ReadOnly}, + {"distributed_tls_write_endpoint", "/api/v1/osquery/distributed/write", kolide.OptionTypeString, kolide.ReadOnly}, + {"pack_delimiter", "/", kolide.OptionTypeString, kolide.ReadOnly}, + // These options may be modified by an admin user + {"aws_access_key_id", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_firehose_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"aws_firehose_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_kinesis_period", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"aws_kinesis_random_partition_key", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"aws_kinesis_stream", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_profile_name", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_region", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_secret_access_key", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_sts_arn_role", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_sts_region", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_sts_session_name", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"aws_sts_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"buffered_log_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"decorations_top_level", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_caching", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_database", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_decorators", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_events", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_kernel", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_logging", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"disable_tables", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"distributed_interval", 10, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"distributed_tls_max_attempts", 3, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"enable_foreign", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"enable_monitor", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"ephemeral", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"events_expiry", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"events_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"events_optimize", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"host_identifier", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"logger_event_type", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"logger_mode", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"logger_path", nil, kolide.OptionTypeString, kolide.NotReadOnly}, + {"logger_plugin", "tls", kolide.OptionTypeString, kolide.NotReadOnly}, + {"logger_secondary_status_only", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"logger_syslog_facility", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"logger_tls_compress", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"logger_tls_endpoint", "/api/v1/osquery/log", kolide.OptionTypeString, kolide.NotReadOnly}, + {"logger_tls_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"logger_tls_period", 10, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"pack_refresh_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"read_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"read_user_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"schedule_default_interval", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"schedule_splay_percent", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"schedule_timeout", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"utc", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"value_max", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + {"verbose", nil, kolide.OptionTypeBool, kolide.NotReadOnly}, + {"worker_threads", nil, kolide.OptionTypeInt, kolide.NotReadOnly}, + } } diff --git a/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go b/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go index 8c0989e59e..6c51d03664 100644 --- a/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go +++ b/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go @@ -21,7 +21,7 @@ func Up_20161223115449(tx *sql.Tx) error { ) VALUES (?, ?, ?, ?) ` - for _, opt := range appstate.Options { + for _, opt := range appstate.Options() { ov := kolide.Option{ Name: opt.Name, ReadOnly: opt.ReadOnly, @@ -44,7 +44,7 @@ func Down_20161223115449(tx *sql.Tx) error { DELETE FROM options WHERE name = ? ` - for _, opt := range appstate.Options { + for _, opt := range appstate.Options() { _, err := tx.Exec(sqlStatement, opt.Name) if err != nil { return err diff --git a/server/datastore/mysql/migrations/data/20161229171615_InsertBuiltinLabels.go b/server/datastore/mysql/migrations/data/20161229171615_InsertBuiltinLabels.go new file mode 100644 index 0000000000..664c6e0752 --- /dev/null +++ b/server/datastore/mysql/migrations/data/20161229171615_InsertBuiltinLabels.go @@ -0,0 +1,48 @@ +package data + +import ( + "database/sql" + + "github.com/kolide/kolide-ose/server/datastore/internal/appstate" +) + +func init() { + MigrationClient.AddMigration(Up_20161229171615, Down_20161229171615) +} + +func Up_20161229171615(tx *sql.Tx) error { + sql := ` + INSERT INTO labels ( + name, + description, + query, + platform, + label_type + ) VALUES (?, ?, ?, ?, ?) +` + + for _, label := range appstate.Labels() { + _, err := tx.Exec(sql, label.Name, label.Description, label.Query, label.Platform, label.LabelType) + if err != nil { + return err + } + } + + return nil +} + +func Down_20161229171615(tx *sql.Tx) error { + sql := ` + DELETE FROM labels + WHERE name = ? AND label_type = ? +` + + for _, label := range appstate.Labels() { + _, err := tx.Exec(sql, label.Name, label.LabelType) + if err != nil { + return err + } + } + + return nil +} diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 9eeb6a7c2e..aae65ac9aa 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -90,14 +90,6 @@ func (d *Datastore) MigrateData() error { return nil } -// Initialize preload data needed by the application -func (d *Datastore) Initialize() error { - if err := d.createBuiltinLabels(); err != nil { - return err - } - return nil -} - // Drop removes database func (d *Datastore) Drop() error { tables := []struct { @@ -190,53 +182,3 @@ func generateMysqlConnectionString(conf config.MysqlConfig) string { conf.Database, ) } - -func (d *Datastore) createBuiltinLabels() error { - // Nuke built in labels and recreate them - _, err := d.db.Exec("DELETE from labels WHERE label_type = ?", kolide.LabelTypeBuiltIn) - if err != nil { - return err - } - - labels := []kolide.Label{ - { - Name: "All Hosts", - Query: "select 1;", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - Platform: "darwin", - Name: "Mac OS X", - Query: "select 1 from osquery_info where build_platform = 'darwin';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - Platform: "ubuntu", - Name: "Ubuntu Linux", - Query: "select 1 from osquery_info where build_platform = 'ubuntu';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - Platform: "centos", - Name: "CentOS Linux", - Query: "select 1 from osquery_info where build_platform = 'centos';", - LabelType: kolide.LabelTypeBuiltIn, - }, - { - Platform: "windows", - Name: "MS Windows", - Query: "select 1 from osquery_info where build_platform = 'windows';", - LabelType: kolide.LabelTypeBuiltIn, - }, - } - - for _, label := range labels { - _, err = d.NewLabel(&label) - if err != nil { - return err - } - } - - return nil - -}