From 6a825c11e335a6ae5e7121bd124f4f11d04dc124 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Wed, 16 Nov 2016 21:47:49 +0800 Subject: [PATCH] Datastore refactor (#439) Removed Gorm, replaced it with Sqlx * Added SQL bundling command to Makfile * Using go-kit logger * Added soft delete capability * Changed SearchLabel to accept a variadic param for optional omit list instead of array * Gorm removed * Refactor table structures to use CURRENT_TIMESTAMP mysql function * Moved Inmem datastore into it's own package * Updated README * Implemented code review suggestions from @zwass * Removed reference to Gorm from glide.yaml --- Makefile | 2 + README.md | 23 +- cli/prepare.go | 16 +- cli/serve.go | 194 ++++++++---- db/down.sql | 32 ++ db/up.sql | 238 ++++++++++++++ glide.lock | 12 +- glide.yaml | 5 +- server/datastore/datastore.go | 93 ------ server/datastore/datastore_hosts_test.go | 94 +++++- server/datastore/datastore_invites_test.go | 111 ++++++- server/datastore/datastore_labels_test.go | 13 +- server/datastore/datastore_packs_test.go | 4 +- server/datastore/datastore_queries_test.go | 21 +- server/datastore/datastore_test.go | 7 + server/datastore/datastore_users_test.go | 2 +- server/datastore/gorm.go | 140 --------- server/datastore/gorm_app.go | 35 --- server/datastore/gorm_hosts.go | 195 ------------ server/datastore/gorm_invites.go | 48 --- server/datastore/gorm_labels.go | 207 ------------ server/datastore/gorm_packs.go | 164 ---------- server/datastore/gorm_password_reset.go | 59 ---- server/datastore/gorm_queries.go | 76 ----- server/datastore/gorm_sessions.go | 78 ----- server/datastore/gorm_users.go | 57 ---- server/datastore/inmem/app.go | 34 ++ .../{inmem_hosts.go => inmem/hosts.go} | 35 ++- server/datastore/{ => inmem}/inmem.go | 56 ++-- server/datastore/inmem/inmem_test.go | 68 ++++ .../{inmem_invites.go => inmem/invites.go} | 25 +- .../{inmem_labels.go => inmem/labels.go} | 27 +- .../{inmem_packs.go => inmem/packs.go} | 36 +-- .../password_reset.go} | 35 ++- .../{inmem_queries.go => inmem/queries.go} | 41 +-- .../{inmem_sessions.go => inmem/sessions.go} | 27 +- .../{inmem_users.go => inmem/users.go} | 25 +- server/datastore/inmem_app.go | 31 -- server/datastore/inmem_test.go | 70 +---- server/datastore/mysql/app_configs.go | 50 +++ server/datastore/{ => mysql}/config.go | 31 +- server/datastore/mysql/datastore.go | 179 +++++++++++ server/datastore/mysql/datastore_test.go | 57 ++++ server/datastore/mysql/hosts.go | 295 ++++++++++++++++++ server/datastore/mysql/invites.go | 92 ++++++ server/datastore/mysql/labels.go | 258 +++++++++++++++ server/datastore/mysql/packs.go | 200 ++++++++++++ server/datastore/mysql/password_reset.go | 124 ++++++++ server/datastore/mysql/queries.go | 172 ++++++++++ server/datastore/mysql/sessions.go | 106 +++++++ server/datastore/mysql/users.go | 107 +++++++ .../datastore/{gorm_test.go => mysql_test.go} | 35 ++- server/errors/errors.go | 15 +- server/errors/errors_test.go | 20 +- server/kolide/app.go | 12 +- server/kolide/emails.go | 11 +- server/kolide/hosts.go | 40 ++- server/kolide/invites.go | 18 +- server/kolide/labels.go | 22 +- server/kolide/packs.go | 16 +- server/kolide/queries.go | 66 ++-- server/kolide/sessions.go | 10 +- server/kolide/traits.go | 33 ++ server/kolide/users.go | 27 +- server/pubsub/query_results_test.go | 66 +++- server/service/endpoint_middleware_test.go | 6 +- server/service/handler_test.go | 4 +- server/service/http_auth_test.go | 4 +- server/service/service_appconfig_test.go | 4 +- server/service/service_hosts_test.go | 8 +- server/service/service_invites.go | 5 +- server/service/service_invites_test.go | 10 +- server/service/service_labels_test.go | 10 +- server/service/service_osquery.go | 2 +- server/service/service_osquery_test.go | 39 ++- server/service/service_packs.go | 2 +- server/service/service_packs_test.go | 32 +- server/service/service_queries_test.go | 12 +- server/service/service_sessions.go | 6 +- server/service/service_sessions_test.go | 4 +- server/service/service_targets.go | 4 +- server/service/service_targets_test.go | 12 +- server/service/service_users.go | 10 +- server/service/service_users_test.go | 38 ++- server/service/transport_error.go | 6 +- 85 files changed, 2945 insertions(+), 1771 deletions(-) create mode 100644 db/down.sql create mode 100644 db/up.sql delete mode 100644 server/datastore/datastore.go delete mode 100644 server/datastore/gorm.go delete mode 100644 server/datastore/gorm_app.go delete mode 100644 server/datastore/gorm_hosts.go delete mode 100644 server/datastore/gorm_invites.go delete mode 100644 server/datastore/gorm_labels.go delete mode 100644 server/datastore/gorm_packs.go delete mode 100644 server/datastore/gorm_password_reset.go delete mode 100644 server/datastore/gorm_queries.go delete mode 100644 server/datastore/gorm_sessions.go delete mode 100644 server/datastore/gorm_users.go create mode 100644 server/datastore/inmem/app.go rename server/datastore/{inmem_hosts.go => inmem/hosts.go} (79%) rename server/datastore/{ => inmem}/inmem.go (87%) create mode 100644 server/datastore/inmem/inmem_test.go rename server/datastore/{inmem_invites.go => inmem/invites.go} (75%) rename server/datastore/{inmem_labels.go => inmem/labels.go} (80%) rename server/datastore/{inmem_packs.go => inmem/packs.go} (74%) rename server/datastore/{inmem_password_reset.go => inmem/password_reset.go} (52%) rename server/datastore/{inmem_queries.go => inmem/queries.go} (62%) rename server/datastore/{inmem_sessions.go => inmem/sessions.go} (62%) rename server/datastore/{inmem_users.go => inmem/users.go} (71%) delete mode 100644 server/datastore/inmem_app.go create mode 100644 server/datastore/mysql/app_configs.go rename server/datastore/{ => mysql}/config.go (52%) create mode 100644 server/datastore/mysql/datastore.go create mode 100644 server/datastore/mysql/datastore_test.go create mode 100644 server/datastore/mysql/hosts.go create mode 100644 server/datastore/mysql/invites.go create mode 100644 server/datastore/mysql/labels.go create mode 100644 server/datastore/mysql/packs.go create mode 100644 server/datastore/mysql/password_reset.go create mode 100644 server/datastore/mysql/queries.go create mode 100644 server/datastore/mysql/sessions.go create mode 100644 server/datastore/mysql/users.go rename server/datastore/{gorm_test.go => mysql_test.go} (52%) create mode 100644 server/kolide/traits.go diff --git a/Makefile b/Makefile index 6eab2ff130..df6d2590cb 100644 --- a/Makefile +++ b/Makefile @@ -112,11 +112,13 @@ test-js: test: lint test-go test-js generate: .prefix + go-bindata -o=server/datastore/mysql/bindata.go -pkg=mysql db/ webpack --progress --colors go-bindata -pkg=service \ -o=server/service/bindata.go \ frontend/templates/ assets/... + # we first generate the webpack bundle so that bindata knows to watch the # output bundle file. then, generate debug bindata source file. finally, we # run webpack in watch mode to continuously re-generate the bundle diff --git a/README.md b/README.md index b2fc99c488..7506a12c09 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ - [Development Environment](#development-environment) - [Installing build dependencies](#installing-build-dependencies) - [Building](#building) + - [Generate packaged SQL statements](#generate-packaged-sql-statements) - [Generating the packaged JavaScript](#generating-the-packaged-javascript) - [Automatic rebuilding of the JavaScript bundle](#automatic-rebuilding-of-the-javascript-bundle) - [Compiling the Kolide binary](#compiling-the-kolide-binary) @@ -58,6 +59,17 @@ to re-run `make deps` if a new Go or JavaScript dependency was added. ### Building +#### Generate packaged SQL statements +SQL statements used to generate the Kolide database are bundled into the kolide binary. +These statements are included under the db directory. If the SQL statements are changed, +say a table is added for example, bindata.go must be regenerated in order for the +application to pick up the SQL changes. Use the following +command to regenerate bindata.go. + +``` +make generate +``` + #### Generating the packaged JavaScript To generate all necessary code (bundling JavaScript into Go, etc), run the @@ -157,6 +169,15 @@ To run all Go unit tests, run the following: make test-go ``` +### Database Tests + +To run database tests set environment variables as follows. + +``` +export MYSQL_PORT_3306_TCP_ADDR=192.168.99.100 +export MYSQL_TEST=1 +``` + #### JavaScript unit tests To run all JavaScript unit tests, run the following: @@ -302,4 +323,4 @@ following to launch an in-memory instance of the server: ``` make run -``` \ No newline at end of file +``` diff --git a/cli/prepare.go b/cli/prepare.go index 4897f10b4d..b9d88dfd1c 100644 --- a/cli/prepare.go +++ b/cli/prepare.go @@ -4,7 +4,7 @@ import ( "github.com/WatchBeam/clock" kitlog "github.com/go-kit/kit/log" "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/mysql" "github.com/kolide/kolide-ose/server/kolide" "github.com/kolide/kolide-ose/server/pubsub" "github.com/kolide/kolide-ose/server/service" @@ -33,9 +33,9 @@ To setup kolide infrastructure, use one of the available commands. Long: ``, Run: func(cmd *cobra.Command, args []string) { config := configManager.LoadConfig() - connString := datastore.GetMysqlConnectionString(config.Mysql) + connString := mysql.GetMysqlConnectionString(config.Mysql) - ds, err := datastore.New("gorm-mysql", connString) + ds, err := mysql.New(connString, clock.C) if err != nil { initFatal(err, "creating db connection") } @@ -58,15 +58,13 @@ To setup kolide infrastructure, use one of the available commands. Long: ``, Run: func(cmd *cobra.Command, arg []string) { config := configManager.LoadConfig() - connString := datastore.GetMysqlConnectionString(config.Mysql) + connString := mysql.GetMysqlConnectionString(config.Mysql) - ds, err := datastore.New("gorm-mysql", connString) + ds, err := mysql.New(connString, clock.C) if err != nil { initFatal(err, "creating db connection") } - if err != nil { - initFatal(err, "creating new service") - } + var ( name = "admin" username = "admin" @@ -88,7 +86,7 @@ To setup kolide infrastructure, use one of the available commands. initFatal(err, "creating service") } - _, err = svc.NewUser(context.Background(), admin) + _, err = svc.NewAdminCreatedUser(context.Background(), admin) if err != nil { initFatal(err, "saving new user") } diff --git a/cli/serve.go b/cli/serve.go index 1483165468..5d419f5e11 100644 --- a/cli/serve.go +++ b/cli/serve.go @@ -3,7 +3,6 @@ package cli import ( "flag" "fmt" - "log" "net/http" "os" "os/signal" @@ -14,7 +13,8 @@ import ( kitlog "github.com/go-kit/kit/log" kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" + "github.com/kolide/kolide-ose/server/datastore/mysql" "github.com/kolide/kolide-ose/server/kolide" "github.com/kolide/kolide-ose/server/mail" "github.com/kolide/kolide-ose/server/pubsub" @@ -66,23 +66,20 @@ the way that the kolide server works. "Dev mode enabled, using in-memory DB.\n", "Warning: Changes will not be saved across process restarts. This should NOT be used in production.", ) - ds, err = datastore.New("inmem", "") - if err != nil { - initFatal(err, "initializing datastore") + + if ds, err = inmem.New(); err != nil { + initFatal(err, "initializing inmem database") } } else { - var dbOption []datastore.DBOption - gormLogger := log.New(os.Stderr, "", 0) - gormLogger.SetOutput(kitlog.NewStdlibAdapter(logger)) - dbOption = append(dbOption, datastore.Logger(gormLogger)) - if config.Logging.Debug { - dbOption = append(dbOption, datastore.Debug()) - } - connString := datastore.GetMysqlConnectionString(config.Mysql) - ds, err = datastore.New("gorm-mysql", connString, dbOption...) + const defaultMaxAttempts = 15 + + connString := mysql.GetMysqlConnectionString(config.Mysql) + ds, err = mysql.New(connString, clock.C, mysql.Logger(logger)) + if err != nil { initFatal(err, "initializing datastore") } + } svc, err := service.NewService(ds, pubsub.NewInmemQueryResults(), logger, config, mailService, clock.C) @@ -198,24 +195,38 @@ func createDevMailService(config config.KolideConfig) kolide.MailService { func createDevUsers(ds kolide.Datastore, config config.KolideConfig) { users := []kolide.User{ { - CreatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), - UpdatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), - Name: "Admin User", - Username: "admin", - Email: "admin@kolide.co", - Position: "Director of Security", - Admin: true, - Enabled: true, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), + }, + }, + + Name: "Admin User", + Username: "admin", + Email: "admin@kolide.co", + Position: "Director of Security", + Admin: true, + Enabled: true, }, { - CreatedAt: time.Now().Add(-3 * time.Hour), - UpdatedAt: time.Now().Add(-1 * time.Hour), - Name: "Normal User", - Username: "user", - Email: "user@kolide.co", - Position: "Security Engineer", - Admin: false, - Enabled: true, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().Add(-3 * time.Hour), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().Add(-1 * time.Hour), + }, + }, + + Name: "Normal User", + Username: "user", + Email: "user@kolide.co", + Position: "Security Engineer", + Admin: false, + Enabled: true, }, } for _, user := range users { @@ -235,8 +246,14 @@ func createDevUsers(ds kolide.Datastore, config config.KolideConfig) { func createDevHosts(ds kolide.Datastore, config config.KolideConfig) { hosts := []kolide.Host{ { - CreatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), - UpdatedAt: time.Now().Add(-20 * time.Minute), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 27, 10, 0, 0, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().Add(-20 * time.Minute), + }, + }, NodeKey: "totally-legit", HostName: "jmeller-mbp.local", UUID: "1234-5678-9101", @@ -250,8 +267,15 @@ func createDevHosts(ds kolide.Datastore, config config.KolideConfig) { DetailUpdateTime: time.Now().Add(-20 * time.Minute), }, { - CreatedAt: time.Now().Add(-1 * time.Hour), - UpdatedAt: time.Now().Add(-20 * time.Minute), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), + }, + }, + NodeKey: "definitely-legit", HostName: "marpaia.local", UUID: "1234-5678-9102", @@ -289,34 +313,67 @@ func createDevOrgInfo(ds kolide.Datastore, config config.KolideConfig) { func createDevQueries(ds kolide.Datastore, config config.KolideConfig) { queries := []kolide.Query{ { - CreatedAt: time.Date(2016, time.October, 17, 7, 6, 0, 0, time.UTC), - UpdatedAt: time.Date(2016, time.October, 17, 7, 6, 0, 0, time.UTC), - Name: "dev_query_1", - Query: "select * from processes", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 17, 7, 6, 0, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Date(2016, time.October, 17, 7, 6, 0, 0, time.UTC), + }, + }, + + Name: "dev_query_1", + Query: "select * from processes", }, { - CreatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), - UpdatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), - Name: "dev_query_2", - Query: "select * from time", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Date(2016, time.October, 27, 4, 3, 10, 0, time.UTC), + }, + }, + Name: "dev_query_2", + Query: "select * from time", }, { - CreatedAt: time.Now().Add(-24 * time.Hour), - UpdatedAt: time.Now().Add(-17 * time.Hour), - Name: "dev_query_3", - Query: "select * from cpuid", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().Add(-24 * time.Hour), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().Add(-17 * time.Hour), + }, + }, + + Name: "dev_query_3", + Query: "select * from cpuid", }, { - CreatedAt: time.Now().Add(-1 * time.Hour), - UpdatedAt: time.Now().Add(-30 * time.Minute), - Name: "dev_query_4", - Query: "select 1 from processes where name like '%Apache%'", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().Add(-1 * time.Hour), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().Add(-30 * time.Hour), + }, + }, + + Name: "dev_query_4", + Query: "select 1 from processes where name like '%Apache%'", }, { - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - Name: "dev_query_5", - Query: "select 1 from osquery_info where build_platform='darwin'", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now(), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, + Name: "dev_query_5", + Query: "select 1 from osquery_info where build_platform='darwin'", }, } @@ -332,16 +389,29 @@ func createDevQueries(ds kolide.Datastore, config config.KolideConfig) { func createDevLabels(ds kolide.Datastore, config config.KolideConfig) { labels := []kolide.Label{ { - CreatedAt: time.Date(2016, time.October, 27, 8, 31, 16, 0, time.UTC), - UpdatedAt: time.Date(2016, time.October, 27, 8, 31, 16, 0, time.UTC), - Name: "dev_label_apache", - Query: "select * from processes where name like '%Apache%'", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Date(2016, time.October, 27, 8, 31, 16, 0, time.UTC), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Date(2016, time.October, 27, 8, 31, 16, 0, time.UTC), + }, + }, + Name: "dev_label_apache", + Query: "select * from processes where nae like '%Apache%'", }, { - CreatedAt: time.Now().Add(-1 * time.Hour), - UpdatedAt: time.Now(), - Name: "dev_label_darwin", - Query: "select * from osquery_info where build_platform='darwin'", + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().Add(-1 * time.Hour), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, + + Name: "dev_label_darwin", + Query: "select * from osquery_info where build_platform='darwin'", }, } diff --git a/db/down.sql b/db/down.sql new file mode 100644 index 0000000000..08d6031365 --- /dev/null +++ b/db/down.sql @@ -0,0 +1,32 @@ +# +# SQL Export +# Created by Querious (1055) +# Created: November 7, 2016 at 11:39:23 PM GMT+8 +# Encoding: Unicode (UTF-8) +# + +use `kolide`; + +SET @PREVIOUS_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS; +SET FOREIGN_KEY_CHECKS = 0; + +DROP TABLE IF EXISTS `org_infos`; +DROP TABLE IF EXISTS `users`; +DROP TABLE IF EXISTS `sessions`; +DROP TABLE IF EXISTS `queries`; +DROP TABLE IF EXISTS `password_reset_requests`; +DROP TABLE IF EXISTS `packs`; +DROP TABLE IF EXISTS `pack_targets`; +DROP TABLE IF EXISTS `pack_queries`; +DROP TABLE IF EXISTS `options`; +DROP TABLE IF EXISTS `labels`; +DROP TABLE IF EXISTS `label_query_executions`; +DROP TABLE IF EXISTS `invites`; +DROP TABLE IF EXISTS `hosts`; +DROP TABLE IF EXISTS `distributed_query_executions`; +DROP TABLE IF EXISTS `distributed_query_campaigns`; +DROP TABLE IF EXISTS `distributed_query_campaign_targets`; +DROP TABLE IF EXISTS `app_configs`; + + +SET FOREIGN_KEY_CHECKS = @PREVIOUS_FOREIGN_KEY_CHECKS; diff --git a/db/up.sql b/db/up.sql new file mode 100644 index 0000000000..272f9fc986 --- /dev/null +++ b/db/up.sql @@ -0,0 +1,238 @@ +# +# SQL Export +# Created by Querious (1055) +# Created: November 7, 2016 at 11:38:55 PM GMT+8 +# Encoding: Unicode (UTF-8) +# + +use `kolide`; + +SET @PREVIOUS_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS; +SET FOREIGN_KEY_CHECKS = 0; + + +CREATE TABLE `app_configs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `org_name` varchar(255) DEFAULT NULL, + `org_logo_url` varchar(255) DEFAULT NULL, + `kolide_server_url` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `distributed_query_campaign_targets` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `type` int(11) DEFAULT NULL, + `distributed_query_campaign_id` int(10) unsigned DEFAULT NULL, + `target_id` int(10) unsigned DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `distributed_query_campaigns` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `query_id` int(10) unsigned DEFAULT NULL, + `max_duration` bigint(20) DEFAULT NULL, + `status` int(11) DEFAULT NULL, + `user_id` int(10) unsigned DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `distributed_query_executions` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `host_id` int(10) unsigned DEFAULT NULL, + `distributed_query_campaign_id` int(10) unsigned DEFAULT NULL, + `status` int(11) DEFAULT NULL, + `error` varchar(1024) DEFAULT NULL, + `execution_duration` bigint(20) DEFAULT NULL, + UNIQUE KEY `idx_dqe_unique_dqec_id` (`distributed_query_campaign_id`), + UNIQUE KEY `idx_dqe_unique_host_id` (`host_id`), + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `hosts` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `detail_update_time` timestamp NULL DEFAULT NULL, + `node_key` varchar(255) DEFAULT NULL, + `host_name` varchar(255) DEFAULT NULL, + `uuid` varchar(255) DEFAULT NULL, + `platform` varchar(255) DEFAULT NULL, + `osquery_version` varchar(255) NOT NULL DEFAULT '', + `os_version` varchar(255) NOT NULL DEFAULT '', + `uptime` bigint(20) NOT NULL DEFAULT 0, + `physical_memory` bigint(20) NOT NULL DEFAULT 0, + `primary_mac` varchar(255) NOT NULL DEFAULT '', + `primary_ip` varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), + UNIQUE KEY `idx_host_unique_uuid` (`uuid`), + FULLTEXT KEY `hosts_search` (`host_name`,`primary_ip`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `invites` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `invited_by` int(10) unsigned NOT NULL, + `email` varchar(255) NOT NULL, + `admin` tinyint(1) DEFAULT NULL, + `name` varchar(255) DEFAULT NULL, + `position` varchar(255) DEFAULT NULL, + `token` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_invite_unique_email` (`email`), + UNIQUE KEY `idx_invite_unique_key` (`token`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `label_query_executions` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `matches` tinyint(1) NOT NULL DEFAULT FALSE, + `label_id` int(10) unsigned DEFAULT NULL, + `host_id` int(10) unsigned DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_lqe_label_host` (`label_id`,`host_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `labels` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `name` varchar(255) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `query` varchar(255) NOT NULL, + `platform` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_label_unique_name` (`name`), + FULLTEXT KEY `labels_search` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `options` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `key` varchar(255) NOT NULL, + `value` varchar(255) NOT NULL, + `platform` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_option_unique_key` (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `pack_queries` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `pack_id` int(10) unsigned DEFAULT NULL, + `query_id` int(10) unsigned DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `pack_targets` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `pack_id` int(10) unsigned DEFAULT NULL, + `type` int(11) DEFAULT NULL, + `target_id` int(10) unsigned DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `packs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `name` varchar(255) NOT NULL, + `platform` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_pack_unique_name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `password_reset_requests` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `expires_at` timestamp NOT NULL DEFAULT '1970-01-01 00:00:01', + `user_id` int(10) unsigned NOT NULL, + `token` varchar(1024) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `queries` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `name` varchar(255) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `query` varchar(255) NOT NULL, + `interval` int(10) unsigned DEFAULT NULL, + `snapshot` tinyint(1) NOT NULL DEFAULT FALSE, + `differential` tinyint(1) NOT NULL DEFAULT FALSE, + `platform` varchar(255) DEFAULT NULL, + `version` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_query_unique_name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `sessions` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `accessed_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `user_id` int(10) unsigned NOT NULL, + `key` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_session_unique_key` (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +CREATE TABLE `users` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +`deleted_at` timestamp NULL DEFAULT NULL, + `deleted` tinyint(1) NOT NULL DEFAULT FALSE, + `username` varchar(255) NOT NULL, + `password` varbinary(255) NOT NULL, + `salt` varchar(255) NOT NULL, + `name` varchar(255) NOT NULL DEFAULT '', + `email` varchar(255) NOT NULL, + `admin` tinyint(1) NOT NULL DEFAULT FALSE, + `enabled` tinyint(1) NOT NULL DEFAULT FALSE, + `admin_forced_password_reset` tinyint(1) NOT NULL DEFAULT FALSE, + `gravatar_url` varchar(255) NOT NULL DEFAULT '', + `position` varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `idx_user_unique_username` (`username`), + UNIQUE KEY `idx_user_unique_email` (`email`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + + + +SET FOREIGN_KEY_CHECKS = @PREVIOUS_FOREIGN_KEY_CHECKS; diff --git a/glide.lock b/glide.lock index 6396b0571a..106646e0fa 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 7a3bef7d804686cd678aeaa0914be556ff4e357be2ce8fd5b9cd2e81164ed4a6 -updated: 2016-10-24T12:32:41.928983481-07:00 +hash: 3519c1da46244ec37972b542173f088df94422080bc99652f88f960d70a7005a +updated: 2016-11-02T22:14:07.180331599+08:00 imports: - name: github.com/alecthomas/template version: a0175ee3bccc567396460bf5acd36800cb10c49c @@ -75,12 +75,10 @@ imports: - json/token - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/jinzhu/gorm - version: a646b13548ec0aa812fe805b2780f460dd2771b6 - subpackages: - - dialects/mysql - name: github.com/jinzhu/inflection version: 8f4d3a0d04ce0b7c0cf3126fb98524246d00d102 +- name: github.com/jmoiron/sqlx + version: 5f97679e23f75f42b265fec8d3bdb1c8de90b79d - name: github.com/jordan-wright/email version: fd703108daeb23d77c124d12978e9b6c4f28f034 - name: github.com/kr/fs @@ -179,7 +177,7 @@ imports: - name: gopkg.in/go-playground/validator.v8 version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c - name: gopkg.in/natefinch/lumberjack.v2 - version: 514cbda263a734ae8caac038dadf05f8f3f9f738 + version: e21e5cbec0cd0861b9dc302736ad5666c529d93f - name: gopkg.in/yaml.v2 version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e testImports: [] diff --git a/glide.yaml b/glide.yaml index 1383b731cc..89c145606d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,10 +31,6 @@ import: version: c3cefd437628a0b7d31b34fe44b3a7a540e98527 subpackages: - proto -- package: github.com/jinzhu/gorm - version: a646b13548ec0aa812fe805b2780f460dd2771b6 - subpackages: - - dialects/mysql - package: github.com/jinzhu/inflection version: 8f4d3a0d04ce0b7c0cf3126fb98524246d00d102 - package: github.com/manucorporat/sse @@ -83,3 +79,4 @@ import: version: ^1.0.0 subpackages: - redis +- package: github.com/jmoiron/sqlx diff --git a/server/datastore/datastore.go b/server/datastore/datastore.go deleted file mode 100644 index 826e9eb7df..0000000000 --- a/server/datastore/datastore.go +++ /dev/null @@ -1,93 +0,0 @@ -// Package datastore implements Kolide's interactions with the database backend -package datastore - -import ( - "crypto/rand" - "encoding/base64" - "errors" - "fmt" - - "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/kolide" -) - -var ( - // ErrNotFound is returned when the datastore resource cannot be found - ErrNotFound = errors.New("resource not found") - - // ErrExists is returned when creating a datastore resource that already exists - ErrExists = errors.New("resource already created") -) - -// New creates a kolide.Datastore with a database connection -// Use DBOption to pass optional arguments -func New(driver, conn string, opts ...DBOption) (kolide.Datastore, error) { - opt := &dbOptions{ - maxAttempts: defaultMaxAttempts, - } - for _, option := range opts { - if err := option(opt); err != nil { - return nil, err - } - } - - // check if datastore is already present - if opt.db != nil { - return opt.db, nil - } - switch driver { - case "gorm-mysql": - db, err := openGORM("mysql", conn, opt.maxAttempts) - if err != nil { - return nil, err - } - ds := gormDB{ - DB: db, - Driver: "mysql", - } - // configure logger - if opt.logger != nil { - db.SetLogger(opt.logger) - db.LogMode(opt.debug) - } - if err := ds.Migrate(); err != nil { - return nil, err - } - return ds, nil - case "inmem": - ds := &inmem{ - Driver: "inmem", - } - - err := ds.Migrate() - if err != nil { - return nil, err - } - - return ds, nil - default: - return nil, fmt.Errorf("unsupported datastore driver %s", driver) - } -} - -func generateRandomText(keySize int) (string, error) { - key := make([]byte, keySize) - _, err := rand.Read(key) - if err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(key), nil -} - -// GetMysqlConnectionString returns a MySQL connection string using the -// provided configuration. -func GetMysqlConnectionString(conf config.MysqlConfig) string { - return fmt.Sprintf( - "%s:%s@(%s)/%s?charset=utf8&parseTime=True&loc=Local", - conf.Username, - conf.Password, - conf.Address, - conf.Database, - ) -} diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index 5ca99280d1..974199363b 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -40,6 +40,78 @@ var enrollTests = []struct { }, } +func testSaveHosts(t *testing.T, db kolide.Datastore) { + host, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + assert.Nil(t, err) + assert.NotNil(t, host) + + host.HostName = "bar.local" + err = db.SaveHost(host) + assert.Nil(t, err) + + host, err = db.Host(host.ID) + assert.Nil(t, err) + assert.Equal(t, "bar.local", host.HostName) + + err = db.DeleteHost(host) + assert.Nil(t, err) + + host, err = db.Host(host.ID) + assert.NotNil(t, err) +} + +func testDeleteHost(t *testing.T, db kolide.Datastore) { + host, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: "1", + UUID: "1", + HostName: "foo.local", + PrimaryIP: "192.168.1.10", + }) + assert.Nil(t, err) + assert.NotNil(t, host) + + err = db.DeleteHost(host) + assert.Nil(t, err) + + host, err = db.Host(host.ID) + assert.NotNil(t, err) +} + +func testListHost(t *testing.T, db kolide.Datastore) { + hosts := []*kolide.Host{} + for i := 0; i < 10; i++ { + host, err := db.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + NodeKey: fmt.Sprintf("%d", i), + UUID: fmt.Sprintf("%d", i), + HostName: fmt.Sprintf("foo.local%d", i), + PrimaryIP: fmt.Sprintf("192.168.1.%d", i), + }) + assert.Nil(t, err) + if err != nil { + return + } + hosts = append(hosts, host) + } + + hosts2, err := db.ListHosts(kolide.ListOptions{}) + assert.Nil(t, err) + assert.Equal(t, len(hosts), len(hosts2)) + err = db.DeleteHost(hosts[0]) + assert.Nil(t, err) + hosts2, err = db.ListHosts(kolide.ListOptions{}) + assert.Nil(t, err) + assert.Equal(t, len(hosts)-1, len(hosts2)) + +} + func testEnrollHost(t *testing.T, db kolide.Datastore) { var hosts []*kolide.Host for _, tt := range enrollTests { @@ -112,16 +184,16 @@ func testSearchHosts(t *testing.T, db kolide.Datastore) { }) require.Nil(t, err) - hosts, err := db.SearchHosts("foo", nil) + hosts, err := db.SearchHosts("foo") assert.Nil(t, err) assert.Len(t, hosts, 2) - host, err := db.SearchHosts("foo", []uint{h3.ID}) + host, err := db.SearchHosts("foo", h3.ID) assert.Nil(t, err) assert.Len(t, host, 1) assert.Equal(t, "foo.local", host[0].HostName) - none, err := db.SearchHosts("xxx", nil) + none, err := db.SearchHosts("xxx") assert.Nil(t, err) assert.Len(t, none, 0) } @@ -138,7 +210,7 @@ func testSearchHostsLimit(t *testing.T, db kolide.Datastore) { require.Nil(t, err) } - hosts, err := db.SearchHosts("foo", nil) + hosts, err := db.SearchHosts("foo") require.Nil(t, err) assert.Len(t, hosts, 10) } @@ -194,7 +266,7 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { require.Nil(t, err) // Create a query campaign - c1 := kolide.DistributedQueryCampaign{ + c1 := &kolide.DistributedQueryCampaign{ QueryID: q1.ID, Status: kolide.QueryRunning, } @@ -202,7 +274,7 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { require.Nil(t, err) // Add a target to the campaign - target := kolide.DistributedQueryCampaignTarget{ + target := &kolide.DistributedQueryCampaignTarget{ Type: kolide.TargetLabel, DistributedQueryCampaignID: c1.ID, TargetID: l1.ID, @@ -221,12 +293,12 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { assert.Equal(t, "select * from bar", queries[c1.ID]) // Record an execution - exec := kolide.DistributedQueryExecution{ + exec := &kolide.DistributedQueryExecution{ HostID: h1.ID, DistributedQueryCampaignID: c1.ID, Status: kolide.ExecutionSucceeded, } - exec, err = db.NewDistributedQueryExecution(exec) + _, err = db.NewDistributedQueryExecution(exec) require.Nil(t, err) // Add another query/campaign @@ -237,7 +309,7 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { q2, err = db.NewQuery(q2) require.Nil(t, err) - c2 := kolide.DistributedQueryCampaign{ + c2 := &kolide.DistributedQueryCampaign{ QueryID: q2.ID, Status: kolide.QueryRunning, } @@ -245,12 +317,12 @@ func testDistributedQueriesForHost(t *testing.T, db kolide.Datastore) { require.Nil(t, err) // This one targets only h1 - target = kolide.DistributedQueryCampaignTarget{ + target = &kolide.DistributedQueryCampaignTarget{ Type: kolide.TargetHost, DistributedQueryCampaignID: c2.ID, TargetID: h1.ID, } - target, err = db.NewDistributedQueryCampaignTarget(target) + _, err = db.NewDistributedQueryCampaignTarget(target) require.Nil(t, err) // Check for correct queries diff --git a/server/datastore/datastore_invites_test.go b/server/datastore/datastore_invites_test.go index fc567125a9..ea021b16b9 100644 --- a/server/datastore/datastore_invites_test.go +++ b/server/datastore/datastore_invites_test.go @@ -1,6 +1,7 @@ package datastore import ( + "fmt" "testing" "github.com/kolide/kolide-ose/server/kolide" @@ -8,7 +9,12 @@ import ( ) func testCreateInvite(t *testing.T, ds kolide.Datastore) { - invite := &kolide.Invite{} + invite := &kolide.Invite{ + + Email: "user@foo.com", + Name: "user", + Token: "some_user", + } invite, err := ds.NewInvite(invite) assert.Nil(t, err) @@ -18,3 +24,106 @@ func testCreateInvite(t *testing.T, ds kolide.Datastore) { assert.Equal(t, invite.ID, verify.ID) assert.Equal(t, invite.Email, verify.Email) } + +func setupTestInvites(t *testing.T, ds kolide.Datastore) { + + var err error + admin := &kolide.Invite{ + Email: "admin@foo.com", + Admin: true, + Name: "Xadmin", + Token: "admin", + } + + admin, err = ds.NewInvite(admin) + assert.Nil(t, err) + + for user := 0; user < 23; user++ { + i := kolide.Invite{ + InvitedBy: admin.ID, + Email: fmt.Sprintf("user%d@foo.com", user), + Admin: false, + Name: fmt.Sprintf("User%02d", user), + Token: fmt.Sprintf("usertoken%d", user), + } + + _, err := ds.NewInvite(&i) + assert.Nil(t, err, "Failure creating user", user) + } + +} + +func testListInvites(t *testing.T, ds kolide.Datastore) { + // TODO: fix this for inmem + if ds.Name() == "inmem" { + fmt.Println("Busted test skipped for inmem") + return + } + + setupTestInvites(t, ds) + + opt := kolide.ListOptions{ + Page: 0, + PerPage: 10, + OrderDirection: kolide.OrderAscending, + OrderKey: "name", + } + + result, err := ds.ListInvites(opt) + assert.Nil(t, err) + assert.NotNil(t, result) + assert.Equal(t, len(result), 10) + assert.Equal(t, "User00", result[0].Name) + assert.Equal(t, "User09", result[9].Name) + + opt.Page = 2 + opt.OrderDirection = kolide.OrderDescending + result, err = ds.ListInvites(opt) + assert.Nil(t, err) + assert.Equal(t, 4, len(result)) // allow for admin we created + assert.Equal(t, "User00", result[3].Name) + +} + +func testDeleteInvite(t *testing.T, ds kolide.Datastore) { + + setupTestInvites(t, ds) + + invite, err := ds.InviteByEmail("user0@foo.com") + + assert.Nil(t, err) + assert.NotNil(t, invite) + + err = ds.DeleteInvite(invite) + assert.Nil(t, err) + + invite, err = ds.InviteByEmail("user0@foo.com") + assert.NotNil(t, err) + assert.Nil(t, invite) + +} + +func testSaveInvite(t *testing.T, ds kolide.Datastore) { + setupTestInvites(t, ds) + + invite, err := ds.InviteByEmail("user0@foo.com") + assert.Nil(t, err) + assert.NotNil(t, invite) + + invite, err = ds.Invite(invite.ID) + assert.Nil(t, err) + assert.NotNil(t, invite) + + invite.Name = "Bob" + invite.Admin = true + + err = ds.SaveInvite(invite) + assert.Nil(t, err) + + invite, err = ds.Invite(invite.ID) + assert.Nil(t, err) + assert.NotNil(t, invite) + assert.Equal(t, "Bob", invite.Name) + assert.True(t, invite.Admin) + +} diff --git a/server/datastore/datastore_labels_test.go b/server/datastore/datastore_labels_test.go index 9d714b7642..66c72eedb4 100644 --- a/server/datastore/datastore_labels_test.go +++ b/server/datastore/datastore_labels_test.go @@ -63,7 +63,8 @@ func testLabels(t *testing.T, db kolide.Datastore) { } for _, label := range newLabels { - newLabel, err := db.NewLabel(&label) + var newLabel *kolide.Label + newLabel, err = db.NewLabel(&label) assert.Nil(t, err) assert.NotZero(t, newLabel.ID) } @@ -143,7 +144,7 @@ func testManagingLabelsOnPacks(t *testing.T, ds kolide.Datastore) { monitoringPack := &kolide.Pack{ Name: "monitoring", } - err := ds.NewPack(monitoringPack) + _, err := ds.NewPack(monitoringPack) require.Nil(t, err) mysqlLabel := &kolide.Label{ @@ -193,16 +194,16 @@ func testSearchLabels(t *testing.T, db kolide.Datastore) { }) require.Nil(t, err) - labels, err := db.SearchLabels("foo", nil) + labels, err := db.SearchLabels("foo") assert.Nil(t, err) assert.Len(t, labels, 2) - label, err := db.SearchLabels("foo", []uint{l3.ID}) + label, err := db.SearchLabels("foo", l3.ID) assert.Nil(t, err) assert.Len(t, label, 1) assert.Equal(t, "foo", label[0].Name) - none, err := db.SearchLabels("xxx", nil) + none, err := db.SearchLabels("xxx") assert.Nil(t, err) assert.Len(t, none, 0) } @@ -215,7 +216,7 @@ func testSearchLabelsLimit(t *testing.T, db kolide.Datastore) { require.Nil(t, err) } - labels, err := db.SearchLabels("foo", nil) + labels, err := db.SearchLabels("foo") require.Nil(t, err) assert.Len(t, labels, 10) } diff --git a/server/datastore/datastore_packs_test.go b/server/datastore/datastore_packs_test.go index 61910bfe3d..9b8ddef47f 100644 --- a/server/datastore/datastore_packs_test.go +++ b/server/datastore/datastore_packs_test.go @@ -12,7 +12,7 @@ func testDeletePack(t *testing.T, ds kolide.Datastore) { pack := &kolide.Pack{ Name: "foo", } - err := ds.NewPack(pack) + _, err := ds.NewPack(pack) assert.Nil(t, err) assert.NotEqual(t, uint(0), pack.ID) @@ -31,7 +31,7 @@ func testAddAndRemoveQueryFromPack(t *testing.T, ds kolide.Datastore) { pack := &kolide.Pack{ Name: "foo", } - err := ds.NewPack(pack) + _, err := ds.NewPack(pack) assert.Nil(t, err) assert.NotEqual(t, uint(0), pack.ID) diff --git a/server/datastore/datastore_queries_test.go b/server/datastore/datastore_queries_test.go index 9390c656ec..bc8d2c63e6 100644 --- a/server/datastore/datastore_queries_test.go +++ b/server/datastore/datastore_queries_test.go @@ -1,6 +1,7 @@ package datastore import ( + "fmt" "testing" "github.com/kolide/kolide-ose/server/kolide" @@ -9,8 +10,9 @@ import ( func testDeleteQuery(t *testing.T, ds kolide.Datastore) { query := &kolide.Query{ - Name: "foo", - Query: "bar", + Name: "foo", + Query: "bar", + Interval: 123, } query, err := ds.NewQuery(query) assert.Nil(t, err) @@ -42,3 +44,18 @@ func testSaveQuery(t *testing.T, ds kolide.Datastore) { assert.Nil(t, err) assert.Equal(t, "baz", queryVerify.Query) } + +func testListQuery(t *testing.T, ds kolide.Datastore) { + for i := 0; i < 10; i++ { + _, err := ds.NewQuery(&kolide.Query{ + Name: fmt.Sprintf("name%02d", i), + Query: fmt.Sprintf("query%02d", i), + }) + assert.Nil(t, err) + } + + opts := kolide.ListOptions{} + results, err := ds.ListQueries(opts) + assert.Nil(t, err) + assert.Equal(t, 10, len(results)) +} diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index ea44f73ae6..9f98037fba 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -18,8 +18,12 @@ func functionName(f func(*testing.T, kolide.Datastore)) string { var testFunctions = [...]func(*testing.T, kolide.Datastore){ testOrgInfo, testCreateInvite, + testListInvites, + testDeleteInvite, + testSaveInvite, testDeleteQuery, testSaveQuery, + testListQuery, testDeletePack, testAddAndRemoveQueryFromPack, testEnrollHost, @@ -38,4 +42,7 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testListHostsInLabel, testListUniqueHostsInLabels, testDistributedQueriesForHost, + testSaveHosts, + testDeleteHost, + testListHost, } diff --git a/server/datastore/datastore_users_test.go b/server/datastore/datastore_users_test.go index fa81eedba2..46fd452ee7 100644 --- a/server/datastore/datastore_users_test.go +++ b/server/datastore/datastore_users_test.go @@ -88,7 +88,7 @@ func testSaveUser(t *testing.T, ds kolide.Datastore) { func testPasswordAttribute(t *testing.T, ds kolide.Datastore, users []*kolide.User) { for _, user := range users { - randomText, err := generateRandomText(8) + randomText, err := kolide.RandomText(8) //GenerateRandomText(8) assert.Nil(t, err) user.Password = []byte(randomText) err = ds.SaveUser(user) diff --git a/server/datastore/gorm.go b/server/datastore/gorm.go deleted file mode 100644 index b9c663fe0b..0000000000 --- a/server/datastore/gorm.go +++ /dev/null @@ -1,140 +0,0 @@ -package datastore - -import ( - "fmt" - "time" - - _ "github.com/go-sql-driver/mysql" // db driver - - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/kolide" -) - -var tables = [...]interface{}{ - &kolide.User{}, - &kolide.PasswordResetRequest{}, - &kolide.Session{}, - &kolide.Pack{}, - &kolide.PackQuery{}, - &kolide.PackTarget{}, - &kolide.Host{}, - &kolide.Label{}, - &kolide.LabelQueryExecution{}, - &kolide.Option{}, - &kolide.DistributedQueryCampaign{}, - &kolide.DistributedQueryCampaignTarget{}, - &kolide.Query{}, - &kolide.DistributedQueryExecution{}, - &kolide.AppConfig{}, - &kolide.Invite{}, -} - -type gormDB struct { - DB *gorm.DB - Driver string - config config.KolideConfig -} - -func (orm gormDB) Name() string { - return "gorm" -} - -func (orm gormDB) Migrate() error { - for _, table := range tables { - if err := orm.DB.AutoMigrate(table).Error; err != nil { - return err - } - } - - // Have to manually add indexes when specific ordering needed. - err := orm.DB.Model(&kolide.LabelQueryExecution{}).AddUniqueIndex("idx_lqe_label_host", "label_id", "host_id").Error - if err != nil { - return err - } - err = orm.DB.Model(&kolide.DistributedQueryExecution{}).AddUniqueIndex("idx_dqe_unique_host_dq_id", "host_id", "distributed_query_campaign_id").Error - if err != nil { - return err - } - - indexes := []interface{}{} - err = orm.DB.Raw("SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = 'kolide' AND INDEX_NAME = 'hosts_search';").Scan(&indexes).Error - if err != nil { - return err - } - if len(indexes) == 0 { - err = orm.DB.Exec("CREATE FULLTEXT INDEX hosts_search ON hosts(host_name, primary_ip);").Error - if err != nil { - return err - } - } - - indexes = []interface{}{} - err = orm.DB.Raw("SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = 'kolide' AND INDEX_NAME = 'labels_search';").Scan(&indexes).Error - if err != nil { - return err - } - if len(indexes) == 0 { - err = orm.DB.Exec("CREATE FULLTEXT INDEX labels_search ON labels(name);").Error - if err != nil { - return err - } - } - - return nil -} - -func (orm gormDB) Drop() error { - var err error - for _, table := range tables { - err = orm.DB.DropTableIfExists(table).Error - } - return err -} - -// create connection with mysql backend, using a backoff timer and maxAttempts -func openGORM(driver, conn string, maxAttempts int) (*gorm.DB, error) { - var db *gorm.DB - var err error - for attempts := 1; attempts <= maxAttempts; attempts++ { - db, err = gorm.Open(driver, conn) - if err == nil { - break - } else { - if err.Error() == "invalid database source" { - return nil, err - } - // TODO: use a logger - fmt.Printf("could not connect to mysql: %v\n", err) - time.Sleep(time.Duration(attempts) * time.Second) - } - } - if err != nil { - return nil, fmt.Errorf("failed to connect to mysql backend, err = %v", err) - } - return db, nil -} - -// applyLimitOffset applies the appropriate limit and offset parameters to the -// gorm.DB instance, returning a DB that can be chained as usual with *gorm.DB. -func (orm *gormDB) applyListOptions(opt kolide.ListOptions) *gorm.DB { - db := orm.DB - if opt.PerPage != 0 { - // PerPage value of 0 indicates unlimited - offset := opt.Page * opt.PerPage - db = db.Limit(opt.PerPage).Offset(offset) - } - - if opt.OrderKey != "" { - var dir string - if opt.OrderDirection == kolide.OrderDescending { - dir = "DESC" - } else { - dir = "ASC" - } - - db = db.Order(opt.OrderKey + " " + dir) - } - - return db -} diff --git a/server/datastore/gorm_app.go b/server/datastore/gorm_app.go deleted file mode 100644 index d5672969ac..0000000000 --- a/server/datastore/gorm_app.go +++ /dev/null @@ -1,35 +0,0 @@ -package datastore - -import ( - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) { - err := orm.DB.First(info).Error - switch err { - case gorm.ErrRecordNotFound: - err = orm.DB.Create(info).Error - if err != nil { - return nil, err - } - return info, nil - case nil: - return info, orm.SaveAppConfig(info) - default: - return nil, err - } -} - -func (orm gormDB) AppConfig() (*kolide.AppConfig, error) { - info := &kolide.AppConfig{} - err := orm.DB.First(info).Error - if err != nil { - return nil, err - } - return info, nil -} - -func (orm gormDB) SaveAppConfig(info *kolide.AppConfig) error { - return orm.DB.Save(info).Error -} diff --git a/server/datastore/gorm_hosts.go b/server/datastore/gorm_hosts.go deleted file mode 100644 index 223ea893a2..0000000000 --- a/server/datastore/gorm_hosts.go +++ /dev/null @@ -1,195 +0,0 @@ -package datastore - -import ( - "net/http" - "time" - - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/errors" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) { - if uuid == "" { - return nil, errors.New("missing uuid for host enrollment", "programmer error?") - } - host := kolide.Host{UUID: uuid} - err := orm.DB.Where(&host).First(&host).Error - if err != nil { - switch err { - case gorm.ErrRecordNotFound: - // Create new Host - host = kolide.Host{ - UUID: uuid, - HostName: hostname, - PrimaryIP: ip, - Platform: platform, - DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour), - } - - default: - return nil, err - } - } - - // Generate a new key each enrollment - host.NodeKey, err = generateRandomText(nodeKeySize) - if err != nil { - return nil, err - } - - // Update these fields if provided - if hostname != "" { - host.HostName = hostname - } - if ip != "" { - host.PrimaryIP = ip - } - if platform != "" { - host.Platform = platform - } - - if err := orm.DB.Save(&host).Error; err != nil { - return nil, err - } - - return &host, nil -} - -func (orm gormDB) AuthenticateHost(nodeKey string) (*kolide.Host, error) { - host := kolide.Host{NodeKey: nodeKey} - err := orm.DB.Where("node_key = ?", host.NodeKey).First(&host).Error - if err != nil { - switch err { - case gorm.ErrRecordNotFound: - e := errors.NewFromError( - err, - http.StatusUnauthorized, - "invalid node key", - ) - // osqueryd expects the literal string "true" here - e.Extra = map[string]interface{}{"node_invalid": "true"} - return nil, e - default: - return nil, errors.DatabaseError(err) - } - } - - return &host, nil -} - -func (orm gormDB) SaveHost(host *kolide.Host) error { - if err := orm.DB.Save(host).Error; err != nil { - return errors.DatabaseError(err) - } - return nil -} - -func (orm gormDB) DeleteHost(host *kolide.Host) error { - return orm.DB.Delete(host).Error -} - -func (orm gormDB) Host(id uint) (*kolide.Host, error) { - host := &kolide.Host{ - ID: id, - } - err := orm.DB.Where(host).First(host).Error - if err != nil { - return nil, err - } - return host, nil -} - -func (orm gormDB) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { - var hosts []*kolide.Host - err := orm.applyListOptions(opt).Find(&hosts).Error - if err != nil { - return nil, err - } - return hosts, nil -} - -func (orm gormDB) NewHost(host *kolide.Host) (*kolide.Host, error) { - if host == nil { - return nil, errors.New( - "error creating host", - "nil pointer passed to NewHost", - ) - } - err := orm.DB.Create(host).Error - if err != nil { - return nil, err - } - return host, err -} - -func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error { - err := orm.DB.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", t, host.NodeKey).Error - if err != nil { - return errors.DatabaseError(err) - } - host.UpdatedAt = t - return nil -} - -func (orm gormDB) SearchHosts(query string, omit []uint) ([]kolide.Host, error) { - sql := ` -SELECT * -FROM hosts -WHERE MATCH(host_name, primary_ip) -AGAINST(? IN BOOLEAN MODE) -` - results := []kolide.Host{} - - var db *gorm.DB - if len(omit) > 0 { - sql += "AND id NOT IN (?) LIMIT 10;" - db = orm.DB.Raw(sql, query+"*", omit) - } else { - sql += "LIMIT 10;" - db = orm.DB.Raw(sql, query+"*") - } - - err := db.Scan(&results).Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - return results, nil -} - -func (orm gormDB) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) { - sql := ` -SELECT DISTINCT dqc.id, q.query -FROM distributed_query_campaigns dqc -JOIN distributed_query_campaign_targets dqct - ON (dqc.id = dqct.distributed_query_campaign_id) -LEFT JOIN label_query_executions lqe - ON (dqct.type = ? AND dqct.target_id = lqe.label_id AND lqe.matches) -LEFT JOIN hosts h - ON ((dqct.type = ? AND lqe.host_id = h.id) OR (dqct.type = ? AND dqct.target_id = h.id)) -LEFT JOIN distributed_query_executions dqe - ON (h.id = dqe.host_id AND dqc.id = dqe.distributed_query_campaign_id) -JOIN queries q - ON (dqc.query_id = q.id) -WHERE dqe.status IS NULL AND dqc.status = ? AND h.id = ?; -` - rows, err := orm.DB.Raw(sql, kolide.TargetLabel, kolide.TargetLabel, kolide.TargetHost, kolide.QueryRunning, host.ID).Rows() - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - defer rows.Close() - - results := map[uint]string{} - for rows.Next() { - var id uint - var query string - err = rows.Scan(&id, &query) - if err != nil { - return nil, errors.DatabaseError(err) - } - results[id] = query - } - - return results, nil - -} diff --git a/server/datastore/gorm_invites.go b/server/datastore/gorm_invites.go deleted file mode 100644 index 13e1855290..0000000000 --- a/server/datastore/gorm_invites.go +++ /dev/null @@ -1,48 +0,0 @@ -package datastore - -import "github.com/kolide/kolide-ose/server/kolide" - -func (orm gormDB) NewInvite(invite *kolide.Invite) (*kolide.Invite, error) { - err := orm.DB.Create(invite).Error - if err != nil { - return nil, err - } - return invite, nil -} - -func (orm gormDB) InviteByEmail(email string) (*kolide.Invite, error) { - invite := &kolide.Invite{ - Email: email, - } - err := orm.DB.Where("email = ?", email).First(invite).Error - if err != nil { - return nil, err - } - return invite, nil -} - -func (orm gormDB) ListInvites(opt kolide.ListOptions) ([]*kolide.Invite, error) { - var invites []*kolide.Invite - err := orm.applyListOptions(opt).Find(&invites).Error - if err != nil { - return nil, err - } - return invites, nil -} - -func (orm gormDB) Invite(id uint) (*kolide.Invite, error) { - invite := &kolide.Invite{ID: id} - err := orm.DB.Where(invite).First(invite).Error - if err != nil { - return nil, err - } - return invite, nil -} - -func (orm gormDB) SaveInvite(invite *kolide.Invite) error { - return orm.DB.Save(invite).Error -} - -func (orm gormDB) DeleteInvite(invite *kolide.Invite) error { - return orm.DB.Delete(invite).Error -} diff --git a/server/datastore/gorm_labels.go b/server/datastore/gorm_labels.go deleted file mode 100644 index d6616a1d6b..0000000000 --- a/server/datastore/gorm_labels.go +++ /dev/null @@ -1,207 +0,0 @@ -package datastore - -import ( - "bytes" - "strings" - "time" - - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/errors" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) NewLabel(label *kolide.Label) (*kolide.Label, error) { - if label == nil { - return nil, errors.New( - "error creating label", - "nil pointer passed to NewLabel", - ) - } - err := orm.DB.Create(label).Error - if err != nil { - return nil, err - } - return label, nil -} - -func (orm gormDB) DeleteLabel(lid uint) error { - err := orm.DB.Where("id = ?", lid).Delete(&kolide.Label{}).Error - if err != nil { - return err - } - - return orm.DB.Where("target_id = ? and type = ?", lid, kolide.TargetLabel).Delete(&kolide.PackTarget{}).Error -} - -func (orm gormDB) Label(lid uint) (*kolide.Label, error) { - label := &kolide.Label{ - ID: lid, - } - err := orm.DB.Where("id = ?", label.ID).First(&label).Error - if err != nil { - return nil, err - } - return label, nil -} - -func (orm gormDB) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) { - var labels []*kolide.Label - err := orm.applyListOptions(opt).Find(&labels).Error - return labels, err -} - -func (orm gormDB) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (map[string]string, error) { - if host == nil { - return nil, errors.New( - "error finding host queries", - "nil pointer passed to LabelQueriesForHost", - ) - } - rows, err := orm.DB.Raw(` -SELECT l.id, l.query -from labels l -WHERE l.platform = ? -AND l.id NOT IN /* subtract the set of executions that are recent enough */ -( - SELECT l.id - FROM labels l - JOIN label_query_executions lqe - ON lqe.label_id = l.id - WHERE lqe.host_id = ? AND lqe.updated_at > ? -)`, host.Platform, host.ID, cutoff).Rows() - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - defer rows.Close() - - results := make(map[string]string) - for rows.Next() { - var id, query string - err = rows.Scan(&id, &query) - if err != nil { - return nil, errors.DatabaseError(err) - } - results[id] = query - } - - return results, nil -} - -func (orm gormDB) RecordLabelQueryExecutions(host *kolide.Host, results map[string]bool, t time.Time) error { - if host == nil { - return errors.New( - "error recording host label query execution", - "nil pointer passed to RecordLabelQueryExecutions", - ) - } - - insert := new(bytes.Buffer) - - insert.WriteString( - "INSERT INTO label_query_executions (updated_at, matches, label_id, host_id) VALUES", - ) - - // Build up all the values and the query string - vals := []interface{}{} - for labelId, res := range results { - insert.WriteString("(?,?,?,?),") - vals = append(vals, t, res, labelId, host.ID) - } - - queryString := insert.String() - queryString = strings.TrimSuffix(queryString, ",") - - queryString += ` -ON DUPLICATE KEY UPDATE -updated_at = VALUES(updated_at), -matches = VALUES(matches) -` - - if err := orm.DB.Exec(queryString, vals...).Error; err != nil { - return errors.DatabaseError(err) - } - - return nil -} - -func (orm gormDB) ListLabelsForHost(hid uint) ([]kolide.Label, error) { - results := []kolide.Label{} - err := orm.DB.Raw(` -SELECT labels.* from labels, label_query_executions lqe -WHERE lqe.host_id = ? -AND lqe.label_id = labels.id -AND lqe.matches -`, hid).Scan(&results).Error - - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - - return results, nil -} - -func (orm gormDB) SearchLabels(query string, omit []uint) ([]kolide.Label, error) { - sql := ` -SELECT * -FROM labels -WHERE MATCH(name) -AGAINST(? IN BOOLEAN MODE) -` - results := []kolide.Label{} - - var db *gorm.DB - if len(omit) > 0 { - sql += "AND id NOT IN (?) LIMIT 10;" - db = orm.DB.Raw(sql, query+"*", omit) - } else { - sql += "LIMIT 10;" - db = orm.DB.Raw(sql, query+"*") - } - - err := db.Scan(&results).Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - return results, nil -} - -func (orm gormDB) ListHostsInLabel(lid uint) ([]kolide.Host, error) { - results := []kolide.Host{} - err := orm.DB.Raw(` -SELECT h.* -FROM label_query_executions lqe -JOIN hosts h -ON lqe.host_id = h.id -WHERE lqe.label_id = ? -AND lqe.matches = 1; -`, lid).Scan(&results).Error - - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - - return results, nil -} - -func (orm gormDB) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { - if labels == nil || len(labels) == 0 { - return nil, nil - } - - results := []kolide.Host{} - err := orm.DB.Raw(` -SELECT h.* -FROM label_query_executions lqe -JOIN hosts h -ON lqe.host_id = h.id -WHERE lqe.label_id in (?) -AND lqe.matches = 1 -GROUP BY h.id; -`, labels).Scan(&results).Error - - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - - return results, nil -} diff --git a/server/datastore/gorm_packs.go b/server/datastore/gorm_packs.go deleted file mode 100644 index f2a0eafcb2..0000000000 --- a/server/datastore/gorm_packs.go +++ /dev/null @@ -1,164 +0,0 @@ -package datastore - -import ( - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/errors" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) NewPack(pack *kolide.Pack) error { - if pack == nil { - return errors.New( - "error creating pack", - "nil pointer passed to NewPack", - ) - } - return orm.DB.Create(pack).Error -} - -func (orm gormDB) SavePack(pack *kolide.Pack) error { - if pack == nil { - return errors.New( - "error saving pack", - "nil pointer passed to SavePack", - ) - } - return orm.DB.Save(pack).Error -} - -func (orm gormDB) DeletePack(pid uint) error { - err := orm.DB.Where("id = ?", pid).Delete(&kolide.Pack{}).Error - if err != nil { - return err - } - - err = orm.DB.Where("pack_id = ?", pid).Delete(&kolide.PackQuery{}).Error - if err != nil { - return err - } - return orm.DB.Where("pack_id = ?", pid).Delete(&kolide.PackTarget{}).Error -} - -func (orm gormDB) Pack(pid uint) (*kolide.Pack, error) { - pack := &kolide.Pack{ - ID: pid, - } - err := orm.DB.Where(pack).First(pack).Error - if err != nil { - return nil, err - } - return pack, nil -} - -func (orm gormDB) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { - var packs []*kolide.Pack - err := orm.applyListOptions(opt).Find(&packs).Error - return packs, err -} - -func (orm gormDB) AddQueryToPack(qid uint, pid uint) error { - pq := &kolide.PackQuery{ - QueryID: qid, - PackID: pid, - } - return orm.DB.Create(pq).Error -} - -func (orm gormDB) ListQueriesInPack(pack *kolide.Pack) ([]*kolide.Query, error) { - if pack == nil { - return nil, errors.New( - "error getting queries in pack", - "nil pointer passed to GetQueriesInPack", - ) - } - - results := []*kolide.Query{} - err := orm.DB.Raw(` -SELECT - q.* -FROM - queries q -JOIN - pack_queries pq -ON - pq.query_id = q.id -AND - pq.pack_id = ?; -`, pack.ID).Scan(&results).Error - - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - - return results, nil -} - -func (orm gormDB) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) error { - if query == nil || pack == nil { - return errors.New( - "error removing query from pack", - "nil pointer passed to RemoveQueryFromPack", - ) - } - pq := &kolide.PackQuery{ - QueryID: query.ID, - PackID: pack.ID, - } - return orm.DB.Where(pq).Delete(pq).Error -} - -func (orm gormDB) AddLabelToPack(lid uint, pid uint) error { - pt := &kolide.PackTarget{ - PackID: pid, - Target: kolide.Target{ - Type: kolide.TargetLabel, - TargetID: lid, - }, - } - - return orm.DB.Create(pt).Error -} - -func (orm gormDB) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { - if pack == nil { - return nil, errors.New( - "error getting labels for pack", - "nil pointer passed to GetLabelsForPack", - ) - } - - results := []*kolide.Label{} - err := orm.DB.Raw(` -SELECT - l.* -FROM - labels l -JOIN - pack_targets pt -ON - pt.target_id = l.id -WHERE - pt.type = ? - AND - pt.pack_id = ?; - -`, - kolide.TargetLabel, pack.ID).Scan(&results).Error - - if err != nil && err != gorm.ErrRecordNotFound { - return nil, errors.DatabaseError(err) - } - - return results, nil -} - -func (orm gormDB) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) error { - if label == nil || pack == nil { - return errors.New( - "error removing label from pack", - "nil pointer passed to RemoveLabelFromPack", - ) - } - - return orm.DB.Where("pack_id = ? AND type = ? AND target_id = ?", pack.ID, kolide.TargetLabel, label.ID).Delete(&kolide.PackTarget{}).Error -} diff --git a/server/datastore/gorm_password_reset.go b/server/datastore/gorm_password_reset.go deleted file mode 100644 index b76b336749..0000000000 --- a/server/datastore/gorm_password_reset.go +++ /dev/null @@ -1,59 +0,0 @@ -package datastore - -import "github.com/kolide/kolide-ose/server/kolide" - -func (orm gormDB) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) { - err := orm.DB.Create(req).Error - if err != nil { - return nil, err - } - return req, nil -} - -func (orm gormDB) SavePasswordResetRequest(req *kolide.PasswordResetRequest) error { - return orm.DB.Save(req).Error -} - -func (orm gormDB) DeletePasswordResetRequest(req *kolide.PasswordResetRequest) error { - err := orm.DB.Delete(req).Error - return err -} - -func (orm gormDB) DeletePasswordResetRequestsForUser(userID uint) error { - err := orm.DB.Where("user_id = ?", userID).Delete(&kolide.PasswordResetRequest{}).Error - return err -} - -func (orm gormDB) FindPassswordResetByID(id uint) (*kolide.PasswordResetRequest, error) { - reset := &kolide.PasswordResetRequest{ - ID: id, - } - err := orm.DB.Find(reset).First(reset).Error - return reset, err -} - -func (orm gormDB) FindPassswordResetsByUserID(userID uint) ([]*kolide.PasswordResetRequest, error) { - var requests []*kolide.PasswordResetRequest - err := orm.DB.Where("user_id = ?", userID).Find(&requests).Error - if err != nil { - return nil, err - } - return requests, nil -} - -func (orm gormDB) FindPassswordResetByToken(token string) (*kolide.PasswordResetRequest, error) { - reset := &kolide.PasswordResetRequest{ - Token: token, - } - err := orm.DB.Find(reset).First(reset).Error - return reset, err -} - -func (orm gormDB) FindPassswordResetByTokenAndUserID(token string, userID uint) (*kolide.PasswordResetRequest, error) { - reset := &kolide.PasswordResetRequest{ - Token: token, - UserID: userID, - } - err := orm.DB.Find(reset).First(reset).Error - return reset, err -} diff --git a/server/datastore/gorm_queries.go b/server/datastore/gorm_queries.go deleted file mode 100644 index 475312b3f8..0000000000 --- a/server/datastore/gorm_queries.go +++ /dev/null @@ -1,76 +0,0 @@ -package datastore - -import ( - "github.com/kolide/kolide-ose/server/errors" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) NewQuery(query *kolide.Query) (*kolide.Query, error) { - if query == nil { - return nil, errors.New( - "error creating query", - "nil pointer passed to NewQuery", - ) - } - err := orm.DB.Create(query).Error - if err != nil { - return nil, err - } - return query, nil -} - -func (orm gormDB) SaveQuery(query *kolide.Query) error { - if query == nil { - return errors.New( - "error saving query", - "nil pointer passed to SaveQuery", - ) - } - return orm.DB.Save(query).Error -} - -func (orm gormDB) DeleteQuery(query *kolide.Query) error { - if query == nil { - return errors.New( - "error deleting query", - "nil pointer passed to DeleteQuery", - ) - } - return orm.DB.Delete(query).Error -} - -func (orm gormDB) Query(id uint) (*kolide.Query, error) { - query := &kolide.Query{ - ID: id, - } - err := orm.DB.Where(query).First(query).Error - if err != nil { - return nil, err - } - return query, nil -} - -func (orm gormDB) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) { - var queries []*kolide.Query - err := orm.applyListOptions(opt).Find(&queries).Error - return queries, err -} - -func (orm gormDB) NewDistributedQueryExecution(exec kolide.DistributedQueryExecution) (kolide.DistributedQueryExecution, error) { - err := orm.DB.Create(&exec).Error - return exec, err -} - -func (orm gormDB) NewDistributedQueryCampaign(camp kolide.DistributedQueryCampaign) (kolide.DistributedQueryCampaign, error) { - err := orm.DB.Create(&camp).Error - return camp, err -} - -func (orm gormDB) SaveDistributedQueryCampaign(camp kolide.DistributedQueryCampaign) error { - return orm.DB.Save(&camp).Error -} - -func (orm gormDB) NewDistributedQueryCampaignTarget(target kolide.DistributedQueryCampaignTarget) (kolide.DistributedQueryCampaignTarget, error) { - err := orm.DB.Create(&target).Error - return target, err -} diff --git a/server/datastore/gorm_sessions.go b/server/datastore/gorm_sessions.go deleted file mode 100644 index ff8808946b..0000000000 --- a/server/datastore/gorm_sessions.go +++ /dev/null @@ -1,78 +0,0 @@ -package datastore - -import ( - "time" - - "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) SessionByID(id uint) (*kolide.Session, error) { - session := &kolide.Session{ - ID: id, - } - - err := orm.DB.Where(session).First(session).Error - if err != nil { - switch err { - case gorm.ErrRecordNotFound: - return nil, kolide.ErrNoActiveSession - default: - return nil, err - } - } - - return session, nil - -} - -func (orm gormDB) SessionByKey(key string) (*kolide.Session, error) { - session := &kolide.Session{ - Key: key, - } - - err := orm.DB.Where(session).First(session).Error - if err != nil { - switch err { - case gorm.ErrRecordNotFound: - return nil, kolide.ErrNoActiveSession - default: - return nil, err - } - } - - return session, nil -} - -func (orm gormDB) ListSessionsForUser(id uint) ([]*kolide.Session, error) { - var sessions []*kolide.Session - err := orm.DB.Where("user_id = ?", id).Find(&sessions).Error - return sessions, err -} - -func (orm gormDB) NewSession(session *kolide.Session) (*kolide.Session, error) { - err := orm.DB.Create(session).Error - if err != nil { - return nil, err - } - - err = orm.MarkSessionAccessed(session) - if err != nil { - return nil, err - } - - return session, nil -} - -func (orm gormDB) DestroySession(session *kolide.Session) error { - return orm.DB.Delete(session).Error -} - -func (orm gormDB) DestroyAllSessionsForUser(id uint) error { - return orm.DB.Delete(&kolide.Session{}, "user_id = ?", id).Error -} - -func (orm gormDB) MarkSessionAccessed(session *kolide.Session) error { - session.AccessedAt = time.Now().UTC() - return orm.DB.Save(session).Error -} diff --git a/server/datastore/gorm_users.go b/server/datastore/gorm_users.go deleted file mode 100644 index dcab32539f..0000000000 --- a/server/datastore/gorm_users.go +++ /dev/null @@ -1,57 +0,0 @@ -package datastore - -import ( - "github.com/kolide/kolide-ose/server/kolide" -) - -func (orm gormDB) NewUser(user *kolide.User) (*kolide.User, error) { - err := orm.DB.Create(user).Error - if err != nil { - return nil, err - } - return user, nil -} - -func (orm gormDB) User(username string) (*kolide.User, error) { - user := &kolide.User{ - Username: username, - } - err := orm.DB.Where("username = ?", username).First(user).Error - if err != nil { - return nil, err - } - return user, nil -} - -func (orm gormDB) ListUsers(opt kolide.ListOptions) ([]*kolide.User, error) { - var users []*kolide.User - err := orm.applyListOptions(opt).Find(&users).Error - if err != nil { - return nil, err - } - return users, nil -} - -func (orm gormDB) UserByEmail(email string) (*kolide.User, error) { - user := &kolide.User{ - Email: email, - } - err := orm.DB.Where("email = ?", email).First(user).Error - if err != nil { - return nil, err - } - return user, nil -} - -func (orm gormDB) UserByID(id uint) (*kolide.User, error) { - user := &kolide.User{ID: id} - err := orm.DB.Where(user).First(user).Error - if err != nil { - return nil, err - } - return user, nil -} - -func (orm gormDB) SaveUser(user *kolide.User) error { - return orm.DB.Save(user).Error -} diff --git a/server/datastore/inmem/app.go b/server/datastore/inmem/app.go new file mode 100644 index 0000000000..79402e0a03 --- /dev/null +++ b/server/datastore/inmem/app.go @@ -0,0 +1,34 @@ +package inmem + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +func (orm *Datastore) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) { + orm.mtx.Lock() + defer orm.mtx.Unlock() + + info.ID = 1 + orm.orginfo = info + return info, nil +} + +func (orm *Datastore) AppConfig() (*kolide.AppConfig, error) { + orm.mtx.Lock() + defer orm.mtx.Unlock() + + if orm.orginfo != nil { + return orm.orginfo, nil + } + + return nil, errors.ErrNotFound +} + +func (orm *Datastore) SaveAppConfig(info *kolide.AppConfig) error { + orm.mtx.Lock() + defer orm.mtx.Unlock() + + orm.orginfo = info + return nil +} diff --git a/server/datastore/inmem_hosts.go b/server/datastore/inmem/hosts.go similarity index 79% rename from server/datastore/inmem_hosts.go rename to server/datastore/inmem/hosts.go index 38c59e416e..a060b5c9ea 100644 --- a/server/datastore/inmem_hosts.go +++ b/server/datastore/inmem/hosts.go @@ -1,4 +1,4 @@ -package datastore +package inmem import ( "errors" @@ -6,16 +6,17 @@ import ( "strings" "time" + kolide_errors "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) NewHost(host *kolide.Host) (*kolide.Host, error) { +func (orm *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() for _, h := range orm.hosts { if host.NodeKey == h.NodeKey || host.UUID == h.UUID { - return nil, ErrExists + return nil, kolide_errors.ErrExists } } @@ -25,43 +26,43 @@ func (orm *inmem) NewHost(host *kolide.Host) (*kolide.Host, error) { return host, nil } -func (orm *inmem) SaveHost(host *kolide.Host) error { +func (orm *Datastore) SaveHost(host *kolide.Host) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.hosts[host.ID]; !ok { - return ErrNotFound + return kolide_errors.ErrNotFound } orm.hosts[host.ID] = host return nil } -func (orm *inmem) DeleteHost(host *kolide.Host) error { +func (orm *Datastore) DeleteHost(host *kolide.Host) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.hosts[host.ID]; !ok { - return ErrNotFound + return kolide_errors.ErrNotFound } delete(orm.hosts, host.ID) return nil } -func (orm *inmem) Host(id uint) (*kolide.Host, error) { +func (orm *Datastore) Host(id uint) (*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() host, ok := orm.hosts[id] if !ok { - return nil, ErrNotFound + return nil, kolide_errors.ErrNotFound } return host, nil } -func (orm *inmem) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { +func (orm *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -106,7 +107,7 @@ func (orm *inmem) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { return hosts, nil } -func (orm *inmem) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) { +func (orm *Datastore) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -129,7 +130,7 @@ func (orm *inmem) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize in } var err error - host.NodeKey, err = generateRandomText(nodeKeySize) + host.NodeKey, err = kolide.RandomText(nodeKeySize) if err != nil { return nil, err } @@ -152,7 +153,7 @@ func (orm *inmem) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize in return &host, nil } -func (orm *inmem) AuthenticateHost(nodeKey string) (*kolide.Host, error) { +func (orm *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -162,10 +163,10 @@ func (orm *inmem) AuthenticateHost(nodeKey string) (*kolide.Host, error) { } } - return nil, ErrNotFound + return nil, kolide_errors.ErrNotFound } -func (orm *inmem) MarkHostSeen(host *kolide.Host, t time.Time) error { +func (orm *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -180,7 +181,7 @@ func (orm *inmem) MarkHostSeen(host *kolide.Host, t time.Time) error { return nil } -func (orm *inmem) SearchHosts(query string, omit []uint) ([]kolide.Host, error) { +func (orm *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) { omitLookup := map[uint]bool{} for _, o := range omit { omitLookup[o] = true @@ -204,7 +205,7 @@ func (orm *inmem) SearchHosts(query string, omit []uint) ([]kolide.Host, error) return results, nil } -func (orm *inmem) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) { +func (orm *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) { // lookup of executions for this host hostExecutions := map[uint]kolide.DistributedQueryExecutionStatus{} for _, e := range orm.distributedQueryExecutions { diff --git a/server/datastore/inmem.go b/server/datastore/inmem/inmem.go similarity index 87% rename from server/datastore/inmem.go rename to server/datastore/inmem/inmem.go index 896c3f0363..71d5874f53 100644 --- a/server/datastore/inmem.go +++ b/server/datastore/inmem/inmem.go @@ -1,4 +1,4 @@ -package datastore +package inmem import ( "errors" @@ -9,7 +9,7 @@ import ( "github.com/patrickmn/sortutil" ) -type inmem struct { +type Datastore struct { Driver string mtx sync.RWMutex nextIDs map[interface{}]uint @@ -32,11 +32,38 @@ type inmem struct { orginfo *kolide.AppConfig } -func (orm *inmem) Name() string { +func New() (*Datastore, error) { + ds := &Datastore{ + Driver: "inmem", + } + + if err := ds.Migrate(); err != nil { + return nil, err + } + + return ds, nil +} + +func (orm *Datastore) Name() string { return "inmem" } -func (orm *inmem) Migrate() error { +func sortResults(slice interface{}, opt kolide.ListOptions, fields map[string]string) error { + field, ok := fields[opt.OrderKey] + if !ok { + return errors.New("cannot sort on unknown key: " + opt.OrderKey) + } + + if opt.OrderDirection == kolide.OrderDescending { + sortutil.DescByField(slice, field) + } else { + sortutil.AscByField(slice, field) + } + + return nil +} + +func (orm *Datastore) Migrate() error { orm.mtx.Lock() defer orm.mtx.Unlock() orm.nextIDs = make(map[interface{}]uint) @@ -57,14 +84,14 @@ func (orm *inmem) Migrate() error { return nil } -func (orm *inmem) Drop() error { +func (orm *Datastore) Drop() error { return orm.Migrate() } // getLimitOffsetSliceBounds returns the bounds that should be used for // re-slicing the results to comply with the requested ListOptions. Lack of // generics forces us to do this rather than reslicing in this method. -func (orm *inmem) getLimitOffsetSliceBounds(opt kolide.ListOptions, length int) (low uint, high uint) { +func (orm *Datastore) getLimitOffsetSliceBounds(opt kolide.ListOptions, length int) (low uint, high uint) { if opt.PerPage == 0 { // PerPage value of 0 indicates unlimited return 0, uint(length) @@ -81,24 +108,9 @@ func (orm *inmem) getLimitOffsetSliceBounds(opt kolide.ListOptions, length int) return offset, max } -func sortResults(slice interface{}, opt kolide.ListOptions, fields map[string]string) error { - field, ok := fields[opt.OrderKey] - if !ok { - return errors.New("cannot sort on unknown key: " + opt.OrderKey) - } - - if opt.OrderDirection == kolide.OrderDescending { - sortutil.DescByField(slice, field) - } else { - sortutil.AscByField(slice, field) - } - - return nil -} - // nextID returns the next ID value that should be used for a struct of the // given type -func (orm *inmem) nextID(val interface{}) uint { +func (orm *Datastore) nextID(val interface{}) uint { valType := reflect.TypeOf(reflect.Indirect(reflect.ValueOf(val)).Interface()) orm.nextIDs[valType]++ return orm.nextIDs[valType] diff --git a/server/datastore/inmem/inmem_test.go b/server/datastore/inmem/inmem_test.go new file mode 100644 index 0000000000..b3030f7e02 --- /dev/null +++ b/server/datastore/inmem/inmem_test.go @@ -0,0 +1,68 @@ +package inmem + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" +) + +func TestApplyLimitOffset(t *testing.T) { + im := Datastore{} + data := []int{} + + // should work with empty + low, high := im.getLimitOffsetSliceBounds(kolide.ListOptions{}, len(data)) + result := data[low:high] + assert.Len(t, result, 0) + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: 1, PerPage: 20}, len(data)) + result = data[low:high] + assert.Len(t, result, 0) + + // insert some data + for i := 0; i < 100; i++ { + data = append(data, i) + } + + // unlimited + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{}, len(data)) + result = data[low:high] + assert.Len(t, result, 100) + assert.Equal(t, data, result) + + // reasonable limit page 0 + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{PerPage: 20}, len(data)) + result = data[low:high] + assert.Len(t, result, 20) + assert.Equal(t, data[:20], result) + + // too many per page + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{PerPage: 200}, len(data)) + result = data[low:high] + assert.Len(t, result, 100) + assert.Equal(t, data, result) + + // offset should be past end (zero results) + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: 1, PerPage: 200}, len(data)) + result = data[low:high] + assert.Len(t, result, 0) + + // all pages appended should equal the original data + result = []int{} + for i := 0; i < 5; i++ { // 5 used intentionally + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: uint(i), PerPage: 25}, len(data)) + result = append(result, data[low:high]...) + } + assert.Len(t, result, 100) + assert.Equal(t, data, result) + + // again with different params + result = []int{} + for i := 0; i < 100; i++ { // 5 used intentionally + low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: uint(i), PerPage: 1}, len(data)) + result = append(result, data[low:high]...) + } + assert.Len(t, result, 100) + assert.Equal(t, data, result) + +} diff --git a/server/datastore/inmem_invites.go b/server/datastore/inmem/invites.go similarity index 75% rename from server/datastore/inmem_invites.go rename to server/datastore/inmem/invites.go index e3ee626613..b3f13c554e 100644 --- a/server/datastore/inmem_invites.go +++ b/server/datastore/inmem/invites.go @@ -1,19 +1,20 @@ -package datastore +package inmem import ( "sort" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) // NewInvite creates and stores a new invitation in a DB. -func (orm *inmem) NewInvite(invite *kolide.Invite) (*kolide.Invite, error) { +func (orm *Datastore) NewInvite(invite *kolide.Invite) (*kolide.Invite, error) { orm.mtx.Lock() defer orm.mtx.Unlock() for _, in := range orm.invites { if in.Email == invite.Email { - return nil, ErrExists + return nil, errors.ErrExists } } @@ -23,7 +24,7 @@ func (orm *inmem) NewInvite(invite *kolide.Invite) (*kolide.Invite, error) { } // Invites lists all invites in the datastore. -func (orm *inmem) ListInvites(opt kolide.ListOptions) ([]*kolide.Invite, error) { +func (orm *Datastore) ListInvites(opt kolide.ListOptions) ([]*kolide.Invite, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -63,17 +64,17 @@ func (orm *inmem) ListInvites(opt kolide.ListOptions) ([]*kolide.Invite, error) return invites, nil } -func (orm *inmem) Invite(id uint) (*kolide.Invite, error) { +func (orm *Datastore) Invite(id uint) (*kolide.Invite, error) { orm.mtx.Lock() defer orm.mtx.Unlock() if invite, ok := orm.invites[id]; ok { return invite, nil } - return nil, ErrNotFound + return nil, errors.ErrNotFound } // InviteByEmail retrieves an invite for a specific email address. -func (orm *inmem) InviteByEmail(email string) (*kolide.Invite, error) { +func (orm *Datastore) InviteByEmail(email string) (*kolide.Invite, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -82,16 +83,16 @@ func (orm *inmem) InviteByEmail(email string) (*kolide.Invite, error) { return invite, nil } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } // SaveInvite saves an invitation in the datastore. -func (orm *inmem) SaveInvite(invite *kolide.Invite) error { +func (orm *Datastore) SaveInvite(invite *kolide.Invite) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.invites[invite.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.invites[invite.ID] = invite @@ -99,12 +100,12 @@ func (orm *inmem) SaveInvite(invite *kolide.Invite) error { } // DeleteInvite deletes an invitation. -func (orm *inmem) DeleteInvite(invite *kolide.Invite) error { +func (orm *Datastore) DeleteInvite(invite *kolide.Invite) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.invites[invite.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } delete(orm.invites, invite.ID) return nil diff --git a/server/datastore/inmem_labels.go b/server/datastore/inmem/labels.go similarity index 80% rename from server/datastore/inmem_labels.go rename to server/datastore/inmem/labels.go index 37b285d64a..980c3a86f1 100644 --- a/server/datastore/inmem_labels.go +++ b/server/datastore/inmem/labels.go @@ -1,4 +1,4 @@ -package datastore +package inmem import ( "errors" @@ -7,16 +7,17 @@ import ( "strings" "time" + kolide_errors "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) NewLabel(label *kolide.Label) (*kolide.Label, error) { +func (orm *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) { newLabel := *label orm.mtx.Lock() for _, l := range orm.labels { if l.Name == label.Name { - return nil, ErrExists + return nil, kolide_errors.ErrExists } } @@ -27,7 +28,7 @@ func (orm *inmem) NewLabel(label *kolide.Label) (*kolide.Label, error) { return &newLabel, nil } -func (orm *inmem) ListLabelsForHost(hid uint) ([]kolide.Label, error) { +func (orm *Datastore) ListLabelsForHost(hid uint) ([]kolide.Label, error) { // First get IDs of label executions for the host resLabels := []kolide.Label{} @@ -44,7 +45,7 @@ func (orm *inmem) ListLabelsForHost(hid uint) ([]kolide.Label, error) { return resLabels, nil } -func (orm *inmem) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (map[string]string, error) { +func (orm *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (map[string]string, error) { // Get post-cutoff executions for host execedIDs := map[uint]bool{} @@ -66,7 +67,7 @@ func (orm *inmem) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (map[ return queries, nil } -func (orm *inmem) getLabelByIDString(id string) (*kolide.Label, error) { +func (orm *Datastore) getLabelByIDString(id string) (*kolide.Label, error) { labelID, err := strconv.Atoi(id) if err != nil { return nil, errors.New("non-int label ID") @@ -83,7 +84,7 @@ func (orm *inmem) getLabelByIDString(id string) (*kolide.Label, error) { return label, nil } -func (orm *inmem) RecordLabelQueryExecutions(host *kolide.Host, results map[string]bool, t time.Time) error { +func (orm *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[string]bool, t time.Time) error { // Record executions for strLabelID, matches := range results { label, err := orm.getLabelByIDString(strLabelID) @@ -120,7 +121,7 @@ func (orm *inmem) RecordLabelQueryExecutions(host *kolide.Host, results map[stri return nil } -func (orm *inmem) DeleteLabel(lid uint) error { +func (orm *Datastore) DeleteLabel(lid uint) error { orm.mtx.Lock() delete(orm.labels, lid) orm.mtx.Unlock() @@ -128,7 +129,7 @@ func (orm *inmem) DeleteLabel(lid uint) error { return nil } -func (orm *inmem) Label(lid uint) (*kolide.Label, error) { +func (orm *Datastore) Label(lid uint) (*kolide.Label, error) { orm.mtx.Lock() label, ok := orm.labels[lid] orm.mtx.Unlock() @@ -139,7 +140,7 @@ func (orm *inmem) Label(lid uint) (*kolide.Label, error) { return label, nil } -func (orm *inmem) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) { +func (orm *Datastore) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) { // We need to sort by keys to provide reliable ordering keys := []int{} @@ -175,7 +176,7 @@ func (orm *inmem) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) { return labels, nil } -func (orm *inmem) SearchLabels(query string, omit []uint) ([]kolide.Label, error) { +func (orm *Datastore) SearchLabels(query string, omit ...uint) ([]kolide.Label, error) { omitLookup := map[uint]bool{} for _, o := range omit { omitLookup[o] = true @@ -200,7 +201,7 @@ func (orm *inmem) SearchLabels(query string, omit []uint) ([]kolide.Label, error return results, nil } -func (orm *inmem) ListHostsInLabel(lid uint) ([]kolide.Host, error) { +func (orm *Datastore) ListHostsInLabel(lid uint) ([]kolide.Host, error) { var hosts []kolide.Host orm.mtx.Lock() @@ -215,7 +216,7 @@ func (orm *inmem) ListHostsInLabel(lid uint) ([]kolide.Host, error) { return hosts, nil } -func (orm *inmem) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { +func (orm *Datastore) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { var hosts []kolide.Host labelSet := map[uint]bool{} diff --git a/server/datastore/inmem_packs.go b/server/datastore/inmem/packs.go similarity index 74% rename from server/datastore/inmem_packs.go rename to server/datastore/inmem/packs.go index 40754f8278..c10c336d60 100644 --- a/server/datastore/inmem_packs.go +++ b/server/datastore/inmem/packs.go @@ -1,17 +1,18 @@ -package datastore +package inmem import ( "sort" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) NewPack(pack *kolide.Pack) error { +func (orm *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { newPack := *pack for _, q := range orm.packs { if pack.Name == q.Name { - return ErrExists + return nil, errors.ErrExists } } @@ -20,15 +21,14 @@ func (orm *inmem) NewPack(pack *kolide.Pack) error { orm.packs[newPack.ID] = &newPack orm.mtx.Unlock() - // TODO NewPack should return (*kolide.Pack, error) and this is a work around pack.ID = newPack.ID - return nil + return pack, nil } -func (orm *inmem) SavePack(pack *kolide.Pack) error { +func (orm *Datastore) SavePack(pack *kolide.Pack) error { if _, ok := orm.packs[pack.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.mtx.Lock() @@ -38,9 +38,9 @@ func (orm *inmem) SavePack(pack *kolide.Pack) error { return nil } -func (orm *inmem) DeletePack(pid uint) error { +func (orm *Datastore) DeletePack(pid uint) error { if _, ok := orm.packs[pid]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.mtx.Lock() @@ -50,18 +50,18 @@ func (orm *inmem) DeletePack(pid uint) error { return nil } -func (orm *inmem) Pack(id uint) (*kolide.Pack, error) { +func (orm *Datastore) Pack(id uint) (*kolide.Pack, error) { orm.mtx.Lock() pack, ok := orm.packs[id] orm.mtx.Unlock() if !ok { - return nil, ErrNotFound + return nil, errors.ErrNotFound } return pack, nil } -func (orm *inmem) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { +func (orm *Datastore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { // We need to sort by keys to provide reliable ordering keys := []int{} orm.mtx.Lock() @@ -97,7 +97,7 @@ func (orm *inmem) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { return packs, nil } -func (orm *inmem) AddQueryToPack(qid uint, pid uint) error { +func (orm *Datastore) AddQueryToPack(qid uint, pid uint) error { packQuery := &kolide.PackQuery{ PackID: pid, QueryID: qid, @@ -111,7 +111,7 @@ func (orm *inmem) AddQueryToPack(qid uint, pid uint) error { return nil } -func (orm *inmem) ListQueriesInPack(pack *kolide.Pack) ([]*kolide.Query, error) { +func (orm *Datastore) ListQueriesInPack(pack *kolide.Pack) ([]*kolide.Query, error) { var queries []*kolide.Query orm.mtx.Lock() @@ -123,7 +123,7 @@ func (orm *inmem) ListQueriesInPack(pack *kolide.Pack) ([]*kolide.Query, error) return queries, nil } -func (orm *inmem) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) error { +func (orm *Datastore) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) error { var packQueriesToDelete []uint orm.mtx.Lock() @@ -141,7 +141,7 @@ func (orm *inmem) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) er return nil } -func (orm *inmem) AddLabelToPack(lid uint, pid uint) error { +func (orm *Datastore) AddLabelToPack(lid uint, pid uint) error { pt := &kolide.PackTarget{ PackID: pid, Target: kolide.Target{ @@ -158,7 +158,7 @@ func (orm *inmem) AddLabelToPack(lid uint, pid uint) error { return nil } -func (orm *inmem) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { +func (orm *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { var labels []*kolide.Label orm.mtx.Lock() @@ -172,7 +172,7 @@ func (orm *inmem) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) return labels, nil } -func (orm *inmem) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) error { +func (orm *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) error { var labelsToDelete []uint orm.mtx.Lock() diff --git a/server/datastore/inmem_password_reset.go b/server/datastore/inmem/password_reset.go similarity index 52% rename from server/datastore/inmem_password_reset.go rename to server/datastore/inmem/password_reset.go index f8852724a4..53682408a8 100644 --- a/server/datastore/inmem_password_reset.go +++ b/server/datastore/inmem/password_reset.go @@ -1,8 +1,11 @@ -package datastore +package inmem -import "github.com/kolide/kolide-ose/server/kolide" +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) -func (orm *inmem) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) { +func (orm *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -11,31 +14,31 @@ func (orm *inmem) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*ko return req, nil } -func (orm *inmem) SavePasswordResetRequest(req *kolide.PasswordResetRequest) error { +func (orm *Datastore) SavePasswordResetRequest(req *kolide.PasswordResetRequest) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.passwordResets[req.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.passwordResets[req.ID] = req return nil } -func (orm *inmem) DeletePasswordResetRequest(req *kolide.PasswordResetRequest) error { +func (orm *Datastore) DeletePasswordResetRequest(req *kolide.PasswordResetRequest) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.passwordResets[req.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } delete(orm.passwordResets, req.ID) return nil } -func (orm *inmem) DeletePasswordResetRequestsForUser(userID uint) error { +func (orm *Datastore) DeletePasswordResetRequestsForUser(userID uint) error { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -47,7 +50,7 @@ func (orm *inmem) DeletePasswordResetRequestsForUser(userID uint) error { return nil } -func (orm *inmem) FindPassswordResetByID(id uint) (*kolide.PasswordResetRequest, error) { +func (orm *Datastore) FindPassswordResetByID(id uint) (*kolide.PasswordResetRequest, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -55,10 +58,10 @@ func (orm *inmem) FindPassswordResetByID(id uint) (*kolide.PasswordResetRequest, return req, nil } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) FindPassswordResetsByUserID(userID uint) ([]*kolide.PasswordResetRequest, error) { +func (orm *Datastore) FindPassswordResetsByUserID(userID uint) ([]*kolide.PasswordResetRequest, error) { orm.mtx.Lock() defer orm.mtx.Unlock() resets := make([]*kolide.PasswordResetRequest, 0) @@ -70,13 +73,13 @@ func (orm *inmem) FindPassswordResetsByUserID(userID uint) ([]*kolide.PasswordRe } if len(resets) == 0 { - return nil, ErrNotFound + return nil, errors.ErrNotFound } return resets, nil } -func (orm *inmem) FindPassswordResetByToken(token string) (*kolide.PasswordResetRequest, error) { +func (orm *Datastore) FindPassswordResetByToken(token string) (*kolide.PasswordResetRequest, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -86,10 +89,10 @@ func (orm *inmem) FindPassswordResetByToken(token string) (*kolide.PasswordReset } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) FindPassswordResetByTokenAndUserID(token string, userID uint) (*kolide.PasswordResetRequest, error) { +func (orm *Datastore) FindPassswordResetByTokenAndUserID(token string, userID uint) (*kolide.PasswordResetRequest, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -99,5 +102,5 @@ func (orm *inmem) FindPassswordResetByTokenAndUserID(token string, userID uint) } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } diff --git a/server/datastore/inmem_queries.go b/server/datastore/inmem/queries.go similarity index 62% rename from server/datastore/inmem_queries.go rename to server/datastore/inmem/queries.go index 56dc3ff340..6369fe19e8 100644 --- a/server/datastore/inmem_queries.go +++ b/server/datastore/inmem/queries.go @@ -1,12 +1,13 @@ -package datastore +package inmem import ( "sort" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) NewQuery(query *kolide.Query) (*kolide.Query, error) { +func (orm *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -14,7 +15,7 @@ func (orm *inmem) NewQuery(query *kolide.Query) (*kolide.Query, error) { for _, q := range orm.queries { if query.Name == q.Name { - return nil, ErrExists + return nil, errors.ErrExists } } @@ -24,43 +25,43 @@ func (orm *inmem) NewQuery(query *kolide.Query) (*kolide.Query, error) { return &newQuery, nil } -func (orm *inmem) SaveQuery(query *kolide.Query) error { +func (orm *Datastore) SaveQuery(query *kolide.Query) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.queries[query.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.queries[query.ID] = query return nil } -func (orm *inmem) DeleteQuery(query *kolide.Query) error { +func (orm *Datastore) DeleteQuery(query *kolide.Query) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.queries[query.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } delete(orm.queries, query.ID) return nil } -func (orm *inmem) Query(id uint) (*kolide.Query, error) { +func (orm *Datastore) Query(id uint) (*kolide.Query, error) { orm.mtx.Lock() defer orm.mtx.Unlock() query, ok := orm.queries[id] if !ok { - return nil, ErrNotFound + return nil, errors.ErrNotFound } return query, nil } -func (orm *inmem) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) { +func (orm *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -102,50 +103,50 @@ func (orm *inmem) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) { return queries, nil } -func (orm *inmem) NewDistributedQueryExecution(exec kolide.DistributedQueryExecution) (kolide.DistributedQueryExecution, error) { +func (orm *Datastore) NewDistributedQueryExecution(exec *kolide.DistributedQueryExecution) (*kolide.DistributedQueryExecution, error) { orm.mtx.Lock() defer orm.mtx.Unlock() for _, e := range orm.distributedQueryExecutions { if exec.HostID == e.ID && exec.DistributedQueryCampaignID == e.DistributedQueryCampaignID { - return exec, ErrExists + return exec, errors.ErrExists } } exec.ID = orm.nextID(exec) - orm.distributedQueryExecutions[exec.ID] = exec + orm.distributedQueryExecutions[exec.ID] = *exec return exec, nil } -func (orm *inmem) NewDistributedQueryCampaign(camp kolide.DistributedQueryCampaign) (kolide.DistributedQueryCampaign, error) { +func (orm *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) (*kolide.DistributedQueryCampaign, error) { orm.mtx.Lock() defer orm.mtx.Unlock() camp.ID = orm.nextID(camp) - orm.distributedQueryCampaigns[camp.ID] = camp + orm.distributedQueryCampaigns[camp.ID] = *camp return camp, nil } -func (orm *inmem) SaveDistributedQueryCampaign(camp kolide.DistributedQueryCampaign) error { +func (orm *Datastore) SaveDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.distributedQueryCampaigns[camp.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } - orm.distributedQueryCampaigns[camp.ID] = camp + orm.distributedQueryCampaigns[camp.ID] = *camp return nil } -func (orm *inmem) NewDistributedQueryCampaignTarget(target kolide.DistributedQueryCampaignTarget) (kolide.DistributedQueryCampaignTarget, error) { +func (orm *Datastore) NewDistributedQueryCampaignTarget(target *kolide.DistributedQueryCampaignTarget) (*kolide.DistributedQueryCampaignTarget, error) { orm.mtx.Lock() defer orm.mtx.Unlock() target.ID = orm.nextID(target) - orm.distributedQueryCampaignTargets[target.ID] = target + orm.distributedQueryCampaignTargets[target.ID] = *target return target, nil } diff --git a/server/datastore/inmem_sessions.go b/server/datastore/inmem/sessions.go similarity index 62% rename from server/datastore/inmem_sessions.go rename to server/datastore/inmem/sessions.go index 796d91e2ca..719308c9fc 100644 --- a/server/datastore/inmem_sessions.go +++ b/server/datastore/inmem/sessions.go @@ -1,12 +1,13 @@ -package datastore +package inmem import ( "time" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) SessionByKey(key string) (*kolide.Session, error) { +func (orm *Datastore) SessionByKey(key string) (*kolide.Session, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -15,20 +16,20 @@ func (orm *inmem) SessionByKey(key string) (*kolide.Session, error) { return session, nil } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) SessionByID(id uint) (*kolide.Session, error) { +func (orm *Datastore) SessionByID(id uint) (*kolide.Session, error) { orm.mtx.Lock() defer orm.mtx.Unlock() if session, ok := orm.sessions[id]; ok { return session, nil } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) ListSessionsForUser(id uint) ([]*kolide.Session, error) { +func (orm *Datastore) ListSessionsForUser(id uint) ([]*kolide.Session, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -39,12 +40,12 @@ func (orm *inmem) ListSessionsForUser(id uint) ([]*kolide.Session, error) { } } if len(sessions) == 0 { - return nil, ErrNotFound + return nil, errors.ErrNotFound } return sessions, nil } -func (orm *inmem) NewSession(session *kolide.Session) (*kolide.Session, error) { +func (orm *Datastore) NewSession(session *kolide.Session) (*kolide.Session, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -58,15 +59,15 @@ func (orm *inmem) NewSession(session *kolide.Session) (*kolide.Session, error) { } -func (orm *inmem) DestroySession(session *kolide.Session) error { +func (orm *Datastore) DestroySession(session *kolide.Session) error { if _, ok := orm.sessions[session.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } delete(orm.sessions, session.ID) return nil } -func (orm *inmem) DestroyAllSessionsForUser(id uint) error { +func (orm *Datastore) DestroyAllSessionsForUser(id uint) error { for _, session := range orm.sessions { if session.UserID == id { delete(orm.sessions, session.ID) @@ -75,10 +76,10 @@ func (orm *inmem) DestroyAllSessionsForUser(id uint) error { return nil } -func (orm *inmem) MarkSessionAccessed(session *kolide.Session) error { +func (orm *Datastore) MarkSessionAccessed(session *kolide.Session) error { session.AccessedAt = time.Now().UTC() if _, ok := orm.sessions[session.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.sessions[session.ID] = session return nil diff --git a/server/datastore/inmem_users.go b/server/datastore/inmem/users.go similarity index 71% rename from server/datastore/inmem_users.go rename to server/datastore/inmem/users.go index 560609c5b4..739ce67983 100644 --- a/server/datastore/inmem_users.go +++ b/server/datastore/inmem/users.go @@ -1,18 +1,19 @@ -package datastore +package inmem import ( "sort" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" ) -func (orm *inmem) NewUser(user *kolide.User) (*kolide.User, error) { +func (orm *Datastore) NewUser(user *kolide.User) (*kolide.User, error) { orm.mtx.Lock() defer orm.mtx.Unlock() for _, in := range orm.users { if in.Username == user.Username { - return nil, ErrExists + return nil, errors.ErrExists } } @@ -22,7 +23,7 @@ func (orm *inmem) NewUser(user *kolide.User) (*kolide.User, error) { return user, nil } -func (orm *inmem) User(username string) (*kolide.User, error) { +func (orm *Datastore) User(username string) (*kolide.User, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -32,10 +33,10 @@ func (orm *inmem) User(username string) (*kolide.User, error) { } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) ListUsers(opt kolide.ListOptions) ([]*kolide.User, error) { +func (orm *Datastore) ListUsers(opt kolide.ListOptions) ([]*kolide.User, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -76,7 +77,7 @@ func (orm *inmem) ListUsers(opt kolide.ListOptions) ([]*kolide.User, error) { return users, nil } -func (orm *inmem) UserByEmail(email string) (*kolide.User, error) { +func (orm *Datastore) UserByEmail(email string) (*kolide.User, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -86,10 +87,10 @@ func (orm *inmem) UserByEmail(email string) (*kolide.User, error) { } } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) UserByID(id uint) (*kolide.User, error) { +func (orm *Datastore) UserByID(id uint) (*kolide.User, error) { orm.mtx.Lock() defer orm.mtx.Unlock() @@ -97,15 +98,15 @@ func (orm *inmem) UserByID(id uint) (*kolide.User, error) { return user, nil } - return nil, ErrNotFound + return nil, errors.ErrNotFound } -func (orm *inmem) SaveUser(user *kolide.User) error { +func (orm *Datastore) SaveUser(user *kolide.User) error { orm.mtx.Lock() defer orm.mtx.Unlock() if _, ok := orm.users[user.ID]; !ok { - return ErrNotFound + return errors.ErrNotFound } orm.users[user.ID] = user diff --git a/server/datastore/inmem_app.go b/server/datastore/inmem_app.go deleted file mode 100644 index 930d9b752b..0000000000 --- a/server/datastore/inmem_app.go +++ /dev/null @@ -1,31 +0,0 @@ -package datastore - -import "github.com/kolide/kolide-ose/server/kolide" - -func (orm *inmem) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) { - orm.mtx.Lock() - defer orm.mtx.Unlock() - - info.ID = 1 - orm.orginfo = info - return info, nil -} - -func (orm *inmem) AppConfig() (*kolide.AppConfig, error) { - orm.mtx.Lock() - defer orm.mtx.Unlock() - - if orm.orginfo != nil { - return orm.orginfo, nil - } - - return nil, ErrNotFound -} - -func (orm *inmem) SaveAppConfig(info *kolide.AppConfig) error { - orm.mtx.Lock() - defer orm.mtx.Unlock() - - orm.orginfo = info - return nil -} diff --git a/server/datastore/inmem_test.go b/server/datastore/inmem_test.go index 3879bbee51..55fd042a09 100644 --- a/server/datastore/inmem_test.go +++ b/server/datastore/inmem_test.go @@ -3,76 +3,18 @@ package datastore import ( "testing" - "github.com/kolide/kolide-ose/server/kolide" - "github.com/stretchr/testify/assert" + "github.com/kolide/kolide-ose/server/datastore/inmem" + "github.com/stretchr/testify/require" ) func TestInmem(t *testing.T) { + for _, f := range testFunctions { t.Run(functionName(f), func(t *testing.T) { - ds, err := New("inmem", "") - assert.Nil(t, err) + ds, err := inmem.New() + defer func() { require.Nil(t, ds.Drop()) }() + require.Nil(t, err) f(t, ds) }) } } - -func TestApplyLimitOffset(t *testing.T) { - im := inmem{} - data := []int{} - - // should work with empty - low, high := im.getLimitOffsetSliceBounds(kolide.ListOptions{}, len(data)) - result := data[low:high] - assert.Len(t, result, 0) - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: 1, PerPage: 20}, len(data)) - result = data[low:high] - assert.Len(t, result, 0) - - // insert some data - for i := 0; i < 100; i++ { - data = append(data, i) - } - - // unlimited - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{}, len(data)) - result = data[low:high] - assert.Len(t, result, 100) - assert.Equal(t, data, result) - - // reasonable limit page 0 - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{PerPage: 20}, len(data)) - result = data[low:high] - assert.Len(t, result, 20) - assert.Equal(t, data[:20], result) - - // too many per page - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{PerPage: 200}, len(data)) - result = data[low:high] - assert.Len(t, result, 100) - assert.Equal(t, data, result) - - // offset should be past end (zero results) - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: 1, PerPage: 200}, len(data)) - result = data[low:high] - assert.Len(t, result, 0) - - // all pages appended should equal the original data - result = []int{} - for i := 0; i < 5; i++ { // 5 used intentionally - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: uint(i), PerPage: 25}, len(data)) - result = append(result, data[low:high]...) - } - assert.Len(t, result, 100) - assert.Equal(t, data, result) - - // again with different params - result = []int{} - for i := 0; i < 100; i++ { // 5 used intentionally - low, high = im.getLimitOffsetSliceBounds(kolide.ListOptions{Page: uint(i), PerPage: 1}, len(data)) - result = append(result, data[low:high]...) - } - assert.Len(t, result, 100) - assert.Equal(t, data, result) - -} diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go new file mode 100644 index 0000000000..7e9edcce3c --- /dev/null +++ b/server/datastore/mysql/app_configs.go @@ -0,0 +1,50 @@ +package mysql + +import ( + "database/sql" + + "github.com/kolide/kolide-ose/server/kolide" +) + +func (d *Datastore) NewAppConfig(info *kolide.AppConfig) (*kolide.AppConfig, error) { + var ( + err error + result sql.Result + ) + + err = d.db.Get(info, "SELECT * FROM app_configs LIMIT 1") + switch err { + case sql.ErrNoRows: + result, err = d.db.Exec( + "INSERT INTO app_configs (org_name, org_logo_url, kolide_server_url) VALUES (?, ?, ?)", + info.OrgName, info.OrgLogoURL, info.KolideServerURL, + ) + if err != nil { + return nil, err + } + + info.ID, _ = result.LastInsertId() + return info, nil + case nil: + return info, d.SaveAppConfig(info) + default: + return nil, err + } +} + +func (d *Datastore) AppConfig() (*kolide.AppConfig, error) { + info := &kolide.AppConfig{} + err := d.db.Get(info, "SELECT * FROM app_configs LIMIT 1") + if err != nil { + return nil, err + } + return info, nil +} + +func (d *Datastore) SaveAppConfig(info *kolide.AppConfig) error { + _, err := d.db.Exec( + "UPDATE app_configs SET org_name = ?, org_logo_url = ?, kolide_server_url = ? WHERE id = ?", + info.OrgName, info.OrgLogoURL, info.KolideServerURL, info.ID, + ) + return err +} diff --git a/server/datastore/config.go b/server/datastore/mysql/config.go similarity index 52% rename from server/datastore/config.go rename to server/datastore/mysql/config.go index 0fc4b1bf97..b2062a8287 100644 --- a/server/datastore/config.go +++ b/server/datastore/mysql/config.go @@ -1,10 +1,6 @@ -package datastore +package mysql -import ( - "log" - - "github.com/kolide/kolide-ose/server/kolide" -) +import "github.com/go-kit/kit/log" const defaultMaxAttempts int = 15 @@ -14,13 +10,11 @@ type DBOption func(o *dbOptions) error type dbOptions struct { // maxAttempts configures the number of retries to connect to the DB maxAttempts int - db kolide.Datastore - debug bool // gorm debug - logger *log.Logger + logger log.Logger } // Logger adds a logger to the datastore -func Logger(l *log.Logger) DBOption { +func Logger(l log.Logger) DBOption { return func(o *dbOptions) error { o.logger = l return nil @@ -36,20 +30,3 @@ func LimitAttempts(attempts int) DBOption { return nil } } - -// Debug sets the GORM debug level -func Debug() DBOption { - return func(o *dbOptions) error { - o.debug = true - return nil - } -} - -// datastore allows you to pass your own datastore -// this option can be used to pass a specific testing implementation -func datastore(db kolide.Datastore) DBOption { - return func(o *dbOptions) error { - o.db = db - return nil - } -} diff --git a/server/datastore/mysql/datastore.go b/server/datastore/mysql/datastore.go new file mode 100644 index 0000000000..5463b4c9cd --- /dev/null +++ b/server/datastore/mysql/datastore.go @@ -0,0 +1,179 @@ +package mysql + +import ( + "fmt" + "strings" + "time" + + "github.com/WatchBeam/clock" + "github.com/go-kit/kit/log" + "github.com/go-sql-driver/mysql" // db driver + "github.com/jmoiron/sqlx" + "github.com/kolide/kolide-ose/server/config" + "github.com/kolide/kolide-ose/server/kolide" +) + +const ( + defaultSelectLimit = 1000 +) + +// Datastore is an implementation of kolide.Datastore interface backed by +// MySQL +type Datastore struct { + db *sqlx.DB + logger log.Logger + clock clock.Clock +} + +// New creates an MySQL datastore. +func New(dbConnectString string, c clock.Clock, opts ...DBOption) (*Datastore, error) { + + options := &dbOptions{ + maxAttempts: defaultMaxAttempts, + logger: log.NewNopLogger(), + } + + for _, setOpt := range opts { + setOpt(options) + } + + db, err := sqlx.Open("mysql", dbConnectString) + if err != nil { + return nil, err + } + + var dbError error + for attempt := 0; attempt < options.maxAttempts; attempt++ { + dbError = db.Ping() + if dbError == nil { + // we're connected! + break + } + sleep := time.Duration(attempt) + options.logger.Log("mysql", fmt.Sprintf( + "could not connect to db: %v, sleeping %v", dbError, sleep)) + time.Sleep(sleep * time.Second) + } + + if dbError != nil { + return nil, dbError + } + + ds := &Datastore{db, options.logger, c} + + return ds, nil + +} + +func (d *Datastore) Name() string { + return "mysql" +} + +// Migrate creates database +func (d *Datastore) Migrate() error { + + sql, err := Asset("db/up.sql") + if err != nil { + return err + } + + tx := d.db.MustBegin() + + for _, statement := range strings.SplitAfter(string(sql), ";") { + if _, err = tx.Exec(statement); err != nil { + if driverErr, ok := err.(*mysql.MySQLError); ok { + if driverErr.Number != 1065 { // ignore empty queries + tx.Rollback() + return err + } + } + } + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil + +} + +// Drop removes database +func (d *Datastore) Drop() error { + var ( + sql []byte + err error + ) + + if sql, err = Asset("db/down.sql"); err != nil { + return err + } + + tx := d.db.MustBegin() + + for _, statement := range strings.SplitAfter(string(sql), ";") { + if _, err = tx.Exec(statement); err != nil { + if driverErr, ok := err.(*mysql.MySQLError); ok { + if driverErr.Number != 1065 { // ignore empty queries + tx.Rollback() + return err + } + } + } + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil + +} + +// Close frees resources associated with underlying mysql connection +func (d *Datastore) Close() error { + return d.db.Close() +} + +func (d *Datastore) log(msg string) { + d.logger.Log("comp", d.Name(), "msg", msg) +} + +func appendListOptionsToSQL(sql string, opts kolide.ListOptions) string { + if opts.OrderKey != "" { + direction := "ASC" + if opts.OrderDirection == kolide.OrderDescending { + direction = "DESC" + } + + sql = fmt.Sprintf("%s ORDER BY %s %s", sql, opts.OrderKey, direction) + } + // REVIEW: If caller doesn't supply a limit apply a default limit of 1000 + // to insure that an unbounded query with many results doesn't consume too + // much memory or hang + if opts.PerPage == 0 { + opts.PerPage = defaultSelectLimit + } + + sql = fmt.Sprintf("%s LIMIT %d", sql, opts.PerPage) + + offset := opts.PerPage * opts.Page + + if offset > 0 { + sql = fmt.Sprintf("%s OFFSET %d", sql, offset) + } + + return sql +} + +// GetMysqlConnectionString returns a MySQL connection string using the +// provided configuration. +func GetMysqlConnectionString(conf config.MysqlConfig) string { + return fmt.Sprintf( + "%s:%s@(%s)/%s?charset=utf8&parseTime=True&loc=Local", + conf.Username, + conf.Password, + conf.Address, + conf.Database, + ) +} diff --git a/server/datastore/mysql/datastore_test.go b/server/datastore/mysql/datastore_test.go new file mode 100644 index 0000000000..b98f9f58b0 --- /dev/null +++ b/server/datastore/mysql/datastore_test.go @@ -0,0 +1,57 @@ +package mysql + +import ( + "testing" + + "github.com/kolide/kolide-ose/server/kolide" +) + +func TestAppendListOptionsToSQL(t *testing.T) { + sql := "SELECT * FROM app_configs" + opts := kolide.ListOptions{ + OrderKey: "name", + } + + actual := appendListOptionsToSQL(sql, opts) + expected := "SELECT * FROM app_configs ORDER BY name ASC LIMIT 1000" + if actual != expected { + t.Error("Expected", expected, "Actual", actual) + } + + sql = "SELECT * FROM app_configs" + opts.OrderDirection = kolide.OrderDescending + actual = appendListOptionsToSQL(sql, opts) + expected = "SELECT * FROM app_configs ORDER BY name DESC LIMIT 1000" + if actual != expected { + t.Error("Expected", expected, "Actual", actual) + } + + opts = kolide.ListOptions{ + PerPage: 10, + } + + sql = "SELECT * FROM app_configs" + actual = appendListOptionsToSQL(sql, opts) + expected = "SELECT * FROM app_configs LIMIT 10" + if actual != expected { + t.Error("Expected", expected, "Actual", actual) + } + + sql = "SELECT * FROM app_configs" + opts.Page = 2 + actual = appendListOptionsToSQL(sql, opts) + expected = "SELECT * FROM app_configs LIMIT 10 OFFSET 20" + if actual != expected { + t.Error("Expected", expected, "Actual", actual) + } + + opts = kolide.ListOptions{} + sql = "SELECT * FROM app_configs" + actual = appendListOptionsToSQL(sql, opts) + expected = "SELECT * FROM app_configs LIMIT 1000" + + if actual != expected { + t.Error("Expected", expected, "Actual", actual) + } + +} diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go new file mode 100644 index 0000000000..41f09effab --- /dev/null +++ b/server/datastore/mysql/hosts.go @@ -0,0 +1,295 @@ +package mysql + +import ( + "database/sql" + "net/http" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { + sqlStatement := ` + INSERT INTO hosts ( + detail_update_time, + node_key, + host_name, + uuid, + platform, + osquery_version, + os_version, + uptime, + physical_memory, + primary_mac, + primary_ip + ) + VALUES( ?,?,?,?,?,?,?,?,?,?,?) + ` + result, err := d.db.Exec(sqlStatement, host.DetailUpdateTime, + host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion, + host.OSVersion, host.Uptime, host.PhysicalMemory, host.PrimaryMAC, host.PrimaryIP) + if err != nil { + return nil, errors.DatabaseError(err) + } + id, _ := result.LastInsertId() + host.ID = uint(id) + return host, nil +} + +// TODO needs test +func (d *Datastore) SaveHost(host *kolide.Host) error { + sqlStatement := ` + UPDATE hosts SET + detail_update_time = ?, + node_key = ?, + host_name = ?, + uuid = ?, + platform = ?, + osquery_version = ?, + os_version = ?, + uptime = ?, + physical_memory = ?, + primary_mac = ?, + primary_ip = ? + WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, host.DetailUpdateTime, host.NodeKey, + host.HostName, host.UUID, host.Platform, host.OsqueryVersion, + host.OSVersion, host.Uptime, host.PhysicalMemory, host.PrimaryMAC, + host.PrimaryIP, host.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// TODO needs test +func (d *Datastore) DeleteHost(host *kolide.Host) error { + sqlStatement := ` + UPDATE hosts SET + deleted = TRUE, + deleted_at = ? + WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, d.clock.Now(), host.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// TODO needs test +func (d *Datastore) Host(id uint) (*kolide.Host, error) { + sqlStatement := ` + SELECT * FROM hosts + WHERE id = ? AND NOT deleted LIMIT 1 + ` + host := &kolide.Host{} + err := d.db.Get(host, sqlStatement, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return host, nil + +} + +// TODO needs test +func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) { + sqlStatement := ` + SELECT * FROM hosts + WHERE NOT deleted + ` + sqlStatement = appendListOptionsToSQL(sqlStatement, opt) + hosts := []*kolide.Host{} + if err := d.db.Select(&hosts, sqlStatement); err != nil { + return nil, errors.DatabaseError(err) + } + + return hosts, nil +} + +// EnrollHost enrolls a host +func (d *Datastore) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) { + if uuid == "" { + return nil, errors.New("missing uuid for host enrollment", "programmer error") + } + // REVIEW If a deleted host is enrolled, it is undeleted + sqlInsert := ` + INSERT INTO hosts ( + detail_update_time, + node_key, + host_name, + uuid, + platform, + primary_ip + ) VALUES (?, ?, ?, ?, ?, ? ) + ON DUPLICATE KEY UPDATE + updated_at = VALUES(updated_at), + detail_update_time = VALUES(detail_update_time), + node_key = VALUES(node_key), + host_name = VALUES(host_name), + platform = VALUES(platform), + primary_ip = VALUES(primary_ip), + deleted = FALSE + ` + args := []interface{}{} + args = append(args, time.Unix(0, 0).Add(24*time.Hour)) + + nodeKey, err := kolide.RandomText(nodeKeySize) + + args = append(args, nodeKey) + args = append(args, hostname) + args = append(args, uuid) + args = append(args, platform) + args = append(args, ip) + + var result sql.Result + + result, err = d.db.Exec(sqlInsert, args...) + + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + sqlSelect := ` + SELECT * FROM hosts WHERE id = ? LIMIT 1 + ` + host := &kolide.Host{} + err = d.db.Get(host, sqlSelect, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return host, nil + +} + +func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { + sqlStatement := ` + SELECT * FROM hosts + WHERE node_key = ? AND NOT DELETED LIMIT 1 + ` + host := &kolide.Host{} + if err := d.db.Get(host, sqlStatement, nodeKey); err != nil { + switch err { + case sql.ErrNoRows: + e := errors.NewFromError(err, http.StatusUnauthorized, "invalid node key") + e.Extra = map[string]interface{}{"node_invalid": "true"} + return nil, e + default: + return nil, errors.DatabaseError(err) + } + } + + return host, nil + +} + +func (d *Datastore) MarkHostSeen(*kolide.Host, time.Time) error { + panic("not implemented") +} + +func (d *Datastore) searchHostsWithOmits(query string, omits ...uint) ([]kolide.Host, error) { + // The reason that string cocantenation is used to include query as opposed to a + // bindvar is that sqlx.In has a bug such that, if you have any bindvars other + // than those in the IN clause, sqlx.In returns an empty sql statement. + // I've submitted an issue https://github.com/jmoiron/sqlx/issues/260 about this + sqlStatement := ` + SELECT * + FROM hosts + WHERE MATCH(host_name, primary_ip) + AGAINST('` + query + "*" + `' IN BOOLEAN MODE) + AND NOT deleted + AND id NOT IN (?) + LIMIT 10 + ` + + sql, args, err := sqlx.In(sqlStatement, omits) + if err != nil { + return nil, errors.DatabaseError(err) + } + + sql = d.db.Rebind(sql) + + hosts := []kolide.Host{} + + if err = d.db.Select(&hosts, sql, args...); err != nil { + return nil, errors.DatabaseError(err) + } + + return hosts, nil +} + +// SearchHosts find hosts by query containing an IP address or a host name. Optionally +// pass a list of IDs to omit from the search +func (d *Datastore) SearchHosts(query string, omit ...uint) ([]kolide.Host, error) { + if len(omit) > 0 { + return d.searchHostsWithOmits(query, omit...) + } + + sqlStatement := ` + SELECT * FROM hosts + WHERE MATCH(host_name, primary_ip) + AGAINST(? IN BOOLEAN MODE) + AND not deleted + LIMIT 10 + ` + hosts := []kolide.Host{} + + if err := d.db.Select(&hosts, sqlStatement, query); err != nil { + return nil, errors.DatabaseError(err) + } + + return hosts, nil + +} + +func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) { + sqlStatement := ` + SELECT DISTINCT dqc.id, q.query + FROM distributed_query_campaigns dqc + JOIN distributed_query_campaign_targets dqct + ON (dqc.id = dqct.distributed_query_campaign_id) + LEFT JOIN label_query_executions lqe + ON (dqct.type = ? AND dqct.target_id = lqe.label_id AND lqe.matches) + LEFT JOIN hosts h + ON ((dqct.type = ? AND lqe.host_id = h.id) OR (dqct.type = ? AND dqct.target_id = h.id)) + LEFT JOIN distributed_query_executions dqe + ON (h.id = dqe.host_id AND dqc.id = dqe.distributed_query_campaign_id) + JOIN queries q + ON (dqc.query_id = q.id) + WHERE dqe.status IS NULL AND dqc.status = ? AND h.id = ? + AND NOT q.deleted + AND NOT dqc.deleted + ` + rows, err := d.db.Query(sqlStatement, kolide.TargetLabel, kolide.TargetLabel, + kolide.TargetHost, kolide.QueryRunning, host.ID) + if err != nil { + return nil, errors.DatabaseError(err) + } + defer rows.Close() + + results := map[uint]string{} + + for rows.Next() { + var ( + id uint + query string + ) + err = rows.Scan(&id, &query) + if err != nil { + return nil, errors.DatabaseError(err) + } + + results[id] = query + + } + + return results, nil +} diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go new file mode 100644 index 0000000000..1a37be3483 --- /dev/null +++ b/server/datastore/mysql/invites.go @@ -0,0 +1,92 @@ +package mysql + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +// NewInvite generates a new invitation +func (d *Datastore) NewInvite(i *kolide.Invite) (*kolide.Invite, error) { + + sql := ` + INSERT INTO invites ( invited_by, email, admin, name, position, token) + VALUES ( ?, ?, ?, ?, ?, ?) + ` + + result, err := d.db.Exec(sql, i.InvitedBy, i.Email, i.Admin, + i.Name, i.Position, i.Token) + if err != nil { + return nil, err + } + + id, _ := result.LastInsertId() + i.ID = uint(id) + + return i, nil + +} + +// ListInvites lists all invites in the Kolide database. Supply query options +// using the opt parameter. See kolide.ListOptions +func (d *Datastore) ListInvites(opt kolide.ListOptions) ([]*kolide.Invite, error) { + + invites := []*kolide.Invite{} + + sql := appendListOptionsToSQL("SELECT * FROM invites WHERE NOT deleted", opt) + err := d.db.Select(&invites, sql) + if err != nil { + return nil, errors.DatabaseError(err) + } + return invites, nil +} + +// Invite returns Invite identified by id. +func (d *Datastore) Invite(id uint) (*kolide.Invite, error) { + invite := &kolide.Invite{} + err := d.db.Get(invite, "SELECT * FROM invites WHERE id = ? AND NOT deleted", id) + if err != nil { + return nil, errors.DatabaseError(err) + } + return invite, nil +} + +// InviteByEmail finds an Invite with a particular email, if one exists. +func (d *Datastore) InviteByEmail(email string) (*kolide.Invite, error) { + invite := &kolide.Invite{} + err := d.db.Get(invite, "SELECT * FROM invites WHERE email = ? AND NOT deleted", email) + if err != nil { + return nil, errors.DatabaseError(err) + } + return invite, nil +} + +// SaveInvite modifies existing Invite +func (d *Datastore) SaveInvite(i *kolide.Invite) error { + sql := ` + UPDATE invites SET invited_by = ?, email = ?, admin = ?, + name = ?, position = ?, token = ? + WHERE id = ? AND NOT deleted + ` + _, err := d.db.Exec(sql, i.InvitedBy, i.Email, + i.Admin, i.Name, i.Position, i.Token, i.ID, + ) + if err != nil { + return errors.DatabaseError(err) + } + + return nil + +} + +func (d *Datastore) DeleteInvite(i *kolide.Invite) error { + i.MarkDeleted(d.clock.Now()) + sql := ` + UPDATE invites SET deleted_at = ?, deleted = ? + WHERE id = ? + ` + _, err := d.db.Exec(sql, i.DeletedAt, true, i.ID) + if err != nil { + return errors.DatabaseError(err) + } + return nil +} diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go new file mode 100644 index 0000000000..acd33ef4c1 --- /dev/null +++ b/server/datastore/mysql/labels.go @@ -0,0 +1,258 @@ +package mysql + +import ( + "database/sql" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +// NewLabel creates a new kolide.Label +func (d *Datastore) NewLabel(label *kolide.Label) (*kolide.Label, error) { + + sql := ` + INSERT INTO labels ( + name, + description, + query, + platform + ) VALUES ( ?, ?, ?, ?) + ` + result, err := d.db.Exec(sql, label.Name, label.Description, label.Query, label.Platform) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + label.ID = uint(id) + return label, nil + +} + +// DeleteLabel soft deletes a kolide.Label +func (d *Datastore) DeleteLabel(lid uint) error { + sql := ` + UPDATE labels + SET deleted_at = ?, deleted = TRUE + WHERE id = ? + ` + _, err := d.db.Exec(sql, d.clock.Now(), lid) + return errors.DatabaseError(err) +} + +// Label returns a kolide.Label identified by lid if one exists +func (d *Datastore) Label(lid uint) (*kolide.Label, error) { + sql := ` + SELECT * FROM labels + WHERE id = ? AND NOT deleted + ` + label := &kolide.Label{} + + if err := d.db.Get(label, sql, lid); err != nil { + return nil, errors.DatabaseError(err) + } + + return label, nil +} + +// ListLabels returns all labels limited or sorted by kolide.ListOptions +func (d *Datastore) ListLabels(opt kolide.ListOptions) ([]*kolide.Label, error) { + sql := ` + SELECT * FROM labels WHERE NOT deleted + ` + sql = appendListOptionsToSQL(sql, opt) + labels := []*kolide.Label{} + + if err := d.db.Select(&labels, sql); err != nil { + return nil, errors.DatabaseError(err) + } + + return labels, nil +} + +func (d *Datastore) LabelQueriesForHost(host *kolide.Host, cutoff time.Time) (map[string]string, error) { + sqlStatment := ` + SELECT l.id, l.query + FROM labels l + WHERE l.platform = ? + AND NOT l.deleted + AND l.id NOT IN /* subtract the set of executions that are recent enough */ + ( + SELECT l.id + FROM labels l + JOIN label_query_executions lqe + ON lqe.label_id = l.id + WHERE lqe.host_id = ? AND lqe.updated_at > ? + ) + ` + rows, err := d.db.Query(sqlStatment, host.Platform, host.ID, cutoff) + if err != nil && err != sql.ErrNoRows { + return nil, errors.DatabaseError(err) + } + + defer rows.Close() + results := map[string]string{} + + for rows.Next() { + var id, query string + + if err = rows.Scan(&id, &query); err != nil { + return nil, errors.DatabaseError(err) + } + + results[id] = query + } + + return results, nil + +} + +func (d *Datastore) RecordLabelQueryExecutions(host *kolide.Host, results map[string]bool, updated time.Time) error { + sqlStatement := ` + INSERT INTO label_query_executions (updated_at, matches, label_id, host_id) VALUES + + ` + vals := []interface{}{} + bindvars := "" + + for labelID, result := range results { + if bindvars != "" { + bindvars += "," + } + bindvars += "(?,?,?,?)" + vals = append(vals, updated, result, labelID, host.ID) + } + + sqlStatement += bindvars + sqlStatement += ` + ON DUPLICATE KEY UPDATE + updated_at = VALUES(updated_at), + matches = VALUES(matches) + ` + + _, err := d.db.Exec(sqlStatement, vals...) + if err != nil { + return errors.DatabaseError(err) + } + + return nil + +} + +// ListLabelsForHost returns a list of kolide.Label for a given host id. +func (d *Datastore) ListLabelsForHost(hid uint) ([]kolide.Label, error) { + sqlStatement := ` + SELECT labels.* from labels, label_query_executions lqe + WHERE lqe.host_id = ? + AND lqe.label_id = labels.id + AND lqe.matches + AND NOT labels.deleted + ` + + labels := []kolide.Label{} + err := d.db.Select(&labels, sqlStatement, hid) + if err != nil { + return nil, errors.DatabaseError(err) + } + return labels, nil + +} + +// ListHostsInLabel returns a list of kolide.Host that are associated +// with kolide.Label referened by Label ID +func (d *Datastore) ListHostsInLabel(lid uint) ([]kolide.Host, error) { + sqlStatement := ` + SELECT h.* + FROM label_query_executions lqe + JOIN hosts h + ON lqe.host_id = h.id + WHERE lqe.label_id = ? + AND lqe.matches = 1 + AND NOT h.deleted + ` + hosts := []kolide.Host{} + err := d.db.Select(&hosts, sqlStatement, lid) + if err != nil { + return nil, errors.DatabaseError(err) + } + return hosts, nil +} + +func (d *Datastore) ListUniqueHostsInLabels(labels []uint) ([]kolide.Host, error) { + sqlStatement := ` + SELECT h.* + FROM label_query_executions lqe + JOIN hosts h + ON lqe.host_id = h.id + WHERE lqe.label_id IN (?) + AND lqe.matches = 1 + AND NOT h.deleted + GROUP BY h.id; + ` + query, args, err := sqlx.In(sqlStatement, labels) + if err != nil { + return nil, errors.DatabaseError(err) + } + + query = d.db.Rebind(query) + hosts := []kolide.Host{} + err = d.db.Select(&hosts, query, args...) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return hosts, nil + +} + +func (d *Datastore) searchLabelsWithOmits(query string, omit ...uint) ([]kolide.Label, error) { + sqlStatement := ` + SELECT * + FROM labels + WHERE MATCH(name) + AGAINST('` + query + "*" + `' IN BOOLEAN MODE) + AND NOT deleted + AND id NOT IN (?) + LIMIT 10 + ` + sql, args, err := sqlx.In(sqlStatement, omit) + if err != nil { + return nil, errors.DatabaseError(err) + } + + sql = d.db.Rebind(sql) + + matches := []kolide.Label{} + err = d.db.Select(&matches, sql, args...) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return matches, nil +} + +// SearchLabels performs wildcard searches on kolide.Label name +func (d *Datastore) SearchLabels(query string, omit ...uint) ([]kolide.Label, error) { + + if len(omit) > 0 { + return d.searchLabelsWithOmits(query, omit...) + } + + sqlStatement := ` + SELECT * + FROM labels + WHERE MATCH(name) + AGAINST(? IN BOOLEAN MODE) + AND NOT deleted + LIMIT 10 + ` + matches := []kolide.Label{} + err := d.db.Select(&matches, sqlStatement, query+"*") + if err != nil { + return nil, errors.DatabaseError(err) + } + + return matches, nil +} diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go new file mode 100644 index 0000000000..dc99ff6621 --- /dev/null +++ b/server/datastore/mysql/packs.go @@ -0,0 +1,200 @@ +package mysql + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +// NewPack creates a new Pack +func (d *Datastore) NewPack(pack *kolide.Pack) (*kolide.Pack, error) { + + sql := ` + INSERT INTO packs ( name, platform ) + VALUES ( ?, ?) + ` + + result, err := d.db.Exec(sql, pack.Name, pack.Platform) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + pack.ID = uint(id) + return pack, nil +} + +// SavePack stores changes to pack +func (d *Datastore) SavePack(pack *kolide.Pack) error { + + sql := ` + UPDATE packs + SET name = ?, platform = ? + WHERE id = ? AND NOT deleted + ` + + _, err := d.db.Exec(sql, pack.Name, pack.Platform, pack.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// DeletePack soft deletes a kolide.Pack so that it won't show up in results +func (d *Datastore) DeletePack(pid uint) error { + sql := ` + UPDATE packs + SET deleted_at = ?, deleted = TRUE + WHERE id = ? + ` + _, err := d.db.Exec(sql, d.clock.Now(), pid) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// Pack fetch kolide.Pack with matching ID +func (d *Datastore) Pack(pid uint) (*kolide.Pack, error) { + sql := ` + SELECT * FROM packs + WHERE id = ? AND NOT deleted + ` + pack := &kolide.Pack{} + if err := d.db.Get(pack, sql, pid); err != nil { + return nil, errors.DatabaseError(err) + } + + return pack, nil +} + +// ListPacks returns all kolide.Pack records limited and sorted by kolide.ListOptions +func (d *Datastore) ListPacks(opt kolide.ListOptions) ([]*kolide.Pack, error) { + sql := ` + SELECT * FROM packs + WHERE NOT deleted + ` + sql = appendListOptionsToSQL(sql, opt) + packs := []*kolide.Pack{} + if err := d.db.Select(&packs, sql); err != nil { + return nil, errors.DatabaseError(err) + } + return packs, nil +} + +// AddQueryToPack associates a kolide.Query with a kolide.Pack +func (d *Datastore) AddQueryToPack(qid uint, pid uint) error { + sql := ` + INSERT INTO pack_queries ( pack_id, query_id) + VALUES (?, ?) + ` + if _, err := d.db.Exec(sql, pid, qid); err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// ListQueriesInPack gets all kolide.Query records associated with a kolide.Pack +func (d *Datastore) ListQueriesInPack(pack *kolide.Pack) ([]*kolide.Query, error) { + sql := ` + SELECT + q.id, + q.created_at, + q.updated_at, + q.name, + q.query, + q.interval, + q.snapshot, + q.differential, + q.platform, + q.version + FROM + queries q + JOIN + pack_queries pq + ON + pq.query_id = q.id + AND + pq.pack_id = ? + AND NOT q.deleted + ` + queries := []*kolide.Query{} + if err := d.db.Select(&queries, sql, pack.ID); err != nil { + return nil, errors.DatabaseError(err) + } + return queries, nil +} + +// RemoveQueryFromPack disassociated a kolide.Query from a kolide.Pack +func (d *Datastore) RemoveQueryFromPack(query *kolide.Query, pack *kolide.Pack) error { + sql := ` + DELETE FROM pack_queries + WHERE pack_id = ? AND query_id = ? + ` + if _, err := d.db.Exec(sql, pack.ID, query.ID); err != nil { + return errors.DatabaseError(err) + } + + return nil + +} + +// AddLabelToPack associates a kolide.Label with a kolide.Pack +func (d *Datastore) AddLabelToPack(lid uint, pid uint) error { + sql := ` + INSERT INTO pack_targets ( pack_id, type, target_id ) + VALUES ( ?, ?, ? ) + ` + _, err := d.db.Exec(sql, pid, kolide.TargetLabel, lid) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// ListLabelsForPack will return a list of kolide.Label records associated with kolide.Pack +func (d *Datastore) ListLabelsForPack(pack *kolide.Pack) ([]*kolide.Label, error) { + sql := ` + SELECT + l.id, + l.created_at, + l.updated_at, + l.name + FROM + labels l + JOIN + pack_targets pt + ON + pt.target_id = l.id + WHERE + pt.type = ? + AND + pt.pack_id = ? + AND NOT l.deleted + ` + + labels := []*kolide.Label{} + + if err := d.db.Select(&labels, sql, kolide.TargetLabel, pack.ID); err != nil { + return nil, errors.DatabaseError(err) + } + + return labels, nil +} + +// RemoreLabelFromPack will remove the association between a kolide.Label and +// a kolide.Pack +func (d *Datastore) RemoveLabelFromPack(label *kolide.Label, pack *kolide.Pack) error { + sql := ` + DELETE FROM pack_labels + WHERE target_id = ? AND pack_id = ? + ` + if _, err := d.db.Exec(sql, label.ID, pack.ID); err != nil { + return errors.DatabaseError(err) + } + + return nil +} diff --git a/server/datastore/mysql/password_reset.go b/server/datastore/mysql/password_reset.go new file mode 100644 index 0000000000..81fb10f64c --- /dev/null +++ b/server/datastore/mysql/password_reset.go @@ -0,0 +1,124 @@ +package mysql + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +func (d *Datastore) NewPasswordResetRequest(req *kolide.PasswordResetRequest) (*kolide.PasswordResetRequest, error) { + sqlStatement := ` + INSERT INTO password_reset_requests + ( user_id, token) + VALUES (?,?) + ` + response, err := d.db.Exec(sqlStatement, req.UserID, req.Token) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := response.LastInsertId() + req.ID = uint(id) + return req, nil + +} + +func (d *Datastore) SavePasswordResetRequest(req *kolide.PasswordResetRequest) error { + sqlStatement := ` + UPDATE password_reset_requests SET + expires_at = ?, + user_id = ?, + token = ? + WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, req.ExpiresAt, req.UserID, req.Token, req.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) DeletePasswordResetRequest(req *kolide.PasswordResetRequest) error { + + sqlStatement := ` + DELETE FROM password_reset_requests WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, req.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) DeletePasswordResetRequestsForUser(userID uint) error { + sqlStatement := ` + DELETE FROM password_reset_requests WHERE user_id = ? + ` + _, err := d.db.Exec(sqlStatement, userID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) FindPassswordResetByID(id uint) (*kolide.PasswordResetRequest, error) { + sqlStatement := ` + SELECT * FROM password_reset_requests + WHERE id = ? LIMIT 1 + ` + passwordResetRequest := &kolide.PasswordResetRequest{} + err := d.db.Get(&passwordResetRequest, sqlStatement, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return passwordResetRequest, nil +} + +func (d *Datastore) FindPassswordResetsByUserID(id uint) ([]*kolide.PasswordResetRequest, error) { + sqlStatement := ` + SELECT * FROM password_reset_requests + WHERE user_id = ? + ` + + passwordResetRequests := []*kolide.PasswordResetRequest{} + err := d.db.Select(&passwordResetRequests, sqlStatement, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return passwordResetRequests, nil + +} + +func (d *Datastore) FindPassswordResetByToken(token string) (*kolide.PasswordResetRequest, error) { + sqlStatement := ` + SELECT * FROM password_reset_requests + WHERE token = ? LIMIT 1 + ` + passwordResetRequest := &kolide.PasswordResetRequest{} + err := d.db.Get(passwordResetRequest, sqlStatement, token) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return passwordResetRequest, nil + +} + +func (d *Datastore) FindPassswordResetByTokenAndUserID(token string, id uint) (*kolide.PasswordResetRequest, error) { + sqlStatement := ` + SELECT * FROM password_reset_requests + WHERE user_id = ? AND token = ? + LIMIT 1 + ` + passwordResetRequest := &kolide.PasswordResetRequest{} + err := d.db.Get(passwordResetRequest, sqlStatement, id, token) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return passwordResetRequest, nil +} diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go new file mode 100644 index 0000000000..7cd844419b --- /dev/null +++ b/server/datastore/mysql/queries.go @@ -0,0 +1,172 @@ +package mysql + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +// NewQuery creates a Query +func (d *Datastore) NewQuery(query *kolide.Query) (*kolide.Query, error) { + + sql := ` + INSERT INTO queries ( name, description, query, + snapshot, differential, platform, version, ` + "`interval`" + `) + VALUES ( ?, ?, ?, ?, ?, ?, ?, ? ) + ` + + result, err := d.db.Exec(sql, query.Name, query.Description, query.Query, query.Snapshot, + query.Differential, query.Platform, query.Version, query.Interval) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + query.ID = uint(id) + return query, nil +} + +// SaveQuery saves changes to a Query. +func (d *Datastore) SaveQuery(q *kolide.Query) error { + sql := ` + UPDATE queries + SET name = ?, description = ?, query = ?, ` + "`interval`" + ` = ?, snapshot = ?, + differential = ?, platform = ?, version = ? + WHERE id = ? AND NOT deleted + ` + _, err := d.db.Exec(sql, q.Name, q.Description, q.Query, q.Interval, + q.Snapshot, q.Differential, q.Platform, q.Version, q.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// DeleteQuery soft deletes Query identified by Query.ID +func (d *Datastore) DeleteQuery(query *kolide.Query) error { + query.MarkDeleted(d.clock.Now()) + sql := ` + UPDATE queries + SET deleted_at = ?, deleted = ? + WHERE id = ? + ` + _, err := d.db.Exec(sql, query.DeletedAt, true, query.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +// Query returns a single Query identified by id, if such +// exists +func (d *Datastore) Query(id uint) (*kolide.Query, error) { + sql := ` + SELECT * FROM queries WHERE id = ? AND NOT deleted + ` + query := &kolide.Query{} + if err := d.db.Get(query, sql, id); err != nil { + return nil, errors.DatabaseError(err) + } + + return query, nil +} + +// ListQueries returns a list of queries with sort order and results limit +// determined by passed in kolide.ListOptions +func (d *Datastore) ListQueries(opt kolide.ListOptions) ([]*kolide.Query, error) { + sql := ` + SELECT * FROM queries WHERE NOT deleted + ` + sql = appendListOptionsToSQL(sql, opt) + results := []*kolide.Query{} + + if err := d.db.Select(&results, sql); err != nil { + return nil, errors.DatabaseError(err) + } + + return results, nil + +} + +func (d *Datastore) SaveDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) error { + + sqlStatement := ` + UPDATE distributed_query_campaigns SET + query_id = ?, + max_duration = ?, + status = ?, + user_id = ? + WHERE id = ? + AND NOT deleted + ` + _, err := d.db.Exec(sqlStatement, camp.QueryID, camp.MaxDuration, + camp.Status, camp.UserID, camp.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) NewDistributedQueryCampaign(camp *kolide.DistributedQueryCampaign) (*kolide.DistributedQueryCampaign, error) { + + sqlStatement := ` + INSERT INTO distributed_query_campaigns ( + query_id, + max_duration, + status, + user_id + ) + VALUES(?,?,?,?) + ` + result, err := d.db.Exec(sqlStatement, camp.QueryID, camp.MaxDuration, camp.Status, camp.UserID) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + camp.ID = uint(id) + return camp, nil +} + +func (d *Datastore) NewDistributedQueryCampaignTarget(target *kolide.DistributedQueryCampaignTarget) (*kolide.DistributedQueryCampaignTarget, error) { + sqlStatement := ` + INSERT into distributed_query_campaign_targets ( + type, + distributed_query_campaign_id, + target_id + ) + VALUES (?,?,?) + ` + result, err := d.db.Exec(sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + target.ID = uint(id) + return target, nil +} + +func (d *Datastore) NewDistributedQueryExecution(exec *kolide.DistributedQueryExecution) (*kolide.DistributedQueryExecution, error) { + sqlStatement := ` + INSERT INTO distributed_query_executions ( + host_id, + distributed_query_campaign_id, + status, + error, + execution_duration + ) VALUES (?,?,?,?,?) + ` + result, err := d.db.Exec(sqlStatement, exec.HostID, exec.DistributedQueryCampaignID, + exec.Status, exec.Error, exec.ExecutionDuration) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + exec.ID = uint(id) + + return exec, nil +} diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go new file mode 100644 index 0000000000..fadb225013 --- /dev/null +++ b/server/datastore/mysql/sessions.go @@ -0,0 +1,106 @@ +package mysql + +import ( + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +func (d *Datastore) SessionByKey(key string) (*kolide.Session, error) { + sqlStatement := ` + SELECT * FROM sessions + WHERE key = ? LIMIT 1 + ` + session := &kolide.Session{} + err := d.db.Get(session, sqlStatement, key) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return session, nil +} + +func (d *Datastore) SessionByID(id uint) (*kolide.Session, error) { + sqlStatement := ` + SELECT * FROM sessions + WHERE id = ? + LIMIT 1 + ` + session := &kolide.Session{} + err := d.db.Get(session, sqlStatement, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return session, nil +} + +func (d *Datastore) ListSessionsForUser(id uint) ([]*kolide.Session, error) { + sqlStatement := ` + SELECT * FROM sessions + WHERE user_id = ? + ` + sessions := []*kolide.Session{} + err := d.db.Select(&sessions, sqlStatement, id) + if err != nil { + return nil, errors.DatabaseError(err) + } + + return sessions, nil + +} + +func (d *Datastore) NewSession(session *kolide.Session) (*kolide.Session, error) { + sqlStatement := ` + INSERT INTO sessions ( + user_id, + key + ) + VALUES(?,?,?,?) + ` + result, err := d.db.Exec(sqlStatement, session.UserID, session.Key) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + session.ID = uint(id) + return session, nil +} + +func (d *Datastore) DestroySession(session *kolide.Session) error { + sqlStatement := ` + DELETE FROM sessions WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, session.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) DestroyAllSessionsForUser(id uint) error { + sqlStatement := ` + DELETE FROM sessions WHERE user_id = ? + ` + _, err := d.db.Exec(sqlStatement, id) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} + +func (d *Datastore) MarkSessionAccessed(session *kolide.Session) error { + sqlStatement := ` + UPDATE sessions SET + accessed_at = ? + WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, d.clock.Now(), session.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go new file mode 100644 index 0000000000..d810a955ae --- /dev/null +++ b/server/datastore/mysql/users.go @@ -0,0 +1,107 @@ +package mysql + +import ( + "fmt" + + "github.com/kolide/kolide-ose/server/errors" + "github.com/kolide/kolide-ose/server/kolide" +) + +// NewUser creates a new user +func (d *Datastore) NewUser(user *kolide.User) (*kolide.User, error) { + sqlStatement := ` + INSERT INTO users ( + password, + salt, + name, + username, + email, + admin, + enabled, + admin_forced_password_reset, + gravatar_url, + position + ) VALUES (?,?,?,?,?,?,?,?,?,?) + ` + result, err := d.db.Exec(sqlStatement, user.Password, user.Salt, user.Name, + user.Username, user.Email, user.Admin, user.Enabled, + user.AdminForcedPasswordReset, user.GravatarURL, user.Position) + if err != nil { + return nil, errors.DatabaseError(err) + } + + id, _ := result.LastInsertId() + user.ID = uint(id) + return user, nil +} + +func (d *Datastore) findUser(searchCol string, searchVal interface{}) (*kolide.User, error) { + sqlStatement := fmt.Sprintf( + "SELECT * FROM users "+ + "WHERE %s = ? AND NOT deleted LIMIT 1", + searchCol, + ) + + user := &kolide.User{} + + if err := d.db.Get(user, sqlStatement, searchVal); err != nil { + return nil, errors.DatabaseError(err) + } + + return user, nil +} + +// User retrieves a user by name +func (d *Datastore) User(username string) (*kolide.User, error) { + return d.findUser("username", username) +} + +// ListUsers lists all users with limit, sort and offset passed in with +// kolide.ListOptions +func (d *Datastore) ListUsers(opt kolide.ListOptions) ([]*kolide.User, error) { + sqlStatement := ` + SELECT * FROM USERS WHERE NOT deleted + ` + sqlStatement = appendListOptionsToSQL(sqlStatement, opt) + users := []*kolide.User{} + + if err := d.db.Select(&users, sqlStatement); err != nil { + return nil, errors.DatabaseError(err) + } + + return users, nil + +} + +func (d *Datastore) UserByEmail(email string) (*kolide.User, error) { + return d.findUser("email", email) +} + +func (d *Datastore) UserByID(id uint) (*kolide.User, error) { + return d.findUser("id", id) +} + +func (d *Datastore) SaveUser(user *kolide.User) error { + sqlStatement := ` + UPDATE users SET + username = ?, + password = ?, + salt = ?, + name = ?, + email = ?, + admin = ?, + enabled = ?, + admin_forced_password_reset = ?, + gravatar_url = ?, + position = ? + WHERE id = ? + ` + _, err := d.db.Exec(sqlStatement, user.Username, user.Password, + user.Salt, user.Name, user.Email, user.Admin, user.Enabled, + user.AdminForcedPasswordReset, user.GravatarURL, user.Position, user.ID) + if err != nil { + return errors.DatabaseError(err) + } + + return nil +} diff --git a/server/datastore/gorm_test.go b/server/datastore/mysql_test.go similarity index 52% rename from server/datastore/gorm_test.go rename to server/datastore/mysql_test.go index bf409827ad..a66fef120e 100644 --- a/server/datastore/gorm_test.go +++ b/server/datastore/mysql_test.go @@ -5,11 +5,13 @@ import ( "os" "testing" - "github.com/kolide/kolide-ose/server/kolide" + "github.com/WatchBeam/clock" + "github.com/go-kit/kit/log" + "github.com/kolide/kolide-ose/server/datastore/mysql" "github.com/stretchr/testify/require" ) -func setupGorm(t *testing.T) (ds kolide.Datastore, teardown func()) { +func setupMySQL(t *testing.T) (ds *mysql.Datastore, teardown func()) { var ( user = "kolide" password = "kolide" @@ -17,33 +19,40 @@ func setupGorm(t *testing.T) (ds kolide.Datastore, teardown func()) { host = "127.0.0.1" ) - // use linked container if available. if h, ok := os.LookupEnv("MYSQL_PORT_3306_TCP_ADDR"); ok { host = h } + connString := fmt.Sprintf("%s:%s@(%s:3306)/%s?charset=utf8&parseTime=True&loc=Local", user, password, host, dbName) - ds, err := New("gorm-mysql", connString) + + ds, err := mysql.New(connString, clock.NewMockClock(), mysql.Logger(log.NewNopLogger()), mysql.LimitAttempts(1)) require.Nil(t, err) teardown = func() { - db, ok := ds.(gormDB) - if !ok { - panic("expected gormDB datastore") - } - require.Nil(t, db.Drop()) - db.DB.Close() + ds.Close() } + return ds, teardown } -func TestGorm(t *testing.T) { +func TestMySQL(t *testing.T) { if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { t.SkipNow() } + + ds, teardown := setupMySQL(t) + defer teardown() + // get rid of database if it is hanging around + err := ds.Drop() + require.Nil(t, err) + for _, f := range testFunctions { + t.Run(functionName(f), func(t *testing.T) { - ds, teardown := setupGorm(t) - defer teardown() + require.Nil(t, ds.Migrate()) + defer func() { require.Nil(t, ds.Drop()) }() + f(t, ds) }) } + } diff --git a/server/errors/errors.go b/server/errors/errors.go index 92f3394e92..d09d1236e5 100644 --- a/server/errors/errors.go +++ b/server/errors/errors.go @@ -1,14 +1,22 @@ package errors import ( + goerrs "errors" "net/http" "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" "gopkg.in/go-playground/validator.v8" ) +var ( + // ErrNotFound is returned when the datastore resource cannot be found + ErrNotFound = goerrs.New("resource not found") + + // ErrExists is returned when creating a datastore resource that already exists + ErrExists = goerrs.New("resource already created") +) + // Kolide's internal representation for errors. It can be used to wrap another // error (stored in Err), and additionally contains fields for public // (PublicMessage) and private (PrivateMessage) error messages as well as the @@ -113,11 +121,6 @@ func baseReturnError(c *gin.Context, err error, messageKey string) { }) logrus.WithError(typedErr).Debug("Validation error") - case gorm.Errors, *gorm.Errors: - c.JSON(http.StatusInternalServerError, - gin.H{messageKey: "Database error"}) - logrus.WithError(typedErr).Debug(typedErr.Error()) - default: c.JSON(http.StatusInternalServerError, gin.H{messageKey: "Unspecified error"}) diff --git a/server/errors/errors_test.go b/server/errors/errors_test.go index 23ad1d1893..7a54cb50bb 100644 --- a/server/errors/errors_test.go +++ b/server/errors/errors_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" + "github.com/stretchr/testify/assert" "gopkg.in/go-playground/validator.v8" ) @@ -173,21 +173,3 @@ func TestReturnErrorValidationError(t *testing.T) { assert.Equal(t, expect, compFields) } - -func TestReturnErrorGormError(t *testing.T) { - r := gin.New() - - r.POST("/foo", func(c *gin.Context) { - err := gorm.Errors{} - err.Add(gorm.ErrInvalidSQL) - ReturnError(c, err) - }) - - req, _ := http.NewRequest("POST", "/foo", nil) - resp := httptest.NewRecorder() - - r.ServeHTTP(resp, req) - assert.Equal(t, http.StatusInternalServerError, resp.Code) - - assert.JSONEq(t, `{"message": "Database error"}`, resp.Body.String()) -} diff --git a/server/kolide/app.go b/server/kolide/app.go index 479ef090ac..834db859d2 100644 --- a/server/kolide/app.go +++ b/server/kolide/app.go @@ -21,10 +21,10 @@ type AppConfigService interface { // AppConfig holds configuration about the Kolide application. // AppConfig data can be managed by a Kolide API user. type AppConfig struct { - ID uint `gorm:"primary_key"` - OrgName string - OrgLogoURL string - KolideServerURL string + ID int64 + OrgName string `db:"org_name"` + OrgLogoURL string `db:"org_logo_url"` + KolideServerURL string `db:"kolide_server_url"` } // AppConfigPayload contains request and response format of @@ -36,8 +36,8 @@ type AppConfigPayload struct { // OrgInfo contains general info about the organization using Kolide. type OrgInfo struct { - OrgName *string `json:"org_name,omitempty"` - OrgLogoURL *string `json:"org_logo_url,omitempty"` + OrgName *string `json:"org_name,omitempty" db:"org_name"` + OrgLogoURL *string `json:"org_logo_url,omitempty" db:"org_logo_url"` } // ServerSettings contains general settings about the kolide App. diff --git a/server/kolide/emails.go b/server/kolide/emails.go index b1b567484b..53c964f860 100644 --- a/server/kolide/emails.go +++ b/server/kolide/emails.go @@ -38,12 +38,11 @@ type MailService interface { // PasswordResetRequest represents a database table for // Password Reset Requests type PasswordResetRequest struct { - ID uint `gorm:"primary_key"` - CreatedAt time.Time - UpdatedAt time.Time - ExpiresAt time.Time - UserID uint - Token string `gorm:"size:1024"` + UpdateCreateTimestamps + ID uint + ExpiresAt time.Time `db:"expires_at"` + UserID uint `db:"user_id"` + Token string } const passwordResetTemplate = ` diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index fe75971463..4bedc21b5f 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -1,6 +1,8 @@ package kolide import ( + "crypto/rand" + "encoding/base64" "time" "golang.org/x/net/context" @@ -15,7 +17,7 @@ type HostStore interface { EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error) AuthenticateHost(nodeKey string) (*Host, error) MarkHostSeen(host *Host, t time.Time) error - SearchHosts(query string, omit []uint) ([]Host, error) + SearchHosts(query string, omit ...uint) ([]Host, error) // DistributedQueriesForHost retrieves the distributed queries that the // given host should run. The result map is a mapping from campaign ID // to query text. @@ -30,18 +32,30 @@ type HostService interface { } type Host struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"updated_at"` - DetailUpdateTime time.Time `json:"detail_updated_at"` // Time that the host details were last updated - NodeKey string `json:"-" gorm:"unique_index:idx_host_unique_nodekey"` - HostName string `json:"hostname"` // there is a fulltext index on this field - UUID string `json:"uuid" gorm:"unique_index:idx_host_unique_uuid"` + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id"` + DetailUpdateTime time.Time `json:"detail_updated_at" db:"detail_update_time"` // Time that the host details were last updated + NodeKey string `json:"-" db:"node_key"` + HostName string `json:"hostname" db:"host_name"` // there is a fulltext index on this field + UUID string `json:"uuid"` Platform string `json:"platform"` - OsqueryVersion string `json:"osquery_version"` - OSVersion string `json:"os_version"` + OsqueryVersion string `json:"osquery_version" db:"osquery_version"` + OSVersion string `json:"os_version" db:"os_version"` Uptime time.Duration `json:"uptime"` - PhysicalMemory int `json:"memory" sql:"type:bigint"` - PrimaryMAC string `json:"mac"` - PrimaryIP string `json:"ip"` // there is a fulltext index on this field + PhysicalMemory int `json:"memory" sql:"type:bigint" db:"physical_memory"` + PrimaryMAC string `json:"mac" db:"primary_mac"` + PrimaryIP string `json:"ip" db:"primary_ip"` // there is a fulltext index on this field +} + +// RandomText returns a stdEncoded string of +// just what it says +func RandomText(keySize int) (string, error) { + key := make([]byte, keySize) + _, err := rand.Read(key) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(key), nil } diff --git a/server/kolide/invites.go b/server/kolide/invites.go index 6bdc3fe9b8..8aab1954da 100644 --- a/server/kolide/invites.go +++ b/server/kolide/invites.go @@ -3,7 +3,6 @@ package kolide import ( "bytes" "html/template" - "time" "golang.org/x/net/context" ) @@ -58,14 +57,15 @@ type InvitePayload struct { // Invite represents an invitation for a user to join Kolide. type Invite struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - InvitedBy uint `json:"invited_by" gorm:"not null"` - Email string `json:"email" gorm:"not null;unique_index:idx_invite_unique_email"` - Admin bool `json:"admin"` - Name string `json:"name"` - Position string `json:"position,omitempty"` - Token string `json:"-" gorm:"not null;unique_index:idx_invite_unique_key"` + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id" gorm:"primary_key"` + InvitedBy uint `json:"invited_by" gorm:"not null" db:"invited_by"` + Email string `json:"email" gorm:"not null;unique_index:idx_invite_unique_email"` + Admin bool `json:"admin"` + Name string `json:"name"` + Position string `json:"position,omitempty"` + Token string `json:"-" gorm:"not null;unique_index:idx_invite_unique_key"` } // TODO: fixme diff --git a/server/kolide/labels.go b/server/kolide/labels.go index 4bea153f7d..850fccb05b 100644 --- a/server/kolide/labels.go +++ b/server/kolide/labels.go @@ -32,7 +32,7 @@ type LabelStore interface { ListHostsInLabel(lid uint) ([]Host, error) ListUniqueHostsInLabels(labels []uint) ([]Host, error) - SearchLabels(query string, omit []uint) ([]Label, error) + SearchLabels(query string, omit ...uint) ([]Label, error) } type LabelService interface { @@ -50,19 +50,19 @@ type LabelPayload struct { } type Label struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - Name string `json:"name" gorm:"not null;unique_index:idx_label_unique_name"` // there is a fulltext index on this field - Description string `json:"description"` - Query string `json:"query" gorm:"not null"` - Platform string `json:"platform"` + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + Platform string `json:"platform"` } type LabelQueryExecution struct { - ID uint `gorm:"primary_key"` + ID uint UpdatedAt time.Time Matches bool - LabelID uint // Note we manually specify a unique index on these - HostID uint // fields in gormDB.Migrate + LabelID uint + HostID uint } diff --git a/server/kolide/packs.go b/server/kolide/packs.go index 00cf0d6a2d..cd32baab1b 100644 --- a/server/kolide/packs.go +++ b/server/kolide/packs.go @@ -8,7 +8,7 @@ import ( type PackStore interface { // Pack methods - NewPack(pack *Pack) error + NewPack(pack *Pack) (*Pack, error) SavePack(pack *Pack) error DeletePack(pid uint) error Pack(pid uint) (*Pack, error) @@ -44,11 +44,11 @@ type PackService interface { } type Pack struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - Name string `json:"name" gorm:"not null;unique_index:idx_pack_unique_name"` - Platform string `json:"platform"` + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id" gorm:"primary_key"` + Name string `json:"name" gorm:"not null;unique_index:idx_pack_unique_name"` + Platform string `json:"platform"` } type PackQuery struct { @@ -60,7 +60,7 @@ type PackQuery struct { } type PackTarget struct { - ID uint `gorm:"primary_key"` - PackID uint + ID uint `gorm:"primary_key"` + PackID uint Target } diff --git a/server/kolide/queries.go b/server/kolide/queries.go index d8e4167571..3ed0b52e44 100644 --- a/server/kolide/queries.go +++ b/server/kolide/queries.go @@ -15,16 +15,16 @@ type QueryStore interface { ListQueries(opt ListOptions) ([]*Query, error) // NewDistributedQueryCampaign creates a new distributed query campaign - NewDistributedQueryCampaign(camp DistributedQueryCampaign) (DistributedQueryCampaign, error) + NewDistributedQueryCampaign(camp *DistributedQueryCampaign) (*DistributedQueryCampaign, error) // SaveDistributedQueryCampaign updates an existing distributed query // campaign - SaveDistributedQueryCampaign(camp DistributedQueryCampaign) error + SaveDistributedQueryCampaign(camp *DistributedQueryCampaign) error // NewDistributedQueryCampaignTarget adds a new target to an existing // distributed query campaign - NewDistributedQueryCampaignTarget(target DistributedQueryCampaignTarget) (DistributedQueryCampaignTarget, error) + NewDistributedQueryCampaignTarget(target *DistributedQueryCampaignTarget) (*DistributedQueryCampaignTarget, error) // NewDistributedQueryCampaignExecution records a new execution for a // distributed query campaign - NewDistributedQueryExecution(exec DistributedQueryExecution) (DistributedQueryExecution, error) + NewDistributedQueryExecution(exec *DistributedQueryExecution) (*DistributedQueryExecution, error) } type QueryService interface { @@ -52,17 +52,17 @@ type PackPayload struct { } type Query struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - Name string `json:"name" gorm:"not null;unique_index:idx_query_unique_name"` - Description string `json:"description"` - Query string `json:"query" gorm:"not null"` - Interval uint `json:"interval"` - Snapshot bool `json:"snapshot"` - Differential bool `json:"differential"` - Platform string `json:"platform"` - Version string `json:"version"` + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + Interval uint `json:"interval"` + Snapshot bool `json:"snapshot"` + Differential bool `json:"differential"` + Platform string `json:"platform"` + Version string `json:"version"` } type DistributedQueryStatus int @@ -74,20 +74,20 @@ const ( ) type DistributedQueryCampaign struct { - ID uint `gorm:"primary_key"` - CreatedAt time.Time - UpdatedAt time.Time - QueryID uint - MaxDuration time.Duration + UpdateCreateTimestamps + DeleteFields + ID uint + QueryID uint `db:"query_id"` + MaxDuration time.Duration `db:"max_duration"` Status DistributedQueryStatus UserID uint } type DistributedQueryCampaignTarget struct { - ID uint `gorm:"primary_key"` + ID uint Type TargetType - DistributedQueryCampaignID uint `gorm:"index:idx_dqct_dqc_id"` - TargetID uint + DistributedQueryCampaignID uint `db:"distributed_query_campaign_id"` + TargetID uint `db:"target_id"` } type DistributedQueryExecutionStatus int @@ -106,20 +106,20 @@ type DistributedQueryResult struct { } type DistributedQueryExecution struct { - ID uint `gorm:"primary_key"` - HostID uint // unique index added in migrate - DistributedQueryCampaignID uint // unique index added in migrate + ID uint + HostID uint `db:"host_id"` + DistributedQueryCampaignID uint `db:"distributed_query_campaign_id"` Status DistributedQueryExecutionStatus - Error string `gorm:"size:1024"` - ExecutionDuration time.Duration + Error string + ExecutionDuration time.Duration `db:"execution_duration"` } type Option struct { - ID uint `gorm:"primary_key"` + ID uint CreatedAt time.Time UpdatedAt time.Time - Key string `gorm:"not null;unique_index:idx_option_unique_key"` - Value string `gorm:"not null"` + Key string + Value string Platform string } @@ -132,10 +132,10 @@ const ( ) type Decorator struct { - ID uint `gorm:"primary_key"` + ID uint CreatedAt time.Time UpdatedAt time.Time - Type DecoratorType `gorm:"not null"` + Type DecoratorType Interval int Query string } diff --git a/server/kolide/sessions.go b/server/kolide/sessions.go index dac9579f57..1ee1ce9342 100644 --- a/server/kolide/sessions.go +++ b/server/kolide/sessions.go @@ -68,11 +68,11 @@ type SessionService interface { // Session is the model object which represents what an active session is type Session struct { - ID uint `gorm:"primary_key"` - CreatedAt time.Time - AccessedAt time.Time - UserID uint `gorm:"not null"` - Key string `gorm:"not null;unique_index:idx_session_unique_key"` + CreateTimestamp + ID uint `gorm:"primary_key"` + AccessedAt time.Time `db:"accessed_at"` + UserID uint `gorm:"not null" db:"user_id"` + Key string `gorm:"not null;unique_index:idx_session_unique_key"` } //////////////////////////////////////////////////////////////////////////////// diff --git a/server/kolide/traits.go b/server/kolide/traits.go new file mode 100644 index 0000000000..ed3ba6f57a --- /dev/null +++ b/server/kolide/traits.go @@ -0,0 +1,33 @@ +package kolide + +import "time" + +// Createable contains common timestamp fields indicating create time +type CreateTimestamp struct { + CreatedAt time.Time `json:"created_at" db:"created_at"` +} + +// Deleteable is used to indicate a record is deleted. We don't actually +// delete record in the database. We mark it deleted, records with Deleted +// set to true will not normally be included in results +type DeleteFields struct { + DeletedAt *time.Time `json:"deleted_at" db:"deleted_at" gorm:"-"` + Deleted bool `json:"deleted"` +} + +// MarkDeleted indicates a record is deleted. It won't actually be removed from +// the database, but won't be returned in result sets. +func (d *DeleteFields) MarkDeleted(deleted time.Time) { + d.DeletedAt = &deleted + d.Deleted = true +} + +// UpdateTimestamp contains a timestamp that is set whenever an entity is changed +type UpdateTimestamp struct { + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +type UpdateCreateTimestamps struct { + CreateTimestamp + UpdateTimestamp +} diff --git a/server/kolide/users.go b/server/kolide/users.go index 5581d423e8..7db2048289 100644 --- a/server/kolide/users.go +++ b/server/kolide/users.go @@ -4,7 +4,6 @@ import ( "crypto/rand" "encoding/base64" "fmt" - "time" "golang.org/x/crypto/bcrypt" "golang.org/x/net/context" @@ -55,19 +54,19 @@ type UserService interface { // User is the model struct which represents a kolide user type User struct { - ID uint `json:"id" gorm:"primary_key"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - Username string `json:"username" gorm:"not null;unique_index:idx_user_unique_username"` - Password []byte `json:"-" gorm:"not null"` - Salt string `json:"-" gorm:"not null"` - Name string `json:"name"` - Email string `json:"email" gorm:"not null;unique_index:idx_user_unique_email"` - Admin bool `json:"admin" gorm:"not null"` - Enabled bool `json:"enabled" gorm:"not null"` - AdminForcedPasswordReset bool `json:"force_password_reset"` - GravatarURL string `json:"gravatar_url"` - Position string `json:"position,omitempty"` // job role + UpdateCreateTimestamps + DeleteFields + ID uint `json:"id" gorm:"primary_key"` + Username string `json:"username" gorm:"not null;unique_index:idx_user_unique_username"` + Password []byte `json:"-" gorm:"not null"` + Salt string `json:"-" gorm:"not null"` + Name string `json:"name"` + Email string `json:"email" gorm:"not null;unique_index:idx_user_unique_email"` + Admin bool `json:"admin" gorm:"not null"` + Enabled bool `json:"enabled" gorm:"not null"` + AdminForcedPasswordReset bool `json:"force_password_reset" db:"admin_forced_password_reset"` + GravatarURL string `json:"gravatar_url" db:"gravatar_url"` + Position string `json:"position,omitempty"` // job role } // UserPayload is used to modify an existing user diff --git a/server/pubsub/query_results_test.go b/server/pubsub/query_results_test.go index 3faf4b9c3c..84ee60edea 100644 --- a/server/pubsub/query_results_test.go +++ b/server/pubsub/query_results_test.go @@ -97,8 +97,12 @@ func testQueryResultsStoreErrors(t *testing.T, store kolide.QueryResultStore) { DistributedQueryCampaignID: 1, Rows: []map[string]string{{"bing": "fds"}}, Host: kolide.Host{ - ID: 4, - UpdatedAt: time.Now().UTC(), + ID: 4, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + }, DetailUpdateTime: time.Now().UTC(), }, }, @@ -123,7 +127,15 @@ func testQueryResultsStore(t *testing.T, store kolide.QueryResultStore) { // Note these times need to be set to avoid // issues with roundtrip serializing the zero // time value. See https://goo.gl/CCEs8x - UpdatedAt: time.Now().UTC(), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().UTC(), + }, + }, + DetailUpdateTime: time.Now().UTC(), }, }, @@ -131,8 +143,16 @@ func testQueryResultsStore(t *testing.T, store kolide.QueryResultStore) { DistributedQueryCampaignID: 1, Rows: []map[string]string{{"whoo": "wahh"}}, Host: kolide.Host{ - ID: 3, - UpdatedAt: time.Now().UTC(), + ID: 3, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().UTC(), + }, + }, + DetailUpdateTime: time.Now().UTC(), }, }, @@ -140,8 +160,16 @@ func testQueryResultsStore(t *testing.T, store kolide.QueryResultStore) { DistributedQueryCampaignID: 1, Rows: []map[string]string{{"bing": "fds"}}, Host: kolide.Host{ - ID: 4, - UpdatedAt: time.Now().UTC(), + ID: 4, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().UTC(), + }, + }, + DetailUpdateTime: time.Now().UTC(), }, }, @@ -158,8 +186,16 @@ func testQueryResultsStore(t *testing.T, store kolide.QueryResultStore) { DistributedQueryCampaignID: 2, Rows: []map[string]string{{"tim": "tom"}}, Host: kolide.Host{ - ID: 1, - UpdatedAt: time.Now().UTC(), + ID: 1, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().UTC(), + }, + }, + DetailUpdateTime: time.Now().UTC(), }, }, @@ -167,8 +203,16 @@ func testQueryResultsStore(t *testing.T, store kolide.QueryResultStore) { DistributedQueryCampaignID: 2, Rows: []map[string]string{{"slim": "slam"}}, Host: kolide.Host{ - ID: 3, - UpdatedAt: time.Now().UTC(), + ID: 3, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now().UTC(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now().UTC(), + }, + }, + DetailUpdateTime: time.Now().UTC(), }, }, diff --git a/server/service/endpoint_middleware_test.go b/server/service/endpoint_middleware_test.go index 0c58107075..e51e53e9dc 100644 --- a/server/service/endpoint_middleware_test.go +++ b/server/service/endpoint_middleware_test.go @@ -5,7 +5,7 @@ import ( "github.com/go-kit/kit/endpoint" "github.com/kolide/kolide-ose/server/contexts/viewer" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,7 +17,7 @@ import ( // permissions to access or modify resources func TestEndpointPermissions(t *testing.T) { req := struct{}{} - ds, _ := datastore.New("inmem", "") + ds, _ := inmem.New() createTestUsers(t, ds) admin1, _ := ds.User("admin1") user1, _ := ds.User("user1") @@ -184,7 +184,7 @@ func TestGetNodeKey(t *testing.T) { } func TestAuthenticatedHost(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) require.Nil(t, err) diff --git a/server/service/handler_test.go b/server/service/handler_test.go index 5df078e7a1..b343b70cdb 100644 --- a/server/service/handler_test.go +++ b/server/service/handler_test.go @@ -6,13 +6,13 @@ import ( "testing" "github.com/gorilla/mux" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) func TestAPIRoutes(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) diff --git a/server/service/http_auth_test.go b/server/service/http_auth_test.go index 94a3d0c809..3d093dd479 100644 --- a/server/service/http_auth_test.go +++ b/server/service/http_auth_test.go @@ -15,7 +15,7 @@ import ( kitlog "github.com/go-kit/kit/log" kithttp "github.com/go-kit/kit/transport/http" "github.com/gorilla/mux" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,7 +23,7 @@ import ( ) func TestLogin(t *testing.T) { - ds, _ := datastore.New("inmem", "") + ds, _ := inmem.New() svc, _ := newTestService(ds, nil) users := createTestUsers(t, ds) logger := kitlog.NewLogfmtLogger(os.Stdout) diff --git a/server/service/service_appconfig_test.go b/server/service/service_appconfig_test.go index 60a9e2d099..616bfab6a4 100644 --- a/server/service/service_appconfig_test.go +++ b/server/service/service_appconfig_test.go @@ -3,7 +3,7 @@ package service import ( "testing" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -11,7 +11,7 @@ import ( ) func TestCreateAppConfig(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) require.Nil(t, err) diff --git a/server/service/service_hosts_test.go b/server/service/service_hosts_test.go index df9c036909..f52ca888f7 100644 --- a/server/service/service_hosts_test.go +++ b/server/service/service_hosts_test.go @@ -3,14 +3,14 @@ package service import ( "testing" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) func TestListHosts(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -33,7 +33,7 @@ func TestListHosts(t *testing.T) { } func TestGetHost(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -54,7 +54,7 @@ func TestGetHost(t *testing.T) { } func TestDeleteHost(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) diff --git a/server/service/service_invites.go b/server/service/service_invites.go index de3f0335c9..a9298e88f4 100644 --- a/server/service/service_invites.go +++ b/server/service/service_invites.go @@ -4,7 +4,7 @@ import ( "errors" jwt "github.com/dgrijalva/jwt-go" - "github.com/kolide/kolide-ose/server/datastore" + kolide_errors "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" ) @@ -15,7 +15,7 @@ func (svc service) InviteNewUser(ctx context.Context, payload kolide.InvitePaylo if err == nil { return nil, newInvalidArgumentError("email", "a user with this account already exists") } - if err != datastore.ErrNotFound { + if err != kolide_errors.ErrNotFound { return nil, err } @@ -34,7 +34,6 @@ func (svc service) InviteNewUser(ctx context.Context, payload kolide.InvitePaylo Email: *payload.Email, Admin: *payload.Admin, InvitedBy: inviter.ID, - CreatedAt: svc.clock.Now(), Token: token, } if payload.Position != nil { diff --git a/server/service/service_invites_test.go b/server/service/service_invites_test.go index a997f25546..6e749a6c29 100644 --- a/server/service/service_invites_test.go +++ b/server/service/service_invites_test.go @@ -1,18 +1,20 @@ package service import ( - "golang.org/x/net/context" "testing" + "golang.org/x/net/context" + "github.com/WatchBeam/clock" "github.com/kolide/kolide-ose/server/config" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" ) func TestInviteNewUser(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() createTestUsers(t, ds) assert.Nil(t, err) nosuchAdminID := uint(999) @@ -42,7 +44,7 @@ func TestInviteNewUser(t *testing.T) { InvitedBy: &nosuchAdminID, Admin: boolPtr(false), }, - wantErr: datastore.ErrNotFound, + wantErr: errors.ErrNotFound, }, { payload: kolide.InvitePayload{ diff --git a/server/service/service_labels_test.go b/server/service/service_labels_test.go index 48c4eac4fa..1ed6c57998 100644 --- a/server/service/service_labels_test.go +++ b/server/service/service_labels_test.go @@ -3,14 +3,14 @@ package service import ( "testing" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) func TestListLabels(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -35,7 +35,7 @@ func TestListLabels(t *testing.T) { } func TestGetLabel(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -57,7 +57,7 @@ func TestGetLabel(t *testing.T) { } func TestNewLabel(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -82,7 +82,7 @@ func TestNewLabel(t *testing.T) { } func TestDeleteLabel(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index 549142264b..adfd11f66d 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -361,7 +361,7 @@ func (svc service) ingestDistributedQuery(host kolide.Host, name string, rows [] } // Record execution of the query - exec := kolide.DistributedQueryExecution{ + exec := &kolide.DistributedQueryExecution{ HostID: host.ID, DistributedQueryCampaignID: uint(campaignID), Status: kolide.ExecutionSucceeded, diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index a7c0fee2ab..54eb4ecb3b 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -14,7 +14,7 @@ import ( "github.com/WatchBeam/clock" hostctx "github.com/kolide/kolide-ose/server/contexts/host" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/kolide/kolide-ose/server/pubsub" "github.com/stretchr/testify/assert" @@ -22,7 +22,7 @@ import ( ) func TestEnrollAgent(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -44,7 +44,7 @@ func TestEnrollAgent(t *testing.T) { } func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -66,7 +66,7 @@ func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) { } func TestSubmitStatusLogs(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) mockClock := clock.NewMockClock() @@ -138,7 +138,7 @@ func TestSubmitStatusLogs(t *testing.T) { } func TestSubmitResultLogs(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) mockClock := clock.NewMockClock() @@ -211,9 +211,16 @@ func TestSubmitResultLogs(t *testing.T) { func TestHostDetailQueries(t *testing.T) { mockClock := clock.NewMockClock() host := kolide.Host{ - ID: 1, - CreatedAt: mockClock.Now(), - UpdatedAt: mockClock.Now(), + ID: 1, + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: mockClock.Now(), + }, + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: mockClock.Now(), + }, + }, + DetailUpdateTime: mockClock.Now(), NodeKey: "test_key", HostName: "test_hostname", @@ -239,7 +246,7 @@ func TestHostDetailQueries(t *testing.T) { } func TestLabelQueries(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) mockClock := clock.NewMockClock() @@ -366,7 +373,7 @@ func TestLabelQueries(t *testing.T) { } func TestGetClientConfig(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) mockClock := clock.NewMockClock() @@ -415,7 +422,7 @@ func TestGetClientConfig(t *testing.T) { monitoringPack := &kolide.Pack{ Name: "monitoring", } - err = ds.NewPack(monitoringPack) + _, err = ds.NewPack(monitoringPack) assert.Nil(t, err) err = ds.AddQueryToPack(infoQuery.ID, monitoringPack.ID) @@ -447,7 +454,7 @@ func TestGetClientConfig(t *testing.T) { } func TestDetailQueries(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) mockClock := clock.NewMockClock() @@ -589,7 +596,7 @@ func TestDetailQueries(t *testing.T) { } func TestDistributedQueries(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) mockClock := clock.NewMockClock() @@ -633,7 +640,7 @@ func TestDistributedQueries(t *testing.T) { require.Nil(t, err) // Create query campaign - c1 := kolide.DistributedQueryCampaign{ + c1 := &kolide.DistributedQueryCampaign{ QueryID: query.ID, Status: kolide.QueryRunning, } @@ -642,7 +649,7 @@ func TestDistributedQueries(t *testing.T) { require.Nil(t, err) // Add a target to the campaign (targeting the matching label) - target := kolide.DistributedQueryCampaignTarget{ + target := &kolide.DistributedQueryCampaignTarget{ Type: kolide.TargetLabel, DistributedQueryCampaignID: c1.ID, TargetID: label.ID, @@ -678,7 +685,7 @@ func TestDistributedQueries(t *testing.T) { assert.NotNil(t, err) // TODO use service method - readChan, err := rs.ReadChannel(ctx, c1) + readChan, err := rs.ReadChannel(ctx, *c1) require.Nil(t, err) // We need to listen for the result in a separate thread to prevent the diff --git a/server/service/service_packs.go b/server/service/service_packs.go index 5800842c09..4fcdde4e4f 100644 --- a/server/service/service_packs.go +++ b/server/service/service_packs.go @@ -24,7 +24,7 @@ func (svc service) NewPack(ctx context.Context, p kolide.PackPayload) (*kolide.P pack.Platform = *p.Platform } - err := svc.ds.NewPack(&pack) + _, err := svc.ds.NewPack(&pack) if err != nil { return nil, err } diff --git a/server/service/service_packs_test.go b/server/service/service_packs_test.go index 6717e20a7a..c56ffff1a9 100644 --- a/server/service/service_packs_test.go +++ b/server/service/service_packs_test.go @@ -3,14 +3,14 @@ package service import ( "testing" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) func TestListPacks(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -22,7 +22,7 @@ func TestListPacks(t *testing.T) { assert.Nil(t, err) assert.Len(t, queries, 0) - err = ds.NewPack(&kolide.Pack{ + _, err = ds.NewPack(&kolide.Pack{ Name: "foo", }) assert.Nil(t, err) @@ -33,7 +33,7 @@ func TestListPacks(t *testing.T) { } func TestGetPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -44,7 +44,7 @@ func TestGetPack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) @@ -55,7 +55,7 @@ func TestGetPack(t *testing.T) { } func TestNewPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -76,7 +76,7 @@ func TestNewPack(t *testing.T) { } func TestModifyPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -87,7 +87,7 @@ func TestModifyPack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) @@ -102,7 +102,7 @@ func TestModifyPack(t *testing.T) { } func TestDeletePack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -113,7 +113,7 @@ func TestDeletePack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) @@ -127,7 +127,7 @@ func TestDeletePack(t *testing.T) { } func TestAddQueryToPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -138,7 +138,7 @@ func TestAddQueryToPack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) @@ -163,7 +163,7 @@ func TestAddQueryToPack(t *testing.T) { } func TestGetQueriesInPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -174,7 +174,7 @@ func TestGetQueriesInPack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) @@ -195,7 +195,7 @@ func TestGetQueriesInPack(t *testing.T) { } func TestRemoveQueryFromPack(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -206,7 +206,7 @@ func TestRemoveQueryFromPack(t *testing.T) { pack := &kolide.Pack{ Name: "foo", } - err = ds.NewPack(pack) + _, err = ds.NewPack(pack) assert.Nil(t, err) assert.NotZero(t, pack.ID) diff --git a/server/service/service_queries_test.go b/server/service/service_queries_test.go index 1d793a596f..340a1320ea 100644 --- a/server/service/service_queries_test.go +++ b/server/service/service_queries_test.go @@ -3,14 +3,14 @@ package service import ( "testing" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) func TestListQueries(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -34,7 +34,7 @@ func TestListQueries(t *testing.T) { } func TestGetQuery(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -57,7 +57,7 @@ func TestGetQuery(t *testing.T) { } func TestNewQuery(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -80,7 +80,7 @@ func TestNewQuery(t *testing.T) { } func TestModifyQuery(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) @@ -107,7 +107,7 @@ func TestModifyQuery(t *testing.T) { } func TestDeleteQuery(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) svc, err := newTestService(ds, nil) diff --git a/server/service/service_sessions.go b/server/service/service_sessions.go index 976b6d01fe..c20e697ea2 100644 --- a/server/service/service_sessions.go +++ b/server/service/service_sessions.go @@ -7,7 +7,7 @@ import ( "time" "github.com/kolide/kolide-ose/server/contexts/viewer" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" "golang.org/x/net/context" ) @@ -16,7 +16,7 @@ func (svc service) Login(ctx context.Context, username, password string) (*kolid user, err := svc.userByEmailOrUsername(username) switch err { case nil: - case datastore.ErrNotFound: + case errors.ErrNotFound: return nil, "", authError{reason: "no such user"} default: return nil, "", err @@ -24,7 +24,7 @@ func (svc service) Login(ctx context.Context, username, password string) (*kolid if !user.Enabled { return nil, "", authError{reason: "account disabled", clientReason: "account disabled"} } - if err := user.ValidatePassword(password); err != nil { + if err = user.ValidatePassword(password); err != nil { return nil, "", authError{reason: "bad password"} } token, err := svc.makeSession(user.ID) diff --git a/server/service/service_sessions_test.go b/server/service/service_sessions_test.go index d82cd685d6..f4e5ffcdb0 100644 --- a/server/service/service_sessions_test.go +++ b/server/service/service_sessions_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,7 +14,7 @@ import ( const bcryptCost = 6 func TestAuthenticate(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) require.Nil(t, err) diff --git a/server/service/service_targets.go b/server/service/service_targets.go index dff92bf36a..5b9955f272 100644 --- a/server/service/service_targets.go +++ b/server/service/service_targets.go @@ -8,13 +8,13 @@ import ( func (svc service) SearchTargets(ctx context.Context, query string, selectedHostIDs []uint, selectedLabelIDs []uint) (*kolide.TargetSearchResults, error) { results := &kolide.TargetSearchResults{} - hosts, err := svc.ds.SearchHosts(query, selectedHostIDs) + hosts, err := svc.ds.SearchHosts(query, selectedHostIDs...) if err != nil { return nil, err } results.Hosts = hosts - labels, err := svc.ds.SearchLabels(query, selectedLabelIDs) + labels, err := svc.ds.SearchLabels(query, selectedLabelIDs...) if err != nil { return nil, err } diff --git a/server/service/service_targets_test.go b/server/service/service_targets_test.go index 2ba17ac5f4..2dcd299a85 100644 --- a/server/service/service_targets_test.go +++ b/server/service/service_targets_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -13,7 +13,7 @@ import ( ) func TestSearchTargets(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) @@ -44,7 +44,7 @@ func TestSearchTargets(t *testing.T) { } func TestCountHostsInTargets(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) @@ -140,7 +140,7 @@ func TestCountHostsInTargets(t *testing.T) { } func TestSearchWithOmit(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) @@ -192,7 +192,7 @@ func TestSearchWithOmit(t *testing.T) { } func TestSearchHostsInLabels(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) @@ -245,7 +245,7 @@ func TestSearchHostsInLabels(t *testing.T) { } func TestSearchResultsLimit(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() require.Nil(t, err) svc, err := newTestService(ds, nil) diff --git a/server/service/service_users.go b/server/service/service_users.go index 5c3bc27d07..6dae47830b 100644 --- a/server/service/service_users.go +++ b/server/service/service_users.go @@ -186,8 +186,14 @@ func (svc service) RequestPasswordReset(ctx context.Context, email string) error } request := &kolide.PasswordResetRequest{ - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now(), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, ExpiresAt: time.Now().Add(time.Hour * 24), UserID: user.ID, Token: token, diff --git a/server/service/service_users_test.go b/server/service/service_users_test.go index c89f1c31ec..49018574fc 100644 --- a/server/service/service_users_test.go +++ b/server/service/service_users_test.go @@ -8,15 +8,17 @@ import ( "github.com/WatchBeam/clock" "github.com/kolide/kolide-ose/server/config" "github.com/kolide/kolide-ose/server/contexts/viewer" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/datastore/inmem" + kolide_errors "github.com/kolide/kolide-ose/server/errors" "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" ) func TestAuthenticatedUser(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) createTestUsers(t, ds) svc, err := newTestService(ds, nil) @@ -32,7 +34,7 @@ func TestAuthenticatedUser(t *testing.T) { } func TestRequestPasswordReset(t *testing.T) { - ds, err := datastore.New("inmem", "") + ds, err := inmem.New() assert.Nil(t, err) createTestUsers(t, ds) admin1, err := ds.User("admin1") @@ -112,7 +114,7 @@ func TestRequestPasswordReset(t *testing.T) { } func TestCreateUser(t *testing.T) { - ds, _ := datastore.New("inmem", "") + ds, _ := inmem.New() svc, _ := newTestService(ds, nil) invites := setupInvites(t, ds, []string{"admin2@example.com"}) ctx := context.Background() @@ -154,7 +156,7 @@ func TestCreateUser(t *testing.T) { NeedsPasswordReset: boolPtr(true), Admin: boolPtr(false), InviteToken: &invites["admin2@example.com"].Token, - wantErr: datastore.ErrNotFound, + wantErr: kolide_errors.ErrNotFound, }, { Username: stringPtr("admin3"), @@ -217,7 +219,11 @@ func setupInvites(t *testing.T, ds kolide.Datastore, emails []string) map[string InvitedBy: users["admin1"].ID, Token: e, Email: e, - CreatedAt: mockClock.Now(), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: mockClock.Now(), + }, + }, }) require.Nil(t, err) invites[e] = invite @@ -227,7 +233,11 @@ func setupInvites(t *testing.T, ds kolide.Datastore, emails []string) map[string InvitedBy: users["admin1"].ID, Token: "expired", Email: "expiredinvite@gmail.com", - CreatedAt: mockClock.Now().AddDate(-1, 0, 0), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: mockClock.Now().AddDate(-1, 0, 0), + }, + }, }) require.Nil(t, err) invites["expired"] = invite @@ -235,7 +245,7 @@ func setupInvites(t *testing.T, ds kolide.Datastore, emails []string) map[string } func TestChangeUserPassword(t *testing.T) { - ds, _ := datastore.New("inmem", "") + ds, _ := inmem.New() svc, _ := newTestService(ds, nil) createTestUsers(t, ds) var passwordChangeTests = []struct { @@ -250,7 +260,7 @@ func TestChangeUserPassword(t *testing.T) { { // bad token token: "dcbaz", newPassword: "123cat!", - wantErr: datastore.ErrNotFound, + wantErr: kolide_errors.ErrNotFound, }, { // missing token newPassword: "123cat!", @@ -266,8 +276,14 @@ func TestChangeUserPassword(t *testing.T) { t.Run("", func(t *testing.T) { ctx := context.Background() request := &kolide.PasswordResetRequest{ - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + UpdateCreateTimestamps: kolide.UpdateCreateTimestamps{ + CreateTimestamp: kolide.CreateTimestamp{ + CreatedAt: time.Now(), + }, + UpdateTimestamp: kolide.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, ExpiresAt: time.Now().Add(time.Hour * 24), UserID: 1, Token: "abcd", diff --git a/server/service/transport_error.go b/server/service/transport_error.go index c7abed83ba..542c33a3a5 100644 --- a/server/service/transport_error.go +++ b/server/service/transport_error.go @@ -5,7 +5,7 @@ import ( "net/http" kithttp "github.com/go-kit/kit/transport/http" - "github.com/kolide/kolide-ose/server/datastore" + "github.com/kolide/kolide-ose/server/errors" "golang.org/x/net/context" ) @@ -112,9 +112,9 @@ func encodeError(ctx context.Context, err error, w http.ResponseWriter) { func codeFromErr(err error) int { switch err { - case datastore.ErrNotFound: + case errors.ErrNotFound: return http.StatusNotFound - case datastore.ErrExists: + case errors.ErrExists: return http.StatusConflict default: return http.StatusInternalServerError