From 35806f1442c36b1b3649e2a0728b200f9cf01d73 Mon Sep 17 00:00:00 2001 From: Zachary Wasserman Date: Thu, 5 Jan 2017 09:27:56 -0800 Subject: [PATCH] Separate tables from data migrations (#745) This PR separates the table migrations from the data population migrations. Table migrations run before data migrations. Now, we have the ability to create the database tables without populating them with data. This can be useful for running "unit" tests against a MySQL store that doesn't have any pre-populated data. When performing real migrations, or for more "integration" style testing, the data migrations can also be executed. Note there are some special cases that must be observed with these migrations, and the README is updated to reflect those. --- README.md | 52 ++++++++++++++++--- cli/prepare.go | 6 ++- glide.lock | 21 ++++---- glide.yaml | 2 +- main.go | 1 - server/datastore/datastore_options_test.go | 4 ++ server/datastore/inmem/inmem.go | 15 ++++-- .../20161223115449_InsertOsqueryOptions.go | 19 ++++--- .../mysql/migrations/data/migration.go | 7 +++ .../20161118193812_CreateTableAppConfigs.go | 10 ++-- ...ateTableDistributedQueryCampaignTargets.go | 6 +-- ...36_CreateTableDistributedQueryCampaigns.go | 6 +-- ...5_CreateTableDistributedQueryExecutions.go | 6 +-- .../20161118212528_CreateTableHosts.go | 6 +-- .../20161118212538_CreateTableInvites.go | 6 +-- ...8212549_CreateTableLabelQueryExecutions.go | 6 +-- .../20161118212557_CreateTableLabels.go | 5 +- .../20161118212604_CreateTableOptions.go | 6 +-- ...61118212613_CreateTableScheduledQueries.go | 6 +-- .../20161118212621_CreateTablePackTargets.go | 6 +-- .../20161118212630_CreateTablePacks.go | 6 +-- ...212641_CreateTablePasswordResetRequests.go | 6 +-- .../20161118212649_CreateTableUsers.go | 6 +-- .../20161118212656_CreateTableSessions.go | 6 +-- .../20161118212758_CreateTableQueries.go | 6 +-- ...1128234849_CreateTableNetworkInterfaces.go | 6 +-- ...61230162221_AddUniqueIndexToPackTargets.go | 6 +-- .../20170104113816_AddSeenTimeToHosts.go | 10 ++-- .../mysql/migrations/tables/migration.go | 7 +++ .../mysql/{datastore.go => mysql.go} | 18 ++++--- server/datastore/mysql_test.go | 3 +- server/kolide/datastore.go | 5 +- server/service/endpoint_test.go | 9 +++- server/service/service_appconfig_test.go | 2 + server/service/service_osquery_test.go | 3 +- 35 files changed, 167 insertions(+), 128 deletions(-) rename server/datastore/mysql/migrations/{ => data}/20161223115449_InsertOsqueryOptions.go (78%) create mode 100644 server/datastore/mysql/migrations/data/migration.go rename server/datastore/mysql/migrations/{ => tables}/20161118193812_CreateTableAppConfigs.go (92%) rename server/datastore/mysql/migrations/{ => tables}/20161118211713_CreateTableDistributedQueryCampaignTargets.go (85%) rename server/datastore/mysql/migrations/{ => tables}/20161118212436_CreateTableDistributedQueryCampaigns.go (88%) rename server/datastore/mysql/migrations/{ => tables}/20161118212515_CreateTableDistributedQueryExecutions.go (88%) rename server/datastore/mysql/migrations/{ => tables}/20161118212528_CreateTableHosts.go (95%) rename server/datastore/mysql/migrations/{ => tables}/20161118212538_CreateTableInvites.go (90%) rename server/datastore/mysql/migrations/{ => tables}/20161118212549_CreateTableLabelQueryExecutions.go (87%) rename server/datastore/mysql/migrations/{ => tables}/20161118212557_CreateTableLabels.go (90%) rename server/datastore/mysql/migrations/{ => tables}/20161118212604_CreateTableOptions.go (85%) rename server/datastore/mysql/migrations/{ => tables}/20161118212613_CreateTableScheduledQueries.go (90%) rename server/datastore/mysql/migrations/{ => tables}/20161118212621_CreateTablePackTargets.go (83%) rename server/datastore/mysql/migrations/{ => tables}/20161118212630_CreateTablePacks.go (89%) rename server/datastore/mysql/migrations/{ => tables}/20161118212641_CreateTablePasswordResetRequests.go (87%) rename server/datastore/mysql/migrations/{ => tables}/20161118212649_CreateTableUsers.go (92%) rename server/datastore/mysql/migrations/{ => tables}/20161118212656_CreateTableSessions.go (86%) rename server/datastore/mysql/migrations/{ => tables}/20161118212758_CreateTableQueries.go (89%) rename server/datastore/mysql/migrations/{ => tables}/20161128234849_CreateTableNetworkInterfaces.go (93%) rename server/datastore/mysql/migrations/{ => tables}/20161230162221_AddUniqueIndexToPackTargets.go (80%) rename server/datastore/mysql/migrations/{ => tables}/20170104113816_AddSeenTimeToHosts.go (70%) create mode 100644 server/datastore/mysql/migrations/tables/migration.go rename server/datastore/mysql/{datastore.go => mysql.go} (92%) diff --git a/README.md b/README.md index 9010e15b7c..270b43f129 100644 --- a/README.md +++ b/README.md @@ -141,20 +141,33 @@ To update, use [`glide up`](https://github.com/Masterminds/glide#glide-update-al #### Database Modifications - * From the project root run the following commands: +##### Adding/Updating tables + +Database schemas are managed by a series of migrations defined in go code. We +use a customized version of the Goose migrations tool to handle these +migrations. + +Note: Once committed to the Kolide repo, table migrations should be considered +immutable. Any changes to an existing table should take place in a new +migration executing ALTERs. + + * From the project root run the following shell commands: + ``` bash - > go get github.com/pressly/goose - > cd server/datastore/mysql/migrations - > goose create AddColumnFooToUsers + go get github.com/kolide/goose + cd server/datastore/mysql/migrations/tables + goose create AddColumnFooToUsers ``` + * Find the file you created in the migrations directory and edit it + ``` go package migration import ( "database/sql" - "github.com/pressly/goose" + "github.com/kolide/goose" ) func init() { @@ -171,12 +184,35 @@ To update, use [`glide up`](https://github.com/Masterminds/glide#glide-update-al return err } ``` - * Update the database by running the following command + + * Update the database by running the following shell commands: + ``` bash - > make build - > build/kolide prepare db + make build + build/kolide prepare db ``` +##### Populating the database + +Populating built in data is also performed through migrations. All table +migrations are performed before any data migrations. + +Note: Data migrations can be mutable. If tables are altered in a way that would +render a data migration invalid (columns changed/removed), data migrations +should be updated to comply with the new schema. Data migrations will not be +re-run when they have already been run against a database, but they must be +updated to maintain compatibility with a fresh DB. + + * From the project root run the following shell commands: + + ``` bash + go get github.com/kolide/goose + cd server/datastore/mysql/migrations/data + goose create PopulateFoo + ``` + + * Proceed as for table migrations, editing and running the newly created + migration file. ### Testing diff --git a/cli/prepare.go b/cli/prepare.go index 0fe8256645..156bd879bf 100644 --- a/cli/prepare.go +++ b/cli/prepare.go @@ -38,9 +38,13 @@ To setup kolide infrastructure, use one of the available commands. initFatal(err, "creating db connection") } - if err := ds.Migrate(); err != nil { + if err := ds.MigrateTables(); err != nil { initFatal(err, "migrating db schema") } + + if err := ds.MigrateData(); err != nil { + initFatal(err, "migrating builtin data") + } }, } diff --git a/glide.lock b/glide.lock index dd023c4e0d..f0454a8da0 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 4a87ddb9e91081df70590230979f58985cc7a9a1f9f6be2e1033e4b84cf2c6e0 -updated: 2016-11-18T19:27:52.047338531+08:00 +hash: e9cd204c0dd7e3e9102a2e0c87cc722101473f0d8fd31b05ec7f88837dd1d529 +updated: 2017-01-04T15:27:20.412164218-08:00 imports: - name: github.com/alecthomas/template version: a0175ee3bccc567396460bf5acd36800cb10c49c @@ -11,6 +11,10 @@ imports: version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 subpackages: - quantile +- name: github.com/davecgh/go-spew + version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + subpackages: + - spew - name: github.com/dgrijalva/jwt-go version: 01aeca54ebda6e0fbfafd0a524d234159c05ec20 - name: github.com/elazarl/go-bindata-assetfs @@ -68,6 +72,8 @@ imports: - reflectx - name: github.com/jordan-wright/email version: fd703108daeb23d77c124d12978e9b6c4f28f034 +- name: github.com/kolide/goose + version: 5221ed7545f6b931a24f99d9bd3e7bcf36aee71b - name: github.com/kr/fs version: 2788f0dbd16903de03cb8186e5c7d97b69ad387b - name: github.com/kr/logfmt @@ -94,12 +100,11 @@ imports: version: 792786c7400a136282c1664665ae0a8db921c6c2 subpackages: - difflib -- name: github.com/pressly/goose - version: d6e8fe0292718c30342b3ed7f324dc21634b3bc6 - name: github.com/prometheus/client_golang version: c5b7fccd204277076155f10851dad72b76a49317 subpackages: - prometheus + - prometheus/promhttp - name: github.com/prometheus/client_model version: fa8ad6fec33561be4280a8f0514318c79d7f6cb6 subpackages: @@ -162,11 +167,7 @@ imports: - name: gopkg.in/go-playground/validator.v8 version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c - name: gopkg.in/natefinch/lumberjack.v2 - version: e21e5cbec0cd0861b9dc302736ad5666c529d93f + version: dd45e6a67c53f673bb49ca8a001fd3a63ceb640e - name: gopkg.in/yaml.v2 version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e -testImports: -- name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 - subpackages: - - spew +testImports: [] diff --git a/glide.yaml b/glide.yaml index 66b4426fbe..416a5c6e77 100644 --- a/glide.yaml +++ b/glide.yaml @@ -59,4 +59,4 @@ import: subpackages: - redis - package: github.com/jmoiron/sqlx -- package: github.com/pressly/goose +- package: github.com/kolide/goose diff --git a/main.go b/main.go index 762fa377a0..1963b0be05 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/kolide/kolide-ose/cli" - _ "github.com/kolide/kolide-ose/server/datastore/mysql/migrations" ) func init() { diff --git a/server/datastore/datastore_options_test.go b/server/datastore/datastore_options_test.go index 506547ba0a..813cfbb3a3 100644 --- a/server/datastore/datastore_options_test.go +++ b/server/datastore/datastore_options_test.go @@ -11,6 +11,8 @@ import ( ) func testOptions(t *testing.T, ds kolide.Datastore) { + require.Nil(t, ds.MigrateData()) + // were options pre-loaded? opts, err := ds.ListOptions() require.Nil(t, err) @@ -58,6 +60,8 @@ func testOptions(t *testing.T, ds kolide.Datastore) { } func testOptionsToConfig(t *testing.T, ds kolide.Datastore) { + require.Nil(t, ds.MigrateData()) + resp, err := ds.GetOsqueryConfigOptions() require.Nil(t, err) assert.Len(t, resp, 10) diff --git a/server/datastore/inmem/inmem.go b/server/datastore/inmem/inmem.go index 90bcb52a36..fd14cb04b9 100644 --- a/server/datastore/inmem/inmem.go +++ b/server/datastore/inmem/inmem.go @@ -43,7 +43,7 @@ func New(config config.KolideConfig) (*Datastore, error) { config: &config, } - if err := ds.Migrate(); err != nil { + if err := ds.MigrateTables(); err != nil { return nil, err } @@ -69,9 +69,10 @@ func sortResults(slice interface{}, opt kolide.ListOptions, fields map[string]st return nil } -func (d *Datastore) Migrate() error { +func (d *Datastore) MigrateTables() error { d.mtx.Lock() defer d.mtx.Unlock() + d.nextIDs = make(map[interface{}]uint) d.users = make(map[uint]*kolide.User) d.sessions = make(map[uint]*kolide.Session) @@ -89,6 +90,13 @@ func (d *Datastore) Migrate() error { d.distributedQueryCampaignTargets = make(map[uint]kolide.DistributedQueryCampaignTarget) d.options = make(map[uint]*kolide.Option) + return nil +} + +func (d *Datastore) MigrateData() error { + d.mtx.Lock() + defer d.mtx.Unlock() + for _, initData := range appstate.Options { opt := kolide.Option{ Name: initData.Name, @@ -107,11 +115,12 @@ func (d *Datastore) Migrate() error { SMTPEnableStartTLS: true, SMTPVerifySSLCerts: true, } + return nil } func (d *Datastore) Drop() error { - return d.Migrate() + return d.MigrateTables() } func (d *Datastore) Initialize() error { diff --git a/server/datastore/mysql/migrations/20161223115449_InsertOsqueryOptions.go b/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go similarity index 78% rename from server/datastore/mysql/migrations/20161223115449_InsertOsqueryOptions.go rename to server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go index 8e37e579c6..8c0989e59e 100644 --- a/server/datastore/mysql/migrations/20161223115449_InsertOsqueryOptions.go +++ b/server/datastore/mysql/migrations/data/20161223115449_InsertOsqueryOptions.go @@ -1,26 +1,25 @@ -package migration +package data import ( "database/sql" "github.com/kolide/kolide-ose/server/datastore/internal/appstate" "github.com/kolide/kolide-ose/server/kolide" - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161223115449, Down_20161223115449) + MigrationClient.AddMigration(Up_20161223115449, Down_20161223115449) } func Up_20161223115449(tx *sql.Tx) error { sqlStatement := ` - INSERT INTO options ( - name, - type, - value, - read_only - ) VALUES( ?, ?, ?, ?) - ` + INSERT INTO options ( + name, + type, + value, + read_only + ) VALUES (?, ?, ?, ?) + ` for _, opt := range appstate.Options { ov := kolide.Option{ diff --git a/server/datastore/mysql/migrations/data/migration.go b/server/datastore/mysql/migrations/data/migration.go new file mode 100644 index 0000000000..89ce700cb7 --- /dev/null +++ b/server/datastore/mysql/migrations/data/migration.go @@ -0,0 +1,7 @@ +package data + +import "github.com/kolide/goose" + +var ( + MigrationClient = goose.New("migration_status_data", goose.MySqlDialect{}) +) diff --git a/server/datastore/mysql/migrations/20161118193812_CreateTableAppConfigs.go b/server/datastore/mysql/migrations/tables/20161118193812_CreateTableAppConfigs.go similarity index 92% rename from server/datastore/mysql/migrations/20161118193812_CreateTableAppConfigs.go rename to server/datastore/mysql/migrations/tables/20161118193812_CreateTableAppConfigs.go index 53e66b6625..e5ea7e976b 100644 --- a/server/datastore/mysql/migrations/20161118193812_CreateTableAppConfigs.go +++ b/server/datastore/mysql/migrations/tables/20161118193812_CreateTableAppConfigs.go @@ -1,13 +1,9 @@ -package migration +package tables -import ( - "database/sql" - - "github.com/pressly/goose" -) +import "database/sql" func init() { - goose.AddMigration(Up_20161118193812, Down_20161118193812) + MigrationClient.AddMigration(Up_20161118193812, Down_20161118193812) } func Up_20161118193812(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118211713_CreateTableDistributedQueryCampaignTargets.go b/server/datastore/mysql/migrations/tables/20161118211713_CreateTableDistributedQueryCampaignTargets.go similarity index 85% rename from server/datastore/mysql/migrations/20161118211713_CreateTableDistributedQueryCampaignTargets.go rename to server/datastore/mysql/migrations/tables/20161118211713_CreateTableDistributedQueryCampaignTargets.go index 19bbd552c5..12cbdc027c 100644 --- a/server/datastore/mysql/migrations/20161118211713_CreateTableDistributedQueryCampaignTargets.go +++ b/server/datastore/mysql/migrations/tables/20161118211713_CreateTableDistributedQueryCampaignTargets.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118211713, Down_20161118211713) + MigrationClient.AddMigration(Up_20161118211713, Down_20161118211713) } func Up_20161118211713(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212436_CreateTableDistributedQueryCampaigns.go b/server/datastore/mysql/migrations/tables/20161118212436_CreateTableDistributedQueryCampaigns.go similarity index 88% rename from server/datastore/mysql/migrations/20161118212436_CreateTableDistributedQueryCampaigns.go rename to server/datastore/mysql/migrations/tables/20161118212436_CreateTableDistributedQueryCampaigns.go index da79be6cb5..40c3abc0f0 100644 --- a/server/datastore/mysql/migrations/20161118212436_CreateTableDistributedQueryCampaigns.go +++ b/server/datastore/mysql/migrations/tables/20161118212436_CreateTableDistributedQueryCampaigns.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212436, Down_20161118212436) + MigrationClient.AddMigration(Up_20161118212436, Down_20161118212436) } func Up_20161118212436(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212515_CreateTableDistributedQueryExecutions.go b/server/datastore/mysql/migrations/tables/20161118212515_CreateTableDistributedQueryExecutions.go similarity index 88% rename from server/datastore/mysql/migrations/20161118212515_CreateTableDistributedQueryExecutions.go rename to server/datastore/mysql/migrations/tables/20161118212515_CreateTableDistributedQueryExecutions.go index 7601b408a1..b5dd18434d 100644 --- a/server/datastore/mysql/migrations/20161118212515_CreateTableDistributedQueryExecutions.go +++ b/server/datastore/mysql/migrations/tables/20161118212515_CreateTableDistributedQueryExecutions.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212515, Down_20161118212515) + MigrationClient.AddMigration(Up_20161118212515, Down_20161118212515) } func Up_20161118212515(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go b/server/datastore/mysql/migrations/tables/20161118212528_CreateTableHosts.go similarity index 95% rename from server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go rename to server/datastore/mysql/migrations/tables/20161118212528_CreateTableHosts.go index 20d0b28061..12484587c3 100644 --- a/server/datastore/mysql/migrations/20161118212528_CreateTableHosts.go +++ b/server/datastore/mysql/migrations/tables/20161118212528_CreateTableHosts.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212528, Down_20161118212528) + MigrationClient.AddMigration(Up_20161118212528, Down_20161118212528) } func Up_20161118212528(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212538_CreateTableInvites.go b/server/datastore/mysql/migrations/tables/20161118212538_CreateTableInvites.go similarity index 90% rename from server/datastore/mysql/migrations/20161118212538_CreateTableInvites.go rename to server/datastore/mysql/migrations/tables/20161118212538_CreateTableInvites.go index 4e9134ddc7..f3fe5cac9f 100644 --- a/server/datastore/mysql/migrations/20161118212538_CreateTableInvites.go +++ b/server/datastore/mysql/migrations/tables/20161118212538_CreateTableInvites.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212538, Down_20161118212538) + MigrationClient.AddMigration(Up_20161118212538, Down_20161118212538) } func Up_20161118212538(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212549_CreateTableLabelQueryExecutions.go b/server/datastore/mysql/migrations/tables/20161118212549_CreateTableLabelQueryExecutions.go similarity index 87% rename from server/datastore/mysql/migrations/20161118212549_CreateTableLabelQueryExecutions.go rename to server/datastore/mysql/migrations/tables/20161118212549_CreateTableLabelQueryExecutions.go index 2e2971d555..92beba6b89 100644 --- a/server/datastore/mysql/migrations/20161118212549_CreateTableLabelQueryExecutions.go +++ b/server/datastore/mysql/migrations/tables/20161118212549_CreateTableLabelQueryExecutions.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212549, Down_20161118212549) + MigrationClient.AddMigration(Up_20161118212549, Down_20161118212549) } func Up_20161118212549(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212557_CreateTableLabels.go b/server/datastore/mysql/migrations/tables/20161118212557_CreateTableLabels.go similarity index 90% rename from server/datastore/mysql/migrations/20161118212557_CreateTableLabels.go rename to server/datastore/mysql/migrations/tables/20161118212557_CreateTableLabels.go index ba1d4696ea..b800b3409f 100644 --- a/server/datastore/mysql/migrations/20161118212557_CreateTableLabels.go +++ b/server/datastore/mysql/migrations/tables/20161118212557_CreateTableLabels.go @@ -1,15 +1,14 @@ -package migration +package tables import ( "database/sql" "fmt" "github.com/kolide/kolide-ose/server/kolide" - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212557, Down_20161118212557) + MigrationClient.AddMigration(Up_20161118212557, Down_20161118212557) } func Up_20161118212557(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212604_CreateTableOptions.go b/server/datastore/mysql/migrations/tables/20161118212604_CreateTableOptions.go similarity index 85% rename from server/datastore/mysql/migrations/20161118212604_CreateTableOptions.go rename to server/datastore/mysql/migrations/tables/20161118212604_CreateTableOptions.go index 0aea7414af..1b404f34c0 100644 --- a/server/datastore/mysql/migrations/20161118212604_CreateTableOptions.go +++ b/server/datastore/mysql/migrations/tables/20161118212604_CreateTableOptions.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212604, Down_20161118212604) + MigrationClient.AddMigration(Up_20161118212604, Down_20161118212604) } func Up_20161118212604(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212613_CreateTableScheduledQueries.go b/server/datastore/mysql/migrations/tables/20161118212613_CreateTableScheduledQueries.go similarity index 90% rename from server/datastore/mysql/migrations/20161118212613_CreateTableScheduledQueries.go rename to server/datastore/mysql/migrations/tables/20161118212613_CreateTableScheduledQueries.go index 967a15673b..66bc821c08 100644 --- a/server/datastore/mysql/migrations/20161118212613_CreateTableScheduledQueries.go +++ b/server/datastore/mysql/migrations/tables/20161118212613_CreateTableScheduledQueries.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212613, Down_20161118212613) + MigrationClient.AddMigration(Up_20161118212613, Down_20161118212613) } func Up_20161118212613(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212621_CreateTablePackTargets.go b/server/datastore/mysql/migrations/tables/20161118212621_CreateTablePackTargets.go similarity index 83% rename from server/datastore/mysql/migrations/20161118212621_CreateTablePackTargets.go rename to server/datastore/mysql/migrations/tables/20161118212621_CreateTablePackTargets.go index 9e2f72305d..f804346cc7 100644 --- a/server/datastore/mysql/migrations/20161118212621_CreateTablePackTargets.go +++ b/server/datastore/mysql/migrations/tables/20161118212621_CreateTablePackTargets.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212621, Down_20161118212621) + MigrationClient.AddMigration(Up_20161118212621, Down_20161118212621) } func Up_20161118212621(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go b/server/datastore/mysql/migrations/tables/20161118212630_CreateTablePacks.go similarity index 89% rename from server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go rename to server/datastore/mysql/migrations/tables/20161118212630_CreateTablePacks.go index 59d8ddbf43..5fca412327 100644 --- a/server/datastore/mysql/migrations/20161118212630_CreateTablePacks.go +++ b/server/datastore/mysql/migrations/tables/20161118212630_CreateTablePacks.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212630, Down_20161118212630) + MigrationClient.AddMigration(Up_20161118212630, Down_20161118212630) } func Up_20161118212630(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212641_CreateTablePasswordResetRequests.go b/server/datastore/mysql/migrations/tables/20161118212641_CreateTablePasswordResetRequests.go similarity index 87% rename from server/datastore/mysql/migrations/20161118212641_CreateTablePasswordResetRequests.go rename to server/datastore/mysql/migrations/tables/20161118212641_CreateTablePasswordResetRequests.go index c4d796d2dc..05b05638ab 100644 --- a/server/datastore/mysql/migrations/20161118212641_CreateTablePasswordResetRequests.go +++ b/server/datastore/mysql/migrations/tables/20161118212641_CreateTablePasswordResetRequests.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212641, Down_20161118212641) + MigrationClient.AddMigration(Up_20161118212641, Down_20161118212641) } func Up_20161118212641(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212649_CreateTableUsers.go b/server/datastore/mysql/migrations/tables/20161118212649_CreateTableUsers.go similarity index 92% rename from server/datastore/mysql/migrations/20161118212649_CreateTableUsers.go rename to server/datastore/mysql/migrations/tables/20161118212649_CreateTableUsers.go index f381947b9d..95acc1bca6 100644 --- a/server/datastore/mysql/migrations/20161118212649_CreateTableUsers.go +++ b/server/datastore/mysql/migrations/tables/20161118212649_CreateTableUsers.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212649, Down_20161118212649) + MigrationClient.AddMigration(Up_20161118212649, Down_20161118212649) } func Up_20161118212649(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212656_CreateTableSessions.go b/server/datastore/mysql/migrations/tables/20161118212656_CreateTableSessions.go similarity index 86% rename from server/datastore/mysql/migrations/20161118212656_CreateTableSessions.go rename to server/datastore/mysql/migrations/tables/20161118212656_CreateTableSessions.go index 63e4022c3b..5d232d0f86 100644 --- a/server/datastore/mysql/migrations/20161118212656_CreateTableSessions.go +++ b/server/datastore/mysql/migrations/tables/20161118212656_CreateTableSessions.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212656, Down_20161118212656) + MigrationClient.AddMigration(Up_20161118212656, Down_20161118212656) } func Up_20161118212656(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161118212758_CreateTableQueries.go b/server/datastore/mysql/migrations/tables/20161118212758_CreateTableQueries.go similarity index 89% rename from server/datastore/mysql/migrations/20161118212758_CreateTableQueries.go rename to server/datastore/mysql/migrations/tables/20161118212758_CreateTableQueries.go index a0932ad195..22fe175451 100644 --- a/server/datastore/mysql/migrations/20161118212758_CreateTableQueries.go +++ b/server/datastore/mysql/migrations/tables/20161118212758_CreateTableQueries.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161118212758, Down_20161118212758) + MigrationClient.AddMigration(Up_20161118212758, Down_20161118212758) } func Up_20161118212758(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161128234849_CreateTableNetworkInterfaces.go b/server/datastore/mysql/migrations/tables/20161128234849_CreateTableNetworkInterfaces.go similarity index 93% rename from server/datastore/mysql/migrations/20161128234849_CreateTableNetworkInterfaces.go rename to server/datastore/mysql/migrations/tables/20161128234849_CreateTableNetworkInterfaces.go index 8bbea8fd24..7199048c7a 100644 --- a/server/datastore/mysql/migrations/20161128234849_CreateTableNetworkInterfaces.go +++ b/server/datastore/mysql/migrations/tables/20161128234849_CreateTableNetworkInterfaces.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161128234849, Down_20161128234849) + MigrationClient.AddMigration(Up_20161128234849, Down_20161128234849) } func Up_20161128234849(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20161230162221_AddUniqueIndexToPackTargets.go b/server/datastore/mysql/migrations/tables/20161230162221_AddUniqueIndexToPackTargets.go similarity index 80% rename from server/datastore/mysql/migrations/20161230162221_AddUniqueIndexToPackTargets.go rename to server/datastore/mysql/migrations/tables/20161230162221_AddUniqueIndexToPackTargets.go index 4c50934586..fe5d32f4f0 100644 --- a/server/datastore/mysql/migrations/20161230162221_AddUniqueIndexToPackTargets.go +++ b/server/datastore/mysql/migrations/tables/20161230162221_AddUniqueIndexToPackTargets.go @@ -1,13 +1,11 @@ -package migration +package tables import ( "database/sql" - - "github.com/pressly/goose" ) func init() { - goose.AddMigration(Up_20161230162221, Down_20161230162221) + MigrationClient.AddMigration(Up_20161230162221, Down_20161230162221) } func Up_20161230162221(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go b/server/datastore/mysql/migrations/tables/20170104113816_AddSeenTimeToHosts.go similarity index 70% rename from server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go rename to server/datastore/mysql/migrations/tables/20170104113816_AddSeenTimeToHosts.go index f70dbb82b8..ad06fd73b1 100644 --- a/server/datastore/mysql/migrations/20170104113816_AddSeenTimeToHosts.go +++ b/server/datastore/mysql/migrations/tables/20170104113816_AddSeenTimeToHosts.go @@ -1,13 +1,9 @@ -package migration +package tables -import ( - "database/sql" - - "github.com/pressly/goose" -) +import "database/sql" func init() { - goose.AddMigration(Up_20170104113816, Down_20170104113816) + MigrationClient.AddMigration(Up_20170104113816, Down_20170104113816) } func Up_20170104113816(tx *sql.Tx) error { diff --git a/server/datastore/mysql/migrations/tables/migration.go b/server/datastore/mysql/migrations/tables/migration.go new file mode 100644 index 0000000000..5fb35fb4ff --- /dev/null +++ b/server/datastore/mysql/migrations/tables/migration.go @@ -0,0 +1,7 @@ +package tables + +import "github.com/kolide/goose" + +var ( + MigrationClient = goose.New("migration_status_tables", goose.MySqlDialect{}) +) diff --git a/server/datastore/mysql/datastore.go b/server/datastore/mysql/mysql.go similarity index 92% rename from server/datastore/mysql/datastore.go rename to server/datastore/mysql/mysql.go index fa363dfbd3..9eeb6a7c2e 100644 --- a/server/datastore/mysql/datastore.go +++ b/server/datastore/mysql/mysql.go @@ -8,8 +8,9 @@ import ( "github.com/go-kit/kit/log" "github.com/jmoiron/sqlx" "github.com/kolide/kolide-ose/server/config" + "github.com/kolide/kolide-ose/server/datastore/mysql/migrations/data" + "github.com/kolide/kolide-ose/server/datastore/mysql/migrations/tables" "github.com/kolide/kolide-ose/server/kolide" - "github.com/pressly/goose" ) const ( @@ -73,17 +74,20 @@ func (d *Datastore) Name() string { return "mysql" } -// Migrate creates database -func (d *Datastore) Migrate() error { - - goose.SetDialect("mysql") - - if err := goose.Run("up", d.db.DB, "."); err != nil { +func (d *Datastore) MigrateTables() error { + if err := tables.MigrationClient.Up(d.db.DB, ""); err != nil { return err } return nil +} +func (d *Datastore) MigrateData() error { + if err := data.MigrationClient.Up(d.db.DB, ""); err != nil { + return err + } + + return nil } // Initialize preload data needed by the application diff --git a/server/datastore/mysql_test.go b/server/datastore/mysql_test.go index 222ec0bbfd..a6e69a4def 100644 --- a/server/datastore/mysql_test.go +++ b/server/datastore/mysql_test.go @@ -9,7 +9,6 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/kolide/kolide-ose/server/config" "github.com/kolide/kolide-ose/server/datastore/mysql" - _ "github.com/kolide/kolide-ose/server/datastore/mysql/migrations" "github.com/stretchr/testify/require" ) @@ -49,7 +48,7 @@ func TestMySQL(t *testing.T) { t.Run(functionName(f), func(t *testing.T) { defer func() { require.Nil(t, ds.Drop()) }() - require.Nil(t, ds.Migrate()) + require.Nil(t, ds.MigrateTables()) f(t, ds) }) diff --git a/server/kolide/datastore.go b/server/kolide/datastore.go index e16509477b..bee7ce7dbf 100644 --- a/server/kolide/datastore.go +++ b/server/kolide/datastore.go @@ -16,7 +16,10 @@ type Datastore interface { OptionStore Name() string Drop() error - Migrate() error + // MigrateTables creates and migrates the table schemas + MigrateTables() error + // MigrateData populates built-in data + MigrateData() error } // NotFoundError is returned when the datastore resource cannot be found. diff --git a/server/service/endpoint_test.go b/server/service/endpoint_test.go index 756a4cab9f..4f3e8176bb 100644 --- a/server/service/endpoint_test.go +++ b/server/service/endpoint_test.go @@ -18,6 +18,7 @@ import ( "github.com/kolide/kolide-ose/server/config" "github.com/kolide/kolide-ose/server/datastore/inmem" "github.com/kolide/kolide-ose/server/kolide" + "github.com/stretchr/testify/require" ) type testResource struct { @@ -29,8 +30,11 @@ type testResource struct { func setupEndpointTest(t *testing.T) *testResource { test := &testResource{} - jwtKey := "CHANGEME" - test.ds, _ = inmem.New(config.TestConfig()) + var err error + test.ds, err = inmem.New(config.TestConfig()) + require.Nil(t, err) + require.Nil(t, test.ds.MigrateData()) + devOrgInfo := &kolide.AppConfig{ OrgName: "Kolide", OrgLogoURL: "http://foo.bar/image.png", @@ -45,6 +49,7 @@ func setupEndpointTest(t *testing.T) *testResource { createTestUsers(t, test.ds) logger := kitlog.NewLogfmtLogger(os.Stdout) + jwtKey := "CHANGEME" opts := []kithttp.ServerOption{ kithttp.ServerBefore(setRequestsContexts(svc, jwtKey)), kithttp.ServerErrorLogger(logger), diff --git a/server/service/service_appconfig_test.go b/server/service/service_appconfig_test.go index 3cba3cb9ac..ca038d4a31 100644 --- a/server/service/service_appconfig_test.go +++ b/server/service/service_appconfig_test.go @@ -14,6 +14,8 @@ import ( func TestCreateAppConfig(t *testing.T) { ds, err := inmem.New(config.TestConfig()) require.Nil(t, err) + require.Nil(t, ds.MigrateData()) + svc, err := newTestService(ds, nil) require.Nil(t, err) var appConfigTests = []struct { diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 514b248288..7a5d22ebc0 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -377,7 +377,8 @@ func TestLabelQueries(t *testing.T) { func TestGetClientConfig(t *testing.T) { ds, err := inmem.New(config.TestConfig()) - assert.Nil(t, err) + require.Nil(t, err) + require.Nil(t, ds.MigrateData()) mockClock := clock.NewMockClock()