From 29b9a7fe88c586c1c4b66baf510169ce62755530 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 4 Aug 2023 13:24:36 -0300 Subject: [PATCH] account for NULL values in scheduled_query columns in data migration (#13142) Prior to 4.35.0, some rows in the scheduled_query table might have a `NULL` value due to a race condition with database replicas and the way `ds.EnsureGlobalPack` and `ApplyPackSpecs` work together. This is no longer the case, but some databases are left in weird states, which were not accounted by this migration. Chaning the migration in-place because that's the approach we took in previous migrations with similar problems. --- changes/13137-migration-fix | 1 + .../20230721161508_QueriesDataMigrator.go | 32 +++++++++---------- ...20230721161508_QueriesDataMigrator_test.go | 9 ++++-- 3 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 changes/13137-migration-fix diff --git a/changes/13137-migration-fix b/changes/13137-migration-fix new file mode 100644 index 0000000000..71a83c969e --- /dev/null +++ b/changes/13137-migration-fix @@ -0,0 +1 @@ +* Fixed a migration to account for columns with NULL values as a result of either creating schedules via the API without providing all values or by a race condition with database replicas. diff --git a/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator.go b/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator.go index 1060b64550..bc8481b02d 100644 --- a/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator.go +++ b/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator.go @@ -50,14 +50,14 @@ func _20230719152138_migrate_global_packs(tx *sql.Tx) error { q.author_id, q.saved, q.observer_can_run, - sq.id AS scheduled_query_id, - sq.interval AS scheduled_query_interval, - sq.snapshot AS scheduled_query_snapshot, - sq.removed AS scheduled_query_removed, - sq.platform AS scheduled_query_platform, - sq.version AS scheduled_query_version, - sq.created_at AS scheduled_created_at, - p.pack_type AS pack_type + sq.id AS scheduled_query_id, + sq.interval AS scheduled_query_interval, + sq.snapshot AS scheduled_query_snapshot, + sq.removed AS scheduled_query_removed, + COALESCE(sq.platform, '') AS scheduled_query_platform, + COALESCE(sq.version, '') AS scheduled_query_version, + sq.created_at AS scheduled_created_at, + p.pack_type AS pack_type FROM queries q INNER JOIN scheduled_queries sq ON q.name = sq.query_name INNER JOIN packs p ON sq.pack_id = p.id @@ -161,14 +161,14 @@ func _20230719152138_migrate_team_packs(tx *sql.Tx) error { q.author_id, q.saved, q.observer_can_run, - sq.id AS scheduled_query_id, - sq.interval AS scheduled_query_interval, - sq.snapshot AS scheduled_query_snapshot, - sq.removed AS scheduled_query_removed, - sq.platform AS scheduled_query_platform, - sq.version AS scheduled_query_version, - sq.created_at AS scheduled_created_at, - p.pack_type AS pack_type + sq.id AS scheduled_query_id, + sq.interval AS scheduled_query_interval, + sq.snapshot AS scheduled_query_snapshot, + sq.removed AS scheduled_query_removed, + COALESCE(sq.platform, '') AS scheduled_query_platform, + COALESCE(sq.version, '') AS scheduled_query_version, + sq.created_at AS scheduled_created_at, + p.pack_type AS pack_type FROM queries q INNER JOIN scheduled_queries sq ON q.team_id IS NULL AND q.name = sq.query_name INNER JOIN packs p ON sq.pack_id = p.id diff --git a/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator_test.go b/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator_test.go index 7dec3b414c..afc433bfe8 100644 --- a/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator_test.go +++ b/server/datastore/mysql/migrations/tables/20230721161508_QueriesDataMigrator_test.go @@ -55,12 +55,15 @@ func TestUp_20230721161508(t *testing.T) { -- Global pack (7,'2023-07-21 20:34:46','2023-07-21 20:34:46',1,2,86400,1,0,'','',NULL,'per_query_perf','per_query_perf','',NULL,''), - (8,'2023-07-21 20:34:51','2023-07-21 20:34:51',1,2,86400,1,0,'','',NULL,'per_query_perf','per_query_perf-1','',NULL,''), + -- NULL platform + (8,'2023-07-21 20:34:51','2023-07-21 20:34:51',1,2,86400,1,0,NULL,'',NULL,'per_query_perf','per_query_perf-1','',NULL,''), -- Team-1 pack (9,'2023-07-21 20:36:08','2023-07-21 20:36:08',3,6,86400,1,0,'','',NULL,'User 1 Query','User 1 Query','',NULL,''), - (10,'2023-07-21 20:36:13','2023-07-21 20:36:13',3,6,86400,1,0,'','',NULL,'User 1 Query','User 1 Query-1','',NULL,''), - (11,'2023-07-21 20:36:25','2023-07-21 20:36:25',3,2,86400,1,0,'','',NULL,'per_query_perf','per_query_perf','',NULL,''), + -- NULL version + (10,'2023-07-21 20:36:13','2023-07-21 20:36:13',3,6,86400,1,0,'',NULL,NULL,'User 1 Query','User 1 Query-1','',NULL,''), + -- NULL platform + (11,'2023-07-21 20:36:25','2023-07-21 20:36:25',3,2,86400,1,0,NULL,'',NULL,'per_query_perf','per_query_perf','',NULL,''), -- Team-2 pack (12,'2023-07-21 20:36:50','2023-07-21 20:36:50',4,5,86400,1,0,'','',NULL,'backup_tool_perf','backup_tool_perf','',NULL,'');