From dc87ac2271e5ab1ae0223346e9c0ae8a25d36be5 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Thu, 14 Mar 2024 18:08:19 -0300 Subject: [PATCH 01/29] add schema for DDM (#17636) #17405 --- .../tables/20240314150853_AddDDMTables.go | 170 ++++++++++++++++++ server/datastore/mysql/schema.sql | 94 +++++++++- 2 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go diff --git a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go new file mode 100644 index 0000000000..5d15da08b2 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go @@ -0,0 +1,170 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240314150853, Down_20240314150853) +} + +func Up_20240314150853(tx *sql.Tx) error { + _, err := tx.Exec(` +CREATE TABLE mdm_apple_declaration_types ( + declaration_type varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + PRIMARY KEY (declaration_type) +) + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` + INSERT INTO mdm_apple_declaration_types + VALUES ('com.apple.configuration'), ('com.apple.activation') + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` +CREATE TABLE mdm_apple_declarations ( + -- declaration_uuid is used as the primary key of the declaration + declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + + -- team_id references the team that owns this declaration + team_id int(10) unsigned NOT NULL DEFAULT '0', + + -- identifier is the "Identifier" field in the declaration, surfaced for convenience. + identifier varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- name is the name of the declaration + name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- declaration_type is the type of the declaration + declaration_type varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- declaration contains a JSON blob with the declaration contents + declaration json NOT NULL, + + -- md5_checksum is an MD5 checksum of the declaration, in binary form + md5_checksum binary(16) NOT NULL, + + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + uploaded_at timestamp NULL DEFAULT NULL, + + PRIMARY KEY (declaration_uuid), + UNIQUE KEY idx_mdm_apple_declaration_team_identifier (team_id, identifier), + UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name), + CONSTRAINT mdm_apple_declaration_declaration_type FOREIGN KEY (declaration_type) REFERENCES mdm_apple_declaration_types (declaration_type) ON DELETE CASCADE +) + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` +CREATE TABLE mdm_declaration_labels ( + -- id is used as the primary key of this table + id int(10) unsigned NOT NULL AUTO_INCREMENT, + + -- declaration_uuid references a declaration + declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + + + -- label name is stored here because we need to list the labels in the UI + -- even if it has been deleted from the labels table. + label_name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- label id is nullable in case it gets deleted from the labels table. + -- A row in this table with label_id = null indicates the "broken" state + label_id int(10) unsigned DEFAULT NULL, + + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + uploaded_at timestamp NULL DEFAULT NULL, + + PRIMARY KEY (id), + UNIQUE KEY idx_mdm_declaration_labels_label_name (declaration_uuid, label_name), + KEY label_id (label_id), + CONSTRAINT mdm_declaration_labels_ibfk_1 FOREIGN KEY (declaration_uuid) REFERENCES mdm_apple_declarations (declaration_uuid) ON DELETE CASCADE, + CONSTRAINT mdm_declaration_labels_ibfk_3 FOREIGN KEY (label_id) REFERENCES labels (id) ON DELETE SET NULL +) + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` +CREATE TABLE mdm_apple_declaration_activation_references ( + -- declaration_uuid is the declaration that contains the references + declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + + -- reference is the declaration_uuid of another declaration + reference varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + + PRIMARY KEY (declaration_uuid, reference), + CONSTRAINT FOREIGN KEY (declaration_uuid) REFERENCES mdm_apple_declarations (declaration_uuid) ON UPDATE CASCADE, + CONSTRAINT FOREIGN KEY (reference) REFERENCES mdm_apple_declarations (declaration_uuid) ON UPDATE CASCADE +) + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` +CREATE TABLE host_mdm_apple_declarations ( + -- host_uuid references a host in the hosts table + host_uuid varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- status represents the status of the declaration in the host + status varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- operation_type is used to signal if the declaration is being added, removed, etc + operation_type varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + + -- detail contains any messages or errors from the protocol or Fleet + detail text COLLATE utf8mb4_unicode_ci, + + -- md5_checksum of the currently implemented declaration + md5_checksum binary(16) NOT NULL, + + -- declaration_uuid references the declaration assigned to the host's team + declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + + PRIMARY KEY (host_uuid, declaration_uuid), + KEY status (status), + KEY operation_type (operation_type), + CONSTRAINT host_mdm_apple_declarations_ibfk_1 FOREIGN KEY (status) REFERENCES mdm_delivery_status (status) ON UPDATE CASCADE, + CONSTRAINT host_mdm_apple_declarations_ibfk_2 FOREIGN KEY (operation_type) REFERENCES mdm_operation_types (operation_type) ON UPDATE CASCADE +) + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + _, err = tx.Exec(` +-- this view is used to pre-compute checksums on a team basis so they can be +-- used as the ServerToken to signal devices if they should fetch declarations. +CREATE VIEW team_declaration_checksum_view AS +SELECT + team_id, + -- since GROUP_CONCAT can be truncated, we calculate the checksum based on + -- the latest updated items and the total number of items + MD5(COUNT(*) + GROUP_CONCAT(HEX(md5_checksum) ORDER BY uploaded_at DESC SEPARATOR '')) AS md5_checksum, + MAX(created_at) AS latest_created_timestamp +FROM + mdm_apple_declarations +GROUP BY + team_id + `) + if err != nil { + return fmt.Errorf("%w", err) + } + + return nil +} + +func Down_20240314150853(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a35608e669..7aa5539edc 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -305,6 +305,22 @@ CREATE TABLE `host_mdm_apple_bootstrap_packages` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `host_mdm_apple_declarations` ( + `host_uuid` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `status` varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `operation_type` varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `detail` text COLLATE utf8mb4_unicode_ci, + `md5_checksum` binary(16) NOT NULL, + `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + PRIMARY KEY (`host_uuid`,`declaration_uuid`), + KEY `status` (`status`), + KEY `operation_type` (`operation_type`), + CONSTRAINT `host_mdm_apple_declarations_ibfk_1` FOREIGN KEY (`status`) REFERENCES `mdm_delivery_status` (`status`) ON UPDATE CASCADE, + CONSTRAINT `host_mdm_apple_declarations_ibfk_2` FOREIGN KEY (`operation_type`) REFERENCES `mdm_operation_types` (`operation_type`) ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `host_mdm_apple_profiles` ( `profile_identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `host_uuid` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, @@ -631,6 +647,44 @@ CREATE TABLE `mdm_apple_configuration_profiles` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mdm_apple_declaration_activation_references` ( + `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `reference` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + PRIMARY KEY (`declaration_uuid`,`reference`), + KEY `reference` (`reference`), + CONSTRAINT `mdm_apple_declaration_activation_references_ibfk_1` FOREIGN KEY (`declaration_uuid`) REFERENCES `mdm_apple_declarations` (`declaration_uuid`) ON UPDATE CASCADE, + CONSTRAINT `mdm_apple_declaration_activation_references_ibfk_2` FOREIGN KEY (`reference`) REFERENCES `mdm_apple_declarations` (`declaration_uuid`) ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mdm_apple_declaration_types` ( + `declaration_type` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + PRIMARY KEY (`declaration_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `mdm_apple_declaration_types` VALUES ('com.apple.activation'),('com.apple.configuration'); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mdm_apple_declarations` ( + `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `team_id` int(10) unsigned NOT NULL DEFAULT '0', + `identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `declaration_type` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + `declaration` json NOT NULL, + `md5_checksum` binary(16) NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `uploaded_at` timestamp NULL DEFAULT NULL, + PRIMARY KEY (`declaration_uuid`), + UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`), + UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`), + KEY `mdm_apple_declaration_declaration_type` (`declaration_type`), + CONSTRAINT `mdm_apple_declaration_declaration_type` FOREIGN KEY (`declaration_type`) REFERENCES `mdm_apple_declaration_types` (`declaration_type`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_apple_default_setup_assistants` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `team_id` int(10) unsigned DEFAULT NULL, @@ -708,6 +762,22 @@ CREATE TABLE `mdm_configuration_profile_labels` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mdm_declaration_labels` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `label_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `label_id` int(10) unsigned DEFAULT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `uploaded_at` timestamp NULL DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_mdm_declaration_labels_label_name` (`declaration_uuid`,`label_name`), + KEY `label_id` (`label_id`), + CONSTRAINT `mdm_declaration_labels_ibfk_1` FOREIGN KEY (`declaration_uuid`) REFERENCES `mdm_apple_declarations` (`declaration_uuid`) ON DELETE CASCADE, + CONSTRAINT `mdm_declaration_labels_ibfk_3` FOREIGN KEY (`label_id`) REFERENCES `labels` (`id`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_delivery_status` ( `status` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL, PRIMARY KEY (`status`) @@ -779,9 +849,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=257 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=258 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314150853,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1400,6 +1470,13 @@ CREATE TABLE `statistics` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE VIEW `team_declaration_checksum_view` AS SELECT + 1 AS `team_id`, + 1 AS `md5_checksum`, + 1 AS `latest_created_timestamp`*/; +SET character_set_client = @saved_cs_client; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `teams` ( @@ -1569,3 +1646,16 @@ CREATE TABLE `wstep_serials` ( /*!50001 SET character_set_client = @saved_cs_client */; /*!50001 SET character_set_results = @saved_cs_results */; /*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP VIEW IF EXISTS `team_declaration_checksum_view`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = utf8mb4 */; +/*!50001 SET character_set_results = utf8mb4 */; +/*!50001 SET collation_connection = utf8mb4_unicode_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`%` SQL SECURITY DEFINER */ +/*!50001 VIEW `team_declaration_checksum_view` AS select `mdm_apple_declarations`.`team_id` AS `team_id`,md5((count(0) + group_concat(hex(`mdm_apple_declarations`.`md5_checksum`) order by `mdm_apple_declarations`.`uploaded_at` DESC separator ''))) AS `md5_checksum`,max(`mdm_apple_declarations`.`created_at`) AS `latest_created_timestamp` from `mdm_apple_declarations` group by `mdm_apple_declarations`.`team_id` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; From d261762e8342b52be41cdc1cce021e52d062b1a9 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 15 Mar 2024 10:51:47 -0300 Subject: [PATCH 02/29] add base types and handlers for DDM (#17657) This includes the base types and the request handlers to reduce the chances of conflicts. --- server/fleet/apple_mdm.go | 77 +++++++++++++++++++++++++++++++++++++ server/service/apple_mdm.go | 22 ++++++++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 02a0b65f47..c6a0061694 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -532,3 +532,80 @@ type SCEPIdentityAssociation struct { EnrollReference string `db:"enroll_reference"` RenewCommandUUID string `db:"renew_command_uuid"` } + +// MDMAppleDeclarationType is the type for the supported declaration types. +type MDMAppleDeclarationType string + +const ( + // MDMAppleConfigurationDeclaration is the value for [configuration][1] declarations + // + // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3813088 + MDMAppleDeclarativeConfiguration MDMAppleDeclarationType = "com.apple.configuration" + + // MDMAppleActivationConfiguration is the value for [activation][1] declarations + // + // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3829708 + MDMAppleDeclarativeActivation MDMAppleDeclarationType = "com.apple.activation" +) + +// MDMAppleDeclaration represents a DDM JSON declaration. +type MDMAppleDeclaration struct { + // DeclarationUUID is the unique identifier of the declaration in + // Fleet. Since we use the same endpoints for declarations and profiles: + // - This is marshalled as profile_uuid + // - The value has a prefix (TODO: @jahzielv to determine and document this) + DeclarationUUID string `db:"declaration_uuid" json:"profile_uuid"` + + // TeamID is the id of the team with which the declaration is associated. A nil team id + // represents a declaration that is not associated with any team. + TeamID *uint `db:"team_id" json:"team_id"` + + // Identifier corresponds to the "Identifier" key of the associated declaration. + // Fleet requires that Identifier must be unique in combination with the Name and TeamID. + Identifier string `db:"identifier" json:"identifier"` + + // Name corresponds to the file name of the associated JSON declaration payload. + // Fleet requires that Name must be unique in combination with the Identifier and TeamID. + Name string `db:"name" json:"name"` + + // DeclarationType is the type of the declaration, at the moment we + // only support configurations and activations. + DeclarationType MDMAppleDeclarationType `db:"declaration_type"` + + // Declaration is the raw JSON content of the declaration + Declaration json.RawMessage `db:"declaration" json:"-"` + + // MD5Checksum is a checsum of the JSON contents + MD5Checksum string `db:"md5_checksum" json:"-"` + + CreatedAt time.Time `db:"created_at" json:"created_at"` + UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"` +} + +// MDMAppleHostDeclaration represents the state of a declaration on a host +type MDMAppleHostDeclaration struct { + // HostUUID is the uuid of the host affected by this declaration + HostUUID string `db:"host_uuid" json:"-"` + + // DeclarationUUID is the unique identifier of the declaration in + // Fleet. Since we use the same endpoints for declarations and profiles: + // - This is marshalled as profile_uuid + // - The value has a prefix (TODO: @jahzielv to determine and document this) + DeclarationUUID string `db:"declaration_uuid" json:"profile_uuid"` + + // Name corresponds to the file name of the associated JSON declaration payload. + Name string `db:"name" json:"name"` + + // Identifier corresponds to the "Identifier" key of the associated declaration. + Identifier string `db:"identifier" json:"-"` + + // Status represent the current state of the declaration, as known by the Fleet server. + Status *MDMDeliveryStatus `db:"status" json:"status"` + + // Operation type represents the operation being performed. + OperationType MDMOperationType `db:"operation_type" json:"operation_type"` + + // Detail contains any messages that must be surfaced to the user, + // either by the MDM protocol or the Fleet server. + Detail string `db:"detail" json:"detail"` +} diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index de9e26403d..3c67f76b4f 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2354,8 +2354,26 @@ func (svc *MDMAppleCheckinAndCommandService) UserAuthenticate(*mdm.Request, *mdm // This method is executed after the request has been handled by nanomdm. // // [1]: https://developer.apple.com/documentation/devicemanagement/declarative_management_checkin -func (svc *MDMAppleCheckinAndCommandService) DeclarativeManagement(*mdm.Request, *mdm.DeclarativeManagement) ([]byte, error) { - return nil, nil +func (svc *MDMAppleCheckinAndCommandService) DeclarativeManagement(r *mdm.Request, cmd *mdm.DeclarativeManagement) ([]byte, error) { + switch cmd.Endpoint { + case "tokens": + return nil, nil + case "declaration-items": + return nil, nil + case "status": + return nil, nil + default: + parts := strings.Split(cmd.Endpoint, "/") + if len(parts) != 3 { + return nil, ctxerr.New(r.Context, "unrecognized DDM endpoint") + } + + declarationType := parts[1] + declarationIdentifier := parts[2] + fmt.Println(declarationType, declarationIdentifier) + + return nil, nil + } } // CommandAndReportResults handles MDM [Commands and Queries][1]. From 48b31a02ae7771c1e96ca725641491daba1a31bd Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Fri, 15 Mar 2024 13:20:15 -0500 Subject: [PATCH 03/29] Add DDM service struct, basic handlers, and test client (#17671) --- cmd/fleet/serve.go | 3 + pkg/mdm/mdmtest/apple.go | 17 ++++++ server/service/apple_mdm.go | 84 +++++++++++++++++++------- server/service/handler.go | 6 +- server/service/integration_mdm_test.go | 24 ++++++++ server/service/testing_utils.go | 4 ++ 6 files changed, 115 insertions(+), 23 deletions(-) diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 729cc31801..22f5802ff8 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -462,6 +462,7 @@ the way that the Fleet server works. mdmStorage *mysql.NanoMDMStorage mdmPushService push.Pusher mdmCheckinAndCommandService *service.MDMAppleCheckinAndCommandService + ddmService *service.MDMAppleDDMService mdmPushCertTopic string ) @@ -546,6 +547,7 @@ the way that the Fleet server works. } commander := apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPushService) mdmCheckinAndCommandService = service.NewMDMAppleCheckinAndCommandService(ds, commander, logger) + ddmService = service.NewMDMAppleDDMService(ds, logger) appCfg.MDM.EnabledAndConfigured = true } @@ -870,6 +872,7 @@ the way that the Fleet server works. scepStorage, logger, mdmCheckinAndCommandService, + ddmService, ); err != nil { initFatal(err, "setup mdm apple services") } diff --git a/pkg/mdm/mdmtest/apple.go b/pkg/mdm/mdmtest/apple.go index 0f15c7c453..c9c0e54273 100644 --- a/pkg/mdm/mdmtest/apple.go +++ b/pkg/mdm/mdmtest/apple.go @@ -386,6 +386,23 @@ func (c *TestAppleMDMClient) TokenUpdate() error { return err } +// DeclarativeManagement sends a DeclarativeManagement checkin request to the server. +// +// The endpoint argument is used as the value for the `Endpoint` key in the request payload. +// +// For more details check https://developer.apple.com/documentation/devicemanagement/declarativemanagementrequest +func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) error { + payload := map[string]any{ + "MessageType": "DeclarativeManagement", + "UDID": c.UUID, + "Topic": "com.apple.mgmt.External." + c.UUID, + "EnrollmentID": "testenrollmentid-" + c.UUID, + "Endpoint": endpoint, + } + _, err := c.request("application/x-apple-aspen-mdm-checkin", payload) + return err +} + // Checkout sends the CheckOut message to the MDM server. func (c *TestAppleMDMClient) Checkout() error { payload := map[string]any{ diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 3c67f76b4f..b8a70eda47 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2225,7 +2225,6 @@ func (svc *MDMAppleCheckinAndCommandService) Authenticate(r *mdm.Request, m *mdm InstalledFromDEP: info.DEPAssignedToFleet, MDMPlatform: fleet.MDMPlatformApple, }) - } // TokenUpdate handles MDM [TokenUpdate][1] requests. @@ -2354,26 +2353,9 @@ func (svc *MDMAppleCheckinAndCommandService) UserAuthenticate(*mdm.Request, *mdm // This method is executed after the request has been handled by nanomdm. // // [1]: https://developer.apple.com/documentation/devicemanagement/declarative_management_checkin -func (svc *MDMAppleCheckinAndCommandService) DeclarativeManagement(r *mdm.Request, cmd *mdm.DeclarativeManagement) ([]byte, error) { - switch cmd.Endpoint { - case "tokens": - return nil, nil - case "declaration-items": - return nil, nil - case "status": - return nil, nil - default: - parts := strings.Split(cmd.Endpoint, "/") - if len(parts) != 3 { - return nil, ctxerr.New(r.Context, "unrecognized DDM endpoint") - } - - declarationType := parts[1] - declarationIdentifier := parts[2] - fmt.Println(declarationType, declarationIdentifier) - - return nil, nil - } +func (svc *MDMAppleCheckinAndCommandService) DeclarativeManagement(r *mdm.Request, dm *mdm.DeclarativeManagement) ([]byte, error) { + // DeclarativeManagement is handled by the MDMAppleDDMService. + return nil, nil } // CommandAndReportResults handles MDM [Commands and Queries][1]. @@ -2964,3 +2946,63 @@ func RenewSCEPCertificates( return nil } + +// MDMAppleDDMService is the service that handles MDM [DeclarativeManagement][1] requests. +// +// [1]: https://developer.apple.com/documentation/devicemanagement/declarative_management_checkin +type MDMAppleDDMService struct { + ds fleet.Datastore + logger kitlog.Logger +} + +func NewMDMAppleDDMService(ds fleet.Datastore, logger kitlog.Logger) *MDMAppleDDMService { + return &MDMAppleDDMService{ + ds: ds, + logger: logger, + } +} + +// DeclarativeManagement handles MDM [DeclarativeManagement][1] requests. +// +// This method is when the request has been handled by nanomdm. +// +// [1]: https://developer.apple.com/documentation/devicemanagement/declarative_management_checkin +func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.DeclarativeManagement) ([]byte, error) { + if dm == nil { + level.Debug(svc.logger).Log("msg", "ddm request received with nil payload") + return nil, nil + } + level.Debug(svc.logger).Log("msg", "ddm request received", "endpoint", dm.Endpoint) + + switch { + case dm.Endpoint == "tokens": + // TODO(sarah): handle tokens + level.Debug(svc.logger).Log("msg", "received tokens request") + return nil, nil + + case dm.Endpoint == "declaration-items": + // TODO(sarah): handle declaration-items + level.Debug(svc.logger).Log("msg", "received declaration-items request") + return nil, nil + + case dm.Endpoint == "status": + // TODO(roberto): handle status + level.Debug(svc.logger).Log("msg", "received status request") + return nil, nil + + case strings.HasPrefix(dm.Endpoint, "declarations"): + // TODO(sarah): handle declarations + level.Debug(svc.logger).Log("msg", "received declarations request") + parts := strings.Split(dm.Endpoint, "/") + if len(parts) != 3 { + return nil, ctxerr.New(r.Context, "unrecognized declarations endpoint") + } + declarationType := parts[1] + declarationIdentifier := parts[2] + level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", declarationType, "identifier", declarationIdentifier) + return nil, nil + + default: + return nil, ctxerr.New(r.Context, "unrecognized ddm endpoint") + } +} diff --git a/server/service/handler.go b/server/service/handler.go index db3208a537..a67cf5cc8f 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -1003,6 +1003,7 @@ func RegisterAppleMDMProtocolServices( scepStorage scep_depot.Depot, logger kitlog.Logger, checkinAndCommandService nanomdm_service.CheckinAndCommandService, + ddmService nanomdm_service.DeclarativeManagement, ) error { scepCACerts, scepCAKey, err := scepStorage.CA([]byte{}) if err != nil { @@ -1011,7 +1012,7 @@ func RegisterAppleMDMProtocolServices( if err := registerSCEP(mux, scepConfig, scepCACerts[0], scepCAKey, scepStorage, logger); err != nil { return fmt.Errorf("scep: %w", err) } - if err := registerMDM(mux, scepCACerts[0], mdmStorage, checkinAndCommandService, logger); err != nil { + if err := registerMDM(mux, scepCACerts[0], mdmStorage, checkinAndCommandService, ddmService, logger); err != nil { return fmt.Errorf("mdm: %w", err) } return nil @@ -1085,6 +1086,7 @@ func registerMDM( scepCACert *x509.Certificate, mdmStorage nanomdm_storage.AllStorage, checkinAndCommandService nanomdm_service.CheckinAndCommandService, + ddmService nanomdm_service.DeclarativeManagement, logger kitlog.Logger, ) error { certVerifier, err := certverify.NewPoolVerifier( @@ -1104,7 +1106,7 @@ func registerMDM( // enrollments and updates the Fleet hosts table accordingly with the UDID and serial number of // the device. // 5. Run actual MDM service operation (checkin handler or command and results handler). - coreMDMService := nanomdm.New(mdmStorage, nanomdm.WithLogger(mdmLogger)) + coreMDMService := nanomdm.New(mdmStorage, nanomdm.WithLogger(mdmLogger), nanomdm.WithDeclarativeManagement(ddmService)) // NOTE: it is critical that the coreMDMService runs first, as the first // service in the multi-service feature is run to completion _before_ running // the other ones in parallel. This way, subsequent services have access to diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index aaa2b73dc0..4aca6ea831 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -12503,3 +12503,27 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } + +// TODO(sarah): Build out this test +func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { + t := s.T() + mdmDevice := mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{ + SCEPChallenge: s.fleetCfg.MDM.AppleSCEPChallenge, + SCEPURL: s.server.URL + apple_mdm.SCEPPath, + MDMURL: s.server.URL + apple_mdm.MDMPath, + }) + err := mdmDevice.Enroll() + require.NoError(t, err) + + err = mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + + err = mdmDevice.DeclarativeManagement("declaration-items") + require.NoError(t, err) + + err = mdmDevice.DeclarativeManagement("status") + require.NoError(t, err) + + err = mdmDevice.DeclarativeManagement("declarations/foo/bar") + require.NoError(t, err) +} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 0ceadc49aa..f6895dc4de 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -339,6 +339,10 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServ commander: apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPusher), logger: kitlog.NewNopLogger(), }, + &MDMAppleDDMService{ + ds: ds, + logger: logger, + }, ) require.NoError(t, err) } From f5cf1566532369008b510bb34ed60f0eea456298 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 15 Mar 2024 17:10:48 -0300 Subject: [PATCH 04/29] improve mdmtest package to handle any kind of command (#17673) it delegates any extra unmarshaling to the caller. We might consider building our own types in the future instead of relying on micromdm, but these are used only for tests right now. --- pkg/mdm/mdmtest/apple.go | 12 +++---- server/service/integration_mdm_test.go | 47 +++++++++++++++++--------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pkg/mdm/mdmtest/apple.go b/pkg/mdm/mdmtest/apple.go index c9c0e54273..b213218980 100644 --- a/pkg/mdm/mdmtest/apple.go +++ b/pkg/mdm/mdmtest/apple.go @@ -31,7 +31,6 @@ import ( httptransport "github.com/go-kit/kit/transport/http" "github.com/google/uuid" "github.com/groob/plist" - micromdm "github.com/micromdm/micromdm/mdm/mdm" "go.mozilla.org/pkcs7" ) @@ -421,7 +420,7 @@ func (c *TestAppleMDMClient) Checkout() error { // receive commands. The server can signal back with either a command to run // or an empty (nil, nil) response body to end the communication // (i.e. no commands to run). -func (c *TestAppleMDMClient) Idle() (*micromdm.CommandPayload, error) { +func (c *TestAppleMDMClient) Idle() (*mdm.Command, error) { payload := map[string]any{ "Status": "Idle", "Topic": "com.apple.mgmt.External." + c.UUID, @@ -437,7 +436,7 @@ func (c *TestAppleMDMClient) Idle() (*micromdm.CommandPayload, error) { // The server can signal back with either a command to run // or an empty (nil, nil) response body to end the communication // (i.e. no commands to run). -func (c *TestAppleMDMClient) Acknowledge(cmdUUID string) (*micromdm.CommandPayload, error) { +func (c *TestAppleMDMClient) Acknowledge(cmdUUID string) (*mdm.Command, error) { payload := map[string]any{ "Status": "Acknowledged", "Topic": "com.apple.mgmt.External." + c.UUID, @@ -490,7 +489,7 @@ func (c *TestAppleMDMClient) GetBootstrapToken() ([]byte, error) { // The server can signal back with either a command to run // or an empty (nil, nil) response body to end the communication // (i.e. no commands to run). -func (c *TestAppleMDMClient) Err(cmdUUID string, errChain []mdm.ErrorChain) (*micromdm.CommandPayload, error) { +func (c *TestAppleMDMClient) Err(cmdUUID string, errChain []mdm.ErrorChain) (*mdm.Command, error) { payload := map[string]any{ "Status": "Error", "Topic": "com.apple.mgmt.External." + c.UUID, @@ -502,7 +501,7 @@ func (c *TestAppleMDMClient) Err(cmdUUID string, errChain []mdm.ErrorChain) (*mi return c.sendAndDecodeCommandResponse(payload) } -func (c *TestAppleMDMClient) sendAndDecodeCommandResponse(payload map[string]any) (*micromdm.CommandPayload, error) { +func (c *TestAppleMDMClient) sendAndDecodeCommandResponse(payload map[string]any) (*mdm.Command, error) { res, err := c.request("", payload) if err != nil { return nil, fmt.Errorf("request error: %w", err) @@ -527,11 +526,12 @@ func (c *TestAppleMDMClient) sendAndDecodeCommandResponse(payload map[string]any if err != nil { return nil, fmt.Errorf("decode command: %w", err) } - var p micromdm.CommandPayload + var p mdm.Command err = plist.Unmarshal(cmd.Raw, &p) if err != nil { return nil, fmt.Errorf("unmarshal command payload: %w", err) } + p.Raw = cmd.Raw return &p, nil } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 4aca6ea831..782ee6dc14 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -1248,7 +1248,7 @@ func (s *integrationMDMTestSuite) TestWindowsProfileRetries() { } func checkNextPayloads(t *testing.T, mdmDevice *mdmtest.TestAppleMDMClient, forceDeviceErr bool) ([][]byte, []string) { - var cmd *micromdm.CommandPayload + var cmd *mdm.Command var err error installs := [][]byte{} removes := []string{} @@ -1273,11 +1273,13 @@ func checkNextPayloads(t *testing.T, mdmDevice *mdmtest.TestAppleMDMClient, forc break } + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) switch cmd.Command.RequestType { case "InstallProfile": - installs = append(installs, cmd.Command.InstallProfile.Payload) + installs = append(installs, fullCmd.Command.InstallProfile.Payload) case "RemoveProfile": - removes = append(removes, cmd.Command.RemoveProfile.Identifier) + removes = append(removes, fullCmd.Command.RemoveProfile.Identifier) } } @@ -2008,13 +2010,12 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() { // run the worker to assign configuration profiles s.awaitTriggerProfileSchedule(t) - var fleetdCmd, installProfileCmd *micromdm.CommandPayload + var fleetdCmd, installProfileCmd *mdm.Command cmd, err := mdmDevice.Idle() require.NoError(t, err) for cmd != nil { if cmd.Command.RequestType == "InstallEnterpriseApplication" && - cmd.Command.InstallEnterpriseApplication.ManifestURL != nil && - strings.Contains(*cmd.Command.InstallEnterpriseApplication.ManifestURL, apple_mdm.FleetdPublicManifestURL) { + strings.Contains(string(cmd.Raw), apple_mdm.FleetdPublicManifestURL) { fleetdCmd = cmd } else if cmd.Command.RequestType == "InstallProfile" { installProfileCmd = cmd @@ -5874,9 +5875,13 @@ func (s *integrationMDMTestSuite) TestBootstrapPackageStatus() { cmd, err := d.device.Idle() require.NoError(t, err) for cmd != nil { + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + // if the command is to install the bootstrap package - if manifest := cmd.Command.InstallEnterpriseApplication.Manifest; manifest != nil { + if manifest := fullCmd.Command.InstallEnterpriseApplication.Manifest; manifest != nil { require.Equal(t, "InstallEnterpriseApplication", cmd.Command.RequestType) + require.NotNil(t, manifest) require.Equal(t, "software-package", (*manifest).ManifestItems[0].Assets[0].Kind) wantURL, err := bp.URL(s.server.URL) require.NoError(t, err) @@ -7406,7 +7411,7 @@ func (s *integrationMDMTestSuite) TestSSO() { s.runWorker() // ask for commands and verify that we get AccountConfiguration - var accCmd *micromdm.CommandPayload + var accCmd *mdm.Command cmd, err := mdmDevice.Idle() require.NoError(t, err) for cmd != nil { @@ -7418,9 +7423,12 @@ func (s *integrationMDMTestSuite) TestSSO() { } require.NotNil(t, accCmd) require.NotNil(t, accCmd.Command) - require.True(t, accCmd.Command.AccountConfiguration.LockPrimaryAccountInfo) - require.Equal(t, "SSO User 1", accCmd.Command.AccountConfiguration.PrimaryAccountFullName) - require.Equal(t, "sso_user", accCmd.Command.AccountConfiguration.PrimaryAccountUserName) + + var fullAccCmd *micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(accCmd.Raw, &fullAccCmd)) + require.True(t, fullAccCmd.Command.AccountConfiguration.LockPrimaryAccountInfo) + require.Equal(t, "SSO User 1", fullAccCmd.Command.AccountConfiguration.PrimaryAccountFullName) + require.Equal(t, "sso_user", fullAccCmd.Command.AccountConfiguration.PrimaryAccountUserName) // report host details for the device var hostResp getHostResponse @@ -11568,10 +11576,12 @@ func (s *integrationMDMTestSuite) TestManualEnrollmentCommands() { cmd, err := mdmDevice.Idle() require.NoError(t, err) for cmd != nil { - if manifest := cmd.Command.InstallEnterpriseApplication.ManifestURL; manifest != nil { + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + if manifest := fullCmd.Command.InstallEnterpriseApplication.ManifestURL; manifest != nil { foundInstallFleetdCommand = true require.Equal(t, "InstallEnterpriseApplication", cmd.Command.RequestType) - require.Contains(t, *cmd.Command.InstallEnterpriseApplication.ManifestURL, apple_mdm.FleetdPublicManifestURL) + require.Contains(t, *fullCmd.Command.InstallEnterpriseApplication.ManifestURL, apple_mdm.FleetdPublicManifestURL) } cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) require.NoError(t, err) @@ -12245,7 +12255,10 @@ func (s *integrationMDMTestSuite) TestDontIgnoreAnyProfileErrors() { for cmd != nil { if cmd.Command.RequestType == "RemoveProfile" { var errChain []mdm.ErrorChain - if cmd.Command.RemoveProfile.Identifier == "I1" { + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + + if fullCmd.Command.RemoveProfile.Identifier == "I1" { errChain = append(errChain, mdm.ErrorChain{ErrorCode: 89, ErrorDomain: "MDMClientError", USEnglishDescription: "Profile with identifier 'I1' not found."}) } else { errChain = append(errChain, mdm.ErrorChain{ErrorCode: 96, ErrorDomain: "MDMClientError", USEnglishDescription: "Cannot replace profile 'I2' because it was not installed by the MDM server."}) @@ -12375,7 +12388,7 @@ func (s *integrationMDMTestSuite) TestSCEPCertExpiration() { require.NoError(t, err) checkRenewCertCommand := func(device *mdmtest.TestAppleMDMClient, enrollRef string) { - var renewCmd *micromdm.CommandPayload + var renewCmd *mdm.Command cmd, err := device.Idle() require.NoError(t, err) for cmd != nil { @@ -12386,7 +12399,9 @@ func (s *integrationMDMTestSuite) TestSCEPCertExpiration() { require.NoError(t, err) } require.NotNil(t, renewCmd) - s.verifyEnrollmentProfile(renewCmd.Command.InstallProfile.Payload, enrollRef) + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(renewCmd.Raw, &fullCmd)) + s.verifyEnrollmentProfile(fullCmd.Command.InstallProfile.Payload, enrollRef) } checkRenewCertCommand(manualEnrolledDevice, "") From e26d23460c0038f9451b8d6b6eb174215d54c5df Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 18 Mar 2024 14:41:33 -0300 Subject: [PATCH 05/29] record declarative checkin command responses (#17693) this is to prevent nanomdm to send the DeclarativeManagement command every time the host checks in. --- server/datastore/mysql/apple_mdm.go | 73 ++++++++++++++++++++++++ server/datastore/mysql/apple_mdm_test.go | 44 ++++++++++++++ server/fleet/datastore.go | 5 ++ server/mdm/apple/commander.go | 22 +++++++ server/mock/datastore_mock.go | 12 ++++ server/service/apple_mdm.go | 3 + 6 files changed, 159 insertions(+) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 61f1979db2..6bc640557c 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -3080,3 +3080,76 @@ WHERE h.uuid = ? return nil } + +func (ds *Datastore) MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, result []byte) error { + err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext( + ctx, + `UPDATE nano_enrollments SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`, + hostUUID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "updating last_seen times") + } + if n, _ := res.RowsAffected(); n == 0 { + return ctxerr.New(ctx, "host is not enrolled in MDM") + } + + // NOTE: DeclarativeManagement checkin commands sent by the device + // don't carry a CommandUUID reference like commands in + // CommandAndReportResults messages do. + // + // In nanomdm's view of the world, a command is pending until + // it receives a result or is deactivated, so we'll grab the + // command_uuid of the oldest DeclarativeManagement command we + // sent and assume this is the response for it. + // + // Other DeclarativeManagement commands will still be in the + // queue and they will trigger DDM syncs when the device checks + // in, so eventually all DDM commands wil get acknowledged. + // + // Alternatively, we could mark all DDM commands as + // acknowledged here, TBD based on the behaviors we see. + var cmdUUID string + err = sqlx.GetContext(ctx, tx, &cmdUUID, ` +SELECT nc.command_uuid +FROM nano_enrollment_queue neq +JOIN nano_commands nc + ON neq.command_uuid = nc.command_uuid +WHERE + id = ? AND + request_type = 'DeclarativeManagement' +ORDER BY neq.created_at ASC +LIMIT 1 + `, hostUUID) + if err != nil { + // it's okay if the host doesn't have matching command enqueued, the + // check-in could be initiated by the device. + if err == sql.ErrNoRows { + return nil + } + + return ctxerr.Wrap(ctx, err, "getting DDM command") + } + + _, err = tx.ExecContext( + ctx, ` +INSERT INTO nano_command_results + (id, command_uuid, status, result) +VALUES + (?, ?, ?, ?) +ON DUPLICATE KEY +UPDATE + status = VALUES(status), + result = VALUES(result)`, + hostUUID, + cmdUUID, + fleet.MDMAppleStatusAcknowledged, + result, + ) + + return ctxerr.Wrap(ctx, err, "updating nano_command_results") + }) + + return ctxerr.Wrap(ctx, err, "saving declarative management response") +} diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index ed38fe7b2a..f36f1335e4 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -69,6 +69,7 @@ func TestMDMApple(t *testing.T) { {"TestMDMAppleDeleteHostDEPAssignments", testMDMAppleDeleteHostDEPAssignments}, {"LockUnlockWipeMacOS", testLockUnlockWipeMacOS}, {"ScreenDEPAssignProfileSerialsForCooldown", testScreenDEPAssignProfileSerialsForCooldown}, + {"MDMAppleRecordDeclarativeCheckIn", testMDMAppleRecordDeclarativeCheckIn}, } for _, c := range cases { @@ -4570,6 +4571,49 @@ func testScreenDEPAssignProfileSerialsForCooldown(t *testing.T, ds *Datastore) { require.Empty(t, assign) } +func testMDMAppleRecordDeclarativeCheckIn(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host, err := ds.NewHost(ctx, &fleet.Host{ + Hostname: "test-host1-name", + OsqueryHostID: ptr.String("1337"), + NodeKey: ptr.String("1337"), + UUID: "test-uuid-1", + TeamID: nil, + Platform: "darwin", + }) + require.NoError(t, err) + + // error if the host is not enrolled + err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte{}) + require.Error(t, err) + + // enroll the host + nanoEnroll(t, ds, host, true) + + // it's okay if the host doesn't have matching command enqueued, the + // check-in could be initiated by the device. + err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte{}) + require.NoError(t, err) + + // enqueue a declarative checkin request + commander, _ := createMDMAppleCommanderAndStorage(t, ds) + cmdUUID := uuid.New().String() + err = commander.DeclarativeManagement(ctx, []string{host.UUID}, cmdUUID) + require.NoError(t, err) + + // record a response from the host + err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte("foo")) + require.NoError(t, err) + + res, err := ds.GetMDMAppleCommandResults(ctx, cmdUUID) + require.NoError(t, err) + require.Len(t, res, 1) + require.Equal(t, host.UUID, res[0].HostUUID) + require.Equal(t, fleet.MDMAppleStatusAcknowledged, res[0].Status) + require.EqualValues(t, []byte("foo"), res[0].Result) +} + func TestMDMAppleProfileVerification(t *testing.T) { ds := CreateMySQLDS(t) ctx := context.Background() diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 4081db8af0..b1bf6a5dfe 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1137,6 +1137,11 @@ type Datastore interface { // host_dep_assignments for host with matching serials. DeleteHostDEPAssignments(ctx context.Context, serials []string) error + // MDMAppleRecordDeclarativeCheckIn records a DeclarativeManagement + // checking from a host, so we know the host received the command to + // start the declarative management sync. + MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, response []byte) error + // UpdateHostDEPAssignProfileResponses receives a profile UUID and threes lists of serials, each representing // one of the three possible responses, and updates the host_dep_assignments table with the corresponding responses. UpdateHostDEPAssignProfileResponses(ctx context.Context, resp *godep.ProfileResponse) error diff --git a/server/mdm/apple/commander.go b/server/mdm/apple/commander.go index 280596a0cb..c162815e8d 100644 --- a/server/mdm/apple/commander.go +++ b/server/mdm/apple/commander.go @@ -226,6 +226,28 @@ func (svc *MDMAppleCommander) AccountConfiguration(ctx context.Context, hostUUID return svc.EnqueueCommand(ctx, hostUUIDs, raw) } +// DeclarativeManagement sends the homonym [command][1] to the device to enable DDM or start a new DDM session. +// +// [1]: https://developer.apple.com/documentation/devicemanagement/declarativemanagementcommand +func (svc *MDMAppleCommander) DeclarativeManagement(ctx context.Context, hostUUIDs []string, uuid string) error { + raw := fmt.Sprintf(` + + + + Command + + RequestType + DeclarativeManagement + + + CommandUUID + %s + + `, uuid) + + return svc.EnqueueCommand(ctx, hostUUIDs, raw) +} + // EnqueueCommand takes care of enqueuing the commands and sending push // notifications to the devices. // diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 1469826979..0e89e3c72a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -748,6 +748,8 @@ type GetMatchingHostSerialsFunc func(ctx context.Context, serials []string) (map type DeleteHostDEPAssignmentsFunc func(ctx context.Context, serials []string) error +type MDMAppleRecordDeclarativeCheckInFunc func(ctx context.Context, hostUUID string, response []byte) error + type UpdateHostDEPAssignProfileResponsesFunc func(ctx context.Context, resp *godep.ProfileResponse) error type ScreenDEPAssignProfileSerialsForCooldownFunc func(ctx context.Context, serials []string) (skipSerials []string, assignSerials []string, err error) @@ -1954,6 +1956,9 @@ type DataStore struct { DeleteHostDEPAssignmentsFunc DeleteHostDEPAssignmentsFunc DeleteHostDEPAssignmentsFuncInvoked bool + MDMAppleRecordDeclarativeCheckInFunc MDMAppleRecordDeclarativeCheckInFunc + MDMAppleRecordDeclarativeCheckInFuncInvoked bool + UpdateHostDEPAssignProfileResponsesFunc UpdateHostDEPAssignProfileResponsesFunc UpdateHostDEPAssignProfileResponsesFuncInvoked bool @@ -4677,6 +4682,13 @@ func (s *DataStore) DeleteHostDEPAssignments(ctx context.Context, serials []stri return s.DeleteHostDEPAssignmentsFunc(ctx, serials) } +func (s *DataStore) MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, response []byte) error { + s.mu.Lock() + s.MDMAppleRecordDeclarativeCheckInFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleRecordDeclarativeCheckInFunc(ctx, hostUUID, response) +} + func (s *DataStore) UpdateHostDEPAssignProfileResponses(ctx context.Context, resp *godep.ProfileResponse) error { s.mu.Lock() s.UpdateHostDEPAssignProfileResponsesFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index b8a70eda47..4ae37d0ed3 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2976,6 +2976,9 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec switch { case dm.Endpoint == "tokens": + if err := svc.ds.MDMAppleRecordDeclarativeCheckIn(r.Context, dm.UDID, dm.Raw); err != nil { + return nil, ctxerr.Wrap(r.Context, err, "recording declarative checkin") + } // TODO(sarah): handle tokens level.Debug(svc.logger).Log("msg", "received tokens request") return nil, nil From 93f040f7da45507dab0128c4c858f4d3a553a86e Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 18 Mar 2024 17:35:18 -0300 Subject: [PATCH 06/29] fix mysql 8+ test (#17702) --- server/datastore/mysql/apple_mdm_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index f36f1335e4..8d0662d91f 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -4603,7 +4603,7 @@ func testMDMAppleRecordDeclarativeCheckIn(t *testing.T, ds *Datastore) { require.NoError(t, err) // record a response from the host - err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte("foo")) + err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte(" Date: Mon, 18 Mar 2024 17:48:07 -0500 Subject: [PATCH 07/29] Add macOS DDM protocol endpoints for `tokens`, `declaration-items`, and `declaration/.../...` (#17679) --- pkg/mdm/mdmtest/apple.go | 6 +- server/datastore/mysql/apple_mdm.go | 64 ++++++- server/fleet/apple_mdm.go | 64 +++++++ server/fleet/datastore.go | 8 + server/mock/datastore_mock.go | 36 ++++ server/service/apple_mdm.go | 99 +++++++++- server/service/integration_mdm_test.go | 251 +++++++++++++++++++++++-- 7 files changed, 502 insertions(+), 26 deletions(-) diff --git a/pkg/mdm/mdmtest/apple.go b/pkg/mdm/mdmtest/apple.go index b213218980..be3759ca64 100644 --- a/pkg/mdm/mdmtest/apple.go +++ b/pkg/mdm/mdmtest/apple.go @@ -390,7 +390,7 @@ func (c *TestAppleMDMClient) TokenUpdate() error { // The endpoint argument is used as the value for the `Endpoint` key in the request payload. // // For more details check https://developer.apple.com/documentation/devicemanagement/declarativemanagementrequest -func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) error { +func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) (*http.Response, error) { payload := map[string]any{ "MessageType": "DeclarativeManagement", "UDID": c.UUID, @@ -398,8 +398,8 @@ func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) error { "EnrollmentID": "testenrollmentid-" + c.UUID, "Endpoint": endpoint, } - _, err := c.request("application/x-apple-aspen-mdm-checkin", payload) - return err + r, err := c.request("application/x-apple-aspen-mdm-checkin", payload) + return r, err } // Checkout sends the CheckOut message to the MDM server. diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 6bc640557c..2d2e267647 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -3081,6 +3082,64 @@ WHERE h.uuid = ? return nil } +func (ds *Datastore) MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) { + const stmt = ` +SELECT + md5_checksum, + latest_created_timestamp +FROM + team_declaration_checksum_view +WHERE + team_id = ?` + + var res fleet.MDMAppleDDMSyncTokens + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, teamID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get DDM checksum by team id") + } + + return &res, nil +} + +func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) { + // TODO: Confirm whether we can use JSON functions in the query (if 5.7 officially unsupported + // by Fleet) + const stmt = ` +SELECT + mad.md5_checksum as server_token, + identifier, + declaration_type, + tv.md5_checksum as declarations_token +FROM + mdm_apple_declarations mad + JOIN team_declaration_checksum_view tv ON mad.team_id = tv.team_id +WHERE + mad.team_id = ?` + + var res []fleet.MDMAppleDDMDeclarationItem + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &res, stmt, teamID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get DDM checksum by team id") + } + + return res, nil +} + +func (ds *Datastore) MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) { + const stmt = ` +SELECT + declaration +FROM + mdm_apple_declarations +WHERE + team_id = ? AND identifier = ? AND declaration_type = ?` + + var res json.RawMessage + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, teamID, identifier, declarationType); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get ddm declaration") + } + + return res, nil +} + func (ds *Datastore) MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, result []byte) error { err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext( @@ -3150,6 +3209,9 @@ UPDATE return ctxerr.Wrap(ctx, err, "updating nano_command_results") }) + if err != nil { + return ctxerr.Wrap(ctx, err, "saving declarative management response") + } - return ctxerr.Wrap(ctx, err, "saving declarative management response") + return nil } diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index c6a0061694..95a2bad009 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -609,3 +609,67 @@ type MDMAppleHostDeclaration struct { // either by the MDM protocol or the Fleet server. Detail string `db:"detail" json:"detail"` } + +// MDMAppleDDMTokensResponse is the response from the DDM tokens endpoint. +// +// https://developer.apple.com/documentation/devicemanagement/tokensresponse +type MDMAppleDDMTokensResponse struct { + SyncTokens MDMAppleDDMSyncTokens +} + +// MDMAppleDDMSyncTokens is dictionary describes the state of declarations on the server. +// +// https://developer.apple.com/documentation/devicemanagement/synchronizationtokens +type MDMAppleDDMSyncTokens struct { + DeclarationsToken string `db:"md5_checksum"` + Timestamp time.Time `db:"latest_created_timestamp"` +} + +// MDMAppleDDMDeclarationItemsResponse is the response from the DDM declaration items endpoint. +// +// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse +type MDMAppleDDMDeclarationItemsResponse struct { + Declarations MDMAppleDDMManifestItems + DeclarationsToken string +} + +// MDMAppleDDMManifestItems is a dictionary that contains the lists of declarations available on the +// server. +// +// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse/manifestdeclarationitems +type MDMAppleDDMManifestItems struct { + Activations []MDMAppleDDMManifest + Assets []MDMAppleDDMManifest + Configurations []MDMAppleDDMManifest + Management []MDMAppleDDMManifest +} + +// MDMAppleDDMManifest is a dictionary that describes a declaration. +// +// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse/manifestdeclarationitems +type MDMAppleDDMManifest struct { + Identifier string + ServerToken string +} + +// MDMAppleDDMDeclarationItem represents a declaration item in the datastore. It is used to +// construct the DDM `declaration-items` endpoint response. +// +// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse +type MDMAppleDDMDeclarationItem struct { + Identifier string `db:"identifier"` + DeclarationType string `db:"declaration_type"` + DeclarationsToken string `db:"declarations_token"` + ServerToken string `db:"server_token"` +} + +// MDMAppleDDMDeclarationResponse represents a declaration in the datastore. It is used for the DDM +// `declaration/.../...` enpoint response. +// +// https://developer.apple.com/documentation/devicemanagement/declarationresponse +type MDMAppleDDMDeclarationResponse struct { + Identifier string `db:"identifier"` + Type string `db:"type"` + Payload json.RawMessage `db:"payload"` + ServerToken string `db:"server_token"` +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index b1bf6a5dfe..61a590ac2c 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1157,6 +1157,14 @@ type Datastore interface { // serials. UpdateDEPAssignProfileRetryPending(ctx context.Context, jobID uint, serials []string) error + // MDMAppleDDMSynchronizationTokens returns the token used to synchronize declarations for the + // specified team or no team. + MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*MDMAppleDDMSyncTokens, error) + // MDMAppleDDMDeclarationItems returns the declaration items for the specified team or no team. + MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]MDMAppleDDMDeclarationItem, error) + // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. + MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) + /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 0e89e3c72a..a7e1f8df11 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -758,6 +758,12 @@ type GetDEPAssignProfileExpiredCooldownsFunc func(ctx context.Context) (map[uint type UpdateDEPAssignProfileRetryPendingFunc func(ctx context.Context, jobID uint, serials []string) error +type MDMAppleDDMSynchronizationTokensFunc func(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) + +type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) + +type MDMAppleDDMDeclarationPayloadFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) + type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error type WSTEPNewSerialFunc func(ctx context.Context) (*big.Int, error) @@ -1971,6 +1977,15 @@ type DataStore struct { UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFuncInvoked bool + MDMAppleDDMSynchronizationTokensFunc MDMAppleDDMSynchronizationTokensFunc + MDMAppleDDMSynchronizationTokensFuncInvoked bool + + MDMAppleDDMDeclarationItemsFunc MDMAppleDDMDeclarationItemsFunc + MDMAppleDDMDeclarationItemsFuncInvoked bool + + MDMAppleDDMDeclarationPayloadFunc MDMAppleDDMDeclarationPayloadFunc + MDMAppleDDMDeclarationPayloadFuncInvoked bool + WSTEPStoreCertificateFunc WSTEPStoreCertificateFunc WSTEPStoreCertificateFuncInvoked bool @@ -4717,6 +4732,27 @@ func (s *DataStore) UpdateDEPAssignProfileRetryPending(ctx context.Context, jobI return s.UpdateDEPAssignProfileRetryPendingFunc(ctx, jobID, serials) } +func (s *DataStore) MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) { + s.mu.Lock() + s.MDMAppleDDMSynchronizationTokensFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleDDMSynchronizationTokensFunc(ctx, teamID) +} + +func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) { + s.mu.Lock() + s.MDMAppleDDMDeclarationItemsFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleDDMDeclarationItemsFunc(ctx, teamID) +} + +func (s *DataStore) MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) { + s.mu.Lock() + s.MDMAppleDDMDeclarationPayloadFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleDDMDeclarationPayloadFunc(ctx, declarationType, identifier, teamID) +} + func (s *DataStore) WSTEPStoreCertificate(ctx context.Context, name string, crt *x509.Certificate) error { s.mu.Lock() s.WSTEPStoreCertificateFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 4ae37d0ed3..77e94f2920 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2974,27 +2974,40 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec } level.Debug(svc.logger).Log("msg", "ddm request received", "endpoint", dm.Endpoint) + if dm.UDID == "" { + return nil, ctxerr.New(r.Context, "missing device id") + } + + h, err := svc.ds.HostLiteByIdentifier(r.Context, dm.UDID) + if err != nil { + return nil, ctxerr.Wrap(r.Context, err, "getting host by identifier") + } + var tid uint + if h.TeamID != nil { + tid = *h.TeamID + } + switch { case dm.Endpoint == "tokens": + level.Debug(svc.logger).Log("msg", "received tokens request") + // TODO: Should we record the checkin for all endpoints or just tokens? if err := svc.ds.MDMAppleRecordDeclarativeCheckIn(r.Context, dm.UDID, dm.Raw); err != nil { return nil, ctxerr.Wrap(r.Context, err, "recording declarative checkin") } - // TODO(sarah): handle tokens - level.Debug(svc.logger).Log("msg", "received tokens request") - return nil, nil + + return svc.handleTokens(r.Context, tid) case dm.Endpoint == "declaration-items": - // TODO(sarah): handle declaration-items level.Debug(svc.logger).Log("msg", "received declaration-items request") - return nil, nil + return svc.handleDeclarationItems(r.Context, tid) case dm.Endpoint == "status": - // TODO(roberto): handle status level.Debug(svc.logger).Log("msg", "received status request") + // TODO(roberto): handle status + return nil, nil case strings.HasPrefix(dm.Endpoint, "declarations"): - // TODO(sarah): handle declarations level.Debug(svc.logger).Log("msg", "received declarations request") parts := strings.Split(dm.Endpoint, "/") if len(parts) != 3 { @@ -3003,9 +3016,79 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec declarationType := parts[1] declarationIdentifier := parts[2] level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", declarationType, "identifier", declarationIdentifier) - return nil, nil + + // TODO: Validate declarationType? + d, err := svc.ds.MDMAppleDDMDeclarationPayload(r.Context, fleet.MDMAppleDeclarationType("com.apple."+declarationType), declarationIdentifier, tid) + if err != nil { + return nil, ctxerr.Wrap(r.Context, err, "getting declaration") + } + b, err := json.Marshal(d) + if err != nil { + return nil, ctxerr.Wrap(r.Context, err, "marshaling declaration") + } + return b, nil default: return nil, ctxerr.New(r.Context, "unrecognized ddm endpoint") } } + +func (svc *MDMAppleDDMService) handleTokens(ctx context.Context, teamID uint) ([]byte, error) { + tok, err := svc.ds.MDMAppleDDMSynchronizationTokens(ctx, teamID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting synchronization tokens") + } + + b, err := json.Marshal(fleet.MDMAppleDDMTokensResponse{ + SyncTokens: *tok, + }) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "marshaling synchronization tokens") + } + + return b, nil +} + +func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, teamID uint) ([]byte, error) { + di, err := svc.ds.MDMAppleDDMDeclarationItems(ctx, teamID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting synchronization tokens") + } + + var dTok string + activations := []fleet.MDMAppleDDMManifest{} + configurations := []fleet.MDMAppleDDMManifest{} + for _, d := range di { + if dTok == "" { + dTok = d.DeclarationsToken + } else if dTok != d.DeclarationsToken { + level.Debug(svc.logger).Log("msg", "inconsistent declarations token", "expected", dTok, "got", d.DeclarationsToken) + } + + manifest := fleet.MDMAppleDDMManifest{Identifier: d.Identifier, ServerToken: d.ServerToken} + switch d.DeclarationType { + case string(fleet.MDMAppleDeclarativeActivation): + activations = append(activations, manifest) + case string(fleet.MDMAppleDeclarativeConfiguration): + configurations = append(configurations, manifest) + default: + level.Debug(svc.logger).Log("msg", "unrecognized declaration type", "type", d.DeclarationType) + return nil, ctxerr.New(ctx, "unrecognized declaration type") + } + } + + b, err := json.Marshal(fleet.MDMAppleDDMDeclarationItemsResponse{ + Declarations: fleet.MDMAppleDDMManifestItems{ + Activations: activations, + Configurations: configurations, + Assets: []fleet.MDMAppleDDMManifest{}, + Management: []fleet.MDMAppleDDMManifest{}, + }, + DeclarationsToken: dTok, + }) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "marshaling synchronization tokens") + } + + return b, nil +} diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 782ee6dc14..bdb8f3f2a7 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -12522,23 +12522,246 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { // TODO(sarah): Build out this test func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { t := s.T() - mdmDevice := mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{ - SCEPChallenge: s.fleetCfg.MDM.AppleSCEPChallenge, - SCEPURL: s.server.URL + apple_mdm.SCEPPath, - MDMURL: s.server.URL + apple_mdm.MDMPath, + _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + insertDeclaration := func(t *testing.T, decl fleet.MDMAppleDeclaration) { + stmt := ` +INSERT INTO mdm_apple_declarations ( + declaration_uuid, + team_id, + identifier, + name, + declaration_type, + declaration, + md5_checksum, + created_at, + uploaded_at +) VALUES (?,?,?,?,?,?,?,?,?)` + + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(context.Background(), stmt, + decl.DeclarationUUID, + decl.TeamID, + decl.Identifier, + decl.Name, + decl.DeclarationType, + decl.Declaration, + decl.MD5Checksum, + decl.CreatedAt, + decl.UploadedAt, + ) + return err + }) + } + + // initialize a time to use for our first declaration, subsequent declarations will be + // incremented by a minute + then := time.Now().UTC().Truncate(time.Second).Add(-1 * time.Hour) + + // insert a declaration with no team + noTeamDeclsByUUID := map[string]fleet.MDMAppleDeclaration{ + "123": { + DeclarationUUID: "123", + TeamID: ptr.Uint(0), + Identifier: "com.example", + Name: "Example", + DeclarationType: fleet.MDMAppleDeclarativeConfiguration, + Declaration: json.RawMessage(`{"foo": "bar"}`), + MD5Checksum: "csum123", + CreatedAt: then, + UploadedAt: then, + }, + } + insertDeclaration(t, noTeamDeclsByUUID["123"]) + + mapDeclsByChecksum := func(byUUID map[string]fleet.MDMAppleDeclaration) map[string]fleet.MDMAppleDeclaration { + byChecksum := make(map[string]fleet.MDMAppleDeclaration) + for _, d := range byUUID { + byChecksum[d.MD5Checksum] = byUUID[d.DeclarationUUID] + } + return byChecksum + } + + parseTokensResp := func(r *http.Response) fleet.MDMAppleDDMTokensResponse { + require.NotNil(t, r) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(b)) + // t.Log("body", string(b)) + + // unmarsal the response to make sure it's valid + var tok fleet.MDMAppleDDMTokensResponse + err = json.NewDecoder(r.Body).Decode(&tok) + require.NoError(t, err) + // t.Log("decoded", tok) + + return tok + } + + parseDeclarationItemsResp := func(r *http.Response) fleet.MDMAppleDDMDeclarationItemsResponse { + require.NotNil(t, r) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(b)) + // t.Log("body", string(b)) + + // unmarsal the response to make sure it's valid + var di fleet.MDMAppleDDMDeclarationItemsResponse + err = json.NewDecoder(r.Body).Decode(&di) + require.NoError(t, err) + // t.Log("decoded", di) + + return di + } + + parseDeclarationResp := func(r *http.Response, expectedBytes []byte) fleet.MDMAppleDDMDeclarationResponse { + require.NotNil(t, r) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + if expectedBytes != nil { + require.Equal(t, expectedBytes, b) + } + r.Body = io.NopCloser(bytes.NewBuffer(b)) + // t.Log("body", string(b)) + + // unmarsal the response to make sure it's valid + var d fleet.MDMAppleDDMDeclarationResponse + err = json.NewDecoder(r.Body).Decode(&d) + require.NoError(t, err) + // t.Logf("decoded: %+v", d) + + return d + } + + checkTokensResp := func(t *testing.T, r fleet.MDMAppleDDMTokensResponse, expectedTimestamp time.Time, prevToken string) { + require.Equal(t, expectedTimestamp, r.SyncTokens.Timestamp) + require.NotEmpty(t, r.SyncTokens.DeclarationsToken) + require.NotEqual(t, prevToken, r.SyncTokens.DeclarationsToken) + } + + checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { + require.Equal(t, expectedDeclTok, r.DeclarationsToken) + require.Empty(t, r.Declarations.Activations) + require.Empty(t, r.Declarations.Assets) + require.Empty(t, r.Declarations.Management) + require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) + for _, m := range r.Declarations.Configurations { + // look up the declaration by the server token (we trim the token to the first seven + // chars to match our keys because response is padded to length 16 with "\u000") + d, ok := expectedDeclsByChecksum[m.ServerToken[0:7]] + require.True(t, ok) + require.Equal(t, d.Identifier, m.Identifier) + } + } + + var currDeclToken string // we'll use this to track the expected token across tests + + t.Run("Tokens", func(t *testing.T) { + // get tokens, timestamp should be the same as the declaration and token should be non-empty + r, err := mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + parsed := parseTokensResp(r) + checkTokensResp(t, parsed, then, "") + currDeclToken = parsed.SyncTokens.DeclarationsToken + + // insert a new declaration + noTeamDeclsByUUID["456"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "456", + TeamID: ptr.Uint(0), + Identifier: "com.example2", + Name: "Example2", + DeclarationType: fleet.MDMAppleDeclarativeConfiguration, + Declaration: json.RawMessage(`{"foo": "baz"}`), + MD5Checksum: "csum456", + CreatedAt: then.Add(1 * time.Minute), + UploadedAt: then.Add(1 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["456"]) + + // get tokens again, timestamp and token should have changed + r, err = mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + parsed = parseTokensResp(r) + checkTokensResp(t, parsed, then.Add(1*time.Minute), currDeclToken) + currDeclToken = parsed.SyncTokens.DeclarationsToken }) - err := mdmDevice.Enroll() - require.NoError(t, err) - err = mdmDevice.DeclarativeManagement("tokens") - require.NoError(t, err) + t.Run("DeclarationItems", func(t *testing.T) { + r, err := mdmDevice.DeclarativeManagement("declaration-items") + require.NoError(t, err) + checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) - err = mdmDevice.DeclarativeManagement("declaration-items") - require.NoError(t, err) + // insert a new declaration + noTeamDeclsByUUID["789"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "789", + TeamID: ptr.Uint(0), + Identifier: "com.example3", + Name: "Example3", + DeclarationType: fleet.MDMAppleDeclarativeConfiguration, + Declaration: json.RawMessage(`{"foo": "bang"}`), + MD5Checksum: "csum789", + CreatedAt: then.Add(2 * time.Minute), + UploadedAt: then.Add(2 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["789"]) - err = mdmDevice.DeclarativeManagement("status") - require.NoError(t, err) + // get tokens again, timestamp and token should have changed + r, err = mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + toks := parseTokensResp(r) + checkTokensResp(t, toks, then.Add(2*time.Minute), currDeclToken) + currDeclToken = toks.SyncTokens.DeclarationsToken - err = mdmDevice.DeclarativeManagement("declarations/foo/bar") - require.NoError(t, err) + r, err = mdmDevice.DeclarativeManagement("declaration-items") + require.NoError(t, err) + checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) + }) + + t.Run("Status", func(t *testing.T) { + _, err := mdmDevice.DeclarativeManagement("status") + require.NoError(t, err) + }) + + t.Run("Declaration", func(t *testing.T) { + want := noTeamDeclsByUUID["123"] + wantBytes, err := json.Marshal(want.Declaration) + require.NoError(t, err) + r, err := mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configuration", want.Identifier)) + require.NoError(t, err) + + _ = parseDeclarationResp(r, wantBytes) + + // insert a new declaration + noTeamDeclsByUUID["abc"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "abc", + TeamID: ptr.Uint(0), + Identifier: "com.example4", + Name: "Example4", + DeclarationType: fleet.MDMAppleDeclarativeConfiguration, + Declaration: json.RawMessage(`{ + "Type": "com.apple.configuration.test", + "Payload": {"foo":"bar"}, + "Identifier": "com.example4", + "ServerToken": "csumabc" + }`), + MD5Checksum: "csumabc", + CreatedAt: then.Add(3 * time.Minute), + UploadedAt: then.Add(3 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["abc"]) + want = noTeamDeclsByUUID["abc"] + wantBytes, err = json.Marshal(want.Declaration) + require.NoError(t, err) + r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configuration", want.Identifier)) + require.NoError(t, err) + + d := parseDeclarationResp(r, wantBytes) + require.Equal(t, want.Identifier, d.Identifier) + require.Equal(t, "com.apple.configuration.test", d.Type) + require.Equal(t, json.RawMessage(`{"foo":"bar"}`), d.Payload) + require.Equal(t, want.MD5Checksum, d.ServerToken) + }) } From d56d0cde31165b17d359a4dd1a094ae1501695db Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:14:21 -0500 Subject: [PATCH 08/29] Update DDM protocol endpoints to use host UUID in support of profile labels (#17719) --- server/datastore/mysql/apple_mdm.go | 57 +++++++++++--------- server/fleet/apple_mdm.go | 13 +++-- server/fleet/datastore.go | 12 ++--- server/mock/datastore_mock.go | 30 +++++------ server/service/apple_mdm.go | 75 ++++++++++++-------------- server/service/integration_mdm_test.go | 27 +++++++++- 6 files changed, 118 insertions(+), 96 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 2d2e267647..58a1966a9f 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -3082,59 +3082,64 @@ WHERE h.uuid = ? return nil } -func (ds *Datastore) MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) { +func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { const stmt = ` SELECT - md5_checksum, - latest_created_timestamp + md5((count(0) + group_concat(hex(mad.md5_checksum) + ORDER BY + mad.uploaded_at DESC separator ''))) AS md5_checksum, + max(mad.created_at) AS latest_created_timestamp FROM - team_declaration_checksum_view + host_mdm_apple_declarations hmad + JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid WHERE - team_id = ?` + hmad.host_uuid = ?` - var res fleet.MDMAppleDDMSyncTokens - if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, teamID); err != nil { - return nil, ctxerr.Wrap(ctx, err, "get DDM checksum by team id") + var res fleet.MDMAppleDDMDeclarationsToken + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get DDM declarations token") } return &res, nil } -func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) { - // TODO: Confirm whether we can use JSON functions in the query (if 5.7 officially unsupported - // by Fleet) +func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { const stmt = ` SELECT - mad.md5_checksum as server_token, - identifier, - declaration_type, - tv.md5_checksum as declarations_token + mad.md5_checksum, + mad.identifier, + mad.declaration_type FROM - mdm_apple_declarations mad - JOIN team_declaration_checksum_view tv ON mad.team_id = tv.team_id + host_mdm_apple_declarations hmad + JOIN mdm_apple_declarations mad ON mad.declaration_uuid = hmad.declaration_uuid WHERE - mad.team_id = ?` + hmad.host_uuid = ? AND operation_type = ?` var res []fleet.MDMAppleDDMDeclarationItem - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &res, stmt, teamID); err != nil { - return nil, ctxerr.Wrap(ctx, err, "get DDM checksum by team id") + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, fleet.MDMOperationTypeInstall); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get DDM declaration items") } return res, nil } -func (ds *Datastore) MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) { +func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) { + // TODO: When hosts table is indexed by uuid, consider joining on hosts to ensure that the + // declaration for the host's current team is returned. In the case where the specified + // identifier is not unique to the team, the cron should ensure that any conflicting + // declarations are removed, but the join would provide an extra layer of safety. const stmt = ` SELECT - declaration + mad.declaration FROM - mdm_apple_declarations + host_mdm_apple_declarations hmad + JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid WHERE - team_id = ? AND identifier = ? AND declaration_type = ?` + host_uuid = ? AND identifier = ? AND declaration_type = ? AND operation_type = ?` var res json.RawMessage - if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, teamID, identifier, declarationType); err != nil { - return nil, ctxerr.Wrap(ctx, err, "get ddm declaration") + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, declarationType, fleet.MDMOperationTypeInstall); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get ddm declarations response") } return res, nil diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 95a2bad009..56116daf6d 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -614,13 +614,13 @@ type MDMAppleHostDeclaration struct { // // https://developer.apple.com/documentation/devicemanagement/tokensresponse type MDMAppleDDMTokensResponse struct { - SyncTokens MDMAppleDDMSyncTokens + SyncTokens MDMAppleDDMDeclarationsToken } -// MDMAppleDDMSyncTokens is dictionary describes the state of declarations on the server. +// MDMAppleDDMDeclarationsToken is dictionary describes the state of declarations on the server. // // https://developer.apple.com/documentation/devicemanagement/synchronizationtokens -type MDMAppleDDMSyncTokens struct { +type MDMAppleDDMDeclarationsToken struct { DeclarationsToken string `db:"md5_checksum"` Timestamp time.Time `db:"latest_created_timestamp"` } @@ -657,10 +657,9 @@ type MDMAppleDDMManifest struct { // // https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse type MDMAppleDDMDeclarationItem struct { - Identifier string `db:"identifier"` - DeclarationType string `db:"declaration_type"` - DeclarationsToken string `db:"declarations_token"` - ServerToken string `db:"server_token"` + Identifier string `db:"identifier"` + DeclarationType string `db:"declaration_type"` + ServerToken string `db:"md5_checksum"` } // MDMAppleDDMDeclarationResponse represents a declaration in the datastore. It is used for the DDM diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 61a590ac2c..aac24814e9 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1157,13 +1157,13 @@ type Datastore interface { // serials. UpdateDEPAssignProfileRetryPending(ctx context.Context, jobID uint, serials []string) error - // MDMAppleDDMSynchronizationTokens returns the token used to synchronize declarations for the - // specified team or no team. - MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*MDMAppleDDMSyncTokens, error) - // MDMAppleDDMDeclarationItems returns the declaration items for the specified team or no team. - MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]MDMAppleDDMDeclarationItem, error) + // MDMAppleDDMDeclarationsToken returns the token used to synchronize declarations for the + // specified host UUID. + MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*MDMAppleDDMDeclarationsToken, error) + // MDMAppleDDMDeclarationItems returns the declaration items for the specified host UUID. + MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]MDMAppleDDMDeclarationItem, error) // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. - MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) + MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index a7e1f8df11..4e6311ed01 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -758,11 +758,11 @@ type GetDEPAssignProfileExpiredCooldownsFunc func(ctx context.Context) (map[uint type UpdateDEPAssignProfileRetryPendingFunc func(ctx context.Context, jobID uint, serials []string) error -type MDMAppleDDMSynchronizationTokensFunc func(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) +type MDMAppleDDMDeclarationsTokenFunc func(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) -type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) +type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) -type MDMAppleDDMDeclarationPayloadFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) +type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error @@ -1977,14 +1977,14 @@ type DataStore struct { UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFuncInvoked bool - MDMAppleDDMSynchronizationTokensFunc MDMAppleDDMSynchronizationTokensFunc - MDMAppleDDMSynchronizationTokensFuncInvoked bool + MDMAppleDDMDeclarationsTokenFunc MDMAppleDDMDeclarationsTokenFunc + MDMAppleDDMDeclarationsTokenFuncInvoked bool MDMAppleDDMDeclarationItemsFunc MDMAppleDDMDeclarationItemsFunc MDMAppleDDMDeclarationItemsFuncInvoked bool - MDMAppleDDMDeclarationPayloadFunc MDMAppleDDMDeclarationPayloadFunc - MDMAppleDDMDeclarationPayloadFuncInvoked bool + MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFunc + MDMAppleDDMDeclarationsResponseFuncInvoked bool WSTEPStoreCertificateFunc WSTEPStoreCertificateFunc WSTEPStoreCertificateFuncInvoked bool @@ -4732,25 +4732,25 @@ func (s *DataStore) UpdateDEPAssignProfileRetryPending(ctx context.Context, jobI return s.UpdateDEPAssignProfileRetryPendingFunc(ctx, jobID, serials) } -func (s *DataStore) MDMAppleDDMSynchronizationTokens(ctx context.Context, teamID uint) (*fleet.MDMAppleDDMSyncTokens, error) { +func (s *DataStore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { s.mu.Lock() - s.MDMAppleDDMSynchronizationTokensFuncInvoked = true + s.MDMAppleDDMDeclarationsTokenFuncInvoked = true s.mu.Unlock() - return s.MDMAppleDDMSynchronizationTokensFunc(ctx, teamID) + return s.MDMAppleDDMDeclarationsTokenFunc(ctx, hostUUID) } -func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, teamID uint) ([]fleet.MDMAppleDDMDeclarationItem, error) { +func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { s.mu.Lock() s.MDMAppleDDMDeclarationItemsFuncInvoked = true s.mu.Unlock() - return s.MDMAppleDDMDeclarationItemsFunc(ctx, teamID) + return s.MDMAppleDDMDeclarationItemsFunc(ctx, hostUUID) } -func (s *DataStore) MDMAppleDDMDeclarationPayload(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, teamID uint) (json.RawMessage, error) { +func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) { s.mu.Lock() - s.MDMAppleDDMDeclarationPayloadFuncInvoked = true + s.MDMAppleDDMDeclarationsResponseFuncInvoked = true s.mu.Unlock() - return s.MDMAppleDDMDeclarationPayloadFunc(ctx, declarationType, identifier, teamID) + return s.MDMAppleDDMDeclarationsResponseFunc(ctx, declarationType, identifier, hostUUID) } func (s *DataStore) WSTEPStoreCertificate(ctx context.Context, name string, crt *x509.Certificate) error { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 77e94f2920..40720ce820 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2978,15 +2978,6 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec return nil, ctxerr.New(r.Context, "missing device id") } - h, err := svc.ds.HostLiteByIdentifier(r.Context, dm.UDID) - if err != nil { - return nil, ctxerr.Wrap(r.Context, err, "getting host by identifier") - } - var tid uint - if h.TeamID != nil { - tid = *h.TeamID - } - switch { case dm.Endpoint == "tokens": level.Debug(svc.logger).Log("msg", "received tokens request") @@ -2995,11 +2986,11 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec return nil, ctxerr.Wrap(r.Context, err, "recording declarative checkin") } - return svc.handleTokens(r.Context, tid) + return svc.handleTokens(r.Context, dm.UDID) case dm.Endpoint == "declaration-items": level.Debug(svc.logger).Log("msg", "received declaration-items request") - return svc.handleDeclarationItems(r.Context, tid) + return svc.handleDeclarationItems(r.Context, dm.UDID) case dm.Endpoint == "status": level.Debug(svc.logger).Log("msg", "received status request") @@ -3009,32 +3000,15 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec case strings.HasPrefix(dm.Endpoint, "declarations"): level.Debug(svc.logger).Log("msg", "received declarations request") - parts := strings.Split(dm.Endpoint, "/") - if len(parts) != 3 { - return nil, ctxerr.New(r.Context, "unrecognized declarations endpoint") - } - declarationType := parts[1] - declarationIdentifier := parts[2] - level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", declarationType, "identifier", declarationIdentifier) - - // TODO: Validate declarationType? - d, err := svc.ds.MDMAppleDDMDeclarationPayload(r.Context, fleet.MDMAppleDeclarationType("com.apple."+declarationType), declarationIdentifier, tid) - if err != nil { - return nil, ctxerr.Wrap(r.Context, err, "getting declaration") - } - b, err := json.Marshal(d) - if err != nil { - return nil, ctxerr.Wrap(r.Context, err, "marshaling declaration") - } - return b, nil + return svc.handleDeclarationsResponse(r.Context, dm.Endpoint, dm.UDID) default: return nil, ctxerr.New(r.Context, "unrecognized ddm endpoint") } } -func (svc *MDMAppleDDMService) handleTokens(ctx context.Context, teamID uint) ([]byte, error) { - tok, err := svc.ds.MDMAppleDDMSynchronizationTokens(ctx, teamID) +func (svc *MDMAppleDDMService) handleTokens(ctx context.Context, hostUUID string) ([]byte, error) { + tok, err := svc.ds.MDMAppleDDMDeclarationsToken(ctx, hostUUID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting synchronization tokens") } @@ -3049,22 +3023,15 @@ func (svc *MDMAppleDDMService) handleTokens(ctx context.Context, teamID uint) ([ return b, nil } -func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, teamID uint) ([]byte, error) { - di, err := svc.ds.MDMAppleDDMDeclarationItems(ctx, teamID) +func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, hostUUID string) ([]byte, error) { + di, err := svc.ds.MDMAppleDDMDeclarationItems(ctx, hostUUID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting synchronization tokens") } - var dTok string activations := []fleet.MDMAppleDDMManifest{} configurations := []fleet.MDMAppleDDMManifest{} for _, d := range di { - if dTok == "" { - dTok = d.DeclarationsToken - } else if dTok != d.DeclarationsToken { - level.Debug(svc.logger).Log("msg", "inconsistent declarations token", "expected", dTok, "got", d.DeclarationsToken) - } - manifest := fleet.MDMAppleDDMManifest{Identifier: d.Identifier, ServerToken: d.ServerToken} switch d.DeclarationType { case string(fleet.MDMAppleDeclarativeActivation): @@ -3077,6 +3044,12 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, teamI } } + // TODO: Look for ways to optimize the declaration item query so that we don't have to get the declarations token separately. + dTok, err := svc.ds.MDMAppleDDMDeclarationsToken(ctx, hostUUID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting declarations token") + } + b, err := json.Marshal(fleet.MDMAppleDDMDeclarationItemsResponse{ Declarations: fleet.MDMAppleDDMManifestItems{ Activations: activations, @@ -3084,7 +3057,7 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, teamI Assets: []fleet.MDMAppleDDMManifest{}, Management: []fleet.MDMAppleDDMManifest{}, }, - DeclarationsToken: dTok, + DeclarationsToken: dTok.DeclarationsToken, }) if err != nil { return nil, ctxerr.Wrap(ctx, err, "marshaling synchronization tokens") @@ -3092,3 +3065,23 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, teamI return b, nil } + +func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, endpoint string, hostUUID string) ([]byte, error) { + parts := strings.Split(endpoint, "/") + if len(parts) != 3 { + return nil, ctxerr.New(ctx, "unrecognized declarations endpoint") + } + level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", parts[1], "identifier", parts[2]) + + // TODO: Validate declarationType? + d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, fleet.MDMAppleDeclarationType("com.apple."+parts[1]), parts[2], hostUUID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting declaration") + } + b, err := json.Marshal(d) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "marshaling declaration") + } + + return b, nil +} diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index bdb8f3f2a7..e22e32c217 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -12519,7 +12519,6 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } -// TODO(sarah): Build out this test func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { t := s.T() _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) @@ -12554,6 +12553,28 @@ INSERT INTO mdm_apple_declarations ( }) } + insertHostDeclaration := func(t *testing.T, hostUUID string, decl fleet.MDMAppleDeclaration) { + stmt := ` +INSERT INTO host_mdm_apple_declarations ( + host_uuid, + status, + operation_type, + md5_checksum, + declaration_uuid +) VALUES (?,?,?,?,?)` + + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(context.Background(), stmt, + hostUUID, + fleet.MDMDeliveryPending, + fleet.MDMOperationTypeInstall, + decl.MD5Checksum, + decl.DeclarationUUID, + ) + return err + }) + } + // initialize a time to use for our first declaration, subsequent declarations will be // incremented by a minute then := time.Now().UTC().Truncate(time.Second).Add(-1 * time.Hour) @@ -12573,6 +12594,7 @@ INSERT INTO mdm_apple_declarations ( }, } insertDeclaration(t, noTeamDeclsByUUID["123"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["123"]) mapDeclsByChecksum := func(byUUID map[string]fleet.MDMAppleDeclaration) map[string]fleet.MDMAppleDeclaration { byChecksum := make(map[string]fleet.MDMAppleDeclaration) @@ -12680,6 +12702,7 @@ INSERT INTO mdm_apple_declarations ( UploadedAt: then.Add(1 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["456"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["456"]) // get tokens again, timestamp and token should have changed r, err = mdmDevice.DeclarativeManagement("tokens") @@ -12707,6 +12730,7 @@ INSERT INTO mdm_apple_declarations ( UploadedAt: then.Add(2 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["789"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["789"]) // get tokens again, timestamp and token should have changed r, err = mdmDevice.DeclarativeManagement("tokens") @@ -12752,6 +12776,7 @@ INSERT INTO mdm_apple_declarations ( UploadedAt: then.Add(3 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["abc"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["abc"]) want = noTeamDeclsByUUID["abc"] wantBytes, err = json.Marshal(want.Declaration) require.NoError(t, err) From 01e3b94e55a31c3740a9de6f22e6a1558e41db16 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:15:07 -0500 Subject: [PATCH 09/29] Update backend APIs for macOS DDM (#17734) Co-authored-by: Jahziel Villasana-Espinoza --- changes/17404-mdm-custom-settings | 1 + cmd/fleetctl/apply_test.go | 4 +- cmd/fleetctl/get_test.go | 2 +- cmd/fleetctl/gitops_test.go | 7 +- server/datastore/mysql/apple_mdm.go | 305 +++++++++++++++++++++++ server/datastore/mysql/apple_mdm_test.go | 100 ++++++++ server/datastore/mysql/mdm.go | 71 +++++- server/datastore/mysql/mdm_test.go | 125 +++++++--- server/datastore/mysql/testing_utils.go | 11 +- server/fleet/apple_mdm.go | 84 ++++++- server/fleet/datastore.go | 5 +- server/fleet/service.go | 2 + server/mdm/mdm.go | 2 +- server/mdm/mdm_test.go | 10 + server/mock/datastore_mock.go | 18 +- server/service/apple_mdm.go | 98 ++++++++ server/service/apple_mdm_test.go | 13 + server/service/integration_mdm_test.go | 171 ++++++++++++- server/service/mdm.go | 132 ++++++++-- server/service/mdm_test.go | 3 +- 20 files changed, 1086 insertions(+), 78 deletions(-) create mode 100644 changes/17404-mdm-custom-settings diff --git a/changes/17404-mdm-custom-settings b/changes/17404-mdm-custom-settings new file mode 100644 index 0000000000..78b0506bd0 --- /dev/null +++ b/changes/17404-mdm-custom-settings @@ -0,0 +1 @@ +- Adds API functionality for creating DDM declarations, both individually and as a batch. \ No newline at end of file diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index cd9340a19c..63e6b61b3b 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -158,7 +158,7 @@ func TestApplyTeamSpecs(t *testing.T) { return nil } - ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile) error { + ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration) error { return nil } @@ -1038,7 +1038,7 @@ func TestApplyAsGitOps(t *testing.T) { teamEnrollSecrets = secrets return nil } - ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile) error { + ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration) error { return nil } ds.BulkSetPendingMDMHostProfilesFunc = func(ctx context.Context, hostIDs, teamIDs []uint, profileUUIDs, hostUUIDs []string) error { diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 8bf99d7742..d6d8ec370d 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -2205,7 +2205,7 @@ func TestGetTeamsYAMLAndApply(t *testing.T) { } return nil, fmt.Errorf("team not found: %s", name) } - ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile) error { + ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration) error { return nil } ds.BulkSetPendingMDMHostProfilesFunc = func(ctx context.Context, hostIDs, teamIDs []uint, profileUUIDs, uuids []string) error { diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 52fdba57ef..1c184acd80 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -25,6 +25,7 @@ func TestBasicGlobalGitOps(t *testing.T) { ds.BatchSetMDMProfilesFunc = func( ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, + macDecls []*fleet.MDMAppleDeclaration, ) error { return nil } @@ -133,7 +134,7 @@ func TestBasicTeamGitOps(t *testing.T) { ds.BatchSetScriptsFunc = func(ctx context.Context, tmID *uint, scripts []*fleet.Script) error { return nil } ds.BatchSetMDMProfilesFunc = func( - ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, + ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration, ) error { return nil } @@ -246,7 +247,7 @@ func TestFullGlobalGitOps(t *testing.T) { var appliedMacProfiles []*fleet.MDMAppleConfigProfile var appliedWinProfiles []*fleet.MDMWindowsConfigProfile ds.BatchSetMDMProfilesFunc = func( - ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, + ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration, ) error { appliedMacProfiles = macProfiles appliedWinProfiles = winProfiles @@ -403,7 +404,7 @@ func TestFullTeamGitOps(t *testing.T) { var appliedMacProfiles []*fleet.MDMAppleConfigProfile var appliedWinProfiles []*fleet.MDMWindowsConfigProfile ds.BatchSetMDMProfilesFunc = func( - ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, + ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration, ) error { appliedMacProfiles = macProfiles appliedWinProfiles = winProfiles diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 58a1966a9f..b03382b546 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -107,6 +107,25 @@ func formatErrorDuplicateConfigProfile(err error, cp *fleet.MDMAppleConfigProfil } } +func formatErrorDuplicateDeclaration(err error, decl *fleet.MDMAppleDeclaration) error { + switch { + case strings.Contains(err.Error(), "idx_mdm_apple_config_prof_team_identifier"): + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadIdentifier", + Identifier: decl.Identifier, + TeamID: decl.TeamID, + } + case strings.Contains(err.Error(), "idx_mdm_apple_config_prof_team_name"): + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + Identifier: decl.Name, + TeamID: decl.TeamID, + } + default: + return err + } +} + func (ds *Datastore) ListMDMAppleConfigProfiles(ctx context.Context, teamID *uint) ([]*fleet.MDMAppleConfigProfile, error) { stmt := ` SELECT @@ -3082,6 +3101,292 @@ WHERE h.uuid = ? return nil } +func (ds *Datastore) batchSetMDMAppleDeclarations(ctx context.Context, tx sqlx.ExtContext, tmID *uint, declarations []*fleet.MDMAppleDeclaration) ([]*fleet.MDMAppleDeclaration, error) { + const insertStmt = ` +INSERT INTO mdm_apple_declarations ( + declaration_uuid, + identifier, + name, + declaration_type, + declaration, + md5_checksum, + uploaded_at, + team_id +) +VALUES ( + ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? +) +ON DUPLICATE KEY UPDATE + uploaded_at = IF(md5_checksum = VALUES(md5_checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), + md5_checksum = VALUES(md5_checksum), + name = VALUES(name), + declaration = VALUES(declaration) +` + + fmtDeleteStmt := ` +DELETE FROM + mdm_apple_declarations +WHERE + team_id = ? AND %s +` + andIdentNotInList := "identifier NOT IN (?)" // added to fmtDeleteStmt if needed + + const loadExistingDecls = ` +SELECT + identifier, + declaration_uuid, + declaration +FROM + mdm_apple_declarations +WHERE + team_id = ? AND + identifier IN (?) +` + + var declTeamID uint + if tmID != nil { + declTeamID = *tmID + } + + var incomingLabels []fleet.DeclarationLabel + + // build a list of identifiers for the incoming declarations, will keep the + // existing ones if there's a match and no change + incomingIdents := make([]string, len(declarations)) + // at the same time, index the incoming declarations keyed by identifier for ease + // or processing + incomingDecls := make(map[string]*fleet.MDMAppleDeclaration, len(declarations)) + for i, p := range declarations { + incomingIdents[i] = p.Identifier + incomingDecls[p.Identifier] = p + } + + var existingDecls []*fleet.MDMAppleDeclaration + + if len(incomingIdents) > 0 { + // load existing declarations that match the incoming declarations by identifiers + stmt, args, err := sqlx.In(loadExistingDecls, declTeamID, incomingIdents) + if err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "inselect") { // TODO(JVE): do we need to create similar errors for testing decls? + if err == nil { + err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + } + return nil, ctxerr.Wrap(ctx, err, "build query to load existing declarations") + } + if err := sqlx.SelectContext(ctx, tx, &existingDecls, stmt, args...); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "select") { + if err == nil { + err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + } + return nil, ctxerr.Wrap(ctx, err, "load existing declarations") + } + } + + // figure out if we need to delete any declarations + keepIdents := make([]any, 0, len(incomingIdents)) + for _, p := range existingDecls { + if newP := incomingDecls[p.Identifier]; newP != nil { + keepIdents = append(keepIdents, p.Identifier) + } + } + + var delArgs []any + var delStmt string + if len(keepIdents) == 0 { + // delete all declarations for the team + delStmt = fmt.Sprintf(fmtDeleteStmt, "TRUE") + delArgs = []any{declTeamID} + } else { + // delete the obsolete declarations (all those that are not in keepIdents) + stmt, args, err := sqlx.In(fmt.Sprintf(fmtDeleteStmt, andIdentNotInList), declTeamID, keepIdents) + // if err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "inselect") { // TODO(JVE): do we need to create similar errors for testing decls? + // if err == nil { + // err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + // } + // return nil, ctxerr.Wrap(ctx, err, "build query to load existing declarations") + // } + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "build query to delete obsolete profiles") + } + delStmt = stmt + delArgs = args + } + + if _, err := tx.ExecContext(ctx, delStmt, delArgs...); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "delete") { + if err == nil { + err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + } + return nil, ctxerr.Wrap(ctx, err, "delete obsolete profiles") + } + + for _, d := range declarations { + checksum := md5ChecksumScriptContent(string(d.Declaration)) + declUUID := "x" + uuid.NewString() + if _, err := tx.ExecContext(ctx, insertStmt, + declUUID, + d.Identifier, + d.Name, + d.DeclarationType, + d.Declaration, + checksum, + declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") { + if err == nil { + err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + } + return nil, ctxerr.Wrapf(ctx, err, "insert new/edited declaration with identifier %q", d.Identifier) + } + + d.DeclarationUUID = declUUID + for _, l := range d.Labels { + l.DeclarationUUID = declUUID + incomingLabels = append(incomingLabels, l) + } + } + + if err := batchSetDeclarationLabelAssociationsDB(ctx, tx, incomingLabels); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "labels") { + if err == nil { + err = errors.New(ds.testBatchSetMDMAppleProfilesErr) + } + return nil, ctxerr.Wrap(ctx, err, "inserting apple profile label associations") + } + + return declarations, nil +} + +func (ds *Datastore) NewMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { + declUUID := "x" + uuid.NewString() + checksum := md5ChecksumScriptContent(string(declaration.Declaration)) + + stmt := ` +INSERT INTO mdm_apple_declarations ( + declaration_uuid, + team_id, + identifier, + name, + declaration_type, + declaration, + md5_checksum, + uploaded_at +) +VALUES ( + ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() +) + ` + + var tmID uint + if declaration.TeamID != nil { + tmID = *declaration.TeamID + } + + err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { + res, err := tx.ExecContext(ctx, stmt, + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.DeclarationType, declaration.Declaration, checksum) + if err != nil { + switch { + case isDuplicate(err): + return ctxerr.Wrap(ctx, formatErrorDuplicateDeclaration(err, declaration)) + default: + return ctxerr.Wrap(ctx, err, "creating new apple mdm declaration") + } + } + + aff, _ := res.RowsAffected() + if aff == 0 { + return &existsError{ + ResourceType: "MDMAppleDeclaration.Name", + Identifier: declaration.Name, + TeamID: declaration.TeamID, + } + } + + for i := range declaration.Labels { + declaration.Labels[i].DeclarationUUID = declUUID + } + if err := batchSetDeclarationLabelAssociationsDB(ctx, tx, declaration.Labels); err != nil { + return ctxerr.Wrap(ctx, err, "inserting mdm declaration label associations") + } + + return nil + }) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "inserting declaration and label associations") + } + + declaration.DeclarationUUID = declUUID + return declaration, nil +} + +func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtContext, declarationLabels []fleet.DeclarationLabel) error { + if len(declarationLabels) == 0 { + return nil + } + + // delete any profile+label tuple that is NOT in the list of provided tuples + // but are associated with the provided profiles (so we don't delete + // unrelated profile+label tuples) + deleteStmt := ` + DELETE FROM mdm_declaration_labels + WHERE (declaration_uuid, label_id) NOT IN (%s) AND + declaration_uuid IN (?) + ` + + upsertStmt := ` + INSERT INTO mdm_declaration_labels + (declaration_uuid, label_id, label_name) + VALUES + %s + ON DUPLICATE KEY UPDATE + label_id = VALUES(label_id) + ` + + var ( + insertBuilder strings.Builder + deleteBuilder strings.Builder + insertParams []any + deleteParams []any + + setProfileUUIDs = make(map[string]struct{}) + ) + for i, pl := range declarationLabels { + if i > 0 { + insertBuilder.WriteString(",") + deleteBuilder.WriteString(",") + } + insertBuilder.WriteString("(?, ?, ?)") + deleteBuilder.WriteString("(?, ?)") + insertParams = append(insertParams, pl.DeclarationUUID, pl.LabelID, pl.LabelName) + deleteParams = append(deleteParams, pl.DeclarationUUID, pl.LabelID) + + setProfileUUIDs[pl.DeclarationUUID] = struct{}{} + } + + _, err := tx.ExecContext(ctx, fmt.Sprintf(upsertStmt, insertBuilder.String()), insertParams...) + if err != nil { + if isChildForeignKeyError(err) { + // one of the provided labels doesn't exist + return foreignKey("mdm_declaration_labels", fmt.Sprintf("(declaration, label)=(%v)", insertParams)) + } + + return ctxerr.Wrap(ctx, err, "setting label associations for declarations") + } + + deleteStmt = fmt.Sprintf(deleteStmt, deleteBuilder.String()) + + profUUIDs := make([]string, 0, len(setProfileUUIDs)) + for k := range setProfileUUIDs { + profUUIDs = append(profUUIDs, k) + } + deleteArgs := append(deleteParams, profUUIDs) + + deleteStmt, args, err := sqlx.In(deleteStmt, deleteArgs...) + if err != nil { + return ctxerr.Wrap(ctx, err, "sqlx.In delete labels for declarations") + } + if _, err := tx.ExecContext(ctx, deleteStmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "deleting labels for declarations") + } + + return nil +} + func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { const stmt = ` SELECT diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 8d0662d91f..990bcc02f2 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -993,6 +993,81 @@ func expectAppleProfiles( return m } +func expectAppleDeclarations( + t *testing.T, + ds *Datastore, + tmID *uint, + want []*fleet.MDMAppleDeclaration, +) map[string]string { + if tmID == nil { + tmID = ptr.Uint(0) + } + + var got []*fleet.MDMAppleDeclaration + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + ctx := context.Background() + return sqlx.SelectContext(ctx, q, &got, `SELECT * FROM mdm_apple_declarations WHERE team_id = ?`, tmID) + }) + + // create map of expected declarations keyed by identifier + wantMap := make(map[string]*fleet.MDMAppleDeclaration, len(want)) + for _, cp := range want { + wantMap[cp.Identifier] = cp + } + + JSONRemarshal := func(bytes []byte) ([]byte, error) { + var ifce interface{} + err := json.Unmarshal(bytes, &ifce) + if err != nil { + return nil, err + } + return json.Marshal(ifce) + } + + // compare only the fields we care about, and build the resulting map of + // declaration identifier as key to declaration UUID as value + m := make(map[string]string) + for _, gotD := range got { + + wantD := wantMap[gotD.Identifier] + + m[gotD.Identifier] = gotD.DeclarationUUID + if gotD.TeamID != nil && *gotD.TeamID == 0 { + gotD.TeamID = nil + } + + // DeclarationUUID is non-empty and starts with "x", but otherwise we don't + // care about it for test assertions. + require.NotEmpty(t, gotD.DeclarationUUID) + require.True(t, strings.HasPrefix(gotD.DeclarationUUID, "x")) + gotD.DeclarationUUID = "" + gotD.MD5Checksum = "" // don't care about md5checksum here + + gotD.CreatedAt = time.Time{} + + gotBytes, err := JSONRemarshal(gotD.Declaration) + require.NoError(t, err) + + wantBytes, err := JSONRemarshal(wantD.Declaration) + require.NoError(t, err) + + require.Equal(t, wantBytes, gotBytes) + + // if an expected uploaded_at timestamp is provided for this declaration, keep + // its value, otherwise clear it as we don't care about asserting its + // value. + if wantD == nil || wantD.UploadedAt.IsZero() { + gotD.UploadedAt = time.Time{} + } + + require.Equal(t, wantD.Name, gotD.Name) + require.Equal(t, wantD.Identifier, gotD.Identifier) + require.Equal(t, wantD.Labels, gotD.Labels) + require.Equal(t, wantD.DeclarationType, gotD.DeclarationType) + } + return m +} + func testBatchSetMDMAppleProfiles(t *testing.T, ds *Datastore) { ctx := context.Background() applyAndExpect := func(newSet []*fleet.MDMAppleConfigProfile, tmID *uint, want []*fleet.MDMAppleConfigProfile) map[string]string { @@ -1171,6 +1246,31 @@ func configProfileForTest(t *testing.T, name, identifier, uuid string, labels .. return cp } +func declForTest(name, identifier, payloadContent string, labels ...*fleet.Label) *fleet.MDMAppleDeclaration { + tmpl := `{ + "Type": "com.apple.configuration.decl%s", + "Identifier": "com.fleet.config%s", + "Payload": { + "ServiceType": "com.apple.service%s" + } + }` + + declBytes := []byte(fmt.Sprintf(tmpl, identifier, identifier, payloadContent)) + + decl := &fleet.MDMAppleDeclaration{ + Declaration: declBytes, + DeclarationType: fleet.MDMAppleDeclarativeConfiguration, + Identifier: fmt.Sprintf("com.fleet.config%s", identifier), + Name: name, + } + + for _, l := range labels { + decl.Labels = append(decl.Labels, fleet.DeclarationLabel{LabelName: l.Name, LabelID: l.ID}) + } + + return decl +} + func teamConfigProfileForTest(t *testing.T, name, identifier, uuid string, teamID uint) *fleet.MDMAppleConfigProfile { prof := configProfileBytesForTest(name, identifier, uuid) cp, err := fleet.NewMDMAppleConfigProfile(configProfileBytesForTest(name, identifier, uuid), &teamID) diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index 8960bb6287..947902bdbe 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -82,7 +82,6 @@ func (ds *Datastore) ListMDMCommands( tmFilter fleet.TeamFilter, listOpts *fleet.MDMCommandListOptions, ) ([]*fleet.MDMCommand, error) { - jointStmt := getCombinedMDMCommandsQuery() + ds.whereFilterHostsByTeams(tmFilter, "h") jointStmt, params := appendListOptionsWithCursorToSQL(jointStmt, nil, &listOpts.ListOptions) var results []*fleet.MDMCommand @@ -102,7 +101,7 @@ func (ds *Datastore) getMDMCommand(ctx context.Context, q sqlx.QueryerContext, c return &cmd, nil } -func (ds *Datastore) BatchSetMDMProfiles(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile) error { +func (ds *Datastore) BatchSetMDMProfiles(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDeclarations []*fleet.MDMAppleDeclaration) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { if err := ds.batchSetMDMWindowsProfilesDB(ctx, tx, tmID, winProfiles); err != nil { return ctxerr.Wrap(ctx, err, "batch set windows profiles") @@ -112,6 +111,10 @@ func (ds *Datastore) BatchSetMDMProfiles(ctx context.Context, tmID *uint, macPro return ctxerr.Wrap(ctx, err, "batch set apple profiles") } + if _, err := ds.batchSetMDMAppleDeclarations(ctx, tx, tmID, macDeclarations); err != nil { + return ctxerr.Wrap(ctx, err, "batch set apple declarations") + } + return nil }) } @@ -122,6 +125,7 @@ func (ds *Datastore) ListMDMConfigProfiles(ctx context.Context, teamID *uint, op var profs []*fleet.MDMConfigProfilePayload + // TODO(roberto): Consider using UNION ALL here, as we know there won't be any duplicates between the tables. const selectStmt = ` SELECT profile_uuid, @@ -164,6 +168,21 @@ FROM ( WHERE team_id = ? AND name NOT IN (?) + + UNION + + SELECT + declaration_uuid AS profile_uuid, + team_id, + name, + 'darwin' AS platform, + identifier, + md5_checksum AS checksum, + created_at, + uploaded_at + FROM mdm_apple_declarations + WHERE team_id = ? + AND declaration_type <> ? ) as combined_profiles ` @@ -183,7 +202,7 @@ FROM ( fleetNames = append(fleetNames, k) } - args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames} + args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames, globalOrTeamID, fleet.MDMAppleDeclarativeActivation} stmt, args := appendListOptionsWithCursorToSQL(selectStmt, args, &opt) stmt, args, err := sqlx.In(stmt, args...) @@ -205,11 +224,16 @@ FROM ( } // load the labels associated with those profiles - var winProfUUIDs, macProfUUIDs []string + var winProfUUIDs, macProfUUIDs, macDeclUUIDs []string for _, prof := range profs { if prof.Platform == "windows" { winProfUUIDs = append(winProfUUIDs, prof.ProfileUUID) } else { + if strings.HasPrefix(prof.ProfileUUID, "x") { + macDeclUUIDs = append(macDeclUUIDs, prof.ProfileUUID) + continue + } + macProfUUIDs = append(macProfUUIDs, prof.ProfileUUID) } } @@ -218,6 +242,13 @@ FROM ( return nil, nil, err } + declLabels, err := ds.listDeclarationLabelsForDeclarations(ctx, macDeclUUIDs) + if err != nil { + return nil, nil, err + } + + labels = append(labels, declLabels...) + // match the labels with their profiles profMap := make(map[string]*fleet.MDMConfigProfilePayload, len(profs)) for _, prof := range profs { @@ -232,6 +263,38 @@ FROM ( return profs, metaData, nil } +// Note: we're using the ConfigurationProfileLabel type here since from the product perspective, MDM +// profiles and declarations are both "profiles". +func (ds *Datastore) listDeclarationLabelsForDeclarations(ctx context.Context, declUUIDs []string) ([]fleet.ConfigurationProfileLabel, error) { + if len(declUUIDs) == 0 { + return []fleet.ConfigurationProfileLabel{}, nil + } + + stmt := ` +SELECT + declaration_uuid AS profile_uuid, + label_name, + label_id +FROM + mdm_declaration_labels +WHERE + declaration_uuid IN (?) +ORDER BY + declaration_uuid, label_name + ` + + stmt, args, err := sqlx.In(stmt, declUUIDs) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "sqlx.In to list labels for declarations") + } + + var labels []fleet.ConfigurationProfileLabel + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &labels, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "select declaration labels") + } + return labels, nil +} + func (ds *Datastore) listProfileLabelsForProfiles(ctx context.Context, winProfUUIDs, macProfUUIDs []string) ([]fleet.ConfigurationProfileLabel, error) { // load the labels associated with those profiles const labelsStmt = ` diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index 6cd8d0c783..f5784bdb10 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -189,15 +189,18 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { applyAndExpect := func( newAppleSet []*fleet.MDMAppleConfigProfile, newWindowsSet []*fleet.MDMWindowsConfigProfile, + newAppleDeclSet []*fleet.MDMAppleDeclaration, tmID *uint, wantApple []*fleet.MDMAppleConfigProfile, wantWindows []*fleet.MDMWindowsConfigProfile, + wantAppleDecl []*fleet.MDMAppleDeclaration, ) { ctx := context.Background() - err := ds.BatchSetMDMProfiles(ctx, tmID, newAppleSet, newWindowsSet) + err := ds.BatchSetMDMProfiles(ctx, tmID, newAppleSet, newWindowsSet, newAppleDeclSet) require.NoError(t, err) expectAppleProfiles(t, ds, tmID, wantApple) expectWindowsProfiles(t, ds, tmID, wantWindows) + expectAppleDeclarations(t, ds, tmID, wantAppleDecl) } withTeamIDApple := func(p *fleet.MDMAppleConfigProfile, tmID uint) *fleet.MDMAppleConfigProfile { @@ -205,30 +208,39 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { return p } + withTeamIDDecl := func(d *fleet.MDMAppleDeclaration, tmID uint) *fleet.MDMAppleDeclaration { + d.TeamID = &tmID + return d + } + withTeamIDWindows := func(p *fleet.MDMWindowsConfigProfile, tmID uint) *fleet.MDMWindowsConfigProfile { p.TeamID = &tmID return p } // empty set for no team (both Apple and Windows) - applyAndExpect(nil, nil, nil, nil, nil) + applyAndExpect(nil, nil, nil, nil, nil, nil, nil) // single Apple and Windows profile set for a specific team applyAndExpect( []*fleet.MDMAppleConfigProfile{configProfileForTest(t, "N1", "I1", "a")}, []*fleet.MDMWindowsConfigProfile{windowsConfigProfileForTest(t, "W1", "l1")}, + []*fleet.MDMAppleDeclaration{declForTest("D1", "D1", "foo")}, ptr.Uint(1), []*fleet.MDMAppleConfigProfile{withTeamIDApple(configProfileForTest(t, "N1", "I1", "a"), 1)}, []*fleet.MDMWindowsConfigProfile{withTeamIDWindows(windowsConfigProfileForTest(t, "W1", "l1"), 1)}, + []*fleet.MDMAppleDeclaration{withTeamIDDecl(declForTest("D1", "D1", "foo"), 1)}, ) // single Apple and Windows profile set for no team applyAndExpect( []*fleet.MDMAppleConfigProfile{configProfileForTest(t, "N1", "I1", "a")}, []*fleet.MDMWindowsConfigProfile{windowsConfigProfileForTest(t, "W1", "l1")}, + []*fleet.MDMAppleDeclaration{declForTest("D1", "D1", "foo")}, nil, []*fleet.MDMAppleConfigProfile{configProfileForTest(t, "N1", "I1", "a")}, []*fleet.MDMWindowsConfigProfile{windowsConfigProfileForTest(t, "W1", "l1")}, + []*fleet.MDMAppleDeclaration{declForTest("D1", "D1", "foo")}, ) // new Apple and Windows profile sets for a specific team @@ -241,6 +253,10 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "W1", "l1"), // unchanged windowsConfigProfileForTest(t, "W2", "l2"), }, + []*fleet.MDMAppleDeclaration{ + declForTest("D1", "D1", "foo"), // unchanged + declForTest("D2", "D2", "foo"), + }, ptr.Uint(1), []*fleet.MDMAppleConfigProfile{ withTeamIDApple(configProfileForTest(t, "N1", "I1", "a"), 1), @@ -250,6 +266,10 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { withTeamIDWindows(windowsConfigProfileForTest(t, "W1", "l1"), 1), withTeamIDWindows(windowsConfigProfileForTest(t, "W2", "l2"), 1), }, + []*fleet.MDMAppleDeclaration{ + withTeamIDDecl(declForTest("D1", "D1", "foo"), 1), + withTeamIDDecl(declForTest("D2", "D2", "foo"), 1), + }, ) // edited profiles, unchanged profiles, and new profiles for a specific team @@ -264,6 +284,11 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "W2", "l2"), // unchanged windowsConfigProfileForTest(t, "W3", "l3"), // new }, + []*fleet.MDMAppleDeclaration{ + declForTest("D1", "D1", "foo-updated"), // content updated + declForTest("D2", "D2", "foo"), // unchanged + declForTest("D3", "D3", "bar"), // new + }, ptr.Uint(1), []*fleet.MDMAppleConfigProfile{ withTeamIDApple(configProfileForTest(t, "N1", "I1", "a-updated"), 1), @@ -275,6 +300,11 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { withTeamIDWindows(windowsConfigProfileForTest(t, "W2", "l2"), 1), withTeamIDWindows(windowsConfigProfileForTest(t, "W3", "l3"), 1), }, + []*fleet.MDMAppleDeclaration{ + withTeamIDDecl(declForTest("D1", "D1", "foo-updated"), 1), + withTeamIDDecl(declForTest("D2", "D2", "foo"), 1), + withTeamIDDecl(declForTest("D3", "D3", "bar"), 1), + }, ) // new Apple and Windows profiles to no team @@ -287,6 +317,10 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "W4", "l4"), windowsConfigProfileForTest(t, "W5", "l5"), }, + []*fleet.MDMAppleDeclaration{ + declForTest("D5", "D4", "foo"), + declForTest("D4", "D5", "foo"), + }, nil, []*fleet.MDMAppleConfigProfile{ configProfileForTest(t, "N4", "I4", "d"), @@ -296,10 +330,14 @@ func testBatchSetMDMProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "W4", "l4"), windowsConfigProfileForTest(t, "W5", "l5"), }, + []*fleet.MDMAppleDeclaration{ + declForTest("D5", "D4", "foo"), + declForTest("D4", "D5", "foo"), + }, ) // Test Case 8: Clear profiles for a specific team - applyAndExpect(nil, nil, ptr.Uint(1), nil, nil) + applyAndExpect(nil, nil, nil, ptr.Uint(1), nil, nil, nil) } func testListMDMConfigProfiles(t *testing.T, ds *Datastore) { @@ -724,7 +762,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "G2w", "L2"), windowsConfigProfileForTest(t, "G3w", "L3"), } - err = ds.BatchSetMDMProfiles(ctx, nil, macGlobalProfiles, winGlobalProfiles) + err = ds.BatchSetMDMProfiles(ctx, nil, macGlobalProfiles, winGlobalProfiles, nil) require.NoError(t, err) macGlobalProfiles, err = ds.ListMDMAppleConfigProfiles(ctx, nil) require.NoError(t, err) @@ -911,7 +949,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "T1.1w", "T1.1"), windowsConfigProfileForTest(t, "T1.2w", "T1.2"), } - err = ds.BatchSetMDMProfiles(ctx, &team1.ID, tm1DarwinProfiles, tm1WindowsProfiles) + err = ds.BatchSetMDMProfiles(ctx, &team1.ID, tm1DarwinProfiles, tm1WindowsProfiles, nil) require.NoError(t, err) tm1Profiles := getProfs(&team1.ID) @@ -1007,7 +1045,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "T1.3w", "T1.3"), } - err = ds.BatchSetMDMProfiles(ctx, &team1.ID, newTm1DarwinProfiles, newTm1WindowsProfiles) + err = ds.BatchSetMDMProfiles(ctx, &team1.ID, newTm1DarwinProfiles, newTm1WindowsProfiles, nil) require.NoError(t, err) newTm1Profiles := getProfs(&team1.ID) @@ -1063,7 +1101,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "T1.3w", "T1.3"), } - err = ds.BatchSetMDMProfiles(ctx, &team1.ID, newTm1DarwinProfiles, newTm1WindowsProfiles) + err = ds.BatchSetMDMProfiles(ctx, &team1.ID, newTm1DarwinProfiles, newTm1WindowsProfiles, nil) require.NoError(t, err) newTm1Profiles = getProfs(&team1.ID) require.Len(t, newTm1Profiles, 6) @@ -1117,7 +1155,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "G4w", "G4"), } - err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles) + err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles, nil) require.NoError(t, err) newGlobalProfiles := getProfs(nil) @@ -1176,7 +1214,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "G5w", "G5"), } - err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles) + err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles, nil) require.NoError(t, err) newGlobalProfiles = getProfs(nil) require.Len(t, newGlobalProfiles, 8) @@ -1270,7 +1308,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "T2.1w", "T2.1"), } - err = ds.BatchSetMDMProfiles(ctx, &team2.ID, tm2DarwinProfiles, tm2WindowsProfiles) + err = ds.BatchSetMDMProfiles(ctx, &team2.ID, tm2DarwinProfiles, tm2WindowsProfiles, nil) require.NoError(t, err) tm2Profiles := getProfs(&team2.ID) require.Len(t, tm2Profiles, 2) @@ -1369,7 +1407,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "G7w", "G7", labels[5]), } - err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles) + err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles, nil) require.NoError(t, err) newGlobalProfiles = getProfs(nil) require.Len(t, newGlobalProfiles, 12) @@ -1459,8 +1497,10 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // make the new Apple host a member of labels[0] and [1] // make the new Windows host a member of labels[3] and [4] err = ds.AsyncBatchInsertLabelMembership(ctx, [][2]uint{ - {labels[0].ID, darwinHosts[3].ID}, {labels[1].ID, darwinHosts[3].ID}, - {labels[3].ID, windowsHosts[3].ID}, {labels[4].ID, windowsHosts[3].ID}, + {labels[0].ID, darwinHosts[3].ID}, + {labels[1].ID, darwinHosts[3].ID}, + {labels[3].ID, windowsHosts[3].ID}, + {labels[4].ID, windowsHosts[3].ID}, }) require.NoError(t, err) @@ -1524,10 +1564,18 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // make the darwinHosts[2] host a member of all labels // make the windowsHosts[2] host a member of all labels err = ds.AsyncBatchInsertLabelMembership(ctx, [][2]uint{ - {labels[0].ID, darwinHosts[2].ID}, {labels[1].ID, darwinHosts[2].ID}, {labels[2].ID, darwinHosts[2].ID}, - {labels[3].ID, darwinHosts[2].ID}, {labels[4].ID, darwinHosts[2].ID}, {labels[5].ID, darwinHosts[2].ID}, - {labels[0].ID, windowsHosts[2].ID}, {labels[1].ID, windowsHosts[2].ID}, {labels[2].ID, windowsHosts[2].ID}, - {labels[3].ID, windowsHosts[2].ID}, {labels[4].ID, windowsHosts[2].ID}, {labels[5].ID, windowsHosts[2].ID}, + {labels[0].ID, darwinHosts[2].ID}, + {labels[1].ID, darwinHosts[2].ID}, + {labels[2].ID, darwinHosts[2].ID}, + {labels[3].ID, darwinHosts[2].ID}, + {labels[4].ID, darwinHosts[2].ID}, + {labels[5].ID, darwinHosts[2].ID}, + {labels[0].ID, windowsHosts[2].ID}, + {labels[1].ID, windowsHosts[2].ID}, + {labels[2].ID, windowsHosts[2].ID}, + {labels[3].ID, windowsHosts[2].ID}, + {labels[4].ID, windowsHosts[2].ID}, + {labels[5].ID, windowsHosts[2].ID}, }) require.NoError(t, err) @@ -1661,8 +1709,10 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // update darwin/windows[2] so they are not members of labels[1][2] and [4][5], which // should remove the G7 label-based profile, but not G6 as it is broken. err = ds.AsyncBatchDeleteLabelMembership(ctx, [][2]uint{ - {labels[1].ID, darwinHosts[2].ID}, {labels[2].ID, darwinHosts[2].ID}, - {labels[4].ID, windowsHosts[2].ID}, {labels[5].ID, windowsHosts[2].ID}, + {labels[1].ID, darwinHosts[2].ID}, + {labels[2].ID, darwinHosts[2].ID}, + {labels[4].ID, windowsHosts[2].ID}, + {labels[5].ID, windowsHosts[2].ID}, }) require.NoError(t, err) @@ -1798,7 +1848,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "T2.2w", "T2.2", labels[4], labels[5]), } - err = ds.BatchSetMDMProfiles(ctx, &team2.ID, tm2DarwinProfiles, tm2WindowsProfiles) + err = ds.BatchSetMDMProfiles(ctx, &team2.ID, tm2DarwinProfiles, tm2WindowsProfiles, nil) require.NoError(t, err) tm2Profiles = getProfs(&team2.ID) require.Len(t, tm2Profiles, 4) @@ -1867,8 +1917,10 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // make darwinHosts[1] and windowsHosts[1] members of the required labels err = ds.AsyncBatchInsertLabelMembership(ctx, [][2]uint{ - {labels[1].ID, darwinHosts[1].ID}, {labels[2].ID, darwinHosts[1].ID}, - {labels[4].ID, windowsHosts[1].ID}, {labels[5].ID, windowsHosts[1].ID}, + {labels[1].ID, darwinHosts[1].ID}, + {labels[2].ID, darwinHosts[1].ID}, + {labels[4].ID, windowsHosts[1].ID}, + {labels[5].ID, windowsHosts[1].ID}, }) require.NoError(t, err) @@ -1999,8 +2051,10 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // remove team 2 hosts membership from labels err = ds.AsyncBatchDeleteLabelMembership(ctx, [][2]uint{ - {labels[1].ID, darwinHosts[1].ID}, {labels[2].ID, darwinHosts[1].ID}, - {labels[4].ID, windowsHosts[1].ID}, {labels[5].ID, windowsHosts[1].ID}, + {labels[1].ID, darwinHosts[1].ID}, + {labels[2].ID, darwinHosts[1].ID}, + {labels[4].ID, windowsHosts[1].ID}, + {labels[5].ID, windowsHosts[1].ID}, }) require.NoError(t, err) @@ -2221,7 +2275,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) configProfileForTest(t, "T1.2", "T1.2", "e"), } - err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil, nil) require.NoError(t, err) profs, _, err := ds.ListMDMConfigProfiles(ctx, &team.ID, fleet.ListOptions{}) @@ -2259,7 +2313,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) label, err := ds.NewLabel(ctx, &fleet.Label{Name: "test_label_1"}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil, nil) require.NoError(t, err) var uid string @@ -2332,7 +2386,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) testLabel3, err := ds.NewLabel(ctx, &fleet.Label{Name: "test_label_3"}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil, nil) require.NoError(t, err) var uid string @@ -2416,7 +2470,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) testLabel4, err := ds.NewLabel(ctx, &fleet.Label{Name: "test_label_4"}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, profiles, nil, nil) require.NoError(t, err) var uid string @@ -2490,7 +2544,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) windowsConfigProfileForTest(t, "T5.2", "T5.2"), } - err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles, nil) require.NoError(t, err) profs, _, err := ds.ListMDMConfigProfiles(ctx, &team.ID, fleet.ListOptions{}) @@ -2528,7 +2582,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) label, err := ds.NewLabel(ctx, &fleet.Label{Name: "test_label_6"}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles, nil) require.NoError(t, err) var uid string @@ -2601,7 +2655,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) testLabel3, err := ds.NewLabel(ctx, &fleet.Label{Name: uuid.NewString()}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles, nil) require.NoError(t, err) var uid string @@ -2685,7 +2739,7 @@ func testGetHostMDMProfilesExpectedForVerification(t *testing.T, ds *Datastore) label, err := ds.NewLabel(ctx, &fleet.Label{Name: uuid.NewString()}) require.NoError(t, err) - err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles) + err = ds.BatchSetMDMProfiles(ctx, &team.ID, nil, profiles, nil) require.NoError(t, err) var uid string @@ -3076,7 +3130,7 @@ func testBatchSetMDMProfilesTransactionError(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "W2", "l2"), } // set the initial profiles without error - err := ds.BatchSetMDMProfiles(ctx, nil, appleProfs, winProfs) + err := ds.BatchSetMDMProfiles(ctx, nil, appleProfs, winProfs, nil) require.NoError(t, err) // now ensure all steps are required (add a profile, delete a profile, set labels) @@ -3092,7 +3146,7 @@ func testBatchSetMDMProfilesTransactionError(t *testing.T, ds *Datastore) { ds.testBatchSetMDMAppleProfilesErr = c.appleErr ds.testBatchSetMDMWindowsProfilesErr = c.windowsErr - err = ds.BatchSetMDMProfiles(ctx, nil, appleProfs, winProfs) + err = ds.BatchSetMDMProfiles(ctx, nil, appleProfs, winProfs, nil) require.ErrorContains(t, err, c.wantErr) }) } @@ -3175,7 +3229,8 @@ func testSCEPRenewalHelpers(t *testing.T, ds *Datastore) { // use the host UUID, just to make sure they're // different from each other, we don't care about the // DER contents here - Raw: []byte(h.UUID)} + Raw: []byte(h.UUID), + } err = scepDepot.Put(cert.Subject.CommonName, cert) require.NoError(t, err) req := mdm.Request{ diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 6a329f7f97..2aa839d1fb 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -313,11 +313,12 @@ func TruncateTables(t testing.TB, ds *Datastore, tables ...string) { // be truncated - a more precise approach must be used for those, e.g. // delete where id > max before test, or something like that. nonEmptyTables := map[string]bool{ - "app_config_json": true, - "migration_status_tables": true, - "osquery_options": true, - "mdm_delivery_status": true, - "mdm_operation_types": true, + "app_config_json": true, + "migration_status_tables": true, + "osquery_options": true, + "mdm_delivery_status": true, + "mdm_operation_types": true, + "mdm_apple_declaration_types": true, } ctx := context.Background() diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 56116daf6d..db460be7e7 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -575,13 +575,72 @@ type MDMAppleDeclaration struct { // Declaration is the raw JSON content of the declaration Declaration json.RawMessage `db:"declaration" json:"-"` - // MD5Checksum is a checsum of the JSON contents + // MD5Checksum is a checksum of the JSON contents MD5Checksum string `db:"md5_checksum" json:"-"` + // Labels are the labels associated with this Declaration + Labels []DeclarationLabel `db:"labels" json:"labels,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"` } +type MDMAppleRawDeclaration struct { + // Type is the "Type" field on the raw declaration JSON. + Type string `json:"Type"` + Identifier string `json:"Identifier"` +} + +// ForbiddenDeclTypes is a set of declaration types that are not allowed to be +// added by users into Fleet. +var ForbiddenDeclTypes = map[string]struct{}{ + "com.apple.configuration.account.caldav": {}, + "com.apple.configuration.account.carddav": {}, + "com.apple.configuration.account.exchange": {}, + "com.apple.configuration.account.google": {}, + "com.apple.configuration.account.ldap": {}, + "com.apple.configuration.account.mail": {}, + "com.apple.configuration.management.test": {}, + "com.apple.configuration.screensharing.connection": {}, + "com.apple.configuration.security.certificate": {}, + "com.apple.configuration.security.identity": {}, + "com.apple.configuration.security.passkey.attestation": {}, + "com.apple.configuration.services.configuration-files": {}, + "com.apple.configuration.watch.enrollment": {}, +} + +func (r *MDMAppleRawDeclaration) ValidateUserProvided() error { + var err error + + // Check against types we don't allow + if r.Type == `com.apple.configuration.softwareupdate.enforcement.specific` { + return NewInvalidArgumentError(r.Type, "Declaration profile can’t include OS updates settings. To control these settings, go to OS updates.") + } + + if _, forbidden := ForbiddenDeclTypes[r.Type]; forbidden { + return NewInvalidArgumentError(r.Type, "Only configuration declarations that don’t require an asset reference are supported.") + } + + if r.Type == "com.apple.configuration.management.status-subscriptions" { + return NewInvalidArgumentError(r.Type, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") + } + + if !strings.HasPrefix(r.Type, string(MDMAppleDeclarativeConfiguration)) { + return NewInvalidArgumentError(r.Type, "Only configuration declarations (com.apple.configuration) are supported.") + } + + return err +} + +func GetRawDeclarationValues(raw []byte) (*MDMAppleRawDeclaration, error) { + var rawDecl MDMAppleRawDeclaration + if err := json.Unmarshal(raw, &rawDecl); err != nil { + return nil, err + } + + return &rawDecl, nil +} + // MDMAppleHostDeclaration represents the state of a declaration on a host type MDMAppleHostDeclaration struct { // HostUUID is the uuid of the host affected by this declaration @@ -610,6 +669,29 @@ type MDMAppleHostDeclaration struct { Detail string `db:"detail" json:"detail"` } +// DeclarationLabel represents the many-to-many relationship between +// declarations and labels. +// TODO(JVE): I think we can remove this type altogether, but double check first (mainly when +// ingesting declarations). +type DeclarationLabel struct { + DeclarationUUID string `db:"profile_uuid" json:"-"` + LabelName string `db:"label_name" json:"name"` + LabelID uint `db:"label_id" json:"id,omitempty"` // omitted if 0 (which is impossible if the label is not broken) + Broken bool `db:"broken" json:"broken,omitempty"` // omitted (not rendered to JSON) if false +} + +func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { + var decl MDMAppleDeclaration + + decl.DeclarationType = MDMAppleDeclarationType(strings.Join(strings.Split(declType, ".")[:3], ".")) + decl.Identifier = ident + decl.Name = name + decl.Declaration = raw + decl.TeamID = teamID + + return &decl +} + // MDMAppleDDMTokensResponse is the response from the DDM tokens endpoint. // // https://developer.apple.com/documentation/devicemanagement/tokensresponse diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index aac24814e9..9ce9691870 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1283,7 +1283,10 @@ type Datastore interface { // BatchSetMDMProfiles sets the MDM Apple or Windows profiles for the given team or // no team in a single transaction. - BatchSetMDMProfiles(ctx context.Context, tmID *uint, macProfiles []*MDMAppleConfigProfile, winProfiles []*MDMWindowsConfigProfile) error + BatchSetMDMProfiles(ctx context.Context, tmID *uint, macProfiles []*MDMAppleConfigProfile, winProfiles []*MDMWindowsConfigProfile, macDeclarations []*MDMAppleDeclaration) error + + // NewMDMAppleDeclaration creates and returns a new MDM Apple declaration. + NewMDMAppleDeclaration(ctx context.Context, declaration *MDMAppleDeclaration) (*MDMAppleDeclaration, error) /////////////////////////////////////////////////////////////////////////////// // Host Script Results diff --git a/server/fleet/service.go b/server/fleet/service.go index 2fef5eeec5..01c870ba8f 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -652,6 +652,8 @@ type Service interface { // NewMDMAppleConfigProfile creates a new configuration profile for the specified team. NewMDMAppleConfigProfile(ctx context.Context, teamID uint, r io.Reader, labels []string) (*MDMAppleConfigProfile, error) + // NewMDMAppleConfigProfileWithPayload creates a new declaration for the specified team. + NewMDMAppleDeclaration(ctx context.Context, teamID uint, r io.Reader, labels []string, name string) (*MDMAppleDeclaration, error) // GetMDMAppleConfigProfileByDeprecatedID retrieves the specified Apple // configuration profile via its numeric ID. This method is deprecated and // should not be used for new endpoints. diff --git a/server/mdm/mdm.go b/server/mdm/mdm.go index 00e515667b..5a15dd8c00 100644 --- a/server/mdm/mdm.go +++ b/server/mdm/mdm.go @@ -50,7 +50,7 @@ func GetRawProfilePlatform(profile []byte) string { bytes.EqualFold(prefix, trimmedProfile[:len(prefix)]) } - if prefixMatches([]byte("= 1 { + tmID = &teamID + } + + validatedLabels, err := svc.validateDeclarationLabels(ctx, labels) + if err != nil { + return nil, err + } + + // TODO(roberto): Maybe GetRawDeclarationValues belongs inside NewMDMAppleDeclaration? We can refactor this in a follow up. + rawDecl, err := fleet.GetRawDeclarationValues(data) + if err != nil { + return nil, err + } + + if err := rawDecl.ValidateUserProvided(); err != nil { + return nil, err + } + + d := fleet.NewMDMAppleDeclaration(data, tmID, name, rawDecl.Type, rawDecl.Identifier) + + // TODO(roberto): Is this already handled in NewMDMAppleDeclaration? Could we add the labels as well? + d.Labels = validatedLabels + d.TeamID = tmID + + decl, err := svc.ds.NewMDMAppleDeclaration(ctx, d) + if err != nil { + return nil, err + } + + return decl, nil +} + +func (svc *Service) batchValidateDeclarationLabels(ctx context.Context, labelNames []string) (map[string]fleet.DeclarationLabel, error) { + if len(labelNames) == 0 { + return nil, nil + } + + labels, err := svc.ds.LabelIDsByName(ctx, labelNames) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "getting label IDs by name") + } + + uniqueNames := make(map[string]bool) + for _, entry := range labelNames { + if _, value := uniqueNames[entry]; !value { + uniqueNames[entry] = true + } + } + + if len(labels) != len(uniqueNames) { + return nil, &fleet.BadRequestError{ + Message: "some or all the labels provided don't exist", + InternalErr: fmt.Errorf("names provided: %v", labelNames), + } + } + + profLabels := make(map[string]fleet.DeclarationLabel) + for labelName, labelID := range labels { + profLabels[labelName] = fleet.DeclarationLabel{ + LabelName: labelName, + LabelID: labelID, + } + } + return profLabels, nil +} + +func (svc *Service) validateDeclarationLabels(ctx context.Context, labelNames []string) ([]fleet.DeclarationLabel, error) { + labelMap, err := svc.batchValidateDeclarationLabels(ctx, labelNames) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "validating declaration labels") + } + + var declLabels []fleet.DeclarationLabel + for _, label := range labelMap { + declLabels = append(declLabels, label) + } + return declLabels, nil +} + type listMDMAppleConfigProfilesRequest struct { TeamID uint `query:"team_id,optional"` } diff --git a/server/service/apple_mdm_test.go b/server/service/apple_mdm_test.go index bd72cbd822..764f4c50ac 100644 --- a/server/service/apple_mdm_test.go +++ b/server/service/apple_mdm_test.go @@ -2727,6 +2727,19 @@ func mobileconfigForTest(name, identifier string) []byte { `, name, identifier, uuid.New().String())) } +func declBytesForTest(identifier string, payloadContent string) []byte { + tmpl := `{ + "Type": "com.apple.configuration.decl%s", + "Identifier": "com.fleet.config%s", + "Payload": { + "ServiceType": "com.apple.service%s" + } + }` + + declBytes := []byte(fmt.Sprintf(tmpl, identifier, identifier, payloadContent)) + return declBytes +} + func mobileconfigForTestWithContent(outerName, outerIdentifier, innerIdentifier, innerType, innerName string) []byte { if innerName == "" { innerName = outerName + ".inner" diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index e22e32c217..8d4c16dced 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9379,8 +9379,8 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { assertAppleProfile("win-team-profile.mobileconfig", "win-team-profile", "test-team-ident-2", 0, nil, http.StatusOK, "") // not an xml nor mobileconfig file - assertWindowsProfile("foo.txt", "./Test", 0, nil, http.StatusBadRequest, "Couldn't upload. The file should be a .mobileconfig or .xml file.") - assertAppleProfile("foo.txt", "foo", "foo-ident", 0, nil, http.StatusBadRequest, "Couldn't upload. The file should be a .mobileconfig or .xml file.") + assertWindowsProfile("foo.txt", "./Test", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") + assertAppleProfile("foo.txt", "foo", "foo-ident", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") // Windows-reserved LocURI assertWindowsProfile("bitlocker.xml", syncml.FleetBitLockerTargetLocURI, 0, nil, http.StatusBadRequest, "Couldn't upload. Custom configuration profiles can't include BitLocker settings.") @@ -12519,6 +12519,173 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } +func (s *integrationMDMTestSuite) TestAppleDDMBatchUpload() { + t := s.T() + tmpl := ` +{ + "Type": "com.apple.configuration.decl%d", + "Identifier": "com.fleet.config%d", + "Payload": { + "ServiceType": "com.apple.bash", + "DataAssetReference": "com.fleet.asset.bash" %s + } +}` + + newDeclBytes := func(i int, payload ...string) []byte { + var p string + if len(payload) > 0 { + p = "," + strings.Join(payload, ",") + } + return []byte(fmt.Sprintf(tmpl, i, i, p)) + } + + var decls [][]byte + + for i := 0; i < 7; i++ { + decls = append(decls, newDeclBytes(i)) + } + + // Non-configuration type should fail + res := s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad", Contents: []byte(`{"Type": "com.apple.activation"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg := extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Only configuration declarations (com.apple.configuration) are supported") + + // "com.apple.configuration.softwareupdate.enforcement.specific" type should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.softwareupdate.enforcement.specific"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Declaration profile can’t include OS updates settings. To control these settings, go to OS updates.") + + // Types from our list of forbidden types should fail + for ft := range fleet.ForbiddenDeclTypes { + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(fmt.Sprintf(`{"Type": "%s"}`, ft))}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Only configuration declarations that don’t require an asset reference are supported.") + } + + // "com.apple.configuration.management.status-subscriptions" type should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.management.status-subscriptions"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") + + // Two different payloads with the same name should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: newDeclBytes(1, `"foo": "bar"`)}, + {Name: "bad2", Contents: newDeclBytes(2, `"baz": "bing"`)}, + }}, http.StatusUnprocessableEntity) + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "A declaration profile with this name already exists.") + + // Same identifier should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N1", Contents: decls[0]}, + {Name: "N2", Contents: decls[0]}, + }}, http.StatusUnprocessableEntity) + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "A declaration profile with this identifier already exists.") + + // Create 2 declarations + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N1", Contents: decls[0]}, + {Name: "N2", Contents: decls[1]}, + }}, http.StatusNoContent) + + var resp listMDMConfigProfilesResponse + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N1", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N2", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // Create 2 new declarations. These should take the place of the first two. + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: decls[2]}, + {Name: "N4", Contents: decls[3]}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N4", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // replace only 1 declaration, the other one should be the same + + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: decls[2]}, + {Name: "N5", Contents: decls[4]}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N5", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // update the declarations + + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: newDeclBytes(2, `"foo": "bar"`)}, + {Name: "N5", Contents: newDeclBytes(4, `"bing": "baz"`)}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N5", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + var createResp createLabelResponse + s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_1"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) + require.NotZero(t, createResp.Label.ID) + require.Equal(t, "label_1", createResp.Label.Name) + lbl1 := createResp.Label.Label + + s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_2"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) + require.NotZero(t, createResp.Label.ID) + require.Equal(t, "label_2", createResp.Label.Name) + lbl2 := createResp.Label.Label + + // Add with labels + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N5", Contents: decls[5], Labels: []string{lbl1.Name, lbl2.Name}}, + {Name: "N6", Contents: decls[6], Labels: []string{lbl1.Name}}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N5", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N6", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + require.Len(t, resp.Profiles[0].Labels, 2) + require.Equal(t, lbl1.Name, resp.Profiles[0].Labels[0].LabelName) + require.Equal(t, lbl2.Name, resp.Profiles[0].Labels[1].LabelName) + require.Len(t, resp.Profiles[1].Labels, 1) + require.Equal(t, lbl1.Name, resp.Profiles[1].Labels[0].LabelName) +} + +// TODO(sarah): Build out this test func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { t := s.T() _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) diff --git a/server/service/mdm.go b/server/service/mdm.go index 82c23aa710..80ae99d97c 100644 --- a/server/service/mdm.go +++ b/server/service/mdm.go @@ -1221,7 +1221,23 @@ func newMDMConfigProfileEndpoint(ctx context.Context, request interface{}, svc f defer ff.Close() fileExt := filepath.Ext(req.Profile.Filename) - if isApple := strings.EqualFold(fileExt, ".mobileconfig"); isApple { + profileName := strings.TrimSuffix(filepath.Base(req.Profile.Filename), fileExt) + isMobileConfig := strings.EqualFold(fileExt, ".mobileconfig") + isJSON := strings.EqualFold(fileExt, ".json") + if isMobileConfig || isJSON { + // Then it's an Apple configuration file + if isJSON { + decl, err := svc.NewMDMAppleDeclaration(ctx, req.TeamID, ff, req.Labels, profileName) + if err != nil { + return &newMDMConfigProfileResponse{Err: err}, nil + } + + return &newMDMConfigProfileResponse{ + ProfileUUID: decl.DeclarationUUID, + }, nil + + } + cp, err := svc.NewMDMAppleConfigProfile(ctx, req.TeamID, ff, req.Labels) if err != nil { return &newMDMConfigProfileResponse{Err: err}, nil @@ -1232,7 +1248,6 @@ func newMDMConfigProfileEndpoint(ctx context.Context, request interface{}, svc f } if isWindows := strings.EqualFold(fileExt, ".xml"); isWindows { - profileName := strings.TrimSuffix(filepath.Base(req.Profile.Filename), fileExt) cp, err := svc.NewMDMWindowsConfigProfile(ctx, req.TeamID, profileName, ff, req.Labels) if err != nil { return &newMDMConfigProfileResponse{Err: err}, nil @@ -1254,7 +1269,7 @@ func (svc *Service) NewMDMUnsupportedConfigProfile(ctx context.Context, teamID u // this is required because we need authorize to return the error, and // svc.authz is only available on the concrete Service struct, not on the // Service interface so it cannot be done in the endpoint itself. - return &fleet.BadRequestError{Message: "Couldn't upload. The file should be a .mobileconfig or .xml file."} + return &fleet.BadRequestError{Message: "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file."} } func (svc *Service) NewMDMWindowsConfigProfile(ctx context.Context, teamID uint, profileName string, r io.Reader, labels []string) (*fleet.MDMWindowsConfigProfile, error) { @@ -1476,7 +1491,7 @@ func (svc *Service) BatchSetMDMProfiles( return ctxerr.Wrap(ctx, err, "validating labels") } - appleProfiles, err := getAppleProfiles(ctx, tmID, appCfg, profiles, labelMap) + appleProfiles, appleDecls, err := getAppleProfiles(ctx, tmID, appCfg, profiles, labelMap) if err != nil { return ctxerr.Wrap(ctx, err, "validating macOS profiles") } @@ -1490,7 +1505,7 @@ func (svc *Service) BatchSetMDMProfiles( return nil } - if err := svc.ds.BatchSetMDMProfiles(ctx, tmID, appleProfiles, windowsProfiles); err != nil { + if err := svc.ds.BatchSetMDMProfiles(ctx, tmID, appleProfiles, windowsProfiles, appleDecls); err != nil { return ctxerr.Wrap(ctx, err, "setting config profiles") } @@ -1576,17 +1591,94 @@ func getAppleProfiles( appCfg *fleet.AppConfig, profiles []fleet.MDMProfileBatchPayload, labelMap map[string]fleet.ConfigurationProfileLabel, -) ([]*fleet.MDMAppleConfigProfile, error) { +) ([]*fleet.MDMAppleConfigProfile, []*fleet.MDMAppleDeclaration, error) { // any duplicate identifier or name in the provided set results in an error profs := make([]*fleet.MDMAppleConfigProfile, 0, len(profiles)) - byName, byIdent := make(map[string]bool, len(profiles)), make(map[string]bool, len(profiles)) + decls := make([]*fleet.MDMAppleDeclaration, 0, len(profiles)) + // we need to keep track of the names and identifiers to check for duplicates so we will use + // a map where the key is the name oridentifier and the value is either "mobileconfig" or + // "declaration" to differentiate between the two types of profiles + byName, byIdent := make(map[string]string, len(profiles)), make(map[string]string, len(profiles)) for _, prof := range profiles { if mdm.GetRawProfilePlatform(prof.Contents) != "darwin" { continue } + + // Check for DDM files + + isJSON := func(b []byte) bool { + var js json.RawMessage + return json.Unmarshal(b, &js) == nil + } + + // TODO(roberto): As a mini optimization, GetRawDeclarationValues could replace isJSON. + if isJSON(prof.Contents) { + rawDecl, err := fleet.GetRawDeclarationValues(prof.Contents) + if err != nil { + return nil, nil, err + } + + if err := rawDecl.ValidateUserProvided(); err != nil { + return nil, nil, err + } + + mdmDecl := fleet.NewMDMAppleDeclaration(prof.Contents, tmID, prof.Name, rawDecl.Type, rawDecl.Identifier) + for _, labelName := range prof.Labels { + if lbl, ok := labelMap[labelName]; ok { + declLabel := fleet.DeclarationLabel{ + LabelName: lbl.LabelName, + LabelID: lbl.LabelID, + } + mdmDecl.Labels = append(mdmDecl.Labels, declLabel) + } + } + + v, ok := byName[mdmDecl.Name] + switch { + case !ok: + byName[mdmDecl.Name] = "declaration" + case v == "mobileconfig": + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Name, "A configuration profile with this name already exists."), + "duplicate mobileconfig profile by name") + case v == "declaration": + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Name, "A declaration profile with this name already exists."), + "duplicate declaration profile by name") + default: + // this should never happen but just in case + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Name, "A profile with this name already exists."), + "duplicate profile by name") + } + + v, ok = byIdent[mdmDecl.Identifier] + switch { + case !ok: + byIdent[mdmDecl.Identifier] = "declaration" + case v == "mobileconfig": + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Identifier, "A configuration profile with this identifier already exists."), + "duplicate mobileconfig profile by identifier") + case v == "declaration": + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Identifier, "A declaration profile with this identifier already exists."), + "duplicate declaration profile by identifier") + default: + // this should never happen but just in case + return nil, nil, ctxerr.Wrap(ctx, + fleet.NewInvalidArgumentError(mdmDecl.Identifier, "A profile with this identifier already exists."), + "duplicate identifier by identifier") + } + + decls = append(decls, mdmDecl) + + continue + } + mdmProf, err := fleet.NewMDMAppleConfigProfile(prof.Contents, tmID) if err != nil { - return nil, ctxerr.Wrap(ctx, + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, err.Error()), "invalid mobileconfig profile") } @@ -1598,29 +1690,31 @@ func getAppleProfiles( } if err := mdmProf.ValidateUserProvided(); err != nil { - return nil, ctxerr.Wrap(ctx, + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, err.Error())) } if mdmProf.Name != prof.Name { - return nil, ctxerr.Wrap(ctx, + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, fmt.Sprintf("Couldn’t edit custom_settings. The name provided for the profile must match the profile PayloadDisplayName: %q", mdmProf.Name)), "duplicate mobileconfig profile by name") } - if byName[mdmProf.Name] { - return nil, ctxerr.Wrap(ctx, + // TODO: confirm error messages + if _, ok := byName[mdmProf.Name]; ok { + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, fmt.Sprintf("Couldn’t edit custom_settings. More than one configuration profile have the same name (PayloadDisplayName): %q", mdmProf.Name)), "duplicate mobileconfig profile by name") } - byName[mdmProf.Name] = true + byName[mdmProf.Name] = "mobileconfig" - if byIdent[mdmProf.Identifier] { - return nil, ctxerr.Wrap(ctx, + // TODO: confirm error messages + if _, ok := byIdent[mdmProf.Identifier]; ok { + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, fmt.Sprintf("Couldn’t edit custom_settings. More than one configuration profile have the same identifier (PayloadIdentifier): %q", mdmProf.Identifier)), "duplicate mobileconfig profile by identifier") } - byIdent[mdmProf.Identifier] = true + byIdent[mdmProf.Identifier] = "mobileconfig" profs = append(profs, mdmProf) } @@ -1632,13 +1726,13 @@ func getAppleProfiles( // custom_settings key, we just return a success response in this // situation. if len(profs) == 0 { - return []*fleet.MDMAppleConfigProfile{}, nil + return []*fleet.MDMAppleConfigProfile{}, []*fleet.MDMAppleDeclaration{}, nil } - return nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError("mdm", "cannot set custom settings: Fleet MDM is not configured")) + return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError("mdm", "cannot set custom settings: Fleet MDM is not configured")) } - return profs, nil + return profs, decls, nil } func getWindowsProfiles( diff --git a/server/service/mdm_test.go b/server/service/mdm_test.go index 04bcb4e8d2..4c2b3738f8 100644 --- a/server/service/mdm_test.go +++ b/server/service/mdm_test.go @@ -1093,7 +1093,7 @@ func TestMDMBatchSetProfiles(t *testing.T) { ds.TeamFunc = func(ctx context.Context, id uint) (*fleet.Team, error) { return &fleet.Team{ID: id, Name: "team"}, nil } - ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile) error { + ds.BatchSetMDMProfilesFunc = func(ctx context.Context, tmID *uint, macProfiles []*fleet.MDMAppleConfigProfile, winProfiles []*fleet.MDMWindowsConfigProfile, macDecls []*fleet.MDMAppleDeclaration) error { return nil } ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activity fleet.ActivityDetails) error { @@ -1300,6 +1300,7 @@ func TestMDMBatchSetProfiles(t *testing.T) { {Name: "N1", Contents: mobileconfigForTest("N1", "I1")}, {Name: "N2", Contents: mobileconfigForTest("N2", "I2")}, {Name: "N3", Contents: mobileconfigForTest("N3", "I3")}, + {Name: "N4", Contents: declBytesForTest("D1", "d1content")}, }, ``, }, From 7dcfef38d47a41f17713fea317f8605f968432c3 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:25:28 -0500 Subject: [PATCH 10/29] Update name validations for `POST /mdm/profiles` endpoint (#17753) Follow up for #17402 --- server/datastore/mysql/apple_mdm.go | 20 ++++++---- server/datastore/mysql/microsoft_mdm.go | 8 +++- server/service/integration_mdm_test.go | 50 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index b03382b546..ca12cadd5b 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -31,6 +31,8 @@ INSERT INTO (SELECT ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? + ) AND NOT EXISTS ( + SELECT 1 FROM mdm_apple_declarations WHERE name = ? AND team_id = ? ) )` @@ -42,7 +44,7 @@ INSERT INTO var profileID int64 err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, - profUUID, teamID, cp.Identifier, cp.Name, cp.Mobileconfig, cp.Mobileconfig, cp.Name, teamID) + profUUID, teamID, cp.Identifier, cp.Name, cp.Mobileconfig, cp.Mobileconfig, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { case isDuplicate(err): @@ -3264,12 +3266,14 @@ INSERT INTO mdm_apple_declarations ( declaration_type, declaration, md5_checksum, - uploaded_at -) -VALUES ( - ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() -) - ` + uploaded_at) +(SELECT ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE + NOT EXISTS ( + SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? + ) AND NOT EXISTS ( + SELECT 1 FROM mdm_apple_configuration_profiles WHERE name = ? AND team_id = ? + ) +)` var tmID uint if declaration.TeamID != nil { @@ -3278,7 +3282,7 @@ VALUES ( err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, - declUUID, tmID, declaration.Identifier, declaration.Name, declaration.DeclarationType, declaration.Declaration, checksum) + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.DeclarationType, declaration.Declaration, checksum, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { case isDuplicate(err): diff --git a/server/datastore/mysql/microsoft_mdm.go b/server/datastore/mysql/microsoft_mdm.go index 9cd2a784db..5c878db43b 100644 --- a/server/datastore/mysql/microsoft_mdm.go +++ b/server/datastore/mysql/microsoft_mdm.go @@ -1495,6 +1495,8 @@ INSERT INTO (SELECT ?, ?, ?, ?, CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_apple_configuration_profiles WHERE name = ? AND team_id = ? + ) AND NOT EXISTS ( + SELECT 1 FROM mdm_apple_declarations WHERE name = ? AND team_id = ? ) )` @@ -1504,7 +1506,7 @@ INSERT INTO } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertProfileStmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID) + res, err := tx.ExecContext(ctx, insertProfileStmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { case isDuplicate(err): @@ -1556,6 +1558,8 @@ INSERT INTO (SELECT ?, ?, ?, ?, CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_apple_configuration_profiles WHERE name = ? AND team_id = ? + ) AND NOT EXISTS ( + SELECT 1 FROM mdm_apple_declarations WHERE name = ? AND team_id = ? ) ) ON DUPLICATE KEY UPDATE @@ -1568,7 +1572,7 @@ ON DUPLICATE KEY UPDATE teamID = *cp.TeamID } - res, err := ds.writer(ctx).ExecContext(ctx, stmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID) + res, err := ds.writer(ctx).ExecContext(ctx, stmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { case isDuplicate(err): diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 8d4c16dced..2c40f7e16c 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9297,6 +9297,39 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { require.Equal(t, "a", string(resp.ProfileUUID[0])) return resp.ProfileUUID } + assertAppleDeclaration := func(filename, name, ident string, teamID uint, labelNames []string, wantStatus int, wantErrMsg string) string { + fields := map[string][]string{ + "labels": labelNames, + } + if teamID > 0 { + fields["team_id"] = []string{fmt.Sprintf("%d", teamID)} + } + + bytes := []byte(fmt.Sprintf(`{ + "Type": "com.apple.configuration.foo", + "Payload": { + "Echo": "f1337" + }, + "Identifier": "%s" +}`, ident)) + + body, headers := generateNewProfileMultipartRequest(t, filename, bytes, s.token, fields) + res := s.DoRawWithHeaders("POST", "/api/latest/fleet/configuration_profiles", body.Bytes(), wantStatus, headers) + + if wantErrMsg != "" { + errMsg := extractServerErrorText(res.Body) + require.Contains(t, errMsg, wantErrMsg) + return "" + } + + var resp newMDMConfigProfileResponse + err := json.NewDecoder(res.Body).Decode(&resp) + require.NoError(t, err) + require.NotEmpty(t, resp.ProfileUUID) + require.Equal(t, "x", string(resp.ProfileUUID[0])) + return resp.ProfileUUID + } + createAppleProfile := func(name, ident string, teamID uint, labelNames []string) string { uid := assertAppleProfile(name+".mobileconfig", name, ident, teamID, labelNames, http.StatusOK, "") @@ -9378,6 +9411,23 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // but no conflict for no-team assertAppleProfile("win-team-profile.mobileconfig", "win-team-profile", "test-team-ident-2", 0, nil, http.StatusOK, "") + // add some macOS declarations + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusOK, "") + // identifier must be unique, it conflicts with existing declaration + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusConflict, "idx_mdm_apple_declaration_team_identifier") + // name is pulled from filename, it conflicts with existing declaration + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "idx_mdm_apple_declaration_team_name") + // uniqueness is checked only within team, so it's fine to have the same name and identifier in different teams + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", testTeam.ID, nil, http.StatusOK, "") + // name is pulled from filename, it conflicts with existing macOS config profile + assertAppleDeclaration("apple-global-profile.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "apple-global-profile already exists") + // name is pulled from filename, it conflicts with existing macOS config profile + assertAppleDeclaration("win-global-profile.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "win-global-profile already exists") + // windows profile name conflicts with existing declaration + assertWindowsProfile("apple-declaration.xml", "./Test", 0, nil, http.StatusConflict, "Couldn't upload. A configuration profile with this name already exists.") + // macOS profile name conflicts with existing declaration + assertAppleProfile("apple-declaration.mobileconfig", "apple-declaration", "test-declaration-ident", 0, nil, http.StatusConflict, "Couldn't upload. A configuration profile with this name already exists.") + // not an xml nor mobileconfig file assertWindowsProfile("foo.txt", "./Test", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") assertAppleProfile("foo.txt", "foo", "foo-ident", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") From 495638b45a2acf87e3d6b3f8eed3505d24a86b2e Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Thu, 21 Mar 2024 13:12:32 -0300 Subject: [PATCH 11/29] bugfixes, integration and cleanup for DDM (#17756) Improvements and fixes I found while integrating this - Renamed db columns to match the profile tables for consistency - Added columns to `host_mdm_apple_declarations` - Removed `team_declaration_checksum_view` - Remove the ad-hoc `MDMAppleRecordDeclarativeCheckIn`, I confused myself by developing this using tests, the device actually sends an `Acknowledged` response, which is recorded by nano - Fixed bugs in the `declaration/../..` endpoints - The prefix for the endpoint is `declaration` without `s` - The response needs to include a `ServerToken`, otherwise the declaration fails --- server/datastore/mysql/apple_mdm.go | 139 ++++-------------- server/datastore/mysql/apple_mdm_test.go | 60 +------- server/datastore/mysql/mdm.go | 10 +- .../tables/20240314150853_AddDDMTables.go | 72 ++++----- server/datastore/mysql/schema.sql | 48 ++---- server/datastore/mysql/testing_utils.go | 12 +- server/fleet/apple_mdm.go | 32 ++-- server/fleet/datastore.go | 7 +- server/mock/datastore_mock.go | 16 +- server/service/apple_mdm.go | 36 +++-- server/service/integration_mdm_test.go | 129 ++++++++-------- 11 files changed, 200 insertions(+), 361 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index ca12cadd5b..f79daed454 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "database/sql" - "encoding/json" "errors" "fmt" "strings" @@ -111,15 +110,15 @@ func formatErrorDuplicateConfigProfile(err error, cp *fleet.MDMAppleConfigProfil func formatErrorDuplicateDeclaration(err error, decl *fleet.MDMAppleDeclaration) error { switch { - case strings.Contains(err.Error(), "idx_mdm_apple_config_prof_team_identifier"): + case strings.Contains(err.Error(), "idx_mdm_apple_declaration_team_identifier"): return &existsError{ - ResourceType: "MDMAppleConfigProfile.PayloadIdentifier", + ResourceType: "MDMAppleDeclaration.Identifier", Identifier: decl.Identifier, TeamID: decl.TeamID, } - case strings.Contains(err.Error(), "idx_mdm_apple_config_prof_team_name"): + case strings.Contains(err.Error(), "idx_mdm_apple_declaration_team_name"): return &existsError{ - ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + ResourceType: "MDMAppleDeclaration.Name", Identifier: decl.Name, TeamID: decl.TeamID, } @@ -3109,9 +3108,9 @@ INSERT INTO mdm_apple_declarations ( declaration_uuid, identifier, name, - declaration_type, - declaration, - md5_checksum, + category, + raw_json, + checksum, uploaded_at, team_id ) @@ -3119,10 +3118,10 @@ VALUES ( ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? ) ON DUPLICATE KEY UPDATE - uploaded_at = IF(md5_checksum = VALUES(md5_checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), - md5_checksum = VALUES(md5_checksum), + uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), + checksum = VALUES(checksum), name = VALUES(name), - declaration = VALUES(declaration) + raw_json = VALUES(raw_json) ` fmtDeleteStmt := ` @@ -3137,7 +3136,7 @@ WHERE SELECT identifier, declaration_uuid, - declaration + raw_json FROM mdm_apple_declarations WHERE @@ -3220,14 +3219,14 @@ WHERE } for _, d := range declarations { - checksum := md5ChecksumScriptContent(string(d.Declaration)) + checksum := md5ChecksumScriptContent(string(d.RawJSON)) declUUID := "x" + uuid.NewString() if _, err := tx.ExecContext(ctx, insertStmt, declUUID, d.Identifier, d.Name, - d.DeclarationType, - d.Declaration, + d.Category, + d.RawJSON, checksum, declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") { if err == nil { @@ -3255,7 +3254,7 @@ WHERE func (ds *Datastore) NewMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { declUUID := "x" + uuid.NewString() - checksum := md5ChecksumScriptContent(string(declaration.Declaration)) + checksum := md5ChecksumScriptContent(string(declaration.RawJSON)) stmt := ` INSERT INTO mdm_apple_declarations ( @@ -3263,9 +3262,9 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - declaration_type, - declaration, - md5_checksum, + category, + raw_json, + checksum, uploaded_at) (SELECT ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( @@ -3282,7 +3281,7 @@ INSERT INTO mdm_apple_declarations ( err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, - declUUID, tmID, declaration.Identifier, declaration.Name, declaration.DeclarationType, declaration.Declaration, checksum, declaration.Name, tmID, declaration.Name, tmID) + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.Category, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { case isDuplicate(err): @@ -3328,13 +3327,13 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont // unrelated profile+label tuples) deleteStmt := ` DELETE FROM mdm_declaration_labels - WHERE (declaration_uuid, label_id) NOT IN (%s) AND - declaration_uuid IN (?) + WHERE (apple_declaration_uuid, label_id) NOT IN (%s) AND + apple_declaration_uuid IN (?) ` upsertStmt := ` INSERT INTO mdm_declaration_labels - (declaration_uuid, label_id, label_name) + (apple_declaration_uuid, label_id, label_name) VALUES %s ON DUPLICATE KEY UPDATE @@ -3394,9 +3393,9 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { const stmt = ` SELECT - md5((count(0) + group_concat(hex(mad.md5_checksum) + md5((count(0) + group_concat(hex(mad.checksum) ORDER BY - mad.uploaded_at DESC separator ''))) AS md5_checksum, + mad.uploaded_at DESC separator ''))) AS checksum, max(mad.created_at) AS latest_created_timestamp FROM host_mdm_apple_declarations hmad @@ -3415,9 +3414,9 @@ WHERE func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { const stmt = ` SELECT - mad.md5_checksum, + HEX(mad.checksum) as checksum, mad.identifier, - mad.declaration_type + mad.category FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON mad.declaration_uuid = hmad.declaration_uuid @@ -3432,100 +3431,24 @@ WHERE return res, nil } -func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) { +func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { // TODO: When hosts table is indexed by uuid, consider joining on hosts to ensure that the // declaration for the host's current team is returned. In the case where the specified // identifier is not unique to the team, the cron should ensure that any conflicting // declarations are removed, but the join would provide an extra layer of safety. const stmt = ` SELECT - mad.declaration + mad.raw_json, HEX(mad.checksum) as checksum FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid WHERE - host_uuid = ? AND identifier = ? AND declaration_type = ? AND operation_type = ?` + host_uuid = ? AND identifier = ? AND category = ? AND operation_type = ?` - var res json.RawMessage + var res fleet.MDMAppleDeclaration if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, declarationType, fleet.MDMOperationTypeInstall); err != nil { return nil, ctxerr.Wrap(ctx, err, "get ddm declarations response") } - return res, nil -} - -func (ds *Datastore) MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, result []byte) error { - err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext( - ctx, - `UPDATE nano_enrollments SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`, - hostUUID, - ) - if err != nil { - return ctxerr.Wrap(ctx, err, "updating last_seen times") - } - if n, _ := res.RowsAffected(); n == 0 { - return ctxerr.New(ctx, "host is not enrolled in MDM") - } - - // NOTE: DeclarativeManagement checkin commands sent by the device - // don't carry a CommandUUID reference like commands in - // CommandAndReportResults messages do. - // - // In nanomdm's view of the world, a command is pending until - // it receives a result or is deactivated, so we'll grab the - // command_uuid of the oldest DeclarativeManagement command we - // sent and assume this is the response for it. - // - // Other DeclarativeManagement commands will still be in the - // queue and they will trigger DDM syncs when the device checks - // in, so eventually all DDM commands wil get acknowledged. - // - // Alternatively, we could mark all DDM commands as - // acknowledged here, TBD based on the behaviors we see. - var cmdUUID string - err = sqlx.GetContext(ctx, tx, &cmdUUID, ` -SELECT nc.command_uuid -FROM nano_enrollment_queue neq -JOIN nano_commands nc - ON neq.command_uuid = nc.command_uuid -WHERE - id = ? AND - request_type = 'DeclarativeManagement' -ORDER BY neq.created_at ASC -LIMIT 1 - `, hostUUID) - if err != nil { - // it's okay if the host doesn't have matching command enqueued, the - // check-in could be initiated by the device. - if err == sql.ErrNoRows { - return nil - } - - return ctxerr.Wrap(ctx, err, "getting DDM command") - } - - _, err = tx.ExecContext( - ctx, ` -INSERT INTO nano_command_results - (id, command_uuid, status, result) -VALUES - (?, ?, ?, ?) -ON DUPLICATE KEY -UPDATE - status = VALUES(status), - result = VALUES(result)`, - hostUUID, - cmdUUID, - fleet.MDMAppleStatusAcknowledged, - result, - ) - - return ctxerr.Wrap(ctx, err, "updating nano_command_results") - }) - if err != nil { - return ctxerr.Wrap(ctx, err, "saving declarative management response") - } - - return nil + return &res, nil } diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 990bcc02f2..bbd5d58097 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -69,7 +69,6 @@ func TestMDMApple(t *testing.T) { {"TestMDMAppleDeleteHostDEPAssignments", testMDMAppleDeleteHostDEPAssignments}, {"LockUnlockWipeMacOS", testLockUnlockWipeMacOS}, {"ScreenDEPAssignProfileSerialsForCooldown", testScreenDEPAssignProfileSerialsForCooldown}, - {"MDMAppleRecordDeclarativeCheckIn", testMDMAppleRecordDeclarativeCheckIn}, } for _, c := range cases { @@ -1041,14 +1040,14 @@ func expectAppleDeclarations( require.NotEmpty(t, gotD.DeclarationUUID) require.True(t, strings.HasPrefix(gotD.DeclarationUUID, "x")) gotD.DeclarationUUID = "" - gotD.MD5Checksum = "" // don't care about md5checksum here + gotD.Checksum = "" // don't care about md5checksum here gotD.CreatedAt = time.Time{} - gotBytes, err := JSONRemarshal(gotD.Declaration) + gotBytes, err := JSONRemarshal(gotD.RawJSON) require.NoError(t, err) - wantBytes, err := JSONRemarshal(wantD.Declaration) + wantBytes, err := JSONRemarshal(wantD.RawJSON) require.NoError(t, err) require.Equal(t, wantBytes, gotBytes) @@ -1063,7 +1062,7 @@ func expectAppleDeclarations( require.Equal(t, wantD.Name, gotD.Name) require.Equal(t, wantD.Identifier, gotD.Identifier) require.Equal(t, wantD.Labels, gotD.Labels) - require.Equal(t, wantD.DeclarationType, gotD.DeclarationType) + require.Equal(t, wantD.Category, gotD.Category) } return m } @@ -1258,10 +1257,10 @@ func declForTest(name, identifier, payloadContent string, labels ...*fleet.Label declBytes := []byte(fmt.Sprintf(tmpl, identifier, identifier, payloadContent)) decl := &fleet.MDMAppleDeclaration{ - Declaration: declBytes, - DeclarationType: fleet.MDMAppleDeclarativeConfiguration, - Identifier: fmt.Sprintf("com.fleet.config%s", identifier), - Name: name, + RawJSON: declBytes, + Category: fleet.MDMAppleDeclarativeConfiguration, + Identifier: fmt.Sprintf("com.fleet.config%s", identifier), + Name: name, } for _, l := range labels { @@ -4671,49 +4670,6 @@ func testScreenDEPAssignProfileSerialsForCooldown(t *testing.T, ds *Datastore) { require.Empty(t, assign) } -func testMDMAppleRecordDeclarativeCheckIn(t *testing.T, ds *Datastore) { - ctx := context.Background() - - host, err := ds.NewHost(ctx, &fleet.Host{ - Hostname: "test-host1-name", - OsqueryHostID: ptr.String("1337"), - NodeKey: ptr.String("1337"), - UUID: "test-uuid-1", - TeamID: nil, - Platform: "darwin", - }) - require.NoError(t, err) - - // error if the host is not enrolled - err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte{}) - require.Error(t, err) - - // enroll the host - nanoEnroll(t, ds, host, true) - - // it's okay if the host doesn't have matching command enqueued, the - // check-in could be initiated by the device. - err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte{}) - require.NoError(t, err) - - // enqueue a declarative checkin request - commander, _ := createMDMAppleCommanderAndStorage(t, ds) - cmdUUID := uuid.New().String() - err = commander.DeclarativeManagement(ctx, []string{host.UUID}, cmdUUID) - require.NoError(t, err) - - // record a response from the host - err = ds.MDMAppleRecordDeclarativeCheckIn(ctx, host.UUID, []byte(" ? + AND category <> ? ) as combined_profiles ` @@ -272,15 +272,15 @@ func (ds *Datastore) listDeclarationLabelsForDeclarations(ctx context.Context, d stmt := ` SELECT - declaration_uuid AS profile_uuid, + apple_declaration_uuid AS profile_uuid, label_name, label_id FROM mdm_declaration_labels WHERE - declaration_uuid IN (?) + apple_declaration_uuid IN (?) ORDER BY - declaration_uuid, label_name + apple_declaration_uuid, label_name ` stmt, args, err := sqlx.In(stmt, declUUIDs) diff --git a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go index 5d15da08b2..b220e91b33 100644 --- a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go +++ b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go @@ -11,21 +11,21 @@ func init() { func Up_20240314150853(tx *sql.Tx) error { _, err := tx.Exec(` -CREATE TABLE mdm_apple_declaration_types ( - declaration_type varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (declaration_type) +CREATE TABLE mdm_apple_declaration_categories ( + declaration_category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + PRIMARY KEY (declaration_category) ) `) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("creating mdm_apple_declaration_categories table: %w", err) } _, err = tx.Exec(` - INSERT INTO mdm_apple_declaration_types + INSERT INTO mdm_apple_declaration_categories VALUES ('com.apple.configuration'), ('com.apple.activation') `) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("inserting default values into mdm_apple_declaration_categories table: %w", err) } _, err = tx.Exec(` @@ -42,14 +42,15 @@ CREATE TABLE mdm_apple_declarations ( -- name is the name of the declaration name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - -- declaration_type is the type of the declaration - declaration_type varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + -- category is the category of the declaration (activation, + -- declaration, management or asset) + category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - -- declaration contains a JSON blob with the declaration contents - declaration json NOT NULL, + -- raw_json contains a JSON blob with the declaration contents + raw_json json NOT NULL, - -- md5_checksum is an MD5 checksum of the declaration, in binary form - md5_checksum binary(16) NOT NULL, + -- checksum is an MD5 checksum of the declaration, in binary form + checksum binary(16) NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, uploaded_at timestamp NULL DEFAULT NULL, @@ -57,11 +58,11 @@ CREATE TABLE mdm_apple_declarations ( PRIMARY KEY (declaration_uuid), UNIQUE KEY idx_mdm_apple_declaration_team_identifier (team_id, identifier), UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name), - CONSTRAINT mdm_apple_declaration_declaration_type FOREIGN KEY (declaration_type) REFERENCES mdm_apple_declaration_types (declaration_type) ON DELETE CASCADE + CONSTRAINT mdm_apple_declaration_category FOREIGN KEY (category) REFERENCES mdm_apple_declaration_categories (declaration_category) ON DELETE CASCADE ) `) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("creating mdm_apple_declarations table: %w", err) } _, err = tx.Exec(` @@ -69,8 +70,8 @@ CREATE TABLE mdm_declaration_labels ( -- id is used as the primary key of this table id int(10) unsigned NOT NULL AUTO_INCREMENT, - -- declaration_uuid references a declaration - declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + -- apple_declaration_uuid references a declaration + apple_declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', -- label name is stored here because we need to list the labels in the UI @@ -85,14 +86,14 @@ CREATE TABLE mdm_declaration_labels ( uploaded_at timestamp NULL DEFAULT NULL, PRIMARY KEY (id), - UNIQUE KEY idx_mdm_declaration_labels_label_name (declaration_uuid, label_name), + UNIQUE KEY idx_mdm_declaration_labels_label_name (apple_declaration_uuid, label_name), KEY label_id (label_id), - CONSTRAINT mdm_declaration_labels_ibfk_1 FOREIGN KEY (declaration_uuid) REFERENCES mdm_apple_declarations (declaration_uuid) ON DELETE CASCADE, + CONSTRAINT mdm_declaration_labels_ibfk_1 FOREIGN KEY (apple_declaration_uuid) REFERENCES mdm_apple_declarations (declaration_uuid) ON DELETE CASCADE, CONSTRAINT mdm_declaration_labels_ibfk_3 FOREIGN KEY (label_id) REFERENCES labels (id) ON DELETE SET NULL ) `) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("creating mdm_declaration_labels table: %w", err) } _, err = tx.Exec(` @@ -109,7 +110,7 @@ CREATE TABLE mdm_apple_declaration_activation_references ( ) `) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("creating mdm_apple_declaration_activation_references table: %w", err) } _, err = tx.Exec(` @@ -126,12 +127,18 @@ CREATE TABLE host_mdm_apple_declarations ( -- detail contains any messages or errors from the protocol or Fleet detail text COLLATE utf8mb4_unicode_ci, - -- md5_checksum of the currently implemented declaration - md5_checksum binary(16) NOT NULL, + -- checksum of the currently implemented declaration + checksum binary(16) NOT NULL, -- declaration_uuid references the declaration assigned to the host's team declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + -- declaration_identifier is the identifier of the declaration + declaration_identifier varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + + -- declaration_name is the name of the declaration + declaration_name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + PRIMARY KEY (host_uuid, declaration_uuid), KEY status (status), KEY operation_type (operation_type), @@ -140,26 +147,7 @@ CREATE TABLE host_mdm_apple_declarations ( ) `) if err != nil { - return fmt.Errorf("%w", err) - } - - _, err = tx.Exec(` --- this view is used to pre-compute checksums on a team basis so they can be --- used as the ServerToken to signal devices if they should fetch declarations. -CREATE VIEW team_declaration_checksum_view AS -SELECT - team_id, - -- since GROUP_CONCAT can be truncated, we calculate the checksum based on - -- the latest updated items and the total number of items - MD5(COUNT(*) + GROUP_CONCAT(HEX(md5_checksum) ORDER BY uploaded_at DESC SEPARATOR '')) AS md5_checksum, - MAX(created_at) AS latest_created_timestamp -FROM - mdm_apple_declarations -GROUP BY - team_id - `) - if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("creatign host_mdm_apple_declarations table %w", err) } return nil diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 7aa5539edc..3882a3061e 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -310,8 +310,10 @@ CREATE TABLE `host_mdm_apple_declarations` ( `status` varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `operation_type` varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `detail` text COLLATE utf8mb4_unicode_ci, - `md5_checksum` binary(16) NOT NULL, + `checksum` binary(16) NOT NULL, `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `declaration_identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `declaration_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', PRIMARY KEY (`host_uuid`,`declaration_uuid`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), @@ -658,12 +660,12 @@ CREATE TABLE `mdm_apple_declaration_activation_references` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; -CREATE TABLE `mdm_apple_declaration_types` ( - `declaration_type` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (`declaration_type`) +CREATE TABLE `mdm_apple_declaration_categories` ( + `declaration_category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + PRIMARY KEY (`declaration_category`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `mdm_apple_declaration_types` VALUES ('com.apple.activation'),('com.apple.configuration'); +INSERT INTO `mdm_apple_declaration_categories` VALUES ('com.apple.activation'),('com.apple.configuration'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_apple_declarations` ( @@ -671,16 +673,16 @@ CREATE TABLE `mdm_apple_declarations` ( `team_id` int(10) unsigned NOT NULL DEFAULT '0', `identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - `declaration_type` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - `declaration` json NOT NULL, - `md5_checksum` binary(16) NOT NULL, + `category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, + `raw_json` json NOT NULL, + `checksum` binary(16) NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `uploaded_at` timestamp NULL DEFAULT NULL, PRIMARY KEY (`declaration_uuid`), UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`), UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`), - KEY `mdm_apple_declaration_declaration_type` (`declaration_type`), - CONSTRAINT `mdm_apple_declaration_declaration_type` FOREIGN KEY (`declaration_type`) REFERENCES `mdm_apple_declaration_types` (`declaration_type`) ON DELETE CASCADE + KEY `mdm_apple_declaration_category` (`category`), + CONSTRAINT `mdm_apple_declaration_category` FOREIGN KEY (`category`) REFERENCES `mdm_apple_declaration_categories` (`declaration_category`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; @@ -764,15 +766,15 @@ CREATE TABLE `mdm_configuration_profile_labels` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_declaration_labels` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `apple_declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `label_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `label_id` int(10) unsigned DEFAULT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `uploaded_at` timestamp NULL DEFAULT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `idx_mdm_declaration_labels_label_name` (`declaration_uuid`,`label_name`), + UNIQUE KEY `idx_mdm_declaration_labels_label_name` (`apple_declaration_uuid`,`label_name`), KEY `label_id` (`label_id`), - CONSTRAINT `mdm_declaration_labels_ibfk_1` FOREIGN KEY (`declaration_uuid`) REFERENCES `mdm_apple_declarations` (`declaration_uuid`) ON DELETE CASCADE, + CONSTRAINT `mdm_declaration_labels_ibfk_1` FOREIGN KEY (`apple_declaration_uuid`) REFERENCES `mdm_apple_declarations` (`declaration_uuid`) ON DELETE CASCADE, CONSTRAINT `mdm_declaration_labels_ibfk_3` FOREIGN KEY (`label_id`) REFERENCES `labels` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; @@ -1470,13 +1472,6 @@ CREATE TABLE `statistics` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -SET @saved_cs_client = @@character_set_client; -SET character_set_client = utf8; -/*!50001 CREATE VIEW `team_declaration_checksum_view` AS SELECT - 1 AS `team_id`, - 1 AS `md5_checksum`, - 1 AS `latest_created_timestamp`*/; -SET character_set_client = @saved_cs_client; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `teams` ( @@ -1646,16 +1641,3 @@ CREATE TABLE `wstep_serials` ( /*!50001 SET character_set_client = @saved_cs_client */; /*!50001 SET character_set_results = @saved_cs_results */; /*!50001 SET collation_connection = @saved_col_connection */; -/*!50001 DROP VIEW IF EXISTS `team_declaration_checksum_view`*/; -/*!50001 SET @saved_cs_client = @@character_set_client */; -/*!50001 SET @saved_cs_results = @@character_set_results */; -/*!50001 SET @saved_col_connection = @@collation_connection */; -/*!50001 SET character_set_client = utf8mb4 */; -/*!50001 SET character_set_results = utf8mb4 */; -/*!50001 SET collation_connection = utf8mb4_unicode_ci */; -/*!50001 CREATE ALGORITHM=UNDEFINED */ -/*!50013 DEFINER=`root`@`%` SQL SECURITY DEFINER */ -/*!50001 VIEW `team_declaration_checksum_view` AS select `mdm_apple_declarations`.`team_id` AS `team_id`,md5((count(0) + group_concat(hex(`mdm_apple_declarations`.`md5_checksum`) order by `mdm_apple_declarations`.`uploaded_at` DESC separator ''))) AS `md5_checksum`,max(`mdm_apple_declarations`.`created_at`) AS `latest_created_timestamp` from `mdm_apple_declarations` group by `mdm_apple_declarations`.`team_id` */; -/*!50001 SET character_set_client = @saved_cs_client */; -/*!50001 SET character_set_results = @saved_cs_results */; -/*!50001 SET collation_connection = @saved_col_connection */; diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 2aa839d1fb..a1eadba60d 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -313,12 +313,12 @@ func TruncateTables(t testing.TB, ds *Datastore, tables ...string) { // be truncated - a more precise approach must be used for those, e.g. // delete where id > max before test, or something like that. nonEmptyTables := map[string]bool{ - "app_config_json": true, - "migration_status_tables": true, - "osquery_options": true, - "mdm_delivery_status": true, - "mdm_operation_types": true, - "mdm_apple_declaration_types": true, + "app_config_json": true, + "migration_status_tables": true, + "osquery_options": true, + "mdm_delivery_status": true, + "mdm_operation_types": true, + "mdm_apple_declaration_categories": true, } ctx := context.Background() diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index db460be7e7..5f9b2993be 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -533,19 +533,19 @@ type SCEPIdentityAssociation struct { RenewCommandUUID string `db:"renew_command_uuid"` } -// MDMAppleDeclarationType is the type for the supported declaration types. -type MDMAppleDeclarationType string +// MDMAppleDeclarationCategory is the type for the supported declaration types. +type MDMAppleDeclarationCategory string const ( // MDMAppleConfigurationDeclaration is the value for [configuration][1] declarations // // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3813088 - MDMAppleDeclarativeConfiguration MDMAppleDeclarationType = "com.apple.configuration" + MDMAppleDeclarativeConfiguration MDMAppleDeclarationCategory = "com.apple.configuration" // MDMAppleActivationConfiguration is the value for [activation][1] declarations // // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3829708 - MDMAppleDeclarativeActivation MDMAppleDeclarationType = "com.apple.activation" + MDMAppleDeclarativeActivation MDMAppleDeclarationCategory = "com.apple.activation" ) // MDMAppleDeclaration represents a DDM JSON declaration. @@ -568,15 +568,15 @@ type MDMAppleDeclaration struct { // Fleet requires that Name must be unique in combination with the Identifier and TeamID. Name string `db:"name" json:"name"` - // DeclarationType is the type of the declaration, at the moment we + // Category is the category of the declaration, at the moment we // only support configurations and activations. - DeclarationType MDMAppleDeclarationType `db:"declaration_type"` + Category MDMAppleDeclarationCategory `db:"category"` - // Declaration is the raw JSON content of the declaration - Declaration json.RawMessage `db:"declaration" json:"-"` + // RawJSON is the raw JSON content of the declaration + RawJSON json.RawMessage `db:"raw_json" json:"-"` - // MD5Checksum is a checksum of the JSON contents - MD5Checksum string `db:"md5_checksum" json:"-"` + // Checksum is a checksum of the JSON contents + Checksum string `db:"checksum" json:"-"` // Labels are the labels associated with this Declaration Labels []DeclarationLabel `db:"labels" json:"labels,omitempty"` @@ -683,10 +683,10 @@ type DeclarationLabel struct { func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { var decl MDMAppleDeclaration - decl.DeclarationType = MDMAppleDeclarationType(strings.Join(strings.Split(declType, ".")[:3], ".")) + decl.Category = MDMAppleDeclarationCategory(strings.Join(strings.Split(declType, ".")[:3], ".")) decl.Identifier = ident decl.Name = name - decl.Declaration = raw + decl.RawJSON = raw decl.TeamID = teamID return &decl @@ -703,7 +703,7 @@ type MDMAppleDDMTokensResponse struct { // // https://developer.apple.com/documentation/devicemanagement/synchronizationtokens type MDMAppleDDMDeclarationsToken struct { - DeclarationsToken string `db:"md5_checksum"` + DeclarationsToken string `db:"checksum"` Timestamp time.Time `db:"latest_created_timestamp"` } @@ -739,9 +739,9 @@ type MDMAppleDDMManifest struct { // // https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse type MDMAppleDDMDeclarationItem struct { - Identifier string `db:"identifier"` - DeclarationType string `db:"declaration_type"` - ServerToken string `db:"md5_checksum"` + Identifier string `db:"identifier"` + Category string `db:"category"` + ServerToken string `db:"checksum"` } // MDMAppleDDMDeclarationResponse represents a declaration in the datastore. It is used for the DDM diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 9ce9691870..b6205af085 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1137,11 +1137,6 @@ type Datastore interface { // host_dep_assignments for host with matching serials. DeleteHostDEPAssignments(ctx context.Context, serials []string) error - // MDMAppleRecordDeclarativeCheckIn records a DeclarativeManagement - // checking from a host, so we know the host received the command to - // start the declarative management sync. - MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, response []byte) error - // UpdateHostDEPAssignProfileResponses receives a profile UUID and threes lists of serials, each representing // one of the three possible responses, and updates the host_dep_assignments table with the corresponding responses. UpdateHostDEPAssignProfileResponses(ctx context.Context, resp *godep.ProfileResponse) error @@ -1163,7 +1158,7 @@ type Datastore interface { // MDMAppleDDMDeclarationItems returns the declaration items for the specified host UUID. MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]MDMAppleDDMDeclarationItem, error) // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. - MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) + MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType MDMAppleDeclarationCategory, identifier string, hostUUID string) (*MDMAppleDeclaration, error) /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 5e8f28fb5b..6532899ba3 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -748,8 +748,6 @@ type GetMatchingHostSerialsFunc func(ctx context.Context, serials []string) (map type DeleteHostDEPAssignmentsFunc func(ctx context.Context, serials []string) error -type MDMAppleRecordDeclarativeCheckInFunc func(ctx context.Context, hostUUID string, response []byte) error - type UpdateHostDEPAssignProfileResponsesFunc func(ctx context.Context, resp *godep.ProfileResponse) error type ScreenDEPAssignProfileSerialsForCooldownFunc func(ctx context.Context, serials []string) (skipSerials []string, assignSerials []string, err error) @@ -762,7 +760,7 @@ type MDMAppleDDMDeclarationsTokenFunc func(ctx context.Context, hostUUID string) type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) -type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) +type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error @@ -1964,9 +1962,6 @@ type DataStore struct { DeleteHostDEPAssignmentsFunc DeleteHostDEPAssignmentsFunc DeleteHostDEPAssignmentsFuncInvoked bool - MDMAppleRecordDeclarativeCheckInFunc MDMAppleRecordDeclarativeCheckInFunc - MDMAppleRecordDeclarativeCheckInFuncInvoked bool - UpdateHostDEPAssignProfileResponsesFunc UpdateHostDEPAssignProfileResponsesFunc UpdateHostDEPAssignProfileResponsesFuncInvoked bool @@ -4702,13 +4697,6 @@ func (s *DataStore) DeleteHostDEPAssignments(ctx context.Context, serials []stri return s.DeleteHostDEPAssignmentsFunc(ctx, serials) } -func (s *DataStore) MDMAppleRecordDeclarativeCheckIn(ctx context.Context, hostUUID string, response []byte) error { - s.mu.Lock() - s.MDMAppleRecordDeclarativeCheckInFuncInvoked = true - s.mu.Unlock() - return s.MDMAppleRecordDeclarativeCheckInFunc(ctx, hostUUID, response) -} - func (s *DataStore) UpdateHostDEPAssignProfileResponses(ctx context.Context, resp *godep.ProfileResponse) error { s.mu.Lock() s.UpdateHostDEPAssignProfileResponsesFuncInvoked = true @@ -4751,7 +4739,7 @@ func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID st return s.MDMAppleDDMDeclarationItemsFunc(ctx, hostUUID) } -func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationType, identifier string, hostUUID string) (json.RawMessage, error) { +func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { s.mu.Lock() s.MDMAppleDDMDeclarationsResponseFuncInvoked = true s.mu.Unlock() diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 3b15a02641..3c3bb3031d 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -3079,11 +3079,6 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec switch { case dm.Endpoint == "tokens": level.Debug(svc.logger).Log("msg", "received tokens request") - // TODO: Should we record the checkin for all endpoints or just tokens? - if err := svc.ds.MDMAppleRecordDeclarativeCheckIn(r.Context, dm.UDID, dm.Raw); err != nil { - return nil, ctxerr.Wrap(r.Context, err, "recording declarative checkin") - } - return svc.handleTokens(r.Context, dm.UDID) case dm.Endpoint == "declaration-items": @@ -3096,7 +3091,7 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec return nil, nil - case strings.HasPrefix(dm.Endpoint, "declarations"): + case strings.HasPrefix(dm.Endpoint, "declaration"): level.Debug(svc.logger).Log("msg", "received declarations request") return svc.handleDeclarationsResponse(r.Context, dm.Endpoint, dm.UDID) @@ -3131,14 +3126,14 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, hostU configurations := []fleet.MDMAppleDDMManifest{} for _, d := range di { manifest := fleet.MDMAppleDDMManifest{Identifier: d.Identifier, ServerToken: d.ServerToken} - switch d.DeclarationType { + switch d.Category { case string(fleet.MDMAppleDeclarativeActivation): activations = append(activations, manifest) case string(fleet.MDMAppleDeclarativeConfiguration): configurations = append(configurations, manifest) default: - level.Debug(svc.logger).Log("msg", "unrecognized declaration type", "type", d.DeclarationType) - return nil, ctxerr.New(ctx, "unrecognized declaration type") + level.Debug(svc.logger).Log("msg", "unrecognized declaration category", "category", d.Category) + return nil, ctxerr.New(ctx, "unrecognized declaration category") } } @@ -3171,15 +3166,30 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e } level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", parts[1], "identifier", parts[2]) - // TODO: Validate declarationType? - d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, fleet.MDMAppleDeclarationType("com.apple."+parts[1]), parts[2], hostUUID) + d, err := svc.ds.MDMAppleDDMDeclarationsResponse( + ctx, fleet.MDMAppleDeclarationCategory("com.apple."+parts[1]), + parts[2], + hostUUID, + ) if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting declaration") } - b, err := json.Marshal(d) + + // unmarshall into a temporary map in order to add the token. + // we do this at this stage because tokens are purely managed by Fleet, + // and we don't want to store a modified version of what's provided by + // the IT admin. + // + // This mimics what we do for CommandUUID, but can be revisited. + var tempd map[string]any + if err := json.Unmarshal(d.RawJSON, &tempd); err != nil { + return nil, ctxerr.Wrap(ctx, err, "unmarshaling stored declaration") + } + tempd["ServerToken"] = d.Checksum + + b, err := json.Marshal(tempd) if err != nil { return nil, ctxerr.Wrap(ctx, err, "marshaling declaration") } - return b, nil } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 2c40f7e16c..93f73a72ed 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9414,9 +9414,9 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // add some macOS declarations assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusOK, "") // identifier must be unique, it conflicts with existing declaration - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusConflict, "idx_mdm_apple_declaration_team_identifier") + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusConflict, "Resource Already Exists: MDMAppleDeclaration.Identifier test-declaration-ident already exists") // name is pulled from filename, it conflicts with existing declaration - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "idx_mdm_apple_declaration_team_name") + assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "Resource Already Exists: MDMAppleDeclaration.Name apple-declaration already exists") // uniqueness is checked only within team, so it's fine to have the same name and identifier in different teams assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", testTeam.ID, nil, http.StatusOK, "") // name is pulled from filename, it conflicts with existing macOS config profile @@ -12740,6 +12740,11 @@ func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { t := s.T() _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + calcChecksum := func(source []byte) string { + csum := fmt.Sprintf("%x", md5.Sum(source)) //nolint:gosec + return strings.ToUpper(csum) + } + insertDeclaration := func(t *testing.T, decl fleet.MDMAppleDeclaration) { stmt := ` INSERT INTO mdm_apple_declarations ( @@ -12747,12 +12752,12 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - declaration_type, - declaration, - md5_checksum, + category, + raw_json, + checksum, created_at, uploaded_at -) VALUES (?,?,?,?,?,?,?,?,?)` +) VALUES (?,?,?,?,?,?,UNHEX(?),?,?)` mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(context.Background(), stmt, @@ -12760,9 +12765,9 @@ INSERT INTO mdm_apple_declarations ( decl.TeamID, decl.Identifier, decl.Name, - decl.DeclarationType, - decl.Declaration, - decl.MD5Checksum, + decl.Category, + decl.RawJSON, + calcChecksum(decl.RawJSON), decl.CreatedAt, decl.UploadedAt, ) @@ -12776,16 +12781,16 @@ INSERT INTO host_mdm_apple_declarations ( host_uuid, status, operation_type, - md5_checksum, + checksum, declaration_uuid -) VALUES (?,?,?,?,?)` +) VALUES (?,?,?,UNHEX(?),?)` mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(context.Background(), stmt, hostUUID, fleet.MDMDeliveryPending, fleet.MDMOperationTypeInstall, - decl.MD5Checksum, + calcChecksum(decl.RawJSON), decl.DeclarationUUID, ) return err @@ -12803,11 +12808,14 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example", Name: "Example", - DeclarationType: fleet.MDMAppleDeclarativeConfiguration, - Declaration: json.RawMessage(`{"foo": "bar"}`), - MD5Checksum: "csum123", - CreatedAt: then, - UploadedAt: then, + Category: fleet.MDMAppleDeclarativeConfiguration, + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"bar"}, + "Identifier": "com.example" + }`), + CreatedAt: then, + UploadedAt: then, }, } insertDeclaration(t, noTeamDeclsByUUID["123"]) @@ -12816,7 +12824,7 @@ INSERT INTO host_mdm_apple_declarations ( mapDeclsByChecksum := func(byUUID map[string]fleet.MDMAppleDeclaration) map[string]fleet.MDMAppleDeclaration { byChecksum := make(map[string]fleet.MDMAppleDeclaration) for _, d := range byUUID { - byChecksum[d.MD5Checksum] = byUUID[d.DeclarationUUID] + byChecksum[calcChecksum(d.RawJSON)] = byUUID[d.DeclarationUUID] } return byChecksum } @@ -12855,24 +12863,19 @@ INSERT INTO host_mdm_apple_declarations ( return di } - parseDeclarationResp := func(r *http.Response, expectedBytes []byte) fleet.MDMAppleDDMDeclarationResponse { + assertDeclarationResponse := func(r *http.Response, expected fleet.MDMAppleDeclaration) { require.NotNil(t, r) - b, err := io.ReadAll(r.Body) - require.NoError(t, err) - defer r.Body.Close() - if expectedBytes != nil { - require.Equal(t, expectedBytes, b) - } - r.Body = io.NopCloser(bytes.NewBuffer(b)) - // t.Log("body", string(b)) - // unmarsal the response to make sure it's valid - var d fleet.MDMAppleDDMDeclarationResponse - err = json.NewDecoder(r.Body).Decode(&d) - require.NoError(t, err) - // t.Logf("decoded: %+v", d) - - return d + // unmarsal the response and assert it's valid + var wantParsed fleet.MDMAppleDDMDeclarationResponse + require.NoError(t, json.Unmarshal(expected.RawJSON, &wantParsed)) + var gotParsed fleet.MDMAppleDDMDeclarationResponse + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed)) + require.EqualValues(t, wantParsed.Payload, gotParsed.Payload) + require.Equal(t, calcChecksum(expected.RawJSON), gotParsed.ServerToken) + require.Contains(t, gotParsed.Type, expected.Category) + require.Equal(t, expected.Identifier, gotParsed.Identifier) + // t.Logf("decoded: %+v", gotParsed) } checkTokensResp := func(t *testing.T, r fleet.MDMAppleDDMTokensResponse, expectedTimestamp time.Time, prevToken string) { @@ -12888,9 +12891,7 @@ INSERT INTO host_mdm_apple_declarations ( require.Empty(t, r.Declarations.Management) require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) for _, m := range r.Declarations.Configurations { - // look up the declaration by the server token (we trim the token to the first seven - // chars to match our keys because response is padded to length 16 with "\u000") - d, ok := expectedDeclsByChecksum[m.ServerToken[0:7]] + d, ok := expectedDeclsByChecksum[m.ServerToken] require.True(t, ok) require.Equal(t, d.Identifier, m.Identifier) } @@ -12912,11 +12913,14 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example2", Name: "Example2", - DeclarationType: fleet.MDMAppleDeclarativeConfiguration, - Declaration: json.RawMessage(`{"foo": "baz"}`), - MD5Checksum: "csum456", - CreatedAt: then.Add(1 * time.Minute), - UploadedAt: then.Add(1 * time.Minute), + Category: fleet.MDMAppleDeclarativeConfiguration, + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"baz"}, + "Identifier": "com.example2" + }`), + CreatedAt: then.Add(1 * time.Minute), + UploadedAt: then.Add(1 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["456"]) insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["456"]) @@ -12940,11 +12944,14 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example3", Name: "Example3", - DeclarationType: fleet.MDMAppleDeclarativeConfiguration, - Declaration: json.RawMessage(`{"foo": "bang"}`), - MD5Checksum: "csum789", - CreatedAt: then.Add(2 * time.Minute), - UploadedAt: then.Add(2 * time.Minute), + Category: fleet.MDMAppleDeclarativeConfiguration, + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"bang"}, + "Identifier": "com.example3" + }`), + CreatedAt: then.Add(2 * time.Minute), + UploadedAt: then.Add(2 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["789"]) insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["789"]) @@ -12968,12 +12975,10 @@ INSERT INTO host_mdm_apple_declarations ( t.Run("Declaration", func(t *testing.T) { want := noTeamDeclsByUUID["123"] - wantBytes, err := json.Marshal(want.Declaration) - require.NoError(t, err) - r, err := mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configuration", want.Identifier)) + r, err := mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) require.NoError(t, err) - _ = parseDeclarationResp(r, wantBytes) + assertDeclarationResponse(r, want) // insert a new declaration noTeamDeclsByUUID["abc"] = fleet.MDMAppleDeclaration{ @@ -12981,29 +12986,21 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example4", Name: "Example4", - DeclarationType: fleet.MDMAppleDeclarativeConfiguration, - Declaration: json.RawMessage(`{ + Category: fleet.MDMAppleDeclarativeConfiguration, + RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.test", "Payload": {"foo":"bar"}, - "Identifier": "com.example4", - "ServerToken": "csumabc" + "Identifier": "com.example4" }`), - MD5Checksum: "csumabc", - CreatedAt: then.Add(3 * time.Minute), - UploadedAt: then.Add(3 * time.Minute), + CreatedAt: then.Add(3 * time.Minute), + UploadedAt: then.Add(3 * time.Minute), } insertDeclaration(t, noTeamDeclsByUUID["abc"]) insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["abc"]) want = noTeamDeclsByUUID["abc"] - wantBytes, err = json.Marshal(want.Declaration) - require.NoError(t, err) - r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configuration", want.Identifier)) + r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) require.NoError(t, err) - d := parseDeclarationResp(r, wantBytes) - require.Equal(t, want.Identifier, d.Identifier) - require.Equal(t, "com.apple.configuration.test", d.Type) - require.Equal(t, json.RawMessage(`{"foo":"bar"}`), d.Payload) - require.Equal(t, want.MD5Checksum, d.ServerToken) + assertDeclarationResponse(r, want) }) } From 60ba78fc9d9919e8957965a248faab6291f7fd11 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:37:43 -0500 Subject: [PATCH 12/29] Additional backend support for DDM profiles (#17775) --- server/datastore/mysql/apple_mdm.go | 81 ++++++++++++-- server/datastore/mysql/apple_mdm_test.go | 6 +- server/datastore/mysql/mdm.go | 60 +++-------- server/datastore/mysql/microsoft_mdm.go | 2 +- server/fleet/apple_mdm.go | 19 ++-- server/fleet/datastore.go | 3 + server/fleet/mdm.go | 18 ++++ server/fleet/service.go | 9 ++ server/mock/datastore_mock.go | 12 +++ server/service/apple_mdm.go | 131 +++++++++++++++++++++-- server/service/integration_mdm_test.go | 65 ++++++++--- server/service/mdm.go | 28 ++++- server/service/mdm_test.go | 2 + 13 files changed, 343 insertions(+), 93 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index f79daed454..ac0801aad8 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -213,7 +213,7 @@ WHERE // get the labels for that profile, except if the profile was loaded by the // old (deprecated) endpoint. if uuid != "" { - labels, err := ds.listProfileLabelsForProfiles(ctx, nil, []string{res.ProfileUUID}) + labels, err := ds.listProfileLabelsForProfiles(ctx, nil, []string{res.ProfileUUID}, nil) if err != nil { return nil, err } @@ -226,11 +226,52 @@ WHERE return &res, nil } +func (ds *Datastore) GetMDMAppleDeclaration(ctx context.Context, declUUID string) (*fleet.MDMAppleDeclaration, error) { + stmt := ` +SELECT + declaration_uuid, + team_id, + name, + identifier, + raw_json, + checksum, + created_at, + uploaded_at +FROM + mdm_apple_declarations +WHERE + declaration_uuid = ? AND category = 'com.apple.configuration'` + + var res fleet.MDMAppleDeclaration + err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, declUUID) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("MDMAppleDeclaration").WithName(declUUID)) + } + + return nil, ctxerr.Wrap(ctx, err, "get mdm apple declaration") + } + + labels, err := ds.listProfileLabelsForProfiles(ctx, nil, nil, []string{res.DeclarationUUID}) + if err != nil { + return nil, err + } + if len(labels) > 0 { + // ensure we leave Labels nil if there are none + res.Labels = labels + } + + return &res, nil +} + func (ds *Datastore) DeleteMDMAppleConfigProfileByDeprecatedID(ctx context.Context, profileID uint) error { return ds.deleteMDMAppleConfigProfileByIDOrUUID(ctx, profileID, "") } func (ds *Datastore) DeleteMDMAppleConfigProfile(ctx context.Context, profileUUID string) error { + if strings.HasPrefix(profileUUID, fleet.MDMAppleDeclarationUUIDPrefix) { + return ds.deleteMDMAppleDeclaration(ctx, profileUUID) + } return ds.deleteMDMAppleConfigProfileByIDOrUUID(ctx, 0, profileUUID) } @@ -260,11 +301,28 @@ func (ds *Datastore) deleteMDMAppleConfigProfileByIDOrUUID(ctx context.Context, return nil } +func (ds *Datastore) deleteMDMAppleDeclaration(ctx context.Context, uuid string) error { + stmt := `DELETE FROM mdm_apple_declarations WHERE declaration_uuid = ?` + + res, err := ds.writer(ctx).ExecContext(ctx, stmt, uuid) + if err != nil { + return ctxerr.Wrap(ctx, err) + } + + deleted, _ := res.RowsAffected() + if deleted != 1 { + return ctxerr.Wrap(ctx, notFound("MDMAppleDeclaration").WithName(uuid)) + } + + return nil +} + func (ds *Datastore) DeleteMDMAppleConfigProfileByTeamAndIdentifier(ctx context.Context, teamID *uint, profileIdentifier string) error { if teamID == nil { teamID = ptr.Uint(0) } + // TODO: add deletion of declarations here or separate method? res, err := ds.writer(ctx).ExecContext(ctx, `DELETE FROM mdm_apple_configuration_profiles WHERE team_id = ? AND identifier = ?`, teamID, profileIdentifier) if err != nil { return ctxerr.Wrap(ctx, err) @@ -3149,7 +3207,7 @@ WHERE declTeamID = *tmID } - var incomingLabels []fleet.DeclarationLabel + var incomingLabels []fleet.ConfigurationProfileLabel // build a list of identifiers for the incoming declarations, will keep the // existing ones if there's a match and no change @@ -3220,7 +3278,7 @@ WHERE for _, d := range declarations { checksum := md5ChecksumScriptContent(string(d.RawJSON)) - declUUID := "x" + uuid.NewString() + declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString() if _, err := tx.ExecContext(ctx, insertStmt, declUUID, d.Identifier, @@ -3237,7 +3295,7 @@ WHERE d.DeclarationUUID = declUUID for _, l := range d.Labels { - l.DeclarationUUID = declUUID + l.ProfileUUID = declUUID incomingLabels = append(incomingLabels, l) } } @@ -3253,7 +3311,7 @@ WHERE } func (ds *Datastore) NewMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { - declUUID := "x" + uuid.NewString() + declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString() checksum := md5ChecksumScriptContent(string(declaration.RawJSON)) stmt := ` @@ -3301,7 +3359,7 @@ INSERT INTO mdm_apple_declarations ( } for i := range declaration.Labels { - declaration.Labels[i].DeclarationUUID = declUUID + declaration.Labels[i].ProfileUUID = declUUID } if err := batchSetDeclarationLabelAssociationsDB(ctx, tx, declaration.Labels); err != nil { return ctxerr.Wrap(ctx, err, "inserting mdm declaration label associations") @@ -3317,7 +3375,7 @@ INSERT INTO mdm_apple_declarations ( return declaration, nil } -func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtContext, declarationLabels []fleet.DeclarationLabel) error { +func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtContext, declarationLabels []fleet.ConfigurationProfileLabel) error { if len(declarationLabels) == 0 { return nil } @@ -3355,10 +3413,10 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont } insertBuilder.WriteString("(?, ?, ?)") deleteBuilder.WriteString("(?, ?)") - insertParams = append(insertParams, pl.DeclarationUUID, pl.LabelID, pl.LabelName) - deleteParams = append(deleteParams, pl.DeclarationUUID, pl.LabelID) + insertParams = append(insertParams, pl.ProfileUUID, pl.LabelID, pl.LabelName) + deleteParams = append(deleteParams, pl.ProfileUUID, pl.LabelID) - setProfileUUIDs[pl.DeclarationUUID] = struct{}{} + setProfileUUIDs[pl.ProfileUUID] = struct{}{} } _, err := tx.ExecContext(ctx, fmt.Sprintf(upsertStmt, insertBuilder.String()), insertParams...) @@ -3447,6 +3505,9 @@ WHERE var res fleet.MDMAppleDeclaration if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, declarationType, fleet.MDMOperationTypeInstall); err != nil { + if err == sql.ErrNoRows { + return nil, notFound(string(declarationType)).WithName(identifier) + } return nil, ctxerr.Wrap(ctx, err, "get ddm declarations response") } diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index bbd5d58097..1bfe5f6574 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1035,10 +1035,10 @@ func expectAppleDeclarations( gotD.TeamID = nil } - // DeclarationUUID is non-empty and starts with "x", but otherwise we don't + // DeclarationUUID is non-empty and starts with "d", but otherwise we don't // care about it for test assertions. require.NotEmpty(t, gotD.DeclarationUUID) - require.True(t, strings.HasPrefix(gotD.DeclarationUUID, "x")) + require.True(t, strings.HasPrefix(gotD.DeclarationUUID, fleet.MDMAppleDeclarationUUIDPrefix)) gotD.DeclarationUUID = "" gotD.Checksum = "" // don't care about md5checksum here @@ -1264,7 +1264,7 @@ func declForTest(name, identifier, payloadContent string, labels ...*fleet.Label } for _, l := range labels { - decl.Labels = append(decl.Labels, fleet.DeclarationLabel{LabelName: l.Name, LabelID: l.ID}) + decl.Labels = append(decl.Labels, fleet.ConfigurationProfileLabel{LabelName: l.Name, LabelID: l.ID}) } return decl diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index f58b0fd3fd..21ff675ed1 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -229,7 +229,7 @@ FROM ( if prof.Platform == "windows" { winProfUUIDs = append(winProfUUIDs, prof.ProfileUUID) } else { - if strings.HasPrefix(prof.ProfileUUID, "x") { + if strings.HasPrefix(prof.ProfileUUID, fleet.MDMAppleDeclarationUUIDPrefix) { macDeclUUIDs = append(macDeclUUIDs, prof.ProfileUUID) continue } @@ -237,18 +237,11 @@ FROM ( macProfUUIDs = append(macProfUUIDs, prof.ProfileUUID) } } - labels, err := ds.listProfileLabelsForProfiles(ctx, winProfUUIDs, macProfUUIDs) + labels, err := ds.listProfileLabelsForProfiles(ctx, winProfUUIDs, macProfUUIDs, macDeclUUIDs) if err != nil { return nil, nil, err } - declLabels, err := ds.listDeclarationLabelsForDeclarations(ctx, macDeclUUIDs) - if err != nil { - return nil, nil, err - } - - labels = append(labels, declLabels...) - // match the labels with their profiles profMap := make(map[string]*fleet.MDMConfigProfilePayload, len(profs)) for _, prof := range profs { @@ -263,39 +256,7 @@ FROM ( return profs, metaData, nil } -// Note: we're using the ConfigurationProfileLabel type here since from the product perspective, MDM -// profiles and declarations are both "profiles". -func (ds *Datastore) listDeclarationLabelsForDeclarations(ctx context.Context, declUUIDs []string) ([]fleet.ConfigurationProfileLabel, error) { - if len(declUUIDs) == 0 { - return []fleet.ConfigurationProfileLabel{}, nil - } - - stmt := ` -SELECT - apple_declaration_uuid AS profile_uuid, - label_name, - label_id -FROM - mdm_declaration_labels -WHERE - apple_declaration_uuid IN (?) -ORDER BY - apple_declaration_uuid, label_name - ` - - stmt, args, err := sqlx.In(stmt, declUUIDs) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "sqlx.In to list labels for declarations") - } - - var labels []fleet.ConfigurationProfileLabel - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &labels, stmt, args...); err != nil { - return nil, ctxerr.Wrap(ctx, err, "select declaration labels") - } - return labels, nil -} - -func (ds *Datastore) listProfileLabelsForProfiles(ctx context.Context, winProfUUIDs, macProfUUIDs []string) ([]fleet.ConfigurationProfileLabel, error) { +func (ds *Datastore) listProfileLabelsForProfiles(ctx context.Context, winProfUUIDs, macProfUUIDs, macDeclUUIDs []string) ([]fleet.ConfigurationProfileLabel, error) { // load the labels associated with those profiles const labelsStmt = ` SELECT @@ -308,6 +269,16 @@ FROM WHERE mcpl.apple_profile_uuid IN (?) OR mcpl.windows_profile_uuid IN (?) +UNION ALL +SELECT + apple_declaration_uuid as profile_uuid, + label_name, + COALESCE(label_id, 0) as label_id, + IF(label_id IS NULL, 1, 0) as broken +FROM + mdm_declaration_labels mdl +WHERE + mdl.apple_declaration_uuid IN (?) ORDER BY profile_uuid, label_name ` @@ -320,8 +291,11 @@ ORDER BY if len(macProfUUIDs) == 0 { macProfUUIDs = []string{"-"} } + if len(macDeclUUIDs) == 0 { + macDeclUUIDs = []string{"-"} + } - stmt, args, err := sqlx.In(labelsStmt, macProfUUIDs, winProfUUIDs) + stmt, args, err := sqlx.In(labelsStmt, macProfUUIDs, winProfUUIDs, macDeclUUIDs) if err != nil { return nil, ctxerr.Wrap(ctx, err, "sqlx.In to list labels for profiles") } diff --git a/server/datastore/mysql/microsoft_mdm.go b/server/datastore/mysql/microsoft_mdm.go index 5c878db43b..e8904bb5a9 100644 --- a/server/datastore/mysql/microsoft_mdm.go +++ b/server/datastore/mysql/microsoft_mdm.go @@ -734,7 +734,7 @@ WHERE return nil, ctxerr.Wrap(ctx, err, "get mdm windows config profile") } - labels, err := ds.listProfileLabelsForProfiles(ctx, []string{res.ProfileUUID}, nil) + labels, err := ds.listProfileLabelsForProfiles(ctx, []string{res.ProfileUUID}, nil, nil) if err != nil { return nil, err } diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 5f9b2993be..78b6cda55d 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -546,6 +546,10 @@ const ( // // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3829708 MDMAppleDeclarativeActivation MDMAppleDeclarationCategory = "com.apple.activation" + + // MDMAppleDeclarationUUIDPrefix is the prefix used to differentiate declaration uuids + // from legacy Apple profile uuids and Windows profile uuids. + MDMAppleDeclarationUUIDPrefix = "d" ) // MDMAppleDeclaration represents a DDM JSON declaration. @@ -579,7 +583,7 @@ type MDMAppleDeclaration struct { Checksum string `db:"checksum" json:"-"` // Labels are the labels associated with this Declaration - Labels []DeclarationLabel `db:"labels" json:"labels,omitempty"` + Labels []ConfigurationProfileLabel `db:"labels" json:"labels,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"` @@ -635,7 +639,7 @@ func (r *MDMAppleRawDeclaration) ValidateUserProvided() error { func GetRawDeclarationValues(raw []byte) (*MDMAppleRawDeclaration, error) { var rawDecl MDMAppleRawDeclaration if err := json.Unmarshal(raw, &rawDecl); err != nil { - return nil, err + return nil, NewInvalidArgumentError("declaration", fmt.Sprintf("Couldn't upload. The file should include valid JSON: %s", err)).WithStatus(http.StatusBadRequest) } return &rawDecl, nil @@ -669,17 +673,6 @@ type MDMAppleHostDeclaration struct { Detail string `db:"detail" json:"detail"` } -// DeclarationLabel represents the many-to-many relationship between -// declarations and labels. -// TODO(JVE): I think we can remove this type altogether, but double check first (mainly when -// ingesting declarations). -type DeclarationLabel struct { - DeclarationUUID string `db:"profile_uuid" json:"-"` - LabelName string `db:"label_name" json:"name"` - LabelID uint `db:"label_id" json:"id,omitempty"` // omitted if 0 (which is impossible if the label is not broken) - Broken bool `db:"broken" json:"broken,omitempty"` // omitted (not rendered to JSON) if false -} - func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { var decl MDMAppleDeclaration diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index b6205af085..5b7cac96c8 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -910,6 +910,9 @@ type Datastore interface { // profile uuid. GetMDMAppleConfigProfile(ctx context.Context, profileUUID string) (*MDMAppleConfigProfile, error) + // GetMDMAppleDeclaration returns the declaration corresponding to the specified uuid. + GetMDMAppleDeclaration(ctx context.Context, declUUID string) (*MDMAppleDeclaration, error) + // ListMDMAppleConfigProfiles lists mdm config profiles associated with the specified team id. // For global config profiles, specify nil as the team id. ListMDMAppleConfigProfiles(ctx context.Context, teamID *uint) ([]*MDMAppleConfigProfile, error) diff --git a/server/fleet/mdm.go b/server/fleet/mdm.go index dbb3808aab..baa501bb4b 100644 --- a/server/fleet/mdm.go +++ b/server/fleet/mdm.go @@ -413,6 +413,24 @@ func NewMDMConfigProfilePayloadFromApple(cp *MDMAppleConfigProfile) *MDMConfigPr } } +func NewMDMConfigProfilePayloadFromAppleDDM(decl *MDMAppleDeclaration) *MDMConfigProfilePayload { + var tid *uint + if decl.TeamID != nil && *decl.TeamID > 0 { + tid = decl.TeamID + } + return &MDMConfigProfilePayload{ + ProfileUUID: decl.DeclarationUUID, + TeamID: tid, + Name: decl.Name, + Identifier: decl.Identifier, + Platform: "darwin", + Checksum: []byte(decl.Checksum), + CreatedAt: decl.CreatedAt, + UploadedAt: decl.UploadedAt, + Labels: decl.Labels, + } +} + // MDMProfileSpec represents the spec used to define configuration // profiles via yaml files. type MDMProfileSpec struct { diff --git a/server/fleet/service.go b/server/fleet/service.go index 01c870ba8f..c38c9550c5 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -654,18 +654,27 @@ type Service interface { NewMDMAppleConfigProfile(ctx context.Context, teamID uint, r io.Reader, labels []string) (*MDMAppleConfigProfile, error) // NewMDMAppleConfigProfileWithPayload creates a new declaration for the specified team. NewMDMAppleDeclaration(ctx context.Context, teamID uint, r io.Reader, labels []string, name string) (*MDMAppleDeclaration, error) + // GetMDMAppleConfigProfileByDeprecatedID retrieves the specified Apple // configuration profile via its numeric ID. This method is deprecated and // should not be used for new endpoints. GetMDMAppleConfigProfileByDeprecatedID(ctx context.Context, profileID uint) (*MDMAppleConfigProfile, error) // GetMDMAppleConfigProfile retrieves the specified configuration profile. GetMDMAppleConfigProfile(ctx context.Context, profileUUID string) (*MDMAppleConfigProfile, error) + + // GetMDMAppleDeclaration retrieves the specified declaration. + GetMDMAppleDeclaration(ctx context.Context, declarationUUID string) (*MDMAppleDeclaration, error) + // DeleteMDMAppleConfigProfileByDeprecatedID deletes the specified Apple // configuration profile via its numeric ID. This method is deprecated and // should not be used for new endpoints. DeleteMDMAppleConfigProfileByDeprecatedID(ctx context.Context, profileID uint) error // DeleteMDMAppleConfigProfile deletes the specified configuration profile. DeleteMDMAppleConfigProfile(ctx context.Context, profileUUID string) error + + // DeleteMDMAppleDeclaration deletes the specified declaration. + DeleteMDMAppleDeclaration(ctx context.Context, declarationUUID string) error + // ListMDMAppleConfigProfiles returns the list of all the configuration profiles for the // specified team. ListMDMAppleConfigProfiles(ctx context.Context, teamID uint) ([]*MDMAppleConfigProfile, error) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 6532899ba3..fdc59496dd 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -620,6 +620,8 @@ type GetMDMAppleConfigProfileByDeprecatedIDFunc func(ctx context.Context, profil type GetMDMAppleConfigProfileFunc func(ctx context.Context, profileUUID string) (*fleet.MDMAppleConfigProfile, error) +type GetMDMAppleDeclarationFunc func(ctx context.Context, declUUID string) (*fleet.MDMAppleDeclaration, error) + type ListMDMAppleConfigProfilesFunc func(ctx context.Context, teamID *uint) ([]*fleet.MDMAppleConfigProfile, error) type DeleteMDMAppleConfigProfileByDeprecatedIDFunc func(ctx context.Context, profileID uint) error @@ -1770,6 +1772,9 @@ type DataStore struct { GetMDMAppleConfigProfileFunc GetMDMAppleConfigProfileFunc GetMDMAppleConfigProfileFuncInvoked bool + GetMDMAppleDeclarationFunc GetMDMAppleDeclarationFunc + GetMDMAppleDeclarationFuncInvoked bool + ListMDMAppleConfigProfilesFunc ListMDMAppleConfigProfilesFunc ListMDMAppleConfigProfilesFuncInvoked bool @@ -4249,6 +4254,13 @@ func (s *DataStore) GetMDMAppleConfigProfile(ctx context.Context, profileUUID st return s.GetMDMAppleConfigProfileFunc(ctx, profileUUID) } +func (s *DataStore) GetMDMAppleDeclaration(ctx context.Context, declUUID string) (*fleet.MDMAppleDeclaration, error) { + s.mu.Lock() + s.GetMDMAppleDeclarationFuncInvoked = true + s.mu.Unlock() + return s.GetMDMAppleDeclarationFunc(ctx, declUUID) +} + func (s *DataStore) ListMDMAppleConfigProfiles(ctx context.Context, teamID *uint) ([]*fleet.MDMAppleConfigProfile, error) { s.mu.Lock() s.ListMDMAppleConfigProfilesFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 3c3bb3031d..b67796b33e 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -31,6 +31,7 @@ import ( "github.com/fleetdm/fleet/v4/server/mdm/apple/mobileconfig" "github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" + nano_service "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/service" "github.com/fleetdm/fleet/v4/server/sso" "github.com/fleetdm/fleet/v4/server/worker" kitlog "github.com/go-kit/log" @@ -399,6 +400,12 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i return nil, ctxerr.Wrap(ctx, err, "check macOS MDM enabled") } + fleetNames := mdm_types.FleetReservedProfileNames() + if _, ok := fleetNames[name]; ok { + err := fleet.NewInvalidArgumentError("declaration", fmt.Sprintf("Profile name %q is not allowed.", name)).WithStatus(http.StatusBadRequest) + return nil, err + } + data, err := io.ReadAll(r) if err != nil { return nil, err @@ -438,7 +445,7 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i return decl, nil } -func (svc *Service) batchValidateDeclarationLabels(ctx context.Context, labelNames []string) (map[string]fleet.DeclarationLabel, error) { +func (svc *Service) batchValidateDeclarationLabels(ctx context.Context, labelNames []string) (map[string]fleet.ConfigurationProfileLabel, error) { if len(labelNames) == 0 { return nil, nil } @@ -462,9 +469,9 @@ func (svc *Service) batchValidateDeclarationLabels(ctx context.Context, labelNam } } - profLabels := make(map[string]fleet.DeclarationLabel) + profLabels := make(map[string]fleet.ConfigurationProfileLabel) for labelName, labelID := range labels { - profLabels[labelName] = fleet.DeclarationLabel{ + profLabels[labelName] = fleet.ConfigurationProfileLabel{ LabelName: labelName, LabelID: labelID, } @@ -472,13 +479,13 @@ func (svc *Service) batchValidateDeclarationLabels(ctx context.Context, labelNam return profLabels, nil } -func (svc *Service) validateDeclarationLabels(ctx context.Context, labelNames []string) ([]fleet.DeclarationLabel, error) { +func (svc *Service) validateDeclarationLabels(ctx context.Context, labelNames []string) ([]fleet.ConfigurationProfileLabel, error) { labelMap, err := svc.batchValidateDeclarationLabels(ctx, labelNames) if err != nil { return nil, ctxerr.Wrap(ctx, err, "validating declaration labels") } - var declLabels []fleet.DeclarationLabel + var declLabels []fleet.ConfigurationProfileLabel for _, label := range labelMap { declLabels = append(declLabels, label) } @@ -613,6 +620,25 @@ func (svc *Service) GetMDMAppleConfigProfile(ctx context.Context, profileUUID st return cp, nil } +func (svc *Service) GetMDMAppleDeclaration(ctx context.Context, profileUUID string) (*fleet.MDMAppleDeclaration, error) { + // first we perform a perform basic authz check + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return nil, err + } + + cp, err := svc.ds.GetMDMAppleDeclaration(ctx, profileUUID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err) + } + + // now we can do a specific authz check based on team id of profile before we return the profile + if err := svc.authz.Authorize(ctx, &fleet.MDMConfigProfileAuthz{TeamID: cp.TeamID}, fleet.ActionRead); err != nil { + return nil, err + } + + return cp, nil +} + type deleteMDMAppleConfigProfileRequest struct { ProfileID uint `url:"profile_id"` } @@ -721,6 +747,88 @@ func (svc *Service) DeleteMDMAppleConfigProfile(ctx context.Context, profileUUID return nil } +func (svc *Service) DeleteMDMAppleDeclaration(ctx context.Context, declUUID string) error { + // first we perform a perform basic authz check + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return ctxerr.Wrap(ctx, err) + } + + // check that Apple MDM is enabled - the middleware of that endpoint checks + // only that any MDM is enabled, maybe it's just Windows + if err := svc.VerifyMDMAppleConfigured(ctx); err != nil { + err := fleet.NewInvalidArgumentError("profile_uuid", fleet.AppleMDMNotConfiguredMessage).WithStatus(http.StatusBadRequest) + return ctxerr.Wrap(ctx, err, "check macOS MDM enabled") + } + + decl, err := svc.ds.GetMDMAppleDeclaration(ctx, declUUID) + if err != nil { + return ctxerr.Wrap(ctx, err) + } + + if _, ok := mdm_types.FleetReservedProfileNames()[decl.Name]; ok { + return &fleet.BadRequestError{ + Message: "profiles managed by Fleet can't be deleted using this endpoint.", + InternalErr: fmt.Errorf("deleting profile %s is not allowed because it's managed by Fleet", decl.Name), + } + } + + // TODO: refine our approach to deleting restricted/forbidden types of declarations so that we + // can check that Fleet-managed aren't being deleted; this can be addressed once we add support + // for more types of declarations + var d fleet.MDMAppleRawDeclaration + if err := json.Unmarshal(decl.RawJSON, &d); err != nil { + return ctxerr.Wrap(ctx, err, "unmarshalling declaration") + } + if err := d.ValidateUserProvided(); err != nil { + return ctxerr.Wrap(ctx, &fleet.BadRequestError{Message: err.Error()}) + } + + var teamName string + teamID := *decl.TeamID + if teamID >= 1 { + tm, err := svc.EnterpriseOverrides.TeamByIDOrName(ctx, &teamID, nil) + if err != nil { + return ctxerr.Wrap(ctx, err) + } + teamName = tm.Name + } + + // now we can do a specific authz check based on team id of profile before we delete the profile + if err := svc.authz.Authorize(ctx, &fleet.MDMConfigProfileAuthz{TeamID: decl.TeamID}, fleet.ActionWrite); err != nil { + return ctxerr.Wrap(ctx, err) + } + + if err := svc.ds.DeleteMDMAppleConfigProfile(ctx, declUUID); err != nil { + return ctxerr.Wrap(ctx, err) + } + + // TODO: confirm if bulk set pending host profiles is needed + // cannot use the profile ID as it is now deleted + if err := svc.ds.BulkSetPendingMDMHostProfiles(ctx, nil, []uint{teamID}, nil, nil); err != nil { + return ctxerr.Wrap(ctx, err, "bulk set pending host profiles") + } + + // TODO: confirm activity type + var ( + actTeamID *uint + actTeamName *string + ) + if teamID > 0 { + actTeamID = &teamID + actTeamName = &teamName + } + if err := svc.ds.NewActivity(ctx, authz.UserFromContext(ctx), &fleet.ActivityTypeDeletedMacosProfile{ + TeamID: actTeamID, + TeamName: actTeamName, + ProfileName: decl.Name, + ProfileIdentifier: decl.Identifier, + }); err != nil { + return ctxerr.Wrap(ctx, err, "logging activity for delete mdm apple declaration") + } + + return nil +} + type getMDMAppleFileVaultSummaryRequest struct { TeamID *uint `query:"team_id,optional"` } @@ -3073,7 +3181,7 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec level.Debug(svc.logger).Log("msg", "ddm request received", "endpoint", dm.Endpoint) if dm.UDID == "" { - return nil, ctxerr.New(r.Context, "missing device id") + return nil, nano_service.NewHTTPStatusError(http.StatusBadRequest, ctxerr.New(r.Context, "missing UDID in request")) } switch { @@ -3091,12 +3199,12 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec return nil, nil - case strings.HasPrefix(dm.Endpoint, "declaration"): + case strings.HasPrefix(dm.Endpoint, "declaration/"): level.Debug(svc.logger).Log("msg", "received declarations request") return svc.handleDeclarationsResponse(r.Context, dm.Endpoint, dm.UDID) default: - return nil, ctxerr.New(r.Context, "unrecognized ddm endpoint") + return nil, nano_service.NewHTTPStatusError(http.StatusBadRequest, ctxerr.New(r.Context, fmt.Sprintf("unrecognized declarations endpoint: %s", dm.Endpoint))) } } @@ -3162,7 +3270,7 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, hostU func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, endpoint string, hostUUID string) ([]byte, error) { parts := strings.Split(endpoint, "/") if len(parts) != 3 { - return nil, ctxerr.New(ctx, "unrecognized declarations endpoint") + return nil, nano_service.NewHTTPStatusError(http.StatusBadRequest, ctxerr.New(ctx, fmt.Sprintf("unrecognized declarations endpoint: %s", endpoint))) } level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", parts[1], "identifier", parts[2]) @@ -3172,7 +3280,10 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e hostUUID, ) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting declaration") + if fleet.IsNotFound(err) { + return nil, nano_service.NewHTTPStatusError(http.StatusNotFound, err) + } + return nil, ctxerr.Wrap(ctx, err, "getting declaration response") } // unmarshall into a temporary map in order to add the token. diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 93f73a72ed..fdf4f12af5 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9297,7 +9297,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { require.Equal(t, "a", string(resp.ProfileUUID[0])) return resp.ProfileUUID } - assertAppleDeclaration := func(filename, name, ident string, teamID uint, labelNames []string, wantStatus int, wantErrMsg string) string { + assertAppleDeclaration := func(filename, ident string, teamID uint, labelNames []string, wantStatus int, wantErrMsg string) string { fields := map[string][]string{ "labels": labelNames, } @@ -9326,7 +9326,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { err := json.NewDecoder(res.Body).Decode(&resp) require.NoError(t, err) require.NotEmpty(t, resp.ProfileUUID) - require.Equal(t, "x", string(resp.ProfileUUID[0])) + require.Equal(t, fleet.MDMAppleDeclarationUUIDPrefix, string(resp.ProfileUUID[0])) return resp.ProfileUUID } @@ -9412,17 +9412,17 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { assertAppleProfile("win-team-profile.mobileconfig", "win-team-profile", "test-team-ident-2", 0, nil, http.StatusOK, "") // add some macOS declarations - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusOK, "") + assertAppleDeclaration("apple-declaration.json", "test-declaration-ident", 0, nil, http.StatusOK, "") // identifier must be unique, it conflicts with existing declaration - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", 0, nil, http.StatusConflict, "Resource Already Exists: MDMAppleDeclaration.Identifier test-declaration-ident already exists") + assertAppleDeclaration("apple-declaration.json", "test-declaration-ident", 0, nil, http.StatusConflict, "test-declaration-ident already exists") // name is pulled from filename, it conflicts with existing declaration - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "Resource Already Exists: MDMAppleDeclaration.Name apple-declaration already exists") + assertAppleDeclaration("apple-declaration.json", "test-declaration-ident-2", 0, nil, http.StatusConflict, "apple-declaration already exists") // uniqueness is checked only within team, so it's fine to have the same name and identifier in different teams - assertAppleDeclaration("apple-declaration.json", "", "test-declaration-ident", testTeam.ID, nil, http.StatusOK, "") + assertAppleDeclaration("apple-declaration.json", "test-declaration-ident", testTeam.ID, nil, http.StatusOK, "") // name is pulled from filename, it conflicts with existing macOS config profile - assertAppleDeclaration("apple-global-profile.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "apple-global-profile already exists") + assertAppleDeclaration("apple-global-profile.json", "test-declaration-ident-2", 0, nil, http.StatusConflict, "apple-global-profile already exists") // name is pulled from filename, it conflicts with existing macOS config profile - assertAppleDeclaration("win-global-profile.json", "", "test-declaration-ident-2", 0, nil, http.StatusConflict, "win-global-profile already exists") + assertAppleDeclaration("win-global-profile.json", "test-declaration-ident-2", 0, nil, http.StatusConflict, "win-global-profile already exists") // windows profile name conflicts with existing declaration assertWindowsProfile("apple-declaration.xml", "./Test", 0, nil, http.StatusConflict, "Couldn't upload. A configuration profile with this name already exists.") // macOS profile name conflicts with existing declaration @@ -9431,6 +9431,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // not an xml nor mobileconfig file assertWindowsProfile("foo.txt", "./Test", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") assertAppleProfile("foo.txt", "foo", "foo-ident", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") + assertAppleDeclaration("foo.txt", "foo-ident", 0, nil, http.StatusBadRequest, "Couldn't add profile. The file should be a .mobileconfig, XML, or JSON file.") // Windows-reserved LocURI assertWindowsProfile("bitlocker.xml", syncml.FleetBitLockerTargetLocURI, 0, nil, http.StatusBadRequest, "Couldn't upload. Custom configuration profiles can't include BitLocker settings.") @@ -9439,11 +9440,13 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // Fleet-reserved profiles for name := range servermdm.FleetReservedProfileNames() { assertAppleProfile(name+".mobileconfig", name, name+"-ident", 0, nil, http.StatusBadRequest, fmt.Sprintf(`name %s is not allowed`, name)) + assertAppleDeclaration(name+".json", name+"-ident", 0, nil, http.StatusBadRequest, fmt.Sprintf(`name %q is not allowed`, name)) assertWindowsProfile(name+".xml", "./Test", 0, nil, http.StatusBadRequest, fmt.Sprintf(`Couldn't upload. Profile name %q is not allowed.`, name)) } // profiles with non-existent labels assertAppleProfile("apple-profile-with-labels.mobileconfig", "apple-profile-with-labels", "ident-with-labels", 0, []string{"does-not-exist"}, http.StatusBadRequest, "some or all the labels provided don't exist") + assertAppleDeclaration("apple-declaration-with-labels.json", "ident-with-labels", 0, []string{"does-not-exist"}, http.StatusBadRequest, "some or all the labels provided don't exist") assertWindowsProfile("win-profile-with-labels.xml", "./Test", 0, []string{"does-not-exist"}, http.StatusBadRequest, "some or all the labels provided don't exist") // create a couple of labels @@ -9456,26 +9459,33 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // profiles mixing existent and non-existent labels assertAppleProfile("apple-profile-with-labels.mobileconfig", "apple-profile-with-labels", "ident-with-labels", 0, []string{"does-not-exist", "foo"}, http.StatusBadRequest, "some or all the labels provided don't exist") + assertAppleDeclaration("apple-declaration-with-labels.json", "ident-with-labels", 0, []string{"does-not-exist", "foo"}, http.StatusBadRequest, "some or all the labels provided don't exist") assertWindowsProfile("win-profile-with-labels.xml", "./Test", 0, []string{"does-not-exist", "bar"}, http.StatusBadRequest, "some or all the labels provided don't exist") // profiles with valid labels uuidAppleWithLabel := assertAppleProfile("apple-profile-with-labels.mobileconfig", "apple-profile-with-labels", "ident-with-labels", 0, []string{"foo"}, http.StatusOK, "") + uuidAppleDDMWithLabel := assertAppleDeclaration("apple-decl-with-labels.json", "ident-decl-with-labels", 0, []string{"foo"}, http.StatusOK, "") uuidWindowsWithLabel := assertWindowsProfile("win-profile-with-labels.xml", "./Test", 0, []string{"foo", "bar"}, http.StatusOK, "") // verify that the label associations have been created // TODO: update when we have datastore methods to get this data var profileLabels []fleet.ConfigurationProfileLabel mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - stmt := `SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, label_id FROM mdm_configuration_profile_labels` + stmt := ` + SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, label_id + FROM mdm_configuration_profile_labels + UNION SELECT apple_declaration_uuid as profile_uuid, label_name, label_id + FROM mdm_declaration_labels ORDER BY profile_uuid, label_name;` return sqlx.SelectContext(context.Background(), q, &profileLabels, stmt) }) require.NotEmpty(t, profileLabels) - require.Len(t, profileLabels, 3) + require.Len(t, profileLabels, 4) require.ElementsMatch( t, []fleet.ConfigurationProfileLabel{ {ProfileUUID: uuidAppleWithLabel, LabelName: labelFoo.Name, LabelID: labelFoo.ID}, + {ProfileUUID: uuidAppleDDMWithLabel, LabelName: labelFoo.Name, LabelID: labelFoo.ID}, {ProfileUUID: uuidWindowsWithLabel, LabelName: labelFoo.Name, LabelID: labelFoo.ID}, {ProfileUUID: uuidWindowsWithLabel, LabelName: labelBar.Name, LabelID: labelBar.ID}, }, @@ -9488,19 +9498,27 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { errMsg := extractServerErrorText(res.Body) require.Contains(t, errMsg, "Couldn't upload. The file should include valid XML:") - // Apple invalid content + // Apple invalid mobileconfig content body, headers = generateNewProfileMultipartRequest(t, "apple.mobileconfig", []byte("\x00\x01\x02"), s.token, nil) res = s.DoRawWithHeaders("POST", "/api/latest/fleet/configuration_profiles", body.Bytes(), http.StatusBadRequest, headers) errMsg = extractServerErrorText(res.Body) require.Contains(t, errMsg, "mobileconfig is not XML nor PKCS7 parseable") + // Apple invalid json declaration + body, headers = generateNewProfileMultipartRequest(t, + "apple.json", []byte("{"), s.token, nil) + res = s.DoRawWithHeaders("POST", "/api/latest/fleet/configuration_profiles", body.Bytes(), http.StatusBadRequest, headers) + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Couldn't upload. The file should include valid JSON:") + // get the existing profiles work expectedProfiles := []fleet.MDMConfigProfilePayload{ {ProfileUUID: noTeamAppleProfUUID, Platform: "darwin", Name: "apple-global-profile", Identifier: "test-global-ident", TeamID: nil}, {ProfileUUID: teamAppleProfUUID, Platform: "darwin", Name: "apple-team-profile", Identifier: "test-team-ident", TeamID: &testTeam.ID}, {ProfileUUID: noTeamWinProfUUID, Platform: "windows", Name: "win-global-profile", TeamID: nil}, {ProfileUUID: teamWinProfUUID, Platform: "windows", Name: "win-team-profile", TeamID: &testTeam.ID}, + {ProfileUUID: uuidAppleDDMWithLabel, Platform: "darwin", Name: "apple-decl-with-labels", Identifier: "ident-decl-with-labels", TeamID: nil, Labels: []fleet.ConfigurationProfileLabel{{LabelID: labelFoo.ID, LabelName: labelFoo.Name}}}, } for _, prof := range expectedProfiles { var getResp getMDMConfigProfileResponse @@ -9519,8 +9537,10 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { resp := s.Do("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", prof.ProfileUUID), nil, http.StatusOK, "alt", "media") require.NotZero(t, resp.ContentLength) require.Contains(t, resp.Header.Get("Content-Disposition"), "attachment;") - if getResp.Platform == "darwin" { + if strings.HasPrefix(prof.ProfileUUID, "a") { require.Contains(t, resp.Header.Get("Content-Type"), "application/x-apple-aspen-config") + } else if strings.HasPrefix(prof.ProfileUUID, fleet.MDMAppleDeclarationUUIDPrefix) { + require.Contains(t, resp.Header.Get("Content-Type"), "application/json") } else { require.Contains(t, resp.Header.Get("Content-Type"), "application/octet-stream") } @@ -9535,6 +9555,9 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // get an unknown Apple profile s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", "ano-such-profile"), nil, http.StatusNotFound, &getResp) s.Do("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", "ano-such-profile"), nil, http.StatusNotFound, "alt", "media") + // get an unknown Apple declaration + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", fmt.Sprintf("%sno-such-profile", fleet.MDMAppleDeclarationUUIDPrefix)), nil, http.StatusNotFound, &getResp) + s.Do("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", fmt.Sprintf("%sno-such-profile", fleet.MDMAppleDeclarationUUIDPrefix)), nil, http.StatusNotFound, "alt", "media") // get an unknown Windows profile s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", "wno-such-profile"), nil, http.StatusNotFound, &getResp) s.Do("GET", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", "wno-such-profile"), nil, http.StatusNotFound, "alt", "media") @@ -9545,6 +9568,11 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", teamAppleProfUUID), nil, http.StatusOK, &deleteResp) // delete non-existing Apple profile s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", "ano-such-profile"), nil, http.StatusNotFound, &deleteResp) + + // delete existing Apple declaration + s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", uuidAppleDDMWithLabel), nil, http.StatusOK, &deleteResp) + // delete non-existing Apple declaration + s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", fmt.Sprintf("%sno-such-profile", fleet.MDMAppleDeclarationUUIDPrefix)), nil, http.StatusNotFound, &deleteResp) // delete existing Windows profiles s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", noTeamWinProfUUID), nil, http.StatusOK, &deleteResp) s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", teamWinProfUUID), nil, http.StatusOK, &deleteResp) @@ -9575,6 +9603,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { return err }) } + // TODO: Add tests for create/delete forbidden declaration types? // make fleet add a FileVault profile acResp := appConfigResponse{} @@ -9596,6 +9625,8 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // try to delete the profile s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", profUUID), nil, http.StatusBadRequest, &deleteResp) + + // TODO: Add tests for OS updates declaration when implemented. } func (s *integrationMDMTestSuite) TestListMDMConfigProfiles() { @@ -13001,6 +13032,16 @@ INSERT INTO host_mdm_apple_declarations ( r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) require.NoError(t, err) + // try getting a non-existent declaration, should fail 404 + _, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", "nonexistent")) + require.Error(t, err) + require.ErrorContains(t, err, "404 Not Found") + + // typo should fail as bad request + _, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configurations", want.Identifier)) + require.Error(t, err) + require.ErrorContains(t, err, "400 Bad Request") + assertDeclarationResponse(r, want) }) } diff --git a/server/service/mdm.go b/server/service/mdm.go index 80ae99d97c..63c71ba286 100644 --- a/server/service/mdm.go +++ b/server/service/mdm.go @@ -1019,6 +1019,25 @@ func getMDMConfigProfileEndpoint(ctx context.Context, request interface{}, svc f }, nil } + if isAppleDeclarationUUID(req.ProfileUUID) { + // TODO: we could potentially combined with the other service methods + decl, err := svc.GetMDMAppleDeclaration(ctx, req.ProfileUUID) + if err != nil { + return &getMDMConfigProfileResponse{Err: err}, nil + } + + if downloadRequested { + return downloadFileResponse{ + content: decl.RawJSON, + contentType: "application/json", + filename: fmt.Sprintf("%s_%s.json", time.Now().Format("2006-01-02"), strings.ReplaceAll(decl.Name, " ", "_")), + }, nil + } + return &getMDMConfigProfileResponse{ + MDMConfigProfilePayload: fleet.NewMDMConfigProfilePayloadFromAppleDDM(decl), + }, nil + } + // Windows config profile cp, err := svc.GetMDMWindowsConfigProfile(ctx, req.ProfileUUID) if err != nil { @@ -1077,6 +1096,9 @@ func deleteMDMConfigProfileEndpoint(ctx context.Context, request interface{}, sv var err error if isAppleProfileUUID(req.ProfileUUID) { err = svc.DeleteMDMAppleConfigProfile(ctx, req.ProfileUUID) + } else if isAppleDeclarationUUID(req.ProfileUUID) { + // TODO: we could potentially combined with the other service methods + err = svc.DeleteMDMAppleDeclaration(ctx, req.ProfileUUID) } else { err = svc.DeleteMDMWindowsConfigProfile(ctx, req.ProfileUUID) } @@ -1157,6 +1179,10 @@ func isAppleProfileUUID(profileUUID string) bool { return strings.HasPrefix(profileUUID, "a") } +func isAppleDeclarationUUID(profileUUID string) bool { + return strings.HasPrefix(profileUUID, fleet.MDMAppleDeclarationUUIDPrefix) +} + //////////////////////////////////////////////////////////////////////////////// // POST /mdm/profiles (Create Apple or Windows MDM Config Profile) //////////////////////////////////////////////////////////////////////////////// @@ -1625,7 +1651,7 @@ func getAppleProfiles( mdmDecl := fleet.NewMDMAppleDeclaration(prof.Contents, tmID, prof.Name, rawDecl.Type, rawDecl.Identifier) for _, labelName := range prof.Labels { if lbl, ok := labelMap[labelName]; ok { - declLabel := fleet.DeclarationLabel{ + declLabel := fleet.ConfigurationProfileLabel{ LabelName: lbl.LabelName, LabelID: lbl.LabelID, } diff --git a/server/service/mdm_test.go b/server/service/mdm_test.go index 4c2b3738f8..472981e518 100644 --- a/server/service/mdm_test.go +++ b/server/service/mdm_test.go @@ -753,6 +753,8 @@ func TestGetMDMDiskEncryptionSummary(t *testing.T) { }) } +// TODO: Add tests for Apple DDM authz? + func TestMDMWindowsConfigProfileAuthz(t *testing.T) { ds := new(mock.Store) // while the config profiles are not premium-only, teams are and we want to test with teams. From d3c843801eb8e9bfc8ae1e66c374318de8fce2ac Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Mon, 25 Mar 2024 19:15:33 +0000 Subject: [PATCH 13/29] update UI to add support for mac ddm profiles (#17730) relates to #17416 update UI to support new DDM profile types. this includes: - updating Custom settings page - updating the os settings modal - [x] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Manual QA for all new/changed functionality --- changes/issue-17416-update-ui-to-support-ddm | 1 + frontend/__mocks__/hostMock.ts | 2 +- frontend/interfaces/mdm.ts | 7 +- .../ProfileStatusAggregateOptions.ts | 3 +- .../cards/CustomSettings/_styles.scss | 2 +- .../ProfileListItem/ProfileListItem.tsx | 18 +++-- .../components/AddProfileGraphic.tsx | 2 +- .../HostDetailsPage/HostDetailsPage.tsx | 26 ++++++- .../OSSettingStatusCell/helpers.ts | 52 ++++++++------ .../OSSettingsTable/OSSettingsTable.tsx | 6 +- .../OSSettingsTable/OSSettingsTableConfig.tsx | 72 +++++++------------ .../OSSettingsIndicator.tsx | 3 +- frontend/services/entities/mdm.ts | 5 ++ 13 files changed, 119 insertions(+), 80 deletions(-) create mode 100644 changes/issue-17416-update-ui-to-support-ddm diff --git a/changes/issue-17416-update-ui-to-support-ddm b/changes/issue-17416-update-ui-to-support-ddm new file mode 100644 index 0000000000..3bbe4eaaa9 --- /dev/null +++ b/changes/issue-17416-update-ui-to-support-ddm @@ -0,0 +1 @@ +- update UI to support macos DDM profiles. diff --git a/frontend/__mocks__/hostMock.ts b/frontend/__mocks__/hostMock.ts index ebd2bb37eb..5912ef451c 100644 --- a/frontend/__mocks__/hostMock.ts +++ b/frontend/__mocks__/hostMock.ts @@ -10,7 +10,7 @@ const DEFAULT_HOST_PROFILE_MOCK: IHostMdmProfile = { detail: "This is verified", }; -export const createMockHostMacMdmProfile = ( +export const createMockHostMdmProfile = ( overrides?: Partial ): IHostMdmProfile => { return { ...DEFAULT_HOST_PROFILE_MOCK, ...overrides }; diff --git a/frontend/interfaces/mdm.ts b/frontend/interfaces/mdm.ts index c7b4071fac..10d87ed75f 100644 --- a/frontend/interfaces/mdm.ts +++ b/frontend/interfaces/mdm.ts @@ -81,6 +81,11 @@ export interface IMdmProfile { } export type MdmProfileStatus = "verified" | "verifying" | "pending" | "failed"; +export type MdmDDMProfileStatus = + | "success" + | "pending" + | "failed" + | "acknowledged"; export type ProfileOperationType = "remove" | "install"; @@ -89,7 +94,7 @@ export interface IHostMdmProfile { name: string; operation_type: ProfileOperationType | null; platform: ProfilePlatform; - status: MdmProfileStatus; + status: MdmProfileStatus | MdmDDMProfileStatus; detail: string; } diff --git a/frontend/pages/ManageControlsPage/OSSettings/ProfileStatusAggregate/ProfileStatusAggregateOptions.ts b/frontend/pages/ManageControlsPage/OSSettings/ProfileStatusAggregate/ProfileStatusAggregateOptions.ts index 8dbe94abf8..c2a917fb59 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/ProfileStatusAggregate/ProfileStatusAggregateOptions.ts +++ b/frontend/pages/ManageControlsPage/OSSettings/ProfileStatusAggregate/ProfileStatusAggregateOptions.ts @@ -14,7 +14,8 @@ const AGGREGATE_STATUS_DISPLAY_OPTIONS: IAggregateDisplayOption[] = [ text: "Verified", iconName: "success", tooltipText: - "These hosts applied all OS settings. Fleet verified with osquery.", + "These hosts applied all OS settings. Fleet verified with osquery. " + + "Declaration profiles are verified with DDM.", }, { value: "verifying", diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/_styles.scss b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/_styles.scss index 3c869da345..9f6c3eef81 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/_styles.scss +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/_styles.scss @@ -186,7 +186,7 @@ flex-direction: column; border-radius: $border-radius; border: 1px solid $ui-fleet-black-10; - overflow-y: scroll; + overflow-y: auto; .loading-spinner { margin: 69.5px auto; diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx index 15b7676701..51c185c521 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx @@ -4,7 +4,7 @@ import FileSaver from "file-saver"; import classnames from "classnames"; import { IMdmProfile } from "interfaces/mdm"; -import mdmAPI from "services/entities/mdm"; +import mdmAPI, { isDDMProfile } from "services/entities/mdm"; import Button from "components/buttons/Button"; import Graphic from "components/Graphic"; @@ -29,11 +29,17 @@ const LabelCount = ({ interface IProfileDetailsProps { platform: string; createdAt: string; + isDDM?: boolean; } -const ProfileDetails = ({ platform, createdAt }: IProfileDetailsProps) => { +const ProfileDetails = ({ + platform, + createdAt, + isDDM, +}: IProfileDetailsProps) => { const getPlatformName = () => { - return platform === "darwin" ? "macOS" : "Windows"; + if (platform === "windows") return "Windows"; + return isDDM ? "macOS (declaration)" : "macOS"; }; return ( @@ -81,7 +87,11 @@ const ProfileListItem = ({
{name}
- +
diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileGraphic.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileGraphic.tsx index c0460e83dd..c75193dcb6 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileGraphic.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileGraphic.tsx @@ -3,7 +3,7 @@ import React from "react"; import Graphic from "components/Graphic"; const ALLOWED_FILE_TYPES_MESSAGE = - "Configuration profile (.mobileconfig for macOS or .xml for Windows)"; + "Configuration profile (.mobileconfig and .json for macOS or .xml for Windows)"; const ProfileGraphic = ({ baseClass, diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 99faef93dd..11b2178bcf 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -13,7 +13,6 @@ import { QueryContext } from "context/query"; import { NotificationContext } from "context/notification"; import activitiesAPI, { - IActivitiesResponse, IPastActivitiesResponse, IUpcomingActivitiesResponse, } from "services/entities/activities"; @@ -53,6 +52,7 @@ import { HOST_ABOUT_DATA, HOST_OSQUERY_DATA, } from "utilities/constants"; +import { createMockHostMdmProfile } from "__mocks__/hostMock"; import Spinner from "components/Spinner"; import TabsWrapper from "components/TabsWrapper"; @@ -753,6 +753,30 @@ const HostDetailsPage = ({ name: host?.mdm.macos_setup?.bootstrap_package_name, }; + // TODO: Remove this when API is ready + if (!host.mdm.profiles) { + host.mdm.profiles = []; + } else { + host.mdm.profiles = [ + createMockHostMdmProfile({ + name: "test.json", + status: "success", + }), + createMockHostMdmProfile({ + name: "test2.json", + status: "pending", + }), + createMockHostMdmProfile({ + name: "test3.json", + status: "failed", + }), + createMockHostMdmProfile({ + name: "test4.json", + status: "acknowledged", + }), + ]; + } + return ( <> diff --git a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingStatusCell/helpers.ts b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingStatusCell/helpers.ts index da989801ef..8f5dd764ee 100644 --- a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingStatusCell/helpers.ts +++ b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingStatusCell/helpers.ts @@ -22,28 +22,35 @@ type OperationTypeOption = Record< type ProfileDisplayConfig = Record; +const MAC_PROFILE_VERIFIED_DISPLAY_CONFIG: ProfileDisplayOption = { + statusText: "Verified", + iconName: "success", + tooltip: (innerProps) => + innerProps.isDiskEncryptionProfile + ? "The host turned disk encryption on and sent the key to Fleet. " + + "Fleet verified with osquery." + : "The host applied the setting. Fleet verified with osquery. " + + "Declaration profiles are verified with DDM.", +} as const; + +const MAC_PROFILE_VERIFYING_DISPLAY_CONFIG: ProfileDisplayOption = { + statusText: "Verifying", + iconName: "success-outline", + tooltip: (innerProps) => + innerProps.isDiskEncryptionProfile + ? "The host acknowledged the MDM command to turn on disk encryption. " + + "Fleet is verifying with osquery and retrieving the disk encryption key. " + + "This may take up to one hour." + : "The host acknowledged the MDM command to apply the setting. Fleet is " + + "verifying with osquery.", +} as const; + export const PROFILE_DISPLAY_CONFIG: ProfileDisplayConfig = { install: { - verified: { - statusText: "Verified", - iconName: "success", - tooltip: (innerProps) => - innerProps.isDiskEncryptionProfile - ? "The host turned disk encryption on and sent the key to Fleet. " + - "Fleet verified with osquery." - : "The host applied the setting. Fleet verified with osquery.", - }, - verifying: { - statusText: "Verifying", - iconName: "success-outline", - tooltip: (innerProps) => - innerProps.isDiskEncryptionProfile - ? "The host acknowledged the MDM command to turn on disk encryption. " + - "Fleet is verifying with osquery and retrieving the disk encryption key. " + - "This may take up to one hour." - : "The host acknowledged the MDM command to apply the setting. Fleet is " + - "verifying with osquery.", - }, + verified: MAC_PROFILE_VERIFIED_DISPLAY_CONFIG, + success: MAC_PROFILE_VERIFIED_DISPLAY_CONFIG, + verifying: MAC_PROFILE_VERIFYING_DISPLAY_CONFIG, + acknowledged: MAC_PROFILE_VERIFYING_DISPLAY_CONFIG, pending: { statusText: "Enforcing (pending)", iconName: "pending-outline", @@ -79,6 +86,8 @@ export const PROFILE_DISPLAY_CONFIG: ProfileDisplayConfig = { action_required: null, // should not be reached verified: null, // should not be reached verifying: null, // should not be reached + success: null, // should not be reached + acknowledged: null, // should not be reached failed: { statusText: "Failed", iconName: "error", @@ -89,7 +98,8 @@ export const PROFILE_DISPLAY_CONFIG: ProfileDisplayConfig = { type WindowsDiskEncryptionDisplayConfig = Omit< OperationTypeOption, - "action_required" + // windows disk encryption does not have these states + "action_required" | "success" | "acknowledged" >; export const WINDOWS_DISK_ENCRYPTION_DISPLAY_CONFIG: WindowsDiskEncryptionDisplayConfig = { diff --git a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTable.tsx b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTable.tsx index d96606e357..7aea4d34ad 100644 --- a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTable.tsx +++ b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTable.tsx @@ -1,12 +1,14 @@ import React from "react"; import TableContainer from "components/TableContainer"; -import tableHeaders, { ITableRowOsSettings } from "./OSSettingsTableConfig"; +import tableHeaders, { + IHostMdmProfileWithAddedStatus, +} from "./OSSettingsTableConfig"; const baseClass = "os-settings-table"; interface IOSSettingsTableProps { - tableData?: ITableRowOsSettings[]; + tableData?: IHostMdmProfileWithAddedStatus[]; } const OSSettingsTable = ({ tableData }: IOSSettingsTableProps) => { diff --git a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTableConfig.tsx b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTableConfig.tsx index 005a8e7e72..5c94da95ab 100644 --- a/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTableConfig.tsx +++ b/frontend/pages/hosts/details/OSSettingsModal/OSSettingsTable/OSSettingsTableConfig.tsx @@ -1,58 +1,44 @@ -import TextCell from "components/TableContainer/DataTable/TextCell"; import React from "react"; +import { Column } from "react-table"; +import { IStringCellProps } from "interfaces/datatable_config"; import { IHostMdmData } from "interfaces/host"; import { FLEET_FILEVAULT_PROFILE_DISPLAY_NAME, // FLEET_FILEVAULT_PROFILE_IDENTIFIER, IHostMdmProfile, + MdmDDMProfileStatus, MdmProfileStatus, ProfilePlatform, isWindowsDiskEncryptionStatus, } from "interfaces/mdm"; import { DEFAULT_EMPTY_CELL_VALUE } from "utilities/constants"; + +import TextCell from "components/TableContainer/DataTable/TextCell"; import TooltipTruncatedTextCell from "components/TableContainer/DataTable/TooltipTruncatedTextCell"; + import OSSettingStatusCell from "./OSSettingStatusCell"; import { generateWinDiskEncryptionProfile } from "../../helpers"; -export interface ITableRowOsSettings extends Omit { - status: OsSettingsTableStatusValue; -} - -export type OsSettingsTableStatusValue = MdmProfileStatus | "action_required"; - export const isMdmProfileStatus = ( status: string ): status is MdmProfileStatus => { return status !== "action_required"; }; -interface IHeaderProps { - column: { - title: string; - isSortedDesc: boolean; - }; +export interface IHostMdmProfileWithAddedStatus + extends Omit { + status: OsSettingsTableStatusValue; } -interface ICellProps { - cell: { - value: string; - }; - row: { - original: ITableRowOsSettings; - }; -} +type ITableColumnConfig = Column; +type ITableStringCellProps = IStringCellProps; -interface IDataColumn { - Header: ((props: IHeaderProps) => JSX.Element) | string; - Cell: (props: ICellProps) => JSX.Element; - id?: string; - title?: string; - accessor?: string; - disableHidden?: boolean; - disableSortBy?: boolean; - sortType?: string; -} +export type INonDDMProfileStatus = MdmProfileStatus | "action_required"; + +export type OsSettingsTableStatusValue = + | MdmDDMProfileStatus + | INonDDMProfileStatus; /** * generates the formatted tooltip for the error column. @@ -107,22 +93,20 @@ const generateErrorTooltip = ( return generateFormattedTooltip(detail); }; -const tableHeaders: IDataColumn[] = [ +const tableHeaders: ITableColumnConfig[] = [ { - title: "Name", Header: "Name", disableSortBy: true, accessor: "name", - Cell: (cellProps: ICellProps): JSX.Element => ( - - ), + Cell: (cellProps: ITableStringCellProps) => { + return ; + }, }, { - title: "Status", Header: "Status", disableSortBy: true, - accessor: "statusText", - Cell: (cellProps: ICellProps) => { + accessor: "status", + Cell: (cellProps: ITableStringCellProps) => { return ( { + Cell: (cellProps: ITableStringCellProps): JSX.Element => { const profile = cellProps.row.original; const value = @@ -165,7 +148,7 @@ const tableHeaders: IDataColumn[] = [ ]; const makeWindowsRows = ({ profiles, os_settings }: IHostMdmData) => { - const rows: ITableRowOsSettings[] = []; + const rows: IHostMdmProfileWithAddedStatus[] = []; if (profiles) { rows.push(...profiles); @@ -190,15 +173,12 @@ const makeWindowsRows = ({ profiles, os_settings }: IHostMdmData) => { return rows; }; -const makeDarwinRows = ({ - profiles, - macos_settings, -}: IHostMdmData): ITableRowOsSettings[] | null => { +const makeDarwinRows = ({ profiles, macos_settings }: IHostMdmData) => { if (!profiles) { return null; } - let rows: ITableRowOsSettings[] = profiles; + let rows: IHostMdmProfileWithAddedStatus[] = profiles; if (macos_settings?.disk_encryption === "action_required") { rows = profiles.map((p) => { // TODO: this is a brittle check for the filevault profile diff --git a/frontend/pages/hosts/details/cards/HostSummary/OSSettingsIndicator/OSSettingsIndicator.tsx b/frontend/pages/hosts/details/cards/HostSummary/OSSettingsIndicator/OSSettingsIndicator.tsx index 6f90c372ab..68043e52a7 100644 --- a/frontend/pages/hosts/details/cards/HostSummary/OSSettingsIndicator/OSSettingsIndicator.tsx +++ b/frontend/pages/hosts/details/cards/HostSummary/OSSettingsIndicator/OSSettingsIndicator.tsx @@ -32,7 +32,8 @@ const STATUS_DISPLAY_OPTIONS: StatusDisplayOptions = { Verified: { iconName: "success", tooltipText: - "The host applied all OS settings. Fleet verified with osquery.", + "The host applied all OS settings. Fleet verified with osquery. " + + "Declaration profiles are verified with DDM.", }, Verifying: { iconName: "success-outline", diff --git a/frontend/services/entities/mdm.ts b/frontend/services/entities/mdm.ts index 29a8e5fab9..6f9a2503b8 100644 --- a/frontend/services/entities/mdm.ts +++ b/frontend/services/entities/mdm.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { DiskEncryptionStatus, + IHostMdmProfile, IMdmProfile, MdmProfileStatus, } from "interfaces/mdm"; @@ -46,6 +47,10 @@ export interface IUploadProfileApiParams { labels?: string[]; } +export const isDDMProfile = (profile: IMdmProfile | IHostMdmProfile) => { + return profile.profile_uuid.startsWith("d"); +}; + const mdmService = { downloadDeviceUserEnrollmentProfile: (token: string) => { const { DEVICE_USER_MDM_ENROLLMENT_PROFILE } = endpoints; From 95df7e2b0ba3b61f157d8bb00c54eeb00b8978da Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 25 Mar 2024 17:32:27 -0300 Subject: [PATCH 14/29] implement DDM cron and protocol bits (#17791) for #17399 --- cmd/fleet/cron.go | 3 + server/datastore/mysql/apple_mdm.go | 373 +++++++++++------- server/datastore/mysql/apple_mdm_test.go | 2 - server/datastore/mysql/mdm.go | 3 +- .../tables/20240314150853_AddDDMTables.go | 25 +- server/datastore/mysql/schema.sql | 13 +- server/fleet/apple_mdm.go | 34 +- server/fleet/datastore.go | 11 +- server/mock/datastore_mock.go | 30 +- server/service/apple_mdm.go | 112 +++++- server/service/integration_mdm_test.go | 291 ++++++++++++-- 11 files changed, 628 insertions(+), 269 deletions(-) diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index ad20a3278d..42fca14d17 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1033,6 +1033,9 @@ func newMDMProfileManager( schedule.WithJob("manage_apple_profiles", func(ctx context.Context) error { return service.ReconcileAppleProfiles(ctx, ds, commander, logger) }), + schedule.WithJob("manage_apple_declarations", func(ctx context.Context) error { + return service.ReconcileAppleDeclarations(ctx, ds, commander, logger) + }), schedule.WithJob("manage_windows_profiles", func(ctx context.Context) error { return service.ReconcileWindowsProfiles(ctx, ds, logger) }), diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index ac0801aad8..10d1b27faf 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -16,8 +16,8 @@ import ( "github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/google/uuid" "github.com/jmoiron/sqlx" ) @@ -240,7 +240,7 @@ SELECT FROM mdm_apple_declarations WHERE - declaration_uuid = ? AND category = 'com.apple.configuration'` + declaration_uuid = ?` var res fleet.MDMAppleDeclaration err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, declUUID) @@ -1490,6 +1490,8 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( return nil } + appleMDMProfilesDesiredStateQuery := generateDesiredStateQuery("profile") + // TODO(mna): the conditions here (and in toRemoveStmt) are subtly different // than the ones in ListMDMAppleProfilesToInstall/Remove, so I'm keeping // those statements distinct to avoid introducing a subtle bug, but we should @@ -1720,20 +1722,30 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( return nil } -const appleMDMProfilesDesiredStateQuery = ` - -- non label-based profiles +// mdmEntityTypeToTable tracks what table should be used in the templates for +// SQL statements based on the given entity type. +var mdmEntityTypeToTable = map[string]string{ + "declaration": "declaration", + "profile": "configuration_profile", +} + +// generateDesiredStateQuery generates a query string that represents the +// desired state of an Apple entity based on its type (profile or declaration) +func generateDesiredStateQuery(entityType string) string { + return fmt.Sprintf(` + -- non label-based entities SELECT - macp.profile_uuid, + mae.%[1]s_uuid, h.uuid as host_uuid, - macp.identifier as profile_identifier, - macp.name as profile_name, - macp.checksum as checksum, - 0 as count_profile_labels, + mae.identifier as %[1]s_identifier, + mae.name as %[1]s_name, + mae.checksum as checksum, + 0 as count_%[1]s_labels, 0 as count_host_labels FROM - mdm_apple_configuration_profiles macp + mdm_apple_%[2]ss mae JOIN hosts h - ON h.team_id = macp.team_id OR (h.team_id IS NULL AND macp.team_id = 0) + ON h.team_id = mae.team_id OR (h.team_id IS NULL AND mae.team_id = 0) JOIN nano_enrollments ne ON ne.device_id = h.uuid WHERE @@ -1742,81 +1754,146 @@ const appleMDMProfilesDesiredStateQuery = ` ne.type = 'Device' AND NOT EXISTS ( SELECT 1 - FROM mdm_configuration_profile_labels mcpl - WHERE mcpl.apple_profile_uuid = macp.profile_uuid + FROM mdm_%[2]s_labels mel + WHERE mel.apple_%[1]s_uuid = mae.%[1]s_uuid ) AND - ( %s ) + ( %[3]s ) UNION - -- label-based profiles where the host is a member of all the labels + -- label-based entities where the host is a member of all the labels SELECT - macp.profile_uuid, + mae.%[1]s_uuid, h.uuid as host_uuid, - macp.identifier as profile_identifier, - macp.name as profile_name, - macp.checksum as checksum, - COUNT(*) as count_profile_labels, + mae.identifier as %[1]s_identifier, + mae.name as %[1]s_name, + mae.checksum as checksum, + COUNT(*) as count_%[1]s_labels, COUNT(lm.label_id) as count_host_labels FROM - mdm_apple_configuration_profiles macp + mdm_apple_%[2]ss mae JOIN hosts h - ON h.team_id = macp.team_id OR (h.team_id IS NULL AND macp.team_id = 0) + ON h.team_id = mae.team_id OR (h.team_id IS NULL AND mae.team_id = 0) JOIN nano_enrollments ne ON ne.device_id = h.uuid - JOIN mdm_configuration_profile_labels mcpl - ON mcpl.apple_profile_uuid = macp.profile_uuid + JOIN mdm_%[2]s_labels mel + ON mel.apple_%[1]s_uuid = mae.%[1]s_uuid LEFT OUTER JOIN label_membership lm - ON lm.label_id = mcpl.label_id AND lm.host_id = h.id + ON lm.label_id = mel.label_id AND lm.host_id = h.id WHERE h.platform = 'darwin' AND ne.enabled = 1 AND ne.type = 'Device' AND - ( %s ) + ( %[3]s ) GROUP BY - macp.profile_uuid, h.uuid, macp.identifier, macp.name, macp.checksum + mae.%[1]s_uuid, h.uuid, mae.identifier, mae.name, mae.checksum HAVING - count_profile_labels > 0 AND count_host_labels = count_profile_labels -` + count_%[1]s_labels > 0 AND count_host_labels = count_%[1]s_labels + + `, entityType, mdmEntityTypeToTable[entityType], "%s") +} + +// generateEntitiesToInstallQuery is a set difference between: +// +// - Set A (ds), the "desired state", can be obtained from a JOIN between +// mdm_apple_x and hosts. +// +// - Set B, the "current state" given by host_mdm_apple_x. +// +// A - B gives us the entities that need to be installed: +// +// - entities that are in A but not in B +// +// - entities which contents have changed, but their identifier are +// the same (by checking the checksums) +// +// - entities that are in A and in B, but with an operation type of +// "remove", regardless of the status. (technically, if status is NULL then +// the entity should be already installed - it has not been queued for +// remove yet -, and same if status is failed, but the proper thing to do +// with it would be to remove the row, not return it as "to install". For +// simplicity of implementation here (and to err on the safer side - the +// entity's content could've changed), we'll return it as "to install" for +// now, which will cause the row to be updated with the correct operation +// type and status). +// +// - entities that are in A and in B, with an operation type of "install" +// and a NULL status. Other statuses mean that the operation is already in +// flight (pending), the operation has been completed but is still subject +// to independent verification by Fleet (verifying), or has reached a terminal +// state (failed or verified). If the entity's content is edited, all +// relevant hosts will be marked as status NULL so that it gets +// re-installed. +// +// Note that for label-based entities, only fully-satisfied entities are +// considered for installation. This means that a broken label-based entity, +// where one of the labels does not exist anymore, will not be considered for +// installation. +func generateEntitiesToInstallQuery(entityType string) string { + return fmt.Sprintf(` + ( %[3]s ) as ds + LEFT JOIN host_mdm_apple_%[1]ss hmae + ON hmae.%[1]s_uuid = ds.%[1]s_uuid AND hmae.host_uuid = ds.host_uuid + WHERE + -- entity has been updated + ( hmae.checksum != ds.checksum ) OR + -- entity in A but not in B + ( hmae.%[1]s_uuid IS NULL AND hmae.host_uuid IS NULL ) OR + -- entities in A and B but with operation type "remove" + ( hmae.host_uuid IS NOT NULL AND ( hmae.operation_type = ? OR hmae.operation_type IS NULL ) ) OR + -- entities in A and B with operation type "install" and NULL status + ( hmae.host_uuid IS NOT NULL AND hmae.operation_type = ? AND hmae.status IS NULL ) +`, entityType, mdmEntityTypeToTable[entityType], fmt.Sprintf(generateDesiredStateQuery(entityType), "TRUE", "TRUE")) +} + +// generateEntitiesToRemoveQuery is a set difference between: +// +// - Set A (ds), the "desired state", can be obtained from a JOIN between +// mdm_apple_configuration_x and hosts. +// +// - Set B, the "current state" given by host_mdm_apple_x. +// +// B - A gives us the entities that need to be removed: +// +// - entities that are in B but not in A, except those with operation type +// "remove" and a terminal state (failed) or a state indicating +// that the operation is in flight (pending) or the operation has been completed +// but is still subject to independent verification by Fleet (verifying) +// or the operation has been completed and independenly verified by Fleet (verified). +// +// Any other case are entities that are in both B and A, and as such are +// processed by the generateEntitiesToInstallQuery query (since they are in +// both, their desired state is necessarily to be installed). +// +// Note that for label-based entities, only those that are fully-sastisfied +// by the host are considered for install (are part of the desired state used +// to compute the ones to remove). However, as a special case, a broken +// label-based entity will NOT be removed from a host where it was +// previously installed. However, if a host used to satisfy a label-based +// entity but no longer does (and that label-based entity is not "broken"), +// the entity will be removed from the host. +func generateEntitiesToRemoveQuery(entityType string) string { + return fmt.Sprintf(` + ( %[3]s ) as ds + RIGHT JOIN host_mdm_apple_%[1]ss hmae + ON hmae.%[1]s_uuid = ds.%[1]s_uuid AND hmae.host_uuid = ds.host_uuid + WHERE + -- entities that are in B but not in A + ds.%[1]s_uuid IS NULL AND ds.host_uuid IS NULL AND + -- except "remove" operations in a terminal state or already pending + ( hmae.operation_type IS NULL OR hmae.operation_type != ? OR hmae.status IS NULL ) AND + -- except "would be removed" entities if they are a broken label-based entities + NOT EXISTS ( + SELECT 1 + FROM mdm_%[2]s_labels mcpl + WHERE + mcpl.apple_%[1]s_uuid = hmae.%[1]s_uuid AND + mcpl.label_id IS NULL + ) +`, entityType, mdmEntityTypeToTable[entityType], fmt.Sprintf(generateDesiredStateQuery(entityType), "TRUE", "TRUE")) +} func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) { - // The query below is a set difference between: - // - // - Set A (ds), the "desired state", can be obtained from a JOIN between - // mdm_apple_configuration_profiles and hosts. - // - // - Set B, the "current state" given by host_mdm_apple_profiles. - // - // A - B gives us the profiles that need to be installed: - // - // - profiles that are in A but not in B - // - // - profiles which contents have changed, but their identifier are - // the same (by checking the checksums) - // - // - profiles that are in A and in B, but with an operation type of - // "remove", regardless of the status. (technically, if status is NULL then - // the profile should be already installed - it has not been queued for - // remove yet -, and same if status is failed, but the proper thing to do - // with it would be to remove the row, not return it as "to install". For - // simplicity of implementation here (and to err on the safer side - the - // profile's content could've changed), we'll return it as "to install" for - // now, which will cause the row to be updated with the correct operation - // type and status). - // - // - profiles that are in A and in B, with an operation type of "install" - // and a NULL status. Other statuses mean that the operation is already in - // flight (pending), the operation has been completed but is still subject - // to independent verification by Fleet (verifying), or has reached a terminal - // state (failed or verified). If the profile's content is edited, all - // relevant hosts will be marked as status NULL so that it gets - // re-installed. - // - // Note that for label-based profiles, only fully-satisfied profiles are - // considered for installation. This means that a broken label-based profile, - // where one of the labels does not exist anymore, will not be considered for - // installation. - query := fmt.Sprintf(` SELECT ds.profile_uuid, @@ -1824,82 +1901,26 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee ds.profile_identifier, ds.profile_name, ds.checksum - FROM ( %s ) as ds - LEFT JOIN host_mdm_apple_profiles hmap - ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid - WHERE - -- profile has been updated - ( hmap.checksum != ds.checksum ) OR - -- profiles in A but not in B - ( hmap.profile_uuid IS NULL AND hmap.host_uuid IS NULL ) OR - -- profiles in A and B but with operation type "remove" - ( hmap.host_uuid IS NOT NULL AND ( hmap.operation_type = ? OR hmap.operation_type IS NULL ) ) OR - -- profiles in A and B with operation type "install" and NULL status - ( hmap.host_uuid IS NOT NULL AND hmap.operation_type = ? AND hmap.status IS NULL ) -`, fmt.Sprintf(appleMDMProfilesDesiredStateQuery, "TRUE", "TRUE")) - + FROM %s `, + generateEntitiesToInstallQuery("profile")) var profiles []*fleet.MDMAppleProfilePayload err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, query, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall) return profiles, err } func (ds *Datastore) ListMDMAppleProfilesToRemove(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) { - // The query below is a set difference between: - // - // - Set A (ds), the "desired state", can be obtained from a JOIN between - // mdm_apple_configuration_profiles and hosts. - // - // - Set B, the "current state" given by host_mdm_apple_profiles. - // - // B - A gives us the profiles that need to be removed: - // - // - profiles that are in B but not in A, except those with operation type - // "remove" and a terminal state (failed) or a state indicating - // that the operation is in flight (pending) or the operation has been completed - // but is still subject to independent verification by Fleet (verifying) - // or the operation has been completed and independenly verified by Fleet (verified). - // - // Any other case are profiles that are in both B and A, and as such are - // processed by the ListMDMAppleProfilesToInstall method (since they are in - // both, their desired state is necessarily to be installed). - // - // Note that for label-based profiles, only those that are fully-sastisfied - // by the host are considered for install (are part of the desired state used - // to compute the ones to remove). However, as a special case, a broken - // label-based profile will NOT be removed from a host where it was - // previously installed. However, if a host used to satisfy a label-based - // profile but no longer does (and that label-based profile is not "broken"), - // the profile will be removed from the host. - query := fmt.Sprintf(` SELECT - hmap.profile_uuid, - hmap.profile_identifier, - hmap.profile_name, - hmap.host_uuid, - hmap.checksum, - hmap.operation_type, - COALESCE(hmap.detail, '') as detail, - hmap.status, - hmap.command_uuid - FROM ( %s ) as ds - RIGHT JOIN host_mdm_apple_profiles hmap - ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid - WHERE - -- profiles that are in B but not in A - ds.profile_uuid IS NULL AND ds.host_uuid IS NULL AND - -- except "remove" operations in a terminal state or already pending - ( hmap.operation_type IS NULL OR hmap.operation_type != ? OR hmap.status IS NULL ) AND - -- except "would be removed" profiles if they are a broken label-based profile - NOT EXISTS ( - SELECT 1 - FROM mdm_configuration_profile_labels mcpl - WHERE - mcpl.apple_profile_uuid = hmap.profile_uuid AND - mcpl.label_id IS NULL - ) -`, fmt.Sprintf(appleMDMProfilesDesiredStateQuery, "TRUE", "TRUE")) - + hmae.profile_uuid, + hmae.profile_identifier, + hmae.profile_name, + hmae.host_uuid, + hmae.checksum, + hmae.operation_type, + COALESCE(hmae.detail, '') as detail, + hmae.status, + hmae.command_uuid + FROM %s`, generateEntitiesToRemoveQuery("profile")) var profiles []*fleet.MDMAppleProfilePayload err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, query, fleet.MDMOperationTypeRemove) return profiles, err @@ -3166,14 +3187,13 @@ INSERT INTO mdm_apple_declarations ( declaration_uuid, identifier, name, - category, raw_json, checksum, uploaded_at, team_id ) VALUES ( - ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? + ?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? ) ON DUPLICATE KEY UPDATE uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), @@ -3283,7 +3303,6 @@ WHERE declUUID, d.Identifier, d.Name, - d.Category, d.RawJSON, checksum, declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") { @@ -3320,11 +3339,10 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - category, raw_json, checksum, uploaded_at) -(SELECT ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE +(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? ) AND NOT EXISTS ( @@ -3339,7 +3357,7 @@ INSERT INTO mdm_apple_declarations ( err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, - declUUID, tmID, declaration.Identifier, declaration.Name, declaration.Category, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { case isDuplicate(err): @@ -3473,8 +3491,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID s const stmt = ` SELECT HEX(mad.checksum) as checksum, - mad.identifier, - mad.category + mad.identifier FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON mad.declaration_uuid = hmad.declaration_uuid @@ -3489,7 +3506,7 @@ WHERE return res, nil } -func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { +func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { // TODO: When hosts table is indexed by uuid, consider joining on hosts to ensure that the // declaration for the host's current team is returned. In the case where the specified // identifier is not unique to the team, the cron should ensure that any conflicting @@ -3501,15 +3518,77 @@ FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid WHERE - host_uuid = ? AND identifier = ? AND category = ? AND operation_type = ?` + host_uuid = ? AND identifier = ? AND operation_type = ?` var res fleet.MDMAppleDeclaration - if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, declarationType, fleet.MDMOperationTypeInstall); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, fleet.MDMOperationTypeInstall); err != nil { if err == sql.ErrNoRows { - return nil, notFound(string(declarationType)).WithName(identifier) + return nil, notFound("MDMAppleDeclaration").WithName(identifier) } return nil, ctxerr.Wrap(ctx, err, "get ddm declarations response") } return &res, nil } + +func (ds *Datastore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { + baseStmt := ` + INSERT INTO host_mdm_apple_declarations + (host_uuid, status, operation_type, checksum, declaration_uuid, declaration_identifier, declaration_name) + VALUES + %s + ON DUPLICATE KEY UPDATE + status = VALUES(status), + operation_type = VALUES(operation_type), + checksum = VALUES(checksum) + ` + var placeholders strings.Builder + var args []any + for _, d := range changedDeclarations { + placeholders.WriteString("(?, 'pending', ?, ?, ?, ?, ?),") + args = append(args, d.HostUUID, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name) + } + _, err := ds.writer(ctx).ExecContext( + ctx, + fmt.Sprintf(baseStmt, strings.TrimSuffix(placeholders.String(), ",")), + args..., + ) + return ctxerr.Wrap(ctx, err, "inserting changed host declaration state") +} + +func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { + stmt := fmt.Sprintf(` + ( + SELECT + ds.host_uuid, + 'install' as operation_type, + ds.checksum, + ds.declaration_uuid, + ds.declaration_identifier as identifier, + ds.declaration_name as name + FROM + %s + ) + UNION ALL + ( + SELECT + hmae.host_uuid, + 'remove' as operation_type, + hmae.checksum, + hmae.declaration_uuid, + hmae.declaration_identifier as identifier, + hmae.declaration_name as name + FROM + %s + ) + `, + generateEntitiesToInstallQuery("declaration"), + generateEntitiesToRemoveQuery("declaration"), + ) + + var decls []*fleet.MDMAppleHostDeclaration + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &decls, stmt, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove); err != nil { + return nil, ctxerr.Wrap(ctx, err, "running sql statement") + } + return decls, nil +} diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 1bfe5f6574..6eed346a69 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1062,7 +1062,6 @@ func expectAppleDeclarations( require.Equal(t, wantD.Name, gotD.Name) require.Equal(t, wantD.Identifier, gotD.Identifier) require.Equal(t, wantD.Labels, gotD.Labels) - require.Equal(t, wantD.Category, gotD.Category) } return m } @@ -1258,7 +1257,6 @@ func declForTest(name, identifier, payloadContent string, labels ...*fleet.Label decl := &fleet.MDMAppleDeclaration{ RawJSON: declBytes, - Category: fleet.MDMAppleDeclarativeConfiguration, Identifier: fmt.Sprintf("com.fleet.config%s", identifier), Name: name, } diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index 21ff675ed1..6316bddd31 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -182,7 +182,6 @@ FROM ( uploaded_at FROM mdm_apple_declarations WHERE team_id = ? - AND category <> ? ) as combined_profiles ` @@ -202,7 +201,7 @@ FROM ( fleetNames = append(fleetNames, k) } - args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames, globalOrTeamID, fleet.MDMAppleDeclarativeActivation} + args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames, globalOrTeamID} stmt, args := appendListOptionsWithCursorToSQL(selectStmt, args, &opt) stmt, args, err := sqlx.In(stmt, args...) diff --git a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go index b220e91b33..29f947a9fa 100644 --- a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go +++ b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go @@ -11,24 +11,6 @@ func init() { func Up_20240314150853(tx *sql.Tx) error { _, err := tx.Exec(` -CREATE TABLE mdm_apple_declaration_categories ( - declaration_category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (declaration_category) -) - `) - if err != nil { - return fmt.Errorf("creating mdm_apple_declaration_categories table: %w", err) - } - - _, err = tx.Exec(` - INSERT INTO mdm_apple_declaration_categories - VALUES ('com.apple.configuration'), ('com.apple.activation') - `) - if err != nil { - return fmt.Errorf("inserting default values into mdm_apple_declaration_categories table: %w", err) - } - - _, err = tx.Exec(` CREATE TABLE mdm_apple_declarations ( -- declaration_uuid is used as the primary key of the declaration declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', @@ -42,10 +24,6 @@ CREATE TABLE mdm_apple_declarations ( -- name is the name of the declaration name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - -- category is the category of the declaration (activation, - -- declaration, management or asset) - category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - -- raw_json contains a JSON blob with the declaration contents raw_json json NOT NULL, @@ -57,8 +35,7 @@ CREATE TABLE mdm_apple_declarations ( PRIMARY KEY (declaration_uuid), UNIQUE KEY idx_mdm_apple_declaration_team_identifier (team_id, identifier), - UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name), - CONSTRAINT mdm_apple_declaration_category FOREIGN KEY (category) REFERENCES mdm_apple_declaration_categories (declaration_category) ON DELETE CASCADE + UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name) ) `) if err != nil { diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 3882a3061e..99b8644db5 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -660,29 +660,18 @@ CREATE TABLE `mdm_apple_declaration_activation_references` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; -CREATE TABLE `mdm_apple_declaration_categories` ( - `declaration_category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (`declaration_category`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -/*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `mdm_apple_declaration_categories` VALUES ('com.apple.activation'),('com.apple.configuration'); -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_apple_declarations` ( `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `team_id` int(10) unsigned NOT NULL DEFAULT '0', `identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - `category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, `raw_json` json NOT NULL, `checksum` binary(16) NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `uploaded_at` timestamp NULL DEFAULT NULL, PRIMARY KEY (`declaration_uuid`), UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`), - UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`), - KEY `mdm_apple_declaration_category` (`category`), - CONSTRAINT `mdm_apple_declaration_category` FOREIGN KEY (`category`) REFERENCES `mdm_apple_declaration_categories` (`declaration_category`) ON DELETE CASCADE + UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 78b6cda55d..093656e58a 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -450,6 +450,8 @@ const ( DEPAssignProfileResponseFailed DEPAssignProfileResponseStatus = "FAILED" ) +const MDMAppleDeclarationUUIDPrefix = "d" + // NanoEnrollment represents a row in the nano_enrollments table managed by // nanomdm. It is meant to be used internally by the server, not to be returned // as part of endpoints, and as a precaution its json-encoding is explicitly @@ -533,25 +535,6 @@ type SCEPIdentityAssociation struct { RenewCommandUUID string `db:"renew_command_uuid"` } -// MDMAppleDeclarationCategory is the type for the supported declaration types. -type MDMAppleDeclarationCategory string - -const ( - // MDMAppleConfigurationDeclaration is the value for [configuration][1] declarations - // - // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3813088 - MDMAppleDeclarativeConfiguration MDMAppleDeclarationCategory = "com.apple.configuration" - - // MDMAppleActivationConfiguration is the value for [activation][1] declarations - // - // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3829708 - MDMAppleDeclarativeActivation MDMAppleDeclarationCategory = "com.apple.activation" - - // MDMAppleDeclarationUUIDPrefix is the prefix used to differentiate declaration uuids - // from legacy Apple profile uuids and Windows profile uuids. - MDMAppleDeclarationUUIDPrefix = "d" -) - // MDMAppleDeclaration represents a DDM JSON declaration. type MDMAppleDeclaration struct { // DeclarationUUID is the unique identifier of the declaration in @@ -572,10 +555,6 @@ type MDMAppleDeclaration struct { // Fleet requires that Name must be unique in combination with the Identifier and TeamID. Name string `db:"name" json:"name"` - // Category is the category of the declaration, at the moment we - // only support configurations and activations. - Category MDMAppleDeclarationCategory `db:"category"` - // RawJSON is the raw JSON content of the declaration RawJSON json.RawMessage `db:"raw_json" json:"-"` @@ -604,7 +583,6 @@ var ForbiddenDeclTypes = map[string]struct{}{ "com.apple.configuration.account.google": {}, "com.apple.configuration.account.ldap": {}, "com.apple.configuration.account.mail": {}, - "com.apple.configuration.management.test": {}, "com.apple.configuration.screensharing.connection": {}, "com.apple.configuration.security.certificate": {}, "com.apple.configuration.security.identity": {}, @@ -629,7 +607,7 @@ func (r *MDMAppleRawDeclaration) ValidateUserProvided() error { return NewInvalidArgumentError(r.Type, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") } - if !strings.HasPrefix(r.Type, string(MDMAppleDeclarativeConfiguration)) { + if !strings.HasPrefix(r.Type, "com.apple.configuration") { return NewInvalidArgumentError(r.Type, "Only configuration declarations (com.apple.configuration) are supported.") } @@ -671,12 +649,15 @@ type MDMAppleHostDeclaration struct { // Detail contains any messages that must be surfaced to the user, // either by the MDM protocol or the Fleet server. Detail string `db:"detail" json:"detail"` + + // Checksum contains the MD5 checksum of the declaration JSON uploaded + // by the IT admin. Fleet uses this value as the ServerToken. + Checksum string `db:"checksum" json:"-"` } func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { var decl MDMAppleDeclaration - decl.Category = MDMAppleDeclarationCategory(strings.Join(strings.Split(declType, ".")[:3], ".")) decl.Identifier = ident decl.Name = name decl.RawJSON = raw @@ -733,7 +714,6 @@ type MDMAppleDDMManifest struct { // https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse type MDMAppleDDMDeclarationItem struct { Identifier string `db:"identifier"` - Category string `db:"category"` ServerToken string `db:"checksum"` } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 5b7cac96c8..e19f7e0b9e 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1161,7 +1161,16 @@ type Datastore interface { // MDMAppleDDMDeclarationItems returns the declaration items for the specified host UUID. MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]MDMAppleDDMDeclarationItem, error) // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. - MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType MDMAppleDeclarationCategory, identifier string, hostUUID string) (*MDMAppleDeclaration, error) + MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*MDMAppleDeclaration, error) + // MDMAppleGetHostsWithChangedDeclarations returns a + // MDMAppleHostDeclaration item for each (host x declaration) pair that + // needs an status change, this includes declarations to install and + // declarations to be removed. Those can be differentiated by the + // OperationType field on each struct. + MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*MDMAppleHostDeclaration, error) + // MDMAppleBatchInsertHostDeclarations tracks the current status of all + // the host declarations provided. + MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*MDMAppleHostDeclaration) error /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index fdc59496dd..7ab7c83a00 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -762,7 +762,11 @@ type MDMAppleDDMDeclarationsTokenFunc func(ctx context.Context, hostUUID string) type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) -type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) +type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) + +type MDMAppleGetHostsWithChangedDeclarationsFunc func(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) + +type MDMAppleBatchInsertHostDeclarationsFunc func(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error @@ -1988,6 +1992,12 @@ type DataStore struct { MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFuncInvoked bool + MDMAppleGetHostsWithChangedDeclarationsFunc MDMAppleGetHostsWithChangedDeclarationsFunc + MDMAppleGetHostsWithChangedDeclarationsFuncInvoked bool + + MDMAppleBatchInsertHostDeclarationsFunc MDMAppleBatchInsertHostDeclarationsFunc + MDMAppleBatchInsertHostDeclarationsFuncInvoked bool + WSTEPStoreCertificateFunc WSTEPStoreCertificateFunc WSTEPStoreCertificateFuncInvoked bool @@ -4751,11 +4761,25 @@ func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID st return s.MDMAppleDDMDeclarationItemsFunc(ctx, hostUUID) } -func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { +func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { s.mu.Lock() s.MDMAppleDDMDeclarationsResponseFuncInvoked = true s.mu.Unlock() - return s.MDMAppleDDMDeclarationsResponseFunc(ctx, declarationType, identifier, hostUUID) + return s.MDMAppleDDMDeclarationsResponseFunc(ctx, identifier, hostUUID) +} + +func (s *DataStore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { + s.mu.Lock() + s.MDMAppleGetHostsWithChangedDeclarationsFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleGetHostsWithChangedDeclarationsFunc(ctx) +} + +func (s *DataStore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { + s.mu.Lock() + s.MDMAppleBatchInsertHostDeclarationsFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleBatchInsertHostDeclarationsFunc(ctx, changedDeclarations) } func (s *DataStore) WSTEPStoreCertificate(ctx context.Context, name string, crt *x509.Certificate) error { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index b67796b33e..e64311c957 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2710,6 +2710,55 @@ func ensureFleetdConfig(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } +func ReconcileAppleDeclarations( + ctx context.Context, + ds fleet.Datastore, + commander *apple_mdm.MDMAppleCommander, + logger kitlog.Logger, +) error { + // once all the declarations are in place, compute the desired state + // and find which hosts need a DDM sync. + changedDeclarations, err := ds.MDMAppleGetHostsWithChangedDeclarations(ctx) + if err != nil { + return ctxerr.Wrap(ctx, err, "find hosts with changed declarations") + } + + if len(changedDeclarations) == 0 { + logger.Log("msg", "no hosts with changed declarations") + return nil + } + + // a host might have more than one declaration to sync, we do this to + // collect unique host UUIDs in order to send a single command to each + // host in the next step + uuidMap := map[string]struct{}{} + for _, d := range changedDeclarations { + uuidMap[d.HostUUID] = struct{}{} + } + uuids := make([]string, 0, len(uuidMap)) + for uuid := range uuidMap { + uuids = append(uuids, uuid) + } + + // mark the host declarations as pending, this serves two purposes: + // + // - support the APIs/methods that track host status (summaries, filters, etc) + // + // - support the DDM endpoints, which use data from the + // `host_mdm_apple_declarations` table to compute which declarations to + // serve + if err := ds.MDMAppleBatchInsertHostDeclarations(ctx, changedDeclarations); err != nil { + return ctxerr.Wrap(ctx, err, "batch insert mdm apple host declarations") + } + + // send a DeclarativeManagement command to start a sync + if err := commander.DeclarativeManagement(ctx, uuids, uuid.NewString()); err != nil { + return ctxerr.Wrap(ctx, err, "issuing DeclarativeManagement command") + } + + return nil +} + func ReconcileAppleProfiles( ctx context.Context, ds fleet.Datastore, @@ -3233,16 +3282,11 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, hostU activations := []fleet.MDMAppleDDMManifest{} configurations := []fleet.MDMAppleDDMManifest{} for _, d := range di { - manifest := fleet.MDMAppleDDMManifest{Identifier: d.Identifier, ServerToken: d.ServerToken} - switch d.Category { - case string(fleet.MDMAppleDeclarativeActivation): - activations = append(activations, manifest) - case string(fleet.MDMAppleDeclarativeConfiguration): - configurations = append(configurations, manifest) - default: - level.Debug(svc.logger).Log("msg", "unrecognized declaration category", "category", d.Category) - return nil, ctxerr.New(ctx, "unrecognized declaration category") - } + configurations = append(configurations, fleet.MDMAppleDDMManifest(d)) + activations = append(activations, fleet.MDMAppleDDMManifest{ + Identifier: fmt.Sprintf("%s.activation", d.Identifier), + ServerToken: d.ServerToken, + }) } // TODO: Look for ways to optimize the declaration item query so that we don't have to get the declarations token separately. @@ -3274,11 +3318,43 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e } level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", parts[1], "identifier", parts[2]) - d, err := svc.ds.MDMAppleDDMDeclarationsResponse( - ctx, fleet.MDMAppleDeclarationCategory("com.apple."+parts[1]), - parts[2], - hostUUID, - ) + switch parts[1] { + case "activation": + return svc.handleActivationDeclaration(ctx, parts, hostUUID) + case "configuration": + return svc.handleConfigurationDeclaration(ctx, parts, hostUUID) + default: + return nil, newNotFoundError() + } +} + +func (svc *MDMAppleDDMService) handleActivationDeclaration(ctx context.Context, parts []string, hostUUID string) ([]byte, error) { + references := strings.TrimSuffix(parts[2], ".activation") + + // ensure the declaration for the requested activation stil exists + d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, references, hostUUID) + if err != nil { + if fleet.IsNotFound(err) { + return nil, nano_service.NewHTTPStatusError(http.StatusNotFound, err) + } + return nil, ctxerr.Wrap(ctx, err, "getting linked configuration for activation declaration") + } + + response := fmt.Sprintf(` +{ + "Identifier": "%s", + "Payload": { + "StandardConfigurations": ["%s"] + }, + "ServerToken": "%s", + "Type": "com.apple.activation.simple" +}`, parts[2], references, d.Checksum) + + return []byte(response), nil +} + +func (svc *MDMAppleDDMService) handleConfigurationDeclaration(ctx context.Context, parts []string, hostUUID string) ([]byte, error) { + d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, parts[2], hostUUID) if err != nil { if fleet.IsNotFound(err) { return nil, nano_service.NewHTTPStatusError(http.StatusNotFound, err) @@ -3286,12 +3362,6 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e return nil, ctxerr.Wrap(ctx, err, "getting declaration response") } - // unmarshall into a temporary map in order to add the token. - // we do this at this stage because tokens are purely managed by Fleet, - // and we don't want to store a modified version of what's provided by - // the IT admin. - // - // This mimics what we do for CommandUUID, but can be revisited. var tempd map[string]any if err := json.Unmarshal(d.RawJSON, &tempd); err != nil { return nil, ctxerr.Wrap(ctx, err, "unmarshaling stored declaration") diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index fdf4f12af5..62039380ad 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -2755,9 +2755,14 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { mdmDeviceA := mdmtest.NewTestMDMClientAppleDirect(mdmEnrollInfo) err := mdmDeviceA.Enroll() require.NoError(t, err) + s.lastActivityOfTypeMatches(fleet.ActivityTypeMDMEnrolled{}.ActivityName(), + fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), 0) + mdmDeviceB := mdmtest.NewTestMDMClientAppleDirect(mdmEnrollInfo) err = mdmDeviceB.Enroll() require.NoError(t, err) + s.lastActivityOfTypeMatches(fleet.ActivityTypeMDMEnrolled{}.ActivityName(), + fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceB.SerialNumber, mdmDeviceB.Model, mdmDeviceB.SerialNumber), 0) // Find the ID of Fleet's MDM solution var mdmID uint @@ -2786,23 +2791,6 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { } } - // Activities are generated for each device - activities := listActivitiesResponse{} - s.DoJSON("GET", "/api/latest/fleet/activities", nil, http.StatusOK, &activities, "order_key", "created_at") - require.GreaterOrEqual(t, len(activities.Activities), 2) - - details := []*json.RawMessage{} - for _, activity := range activities.Activities { - if activity.Type == "mdm_enrolled" { - require.Nil(t, activity.ActorID) - require.Nil(t, activity.ActorFullName) - details = append(details, activity.Details) - } - } - require.Len(t, details, 2) - require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), string(*details[len(details)-2])) - require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceB.SerialNumber, mdmDeviceB.Model, mdmDeviceB.SerialNumber), string(*details[len(details)-1])) - // set an enroll secret var applyResp applyEnrollSecretSpecResponse s.DoJSON("POST", "/api/latest/fleet/spec/enroll_secret", applyEnrollSecretSpecRequest{ @@ -2839,7 +2827,7 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { require.NoError(t, err) // An activity is created - activities = listActivitiesResponse{} + activities := listActivitiesResponse{} s.DoJSON("GET", "/api/latest/fleet/activities", nil, http.StatusOK, &activities) found := false @@ -2848,7 +2836,6 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { found = true require.Nil(t, activity.ActorID) require.Nil(t, activity.ActorFullName) - details = append(details, activity.Details) require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), string(*activity.Details)) } } @@ -9472,9 +9459,9 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { var profileLabels []fleet.ConfigurationProfileLabel mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { stmt := ` - SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, label_id + SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, COALESCE(label_id, 0) as label_id FROM mdm_configuration_profile_labels - UNION SELECT apple_declaration_uuid as profile_uuid, label_name, label_id + UNION SELECT apple_declaration_uuid as profile_uuid, label_name, COALESCE(label_id, 0) as label_id FROM mdm_declaration_labels ORDER BY profile_uuid, label_name;` return sqlx.SelectContext(context.Background(), q, &profileLabels, stmt) }) @@ -12783,12 +12770,11 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - category, raw_json, checksum, created_at, uploaded_at -) VALUES (?,?,?,?,?,?,UNHEX(?),?,?)` +) VALUES (?,?,?,?,?,UNHEX(?),?,?)` mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(context.Background(), stmt, @@ -12796,7 +12782,6 @@ INSERT INTO mdm_apple_declarations ( decl.TeamID, decl.Identifier, decl.Name, - decl.Category, decl.RawJSON, calcChecksum(decl.RawJSON), decl.CreatedAt, @@ -12839,7 +12824,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example", Name: "Example", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"bar"}, @@ -12904,7 +12888,6 @@ INSERT INTO host_mdm_apple_declarations ( require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed)) require.EqualValues(t, wantParsed.Payload, gotParsed.Payload) require.Equal(t, calcChecksum(expected.RawJSON), gotParsed.ServerToken) - require.Contains(t, gotParsed.Type, expected.Category) require.Equal(t, expected.Identifier, gotParsed.Identifier) // t.Logf("decoded: %+v", gotParsed) } @@ -12917,7 +12900,8 @@ INSERT INTO host_mdm_apple_declarations ( checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { require.Equal(t, expectedDeclTok, r.DeclarationsToken) - require.Empty(t, r.Declarations.Activations) + // TODO(roberto): better assertions + require.NotEmpty(t, r.Declarations.Activations) require.Empty(t, r.Declarations.Assets) require.Empty(t, r.Declarations.Management) require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) @@ -12944,7 +12928,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example2", Name: "Example2", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"baz"}, @@ -12975,7 +12958,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example3", Name: "Example3", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"bang"}, @@ -13017,7 +12999,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example4", Name: "Example4", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.test", "Payload": {"foo":"bar"}, @@ -13045,3 +13026,253 @@ INSERT INTO host_mdm_apple_declarations ( assertDeclarationResponse(r, want) }) } + +func (s *integrationMDMTestSuite) TestAppleDDMReconciliation() { + t := s.T() + ctx := context.Background() + // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG + logger := kitlog.NewJSONLogger(os.Stdout) + + // TODO: use endpoints once those are available. + addDeclaration := func(identifier string, teamID uint) { + stmt := ` + INSERT INTO mdm_apple_declarations + (declaration_uuid, team_id, identifier, name, raw_json, checksum) + VALUES + (UUID(), ?, ?, UUID(), ?, HEX(MD5(raw_json)) )` + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, stmt, teamID, identifier, declarationForTest(identifier)) + return err + }) + } + + deleteDeclaration := func(identifier string, teamID uint) { + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM mdm_apple_declarations WHERE team_id = ? AND identifier = ?", teamID, identifier) + return err + }) + } + + // create a team + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + } + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", team, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + team = createTeamResp.Team + + t.Cleanup(func() { + // delete declarations to not affect other tests + deleteDeclaration("I2", 0) + deleteDeclaration("I1", team.ID) + deleteDeclaration("I2", team.ID) + deleteDeclaration("I3", team.ID) + }) + + checkNoCommands := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.Nil(t, cmd) + } + + checkDDMSync := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.NotNil(t, cmd) + require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) + cmd, err = d.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + require.Nil(t, cmd) + _, err = d.DeclarativeManagement("tokens") + require.NoError(t, err) + } + + // create a windows host + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 1, + OsqueryHostID: ptr.String("non-macos-host"), + NodeKey: ptr.String("non-macos-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.non.macos", t.Name()), + Platform: "windows", + }) + require.NoError(t, err) + + // create a windows host that's enrolled in MDM + _, _ = createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) + + // create a linux host + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("linux-host"), + NodeKey: ptr.String("linux-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.linux", t.Name()), + Platform: "linux", + }) + require.NoError(t, err) + + // create a host that's not enrolled into MDM + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("not-mdm-enrolled"), + NodeKey: ptr.String("not-mdm-enrolled"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.not.enrolled", t.Name()), + Platform: "darwin", + }) + require.NoError(t, err) + + // create a host and then enroll in MDM. + mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + // trigger the reconciler, no error + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // declarativeManagement command is not sent. + checkNoCommands(device) + + // add global declarations + addDeclaration("I1", 0) + addDeclaration("I2", 0) + + // reconcile again, this time new declarations were added + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // TODO: check command is pending + + // declarativeManagement command is sent + checkDDMSync(device) + + // reconcile again, commands for the uploaded declarations are already sent + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // no new commands are sent + checkNoCommands(device) + + // delete a declaration + deleteDeclaration("I1", 0) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // a DDM sync is triggered + checkDDMSync(device) + + // add a new host + _, deviceTwo := createHostThenEnrollMDM(s.ds, s.server.URL, t) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkDDMSync(deviceTwo) + + // add device to the team + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHost.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // DDM sync is triggered only for the transferred host + // because the team doesn't have any declarations + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // nobody receives commands this time + checkNoCommands(device) + checkNoCommands(deviceTwo) + + // add declarations to the team + addDeclaration("I1", team.ID) + addDeclaration("I2", team.ID) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered for the host in the team + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // add a new host, this one belongs to the team + mdmHostThree, deviceThree := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHostThree.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) + + // no new commands after another reconciliation + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkNoCommands(deviceThree) + + // TODO: use proper APIs for this + // add a new label + label declaration + addDeclaration("I3", team.ID) + label, err := s.ds.NewLabel(ctx, &fleet.Label{Name: t.Name(), Query: "select 1;"}) + require.NoError(t, err) + // update label with host membership + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + "INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, ?)", + mdmHostThree.ID, + label.ID, + ) + return err + }, + ) + + // update declaration <-> label mapping + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + `INSERT INTO + mdm_declaration_labels (apple_declaration_uuid, label_name, label_id) + VALUES ((SELECT declaration_uuid FROM mdm_apple_declarations WHERE team_id = ? and identifier = ?), ?, ?)`, + team.ID, + "I3", + label.Name, + label.ID, + ) + return err + }, + ) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the host with the label + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) +} + +func declarationForTest(identifier string) []byte { + return []byte(fmt.Sprintf(` +{ + "Type": "com.apple.configuration.management.test", + "Payload": { + "Echo": "foo" + }, + "Identifier": "%s" +}`, identifier)) +} From 44727ace3b564d1f5046491c0831965bcd059873 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 25 Mar 2024 17:36:26 -0300 Subject: [PATCH 15/29] fix issues with ddm CLI (#17826) for #17404. I couldn't find tests for this portion of the code, so full tests for this section will need to be added during freeze. --- server/datastore/mysql/apple_mdm.go | 51 +++++++++++++++++++++-------- server/service/client.go | 5 +-- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 10d1b27faf..8432c068bd 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -3181,7 +3181,7 @@ WHERE h.uuid = ? return nil } -func (ds *Datastore) batchSetMDMAppleDeclarations(ctx context.Context, tx sqlx.ExtContext, tmID *uint, declarations []*fleet.MDMAppleDeclaration) ([]*fleet.MDMAppleDeclaration, error) { +func (ds *Datastore) batchSetMDMAppleDeclarations(ctx context.Context, tx sqlx.ExtContext, tmID *uint, incomingDeclarations []*fleet.MDMAppleDeclaration) ([]*fleet.MDMAppleDeclaration, error) { const insertStmt = ` INSERT INTO mdm_apple_declarations ( declaration_uuid, @@ -3227,15 +3227,13 @@ WHERE declTeamID = *tmID } - var incomingLabels []fleet.ConfigurationProfileLabel - // build a list of identifiers for the incoming declarations, will keep the // existing ones if there's a match and no change - incomingIdents := make([]string, len(declarations)) + incomingIdents := make([]string, len(incomingDeclarations)) // at the same time, index the incoming declarations keyed by identifier for ease // or processing - incomingDecls := make(map[string]*fleet.MDMAppleDeclaration, len(declarations)) - for i, p := range declarations { + incomingDecls := make(map[string]*fleet.MDMAppleDeclaration, len(incomingDeclarations)) + for i, p := range incomingDeclarations { incomingIdents[i] = p.Identifier incomingDecls[p.Identifier] = p } @@ -3293,10 +3291,10 @@ WHERE if err == nil { err = errors.New(ds.testBatchSetMDMAppleProfilesErr) } - return nil, ctxerr.Wrap(ctx, err, "delete obsolete profiles") + return nil, ctxerr.Wrap(ctx, err, "delete obsolete declarations") } - for _, d := range declarations { + for _, d := range incomingDeclarations { checksum := md5ChecksumScriptContent(string(d.RawJSON)) declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString() if _, err := tx.ExecContext(ctx, insertStmt, @@ -3311,11 +3309,36 @@ WHERE } return nil, ctxerr.Wrapf(ctx, err, "insert new/edited declaration with identifier %q", d.Identifier) } + } - d.DeclarationUUID = declUUID - for _, l := range d.Labels { - l.ProfileUUID = declUUID - incomingLabels = append(incomingLabels, l) + incomingLabels := []fleet.ConfigurationProfileLabel{} + if len(incomingIdents) > 0 { + var newlyInsertedDecls []*fleet.MDMAppleDeclaration + // load current declarations (again) that match the incoming declarations by name to grab their uuids + // this is an easy way to grab the identifiers for both the existing declarations and the new ones we generated. + // + // TODO(roberto): if we're a bit careful, we can harvest this + // information without this extra request in the previous DB + // calls. Due to time constraints, I'm leaving that + // optimization for a later iteration. + stmt, args, err := sqlx.In(loadExistingDecls, declTeamID, incomingIdents) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "build query to load newly inserted declarations") + } + if err := sqlx.SelectContext(ctx, tx, &newlyInsertedDecls, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "load newly inserted declarations") + } + + for _, newlyInsertedDecl := range newlyInsertedDecls { + incomingDecl, ok := incomingDecls[newlyInsertedDecl.Identifier] + if !ok { + return nil, ctxerr.Wrapf(ctx, err, "declaration %q is in the database but was not incoming", newlyInsertedDecl.Identifier) + } + + for _, label := range incomingDecl.Labels { + label.ProfileUUID = newlyInsertedDecl.DeclarationUUID + incomingLabels = append(incomingLabels, label) + } } } @@ -3323,10 +3346,10 @@ WHERE if err == nil { err = errors.New(ds.testBatchSetMDMAppleProfilesErr) } - return nil, ctxerr.Wrap(ctx, err, "inserting apple profile label associations") + return nil, ctxerr.Wrap(ctx, err, "inserting apple declaration label associations") } - return declarations, nil + return incomingDeclarations, nil } func (ds *Datastore) NewMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { diff --git a/server/service/client.go b/server/service/client.go index f26bfcc051..bc8c9f17cf 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -294,8 +294,9 @@ func getProfilesContents(baseDir string, profiles []fleet.MDMProfileSpec) ([]fle } // by default, use the file name. macOS profiles use their PayloadDisplayName - name := strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) - if mdm.GetRawProfilePlatform(fileContents) == "darwin" { + ext := filepath.Ext(filePath) + name := strings.TrimSuffix(filepath.Base(filePath), ext) + if mdm.GetRawProfilePlatform(fileContents) == "darwin" && ext == ".mobileconfig" { mc, err := fleet.NewMDMAppleConfigProfile(fileContents, nil) if err != nil { return nil, fmt.Errorf("applying fleet config: %w", err) From f0ad942a57c96fde9929478b444412cd683ee2a6 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Tue, 26 Mar 2024 10:40:35 -0300 Subject: [PATCH 16/29] implement status reports for DDM commands (#17831) for #17408 --- pkg/mdm/mdmtest/apple.go | 11 +- server/datastore/mysql/apple_mdm.go | 85 ++++++++++++++- server/fleet/apple_mdm.go | 95 ++++++++++++++++- server/fleet/datastore.go | 10 ++ server/mdm/apple/util.go | 10 ++ server/mock/datastore_mock.go | 24 +++++ server/service/apple_mdm.go | 65 +++++++++++- server/service/integration_mdm_test.go | 140 ++++++++++++++++++++++++- 8 files changed, 429 insertions(+), 11 deletions(-) diff --git a/pkg/mdm/mdmtest/apple.go b/pkg/mdm/mdmtest/apple.go index be3759ca64..deb52664b7 100644 --- a/pkg/mdm/mdmtest/apple.go +++ b/pkg/mdm/mdmtest/apple.go @@ -9,6 +9,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -21,6 +22,7 @@ import ( "time" "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/fleetdm/fleet/v4/server/fleet" apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" "github.com/fleetdm/fleet/v4/server/mdm/scep/cryptoutil/x509util" @@ -390,7 +392,7 @@ func (c *TestAppleMDMClient) TokenUpdate() error { // The endpoint argument is used as the value for the `Endpoint` key in the request payload. // // For more details check https://developer.apple.com/documentation/devicemanagement/declarativemanagementrequest -func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) (*http.Response, error) { +func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string, data ...fleet.MDMAppleDDMStatusReport) (*http.Response, error) { payload := map[string]any{ "MessageType": "DeclarativeManagement", "UDID": c.UUID, @@ -398,6 +400,13 @@ func (c *TestAppleMDMClient) DeclarativeManagement(endpoint string) (*http.Respo "EnrollmentID": "testenrollmentid-" + c.UUID, "Endpoint": endpoint, } + if len(data) != 0 { + rawData, err := json.Marshal(data[0]) + if err != nil { + return nil, fmt.Errorf("marshaling status report: %w", err) + } + payload["Data"] = rawData + } r, err := c.request("application/x-apple-aspen-mdm-checkin", payload) return r, err } diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 8432c068bd..bf80e95f65 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -3587,8 +3587,8 @@ func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context 'install' as operation_type, ds.checksum, ds.declaration_uuid, - ds.declaration_identifier as identifier, - ds.declaration_name as name + ds.declaration_identifier, + ds.declaration_name FROM %s ) @@ -3599,8 +3599,8 @@ func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context 'remove' as operation_type, hmae.checksum, hmae.declaration_uuid, - hmae.declaration_identifier as identifier, - hmae.declaration_name as name + hmae.declaration_identifier, + hmae.declaration_name FROM %s ) @@ -3615,3 +3615,80 @@ func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context } return decls, nil } + +func (ds *Datastore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error { + getHostDeclarationsStmt := ` + SELECT host_uuid, status, operation_type, HEX(checksum) as checksum, declaration_uuid, declaration_identifier, declaration_name + FROM host_mdm_apple_declarations + WHERE host_uuid = ? + ` + + updateHostDeclarationsStmt := ` +INSERT INTO host_mdm_apple_declarations + (host_uuid, declaration_uuid, status, operation_type, detail, declaration_name, declaration_identifier, checksum) +VALUES + %s +ON DUPLICATE KEY UPDATE + status = VALUES(status), + operation_type = VALUES(operation_type), + detail = VALUES(detail) + ` + + deletePendingRemovesStmt := ` + DELETE FROM host_mdm_apple_declarations + WHERE host_uuid = ? AND operation_type = 'remove' AND status = 'pending' + ` + + var current []*fleet.MDMAppleHostDeclaration + if err := sqlx.SelectContext(ctx, ds.reader(ctx), ¤t, getHostDeclarationsStmt, hostUUID); err != nil { + return ctxerr.Wrap(ctx, err, "getting current host declarations") + } + + updatesByChecksum := make(map[string]*fleet.MDMAppleHostDeclaration, len(updates)) + for _, u := range updates { + updatesByChecksum[u.Checksum] = u + } + + var args []any + var insertVals strings.Builder + for _, c := range current { + if u, ok := updatesByChecksum[c.Checksum]; ok { + insertVals.WriteString("(?, ?, ?, ?, ?, ?, ?, UNHEX(?)),") + args = append(args, hostUUID, c.DeclarationUUID, u.Status, u.OperationType, u.Detail, c.Identifier, c.Name, c.Checksum) + } + } + + err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + if len(args) != 0 { + stmt := fmt.Sprintf(updateHostDeclarationsStmt, strings.TrimSuffix(insertVals.String(), ",")) + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "updating existing declarations") + } + } + + if _, err := tx.ExecContext(ctx, deletePendingRemovesStmt, hostUUID); err != nil { + return ctxerr.Wrap(ctx, err, "deleting pending removals") + } + + return nil + }) + + return ctxerr.Wrap(ctx, err, "updating host declarations") +} + +func (ds *Datastore) MDMAppleSetDeclarationsAsVerifying(ctx context.Context, hostUUID string) error { + stmt := ` + UPDATE host_mdm_apple_declarations + SET status = ? + WHERE + operation_type = ? + AND status = ? + AND host_uuid = ? + ` + + _, err := ds.writer(ctx).ExecContext( + ctx, stmt, fleet.MDMDeliveryVerifying, + fleet.MDMOperationTypeInstall, fleet.MDMDeliveryPending, hostUUID, + ) + return ctxerr.Wrap(ctx, err, "updating host declaration status to verifying") +} diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 093656e58a..88ea0f059e 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -635,10 +635,10 @@ type MDMAppleHostDeclaration struct { DeclarationUUID string `db:"declaration_uuid" json:"profile_uuid"` // Name corresponds to the file name of the associated JSON declaration payload. - Name string `db:"name" json:"name"` + Name string `db:"declaration_name" json:"name"` // Identifier corresponds to the "Identifier" key of the associated declaration. - Identifier string `db:"identifier" json:"-"` + Identifier string `db:"declaration_identifier" json:"-"` // Status represent the current state of the declaration, as known by the Fleet server. Status *MDMDeliveryStatus `db:"status" json:"status"` @@ -727,3 +727,94 @@ type MDMAppleDDMDeclarationResponse struct { Payload json.RawMessage `db:"payload"` ServerToken string `db:"server_token"` } + +// MDMAppleDDMStatusReport represents a report of the device's current state. +// +// https://developer.apple.com/documentation/devicemanagement/statusreport +type MDMAppleDDMStatusReport struct { + StatusItems MDMAppleDDMStatusItems `json:"StatusItems"` + Errors []MDMAppleDDMErrors `json:"Errors"` +} + +// MDMAppleDDMStatusItems are the status items for a report. +// +// https://developer.apple.com/documentation/devicemanagement/statusreport/statusitems +type MDMAppleDDMStatusItems struct { + Management MDMAppleDDMStatusManagement `json:"management"` +} + +// MDMAppleDDMStatusManagement represents status report of the client's +// processed declarations. +// +// https://developer.apple.com/documentation/devicemanagement/statusmanagementdeclarations +type MDMAppleDDMStatusManagement struct { + Declarations MDMAppleDDMStatusDeclarations `json:"declarations"` +} + +// MDMAppleDDMStatusDeclarations represents a collection of the client's +// processed declarations. +// +// https://developer.apple.com/documentation/devicemanagement/statusmanagementdeclarationsdeclarationsobject +type MDMAppleDDMStatusDeclarations struct { + // Activations is an array of declarations that represent the client's + // processed activation types. + Activations []MDMAppleDDMStatusDeclaration `json:"activations"` + // Configurations is an array of declarations that represent the + // client's processed configuration types. + Configurations []MDMAppleDDMStatusDeclaration `json:"configurations"` + // Assets is an array of declarations that represent the client's + // processed assets. + Assets []MDMAppleDDMStatusDeclaration `json:"assets"` + // Management is an array of declarations that represent the client's + // processed declaration types. + Management []MDMAppleDDMStatusDeclaration `json:"management"` +} + +type MDMAppleDeclarationValidity string + +const ( + MDMAppleDeclarationValid MDMAppleDeclarationValidity = "valid" + MDMAppleDeclarationInvalid MDMAppleDeclarationValidity = "invalid" + MDMAppleDeclarationUnknown MDMAppleDeclarationValidity = "valid" +) + +// MDMAppleDDMStatusDeclaration represents a processed declaration for the client. +// +// https://developer.apple.com/documentation/devicemanagement/statusmanagementdeclarationsdeclarationobject +type MDMAppleDDMStatusDeclaration struct { + // Active signals if the declaration is active on the device. + Active bool `json:"active"` + // Identifier is the identifier of the declaration this status report refers to. + Identifier string `json:"identifier"` + // Valid defines the validity of the declaration. If it's invalid, the + // reasons property contains more details. + Valid MDMAppleDeclarationValidity `json:"valid"` + // ServerToken of the declaration this status report refers to. + ServerToken string `json:"server-token"` + // Reasons are the details of any client errors. + Reasons []MDMAppleDDMStatusErrorReason `json:"reasons,omitempty"` +} + +// A status report's error that contains the status item and the reasons for +// the error. +// +// https://developer.apple.com/documentation/devicemanagement/statusreport/error +type MDMAppleDDMErrors struct { + // StatusItem is the status item that this error pertains to. + StatusItem string `json:"StatusItem"` + // Reasons is an array of reasons for the error. + Reasons []MDMAppleDDMStatusErrorReason `json:"Reasons"` +} + +// A status report that contains details about an error. +// +// https://developer.apple.com/documentation/devicemanagement/statusreason +type MDMAppleDDMStatusErrorReason struct { + // Code is the error code for this error. + Code string `json:"Code"` + // Description is a short error description. + Description string `json:"Description"` + // Details is a dictionary that contains further details about this + // error. + Details map[string]any `json:"Details"` +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e19f7e0b9e..fbd863679a 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1171,6 +1171,16 @@ type Datastore interface { // MDMAppleBatchInsertHostDeclarations tracks the current status of all // the host declarations provided. MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*MDMAppleHostDeclaration) error + // MDMAppleStoreDDMStatusReport receives a host.uuid and a slice + // of declarations, and updates the tracked host declaration status for + // matching declarations. + // + // It also takes care of cleaning up all host declarations that are + // pending removal. + MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*MDMAppleHostDeclaration) error + // MDMAppleSetDeclarationsAsVerifying updates all + // ("pending", "install") declarations for a host to be ("verifying", "install") + MDMAppleSetDeclarationsAsVerifying(ctx context.Context, hostUUID string) error /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mdm/apple/util.go b/server/mdm/apple/util.go index 801ca37d67..bc8b2bc6e4 100644 --- a/server/mdm/apple/util.go +++ b/server/mdm/apple/util.go @@ -101,6 +101,7 @@ func GenerateRandomPin(length int) string { return fmt.Sprintf(f, v) } +// FmtErrorChain formats Command error message for macOS MDM v1 func FmtErrorChain(chain []mdm.ErrorChain) string { var sb strings.Builder for _, mdmErr := range chain { @@ -113,6 +114,15 @@ func FmtErrorChain(chain []mdm.ErrorChain) string { return sb.String() } +// FmtDDMError formats a DDM error message +func FmtDDMError(reasons []fleet.MDMAppleDDMStatusErrorReason) string { + var errMsg strings.Builder + for _, r := range reasons { + errMsg.WriteString(fmt.Sprintf("%s: %s %+v\n", r.Code, r.Description, r.Details)) + } + return errMsg.String() +} + func EnrollURL(token string, appConfig *fleet.AppConfig) (string, error) { enrollURL, err := url.Parse(appConfig.ServerSettings.ServerURL) if err != nil { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 7ab7c83a00..bb7ae7c5fb 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -768,6 +768,10 @@ type MDMAppleGetHostsWithChangedDeclarationsFunc func(ctx context.Context) ([]*f type MDMAppleBatchInsertHostDeclarationsFunc func(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error +type MDMAppleStoreDDMStatusReportFunc func(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error + +type MDMAppleSetDeclarationsAsVerifyingFunc func(ctx context.Context, hostUUID string) error + type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error type WSTEPNewSerialFunc func(ctx context.Context) (*big.Int, error) @@ -1998,6 +2002,12 @@ type DataStore struct { MDMAppleBatchInsertHostDeclarationsFunc MDMAppleBatchInsertHostDeclarationsFunc MDMAppleBatchInsertHostDeclarationsFuncInvoked bool + MDMAppleStoreDDMStatusReportFunc MDMAppleStoreDDMStatusReportFunc + MDMAppleStoreDDMStatusReportFuncInvoked bool + + MDMAppleSetDeclarationsAsVerifyingFunc MDMAppleSetDeclarationsAsVerifyingFunc + MDMAppleSetDeclarationsAsVerifyingFuncInvoked bool + WSTEPStoreCertificateFunc WSTEPStoreCertificateFunc WSTEPStoreCertificateFuncInvoked bool @@ -4782,6 +4792,20 @@ func (s *DataStore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, cha return s.MDMAppleBatchInsertHostDeclarationsFunc(ctx, changedDeclarations) } +func (s *DataStore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error { + s.mu.Lock() + s.MDMAppleStoreDDMStatusReportFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleStoreDDMStatusReportFunc(ctx, hostUUID, updates) +} + +func (s *DataStore) MDMAppleSetDeclarationsAsVerifying(ctx context.Context, hostUUID string) error { + s.mu.Lock() + s.MDMAppleSetDeclarationsAsVerifyingFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleSetDeclarationsAsVerifyingFunc(ctx, hostUUID) +} + func (s *DataStore) WSTEPStoreCertificate(ctx context.Context, name string, crt *x509.Certificate) error { s.mu.Lock() s.WSTEPStoreCertificateFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index e64311c957..7ddd09273e 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2615,7 +2615,13 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ cmdResult.Status == fleet.MDMAppleStatusCommandFormatError { return nil, svc.ds.UpdateHostLockWipeStatusFromAppleMDMResult(r.Context, cmdResult.UDID, cmdResult.CommandUUID, requestType, cmdResult.Status == fleet.MDMAppleStatusAcknowledged) } + case "DeclarativeManagement": + // set "pending-install" profiles to "verifying" + err := svc.ds.MDMAppleSetDeclarationsAsVerifying(r.Context, cmdResult.UDID) + return nil, ctxerr.Wrap(r.Context, err, "update declaration status on DeclarativeManagement ack") + } + return nil, nil } @@ -2756,6 +2762,8 @@ func ReconcileAppleDeclarations( return ctxerr.Wrap(ctx, err, "issuing DeclarativeManagement command") } + logger.Log("msg", "sent DeclarativeManagement command", "host_number", len(uuids)) + return nil } @@ -3244,9 +3252,7 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec case dm.Endpoint == "status": level.Debug(svc.logger).Log("msg", "received status request") - // TODO(roberto): handle status - - return nil, nil + return nil, svc.handleDeclarationStatus(r.Context, dm) case strings.HasPrefix(dm.Endpoint, "declaration/"): level.Debug(svc.logger).Log("msg", "received declarations request") @@ -3374,3 +3380,56 @@ func (svc *MDMAppleDDMService) handleConfigurationDeclaration(ctx context.Contex } return b, nil } + +func (svc *MDMAppleDDMService) handleDeclarationStatus(ctx context.Context, dm *mdm.DeclarativeManagement) error { + var status fleet.MDMAppleDDMStatusReport + if err := json.Unmarshal(dm.Data, &status); err != nil { + return ctxerr.Wrap(ctx, err, "unmarshalling response") + } + + configurationReports := status.StatusItems.Management.Declarations.Configurations + updates := make([]*fleet.MDMAppleHostDeclaration, len(configurationReports)) + for i, r := range configurationReports { + var status fleet.MDMDeliveryStatus + var detail string + switch { + case r.Active && r.Valid == fleet.MDMAppleDeclarationValid: + status = fleet.MDMDeliveryVerified + case r.Valid == fleet.MDMAppleDeclarationInvalid: + status = fleet.MDMDeliveryFailed + detail = apple_mdm.FmtDDMError(r.Reasons) + default: + status = fleet.MDMDeliveryVerifying + } + + updates[i] = &fleet.MDMAppleHostDeclaration{ + Status: &status, + OperationType: fleet.MDMOperationTypeInstall, + Detail: detail, + Checksum: r.ServerToken, + } + } + + if len(updates) == 0 { + return nil + } + + // MDMAppleStoreDDMStatusReport takes care of cleaning ("pending", "remove") + // pairs for the host. + // + // TODO(roberto): in the DDM documentation, it's mentioned that status + // report will give you a "remove" status so the server can track + // removals. In my testing, I never saw this (after spending + // considerable time trying to make it work.) + // + // My current guess is that the documentation is implicitly referring + // to asset declarations (which deliver tangible "assets" to the host) + // + // The best indication I found so far, is that if the declaration is + // not in the report, then it's implicitly removed. + if err := svc.ds.MDMAppleStoreDDMStatusReport(ctx, dm.UDID, updates); err != nil { + return ctxerr.Wrap(ctx, err, "updating host declaration status with reports") + } + + return nil +} diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 62039380ad..e88d610ed4 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -12982,7 +12982,7 @@ INSERT INTO host_mdm_apple_declarations ( }) t.Run("Status", func(t *testing.T) { - _, err := mdmDevice.DeclarativeManagement("status") + _, err := mdmDevice.DeclarativeManagement("status", fleet.MDMAppleDDMStatusReport{}) require.NoError(t, err) }) @@ -13266,6 +13266,144 @@ func (s *integrationMDMTestSuite) TestAppleDDMReconciliation() { checkDDMSync(deviceThree) } +func (s *integrationMDMTestSuite) TestAppleDDMStatusReport() { + t := s.T() + ctx := context.Background() + // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG + logger := kitlog.NewJSONLogger(os.Stdout) + + assertHostDeclarations := func(hostUUID string, wantDecls []*fleet.MDMAppleHostDeclaration) { + var gotDecls []*fleet.MDMAppleHostDeclaration + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.SelectContext(context.Background(), q, &gotDecls, `SELECT declaration_identifier, status, operation_type FROM host_mdm_apple_declarations WHERE host_uuid = ?`, hostUUID) + }) + require.ElementsMatch(t, wantDecls, gotDecls) + } + + // create a host and then enroll in MDM. + mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + declarations := []fleet.MDMProfileBatchPayload{ + {Name: "N1.json", Contents: declarationForTest("I1")}, + {Name: "N2.json", Contents: declarationForTest("I2")}, + } + // add global declarations + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) + + // reconcile profiles + err := ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // declarations are ("install", "pending") after the cron run + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host gets a DDM sync call + cmd, err := device.Idle() + require.NoError(t, err) + require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) + _, err = device.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + + r, err := device.DeclarativeManagement("declaration-items") + require.NoError(t, err) + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var items fleet.MDMAppleDDMDeclarationItemsResponse + require.NoError(t, json.Unmarshal(body, &items)) + + var i1ServerToken, i2ServerToken string + for _, d := range items.Declarations.Configurations { + switch d.Identifier { + case "I1": + i1ServerToken = d.ServerToken + case "I2": + i2ServerToken = d.ServerToken + } + } + + // declarations are ("install", "verifying") after the ack + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a partial DDM report + report := fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a report with a wrong (could be old) server token for I2, nothing changes + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I2", ServerToken: "foo"}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a full report, declaration I2 is invalid + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I2", ServerToken: i2ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // do a batch request, this time I2 is deleted + declarations = []fleet.MDMProfileBatchPayload{ + {Name: "N1.json", Contents: declarationForTest("I1")}, + } + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) + + // reconcile profiles + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + }) + + // host sends a report, declaration I2 is removed from the hosts_* table + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a report, declaration I1 is failing after a while + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, + }) +} + func declarationForTest(identifier string) []byte { return []byte(fmt.Sprintf(` { From bb63da41b70ebefd76d3da4b51ee67166216ca6f Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Tue, 26 Mar 2024 17:54:23 +0000 Subject: [PATCH 17/29] add ddm activities to the UI (#17864) relates to #17409 adds the ddm activities to the UI. - [x] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Manual QA for all new/changed functionality --- changes/issue-17409-add-ddm-activities-to-ui | 1 + frontend/interfaces/activity.ts | 3 + .../ActivityItem/ActivityItem.tsx | 65 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 changes/issue-17409-add-ddm-activities-to-ui diff --git a/changes/issue-17409-add-ddm-activities-to-ui b/changes/issue-17409-add-ddm-activities-to-ui new file mode 100644 index 0000000000..0c0c267a32 --- /dev/null +++ b/changes/issue-17409-add-ddm-activities-to-ui @@ -0,0 +1 @@ +- add ddm activities to the fleet UI diff --git a/frontend/interfaces/activity.ts b/frontend/interfaces/activity.ts index 143efeaf0a..dd871d7be6 100644 --- a/frontend/interfaces/activity.ts +++ b/frontend/interfaces/activity.ts @@ -67,6 +67,9 @@ export enum ActivityType { LockedHost = "locked_host", UnlockedHost = "unlocked_host", WipedHost = "wiped_host", + CreatedDeclarationProfile = "created_declaration_profile", + DeletedDeclarationProfile = "deleted_declaration_profile", + EditedDeclarationProfile = "edited_declaration_profile", } // This is a subset of ActivityType that are shown only for the host past activities diff --git a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx index 7c13965bdb..a3941ac7af 100644 --- a/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx +++ b/frontend/pages/DashboardPage/cards/ActivityFeed/ActivityItem/ActivityItem.tsx @@ -763,6 +763,55 @@ const TAGGED_TEMPLATES = { ); }, + createdDeclarationProfile: (activity: IActivity, isPremiumTier: boolean) => { + return ( + <> + {" "} + added declaration (DDM) profile + {activity.details?.profile_name} + {" "} + to{" "} + {getProfileMessageSuffix( + isPremiumTier, + "darwin", + activity.details?.team_name + )} + . + + ); + }, + deletedDeclarationProfile: (activity: IActivity, isPremiumTier: boolean) => { + return ( + <> + {" "} + removed declaration (DDM) profile{" "} + {activity.details?.profile_name} from{" "} + {getProfileMessageSuffix( + isPremiumTier, + "darwin", + activity.details?.team_name + )} + . + + ); + }, + editedDeclarationProfile: (activity: IActivity, isPremiumTier: boolean) => { + return ( + <> + {" "} + edited declaration (DDM) profile + {activity.details?.profile_name} + {" "} + for{" "} + {getProfileMessageSuffix( + isPremiumTier, + "darwin", + activity.details?.team_name + )}{" "} + via fleetctl. + + ); + }, }; const getDetail = ( @@ -918,6 +967,22 @@ const getDetail = ( case ActivityType.WipedHost: { return TAGGED_TEMPLATES.wipedHost(activity); } + case ActivityType.CreatedDeclarationProfile: { + return TAGGED_TEMPLATES.createdDeclarationProfile( + activity, + isPremiumTier + ); + } + case ActivityType.DeletedDeclarationProfile: { + return TAGGED_TEMPLATES.deletedDeclarationProfile( + activity, + isPremiumTier + ); + } + case ActivityType.EditedDeclarationProfile: { + return TAGGED_TEMPLATES.editedDeclarationProfile(activity, isPremiumTier); + } + default: { return TAGGED_TEMPLATES.defaultActivityTemplate(activity); } From 1edd9f07bb07fc09b606ea369bd978fcc07f1b51 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:54:47 -0500 Subject: [PATCH 18/29] Update host details, list host filters, and MDM summary to include macOS declarations (#17866) Issue #17619 --------- Co-authored-by: Roberto Dip --- server/datastore/mysql/apple_mdm.go | 292 +++++++++++++++--- server/datastore/mysql/apple_mdm_test.go | 24 +- server/datastore/mysql/hosts.go | 60 ++-- server/datastore/mysql/labels_test.go | 5 + server/datastore/mysql/mdm_test.go | 372 ++++++++++++++++++++++- 5 files changed, 662 insertions(+), 91 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index bf80e95f65..eb7143e2b0 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -352,16 +352,38 @@ COALESCE(detail, '') AS detail FROM host_mdm_apple_profiles WHERE +host_uuid = ? AND NOT (operation_type = '%s' AND COALESCE(status, '%s') IN('%s', '%s')) + +UNION ALL +SELECT +declaration_uuid AS profile_uuid, +declaration_name AS name, +declaration_identifier AS identifier, +-- internally, a NULL status implies that the cron needs to pick up +-- this profile, for the user that difference doesn't exist, the +-- profile is effectively pending. This is consistent with all our +-- aggregation functions. +COALESCE(status, '%s') AS status, +COALESCE(operation_type, '') AS operation_type, +COALESCE(detail, '') AS detail +FROM +host_mdm_apple_declarations +WHERE host_uuid = ? AND NOT (operation_type = '%s' AND COALESCE(status, '%s') IN('%s', '%s'))`, fleet.MDMDeliveryPending, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, fleet.MDMDeliveryVerifying, fleet.MDMDeliveryVerified, + fleet.MDMDeliveryPending, + fleet.MDMOperationTypeRemove, + fleet.MDMDeliveryPending, + fleet.MDMDeliveryVerifying, + fleet.MDMDeliveryVerified, ) var profiles []fleet.HostMDMAppleProfile - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, stmt, hostUUID); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, stmt, hostUUID, hostUUID); err != nil { return nil, err } return profiles, nil @@ -2190,57 +2212,251 @@ func subqueryAppleProfileStatus(status fleet.MDMDeliveryStatus) (string, []any, return query, args, nil } +// subqueryAppleDeclarationStatus builds out the subquery for declaration status +func subqueryAppleDeclarationStatus() (string, []any, error) { + const declNamedStmt = ` + CASE WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d1 + WHERE + h.uuid = d1.host_uuid + AND d1.status = :failed) THEN + 'declarations_failed' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d2 + WHERE + h.uuid = d2.host_uuid + AND(d2.status IS NULL + OR d2.status = :pending) + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d3 + WHERE + h.uuid = d3.host_uuid + AND d3.status = :failed)) THEN + 'declarations_pending' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d4 + WHERE + h.uuid = d4.host_uuid + AND d4.status = :verifying + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d5 + WHERE (h.uuid = d5.host_uuid + AND(d5.status IS NULL + OR d5.status IN(:pending, :failed))))) THEN + 'declarations_verifying' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d6 + WHERE + h.uuid = d6.host_uuid + AND d6.status = :verified + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d7 + WHERE (h.uuid = d7.host_uuid + AND(d7.status IS NULL + OR d7.status IN(:pending, :failed, :verifying))))) THEN + 'declarations_verified' + ELSE + '' + END` + + // TODO: do we need to differentiate between install and remove? + arg := map[string]any{ + // "install": fleet.MDMOperationTypeInstall, + // "remove": fleet.MDMOperationTypeRemove, + "verifying": fleet.MDMDeliveryVerifying, + "failed": fleet.MDMDeliveryFailed, + "verified": fleet.MDMDeliveryVerified, + "pending": fleet.MDMDeliveryPending, + } + query, args, err := sqlx.Named(declNamedStmt, arg) + if err != nil { + return "", nil, fmt.Errorf("subqueryAppleDeclarationStatus: %w", err) + } + + return query, args, nil +} + +func subqueryOSSettingsStatusMac() (string, []any, error) { + var profArgs []any + profFailed, profFailedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profFailedArgs...) + + profPending, profPendingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryPending) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profPendingArgs...) + + profVerifying, profVerifyingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profVerifyingArgs...) + + profVerified, profVerifiedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profVerifiedArgs...) + + profStmt := fmt.Sprintf(` + CASE WHEN EXISTS (%s) THEN + 'profiles_failed' + WHEN EXISTS (%s) THEN + 'profiles_pending' + WHEN EXISTS (%s) THEN + 'profiles_verifying' + WHEN EXISTS (%s) THEN + 'profiles_verified' + ELSE + '' + END`, + profFailed, + profPending, + profVerifying, + profVerified, + ) + + declStmt, declArgs, err := subqueryAppleDeclarationStatus() + if err != nil { + return "", nil, err + } + + stmt := fmt.Sprintf(` + CASE (%s) + WHEN 'profiles_failed' THEN + 'failed' + WHEN 'profiles_pending' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + ELSE + 'pending' + END) + WHEN 'profiles_verifying' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + WHEN 'declarations_pending' THEN + 'pending' + ELSE + 'verifying' + END) + WHEN 'profiles_verified' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + WHEN 'declarations_pending' THEN + 'pending' + WHEN 'declarations_verifying' THEN + 'verifying' + ELSE + 'verified' + END) + ELSE + REPLACE((%s), 'declarations_', '') + END`, profStmt, declStmt, declStmt, declStmt, declStmt) + + args := append(profArgs, declArgs...) + args = append(args, declArgs...) + args = append(args, declArgs...) + args = append(args, declArgs...) + + // FIXME(roberto): we found issues in MySQL 5.7.17 (only that version, + // which we must support for now) with prepared statements on this + // query. The results returned by the DB were always different what + // expected unless the arguments are inlined in the query. + // + // We decided to do this given: + // + // - The time constraints we were given to develop DDM + // - The fact that all the variables in this query are really strings managed by us + // - The imminent deprecation of MySQL 5.7 + return fmt.Sprintf(strings.Replace(stmt, "?", "'%s'", -1), args...), []any{}, nil +} + func (ds *Datastore) GetMDMAppleProfilesSummary(ctx context.Context, teamID *uint) (*fleet.MDMProfilesSummary, error) { - var args []interface{} - - subqueryFailed, subqueryFailedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) + subquery, args, err := subqueryOSSettingsStatusMac() if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building failed subquery") + return nil, ctxerr.Wrap(ctx, err, "building os settings subquery") } - args = append(args, subqueryFailedArgs...) - - subqueryPending, subqueryPendingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building pending subquery") - } - args = append(args, subqueryPendingArgs...) - - subqueryVerifying, subqueryVerifyingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building verifying subquery") - } - args = append(args, subqueryVerifyingArgs...) - - subqueryVerified, subqueryVerifiedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building verified subquery") - } - args = append(args, subqueryVerifiedArgs...) sqlFmt := ` - SELECT - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS failed, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS pending, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS verifying, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS verified - FROM - hosts h - WHERE - h.platform = 'darwin' AND %s` +SELECT + %s as status, + COUNT(id) as count +FROM + hosts h +GROUP BY status, platform, team_id HAVING platform = 'darwin' AND status IN (?, ?, ?, ?) AND %s` - teamFilter := "h.team_id IS NULL" + args = append(args, fleet.MDMDeliveryFailed, fleet.MDMDeliveryPending, fleet.MDMDeliveryVerifying, fleet.MDMDeliveryVerified) + + teamFilter := "team_id IS NULL" if teamID != nil && *teamID > 0 { - teamFilter = "h.team_id = ?" + teamFilter = "team_id = ?" args = append(args, *teamID) } - stmt := fmt.Sprintf(sqlFmt, subqueryFailed, subqueryPending, subqueryVerifying, subqueryVerified, teamFilter) - var res fleet.MDMProfilesSummary - err = sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, args...) + stmt := fmt.Sprintf(sqlFmt, subquery, teamFilter) + + var dest []struct { + Count uint `db:"count"` + Status string `db:"status"` + } + + err = sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt, args...) if err != nil { return nil, err } + byStatus := make(map[string]uint) + for _, s := range dest { + if _, ok := byStatus[s.Status]; ok { + return nil, fmt.Errorf("duplicate status %s", s.Status) + } + byStatus[s.Status] = s.Count + } + + var res fleet.MDMProfilesSummary + for s, c := range byStatus { + switch fleet.MDMDeliveryStatus(s) { + case fleet.MDMDeliveryFailed: + res.Failed = c + case fleet.MDMDeliveryPending: + res.Pending = c + case fleet.MDMDeliveryVerifying: + res.Verifying = c + case fleet.MDMDeliveryVerified: + res.Verified = c + default: + return nil, fmt.Errorf("unknown status %s", s) + } + } + return &res, nil } diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 6eed346a69..d989cc2719 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1787,13 +1787,19 @@ func testAggregateMacOSSettingsStatusWithFileVault(t *testing.T, ds *Datastore) expectedIDs = append(expectedIDs, h.ID) } - gotHosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{MacOSSettingsFilter: status, TeamFilter: teamID}) + gotHosts, err := ds.ListHosts( + ctx, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, + fleet.HostListOptions{MacOSSettingsFilter: status, TeamFilter: teamID}, + ) gotIDs := []uint{} for _, h := range gotHosts { gotIDs = append(gotIDs, h.ID) } - return assert.NoError(t, err) && assert.Len(t, gotHosts, len(expected)) && assert.ElementsMatch(t, expectedIDs, gotIDs) + return assert.NoError(t, err) && + assert.Len(t, gotHosts, len(expected)) && + assert.ElementsMatch(t, expectedIDs, gotIDs) } var hosts []*fleet.Host @@ -2615,7 +2621,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2641,7 +2647,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2669,7 +2675,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2691,7 +2697,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(1), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2713,7 +2719,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(3), allProfilesSummary.Pending) require.Equal(t, uint(1), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2743,7 +2749,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, &tm.ID) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(0), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2769,7 +2775,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, &tm.ID) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(0), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(0), allProfilesSummary.Verifying) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index df2f4395bc..5c78af5b15 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1210,34 +1210,22 @@ func filterHostsByMacOSSettingsStatus(sql string, opt fleet.HostListOptions, par return sql, params, nil } - newSQL := "" + whereStatus := "" + // macOS settings filter is not compatible with the "all teams" option so append the "no + // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) if opt.TeamFilter == nil { - // macOS settings filter is not compatible with the "all teams" option so append the "no - // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) - newSQL += ` AND h.team_id IS NULL` + whereStatus += ` AND h.team_id IS NULL` } - var subquery string - var subqueryParams []any - var err error - switch opt.MacOSSettingsFilter { - case fleet.OSSettingsFailed: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) - case fleet.OSSettingsPending: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - case fleet.OSSettingsVerifying: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - case fleet.OSSettingsVerified: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - } + subqueryStatus, paramsStatus, err := subqueryOSSettingsStatusMac() if err != nil { - return "", nil, fmt.Errorf("building subquery for %s filter: %w", opt.MacOSSettingsFilter, err) - } - if subquery != "" { - newSQL += fmt.Sprintf(` AND EXISTS (%s)`, subquery) + return "", nil, err } - return sql + newSQL, append(params, subqueryParams...), nil + whereStatus += fmt.Sprintf(` AND %s = ?`, subqueryStatus) + paramsStatus = append(paramsStatus, opt.MacOSSettingsFilter) + + return sql + whereStatus, append(params, paramsStatus...), nil } func filterHostsByMacOSDiskEncryptionStatus(sql string, opt fleet.HostListOptions, params []interface{}) (string, []interface{}) { @@ -1285,30 +1273,16 @@ func (ds *Datastore) filterHostsByOSSettingsStatus(sql string, opt fleet.HostLis sqlFmt += ` AND h.team_id IS NULL` } var whereMacOS, whereWindows string - sqlFmt += ` AND ((h.platform = 'windows' AND (%s)) OR (h.platform = 'darwin' AND (%s)))` + sqlFmt += ` +AND ((h.platform = 'windows' AND (%s)) +OR (h.platform = 'darwin' AND (%s)))` - // construct the WHERE for macOS - var subqueryMacOS string - var paramsMacOS []interface{} - var err error - switch opt.OSSettingsFilter { - case fleet.OSSettingsFailed: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) - case fleet.OSSettingsPending: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - case fleet.OSSettingsVerifying: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - case fleet.OSSettingsVerified: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - } + whereMacOS, paramsMacOS, err := subqueryOSSettingsStatusMac() if err != nil { - return "", nil, fmt.Errorf("building subquery for %s filter: %w", opt.OSSettingsFilter, err) - } - if subqueryMacOS != "" { - whereMacOS = "EXISTS (" + subqueryMacOS + ")" - } else { - whereMacOS = "FALSE" + return "", nil, err } + whereMacOS += ` = ?` + paramsMacOS = append(paramsMacOS, opt.OSSettingsFilter) // construct the WHERE for windows whereWindows = `hmdm.name = ? AND hmdm.enrolled = 1 AND hmdm.is_server = 0` diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index eb1ddcd9fa..ea9d455c9a 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -364,6 +364,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("1"), UUID: "1", Hostname: "foo.local", + Platform: "darwin", }) require.NoError(t, err) @@ -377,6 +378,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("2"), UUID: "2", Hostname: "bar.local", + Platform: "darwin", }) require.NoError(t, err) h3, err := db.NewHost(context.Background(), &fleet.Host{ @@ -388,6 +390,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("3"), UUID: "3", Hostname: "baz.local", + Platform: "darwin", }) require.NoError(t, err) @@ -427,6 +430,7 @@ func testLabelsListHostsInLabelAndTeamFilter(deferred bool, t *testing.T, db *Da NodeKey: ptr.String("1"), UUID: "1", Hostname: "foo.local", + Platform: "darwin", }) require.Nil(t, err) @@ -440,6 +444,7 @@ func testLabelsListHostsInLabelAndTeamFilter(deferred bool, t *testing.T, db *Da NodeKey: ptr.String("2"), UUID: "2", Hostname: "bar.local", + Platform: "darwin", }) require.Nil(t, err) diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index f5784bdb10..b962cf8f68 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strconv" + "strings" "testing" "time" @@ -43,12 +44,12 @@ func TestMDMShared(t *testing.T) { {"TestMDMEULA", testMDMEULA}, {"TestGetHostCertAssociationsToExpire", testSCEPRenewalHelpers}, {"TestSCEPRenewalHelpers", testSCEPRenewalHelpers}, + {"TestMDMProfilesSummaryAndHostFilters", testMDMProfilesSummaryAndHostFilters}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { defer TruncateTables(t, ds) - c.fn(t, ds) }) } @@ -3330,3 +3331,372 @@ func testSCEPRenewalHelpers(t *testing.T, ds *Datastore) { require.NoError(t, err) checkSCEPRenew(assocs[0], nil) } + +func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) { + // TODO: Expand this test to include: + // - more scenarios for windows + // - disk encryption (mac and windows) + // - more scenarios for labels + + ctx := context.Background() + + checkSummaryWindows := func(t *testing.T, teamID *uint, expected fleet.MDMProfilesSummary) { + ps, err := ds.GetMDMWindowsProfilesSummary(ctx, teamID) + require.NoError(t, err) + require.NotNil(t, ps) + require.Equal(t, expected, *ps) + } + + checkSummaryMac := func(t *testing.T, teamID *uint, expected fleet.MDMProfilesSummary) { + ps, err := ds.GetMDMAppleProfilesSummary(ctx, teamID) + require.NoError(t, err) + require.NotNil(t, ps) + require.Equal(t, expected, *ps) + } + + checkListHostsFilterOSSettings := func(t *testing.T, teamID *uint, status fleet.OSSettingsStatus, expectedIDs []uint) { + gotHosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status}) + require.NoError(t, err) + if len(expectedIDs) != len(gotHosts) { + gotIDs := make([]uint, len(gotHosts)) + for _, h := range gotHosts { + gotIDs = append(gotIDs, h.ID) + } + require.Len(t, gotHosts, len(expectedIDs), fmt.Sprintf("status: %s expected: %v got: %v", status, expectedIDs, gotIDs)) + + } + for _, h := range gotHosts { + require.Contains(t, expectedIDs, h.ID) + } + + count, err := ds.CountHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status}) + require.NoError(t, err) + require.Equal(t, len(expectedIDs), count, "status: %s", status) + } + + type hostIDsByProfileStatus map[fleet.MDMDeliveryStatus][]uint + + checkExpected := func(t *testing.T, teamID *uint, ep hostIDsByProfileStatus) { + expectSummaryWindows := map[fleet.MDMDeliveryStatus]uint{} + expectSummaryMac := map[fleet.MDMDeliveryStatus]uint{} + for status, ids := range ep { + if len(ids) > 0 { + for _, id := range ids { + if id < 5 { + expectSummaryWindows[status]++ + } else { + expectSummaryMac[status]++ + } + } + } + } + checkSummaryMac(t, teamID, fleet.MDMProfilesSummary{ + Pending: expectSummaryMac[fleet.MDMDeliveryPending], + Failed: expectSummaryMac[fleet.MDMDeliveryFailed], + Verifying: expectSummaryMac[fleet.MDMDeliveryVerifying], + Verified: expectSummaryMac[fleet.MDMDeliveryVerified], + }) + + checkSummaryWindows(t, teamID, fleet.MDMProfilesSummary{ + Pending: expectSummaryWindows[fleet.MDMDeliveryPending], + Failed: expectSummaryWindows[fleet.MDMDeliveryFailed], + Verifying: expectSummaryWindows[fleet.MDMDeliveryVerifying], + Verified: expectSummaryWindows[fleet.MDMDeliveryVerified], + }) + + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerified, ep[fleet.MDMDeliveryVerified]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerifying, ep[fleet.MDMDeliveryVerifying]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsFailed, ep[fleet.MDMDeliveryFailed]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsPending, ep[fleet.MDMDeliveryPending]) + } + + // checkWinHostProfiles := func(t *testing.T, hostUUID string, statusByProfUUID map[string]string) { + // profs, err := ds.GetHostMDMWindowsProfiles(ctx, hostUUID) + // require.NoError(t, err) + // require.Len(t, profs, len(statusByProfUUID)) + // for _, prof := range profs { + // ep, ok := statusByProfUUID[prof.ProfileUUID] + // require.True(t, ok) + // require.Equal(t, ep, prof.Status) + // } + // } + + checkMacHostProfiles := func(t *testing.T, hostUUID string, statusByProfUUID map[string]string) { + profs, err := ds.GetHostMDMAppleProfiles(ctx, hostUUID) + require.NoError(t, err) + require.Len(t, profs, len(statusByProfUUID)) + for _, prof := range profs { + ep, ok := statusByProfUUID[prof.ProfileUUID] + require.True(t, ok) + require.NotNil(t, prof.Status) + require.Equal(t, fleet.MDMDeliveryStatus(ep), *prof.Status) + } + } + + upsertHostProfileStatus := func(t *testing.T, hostUUID string, profUUID string, status *fleet.MDMDeliveryStatus) { + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + var table string + var profType string + switch { + case strings.HasPrefix(profUUID, "a"): + table = "host_mdm_apple_profiles" + profType = "profile" + case strings.HasPrefix(profUUID, "w"): + table = "host_mdm_windows_profiles" + profType = "profile" + case strings.HasPrefix(profUUID, "d"): + table = "host_mdm_apple_declarations" + profType = "declaration" + default: + require.FailNow(t, "unknown profile type") + } + stmt := fmt.Sprintf(`INSERT INTO %s (host_uuid, %s_uuid, status) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE status = ?`, table, profType) + _, err := q.ExecContext(ctx, stmt, hostUUID, profUUID, status, status) + if err != nil { + require.NoError(t, err) + return err + } + stmt = fmt.Sprintf(`UPDATE %s SET operation_type = ? WHERE host_uuid = ? AND %s_uuid = ?`, table, profType) + _, err = q.ExecContext(ctx, stmt, fleet.MDMOperationTypeInstall, hostUUID, profUUID) + require.NoError(t, err) + return err + }) + } + + cleanupTables := func(t *testing.T) { + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_windows_profiles`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_profiles`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_declarations`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_disk_encryption_keys`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_disks`) + return err + }) + } + + // updateHostDisks := func(t *testing.T, hostID uint, encrypted bool, updated_at time.Time) { + // ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + // stmt := `UPDATE host_disks SET encrypted = ?, updated_at = ? where host_id = ?` + // _, err := q.ExecContext(ctx, stmt, encrypted, updated_at, hostID) + // return err + // }) + // } + + // Create some hosts + var hosts []*fleet.Host + macHostsByID := make(map[uint]*fleet.Host, 5) + winHostsByID := make(map[uint]*fleet.Host, 5) + for i := 0; i < 10; i++ { + p := "windows" + if i >= 5 { + p = "darwin" + } + u := uuid.New().String() + h, err := ds.NewHost(ctx, &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: &u, + UUID: u, + Hostname: u, + Platform: p, + }) + require.NoError(t, err) + require.NotNil(t, h) + hosts = append(hosts, h) + if p == "darwin" { + macHostsByID[h.ID] = h + } else { + winHostsByID[h.ID] = h + } + + require.NoError(t, ds.SetOrUpdateMDMData(ctx, h.ID, false, true, "https://example.com", false, fleet.WellKnownMDMFleet, "")) + } + + checkExpected(t, nil, nil) + + upsertHostProfileStatus(t, hosts[0].UUID, "w1", &fleet.MDMDeliveryPending) + + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + }) + + // add some mac profiles with different statuses + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryFailed) + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryPending) + upsertHostProfileStatus(t, hosts[9].UUID, "a3", &fleet.MDMDeliveryVerifying) + upsertHostProfileStatus(t, hosts[9].UUID, "a4", &fleet.MDMDeliveryVerified) + + // add some mac declarations with different statuses + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryFailed) + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryPending) + upsertHostProfileStatus(t, hosts[9].UUID, "d3", &fleet.MDMDeliveryVerifying) + upsertHostProfileStatus(t, hosts[9].UUID, "d4", &fleet.MDMDeliveryVerified) + + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + expectedHostProfiles := map[string]string{ + "a1": "failed", + "a2": "pending", + "a3": "verifying", + "a4": "verified", + "d1": "failed", + "d2": "pending", + "d3": "verifying", + "d4": "verified", + } + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac profile to pending, still failed because of failed declaration + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryPending) + expectedHostProfiles["a1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac declaration to pending, now host stsatus is pending + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryPending) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac declaration to failed, host status is now failed + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryFailed) + expectedHostProfiles["d2"] = "failed" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac declaration to verifying, host status is now pending + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["d2"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac profiles to verifying, host status is still pending because d1 is still pending + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["a1"] = "verifying" + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["a2"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac declarations to verifying, host status is now verifying + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["d1"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerifying: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set a mac profile to failed, host status is now failed + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryFailed) + expectedHostProfiles["a1"] = "failed" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set mac profiles to verified, host status is now verifying because declarations are still + // verifying + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a1"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a2"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a3", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a3"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a4", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a4"] = "verified" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerifying: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set mac declarations to verified, host status is now verified + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d1"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d2"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "d3", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d3"] = "verified" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerified: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set a mac declaration to nil, host status is now pending + upsertHostProfileStatus(t, hosts[9].UUID, "d1", nil) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if we remove mac declarations + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_declarations`) + return err + }) + delete(expectedHostProfiles, "d1") + delete(expectedHostProfiles, "d2") + delete(expectedHostProfiles, "d3") + delete(expectedHostProfiles, "d4") + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerified: []uint{hosts[9].ID}, // all profiles were verified + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if we remove mac profiles + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_profiles`) + return err + }) + delete(expectedHostProfiles, "a1") + delete(expectedHostProfiles, "a2") + delete(expectedHostProfiles, "a3") + delete(expectedHostProfiles, "a4") + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if declarations but no profiles + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryPending) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + cleanupTables(t) +} From 7b13d9ce173f586d8d51ee217bd6ba8730f5fc3c Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:09:09 -0400 Subject: [PATCH 19/29] Add mdm_apple_declarative_requests table to log DDM requests (#17844) #17792 --- server/datastore/mysql/apple_mdm.go | 18 ++++++++ ...240322145615_CreateTableNanoDDMRequests.go | 36 ++++++++++++++++ server/datastore/mysql/schema.sql | 17 +++++++- server/fleet/datastore.go | 3 ++ server/mock/datastore_mock.go | 12 ++++++ server/service/apple_mdm.go | 4 ++ server/service/integration_mdm_test.go | 43 +++++++++++++++++-- 7 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240322145615_CreateTableNanoDDMRequests.go diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index eb7143e2b0..ad86960da7 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -3770,6 +3770,24 @@ WHERE return &res, nil } +func (ds *Datastore) InsertMDMAppleDDMRequest(ctx context.Context, hostUUID, messageType, rawJSON string) error { + const stmt = ` +INSERT INTO + mdm_apple_declarative_requests ( + enrollment_id, + message_type, + raw_json + ) +VALUES + (?, ?, ?) +` + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostUUID, messageType, rawJSON); err != nil { + return ctxerr.Wrap(ctx, err, "writing apple declarative request to db") + } + + return nil +} + func (ds *Datastore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { baseStmt := ` INSERT INTO host_mdm_apple_declarations diff --git a/server/datastore/mysql/migrations/tables/20240322145615_CreateTableNanoDDMRequests.go b/server/datastore/mysql/migrations/tables/20240322145615_CreateTableNanoDDMRequests.go new file mode 100644 index 0000000000..927767e93f --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240322145615_CreateTableNanoDDMRequests.go @@ -0,0 +1,36 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240322145615, Down_20240322145615) +} + +func Up_20240322145615(tx *sql.Tx) error { + _, err := tx.Exec(` +CREATE TABLE mdm_apple_declarative_requests ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + enrollment_id VARCHAR(255) NOT NULL, + -- Should be one of "tokens", "declaration-items", "status", or "declaration/…/…" where the ellipses reference a declaration on the server + message_type VARCHAR(255) NOT NULL, + -- json payload + raw_json TEXT, + PRIMARY KEY (id), + CONSTRAINT mdm_apple_declarative_requests_enrollment_id FOREIGN KEY (enrollment_id) REFERENCES nano_enrollments (id) ON DELETE CASCADE +) +`) + + if err != nil { + return fmt.Errorf("creating mdm_apple_declarative_requsts: %w", err) + } + + return nil +} + +func Down_20240322145615(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 99b8644db5..86163de3fe 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -676,6 +676,19 @@ CREATE TABLE `mdm_apple_declarations` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `mdm_apple_declarative_requests` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `enrollment_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `message_type` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `raw_json` text COLLATE utf8mb4_unicode_ci, + PRIMARY KEY (`id`), + KEY `mdm_apple_declarative_requests_enrollment_id` (`enrollment_id`), + CONSTRAINT `mdm_apple_declarative_requests_enrollment_id` FOREIGN KEY (`enrollment_id`) REFERENCES `nano_enrollments` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_apple_default_setup_assistants` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `team_id` int(10) unsigned DEFAULT NULL, @@ -840,9 +853,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=258 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=259 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314150853,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314150853,1,'2020-01-01 01:01:01'),(258,20240322145615,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index fbd863679a..2c833ecae7 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1155,6 +1155,9 @@ type Datastore interface { // serials. UpdateDEPAssignProfileRetryPending(ctx context.Context, jobID uint, serials []string) error + // InsertMDMAppleDDMRequest inserts a DDM request. + InsertMDMAppleDDMRequest(ctx context.Context, hostUUID, messageType, rawJSON string) error + // MDMAppleDDMDeclarationsToken returns the token used to synchronize declarations for the // specified host UUID. MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*MDMAppleDDMDeclarationsToken, error) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index bb7ae7c5fb..7022bc01ca 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -758,6 +758,8 @@ type GetDEPAssignProfileExpiredCooldownsFunc func(ctx context.Context) (map[uint type UpdateDEPAssignProfileRetryPendingFunc func(ctx context.Context, jobID uint, serials []string) error +type InsertMDMAppleDDMRequestFunc func(ctx context.Context, hostUUID string, messageType string, rawJSON string) error + type MDMAppleDDMDeclarationsTokenFunc func(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) @@ -1987,6 +1989,9 @@ type DataStore struct { UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFunc UpdateDEPAssignProfileRetryPendingFuncInvoked bool + InsertMDMAppleDDMRequestFunc InsertMDMAppleDDMRequestFunc + InsertMDMAppleDDMRequestFuncInvoked bool + MDMAppleDDMDeclarationsTokenFunc MDMAppleDDMDeclarationsTokenFunc MDMAppleDDMDeclarationsTokenFuncInvoked bool @@ -4757,6 +4762,13 @@ func (s *DataStore) UpdateDEPAssignProfileRetryPending(ctx context.Context, jobI return s.UpdateDEPAssignProfileRetryPendingFunc(ctx, jobID, serials) } +func (s *DataStore) InsertMDMAppleDDMRequest(ctx context.Context, hostUUID string, messageType string, rawJSON string) error { + s.mu.Lock() + s.InsertMDMAppleDDMRequestFuncInvoked = true + s.mu.Unlock() + return s.InsertMDMAppleDDMRequestFunc(ctx, hostUUID, messageType, rawJSON) +} + func (s *DataStore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { s.mu.Lock() s.MDMAppleDDMDeclarationsTokenFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 7ddd09273e..49092b559e 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -3237,6 +3237,10 @@ func (svc *MDMAppleDDMService) DeclarativeManagement(r *mdm.Request, dm *mdm.Dec } level.Debug(svc.logger).Log("msg", "ddm request received", "endpoint", dm.Endpoint) + if err := svc.ds.InsertMDMAppleDDMRequest(r.Context, dm.UDID, dm.Endpoint, string(dm.Data)); err != nil { + return nil, ctxerr.Wrap(r.Context, err, "insert ddm request history") + } + if dm.UDID == "" { return nil, nano_service.NewHTTPStatusError(http.StatusBadRequest, ctxerr.New(r.Context, "missing UDID in request")) } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index e88d610ed4..8d68ebaf80 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -12912,9 +12912,30 @@ INSERT INTO host_mdm_apple_declarations ( } } + checkRequestsDatabase := func(t *testing.T, messageType, enrollmentID string, expectedCount int) { + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + var count int + if err := sqlx.GetContext( + context.Background(), + q, + &count, + "SELECT count(*) AS count FROM mdm_apple_declarative_requests WHERE enrollment_id = ? AND message_type = ?", + enrollmentID, + messageType, + ); err != nil { + return err + } + + require.Equal(t, expectedCount, count, "unexpected db row count for declaration requests") + + return nil + }) + } + var currDeclToken string // we'll use this to track the expected token across tests t.Run("Tokens", func(t *testing.T) { + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 0) // get tokens, timestamp should be the same as the declaration and token should be non-empty r, err := mdmDevice.DeclarativeManagement("tokens") require.NoError(t, err) @@ -12938,6 +12959,7 @@ INSERT INTO host_mdm_apple_declarations ( } insertDeclaration(t, noTeamDeclsByUUID["456"]) insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["456"]) + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 1) // get tokens again, timestamp and token should have changed r, err = mdmDevice.DeclarativeManagement("tokens") @@ -12945,9 +12967,11 @@ INSERT INTO host_mdm_apple_declarations ( parsed = parseTokensResp(r) checkTokensResp(t, parsed, then.Add(1*time.Minute), currDeclToken) currDeclToken = parsed.SyncTokens.DeclarationsToken + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 2) }) t.Run("DeclarationItems", func(t *testing.T) { + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 0) r, err := mdmDevice.DeclarativeManagement("declaration-items") require.NoError(t, err) checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) @@ -12968,6 +12992,7 @@ INSERT INTO host_mdm_apple_declarations ( } insertDeclaration(t, noTeamDeclsByUUID["789"]) insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["789"]) + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 1) // get tokens again, timestamp and token should have changed r, err = mdmDevice.DeclarativeManagement("tokens") @@ -12979,16 +13004,21 @@ INSERT INTO host_mdm_apple_declarations ( r, err = mdmDevice.DeclarativeManagement("declaration-items") require.NoError(t, err) checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 2) }) t.Run("Status", func(t *testing.T) { + checkRequestsDatabase(t, "status", mdmDevice.UUID, 0) _, err := mdmDevice.DeclarativeManagement("status", fleet.MDMAppleDDMStatusReport{}) require.NoError(t, err) + checkRequestsDatabase(t, "status", mdmDevice.UUID, 1) }) t.Run("Declaration", func(t *testing.T) { want := noTeamDeclsByUUID["123"] - r, err := mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) + declarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier) + checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 0) + r, err := mdmDevice.DeclarativeManagement(declarationPath) require.NoError(t, err) assertDeclarationResponse(r, want) @@ -13012,16 +13042,23 @@ INSERT INTO host_mdm_apple_declarations ( want = noTeamDeclsByUUID["abc"] r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) require.NoError(t, err) + checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 1) // try getting a non-existent declaration, should fail 404 - _, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", "nonexistent")) + nonExistantDeclarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", "nonexistent") + checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 0) + _, err = mdmDevice.DeclarativeManagement(nonExistantDeclarationPath) require.Error(t, err) require.ErrorContains(t, err, "404 Not Found") + checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 1) // typo should fail as bad request - _, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declarations/%s/%s", "configurations", want.Identifier)) + typoDeclarationPath := fmt.Sprintf("declarations/%s/%s", "configurations", want.Identifier) + checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 0) + _, err = mdmDevice.DeclarativeManagement(typoDeclarationPath) require.Error(t, err) require.ErrorContains(t, err, "400 Bad Request") + checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 1) assertDeclarationResponse(r, want) }) From fdc5aa57c2f48d5c6dbde0e52e84d3a87bd7bab8 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 27 Mar 2024 09:53:43 -0300 Subject: [PATCH 20/29] add ddm declarations in the API (#17880) for #17409 --- server/fleet/activities.go | 50 ++++++++++++++++++++++++++ server/service/apple_mdm.go | 39 +++++++++++++++----- server/service/integration_mdm_test.go | 23 ++++++++++-- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/server/fleet/activities.go b/server/fleet/activities.go index 2fb7a4dbc0..8dacdf6862 100644 --- a/server/fleet/activities.go +++ b/server/fleet/activities.go @@ -1317,6 +1317,56 @@ func (a ActivityTypeWipedHost) Documentation() (activity, details, detailsExampl }` } +type ActivityTypeCreatedDeclarationProfile struct { + ProfileName string `json:"profile_name"` + Identifier string `json:"identifier"` + TeamID *uint `json:"team_id"` + TeamName *string `json:"team_name"` +} + +func (a ActivityTypeCreatedDeclarationProfile) ActivityName() string { + return "created_declaration_profile" +} + +func (a ActivityTypeCreatedDeclarationProfile) Documentation() (activity string, details string, detailsExample string) { + return `Generated when a user adds a new macOS declaration to a team (or no team).`, + `This activity contains the following fields: +- "profile_name": Name of the declaration. +- "identifier": Identifier of the declaration. +- "team_id": The ID of the team that the declaration applies to, ` + "`null`" + ` if it applies to devices that are not in a team. +- "team_name": The name of the team that the declaration applies to, ` + "`null`" + ` if it applies to devices that are not in a team.`, `{ + "profile_name": "Passcode requirements", + "profile_identifier": "com.my.declaration", + "team_id": 123, + "team_name": "Workstations" +}` +} + +type ActivityTypeDeletedDeclarationProfile struct { + ProfileName string `json:"profile_name"` + Identifier string `json:"identifier"` + TeamID *uint `json:"team_id"` + TeamName *string `json:"team_name"` +} + +func (a ActivityTypeDeletedDeclarationProfile) ActivityName() string { + return "deleted_declaration_profile" +} + +func (a ActivityTypeDeletedDeclarationProfile) Documentation() (activity string, details string, detailsExample string) { + return `Generated when a user removes a macOS declaration from a team (or no team).`, + `This activity contains the following fields: +- "profile_name": Name of the declaration. +- "identifier": Identifier of the declaration. +- "team_id": The ID of the team that the declaration applies to, ` + "`null`" + ` if it applies to devices that are not in a team. +- "team_name": The name of the team that the declaration applies to, ` + "`null`" + ` if it applies to devices that are not in a team.`, `{ + "profile_name": "Passcode requirements", + "profile_identifier": "com.my.declaration", + "team_id": 123, + "team_name": "Workstations" +}` +} + // LogRoleChangeActivities logs activities for each role change, globally and one for each change in teams. func LogRoleChangeActivities(ctx context.Context, ds Datastore, adminUser *User, oldGlobalRole *string, oldTeamRoles []UserTeam, user *User) error { if user.GlobalRole != nil && (oldGlobalRole == nil || *oldGlobalRole != *user.GlobalRole) { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 49092b559e..1a05d51f57 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -406,6 +406,15 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i return nil, err } + var teamName string + if teamID >= 1 { + tm, err := svc.EnterpriseOverrides.TeamByIDOrName(ctx, &teamID, nil) + if err != nil { + return nil, ctxerr.Wrap(ctx, err) + } + teamName = tm.Name + } + data, err := io.ReadAll(r) if err != nil { return nil, err @@ -442,6 +451,23 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i return nil, err } + var ( + actTeamID *uint + actTeamName *string + ) + if teamID > 0 { + actTeamID = &teamID + actTeamName = &teamName + } + if err := svc.ds.NewActivity(ctx, authz.UserFromContext(ctx), &fleet.ActivityTypeCreatedDeclarationProfile{ + TeamID: actTeamID, + TeamName: actTeamName, + ProfileName: decl.Name, + Identifier: decl.Identifier, + }); err != nil { + return nil, ctxerr.Wrap(ctx, err, "logging activity for create mdm apple declaration") + } + return decl, nil } @@ -802,13 +828,10 @@ func (svc *Service) DeleteMDMAppleDeclaration(ctx context.Context, declUUID stri return ctxerr.Wrap(ctx, err) } - // TODO: confirm if bulk set pending host profiles is needed - // cannot use the profile ID as it is now deleted if err := svc.ds.BulkSetPendingMDMHostProfiles(ctx, nil, []uint{teamID}, nil, nil); err != nil { return ctxerr.Wrap(ctx, err, "bulk set pending host profiles") } - // TODO: confirm activity type var ( actTeamID *uint actTeamName *string @@ -817,11 +840,11 @@ func (svc *Service) DeleteMDMAppleDeclaration(ctx context.Context, declUUID stri actTeamID = &teamID actTeamName = &teamName } - if err := svc.ds.NewActivity(ctx, authz.UserFromContext(ctx), &fleet.ActivityTypeDeletedMacosProfile{ - TeamID: actTeamID, - TeamName: actTeamName, - ProfileName: decl.Name, - ProfileIdentifier: decl.Identifier, + if err := svc.ds.NewActivity(ctx, authz.UserFromContext(ctx), &fleet.ActivityTypeDeletedDeclarationProfile{ + TeamID: actTeamID, + TeamName: actTeamName, + ProfileName: decl.Name, + Identifier: decl.Identifier, }); err != nil { return ctxerr.Wrap(ctx, err, "logging activity for delete mdm apple declaration") } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 8d68ebaf80..3820aff4f0 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9331,6 +9331,20 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { return uid } + createAppleDeclaration := func(name, ident string, teamID uint, labelNames []string) string { + uid := assertAppleDeclaration(name+".json", ident, teamID, labelNames, http.StatusOK, "") + + var wantJSON string + if teamID == 0 { + wantJSON = fmt.Sprintf(`{"team_id": null, "team_name": null, "profile_name": %q, "identifier": %q}`, name, ident) + } else { + wantJSON = fmt.Sprintf(`{"team_id": %d, "team_name": %q, "profile_name": %q, "identifier": %q}`, teamID, testTeam.Name, name, ident) + } + s.lastActivityOfTypeMatches(fleet.ActivityTypeCreatedDeclarationProfile{}.ActivityName(), wantJSON, 0) + + return uid + } + assertWindowsProfile := func(filename, locURI string, teamID uint, labelNames []string, wantStatus int, wantErrMsg string) string { fields := map[string][]string{ "labels": labelNames, @@ -9399,7 +9413,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { assertAppleProfile("win-team-profile.mobileconfig", "win-team-profile", "test-team-ident-2", 0, nil, http.StatusOK, "") // add some macOS declarations - assertAppleDeclaration("apple-declaration.json", "test-declaration-ident", 0, nil, http.StatusOK, "") + createAppleDeclaration("apple-declaration", "test-declaration-ident", 0, nil) // identifier must be unique, it conflicts with existing declaration assertAppleDeclaration("apple-declaration.json", "test-declaration-ident", 0, nil, http.StatusConflict, "test-declaration-ident already exists") // name is pulled from filename, it conflicts with existing declaration @@ -9451,7 +9465,7 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // profiles with valid labels uuidAppleWithLabel := assertAppleProfile("apple-profile-with-labels.mobileconfig", "apple-profile-with-labels", "ident-with-labels", 0, []string{"foo"}, http.StatusOK, "") - uuidAppleDDMWithLabel := assertAppleDeclaration("apple-decl-with-labels.json", "ident-decl-with-labels", 0, []string{"foo"}, http.StatusOK, "") + uuidAppleDDMWithLabel := createAppleDeclaration("apple-decl-with-labels", "ident-decl-with-labels", 0, []string{"foo"}) uuidWindowsWithLabel := assertWindowsProfile("win-profile-with-labels.xml", "./Test", 0, []string{"foo", "bar"}, http.StatusOK, "") // verify that the label associations have been created @@ -9558,6 +9572,11 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { // delete existing Apple declaration s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", uuidAppleDDMWithLabel), nil, http.StatusOK, &deleteResp) + s.lastActivityOfTypeMatches( + fleet.ActivityTypeDeletedDeclarationProfile{}.ActivityName(), + `{"profile_name": "apple-decl-with-labels", "identifier": "ident-decl-with-labels", "team_id": null, "team_name": null}`, + 0, + ) // delete non-existing Apple declaration s.DoJSON("DELETE", fmt.Sprintf("/api/latest/fleet/configuration_profiles/%s", fmt.Sprintf("%sno-such-profile", fleet.MDMAppleDeclarationUUIDPrefix)), nil, http.StatusNotFound, &deleteResp) // delete existing Windows profiles From 0be9f085b0c1cf962d3fb687d26501a5e6316927 Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 27 Mar 2024 10:44:22 -0300 Subject: [PATCH 21/29] batch set declarations as pending when they're uploaded (#17876) #17685 --- server/datastore/mysql/apple_mdm.go | 165 +- server/datastore/mysql/mdm.go | 14 + server/datastore/mysql/mdm_test.go | 3807 ++++++++++++++++++++---- server/datastore/mysql/mysql.go | 54 + server/datastore/mysql/mysql_test.go | 52 + server/fleet/datastore.go | 11 +- server/mock/datastore_mock.go | 24 +- server/service/apple_mdm.go | 37 +- server/service/integration_ddm_test.go | 909 ++++++ server/service/integration_mdm_test.go | 867 +----- 10 files changed, 4381 insertions(+), 1559 deletions(-) create mode 100644 server/service/integration_ddm_test.go diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index ad86960da7..b22f178300 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -2012,11 +2012,11 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload return err } - var ( - args []any - sb strings.Builder - batchCount int - ) + generateValueArgs := func(p *fleet.MDMAppleBulkUpsertHostProfilePayload) (string, []any) { + valuePart := "(?, ?, ?, ?, ?, ?, ?, ?, ?)," + args := []any{p.ProfileUUID, p.ProfileIdentifier, p.ProfileName, p.HostUUID, p.Status, p.OperationType, p.Detail, p.CommandUUID, p.Checksum} + return valuePart, args + } const defaultBatchSize = 1000 // results in this times 9 placeholders batchSize := defaultBatchSize @@ -2024,30 +2024,10 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload batchSize = ds.testUpsertMDMDesiredProfilesBatchSize } - resetBatch := func() { - batchCount = 0 - args = args[:0] - sb.Reset() + if err := batchProcessDB(payload, batchSize, generateValueArgs, executeUpsertBatch); err != nil { + return err } - for _, p := range payload { - args = append(args, p.ProfileUUID, p.ProfileIdentifier, p.ProfileName, p.HostUUID, p.Status, p.OperationType, p.Detail, p.CommandUUID, p.Checksum) - sb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),") - batchCount++ - - if batchCount >= batchSize { - if err := executeUpsertBatch(sb.String(), args); err != nil { - return err - } - resetBatch() - } - } - - if batchCount > 0 { - if err := executeUpsertBatch(sb.String(), args); err != nil { - return err - } - } return nil } @@ -3770,25 +3750,71 @@ WHERE return &res, nil } -func (ds *Datastore) InsertMDMAppleDDMRequest(ctx context.Context, hostUUID, messageType, rawJSON string) error { - const stmt = ` -INSERT INTO - mdm_apple_declarative_requests ( - enrollment_id, - message_type, - raw_json - ) -VALUES - (?, ?, ?) -` - if _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostUUID, messageType, rawJSON); err != nil { - return ctxerr.Wrap(ctx, err, "writing apple declarative request to db") +func (ds *Datastore) MDMAppleBatchSetHostDeclarationState(ctx context.Context) ([]string, error) { + var uuids []string + + const defaultBatchSize = 1000 + batchSize := defaultBatchSize + if ds.testUpsertMDMDesiredProfilesBatchSize > 0 { + batchSize = ds.testUpsertMDMDesiredProfilesBatchSize } - return nil + err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + var err error + uuids, err = mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, &fleet.MDMDeliveryPending) + return err + }) + + return uuids, ctxerr.Wrap(ctx, err, "upserting host declaration state") } -func (ds *Datastore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { +func mdmAppleBatchSetHostDeclarationStateDB(ctx context.Context, tx sqlx.ExtContext, batchSize int, status *fleet.MDMDeliveryStatus) ([]string, error) { + // once all the declarations are in place, compute the desired state + // and find which hosts need a DDM sync. + changedDeclarations, err := mdmAppleGetHostsWithChangedDeclarationsDB(ctx, tx) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "find hosts with changed declarations") + } + + if len(changedDeclarations) == 0 { + return []string{}, nil + } + + // a host might have more than one declaration to sync, we do this to + // collect unique host UUIDs in order to send a single command to each + // host in the next step + uuidMap := map[string]struct{}{} + for _, d := range changedDeclarations { + uuidMap[d.HostUUID] = struct{}{} + } + uuids := make([]string, 0, len(uuidMap)) + for uuid := range uuidMap { + uuids = append(uuids, uuid) + } + + // mark the host declarations as pending, this serves two purposes: + // + // - support the APIs/methods that track host status (summaries, filters, etc) + // + // - support the DDM endpoints, which use data from the + // `host_mdm_apple_declarations` table to compute which declarations to + // serve + if err := mdmAppleBatchSetPendingHostDeclarationsDB(ctx, tx, batchSize, changedDeclarations, status); err != nil { + return nil, ctxerr.Wrap(ctx, err, "batch insert mdm apple host declarations") + } + + return uuids, nil +} + +// mdmAppleBatchSetPendingHostDeclarationsDB tracks the current status of all +// the host declarations provided. +func mdmAppleBatchSetPendingHostDeclarationsDB( + ctx context.Context, + tx sqlx.ExtContext, + batchSize int, + changedDeclarations []*fleet.MDMAppleHostDeclaration, + status *fleet.MDMDeliveryStatus, +) error { baseStmt := ` INSERT INTO host_mdm_apple_declarations (host_uuid, status, operation_type, checksum, declaration_uuid, declaration_identifier, declaration_name) @@ -3799,21 +3825,32 @@ func (ds *Datastore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, ch operation_type = VALUES(operation_type), checksum = VALUES(checksum) ` - var placeholders strings.Builder - var args []any - for _, d := range changedDeclarations { - placeholders.WriteString("(?, 'pending', ?, ?, ?, ?, ?),") - args = append(args, d.HostUUID, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name) + + executeUpsertBatch := func(valuePart string, args []any) error { + _, err := tx.ExecContext( + ctx, + fmt.Sprintf(baseStmt, strings.TrimSuffix(valuePart, ",")), + args..., + ) + return err } - _, err := ds.writer(ctx).ExecContext( - ctx, - fmt.Sprintf(baseStmt, strings.TrimSuffix(placeholders.String(), ",")), - args..., - ) + + generateValueArgs := func(d *fleet.MDMAppleHostDeclaration) (string, []any) { + valuePart := "(?, ?, ?, ?, ?, ?, ?)," + args := []any{d.HostUUID, status, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name} + return valuePart, args + } + + err := batchProcessDB(changedDeclarations, batchSize, generateValueArgs, executeUpsertBatch) return ctxerr.Wrap(ctx, err, "inserting changed host declaration state") } -func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { +// mdmAppleGetHostsWithChangedDeclarationsDB returns a +// MDMAppleHostDeclaration item for each (host x declaration) pair that +// needs an status change, this includes declarations to install and +// declarations to be removed. Those can be differentiated by the +// OperationType field on each struct. +func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtContext) ([]*fleet.MDMAppleHostDeclaration, error) { stmt := fmt.Sprintf(` ( SELECT @@ -3844,7 +3881,7 @@ func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context ) var decls []*fleet.MDMAppleHostDeclaration - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &decls, stmt, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove); err != nil { + if err := sqlx.SelectContext(ctx, tx, &decls, stmt, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove); err != nil { return nil, ctxerr.Wrap(ctx, err, "running sql statement") } return decls, nil @@ -3870,7 +3907,7 @@ ON DUPLICATE KEY UPDATE deletePendingRemovesStmt := ` DELETE FROM host_mdm_apple_declarations - WHERE host_uuid = ? AND operation_type = 'remove' AND status = 'pending' + WHERE host_uuid = ? AND operation_type = 'remove' AND (status = 'pending' OR status IS NULL) ` var current []*fleet.MDMAppleHostDeclaration @@ -3926,3 +3963,21 @@ func (ds *Datastore) MDMAppleSetDeclarationsAsVerifying(ctx context.Context, hos ) return ctxerr.Wrap(ctx, err, "updating host declaration status to verifying") } + +func (ds *Datastore) InsertMDMAppleDDMRequest(ctx context.Context, hostUUID, messageType, rawJSON string) error { + const stmt = ` +INSERT INTO + mdm_apple_declarative_requests ( + enrollment_id, + message_type, + raw_json + ) +VALUES + (?, ?, ?) +` + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostUUID, messageType, rawJSON); err != nil { + return ctxerr.Wrap(ctx, err, "writing apple declarative request to db") + } + + return nil +} diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index 6316bddd31..0ef788c464 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -451,6 +451,20 @@ WHERE return ctxerr.Wrap(ctx, err, "bulk set pending windows host profiles") } + const defaultBatchSize = 1000 + batchSize := defaultBatchSize + if ds.testUpsertMDMDesiredProfilesBatchSize > 0 { + batchSize = ds.testUpsertMDMDesiredProfilesBatchSize + } + // TODO(roberto): this method currently sets the state of all + // declarations for all hosts. I don't see an immediate concern + // (and my hunch is that we could even do the same for + // profiles) but this could be optimized to use only a provided + // set of host uuids. + if _, err := mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, nil); err != nil { + return ctxerr.Wrap(ctx, err, "bulk set pending apple declarations") + } + return nil }) } diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index b962cf8f68..3f4988d13e 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -651,9 +651,30 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { wantProfs = sortProfs(wantProfs) for i, wp := range wantProfs { gp := gotProfs[i] - require.Equal(t, wp.ProfileUUID, gp.ProfileUUID, "host uuid: %s, prof id or name: %s", h.UUID, gp.IdentifierOrName) - require.Equal(t, wp.Status, gp.Status, "host uuid: %s, prof id or name: %s", h.UUID, gp.IdentifierOrName) - require.Equal(t, wp.OperationType, gp.OperationType, "host uuid: %s, prof id or name: %s", h.UUID, gp.IdentifierOrName) + require.Equal( + t, + wp.ProfileUUID, + gp.ProfileUUID, + "host uuid: %s, prof id or name: %s", + h.UUID, + gp.IdentifierOrName, + ) + require.Equal( + t, + wp.Status, + gp.Status, + "host uuid: %s, prof id or name: %s", + h.UUID, + gp.IdentifierOrName, + ) + require.Equal( + t, + wp.OperationType, + gp.OperationType, + "host uuid: %s, prof id or name: %s", + h.UUID, + gp.IdentifierOrName, + ) } } } @@ -758,18 +779,22 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { configProfileForTest(t, "G2a", "G2a", "b"), configProfileForTest(t, "G3a", "G3a", "c"), } + macGlobalDeclarations := []*fleet.MDMAppleDeclaration{ + declForTest("G1d", "G1d", "foo"), + declForTest("G2d", "G2d", "bar"), + } winGlobalProfiles := []*fleet.MDMWindowsConfigProfile{ windowsConfigProfileForTest(t, "G1w", "L1"), windowsConfigProfileForTest(t, "G2w", "L2"), windowsConfigProfileForTest(t, "G3w", "L3"), } - err = ds.BatchSetMDMProfiles(ctx, nil, macGlobalProfiles, winGlobalProfiles, nil) + err = ds.BatchSetMDMProfiles(ctx, nil, macGlobalProfiles, winGlobalProfiles, macGlobalDeclarations) require.NoError(t, err) macGlobalProfiles, err = ds.ListMDMAppleConfigProfiles(ctx, nil) require.NoError(t, err) require.Len(t, macGlobalProfiles, 3) globalProfiles := getProfs(nil) - require.Len(t, globalProfiles, 6) + require.Len(t, globalProfiles, 8) // list profiles to install, should result in the global profiles for all // enrolled hosts @@ -793,36 +818,153 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -851,37 +993,148 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.Len(t, toRemoveWindows, 3) // update status of the moved host (team has no profiles) - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(darwinHosts[0], windowsHosts[0]), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts(darwinHosts[0], windowsHosts[0]), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, // windows profiles are directly deleted without a pending state (there's no on-host removal of profiles) windowsHosts[0]: {}, windowsHosts[1]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -911,23 +1164,110 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.Len(t, toRemoveWindows, 3) // update status of the moved host via its uuid (team has no profiles) - err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, nil, nil, []string{darwinHosts[1].UUID, windowsHosts[1].UUID}) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + nil, + nil, + nil, + []string{darwinHosts[1].UUID, windowsHosts[1].UUID}, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, @@ -935,9 +1275,21 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // windows profiles are directly deleted without a pending state windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -978,33 +1330,144 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, + { + ProfileUUID: tm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm1Profiles[0].Identifier, + }, + { + ProfileUUID: tm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm1Profiles[1].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: tm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1022,15 +1485,15 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // rows in this test since we don't have command uuids. err = ds.BulkUpsertMDMAppleHostProfiles(ctx, []*fleet.MDMAppleBulkUpsertHostProfilePayload{ { - HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[0].ProfileUUID, + HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[0].ProfileUUID, ProfileIdentifier: darwinGlobalProfiles[0].Identifier, Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeRemove, Checksum: []byte("csum"), }, { - HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[1].ProfileUUID, + HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[1].ProfileUUID, ProfileIdentifier: darwinGlobalProfiles[1].Identifier, Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeRemove, Checksum: []byte("csum"), }, { - HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[2].ProfileUUID, + HostUUID: darwinHosts[0].UUID, ProfileUUID: darwinGlobalProfiles[2].ProfileUUID, ProfileIdentifier: darwinGlobalProfiles[2].Identifier, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove, Checksum: []byte("csum"), }, }) @@ -1058,32 +1521,138 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: tm1Profiles[0].Identifier, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[0].Identifier, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1113,33 +1682,143 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[0].Identifier, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[1].Identifier, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[1].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[3].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: globalProfiles[4].Identifier, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: globalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: globalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: globalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1156,6 +1835,7 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { windowsConfigProfileForTest(t, "G4w", "G4"), } + // TODO(roberto): add new darwin declarations for this and all subsequent assertions err = ds.BatchSetMDMProfiles(ctx, nil, newDarwinGlobalProfiles, newWindowsGlobalProfiles, nil) require.NoError(t, err) @@ -1166,36 +1846,112 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, []uint{0}, nil, nil) require.NoError(t, err) + require.NoError(t, ds.MDMAppleStoreDDMStatusReport(ctx, darwinHosts[0].UUID, nil)) + require.NoError(t, ds.MDMAppleStoreDDMStatusReport(ctx, darwinHosts[1].UUID, nil)) + require.NoError(t, ds.MDMAppleStoreDDMStatusReport(ctx, darwinHosts[2].UUID, nil)) + assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[0].Identifier, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[1].Identifier, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: newTm1Profiles[2].Identifier, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1227,35 +1983,107 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1266,36 +2094,112 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: {}, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1320,39 +2224,123 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1420,51 +2408,150 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // simulate an entry with some values set to NULL ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(ctx, `UPDATE host_mdm_apple_profiles SET detail = NULL WHERE profile_uuid = ?`, globalProfiles[2].ProfileUUID) + _, err := q.ExecContext( + ctx, + `UPDATE host_mdm_apple_profiles SET detail = NULL WHERE profile_uuid = ?`, + globalProfiles[2].ProfileUUID, + ) return err }) // do a sync of all hosts, should not change anything as no host is a member // of the new label-based profiles (indices change due to new Apple and // Windows profiles) - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts( + append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm2Profiles[0].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1507,58 +2594,189 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { // do a full sync, the new global hosts get the standard global profiles and // also the label-based profile that they are a member of - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts( + append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1581,62 +2799,208 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.NoError(t, err) // do a sync of those hosts, they will get the two label-based profiles of their platform - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(darwinHosts[2], windowsHosts[2]), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts(darwinHosts[2], windowsHosts[2]), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[11].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[11].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1645,65 +3009,217 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { require.NoError(t, ds.DeleteLabel(ctx, labels[3].Name)) // sync the affected profiles - err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, nil, []string{newGlobalProfiles[4].ProfileUUID}, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + nil, + nil, + []string{newGlobalProfiles[4].ProfileUUID}, + nil, + ) require.NoError(t, err) - err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, nil, []string{newGlobalProfiles[10].ProfileUUID}, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + nil, + nil, + []string{newGlobalProfiles[10].ProfileUUID}, + nil, + ) require.NoError(t, err) // nothing changes - broken label-based profiles are simply ignored assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[11].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[11].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1717,61 +3233,203 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { }) require.NoError(t, err) - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(darwinHosts[2], windowsHosts[2]), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts(darwinHosts[2], windowsHosts[2]), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1780,62 +3438,206 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { setProfileLabels(t, newGlobalProfiles[4], labels[1]) setProfileLabels(t, newGlobalProfiles[10], labels[4]) - err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, nil, []string{newGlobalProfiles[4].ProfileUUID}, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + nil, + nil, + []string{newGlobalProfiles[4].ProfileUUID}, + nil, + ) require.NoError(t, err) - err = ds.BulkSetPendingMDMHostProfiles(ctx, nil, nil, []string{newGlobalProfiles[10].ProfileUUID}, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + nil, + nil, + []string{newGlobalProfiles[10].ProfileUUID}, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1864,55 +3666,187 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1931,57 +3865,202 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm2Profiles[0].Identifier, + }, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm2Profiles[1].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -1996,57 +4075,202 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[0].Identifier, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[2].Identifier, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + IdentifierOrName: globalProfiles[4].Identifier, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm2Profiles[0].Identifier, + }, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + IdentifierOrName: tm2Profiles[1].Identifier, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -2066,57 +4290,197 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) @@ -2131,115 +4495,394 @@ func testBulkSetPendingMDMHostProfiles(t *testing.T, ds *Datastore) { assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) // sanity-check, a full sync does not change anything - err = ds.BulkSetPendingMDMHostProfiles(ctx, hostIDsFromHosts(append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), nil, nil, nil) + err = ds.BulkSetPendingMDMHostProfiles( + ctx, + hostIDsFromHosts( + append(darwinHosts, append(windowsHosts, unenrolledHost, linuxHost)...)...), + nil, + nil, + nil, + ) require.NoError(t, err) assertHostProfiles(map[*fleet.Host][]anyProfile{ darwinHosts[0]: { - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newTm1Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryFailed, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newTm1Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, darwinHosts[1]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: globalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: tm2Profiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: tm2Profiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: globalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: tm2Profiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: tm2Profiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[2]: { - {ProfileUUID: globalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - {ProfileUUID: newGlobalProfiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + { + ProfileUUID: globalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, + { + ProfileUUID: newGlobalProfiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeRemove, + }, }, darwinHosts[3]: { - {ProfileUUID: newGlobalProfiles[0].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[1].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[0].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[1].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, unenrolledHost: {}, linuxHost: {}, windowsHosts[0]: { - {ProfileUUID: newTm1Profiles[3].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[4].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newTm1Profiles[5].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newTm1Profiles[3].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[4].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newTm1Profiles[5].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[1]: { - {ProfileUUID: tm2Profiles[2].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: tm2Profiles[2].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[2]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, windowsHosts[3]: { - {ProfileUUID: newGlobalProfiles[6].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[7].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[8].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[9].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {ProfileUUID: newGlobalProfiles[10].ProfileUUID, Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + { + ProfileUUID: newGlobalProfiles[6].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[7].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[8].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[9].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, + { + ProfileUUID: newGlobalProfiles[10].ProfileUUID, + Status: &fleet.MDMDeliveryPending, + OperationType: fleet.MDMOperationTypeInstall, + }, }, }) } diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 8c4d7c62c2..866f9d6d1d 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -1281,3 +1281,57 @@ func (ds *Datastore) optimisticGetOrInsert(ctx context.Context, readStmt, insert } return id, nil } + +// batchProcessDB abstracts the batch processing logic, for a given payload: +// +// - generateValueArgs will get called for each item, the expected return values are: +// - a string containing the placeholders for each item in the batch +// - a slice of arguments containing one item for each placeholder +// +// - executeBatch will get called on each batch to perform the operation in the db +// +// TODO(roberto): use this function in all the functions where we do ad-hoc +// batch processing. +func batchProcessDB[T any]( + payload []T, + batchSize int, + generateValueArgs func(T) (string, []any), + executeBatch func(string, []any) error, +) error { + if len(payload) == 0 { + return nil + } + + var ( + args []any + sb strings.Builder + batchCount int + ) + + resetBatch := func() { + batchCount = 0 + args = args[:0] + sb.Reset() + } + + for _, item := range payload { + valuePart, itemArgs := generateValueArgs(item) + args = append(args, itemArgs...) + sb.WriteString(valuePart) + batchCount++ + + if batchCount >= batchSize { + if err := executeBatch(sb.String(), args); err != nil { + return err + } + resetBatch() + } + } + + if batchCount > 0 { + if err := executeBatch(sb.String(), args); err != nil { + return err + } + } + return nil +} diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index 171426016d..de5eee8905 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -1229,3 +1229,55 @@ func TestWhereFilterGlobalOrTeamIDByTeams(t *testing.T) { }) } } + +func TestBatchProcessDB(t *testing.T) { + type testData struct { + id int + value string + } + + payload := []interface{}{ + &testData{id: 1, value: "a"}, + &testData{id: 2, value: "b"}, + &testData{id: 3, value: "c"}, + } + + generateValueArgs := func(item interface{}) (string, []any) { + p := item.(*testData) + valuePart := "(?, ?)," + args := []any{p.id, p.value} + return valuePart, args + } + + t.Run("TestEmptyPayload", func(t *testing.T) { + executeBatch := func(valuePart string, args []any) error { + return errors.New("execute shouldn't be called for an empty payload") + } + err := batchProcessDB([]interface{}{}, 1000, generateValueArgs, executeBatch) + require.NoError(t, err) + }) + + t.Run("TestSingleBatch", func(t *testing.T) { + callCount := 0 + executeBatch := func(valuePart string, args []any) error { + callCount++ + require.Equal(t, 2, len(args)/2) // each item adds 2 args + return nil + } + err := batchProcessDB(payload[:2], 2, generateValueArgs, executeBatch) + require.NoError(t, err) + require.Equal(t, 1, callCount) + }) + + t.Run("TestMultipleBatches", func(t *testing.T) { + callCount := 0 + executeBatch := func(valuePart string, args []any) error { + callCount++ + require.Equal(t, 2/callCount, len(args)/2) // each item adds 2 args + return nil + } + err := batchProcessDB(payload, 2, generateValueArgs, executeBatch) + require.NoError(t, err) + require.Equal(t, 2, callCount) + }) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 2c833ecae7..7e3fac09f8 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1165,15 +1165,8 @@ type Datastore interface { MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]MDMAppleDDMDeclarationItem, error) // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*MDMAppleDeclaration, error) - // MDMAppleGetHostsWithChangedDeclarations returns a - // MDMAppleHostDeclaration item for each (host x declaration) pair that - // needs an status change, this includes declarations to install and - // declarations to be removed. Those can be differentiated by the - // OperationType field on each struct. - MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*MDMAppleHostDeclaration, error) - // MDMAppleBatchInsertHostDeclarations tracks the current status of all - // the host declarations provided. - MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*MDMAppleHostDeclaration) error + //MDMAppleBatchSetHostDeclarationState + MDMAppleBatchSetHostDeclarationState(ctx context.Context) ([]string, error) // MDMAppleStoreDDMStatusReport receives a host.uuid and a slice // of declarations, and updates the tracked host declaration status for // matching declarations. diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 7022bc01ca..d3f0e3824d 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -766,9 +766,7 @@ type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) -type MDMAppleGetHostsWithChangedDeclarationsFunc func(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) - -type MDMAppleBatchInsertHostDeclarationsFunc func(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error +type MDMAppleBatchSetHostDeclarationStateFunc func(ctx context.Context) ([]string, error) type MDMAppleStoreDDMStatusReportFunc func(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error @@ -2001,11 +1999,8 @@ type DataStore struct { MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFuncInvoked bool - MDMAppleGetHostsWithChangedDeclarationsFunc MDMAppleGetHostsWithChangedDeclarationsFunc - MDMAppleGetHostsWithChangedDeclarationsFuncInvoked bool - - MDMAppleBatchInsertHostDeclarationsFunc MDMAppleBatchInsertHostDeclarationsFunc - MDMAppleBatchInsertHostDeclarationsFuncInvoked bool + MDMAppleBatchSetHostDeclarationStateFunc MDMAppleBatchSetHostDeclarationStateFunc + MDMAppleBatchSetHostDeclarationStateFuncInvoked bool MDMAppleStoreDDMStatusReportFunc MDMAppleStoreDDMStatusReportFunc MDMAppleStoreDDMStatusReportFuncInvoked bool @@ -4790,18 +4785,11 @@ func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identif return s.MDMAppleDDMDeclarationsResponseFunc(ctx, identifier, hostUUID) } -func (s *DataStore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { +func (s *DataStore) MDMAppleBatchSetHostDeclarationState(ctx context.Context) ([]string, error) { s.mu.Lock() - s.MDMAppleGetHostsWithChangedDeclarationsFuncInvoked = true + s.MDMAppleBatchSetHostDeclarationStateFuncInvoked = true s.mu.Unlock() - return s.MDMAppleGetHostsWithChangedDeclarationsFunc(ctx) -} - -func (s *DataStore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { - s.mu.Lock() - s.MDMAppleBatchInsertHostDeclarationsFuncInvoked = true - s.mu.Unlock() - return s.MDMAppleBatchInsertHostDeclarationsFunc(ctx, changedDeclarations) + return s.MDMAppleBatchSetHostDeclarationStateFunc(ctx) } func (s *DataStore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 1a05d51f57..1f19ede881 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2745,47 +2745,24 @@ func ReconcileAppleDeclarations( commander *apple_mdm.MDMAppleCommander, logger kitlog.Logger, ) error { - // once all the declarations are in place, compute the desired state - // and find which hosts need a DDM sync. - changedDeclarations, err := ds.MDMAppleGetHostsWithChangedDeclarations(ctx) + + // batch set declarations as pending + changedHosts, err := ds.MDMAppleBatchSetHostDeclarationState(ctx) if err != nil { - return ctxerr.Wrap(ctx, err, "find hosts with changed declarations") + return ctxerr.Wrap(ctx, err, "updating host declaration state") } - if len(changedDeclarations) == 0 { + if len(changedHosts) == 0 { logger.Log("msg", "no hosts with changed declarations") return nil } - // a host might have more than one declaration to sync, we do this to - // collect unique host UUIDs in order to send a single command to each - // host in the next step - uuidMap := map[string]struct{}{} - for _, d := range changedDeclarations { - uuidMap[d.HostUUID] = struct{}{} - } - uuids := make([]string, 0, len(uuidMap)) - for uuid := range uuidMap { - uuids = append(uuids, uuid) - } - - // mark the host declarations as pending, this serves two purposes: - // - // - support the APIs/methods that track host status (summaries, filters, etc) - // - // - support the DDM endpoints, which use data from the - // `host_mdm_apple_declarations` table to compute which declarations to - // serve - if err := ds.MDMAppleBatchInsertHostDeclarations(ctx, changedDeclarations); err != nil { - return ctxerr.Wrap(ctx, err, "batch insert mdm apple host declarations") - } - // send a DeclarativeManagement command to start a sync - if err := commander.DeclarativeManagement(ctx, uuids, uuid.NewString()); err != nil { + if err := commander.DeclarativeManagement(ctx, changedHosts, uuid.NewString()); err != nil { return ctxerr.Wrap(ctx, err, "issuing DeclarativeManagement command") } - logger.Log("msg", "sent DeclarativeManagement command", "host_number", len(uuids)) + logger.Log("msg", "sent DeclarativeManagement command", "host_number", len(changedHosts)) return nil } diff --git a/server/service/integration_ddm_test.go b/server/service/integration_ddm_test.go new file mode 100644 index 0000000000..e8c77a192d --- /dev/null +++ b/server/service/integration_ddm_test.go @@ -0,0 +1,909 @@ +package service + +import ( + "bytes" + "context" + "crypto/md5" // nolint:gosec // used only for tests + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/pkg/mdm/mdmtest" + "github.com/fleetdm/fleet/v4/server/datastore/mysql" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + kitlog "github.com/go-kit/log" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +func (s *integrationMDMTestSuite) TestAppleDDMBatchUpload() { + t := s.T() + tmpl := ` +{ + "Type": "com.apple.configuration.decl%d", + "Identifier": "com.fleet.config%d", + "Payload": { + "ServiceType": "com.apple.bash", + "DataAssetReference": "com.fleet.asset.bash" %s + } +}` + // TODO: figure out the best way to do this. We might even consider + // starting a different test suite. + t.Cleanup(func() { s.cleanupDeclarations(t) }) + + newDeclBytes := func(i int, payload ...string) []byte { + var p string + if len(payload) > 0 { + p = "," + strings.Join(payload, ",") + } + return []byte(fmt.Sprintf(tmpl, i, i, p)) + } + + var decls [][]byte + + for i := 0; i < 7; i++ { + decls = append(decls, newDeclBytes(i)) + } + + // Non-configuration type should fail + res := s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad", Contents: []byte(`{"Type": "com.apple.activation"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg := extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Only configuration declarations (com.apple.configuration) are supported") + + // "com.apple.configuration.softwareupdate.enforcement.specific" type should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.softwareupdate.enforcement.specific"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Declaration profile can’t include OS updates settings. To control these settings, go to OS updates.") + + // Types from our list of forbidden types should fail + for ft := range fleet.ForbiddenDeclTypes { + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(fmt.Sprintf(`{"Type": "%s"}`, ft))}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Only configuration declarations that don’t require an asset reference are supported.") + } + + // "com.apple.configuration.management.status-subscriptions" type should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.management.status-subscriptions"}`)}, + }}, http.StatusUnprocessableEntity) + + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") + + // Two different payloads with the same name should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "bad2", Contents: newDeclBytes(1, `"foo": "bar"`)}, + {Name: "bad2", Contents: newDeclBytes(2, `"baz": "bing"`)}, + }}, http.StatusUnprocessableEntity) + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "A declaration profile with this name already exists.") + + // Same identifier should fail + res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N1", Contents: decls[0]}, + {Name: "N2", Contents: decls[0]}, + }}, http.StatusUnprocessableEntity) + errMsg = extractServerErrorText(res.Body) + require.Contains(t, errMsg, "A declaration profile with this identifier already exists.") + + // Create 2 declarations + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N1", Contents: decls[0]}, + {Name: "N2", Contents: decls[1]}, + }}, http.StatusNoContent) + + var resp listMDMConfigProfilesResponse + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N1", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N2", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // Create 2 new declarations. These should take the place of the first two. + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: decls[2]}, + {Name: "N4", Contents: decls[3]}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N4", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // replace only 1 declaration, the other one should be the same + + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: decls[2]}, + {Name: "N5", Contents: decls[4]}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N5", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + // update the declarations + + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N3", Contents: newDeclBytes(2, `"foo": "bar"`)}, + {Name: "N5", Contents: newDeclBytes(4, `"bing": "baz"`)}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N3", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N5", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + + var createResp createLabelResponse + s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_1"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) + require.NotZero(t, createResp.Label.ID) + require.Equal(t, "label_1", createResp.Label.Name) + lbl1 := createResp.Label.Label + + s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_2"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) + require.NotZero(t, createResp.Label.ID) + require.Equal(t, "label_2", createResp.Label.Name) + lbl2 := createResp.Label.Label + + // Add with labels + s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ + {Name: "N5", Contents: decls[5], Labels: []string{lbl1.Name, lbl2.Name}}, + {Name: "N6", Contents: decls[6], Labels: []string{lbl1.Name}}, + }}, http.StatusNoContent) + + s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) + + require.Len(t, resp.Profiles, 2) + require.Equal(t, "N5", resp.Profiles[0].Name) + require.Equal(t, "darwin", resp.Profiles[0].Platform) + require.Equal(t, "N6", resp.Profiles[1].Name) + require.Equal(t, "darwin", resp.Profiles[1].Platform) + require.Len(t, resp.Profiles[0].Labels, 2) + require.Equal(t, lbl1.Name, resp.Profiles[0].Labels[0].LabelName) + require.Equal(t, lbl2.Name, resp.Profiles[0].Labels[1].LabelName) + require.Len(t, resp.Profiles[1].Labels, 1) + require.Equal(t, lbl1.Name, resp.Profiles[1].Labels[0].LabelName) +} + +func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { + t := s.T() + _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + calcChecksum := func(source []byte) string { + csum := fmt.Sprintf("%x", md5.Sum(source)) //nolint:gosec + return strings.ToUpper(csum) + } + + t.Cleanup(func() { s.cleanupDeclarations(t) }) + + insertDeclaration := func(t *testing.T, decl fleet.MDMAppleDeclaration) { + stmt := ` +INSERT INTO mdm_apple_declarations ( + declaration_uuid, + team_id, + identifier, + name, + raw_json, + checksum, + created_at, + uploaded_at +) VALUES (?,?,?,?,?,UNHEX(?),?,?)` + + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(context.Background(), stmt, + decl.DeclarationUUID, + decl.TeamID, + decl.Identifier, + decl.Name, + decl.RawJSON, + calcChecksum(decl.RawJSON), + decl.CreatedAt, + decl.UploadedAt, + ) + return err + }) + } + + insertHostDeclaration := func(t *testing.T, hostUUID string, decl fleet.MDMAppleDeclaration) { + stmt := ` +INSERT INTO host_mdm_apple_declarations ( + host_uuid, + status, + operation_type, + checksum, + declaration_uuid +) VALUES (?,?,?,UNHEX(?),?)` + + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(context.Background(), stmt, + hostUUID, + fleet.MDMDeliveryPending, + fleet.MDMOperationTypeInstall, + calcChecksum(decl.RawJSON), + decl.DeclarationUUID, + ) + return err + }) + } + + // initialize a time to use for our first declaration, subsequent declarations will be + // incremented by a minute + then := time.Now().UTC().Truncate(time.Second).Add(-1 * time.Hour) + + // insert a declaration with no team + noTeamDeclsByUUID := map[string]fleet.MDMAppleDeclaration{ + "123": { + DeclarationUUID: "123", + TeamID: ptr.Uint(0), + Identifier: "com.example", + Name: "Example", + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"bar"}, + "Identifier": "com.example" + }`), + CreatedAt: then, + UploadedAt: then, + }, + } + insertDeclaration(t, noTeamDeclsByUUID["123"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["123"]) + + mapDeclsByChecksum := func(byUUID map[string]fleet.MDMAppleDeclaration) map[string]fleet.MDMAppleDeclaration { + byChecksum := make(map[string]fleet.MDMAppleDeclaration) + for _, d := range byUUID { + byChecksum[calcChecksum(d.RawJSON)] = byUUID[d.DeclarationUUID] + } + return byChecksum + } + + parseTokensResp := func(r *http.Response) fleet.MDMAppleDDMTokensResponse { + require.NotNil(t, r) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(b)) + // t.Log("body", string(b)) + + // unmarsal the response to make sure it's valid + var tok fleet.MDMAppleDDMTokensResponse + err = json.NewDecoder(r.Body).Decode(&tok) + require.NoError(t, err) + // t.Log("decoded", tok) + + return tok + } + + parseDeclarationItemsResp := func(r *http.Response) fleet.MDMAppleDDMDeclarationItemsResponse { + require.NotNil(t, r) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + defer r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(b)) + // t.Log("body", string(b)) + + // unmarsal the response to make sure it's valid + var di fleet.MDMAppleDDMDeclarationItemsResponse + err = json.NewDecoder(r.Body).Decode(&di) + require.NoError(t, err) + // t.Log("decoded", di) + + return di + } + + assertDeclarationResponse := func(r *http.Response, expected fleet.MDMAppleDeclaration) { + require.NotNil(t, r) + + // unmarsal the response and assert it's valid + var wantParsed fleet.MDMAppleDDMDeclarationResponse + require.NoError(t, json.Unmarshal(expected.RawJSON, &wantParsed)) + var gotParsed fleet.MDMAppleDDMDeclarationResponse + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed)) + require.EqualValues(t, wantParsed.Payload, gotParsed.Payload) + require.Equal(t, calcChecksum(expected.RawJSON), gotParsed.ServerToken) + require.Equal(t, expected.Identifier, gotParsed.Identifier) + // t.Logf("decoded: %+v", gotParsed) + } + + checkTokensResp := func(t *testing.T, r fleet.MDMAppleDDMTokensResponse, expectedTimestamp time.Time, prevToken string) { + require.Equal(t, expectedTimestamp, r.SyncTokens.Timestamp) + require.NotEmpty(t, r.SyncTokens.DeclarationsToken) + require.NotEqual(t, prevToken, r.SyncTokens.DeclarationsToken) + } + + checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { + require.Equal(t, expectedDeclTok, r.DeclarationsToken) + // TODO(roberto): better assertions + require.NotEmpty(t, r.Declarations.Activations) + require.Empty(t, r.Declarations.Assets) + require.Empty(t, r.Declarations.Management) + require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) + for _, m := range r.Declarations.Configurations { + d, ok := expectedDeclsByChecksum[m.ServerToken] + require.True(t, ok) + require.Equal(t, d.Identifier, m.Identifier) + } + } + + checkRequestsDatabase := func(t *testing.T, messageType, enrollmentID string, expectedCount int) { + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + var count int + if err := sqlx.GetContext( + context.Background(), + q, + &count, + "SELECT count(*) AS count FROM mdm_apple_declarative_requests WHERE enrollment_id = ? AND message_type = ?", + enrollmentID, + messageType, + ); err != nil { + return err + } + + require.Equal(t, expectedCount, count, "unexpected db row count for declaration requests") + + return nil + }) + } + + var currDeclToken string // we'll use this to track the expected token across tests + + t.Run("Tokens", func(t *testing.T) { + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 0) + // get tokens, timestamp should be the same as the declaration and token should be non-empty + r, err := mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + parsed := parseTokensResp(r) + checkTokensResp(t, parsed, then, "") + currDeclToken = parsed.SyncTokens.DeclarationsToken + + // insert a new declaration + noTeamDeclsByUUID["456"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "456", + TeamID: ptr.Uint(0), + Identifier: "com.example2", + Name: "Example2", + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"baz"}, + "Identifier": "com.example2" + }`), + CreatedAt: then.Add(1 * time.Minute), + UploadedAt: then.Add(1 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["456"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["456"]) + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 1) + + // get tokens again, timestamp and token should have changed + r, err = mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + parsed = parseTokensResp(r) + checkTokensResp(t, parsed, then.Add(1*time.Minute), currDeclToken) + currDeclToken = parsed.SyncTokens.DeclarationsToken + checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 2) + }) + + t.Run("DeclarationItems", func(t *testing.T) { + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 0) + r, err := mdmDevice.DeclarativeManagement("declaration-items") + require.NoError(t, err) + checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) + + // insert a new declaration + noTeamDeclsByUUID["789"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "789", + TeamID: ptr.Uint(0), + Identifier: "com.example3", + Name: "Example3", + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.declaration-items.test", + "Payload": {"foo":"bang"}, + "Identifier": "com.example3" + }`), + CreatedAt: then.Add(2 * time.Minute), + UploadedAt: then.Add(2 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["789"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["789"]) + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 1) + + // get tokens again, timestamp and token should have changed + r, err = mdmDevice.DeclarativeManagement("tokens") + require.NoError(t, err) + toks := parseTokensResp(r) + checkTokensResp(t, toks, then.Add(2*time.Minute), currDeclToken) + currDeclToken = toks.SyncTokens.DeclarationsToken + + r, err = mdmDevice.DeclarativeManagement("declaration-items") + require.NoError(t, err) + checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) + checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 2) + }) + + t.Run("Status", func(t *testing.T) { + checkRequestsDatabase(t, "status", mdmDevice.UUID, 0) + _, err := mdmDevice.DeclarativeManagement("status", fleet.MDMAppleDDMStatusReport{}) + require.NoError(t, err) + checkRequestsDatabase(t, "status", mdmDevice.UUID, 1) + }) + + t.Run("Declaration", func(t *testing.T) { + want := noTeamDeclsByUUID["123"] + declarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier) + checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 0) + r, err := mdmDevice.DeclarativeManagement(declarationPath) + require.NoError(t, err) + + assertDeclarationResponse(r, want) + + // insert a new declaration + noTeamDeclsByUUID["abc"] = fleet.MDMAppleDeclaration{ + DeclarationUUID: "abc", + TeamID: ptr.Uint(0), + Identifier: "com.example4", + Name: "Example4", + RawJSON: json.RawMessage(`{ + "Type": "com.apple.configuration.test", + "Payload": {"foo":"bar"}, + "Identifier": "com.example4" + }`), + CreatedAt: then.Add(3 * time.Minute), + UploadedAt: then.Add(3 * time.Minute), + } + insertDeclaration(t, noTeamDeclsByUUID["abc"]) + insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["abc"]) + want = noTeamDeclsByUUID["abc"] + r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) + require.NoError(t, err) + checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 1) + + // try getting a non-existent declaration, should fail 404 + nonExistantDeclarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", "nonexistent") + checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 0) + _, err = mdmDevice.DeclarativeManagement(nonExistantDeclarationPath) + require.Error(t, err) + require.ErrorContains(t, err, "404 Not Found") + checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 1) + + // typo should fail as bad request + typoDeclarationPath := fmt.Sprintf("declarations/%s/%s", "configurations", want.Identifier) + checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 0) + _, err = mdmDevice.DeclarativeManagement(typoDeclarationPath) + require.Error(t, err) + require.ErrorContains(t, err, "400 Bad Request") + checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 1) + + assertDeclarationResponse(r, want) + }) +} + +func (s *integrationMDMTestSuite) TestAppleDDMReconciliation() { + t := s.T() + ctx := context.Background() + // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG + logger := kitlog.NewJSONLogger(os.Stdout) + + // TODO: use endpoints once those are available. + addDeclaration := func(identifier string, teamID uint) { + stmt := ` + INSERT INTO mdm_apple_declarations + (declaration_uuid, team_id, identifier, name, raw_json, checksum) + VALUES + (UUID(), ?, ?, UUID(), ?, HEX(MD5(raw_json)) )` + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, stmt, teamID, identifier, declarationForTest(identifier)) + return err + }) + } + + deleteDeclaration := func(identifier string, teamID uint) { + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM mdm_apple_declarations WHERE team_id = ? AND identifier = ?", teamID, identifier) + return err + }) + } + + // create a team + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + } + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", team, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + team = createTeamResp.Team + + // TODO: figure out the best way to do this. We might even consider + // starting a different test suite. + t.Cleanup(func() { s.cleanupDeclarations(t) }) + + checkNoCommands := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.Nil(t, cmd) + } + + checkDDMSync := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.NotNil(t, cmd) + require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) + cmd, err = d.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + require.Nil(t, cmd) + _, err = d.DeclarativeManagement("tokens") + require.NoError(t, err) + } + + // create a windows host + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 1, + OsqueryHostID: ptr.String("non-macos-host"), + NodeKey: ptr.String("non-macos-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.non.macos", t.Name()), + Platform: "windows", + }) + require.NoError(t, err) + + // create a windows host that's enrolled in MDM + _, _ = createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) + + // create a linux host + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("linux-host"), + NodeKey: ptr.String("linux-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.linux", t.Name()), + Platform: "linux", + }) + require.NoError(t, err) + + // create a host that's not enrolled into MDM + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("not-mdm-enrolled"), + NodeKey: ptr.String("not-mdm-enrolled"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.not.enrolled", t.Name()), + Platform: "darwin", + }) + require.NoError(t, err) + + // create a host and then enroll in MDM. + mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + // trigger the reconciler, no error + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // declarativeManagement command is not sent. + checkNoCommands(device) + + // add global declarations + addDeclaration("I1", 0) + addDeclaration("I2", 0) + + // reconcile again, this time new declarations were added + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // TODO: check command is pending + + // declarativeManagement command is sent + checkDDMSync(device) + + // reconcile again, commands for the uploaded declarations are already sent + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // no new commands are sent + checkNoCommands(device) + + // delete a declaration + deleteDeclaration("I1", 0) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // a DDM sync is triggered + checkDDMSync(device) + + // add a new host + _, deviceTwo := createHostThenEnrollMDM(s.ds, s.server.URL, t) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkDDMSync(deviceTwo) + + // add device to the team + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHost.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // DDM sync is triggered only for the transferred host + // because the team doesn't have any declarations + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // nobody receives commands this time + checkNoCommands(device) + checkNoCommands(deviceTwo) + + // add declarations to the team + addDeclaration("I1", team.ID) + addDeclaration("I2", team.ID) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered for the host in the team + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // add a new host, this one belongs to the team + mdmHostThree, deviceThree := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHostThree.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) + + // no new commands after another reconciliation + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkNoCommands(deviceThree) + + // TODO: use proper APIs for this + // add a new label + label declaration + addDeclaration("I3", team.ID) + label, err := s.ds.NewLabel(ctx, &fleet.Label{Name: t.Name(), Query: "select 1;"}) + require.NoError(t, err) + // update label with host membership + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + "INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, ?)", + mdmHostThree.ID, + label.ID, + ) + return err + }, + ) + + // update declaration <-> label mapping + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + `INSERT INTO + mdm_declaration_labels (apple_declaration_uuid, label_name, label_id) + VALUES ((SELECT declaration_uuid FROM mdm_apple_declarations WHERE team_id = ? and identifier = ?), ?, ?)`, + team.ID, + "I3", + label.Name, + label.ID, + ) + return err + }, + ) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the host with the label + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) +} + +func (s *integrationMDMTestSuite) TestAppleDDMStatusReport() { + t := s.T() + ctx := context.Background() + // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG + logger := kitlog.NewJSONLogger(os.Stdout) + + // TODO: figure out the best way to do this. We might even consider + // starting a different test suite. + t.Cleanup(func() { s.cleanupDeclarations(t) }) + + assertHostDeclarations := func(hostUUID string, wantDecls []*fleet.MDMAppleHostDeclaration) { + var gotDecls []*fleet.MDMAppleHostDeclaration + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.SelectContext(context.Background(), q, &gotDecls, `SELECT declaration_identifier, status, operation_type FROM host_mdm_apple_declarations WHERE host_uuid = ?`, hostUUID) + }) + require.ElementsMatch(t, wantDecls, gotDecls) + } + + // create a host and then enroll in MDM. + mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + declarations := []fleet.MDMProfileBatchPayload{ + {Name: "N1.json", Contents: declarationForTest("I1")}, + {Name: "N2.json", Contents: declarationForTest("I2")}, + } + // add global declarations + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) + + // reconcile profiles + err := ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // declarations are ("install", "pending") after the cron run + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host gets a DDM sync call + cmd, err := device.Idle() + require.NoError(t, err) + require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) + _, err = device.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + + r, err := device.DeclarativeManagement("declaration-items") + require.NoError(t, err) + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var items fleet.MDMAppleDDMDeclarationItemsResponse + require.NoError(t, json.Unmarshal(body, &items)) + + var i1ServerToken, i2ServerToken string + for _, d := range items.Declarations.Configurations { + switch d.Identifier { + case "I1": + i1ServerToken = d.ServerToken + case "I2": + i2ServerToken = d.ServerToken + } + } + + // declarations are ("install", "verifying") after the ack + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a partial DDM report + report := fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a report with a wrong (could be old) server token for I2, nothing changes + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I2", ServerToken: "foo"}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a full report, declaration I2 is invalid + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I2", ServerToken: i2ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // do a batch request, this time I2 is deleted + declarations = []fleet.MDMProfileBatchPayload{ + {Name: "N1.json", Contents: declarationForTest("I1")}, + } + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) + + // reconcile profiles + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, + }) + + // host sends a report, declaration I2 is removed from the hosts_* table + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, + }) + + // host sends a report, declaration I1 is failing after a while + report = fleet.MDMAppleDDMStatusReport{} + report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ + {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I1", ServerToken: i1ServerToken}, + } + _, err = device.DeclarativeManagement("status", report) + require.NoError(t, err) + assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ + {Identifier: "I1", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, + }) +} + +func declarationForTest(identifier string) []byte { + return []byte(fmt.Sprintf(` +{ + "Type": "com.apple.configuration.management.test", + "Payload": { + "Echo": "foo" + }, + "Identifier": "%s" +}`, identifier)) +} + +func (s *integrationMDMTestSuite) cleanupDeclarations(t *testing.T) { + ctx := context.Background() + // TODO: figure out the best way to do this. We might even consider + // starting a different test suite. + // delete declarations to not affect other tests + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM mdm_apple_declarations") + return err + }) + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM host_mdm_apple_declarations") + return err + }) + +} diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 3820aff4f0..5b4d6250c0 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -9259,6 +9259,8 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { testTeam, err := s.ds.NewTeam(ctx, &fleet.Team{Name: "TestTeam"}) require.NoError(t, err) + t.Cleanup(func() { s.cleanupDeclarations(t) }) + assertAppleProfile := func(filename, name, ident string, teamID uint, labelNames []string, wantStatus int, wantErrMsg string) string { fields := map[string][]string{ "labels": labelNames, @@ -12605,868 +12607,3 @@ func (s *integrationMDMTestSuite) TestIsServerBitlockerStatus() { require.Equal(t, fleet.DiskEncryptionEnforcing, *hr.Host.MDM.OSSettings.DiskEncryption.Status) } - -func (s *integrationMDMTestSuite) TestAppleDDMBatchUpload() { - t := s.T() - tmpl := ` -{ - "Type": "com.apple.configuration.decl%d", - "Identifier": "com.fleet.config%d", - "Payload": { - "ServiceType": "com.apple.bash", - "DataAssetReference": "com.fleet.asset.bash" %s - } -}` - - newDeclBytes := func(i int, payload ...string) []byte { - var p string - if len(payload) > 0 { - p = "," + strings.Join(payload, ",") - } - return []byte(fmt.Sprintf(tmpl, i, i, p)) - } - - var decls [][]byte - - for i := 0; i < 7; i++ { - decls = append(decls, newDeclBytes(i)) - } - - // Non-configuration type should fail - res := s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "bad", Contents: []byte(`{"Type": "com.apple.activation"}`)}, - }}, http.StatusUnprocessableEntity) - - errMsg := extractServerErrorText(res.Body) - require.Contains(t, errMsg, "Only configuration declarations (com.apple.configuration) are supported") - - // "com.apple.configuration.softwareupdate.enforcement.specific" type should fail - res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.softwareupdate.enforcement.specific"}`)}, - }}, http.StatusUnprocessableEntity) - - errMsg = extractServerErrorText(res.Body) - require.Contains(t, errMsg, "Declaration profile can’t include OS updates settings. To control these settings, go to OS updates.") - - // Types from our list of forbidden types should fail - for ft := range fleet.ForbiddenDeclTypes { - res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "bad2", Contents: []byte(fmt.Sprintf(`{"Type": "%s"}`, ft))}, - }}, http.StatusUnprocessableEntity) - - errMsg = extractServerErrorText(res.Body) - require.Contains(t, errMsg, "Only configuration declarations that don’t require an asset reference are supported.") - } - - // "com.apple.configuration.management.status-subscriptions" type should fail - res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "bad2", Contents: []byte(`{"Type": "com.apple.configuration.management.status-subscriptions"}`)}, - }}, http.StatusUnprocessableEntity) - - errMsg = extractServerErrorText(res.Body) - require.Contains(t, errMsg, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") - - // Two different payloads with the same name should fail - res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "bad2", Contents: newDeclBytes(1, `"foo": "bar"`)}, - {Name: "bad2", Contents: newDeclBytes(2, `"baz": "bing"`)}, - }}, http.StatusUnprocessableEntity) - errMsg = extractServerErrorText(res.Body) - require.Contains(t, errMsg, "A declaration profile with this name already exists.") - - // Same identifier should fail - res = s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N1", Contents: decls[0]}, - {Name: "N2", Contents: decls[0]}, - }}, http.StatusUnprocessableEntity) - errMsg = extractServerErrorText(res.Body) - require.Contains(t, errMsg, "A declaration profile with this identifier already exists.") - - // Create 2 declarations - s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N1", Contents: decls[0]}, - {Name: "N2", Contents: decls[1]}, - }}, http.StatusNoContent) - - var resp listMDMConfigProfilesResponse - s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) - - require.Len(t, resp.Profiles, 2) - require.Equal(t, "N1", resp.Profiles[0].Name) - require.Equal(t, "darwin", resp.Profiles[0].Platform) - require.Equal(t, "N2", resp.Profiles[1].Name) - require.Equal(t, "darwin", resp.Profiles[1].Platform) - - // Create 2 new declarations. These should take the place of the first two. - s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N3", Contents: decls[2]}, - {Name: "N4", Contents: decls[3]}, - }}, http.StatusNoContent) - - s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) - - require.Len(t, resp.Profiles, 2) - require.Equal(t, "N3", resp.Profiles[0].Name) - require.Equal(t, "darwin", resp.Profiles[0].Platform) - require.Equal(t, "N4", resp.Profiles[1].Name) - require.Equal(t, "darwin", resp.Profiles[1].Platform) - - // replace only 1 declaration, the other one should be the same - - s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N3", Contents: decls[2]}, - {Name: "N5", Contents: decls[4]}, - }}, http.StatusNoContent) - - s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) - - require.Len(t, resp.Profiles, 2) - require.Equal(t, "N3", resp.Profiles[0].Name) - require.Equal(t, "darwin", resp.Profiles[0].Platform) - require.Equal(t, "N5", resp.Profiles[1].Name) - require.Equal(t, "darwin", resp.Profiles[1].Platform) - - // update the declarations - - s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N3", Contents: newDeclBytes(2, `"foo": "bar"`)}, - {Name: "N5", Contents: newDeclBytes(4, `"bing": "baz"`)}, - }}, http.StatusNoContent) - - s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) - - require.Len(t, resp.Profiles, 2) - require.Equal(t, "N3", resp.Profiles[0].Name) - require.Equal(t, "darwin", resp.Profiles[0].Platform) - require.Equal(t, "N5", resp.Profiles[1].Name) - require.Equal(t, "darwin", resp.Profiles[1].Platform) - - var createResp createLabelResponse - s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_1"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) - require.NotZero(t, createResp.Label.ID) - require.Equal(t, "label_1", createResp.Label.Name) - lbl1 := createResp.Label.Label - - s.DoJSON("POST", "/api/latest/fleet/labels", &fleet.LabelPayload{Name: ptr.String("label_2"), Query: ptr.String("select 1")}, http.StatusOK, &createResp) - require.NotZero(t, createResp.Label.ID) - require.Equal(t, "label_2", createResp.Label.Name) - lbl2 := createResp.Label.Label - - // Add with labels - s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{ - {Name: "N5", Contents: decls[5], Labels: []string{lbl1.Name, lbl2.Name}}, - {Name: "N6", Contents: decls[6], Labels: []string{lbl1.Name}}, - }}, http.StatusNoContent) - - s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp) - - require.Len(t, resp.Profiles, 2) - require.Equal(t, "N5", resp.Profiles[0].Name) - require.Equal(t, "darwin", resp.Profiles[0].Platform) - require.Equal(t, "N6", resp.Profiles[1].Name) - require.Equal(t, "darwin", resp.Profiles[1].Platform) - require.Len(t, resp.Profiles[0].Labels, 2) - require.Equal(t, lbl1.Name, resp.Profiles[0].Labels[0].LabelName) - require.Equal(t, lbl2.Name, resp.Profiles[0].Labels[1].LabelName) - require.Len(t, resp.Profiles[1].Labels, 1) - require.Equal(t, lbl1.Name, resp.Profiles[1].Labels[0].LabelName) -} - -// TODO(sarah): Build out this test -func (s *integrationMDMTestSuite) TestMDMAppleDeviceManagementRequests() { - t := s.T() - _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) - - calcChecksum := func(source []byte) string { - csum := fmt.Sprintf("%x", md5.Sum(source)) //nolint:gosec - return strings.ToUpper(csum) - } - - insertDeclaration := func(t *testing.T, decl fleet.MDMAppleDeclaration) { - stmt := ` -INSERT INTO mdm_apple_declarations ( - declaration_uuid, - team_id, - identifier, - name, - raw_json, - checksum, - created_at, - uploaded_at -) VALUES (?,?,?,?,?,UNHEX(?),?,?)` - - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(context.Background(), stmt, - decl.DeclarationUUID, - decl.TeamID, - decl.Identifier, - decl.Name, - decl.RawJSON, - calcChecksum(decl.RawJSON), - decl.CreatedAt, - decl.UploadedAt, - ) - return err - }) - } - - insertHostDeclaration := func(t *testing.T, hostUUID string, decl fleet.MDMAppleDeclaration) { - stmt := ` -INSERT INTO host_mdm_apple_declarations ( - host_uuid, - status, - operation_type, - checksum, - declaration_uuid -) VALUES (?,?,?,UNHEX(?),?)` - - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - _, err := q.ExecContext(context.Background(), stmt, - hostUUID, - fleet.MDMDeliveryPending, - fleet.MDMOperationTypeInstall, - calcChecksum(decl.RawJSON), - decl.DeclarationUUID, - ) - return err - }) - } - - // initialize a time to use for our first declaration, subsequent declarations will be - // incremented by a minute - then := time.Now().UTC().Truncate(time.Second).Add(-1 * time.Hour) - - // insert a declaration with no team - noTeamDeclsByUUID := map[string]fleet.MDMAppleDeclaration{ - "123": { - DeclarationUUID: "123", - TeamID: ptr.Uint(0), - Identifier: "com.example", - Name: "Example", - RawJSON: json.RawMessage(`{ - "Type": "com.apple.configuration.declaration-items.test", - "Payload": {"foo":"bar"}, - "Identifier": "com.example" - }`), - CreatedAt: then, - UploadedAt: then, - }, - } - insertDeclaration(t, noTeamDeclsByUUID["123"]) - insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["123"]) - - mapDeclsByChecksum := func(byUUID map[string]fleet.MDMAppleDeclaration) map[string]fleet.MDMAppleDeclaration { - byChecksum := make(map[string]fleet.MDMAppleDeclaration) - for _, d := range byUUID { - byChecksum[calcChecksum(d.RawJSON)] = byUUID[d.DeclarationUUID] - } - return byChecksum - } - - parseTokensResp := func(r *http.Response) fleet.MDMAppleDDMTokensResponse { - require.NotNil(t, r) - b, err := io.ReadAll(r.Body) - require.NoError(t, err) - defer r.Body.Close() - r.Body = io.NopCloser(bytes.NewBuffer(b)) - // t.Log("body", string(b)) - - // unmarsal the response to make sure it's valid - var tok fleet.MDMAppleDDMTokensResponse - err = json.NewDecoder(r.Body).Decode(&tok) - require.NoError(t, err) - // t.Log("decoded", tok) - - return tok - } - - parseDeclarationItemsResp := func(r *http.Response) fleet.MDMAppleDDMDeclarationItemsResponse { - require.NotNil(t, r) - b, err := io.ReadAll(r.Body) - require.NoError(t, err) - defer r.Body.Close() - r.Body = io.NopCloser(bytes.NewBuffer(b)) - // t.Log("body", string(b)) - - // unmarsal the response to make sure it's valid - var di fleet.MDMAppleDDMDeclarationItemsResponse - err = json.NewDecoder(r.Body).Decode(&di) - require.NoError(t, err) - // t.Log("decoded", di) - - return di - } - - assertDeclarationResponse := func(r *http.Response, expected fleet.MDMAppleDeclaration) { - require.NotNil(t, r) - - // unmarsal the response and assert it's valid - var wantParsed fleet.MDMAppleDDMDeclarationResponse - require.NoError(t, json.Unmarshal(expected.RawJSON, &wantParsed)) - var gotParsed fleet.MDMAppleDDMDeclarationResponse - require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed)) - require.EqualValues(t, wantParsed.Payload, gotParsed.Payload) - require.Equal(t, calcChecksum(expected.RawJSON), gotParsed.ServerToken) - require.Equal(t, expected.Identifier, gotParsed.Identifier) - // t.Logf("decoded: %+v", gotParsed) - } - - checkTokensResp := func(t *testing.T, r fleet.MDMAppleDDMTokensResponse, expectedTimestamp time.Time, prevToken string) { - require.Equal(t, expectedTimestamp, r.SyncTokens.Timestamp) - require.NotEmpty(t, r.SyncTokens.DeclarationsToken) - require.NotEqual(t, prevToken, r.SyncTokens.DeclarationsToken) - } - - checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { - require.Equal(t, expectedDeclTok, r.DeclarationsToken) - // TODO(roberto): better assertions - require.NotEmpty(t, r.Declarations.Activations) - require.Empty(t, r.Declarations.Assets) - require.Empty(t, r.Declarations.Management) - require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) - for _, m := range r.Declarations.Configurations { - d, ok := expectedDeclsByChecksum[m.ServerToken] - require.True(t, ok) - require.Equal(t, d.Identifier, m.Identifier) - } - } - - checkRequestsDatabase := func(t *testing.T, messageType, enrollmentID string, expectedCount int) { - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - var count int - if err := sqlx.GetContext( - context.Background(), - q, - &count, - "SELECT count(*) AS count FROM mdm_apple_declarative_requests WHERE enrollment_id = ? AND message_type = ?", - enrollmentID, - messageType, - ); err != nil { - return err - } - - require.Equal(t, expectedCount, count, "unexpected db row count for declaration requests") - - return nil - }) - } - - var currDeclToken string // we'll use this to track the expected token across tests - - t.Run("Tokens", func(t *testing.T) { - checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 0) - // get tokens, timestamp should be the same as the declaration and token should be non-empty - r, err := mdmDevice.DeclarativeManagement("tokens") - require.NoError(t, err) - parsed := parseTokensResp(r) - checkTokensResp(t, parsed, then, "") - currDeclToken = parsed.SyncTokens.DeclarationsToken - - // insert a new declaration - noTeamDeclsByUUID["456"] = fleet.MDMAppleDeclaration{ - DeclarationUUID: "456", - TeamID: ptr.Uint(0), - Identifier: "com.example2", - Name: "Example2", - RawJSON: json.RawMessage(`{ - "Type": "com.apple.configuration.declaration-items.test", - "Payload": {"foo":"baz"}, - "Identifier": "com.example2" - }`), - CreatedAt: then.Add(1 * time.Minute), - UploadedAt: then.Add(1 * time.Minute), - } - insertDeclaration(t, noTeamDeclsByUUID["456"]) - insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["456"]) - checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 1) - - // get tokens again, timestamp and token should have changed - r, err = mdmDevice.DeclarativeManagement("tokens") - require.NoError(t, err) - parsed = parseTokensResp(r) - checkTokensResp(t, parsed, then.Add(1*time.Minute), currDeclToken) - currDeclToken = parsed.SyncTokens.DeclarationsToken - checkRequestsDatabase(t, "tokens", mdmDevice.UUID, 2) - }) - - t.Run("DeclarationItems", func(t *testing.T) { - checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 0) - r, err := mdmDevice.DeclarativeManagement("declaration-items") - require.NoError(t, err) - checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) - - // insert a new declaration - noTeamDeclsByUUID["789"] = fleet.MDMAppleDeclaration{ - DeclarationUUID: "789", - TeamID: ptr.Uint(0), - Identifier: "com.example3", - Name: "Example3", - RawJSON: json.RawMessage(`{ - "Type": "com.apple.configuration.declaration-items.test", - "Payload": {"foo":"bang"}, - "Identifier": "com.example3" - }`), - CreatedAt: then.Add(2 * time.Minute), - UploadedAt: then.Add(2 * time.Minute), - } - insertDeclaration(t, noTeamDeclsByUUID["789"]) - insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["789"]) - checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 1) - - // get tokens again, timestamp and token should have changed - r, err = mdmDevice.DeclarativeManagement("tokens") - require.NoError(t, err) - toks := parseTokensResp(r) - checkTokensResp(t, toks, then.Add(2*time.Minute), currDeclToken) - currDeclToken = toks.SyncTokens.DeclarationsToken - - r, err = mdmDevice.DeclarativeManagement("declaration-items") - require.NoError(t, err) - checkDeclarationItemsResp(t, parseDeclarationItemsResp(r), currDeclToken, mapDeclsByChecksum(noTeamDeclsByUUID)) - checkRequestsDatabase(t, "declaration-items", mdmDevice.UUID, 2) - }) - - t.Run("Status", func(t *testing.T) { - checkRequestsDatabase(t, "status", mdmDevice.UUID, 0) - _, err := mdmDevice.DeclarativeManagement("status", fleet.MDMAppleDDMStatusReport{}) - require.NoError(t, err) - checkRequestsDatabase(t, "status", mdmDevice.UUID, 1) - }) - - t.Run("Declaration", func(t *testing.T) { - want := noTeamDeclsByUUID["123"] - declarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier) - checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 0) - r, err := mdmDevice.DeclarativeManagement(declarationPath) - require.NoError(t, err) - - assertDeclarationResponse(r, want) - - // insert a new declaration - noTeamDeclsByUUID["abc"] = fleet.MDMAppleDeclaration{ - DeclarationUUID: "abc", - TeamID: ptr.Uint(0), - Identifier: "com.example4", - Name: "Example4", - RawJSON: json.RawMessage(`{ - "Type": "com.apple.configuration.test", - "Payload": {"foo":"bar"}, - "Identifier": "com.example4" - }`), - CreatedAt: then.Add(3 * time.Minute), - UploadedAt: then.Add(3 * time.Minute), - } - insertDeclaration(t, noTeamDeclsByUUID["abc"]) - insertHostDeclaration(t, mdmDevice.UUID, noTeamDeclsByUUID["abc"]) - want = noTeamDeclsByUUID["abc"] - r, err = mdmDevice.DeclarativeManagement(fmt.Sprintf("declaration/%s/%s", "configuration", want.Identifier)) - require.NoError(t, err) - checkRequestsDatabase(t, declarationPath, mdmDevice.UUID, 1) - - // try getting a non-existent declaration, should fail 404 - nonExistantDeclarationPath := fmt.Sprintf("declaration/%s/%s", "configuration", "nonexistent") - checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 0) - _, err = mdmDevice.DeclarativeManagement(nonExistantDeclarationPath) - require.Error(t, err) - require.ErrorContains(t, err, "404 Not Found") - checkRequestsDatabase(t, nonExistantDeclarationPath, mdmDevice.UUID, 1) - - // typo should fail as bad request - typoDeclarationPath := fmt.Sprintf("declarations/%s/%s", "configurations", want.Identifier) - checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 0) - _, err = mdmDevice.DeclarativeManagement(typoDeclarationPath) - require.Error(t, err) - require.ErrorContains(t, err, "400 Bad Request") - checkRequestsDatabase(t, typoDeclarationPath, mdmDevice.UUID, 1) - - assertDeclarationResponse(r, want) - }) -} - -func (s *integrationMDMTestSuite) TestAppleDDMReconciliation() { - t := s.T() - ctx := context.Background() - // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG - logger := kitlog.NewJSONLogger(os.Stdout) - - // TODO: use endpoints once those are available. - addDeclaration := func(identifier string, teamID uint) { - stmt := ` - INSERT INTO mdm_apple_declarations - (declaration_uuid, team_id, identifier, name, raw_json, checksum) - VALUES - (UUID(), ?, ?, UUID(), ?, HEX(MD5(raw_json)) )` - mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, stmt, teamID, identifier, declarationForTest(identifier)) - return err - }) - } - - deleteDeclaration := func(identifier string, teamID uint) { - mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, "DELETE FROM mdm_apple_declarations WHERE team_id = ? AND identifier = ?", teamID, identifier) - return err - }) - } - - // create a team - teamName := t.Name() + "team1" - team := &fleet.Team{ - Name: teamName, - } - var createTeamResp teamResponse - s.DoJSON("POST", "/api/latest/fleet/teams", team, http.StatusOK, &createTeamResp) - require.NotZero(t, createTeamResp.Team.ID) - team = createTeamResp.Team - - t.Cleanup(func() { - // delete declarations to not affect other tests - deleteDeclaration("I2", 0) - deleteDeclaration("I1", team.ID) - deleteDeclaration("I2", team.ID) - deleteDeclaration("I3", team.ID) - }) - - checkNoCommands := func(d *mdmtest.TestAppleMDMClient) { - cmd, err := d.Idle() - require.NoError(t, err) - require.Nil(t, cmd) - } - - checkDDMSync := func(d *mdmtest.TestAppleMDMClient) { - cmd, err := d.Idle() - require.NoError(t, err) - require.NotNil(t, cmd) - require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) - cmd, err = d.Acknowledge(cmd.CommandUUID) - require.NoError(t, err) - require.Nil(t, cmd) - _, err = d.DeclarativeManagement("tokens") - require.NoError(t, err) - } - - // create a windows host - _, err := s.ds.NewHost(context.Background(), &fleet.Host{ - ID: 1, - OsqueryHostID: ptr.String("non-macos-host"), - NodeKey: ptr.String("non-macos-host"), - UUID: uuid.New().String(), - Hostname: fmt.Sprintf("%sfoo.local.non.macos", t.Name()), - Platform: "windows", - }) - require.NoError(t, err) - - // create a windows host that's enrolled in MDM - _, _ = createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) - - // create a linux host - _, err = s.ds.NewHost(context.Background(), &fleet.Host{ - ID: 2, - OsqueryHostID: ptr.String("linux-host"), - NodeKey: ptr.String("linux-host"), - UUID: uuid.New().String(), - Hostname: fmt.Sprintf("%sfoo.local.linux", t.Name()), - Platform: "linux", - }) - require.NoError(t, err) - - // create a host that's not enrolled into MDM - _, err = s.ds.NewHost(context.Background(), &fleet.Host{ - ID: 2, - OsqueryHostID: ptr.String("not-mdm-enrolled"), - NodeKey: ptr.String("not-mdm-enrolled"), - UUID: uuid.New().String(), - Hostname: fmt.Sprintf("%sfoo.local.not.enrolled", t.Name()), - Platform: "darwin", - }) - require.NoError(t, err) - - // create a host and then enroll in MDM. - mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) - - // trigger the reconciler, no error - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - - // declarativeManagement command is not sent. - checkNoCommands(device) - - // add global declarations - addDeclaration("I1", 0) - addDeclaration("I2", 0) - - // reconcile again, this time new declarations were added - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - - // TODO: check command is pending - - // declarativeManagement command is sent - checkDDMSync(device) - - // reconcile again, commands for the uploaded declarations are already sent - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // no new commands are sent - checkNoCommands(device) - - // delete a declaration - deleteDeclaration("I1", 0) - // reconcile again - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // a DDM sync is triggered - checkDDMSync(device) - - // add a new host - _, deviceTwo := createHostThenEnrollMDM(s.ds, s.server.URL, t) - // reconcile again - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // DDM sync is triggered only for the new host - checkNoCommands(device) - checkDDMSync(deviceTwo) - - // add device to the team - s.Do("POST", "/api/v1/fleet/hosts/transfer", - addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHost.ID}}, http.StatusOK) - - // reconcile - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - - // DDM sync is triggered only for the transferred host - // because the team doesn't have any declarations - checkDDMSync(device) - checkNoCommands(deviceTwo) - - // reconcile - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // nobody receives commands this time - checkNoCommands(device) - checkNoCommands(deviceTwo) - - // add declarations to the team - addDeclaration("I1", team.ID) - addDeclaration("I2", team.ID) - - // reconcile - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // DDM sync is triggered for the host in the team - checkDDMSync(device) - checkNoCommands(deviceTwo) - - // add a new host, this one belongs to the team - mdmHostThree, deviceThree := createHostThenEnrollMDM(s.ds, s.server.URL, t) - s.Do("POST", "/api/v1/fleet/hosts/transfer", - addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHostThree.ID}}, http.StatusOK) - - // reconcile - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // DDM sync is triggered only for the new host - checkNoCommands(device) - checkNoCommands(deviceTwo) - checkDDMSync(deviceThree) - - // no new commands after another reconciliation - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - checkNoCommands(device) - checkNoCommands(deviceTwo) - checkNoCommands(deviceThree) - - // TODO: use proper APIs for this - // add a new label + label declaration - addDeclaration("I3", team.ID) - label, err := s.ds.NewLabel(ctx, &fleet.Label{Name: t.Name(), Query: "select 1;"}) - require.NoError(t, err) - // update label with host membership - mysql.ExecAdhocSQL( - t, s.ds, func(db sqlx.ExtContext) error { - _, err := db.ExecContext( - context.Background(), - "INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, ?)", - mdmHostThree.ID, - label.ID, - ) - return err - }, - ) - - // update declaration <-> label mapping - mysql.ExecAdhocSQL( - t, s.ds, func(db sqlx.ExtContext) error { - _, err := db.ExecContext( - context.Background(), - `INSERT INTO - mdm_declaration_labels (apple_declaration_uuid, label_name, label_id) - VALUES ((SELECT declaration_uuid FROM mdm_apple_declarations WHERE team_id = ? and identifier = ?), ?, ?)`, - team.ID, - "I3", - label.Name, - label.ID, - ) - return err - }, - ) - - // reconcile - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - // DDM sync is triggered only for the host with the label - checkNoCommands(device) - checkNoCommands(deviceTwo) - checkDDMSync(deviceThree) -} - -func (s *integrationMDMTestSuite) TestAppleDDMStatusReport() { - t := s.T() - ctx := context.Background() - // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG - logger := kitlog.NewJSONLogger(os.Stdout) - - assertHostDeclarations := func(hostUUID string, wantDecls []*fleet.MDMAppleHostDeclaration) { - var gotDecls []*fleet.MDMAppleHostDeclaration - mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { - return sqlx.SelectContext(context.Background(), q, &gotDecls, `SELECT declaration_identifier, status, operation_type FROM host_mdm_apple_declarations WHERE host_uuid = ?`, hostUUID) - }) - require.ElementsMatch(t, wantDecls, gotDecls) - } - - // create a host and then enroll in MDM. - mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) - - declarations := []fleet.MDMProfileBatchPayload{ - {Name: "N1.json", Contents: declarationForTest("I1")}, - {Name: "N2.json", Contents: declarationForTest("I2")}, - } - // add global declarations - s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) - - // reconcile profiles - err := ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - - // declarations are ("install", "pending") after the cron run - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // host gets a DDM sync call - cmd, err := device.Idle() - require.NoError(t, err) - require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) - _, err = device.Acknowledge(cmd.CommandUUID) - require.NoError(t, err) - - r, err := device.DeclarativeManagement("declaration-items") - require.NoError(t, err) - body, err := io.ReadAll(r.Body) - require.NoError(t, err) - var items fleet.MDMAppleDDMDeclarationItemsResponse - require.NoError(t, json.Unmarshal(body, &items)) - - var i1ServerToken, i2ServerToken string - for _, d := range items.Declarations.Configurations { - switch d.Identifier { - case "I1": - i1ServerToken = d.ServerToken - case "I2": - i2ServerToken = d.ServerToken - } - } - - // declarations are ("install", "verifying") after the ack - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // host sends a partial DDM report - report := fleet.MDMAppleDDMStatusReport{} - report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ - {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, - } - _, err = device.DeclarativeManagement("status", report) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // host sends a report with a wrong (could be old) server token for I2, nothing changes - report = fleet.MDMAppleDDMStatusReport{} - report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ - {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I2", ServerToken: "foo"}, - } - _, err = device.DeclarativeManagement("status", report) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryVerifying, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // host sends a full report, declaration I2 is invalid - report = fleet.MDMAppleDDMStatusReport{} - report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ - {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, - {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I2", ServerToken: i2ServerToken}, - } - _, err = device.DeclarativeManagement("status", report) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // do a batch request, this time I2 is deleted - declarations = []fleet.MDMProfileBatchPayload{ - {Name: "N1.json", Contents: declarationForTest("I1")}, - } - s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: declarations}, http.StatusNoContent) - - // reconcile profiles - err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, - {Identifier: "I2", Status: &fleet.MDMDeliveryPending, OperationType: fleet.MDMOperationTypeRemove}, - }) - - // host sends a report, declaration I2 is removed from the hosts_* table - report = fleet.MDMAppleDDMStatusReport{} - report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ - {Active: true, Valid: fleet.MDMAppleDeclarationValid, Identifier: "I1", ServerToken: i1ServerToken}, - } - _, err = device.DeclarativeManagement("status", report) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryVerified, OperationType: fleet.MDMOperationTypeInstall}, - }) - - // host sends a report, declaration I1 is failing after a while - report = fleet.MDMAppleDDMStatusReport{} - report.StatusItems.Management.Declarations.Configurations = []fleet.MDMAppleDDMStatusDeclaration{ - {Active: false, Valid: fleet.MDMAppleDeclarationInvalid, Identifier: "I1", ServerToken: i1ServerToken}, - } - _, err = device.DeclarativeManagement("status", report) - require.NoError(t, err) - assertHostDeclarations(mdmHost.UUID, []*fleet.MDMAppleHostDeclaration{ - {Identifier: "I1", Status: &fleet.MDMDeliveryFailed, OperationType: fleet.MDMOperationTypeInstall}, - }) -} - -func declarationForTest(identifier string) []byte { - return []byte(fmt.Sprintf(` -{ - "Type": "com.apple.configuration.management.test", - "Payload": { - "Echo": "foo" - }, - "Identifier": "%s" -}`, identifier)) -} From 88b6ac9b179f995cb13b9812ab37e1dd64e0e04e Mon Sep 17 00:00:00 2001 From: Gabriel Hernandez Date: Wed, 27 Mar 2024 13:46:00 +0000 Subject: [PATCH 22/29] Feat UI ddm integration (#17885) relate to #17416 update to uploading, downloading, and deleting ddm profiles. - [x] Manual QA for all new/changed functionality --- .../ProfileListItem/ProfileListItem.tsx | 21 +++++++++++++--- .../components/AddProfileModal.tsx | 2 +- .../components/ProfileUploader/helpers.tsx | 3 +++ .../HostDetailsPage/HostDetailsPage.tsx | 24 ------------------- 4 files changed, 22 insertions(+), 28 deletions(-) diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx index 51c185c521..b61ea7c2f0 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileListItem/ProfileListItem.tsx @@ -53,6 +53,21 @@ const ProfileDetails = ({ ); }; +const createProfileExtension = (profile: IMdmProfile) => { + if (isDDMProfile(profile)) { + return "json"; + } + return profile.platform === "darwin" ? "mobileconfig" : "xml"; +}; + +const createFileContent = async (profile: IMdmProfile) => { + const content = await mdmAPI.downloadProfile(profile.profile_uuid); + if (isDDMProfile(profile)) { + return JSON.stringify(content, null, 2); + } + return content; +}; + interface IProfileListItemProps { isPremium: boolean; profile: IMdmProfile; @@ -68,13 +83,13 @@ const ProfileListItem = ({ onDelete, setProfileLabelsModalData, }: IProfileListItemProps) => { - const { created_at, labels, name, platform, profile_uuid } = profile; + const { created_at, labels, name, platform } = profile; const subClass = "list-item"; const onClickDownload = async () => { - const fileContent = await mdmAPI.downloadProfile(profile_uuid); + const fileContent = await createFileContent(profile); const formatDate = format(new Date(), "yyyy-MM-dd"); - const extension = platform === "darwin" ? "mobileconfig" : "xml"; + const extension = createProfileExtension(profile); const filename = `${formatDate}_${name}.${extension}`; const file = new File([fileContent], filename); FileSaver.saveAs(file); diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx index 883479185a..403ef83c16 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/components/AddProfileModal.tsx @@ -52,7 +52,7 @@ const FileChooser = ({ { diff --git a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/helpers.tsx b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/helpers.tsx index 3f12067abf..bef5542e71 100644 --- a/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/helpers.tsx +++ b/frontend/pages/ManageControlsPage/OSSettings/cards/CustomSettings/components/ProfileUploader/helpers.tsx @@ -78,6 +78,9 @@ export const parseFile = async (file: File): Promise<[string, string]> => { // } return [name, "macOS"]; } + case "json": { + return [name, "macOS"]; + } default: { throw new Error(`Invalid file type: ${ext}`); } diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 11b2178bcf..4f03cb495a 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -753,30 +753,6 @@ const HostDetailsPage = ({ name: host?.mdm.macos_setup?.bootstrap_package_name, }; - // TODO: Remove this when API is ready - if (!host.mdm.profiles) { - host.mdm.profiles = []; - } else { - host.mdm.profiles = [ - createMockHostMdmProfile({ - name: "test.json", - status: "success", - }), - createMockHostMdmProfile({ - name: "test2.json", - status: "pending", - }), - createMockHostMdmProfile({ - name: "test3.json", - status: "failed", - }), - createMockHostMdmProfile({ - name: "test4.json", - status: "acknowledged", - }), - ]; - } - return ( <> From 15e671da9a20b0bd36c3a13e4c5e624e51e2e6bc Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 27 Mar 2024 11:04:47 -0300 Subject: [PATCH 23/29] add missing activities --- server/fleet/activities.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/fleet/activities.go b/server/fleet/activities.go index 8dacdf6862..abd4835a4d 100644 --- a/server/fleet/activities.go +++ b/server/fleet/activities.go @@ -84,6 +84,9 @@ var ActivityDetailsList = []ActivityDetails{ ActivityTypeLockedHost{}, ActivityTypeUnlockedHost{}, ActivityTypeWipedHost{}, + + ActivityTypeCreatedDeclarationProfile{}, + ActivityTypeDeletedDeclarationProfile{}, } type ActivityDetails interface { From 96d67af9d86affc02cbe5dff00cb2d83689f1202 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Wed, 27 Mar 2024 08:17:12 -0700 Subject: [PATCH 24/29] Handbook: Update vulnerability-management.ejs (#17853) Thanks Paddy Harrington for the tip! --------- Co-authored-by: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> --- website/views/pages/vulnerability-management.ejs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/views/pages/vulnerability-management.ejs b/website/views/pages/vulnerability-management.ejs index 27c0205659..6971d60fe4 100644 --- a/website/views/pages/vulnerability-management.ejs +++ b/website/views/pages/vulnerability-management.ejs @@ -115,9 +115,10 @@

Up-to-date data without scans

-

Traditional network vulnerability scans can clog networks. Fleet does things differently.

+

Traditional network vulnerability scans can clog your network and even haunt your printers with pages full of wingdings. Fleet does things differently.

-

Lightweight enough for the most brittle environments (OT, data centers, embedded/BTS, low-latency gaming servers).

+

Eliminate the risk of side effects from scanning the network.

+

Lightweight enough for the most brittle environments (OT, data centers, embedded/BTS, low-latency gaming servers).

Quickly pull data about important CVEs and zero days during an incident or audit.

From 0d15132ae4e4f7a8729a02ca77686b663f3f9e6e Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Wed, 27 Mar 2024 12:21:08 -0300 Subject: [PATCH 25/29] fix tests and lint --- server/service/integration_mdm_dep_test.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/server/service/integration_mdm_dep_test.go b/server/service/integration_mdm_dep_test.go index 4e83e3c0fe..e3f36ea5d7 100644 --- a/server/service/integration_mdm_dep_test.go +++ b/server/service/integration_mdm_dep_test.go @@ -25,6 +25,7 @@ import ( "github.com/fleetdm/fleet/v4/server/worker" kitlog "github.com/go-kit/log" "github.com/google/uuid" + "github.com/groob/plist" "github.com/jmoiron/sqlx" micromdm "github.com/micromdm/micromdm/mdm/mdm" "github.com/stretchr/testify/require" @@ -264,7 +265,9 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de //default: // fmt.Println(">>>> device received command: ", cmd.CommandUUID, cmd.Command.RequestType) //} - cmds = append(cmds, cmd) + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + cmds = append(cmds, &fullCmd) cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) require.NoError(t, err) } @@ -327,7 +330,9 @@ func (s *integrationMDMTestSuite) runDEPEnrollReleaseDeviceTest(t *testing.T, de cmd, err = mdmDevice.Idle() require.NoError(t, err) for cmd != nil { - cmds = append(cmds, cmd) + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + cmds = append(cmds, &fullCmd) cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) require.NoError(t, err) } @@ -374,12 +379,14 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() { cmd, err := mdmDevice.Idle() require.NoError(t, err) for cmd != nil { - if cmd.Command.RequestType == "InstallEnterpriseApplication" && - cmd.Command.InstallEnterpriseApplication.ManifestURL != nil && - strings.Contains(*cmd.Command.InstallEnterpriseApplication.ManifestURL, apple_mdm.FleetdPublicManifestURL) { - fleetdCmd = cmd + var fullCmd micromdm.CommandPayload + require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd)) + if fullCmd.Command.RequestType == "InstallEnterpriseApplication" && + fullCmd.Command.InstallEnterpriseApplication.ManifestURL != nil && + strings.Contains(*fullCmd.Command.InstallEnterpriseApplication.ManifestURL, apple_mdm.FleetdPublicManifestURL) { + fleetdCmd = &fullCmd } else if cmd.Command.RequestType == "InstallProfile" { - installProfileCmd = cmd + installProfileCmd = &fullCmd } cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) require.NoError(t, err) From 1777b9c5d1c6169efe1149cf537d39192058872a Mon Sep 17 00:00:00 2001 From: Noah Talerman <47070608+noahtalerman@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:21:19 -0400 Subject: [PATCH 26/29] Update macos-install-wine.sh (#17828) - Add note about running the script in user context - Add redirect so that we can change the link later --- orbit/pkg/packaging/windows.go | 2 +- scripts/macos-install-wine.sh | 3 ++- website/config/routes.js | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/orbit/pkg/packaging/windows.go b/orbit/pkg/packaging/windows.go index 731e57efe0..b35a00c5bb 100644 --- a/orbit/pkg/packaging/windows.go +++ b/orbit/pkg/packaging/windows.go @@ -222,7 +222,7 @@ func checkWine(wineChecked bool) error { cmd := exec.Command(wix.WineCmd, "--version") if err := cmd.Run(); err != nil { return fmt.Errorf( - "%s failed. Is Wine installed? Creating a fleetd agent for Windows (.msi) requires Wine. To install Wine see the script here: https://github.com/fleetdm/fleet/blob/fleet-v4.44.0/scripts/macos-install-wine.sh %w", + "%s failed. Is Wine installed? Creating a fleetd agent for Windows (.msi) requires Wine. To install Wine see the script here: https://fleetdm.com/install-wine %w", wix.WineCmd, err, ) } diff --git a/scripts/macos-install-wine.sh b/scripts/macos-install-wine.sh index fd5c55532b..5a29f352d2 100755 --- a/scripts/macos-install-wine.sh +++ b/scripts/macos-install-wine.sh @@ -2,8 +2,9 @@ set -eo pipefail +# Run this script in user context (not root). # Reference: https://wiki.winehq.org/MacOS -# NOTE: Wine can be installed without brew via a distribution such as https://github.com/Gcenx/macOS_Wine_builds/releases/tag/9.0, or by building from source. +# Wine can be installed without brew via a distribution such as https://github.com/Gcenx/macOS_Wine_builds/releases/tag/9.0, or by building from source. # Check if brew is installed if ! command -v brew >/dev/null 2>&1 ; then diff --git a/website/config/routes.js b/website/config/routes.js index 4a69f469ae..6a486f1db6 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -511,6 +511,7 @@ module.exports.routes = { 'GET /learn-more-about/enrolling-hosts': '/docs/using-fleet/adding-hosts', 'GET /learn-more-about/setup-assistant': '/docs/using-fleet/mdm-macos-setup-experience#macos-setup-assistant', 'GET /learn-more-about/policy-automations': '/docs/using-fleet/automations', + 'GET /install-wine': 'https://github.com/fleetdm/fleet/blob/main/scripts/macos-install-wine.sh', // Sitemap // ============================================================================================================= From f2b3a58bff85002cbbf7ce24c3701c48b4361451 Mon Sep 17 00:00:00 2001 From: George Karr Date: Wed, 27 Mar 2024 10:41:51 -0500 Subject: [PATCH 27/29] Updating changelog for 4.47.3 (#17871) --- CHANGELOG.md | 6 ++++++ charts/fleet/Chart.yaml | 2 +- charts/fleet/values.yaml | 2 +- infrastructure/dogfood/terraform/aws/variables.tf | 2 +- infrastructure/dogfood/terraform/gcp/variables.tf | 2 +- terraform/byo-vpc/README.md | 2 +- terraform/byo-vpc/byo-db/byo-ecs/variables.tf | 2 +- terraform/byo-vpc/byo-db/variables.tf | 2 +- terraform/byo-vpc/example/main.tf | 2 +- terraform/byo-vpc/variables.tf | 2 +- terraform/example/main.tf | 4 ++-- terraform/variables.tf | 2 +- tools/fleetctl-npm/package.json | 2 +- 13 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7115220bd..521746d6c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## Fleet 4.47.3 (Mar 26, 2024) + +### Bug fixes + +* Fixed a bug where valid Windows MDM enrollments would show up as unmanaged (EnrollmentState 3). + ## Fleet 4.47.2 (Mar 22, 2024) ### Bug fixes diff --git a/charts/fleet/Chart.yaml b/charts/fleet/Chart.yaml index 92a097ab51..f0558d302e 100644 --- a/charts/fleet/Chart.yaml +++ b/charts/fleet/Chart.yaml @@ -8,7 +8,7 @@ version: v6.0.2 home: https://github.com/fleetdm/fleet sources: - https://github.com/fleetdm/fleet.git -appVersion: v4.47.2 +appVersion: v4.47.3 dependencies: - name: mysql condition: mysql.enabled diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 5c9e45f593..af4099168f 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -2,7 +2,7 @@ # All settings related to how Fleet is deployed in Kubernetes hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy -imageTag: v4.47.2 # Version of Fleet to deploy +imageTag: v4.47.3 # Version of Fleet to deploy podAnnotations: {} # Additional annotations to add to the Fleet pod serviceAccountAnnotations: {} # Additional annotations to add to the Fleet service account resources: diff --git a/infrastructure/dogfood/terraform/aws/variables.tf b/infrastructure/dogfood/terraform/aws/variables.tf index 2775e36c4e..991c9135de 100644 --- a/infrastructure/dogfood/terraform/aws/variables.tf +++ b/infrastructure/dogfood/terraform/aws/variables.tf @@ -56,7 +56,7 @@ variable "database_name" { variable "fleet_image" { description = "the name of the container image to run" - default = "fleetdm/fleet:v4.47.2" + default = "fleetdm/fleet:v4.47.3" } variable "software_inventory" { diff --git a/infrastructure/dogfood/terraform/gcp/variables.tf b/infrastructure/dogfood/terraform/gcp/variables.tf index 98f012a0cd..b3cddde5ca 100644 --- a/infrastructure/dogfood/terraform/gcp/variables.tf +++ b/infrastructure/dogfood/terraform/gcp/variables.tf @@ -68,5 +68,5 @@ variable "redis_mem" { } variable "image" { - default = "fleet:v4.47.2" + default = "fleet:v4.47.3" } diff --git a/terraform/byo-vpc/README.md b/terraform/byo-vpc/README.md index 902b942aa2..27920c1819 100644 --- a/terraform/byo-vpc/README.md +++ b/terraform/byo-vpc/README.md @@ -33,7 +33,7 @@ No requirements. |------|-------------|------|---------|:--------:| | [alb\_config](#input\_alb\_config) | n/a | 
object({
    name                 = optional(string, "fleet")
    subnets              = list(string)
    security_groups      = optional(list(string), [])
    access_logs          = optional(map(string), {})
    certificate_arn      = string
    allowed_cidrs        = optional(list(string), ["0.0.0.0/0"])
    allowed_ipv6_cidrs   = optional(list(string), ["::/0"])
    egress_cidrs         = optional(list(string), ["0.0.0.0/0"])
    egress_ipv6_cidrs    = optional(list(string), ["::/0"])
    extra_target_groups  = optional(any, [])
    https_listener_rules = optional(any, [])
    tls_policy           = optional(string, "ELBSecurityPolicy-TLS-1-2-2017-01")
    idle_timeout         = optional(number, 60)
  })
| n/a | yes | | [ecs\_cluster](#input\_ecs\_cluster) | The config for the terraform-aws-modules/ecs/aws module | 
object({
    autoscaling_capacity_providers = optional(any, {})
    cluster_configuration = optional(any, {
      execute_command_configuration = {
        logging = "OVERRIDE"
        log_configuration = {
          cloud_watch_log_group_name = "/aws/ecs/aws-ec2"
        }
      }
    })
    cluster_name = optional(string, "fleet")
    cluster_settings = optional(map(string), {
      "name" : "containerInsights",
      "value" : "enabled",
    })
    create                                = optional(bool, true)
    default_capacity_provider_use_fargate = optional(bool, true)
    fargate_capacity_providers = optional(any, {
      FARGATE = {
        default_capacity_provider_strategy = {
          weight = 100
        }
      }
      FARGATE_SPOT = {
        default_capacity_provider_strategy = {
          weight = 0
        }
      }
    })
    tags = optional(map(string))
  })
| 
{
  "autoscaling_capacity_providers": {},
  "cluster_configuration": {
    "execute_command_configuration": {
      "log_configuration": {
        "cloud_watch_log_group_name": "/aws/ecs/aws-ec2"
      },
      "logging": "OVERRIDE"
    }
  },
  "cluster_name": "fleet",
  "cluster_settings": {
    "name": "containerInsights",
    "value": "enabled"
  },
  "create": true,
  "default_capacity_provider_use_fargate": true,
  "fargate_capacity_providers": {
    "FARGATE": {
      "default_capacity_provider_strategy": {
        "weight": 100
      }
    },
    "FARGATE_SPOT": {
      "default_capacity_provider_strategy": {
        "weight": 0
      }
    }
  },
  "tags": {}
}
| no | -| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | 
object({
    mem                          = optional(number, 4096)
    cpu                          = optional(number, 512)
    image                        = optional(string, "fleetdm/fleet:v4.47.2")
    family                       = optional(string, "fleet")
    sidecars                     = optional(list(any), [])
    depends_on                   = optional(list(any), [])
    mount_points                 = optional(list(any), [])
    volumes                      = optional(list(any), [])
    extra_environment_variables  = optional(map(string), {})
    extra_iam_policies           = optional(list(string), [])
    extra_execution_iam_policies = optional(list(string), [])
    extra_secrets                = optional(map(string), {})
    security_groups              = optional(list(string), null)
    security_group_name          = optional(string, "fleet")
    iam_role_arn                 = optional(string, null)
    service = optional(object({
      name = optional(string, "fleet")
      }), {
      name = "fleet"
    })
    database = optional(object({
      password_secret_arn = string
      user                = string
      database            = string
      address             = string
      rr_address          = optional(string, null)
      }), {
      password_secret_arn = null
      user                = null
      database            = null
      address             = null
      rr_address          = null
    })
    redis = optional(object({
      address = string
      use_tls = optional(bool, true)
      }), {
      address = null
      use_tls = true
    })
    awslogs = optional(object({
      name      = optional(string, null)
      region    = optional(string, null)
      create    = optional(bool, true)
      prefix    = optional(string, "fleet")
      retention = optional(number, 5)
      }), {
      name      = null
      region    = null
      prefix    = "fleet"
      retention = 5
    })
    loadbalancer = optional(object({
      arn = string
      }), {
      arn = null
    })
    extra_load_balancers = optional(list(any), [])
    networking = optional(object({
      subnets         = list(string)
      security_groups = optional(list(string), null)
      }), {
      subnets         = null
      security_groups = null
    })
    autoscaling = optional(object({
      max_capacity                 = optional(number, 5)
      min_capacity                 = optional(number, 1)
      memory_tracking_target_value = optional(number, 80)
      cpu_tracking_target_value    = optional(number, 80)
      }), {
      max_capacity                 = 5
      min_capacity                 = 1
      memory_tracking_target_value = 80
      cpu_tracking_target_value    = 80
    })
    iam = optional(object({
      role = optional(object({
        name        = optional(string, "fleet-role")
        policy_name = optional(string, "fleet-iam-policy")
        }), {
        name        = "fleet-role"
        policy_name = "fleet-iam-policy"
      })
      execution = optional(object({
        name        = optional(string, "fleet-execution-role")
        policy_name = optional(string, "fleet-execution-role")
        }), {
        name        = "fleet-execution-role"
        policy_name = "fleet-iam-policy-execution"
      })
      }), {
      name = "fleetdm-execution-role"
    })
  })
| 
{
  "autoscaling": {
    "cpu_tracking_target_value": 80,
    "max_capacity": 5,
    "memory_tracking_target_value": 80,
    "min_capacity": 1
  },
  "awslogs": {
    "create": true,
    "name": null,
    "prefix": "fleet",
    "region": null,
    "retention": 5
  },
  "cpu": 256,
  "database": {
    "address": null,
    "database": null,
    "password_secret_arn": null,
    "rr_address": null,
    "user": null
  },
  "depends_on": [],
  "extra_environment_variables": {},
  "extra_execution_iam_policies": [],
  "extra_iam_policies": [],
  "extra_load_balancers": [],
  "extra_secrets": {},
  "family": "fleet",
  "iam": {
    "execution": {
      "name": "fleet-execution-role",
      "policy_name": "fleet-iam-policy-execution"
    },
    "role": {
      "name": "fleet-role",
      "policy_name": "fleet-iam-policy"
    }
  },
  "iam_role_arn": null,
  "image": "fleetdm/fleet:v4.31.1",
  "loadbalancer": {
    "arn": null
  },
  "mem": 512,
  "mount_points": [],
  "networking": {
    "security_groups": null,
    "subnets": null
  },
  "redis": {
    "address": null,
    "use_tls": true
  },
  "security_group_name": "fleet",
  "security_groups": null,
  "service": {
    "name": "fleet"
  },
  "sidecars": [],
  "volumes": []
}
| no | +| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | 
object({
    mem                          = optional(number, 4096)
    cpu                          = optional(number, 512)
    image                        = optional(string, "fleetdm/fleet:v4.47.3")
    family                       = optional(string, "fleet")
    sidecars                     = optional(list(any), [])
    depends_on                   = optional(list(any), [])
    mount_points                 = optional(list(any), [])
    volumes                      = optional(list(any), [])
    extra_environment_variables  = optional(map(string), {})
    extra_iam_policies           = optional(list(string), [])
    extra_execution_iam_policies = optional(list(string), [])
    extra_secrets                = optional(map(string), {})
    security_groups              = optional(list(string), null)
    security_group_name          = optional(string, "fleet")
    iam_role_arn                 = optional(string, null)
    service = optional(object({
      name = optional(string, "fleet")
      }), {
      name = "fleet"
    })
    database = optional(object({
      password_secret_arn = string
      user                = string
      database            = string
      address             = string
      rr_address          = optional(string, null)
      }), {
      password_secret_arn = null
      user                = null
      database            = null
      address             = null
      rr_address          = null
    })
    redis = optional(object({
      address = string
      use_tls = optional(bool, true)
      }), {
      address = null
      use_tls = true
    })
    awslogs = optional(object({
      name      = optional(string, null)
      region    = optional(string, null)
      create    = optional(bool, true)
      prefix    = optional(string, "fleet")
      retention = optional(number, 5)
      }), {
      name      = null
      region    = null
      prefix    = "fleet"
      retention = 5
    })
    loadbalancer = optional(object({
      arn = string
      }), {
      arn = null
    })
    extra_load_balancers = optional(list(any), [])
    networking = optional(object({
      subnets         = list(string)
      security_groups = optional(list(string), null)
      }), {
      subnets         = null
      security_groups = null
    })
    autoscaling = optional(object({
      max_capacity                 = optional(number, 5)
      min_capacity                 = optional(number, 1)
      memory_tracking_target_value = optional(number, 80)
      cpu_tracking_target_value    = optional(number, 80)
      }), {
      max_capacity                 = 5
      min_capacity                 = 1
      memory_tracking_target_value = 80
      cpu_tracking_target_value    = 80
    })
    iam = optional(object({
      role = optional(object({
        name        = optional(string, "fleet-role")
        policy_name = optional(string, "fleet-iam-policy")
        }), {
        name        = "fleet-role"
        policy_name = "fleet-iam-policy"
      })
      execution = optional(object({
        name        = optional(string, "fleet-execution-role")
        policy_name = optional(string, "fleet-execution-role")
        }), {
        name        = "fleet-execution-role"
        policy_name = "fleet-iam-policy-execution"
      })
      }), {
      name = "fleetdm-execution-role"
    })
  })
| 
{
  "autoscaling": {
    "cpu_tracking_target_value": 80,
    "max_capacity": 5,
    "memory_tracking_target_value": 80,
    "min_capacity": 1
  },
  "awslogs": {
    "create": true,
    "name": null,
    "prefix": "fleet",
    "region": null,
    "retention": 5
  },
  "cpu": 256,
  "database": {
    "address": null,
    "database": null,
    "password_secret_arn": null,
    "rr_address": null,
    "user": null
  },
  "depends_on": [],
  "extra_environment_variables": {},
  "extra_execution_iam_policies": [],
  "extra_iam_policies": [],
  "extra_load_balancers": [],
  "extra_secrets": {},
  "family": "fleet",
  "iam": {
    "execution": {
      "name": "fleet-execution-role",
      "policy_name": "fleet-iam-policy-execution"
    },
    "role": {
      "name": "fleet-role",
      "policy_name": "fleet-iam-policy"
    }
  },
  "iam_role_arn": null,
  "image": "fleetdm/fleet:v4.31.1",
  "loadbalancer": {
    "arn": null
  },
  "mem": 512,
  "mount_points": [],
  "networking": {
    "security_groups": null,
    "subnets": null
  },
  "redis": {
    "address": null,
    "use_tls": true
  },
  "security_group_name": "fleet",
  "security_groups": null,
  "service": {
    "name": "fleet"
  },
  "sidecars": [],
  "volumes": []
}
| no | | [migration\_config](#input\_migration\_config) | The configuration object for Fleet's migration task. | 
object({
    mem = number
    cpu = number
  })
| 
{
  "cpu": 1024,
  "mem": 2048
}
| no | | [rds\_config](#input\_rds\_config) | The config for the terraform-aws-modules/rds-aurora/aws module | 
object({
    name                            = optional(string, "fleet")
    engine_version                  = optional(string, "8.0.mysql_aurora.3.02.2")
    instance_class                  = optional(string, "db.t4g.large")
    subnets                         = optional(list(string), [])
    allowed_security_groups         = optional(list(string), [])
    allowed_cidr_blocks             = optional(list(string), [])
    apply_immediately               = optional(bool, true)
    monitoring_interval             = optional(number, 10)
    db_parameter_group_name         = optional(string)
    db_parameters                   = optional(map(string), {})
    db_cluster_parameter_group_name = optional(string)
    db_cluster_parameters           = optional(map(string), {})
    enabled_cloudwatch_logs_exports = optional(list(string), [])
    master_username                 = optional(string, "fleet")
    snapshot_identifier             = optional(string)
    cluster_tags                    = optional(map(string), {})
    preferred_maintenance_window    = optional(string, "thu:23:00-fri:00:00")
  })
| 
{
  "allowed_cidr_blocks": [],
  "allowed_security_groups": [],
  "apply_immediately": true,
  "cluster_tags": {},
  "db_cluster_parameter_group_name": null,
  "db_cluster_parameters": {},
  "db_parameter_group_name": null,
  "db_parameters": {},
  "enabled_cloudwatch_logs_exports": [],
  "engine_version": "8.0.mysql_aurora.3.02.2",
  "instance_class": "db.t4g.large",
  "master_username": "fleet",
  "monitoring_interval": 10,
  "name": "fleet",
  "preferred_maintenance_window": "thu:23:00-fri:00:00",
  "snapshot_identifier": null,
  "subnets": []
}
| no | | [redis\_config](#input\_redis\_config) | n/a | 
object({
    name                          = optional(string, "fleet")
    replication_group_id          = optional(string)
    elasticache_subnet_group_name = optional(string, "")
    allowed_security_group_ids    = optional(list(string), [])
    subnets                       = list(string)
    allowed_cidrs                 = list(string)
    availability_zones            = optional(list(string), [])
    cluster_size                  = optional(number, 3)
    instance_type                 = optional(string, "cache.m5.large")
    apply_immediately             = optional(bool, true)
    automatic_failover_enabled    = optional(bool, false)
    engine_version                = optional(string, "6.x")
    family                        = optional(string, "redis6.x")
    at_rest_encryption_enabled    = optional(bool, true)
    transit_encryption_enabled    = optional(bool, true)
    parameter = optional(list(object({
      name  = string
      value = string
    })), [])
    log_delivery_configuration = optional(list(map(any)), [])
    tags                       = optional(map(string), {})
  })
| 
{
  "allowed_cidrs": null,
  "allowed_security_group_ids": [],
  "apply_immediately": true,
  "at_rest_encryption_enabled": true,
  "automatic_failover_enabled": false,
  "availability_zones": [],
  "cluster_size": 3,
  "elasticache_subnet_group_name": "",
  "engine_version": "6.x",
  "family": "redis6.x",
  "instance_type": "cache.m5.large",
  "log_delivery_configuration": [],
  "name": "fleet",
  "parameter": [],
  "replication_group_id": null,
  "subnets": null,
  "tags": {},
  "transit_encryption_enabled": true
}
| no | diff --git a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf index 0fac721d46..367832f457 100644 --- a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf +++ b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf @@ -13,7 +13,7 @@ variable "fleet_config" { type = object({ mem = optional(number, 4096) cpu = optional(number, 512) - image = optional(string, "fleetdm/fleet:v4.47.2") + image = optional(string, "fleetdm/fleet:v4.47.3") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) diff --git a/terraform/byo-vpc/byo-db/variables.tf b/terraform/byo-vpc/byo-db/variables.tf index 090ba32812..7c8ff1738f 100644 --- a/terraform/byo-vpc/byo-db/variables.tf +++ b/terraform/byo-vpc/byo-db/variables.tf @@ -74,7 +74,7 @@ variable "fleet_config" { type = object({ mem = optional(number, 4096) cpu = optional(number, 512) - image = optional(string, "fleetdm/fleet:v4.47.2") + image = optional(string, "fleetdm/fleet:v4.47.3") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) diff --git a/terraform/byo-vpc/example/main.tf b/terraform/byo-vpc/example/main.tf index 51c04fed72..daefdcbc54 100644 --- a/terraform/byo-vpc/example/main.tf +++ b/terraform/byo-vpc/example/main.tf @@ -17,7 +17,7 @@ provider "aws" { } locals { - fleet_image = "fleetdm/fleet:v4.47.2" + fleet_image = "fleetdm/fleet:v4.47.3" domain_name = "example.com" } diff --git a/terraform/byo-vpc/variables.tf b/terraform/byo-vpc/variables.tf index e886db4212..f74561183f 100644 --- a/terraform/byo-vpc/variables.tf +++ b/terraform/byo-vpc/variables.tf @@ -167,7 +167,7 @@ variable "fleet_config" { type = object({ mem = optional(number, 4096) cpu = optional(number, 512) - image = optional(string, "fleetdm/fleet:v4.47.2") + image = optional(string, "fleetdm/fleet:v4.47.3") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) diff --git a/terraform/example/main.tf b/terraform/example/main.tf index 9339221dcb..5a2fcf5491 100644 --- a/terraform/example/main.tf +++ b/terraform/example/main.tf @@ -59,8 +59,8 @@ module "fleet" { fleet_config = { # To avoid pull-rate limiting from dockerhub, consider using our quay.io mirror - # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.47.2" - image = "fleetdm/fleet:v4.47.2" # override default to deploy the image you desire + # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.47.3" + image = "fleetdm/fleet:v4.47.3" # override default to deploy the image you desire # See https://fleetdm.com/docs/deploy/reference-architectures#aws for appropriate scaling # memory and cpu. autoscaling = { diff --git a/terraform/variables.tf b/terraform/variables.tf index b77887ddb2..e8b6e29b67 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -215,7 +215,7 @@ variable "fleet_config" { type = object({ mem = optional(number, 4096) cpu = optional(number, 512) - image = optional(string, "fleetdm/fleet:v4.47.2") + image = optional(string, "fleetdm/fleet:v4.47.3") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json index af247c6058..19de34bf6c 100644 --- a/tools/fleetctl-npm/package.json +++ b/tools/fleetctl-npm/package.json @@ -1,6 +1,6 @@ { "name": "fleetctl", - "version": "v4.47.2", + "version": "v4.47.3", "description": "Installer for the fleetctl CLI tool", "bin": { "fleetctl": "./run.js" From 4905f5db018290891441bd5728a9d2398977a521 Mon Sep 17 00:00:00 2001 From: Sharon Katz <121527325+sharon-fdm@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:52:07 -0400 Subject: [PATCH 28/29] Test release for beta (#17894) --- ee/fleetd-chrome/package-lock.json | 4 ++-- ee/fleetd-chrome/package.json | 2 +- ee/fleetd-chrome/updates-beta.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ee/fleetd-chrome/package-lock.json b/ee/fleetd-chrome/package-lock.json index f10cd0199c..1eef8fb7d6 100644 --- a/ee/fleetd-chrome/package-lock.json +++ b/ee/fleetd-chrome/package-lock.json @@ -1,12 +1,12 @@ { "name": "fleetd-for-chrome", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "fleetd-for-chrome", - "version": "1.2.0", + "version": "1.2.1", "dependencies": { "dotenv": "^16.0.3", "wa-sqlite": "github:rhashimoto/wa-sqlite#v0.9.11" diff --git a/ee/fleetd-chrome/package.json b/ee/fleetd-chrome/package.json index ad2d9a08f3..eba4f66075 100644 --- a/ee/fleetd-chrome/package.json +++ b/ee/fleetd-chrome/package.json @@ -1,7 +1,7 @@ { "name": "fleetd-for-chrome", "description": "Extension for Fleetd on ChromeOS", - "version": "1.2.0", + "version": "1.2.1", "dependencies": { "dotenv": "^16.0.3", "wa-sqlite": "github:rhashimoto/wa-sqlite#v0.9.11" diff --git a/ee/fleetd-chrome/updates-beta.xml b/ee/fleetd-chrome/updates-beta.xml index 03bddca04f..fdb6c2e6fd 100644 --- a/ee/fleetd-chrome/updates-beta.xml +++ b/ee/fleetd-chrome/updates-beta.xml @@ -1,6 +1,6 @@ - + - \ No newline at end of file + From a8f5bd9281aaa725ced0cb09632e1d5687381e9f Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:38:49 -0400 Subject: [PATCH 29/29] [released bug] Fleet UI: Fix numerical type sort on Query Report (#17891) --- changes/17288-fix-sort-of-sql-results | 1 + .../components/QueryReport/QueryReport.tsx | 7 +++- .../QueryReport/QueryReportTableConfig.tsx | 7 +++- .../components/QueryResults/QueryResults.tsx | 28 +++---------- .../QueryResults/QueryResultsTableConfig.tsx | 23 ++--------- frontend/utilities/helpers.tsx | 40 +++++++++++++++++++ 6 files changed, 60 insertions(+), 46 deletions(-) create mode 100644 changes/17288-fix-sort-of-sql-results diff --git a/changes/17288-fix-sort-of-sql-results b/changes/17288-fix-sort-of-sql-results new file mode 100644 index 0000000000..ededd089b4 --- /dev/null +++ b/changes/17288-fix-sort-of-sql-results @@ -0,0 +1 @@ +* UI fix of sql result sort for both string and numerical columns on live query results, live policy results, and query report \ No newline at end of file diff --git a/frontend/pages/queries/details/components/QueryReport/QueryReport.tsx b/frontend/pages/queries/details/components/QueryReport/QueryReport.tsx index 2c78ef99b4..7ccd7fb166 100644 --- a/frontend/pages/queries/details/components/QueryReport/QueryReport.tsx +++ b/frontend/pages/queries/details/components/QueryReport/QueryReport.tsx @@ -8,6 +8,7 @@ import { generateCSVFilename, generateCSVQueryResults, } from "utilities/generate_csv"; +import { getTableColumnsFromSql } from "utilities/helpers"; import { IQueryReport, IQueryReportResultRow } from "interfaces/query_report"; import Button from "components/buttons/Button"; @@ -52,9 +53,13 @@ const QueryReport = ({ useEffect(() => { if (queryReport && queryReport.results && queryReport.results.length > 0) { + const tableColumns = getTableColumnsFromSql(lastEditedQueryBody); + const newColumnConfigs = generateReportColumnConfigsFromResults( - flattenResults(queryReport.results) + flattenResults(queryReport.results), + tableColumns ); + // Update tableHeaders if new headers are found if (newColumnConfigs !== columnConfigs) { setColumnConfigs(newColumnConfigs); diff --git a/frontend/pages/queries/details/components/QueryReport/QueryReportTableConfig.tsx b/frontend/pages/queries/details/components/QueryReport/QueryReportTableConfig.tsx index 95ac94c664..b46383ecf6 100644 --- a/frontend/pages/queries/details/components/QueryReport/QueryReportTableConfig.tsx +++ b/frontend/pages/queries/details/components/QueryReport/QueryReportTableConfig.tsx @@ -9,10 +9,12 @@ import DefaultColumnFilter from "components/TableContainer/DataTable/DefaultColu import HeaderCell from "components/TableContainer/DataTable/HeaderCell/HeaderCell"; import { + getSortTypeFromColumnType, getUniqueColumnNamesFromRows, humanHostLastSeen, internallyTruncateText, } from "utilities/helpers"; +import { IQueryTableColumn } from "interfaces/osquery_table"; import { IHeaderProps, IWebSocketData } from "interfaces/datatable_config"; type IQueryReportTableColumnConfig = Column; @@ -40,7 +42,8 @@ const _unshiftHostname = (headers: IQueryReportTableColumnConfig[]) => { }; const generateReportColumnConfigsFromResults = ( - results: IWebSocketData[] + results: IWebSocketData[], + tableColumns?: IQueryTableColumn[] | [] ): IQueryReportTableColumnConfig[] => { /* Results include an array of objects, each representing a table row Each key value pair in an object represents a column name and value @@ -79,7 +82,7 @@ const generateReportColumnConfigsFromResults = ( Filter: DefaultColumnFilter, // Component hides filter for last_fetched filterType: "text", disableSortBy: false, - sortType: "caseInsensitive", + sortType: getSortTypeFromColumnType(key, tableColumns), }; } ); diff --git a/frontend/pages/queries/edit/components/QueryResults/QueryResults.tsx b/frontend/pages/queries/edit/components/QueryResults/QueryResults.tsx index 786563c05b..c27ab54685 100644 --- a/frontend/pages/queries/edit/components/QueryResults/QueryResults.tsx +++ b/frontend/pages/queries/edit/components/QueryResults/QueryResults.tsx @@ -5,16 +5,14 @@ import classnames from "classnames"; import FileSaver from "file-saver"; import { QueryContext } from "context/query"; import { useDebouncedCallback } from "use-debounce"; -import { find } from "lodash"; import { generateCSVFilename, generateCSVQueryResults, } from "utilities/generate_csv"; -import { osqueryTables } from "utilities/osquery_tables"; +import { getTableColumnsFromSql } from "utilities/helpers"; import { ICampaign, ICampaignError } from "interfaces/campaign"; import { ITarget } from "interfaces/target"; -import { IQueryTableColumn } from "interfaces/osquery_table"; import Button from "components/buttons/Button"; import Icon from "components/Icon/Icon"; @@ -25,7 +23,6 @@ import QueryResultsHeading from "components/queries/queryResults/QueryResultsHea import AwaitingResults from "components/queries/queryResults/AwaitingResults"; import InfoBanner from "components/InfoBanner"; import CustomLink from "components/CustomLink"; -import { checkTable } from "utilities/sql_tools"; import generateColumnConfigsFromRows from "./QueryResultsTableConfig"; @@ -77,9 +74,6 @@ const QueryResults = ({ const [queryResultsForTableRender, setQueryResultsForTableRender] = useState( queryResults ); - const [osqueryTableColumns, setOsqueryTableColumns] = useState< - IQueryTableColumn[] | [] - >([]); // immediately reset results const onRunAgain = useCallback(() => { @@ -98,32 +92,20 @@ const QueryResults = ({ debounceQueryResults(queryResults); }, [queryResults, debounceQueryResults]); - // Set table/s columns from SQL - useEffect(() => { - const tableNames = - (lastEditedQueryBody && checkTable(lastEditedQueryBody).tables) || []; - - let columns: IQueryTableColumn[] | [] = []; - tableNames.forEach((tableName: string) => { - const tableColumns = - find(osqueryTables, { name: tableName })?.columns || []; - columns = [...columns, ...tableColumns]; - }); - setOsqueryTableColumns(columns); - }, [lastEditedQueryBody]); - useEffect(() => { if (queryResults && queryResults.length > 0) { + const tableColumns = getTableColumnsFromSql(lastEditedQueryBody); + const newResultsColumnConfigs = generateColumnConfigsFromRows( queryResults, - osqueryTableColumns + tableColumns ); // Update tableHeaders if new headers are found if (newResultsColumnConfigs !== resultsColumnConfigs) { setResultsColumnConfigs(newResultsColumnConfigs); } } - }, [queryResults]); // Cannot use tableHeaders as it will cause infinite loop with setTableHeaders + }, [queryResults, lastEditedQueryBody]); // Cannot use tableHeaders as it will cause infinite loop with setTableHeaders useEffect(() => { if (errorColumnConfigs?.length === 0 && !!errors?.length) { diff --git a/frontend/pages/queries/edit/components/QueryResults/QueryResultsTableConfig.tsx b/frontend/pages/queries/edit/components/QueryResults/QueryResultsTableConfig.tsx index 75327cea21..fd29edceb4 100644 --- a/frontend/pages/queries/edit/components/QueryResults/QueryResultsTableConfig.tsx +++ b/frontend/pages/queries/edit/components/QueryResults/QueryResultsTableConfig.tsx @@ -4,11 +4,11 @@ import React from "react"; import { CellProps, Column, HeaderProps } from "react-table"; -import { find } from "lodash"; import DefaultColumnFilter from "components/TableContainer/DataTable/DefaultColumnFilter"; import HeaderCell from "components/TableContainer/DataTable/HeaderCell/HeaderCell"; import { + getSortTypeFromColumnType, getUniqueColumnNamesFromRows, internallyTruncateText, } from "utilities/helpers"; @@ -34,28 +34,11 @@ const _unshiftHostname = (columns: Column[]) => { return newHeaders; }; -// Sorts numerical columns correctly while perserving case insensitive sort for text columns -const sortType = ( - colName: string | number | symbol, - osqueryTableColumns?: IQueryTableColumn[] | [] -) => { - if (typeof colName === "string" && !!osqueryTableColumns) { - const numberTypes = ["integer", "bigint", "unsigned_bigint", "double"]; - - const type = find(osqueryTableColumns, { name: colName })?.type; - - if (type && numberTypes.includes(type)) { - return "alphanumeric"; - } - } - return "caseInsensitive"; -}; - const generateColumnConfigsFromRows = >( // TODO - narrow typing down this entire chain of logic // typed as any[] to accomodate loose typing of websocket API results: T[], // {col:val, ...} for each row of query results - osqueryTableColumns?: IQueryTableColumn[] | [] + tableColumns?: IQueryTableColumn[] | [] ): Column[] => { const uniqueColumnNames = getUniqueColumnNamesFromRows(results); const columnsConfigs = uniqueColumnNames.map>((colName) => { @@ -76,7 +59,7 @@ const generateColumnConfigsFromRows = >( }, Filter: DefaultColumnFilter, disableSortBy: false, - sortType: sortType(colName, osqueryTableColumns), + sortType: getSortTypeFromColumnType(colName, tableColumns), }; }); return _unshiftHostname(columnsConfigs); diff --git a/frontend/utilities/helpers.tsx b/frontend/utilities/helpers.tsx index a7ece58a81..2b9dc59ba4 100644 --- a/frontend/utilities/helpers.tsx +++ b/frontend/utilities/helpers.tsx @@ -2,6 +2,7 @@ import React from "react"; import { isEmpty, flatMap, + find, omit, pick, size, @@ -25,6 +26,7 @@ import { buildQueryStringFromParams } from "utilities/url"; import { IHost } from "interfaces/host"; import { ILabel } from "interfaces/label"; import { IPack } from "interfaces/pack"; +import { IQueryTableColumn } from "interfaces/osquery_table"; import { IScheduledQuery, IPackQueryFormData, @@ -39,6 +41,8 @@ import { UserRole } from "interfaces/user"; import stringUtils from "utilities/strings"; import sortUtils from "utilities/sort"; +import { checkTable } from "utilities/sql_tools"; +import { osqueryTables } from "utilities/osquery_tables"; import { DEFAULT_EMPTY_CELL_VALUE, DEFAULT_GRAVATAR_LINK, @@ -887,6 +891,40 @@ export const getUniqueColumnNamesFromRows = < // can allow additional dropdown value types in the future type DropdownOptionValue = IDropdownOption["value"]; +/** Generates the column schema for a sql query */ +export const getTableColumnsFromSql = ( + sql: string +): IQueryTableColumn[] | [] => { + const tableNames = (sql && checkTable(sql).tables) || []; + + let sqlColumns: IQueryTableColumn[] | [] = []; + tableNames.forEach((tableName: string) => { + const tableColumns = + find(osqueryTables, { name: tableName })?.columns || []; + sqlColumns = [...sqlColumns, ...tableColumns]; + }); + // TODO: Edge case of tables sharing column names with different typing not considered + + return sqlColumns; +}; + +/** Sorts sql results numerical columns correctly while perserving case insensitive sort for text columns */ +export const getSortTypeFromColumnType = ( + colName: string | number | symbol, + tableColumns?: IQueryTableColumn[] | [] +) => { + if (typeof colName === "string") { + const numberTypes = ["integer", "bigint", "unsigned_bigint", "double"]; + + const type = find(tableColumns, { name: colName })?.type; + + if (type && numberTypes.includes(type)) { + return "alphanumeric"; + } + } + return "caseInsensitive"; +}; + export function getCustomDropdownOptions( defaultOptions: IDropdownOption[], customValue: DropdownOptionValue, @@ -918,6 +956,8 @@ export default { generateRole, generateTeam, getUniqueColumnNamesFromRows, + getTableColumnsFromSql, + getSortTypeFromColumnType, getCustomDropdownOptions, greyCell, humanHostLastSeen,