Update Apple config/DDM profiles if secret variables changed (#24995)

#24900 

This PR includes and depends on PR #25012, which should be
reviewed/merged before this one.

Windows profiles are not included in this PR due to issue #25030

This PR adds the following functionality: Apple config/DDM profile is
resent to the device when the profile contains secret variables, and the
values of those variables have changed. For example.
- Upload secret variables
- Upload profile
- Device gets profile
- Upload the same profile
- Nothing happens
- Upload a different secret variable value
- Upload the same profile
- Device gets updated profile

# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [x] Added/updated tests
- [x] If database migrations are included, checked table schema to
confirm autoupdate
- For database migrations:
- [x] Checked schema for all modified table for columns that will
auto-update timestamps during migration.
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Victor Lyuboslavsky 2024-12-30 17:58:39 -06:00 committed by GitHub
parent a42189e50d
commit bd51e858ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 567 additions and 291 deletions

View file

@ -2,3 +2,4 @@ Added ability to use secrets ($FLEET_SECRET_YOURNAME) in scripts and profiles.
- Added `/fleet/spec/secret_variables` API endpoint.
- fleetctl gitops identifies secrets in scripts and profiles and saves them on the Fleet server.
- secret values are populated when scripts and profiles are sent to devices.
- When fleetctl gitops updates profiles, if the secret value has changed, the profile is updated on the host.

View file

@ -208,8 +208,8 @@ func TestApplyTeamSpecs(t *testing.T) {
ds.DeleteMDMAppleDeclarationByNameFunc = func(ctx context.Context, teamID *uint, name string) error {
return nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
filename := writeTmpYml(t, `
@ -1362,8 +1362,8 @@ func TestApplyAsGitOps(t *testing.T) {
ds.ListVPPTokensFunc = func(ctx context.Context) ([]*fleet.VPPTokenDB, error) {
return []*fleet.VPPTokenDB{}, nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
// Apply global config.

View file

@ -660,8 +660,8 @@ func TestGitOpsFullGlobal(t *testing.T) {
return []*fleet.ABMToken{}, nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
const (
@ -865,8 +865,8 @@ func TestGitOpsFullTeam(t *testing.T) {
return nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
// Queries
@ -2599,8 +2599,8 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
ds.SetSetupExperienceScriptFunc = func(ctx context.Context, script *fleet.Script) error {
return nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
t.Setenv("FLEET_SERVER_URL", fleetServerURL)

View file

@ -39,8 +39,8 @@ func (ds *Datastore) NewMDMAppleConfigProfile(ctx context.Context, cp fleet.MDMA
profUUID := "a" + uuid.New().String()
stmt := `
INSERT INTO
mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at)
(SELECT ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP() FROM DUAL WHERE
mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at, secrets_updated_at)
(SELECT ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP(), ? FROM DUAL WHERE
NOT EXISTS (
SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ?
) AND NOT EXISTS (
@ -56,7 +56,8 @@ INSERT INTO
var profileID int64
err := ds.withTx(ctx, func(tx sqlx.ExtContext) error {
res, err := tx.ExecContext(ctx, stmt,
profUUID, teamID, cp.Identifier, cp.Name, cp.Mobileconfig, cp.Mobileconfig, cp.Name, teamID, cp.Name, teamID)
profUUID, teamID, cp.Identifier, cp.Name, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name,
teamID)
if err != nil {
switch {
case IsDuplicate(err):
@ -213,7 +214,8 @@ SELECT
mobileconfig,
checksum,
created_at,
uploaded_at
uploaded_at,
secrets_updated_at
FROM
mdm_apple_configuration_profiles
WHERE
@ -277,8 +279,10 @@ SELECT
identifier,
raw_json,
checksum,
token,
created_at,
uploaded_at
uploaded_at,
secrets_updated_at
FROM
mdm_apple_declarations
WHERE
@ -1701,7 +1705,8 @@ func (ds *Datastore) batchSetMDMAppleProfilesDB(
SELECT
identifier,
profile_uuid,
mobileconfig
mobileconfig,
secrets_updated_at
FROM
mdm_apple_configuration_profiles
WHERE
@ -1720,13 +1725,14 @@ WHERE
const insertNewOrEditedProfile = `
INSERT INTO
mdm_apple_configuration_profiles (
profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at
profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at, secrets_updated_at
)
VALUES
-- see https://stackoverflow.com/a/51393124/1094941
( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP() )
( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(6), ?)
ON DUPLICATE KEY UPDATE
uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()),
uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)),
secrets_updated_at = VALUES(secrets_updated_at),
checksum = VALUES(checksum),
name = VALUES(name),
mobileconfig = VALUES(mobileconfig)
@ -1809,7 +1815,7 @@ ON DUPLICATE KEY UPDATE
// insert the new profiles and the ones that have changed
for _, p := range incomingProfs {
if result, err = tx.ExecContext(ctx, insertNewOrEditedProfile, profTeamID, p.Identifier, p.Name,
p.Mobileconfig); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") {
p.Mobileconfig, p.SecretsUpdatedAt); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") {
if err == nil {
err = errors.New(ds.testBatchSetMDMAppleProfilesErr)
}
@ -1988,13 +1994,14 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
ds.host_platform as host_platform,
ds.profile_identifier as profile_identifier,
ds.profile_name as profile_name,
ds.checksum as checksum
ds.checksum as checksum,
ds.secrets_updated_at as secrets_updated_at
FROM ( %s ) as ds
LEFT JOIN host_mdm_apple_profiles hmap
ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid
WHERE
-- profile has been updated
( hmap.checksum != ds.checksum ) OR
-- profile or secret variables have been updated
( hmap.checksum != ds.checksum ) OR IFNULL(hmap.secrets_updated_at < ds.secrets_updated_at, FALSE) OR
-- profiles in A but not in B
( hmap.profile_uuid IS NULL AND hmap.host_uuid IS NULL ) OR
-- profiles in A and B but with operation type "remove"
@ -2060,6 +2067,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
narrowByProfiles = "AND hmap.profile_uuid IN (?)"
}
// Note: We do not need secrets_updated_at in the remove statement
toRemoveStmt := fmt.Sprintf(`
SELECT
hmap.profile_uuid as profile_uuid,
@ -2176,46 +2184,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
profilesToInsert := make(map[string]*fleet.MDMAppleProfilePayload)
executeUpsertBatch := func(valuePart string, args []any) error {
// Check if the update needs to be done at all.
selectStmt := fmt.Sprintf(`
SELECT
host_uuid,
profile_uuid,
profile_identifier,
status,
COALESCE(operation_type, '') AS operation_type,
COALESCE(detail, '') AS detail,
command_uuid,
profile_name,
checksum,
profile_uuid
FROM host_mdm_apple_profiles WHERE (host_uuid, profile_uuid) IN (%s)`,
strings.TrimSuffix(strings.Repeat("(?,?),", len(profilesToInsert)), ","))
var selectArgs []any
for _, p := range profilesToInsert {
selectArgs = append(selectArgs, p.HostUUID, p.ProfileUUID)
}
var existingProfiles []fleet.MDMAppleProfilePayload
if err := sqlx.SelectContext(ctx, tx, &existingProfiles, selectStmt, selectArgs...); err != nil {
return ctxerr.Wrap(ctx, err, "bulk set pending profile status select existing")
}
var updateNeeded bool
if len(existingProfiles) == len(profilesToInsert) {
for _, exist := range existingProfiles {
insert, ok := profilesToInsert[fmt.Sprintf("%s\n%s", exist.HostUUID, exist.ProfileUUID)]
if !ok || !exist.Equal(*insert) {
updateNeeded = true
break
}
}
} else {
updateNeeded = true
}
if !updateNeeded {
// All profiles are already in the database, no need to update.
return nil
}
// If this call is made, we assume the update must be done -- a new profile was added or existing one modified.
updatedDB = true
baseStmt := fmt.Sprintf(`
INSERT INTO host_mdm_apple_profiles (
@ -2224,6 +2193,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
profile_identifier,
profile_name,
checksum,
secrets_updated_at,
operation_type,
status,
command_uuid,
@ -2235,6 +2205,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
status = VALUES(status),
command_uuid = VALUES(command_uuid),
checksum = VALUES(checksum),
secrets_updated_at = VALUES(secrets_updated_at),
detail = VALUES(detail)
`, strings.TrimSuffix(valuePart, ","))
@ -2271,14 +2242,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
HostUUID: p.HostUUID,
HostPlatform: p.HostPlatform,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
Status: pp.Status,
OperationType: pp.OperationType,
Detail: pp.Detail,
CommandUUID: pp.CommandUUID,
}
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum,
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt,
pp.OperationType, pp.Status, pp.CommandUUID, pp.Detail)
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),")
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),")
batchCount++
if batchCount >= batchSize {
@ -2298,14 +2270,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
HostUUID: p.HostUUID,
HostPlatform: p.HostPlatform,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
OperationType: fleet.MDMOperationTypeInstall,
Status: nil,
CommandUUID: "",
Detail: "",
}
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum,
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt,
fleet.MDMOperationTypeInstall, nil, "", "")
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),")
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),")
batchCount++
if batchCount >= batchSize {
@ -2334,14 +2307,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB(
HostUUID: p.HostUUID,
HostPlatform: p.HostPlatform,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
OperationType: fleet.MDMOperationTypeRemove,
Status: nil,
CommandUUID: "",
Detail: "",
}
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum,
pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt,
fleet.MDMOperationTypeRemove, nil, "", "")
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),")
psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),")
batchCount++
if batchCount >= batchSize {
@ -2404,6 +2378,7 @@ func generateDesiredStateQuery(entityType string) string {
mae.identifier as ${entityIdentifierColumn},
mae.name as ${entityNameColumn},
mae.checksum as checksum,
mae.secrets_updated_at as secrets_updated_at,
0 as ${countEntityLabelsColumn},
0 as count_non_broken_labels,
0 as count_host_labels,
@ -2437,6 +2412,7 @@ func generateDesiredStateQuery(entityType string) string {
mae.identifier as ${entityIdentifierColumn},
mae.name as ${entityNameColumn},
mae.checksum as checksum,
mae.secrets_updated_at as secrets_updated_at,
COUNT(*) as ${countEntityLabelsColumn},
COUNT(mel.label_id) as count_non_broken_labels,
COUNT(lm.label_id) as count_host_labels,
@ -2457,7 +2433,7 @@ func generateDesiredStateQuery(entityType string) string {
ne.type = 'Device' AND
( %s )
GROUP BY
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at
HAVING
${countEntityLabelsColumn} > 0 AND count_host_labels = ${countEntityLabelsColumn}
@ -2475,6 +2451,7 @@ func generateDesiredStateQuery(entityType string) string {
mae.identifier as ${entityIdentifierColumn},
mae.name as ${entityNameColumn},
mae.checksum as checksum,
mae.secrets_updated_at as secrets_updated_at,
COUNT(*) as ${countEntityLabelsColumn},
COUNT(mel.label_id) as count_non_broken_labels,
COUNT(lm.label_id) as count_host_labels,
@ -2499,7 +2476,7 @@ func generateDesiredStateQuery(entityType string) string {
ne.type = 'Device' AND
( %s )
GROUP BY
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at
HAVING
-- considers only the profiles with labels, without any broken label, with results reported after all labels were created and with the host not in any label
${countEntityLabelsColumn} > 0 AND ${countEntityLabelsColumn} = count_non_broken_labels AND ${countEntityLabelsColumn} = count_host_updated_after_labels AND count_host_labels = 0
@ -2516,6 +2493,7 @@ func generateDesiredStateQuery(entityType string) string {
mae.identifier as ${entityIdentifierColumn},
mae.name as ${entityNameColumn},
mae.checksum as checksum,
mae.secrets_updated_at as secrets_updated_at,
COUNT(*) as ${countEntityLabelsColumn},
COUNT(mel.label_id) as count_non_broken_labels,
COUNT(lm.label_id) as count_host_labels,
@ -2536,7 +2514,7 @@ func generateDesiredStateQuery(entityType string) string {
ne.type = 'Device' AND
( %s )
GROUP BY
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum
mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at
HAVING
${countEntityLabelsColumn} > 0 AND count_host_labels >= 1
`, func(s string) string { return dynamicNames[s] })
@ -2590,7 +2568,7 @@ func generateEntitiesToInstallQuery(entityType string) string {
ON hmae.${entityUUIDColumn} = ds.${entityUUIDColumn} AND hmae.host_uuid = ds.host_uuid
WHERE
-- entity has been updated
( hmae.checksum != ds.checksum ) OR
( hmae.checksum != ds.checksum ) OR IFNULL(hmae.secrets_updated_at < ds.secrets_updated_at, FALSE) OR
-- entity in A but not in B
( hmae.${entityUUIDColumn} IS NULL AND hmae.host_uuid IS NULL ) OR
-- entities in A and B but with operation type "remove"
@ -2661,7 +2639,8 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee
ds.host_platform,
ds.profile_identifier,
ds.profile_name,
ds.checksum
ds.checksum,
ds.secrets_updated_at
FROM %s `,
generateEntitiesToInstallQuery("profile"))
var profiles []*fleet.MDMAppleProfilePayload
@ -2670,6 +2649,8 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee
}
func (ds *Datastore) ListMDMAppleProfilesToRemove(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) {
// Note: although some of these values (like secrets_updated_at) are not strictly necessary for profile removal,
// we are keeping them here for consistency.
query := fmt.Sprintf(`
SELECT
hmae.profile_uuid,
@ -2677,6 +2658,7 @@ func (ds *Datastore) ListMDMAppleProfilesToRemove(ctx context.Context) ([]*fleet
hmae.profile_name,
hmae.host_uuid,
hmae.checksum,
hmae.secrets_updated_at,
hmae.operation_type,
COALESCE(hmae.detail, '') as detail,
hmae.status,
@ -2718,6 +2700,7 @@ func (ds *Datastore) GetMDMAppleProfilesContents(ctx context.Context, uuids []st
return results, nil
}
// BulkUpsertMDMAppleHostProfiles is used to update the status of profile delivery to hosts.
func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload []*fleet.MDMAppleBulkUpsertHostProfilePayload) error {
if len(payload) == 0 {
return nil
@ -2734,7 +2717,8 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload
operation_type,
detail,
command_uuid,
checksum
checksum,
secrets_updated_at
)
VALUES %s
ON DUPLICATE KEY UPDATE
@ -2742,6 +2726,7 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload
operation_type = VALUES(operation_type),
detail = VALUES(detail),
checksum = VALUES(checksum),
secrets_updated_at = VALUES(secrets_updated_at),
profile_identifier = VALUES(profile_identifier),
profile_name = VALUES(profile_name),
command_uuid = VALUES(command_uuid)`,
@ -2761,12 +2746,13 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload
}
generateValueArgs := func(p *fleet.MDMAppleBulkUpsertHostProfilePayload) (string, []any) {
valuePart := "(?, ?, ?, ?, ?, ?, ?, ?, ?),"
args := []any{p.ProfileUUID, p.ProfileIdentifier, p.ProfileName, p.HostUUID, p.Status, p.OperationType, p.Detail, p.CommandUUID, p.Checksum}
valuePart := "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),"
args := []any{p.ProfileUUID, p.ProfileIdentifier, p.ProfileName, p.HostUUID, p.Status, p.OperationType, p.Detail, p.CommandUUID,
p.Checksum, p.SecretsUpdatedAt}
return valuePart, args
}
const defaultBatchSize = 1000 // results in this times 9 placeholders
const defaultBatchSize = 1000 // number of parameters is this times number of placeholders
batchSize := defaultBatchSize
if ds.testUpsertMDMDesiredProfilesBatchSize > 0 {
batchSize = ds.testUpsertMDMDesiredProfilesBatchSize
@ -3221,19 +3207,20 @@ func (ds *Datastore) BulkUpsertMDMAppleConfigProfiles(ctx context.Context, paylo
teamID = *cp.TeamID
}
args = append(args, teamID, cp.Identifier, cp.Name, cp.Mobileconfig)
args = append(args, teamID, cp.Identifier, cp.Name, cp.Mobileconfig, cp.SecretsUpdatedAt)
// see https://stackoverflow.com/a/51393124/1094941
sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP() ),")
sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(), ?),")
}
stmt := fmt.Sprintf(`
INSERT INTO
mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at)
mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, mobileconfig, checksum, uploaded_at, secrets_updated_at)
VALUES %s
ON DUPLICATE KEY UPDATE
uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()),
mobileconfig = VALUES(mobileconfig),
checksum = VALUES(checksum)
checksum = VALUES(checksum),
secrets_updated_at = VALUES(secrets_updated_at)
`, strings.TrimSuffix(sb.String(), ","))
if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
@ -4230,15 +4217,17 @@ INSERT INTO mdm_apple_declarations (
name,
raw_json,
checksum,
secrets_updated_at,
uploaded_at,
team_id
)
VALUES (
?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),?
?,?,?,?,UNHEX(MD5(raw_json)),?,CURRENT_TIMESTAMP(),?
)
ON DUPLICATE KEY UPDATE
uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()),
checksum = VALUES(checksum),
secrets_updated_at = VALUES(secrets_updated_at),
name = VALUES(name),
identifier = VALUES(identifier),
raw_json = VALUES(raw_json)
@ -4338,14 +4327,13 @@ WHERE
}
for _, d := range incomingDeclarations {
checksum := md5ChecksumScriptContent(string(d.RawJSON))
declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString()
if result, err = tx.ExecContext(ctx, insertStmt,
declUUID,
d.Identifier,
d.Name,
d.RawJSON,
checksum,
d.SecretsUpdatedAt,
declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") {
if err == nil {
err = errors.New(ds.testBatchSetMDMAppleProfilesErr)
@ -4421,8 +4409,9 @@ INSERT INTO mdm_apple_declarations (
name,
raw_json,
checksum,
secrets_updated_at,
uploaded_at)
(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE
(SELECT ?,?,?,?,?,UNHEX(MD5(?)),?,CURRENT_TIMESTAMP() FROM DUAL WHERE
NOT EXISTS (
SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ?
) AND NOT EXISTS (
@ -4442,8 +4431,9 @@ INSERT INTO mdm_apple_declarations (
name,
raw_json,
checksum,
secrets_updated_at,
uploaded_at)
(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE
(SELECT ?,?,?,?,?,UNHEX(MD5(?)),?,CURRENT_TIMESTAMP() FROM DUAL WHERE
NOT EXISTS (
SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ?
) AND NOT EXISTS (
@ -4461,7 +4451,6 @@ ON DUPLICATE KEY UPDATE
func (ds *Datastore) insertOrUpsertMDMAppleDeclaration(ctx context.Context, insOrUpsertStmt string, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) {
declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString()
checksum := md5ChecksumScriptContent(string(declaration.RawJSON))
var tmID uint
if declaration.TeamID != nil {
@ -4472,7 +4461,9 @@ func (ds *Datastore) insertOrUpsertMDMAppleDeclaration(ctx context.Context, insO
err := ds.withTx(ctx, func(tx sqlx.ExtContext) error {
res, err := tx.ExecContext(ctx, insOrUpsertStmt,
declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID)
declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, declaration.RawJSON,
declaration.SecretsUpdatedAt,
declaration.Name, tmID, declaration.Name, tmID)
if err != nil {
switch {
case IsDuplicate(err):
@ -4652,9 +4643,9 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont
func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) {
const stmt = `
SELECT
COALESCE(MD5((count(0) + GROUP_CONCAT(HEX(mad.checksum)
COALESCE(MD5((count(0) + GROUP_CONCAT(HEX(mad.token)
ORDER BY
mad.uploaded_at DESC separator ''))), '') AS checksum,
mad.uploaded_at DESC separator ''))), '') AS token,
COALESCE(MAX(mad.created_at), NOW()) AS latest_created_timestamp
FROM
host_mdm_apple_declarations hmad
@ -4681,7 +4672,7 @@ WHERE
func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) {
const stmt = `
SELECT
HEX(mad.checksum) as checksum,
HEX(mad.token) as token,
mad.identifier
FROM
host_mdm_apple_declarations hmad
@ -4704,7 +4695,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identi
// declarations are removed, but the join would provide an extra layer of safety.
const stmt = `
SELECT
mad.raw_json, HEX(mad.checksum) as checksum
mad.raw_json, HEX(mad.token) as token
FROM
host_mdm_apple_declarations hmad
JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid
@ -4792,13 +4783,14 @@ func mdmAppleBatchSetPendingHostDeclarationsDB(
) (updatedDB bool, err error) {
baseStmt := `
INSERT INTO host_mdm_apple_declarations
(host_uuid, status, operation_type, checksum, declaration_uuid, declaration_identifier, declaration_name)
(host_uuid, status, operation_type, checksum, secrets_updated_at, declaration_uuid, declaration_identifier, declaration_name)
VALUES
%s
ON DUPLICATE KEY UPDATE
status = VALUES(status),
operation_type = VALUES(operation_type),
checksum = VALUES(checksum)
checksum = VALUES(checksum),
secrets_updated_at = VALUES(secrets_updated_at)
`
profilesToInsert := make(map[string]*fleet.MDMAppleHostDeclaration)
@ -4813,6 +4805,7 @@ func mdmAppleBatchSetPendingHostDeclarationsDB(
COALESCE(operation_type, '') AS operation_type,
COALESCE(detail, '') AS detail,
checksum,
secrets_updated_at,
declaration_uuid,
declaration_identifier,
declaration_name
@ -4855,17 +4848,18 @@ func mdmAppleBatchSetPendingHostDeclarationsDB(
generateValueArgs := func(d *fleet.MDMAppleHostDeclaration) (string, []any) {
profilesToInsert[fmt.Sprintf("%s\n%s", d.HostUUID, d.DeclarationUUID)] = &fleet.MDMAppleHostDeclaration{
HostUUID: d.HostUUID,
DeclarationUUID: d.DeclarationUUID,
Name: d.Name,
Identifier: d.Identifier,
Status: status,
OperationType: d.OperationType,
Detail: d.Detail,
Checksum: d.Checksum,
HostUUID: d.HostUUID,
DeclarationUUID: d.DeclarationUUID,
Name: d.Name,
Identifier: d.Identifier,
Status: status,
OperationType: d.OperationType,
Detail: d.Detail,
Checksum: d.Checksum,
SecretsUpdatedAt: d.SecretsUpdatedAt,
}
valuePart := "(?, ?, ?, ?, ?, ?, ?),"
args := []any{d.HostUUID, status, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name}
valuePart := "(?, ?, ?, ?, ?, ?, ?, ?),"
args := []any{d.HostUUID, status, d.OperationType, d.Checksum, d.SecretsUpdatedAt, d.DeclarationUUID, d.Identifier, d.Name}
return valuePart, args
}
@ -4875,9 +4869,14 @@ func mdmAppleBatchSetPendingHostDeclarationsDB(
// mdmAppleGetHostsWithChangedDeclarationsDB returns a
// MDMAppleHostDeclaration item for each (host x declaration) pair that
// needs an status change, this includes declarations to install and
// needs a status change, this includes declarations to install and
// declarations to be removed. Those can be differentiated by the
// OperationType field on each struct.
//
// Note (2024/12/24): This method returns some rows that DO NOT NEED TO BE UPDATED.
// We should optimize this method to only return the rows that need to be updated.
// Then we can eliminate the subsequent check for updates in the caller.
// The check for updates is needed to log the correct activity item -- whether declarations were updated or not.
func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtContext) ([]*fleet.MDMAppleHostDeclaration, error) {
stmt := fmt.Sprintf(`
(
@ -4885,6 +4884,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC
ds.host_uuid,
'install' as operation_type,
ds.checksum,
ds.secrets_updated_at,
ds.declaration_uuid,
ds.declaration_identifier,
ds.declaration_name
@ -4897,6 +4897,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC
hmae.host_uuid,
'remove' as operation_type,
hmae.checksum,
hmae.secrets_updated_at,
hmae.declaration_uuid,
hmae.declaration_identifier,
hmae.declaration_name
@ -4915,16 +4916,17 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC
return decls, nil
}
// MDMAppleStoreDDMStatusReport updates the status of the host's declarations.
func (ds *Datastore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error {
getHostDeclarationsStmt := `
SELECT host_uuid, status, operation_type, HEX(checksum) as checksum, declaration_uuid, declaration_identifier, declaration_name
SELECT host_uuid, status, operation_type, HEX(checksum) as checksum, secrets_updated_at, declaration_uuid, declaration_identifier, declaration_name
FROM host_mdm_apple_declarations
WHERE host_uuid = ?
`
updateHostDeclarationsStmt := `
INSERT INTO host_mdm_apple_declarations
(host_uuid, declaration_uuid, status, operation_type, detail, declaration_name, declaration_identifier, checksum)
(host_uuid, declaration_uuid, status, operation_type, detail, declaration_name, declaration_identifier, checksum, secrets_updated_at)
VALUES
%s
ON DUPLICATE KEY UPDATE
@ -4952,8 +4954,9 @@ ON DUPLICATE KEY UPDATE
var insertVals strings.Builder
for _, c := range current {
if u, ok := updatesByChecksum[c.Checksum]; ok {
insertVals.WriteString("(?, ?, ?, ?, ?, ?, ?, UNHEX(?)),")
args = append(args, hostUUID, c.DeclarationUUID, u.Status, u.OperationType, u.Detail, c.Identifier, c.Name, c.Checksum)
insertVals.WriteString("(?, ?, ?, ?, ?, ?, ?, UNHEX(?), ?),")
args = append(args, hostUUID, c.DeclarationUUID, u.Status, u.OperationType, u.Detail, c.Identifier, c.Name, c.Checksum,
c.SecretsUpdatedAt)
}
}

View file

@ -1056,6 +1056,7 @@ func expectAppleProfiles(
gotp.ProfileUUID = ""
gotp.CreatedAt = time.Time{}
gotp.SecretsUpdatedAt = nil
// if an expected uploaded_at timestamp is provided for this profile, keep
// its value, otherwise clear it as we don't care about asserting its
@ -1119,7 +1120,7 @@ func expectAppleDeclarations(
require.NotEmpty(t, gotD.DeclarationUUID)
require.True(t, strings.HasPrefix(gotD.DeclarationUUID, fleet.MDMAppleDeclarationUUIDPrefix))
gotD.DeclarationUUID = ""
gotD.Checksum = "" // don't care about md5checksum here
gotD.Token = "" // don't care about md5checksum here
gotD.CreatedAt = time.Time{}
@ -1399,6 +1400,7 @@ func testMDMAppleProfileManagement(t *testing.T, ds *Datastore) {
for _, p := range got {
require.NotEmpty(t, p.Checksum)
p.Checksum = nil
p.SecretsUpdatedAt = nil
}
require.ElementsMatch(t, want, got)
}
@ -7336,7 +7338,9 @@ func testMDMAppleProfileLabels(t *testing.T, ds *Datastore) {
matchProfiles := func(want, got []*fleet.MDMAppleProfilePayload) {
// match only the fields we care about
for _, p := range got {
assert.NotEmpty(t, p.Checksum)
p.Checksum = nil
p.SecretsUpdatedAt = nil
}
require.ElementsMatch(t, want, got)
}

View file

@ -543,6 +543,10 @@ OR
// (and my hunch is that we could even do the same for
// profiles) but this could be optimized to use only a provided
// set of host uuids.
//
// Note(victor): Why is the status being set to nil? Shouldn't it be set to pending?
// Or at least pending for install and nil for remove profiles. Please update this comment if you know.
// This method is called bulkSetPendingMDMHostProfilesDB, so it is confusing that the status is NOT explicitly set to pending.
_, updates.AppleDeclaration, err = mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, nil)
if err != nil {
return updates, ctxerr.Wrap(ctx, err, "bulk set pending apple declarations")

View file

@ -1777,6 +1777,7 @@ WHERE
team_id = ?
`
// For Windows profiles, if team_id and name are the same, we do an update. Otherwise, we do an insert.
const insertNewOrEditedProfile = `
INSERT INTO
mdm_windows_configuration_profiles (

View file

@ -0,0 +1,54 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20241230000000, Down_20241230000000)
}
func Up_20241230000000(tx *sql.Tx) error {
// Using DATETIME instead of TIMESTAMP for secrets_updated_at to avoid future Y2K38 issues,
// since this date is used to detect if profile needs to be updated.
// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
_, err := tx.Exec(`ALTER TABLE mdm_apple_configuration_profiles
ADD COLUMN secrets_updated_at DATETIME(6) NULL,
MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`)
if err != nil {
return fmt.Errorf("failed to alter mdm_apple_configuration_profiles table: %w", err)
}
_, err = tx.Exec(`ALTER TABLE host_mdm_apple_profiles
ADD COLUMN secrets_updated_at DATETIME(6) NULL`)
if err != nil {
return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_profiles table: %w", err)
}
// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
_, err = tx.Exec(`ALTER TABLE mdm_apple_declarations
ADD COLUMN secrets_updated_at DATETIME(6) NULL,
MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL,
-- token is used to identify if declaration needs to be re-applied
ADD COLUMN token BINARY(16) GENERATED ALWAYS AS
(UNHEX(MD5(CONCAT(raw_json, IFNULL(secrets_updated_at, ''))))) STORED NULL`)
if err != nil {
return fmt.Errorf("failed to alter mdm_apple_declarations table: %w", err)
}
_, err = tx.Exec(`ALTER TABLE host_mdm_apple_declarations
ADD COLUMN secrets_updated_at DATETIME(6) NULL`)
if err != nil {
return fmt.Errorf("failed to alter host_mdm_apple_declarations table: %w", err)
}
return nil
}
func Down_20241230000000(_ *sql.Tx) error {
return nil
}

File diff suppressed because one or more lines are too long

View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"time"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
@ -15,24 +16,69 @@ func (ds *Datastore) UpsertSecretVariables(ctx context.Context, secretVariables
return nil
}
values := strings.TrimSuffix(strings.Repeat("(?,?),", len(secretVariables)), ",")
// The secret variables should rarely change, so we do not use a transaction here.
// When we encrypt a secret variable, it is salted, so the encrypted data is different each time.
// In order to keep the updated_at timestamp correct, we need to compare the encrypted value
// with the existing value in the database. If the values are the same, we do not update the row.
stmt := fmt.Sprintf(`
INSERT INTO secret_variables (name, value)
VALUES %s
ON DUPLICATE KEY UPDATE value = VALUES(value)`, values)
args := make([]interface{}, 0, len(secretVariables)*2)
var names []string
for _, secretVariable := range secretVariables {
valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
if err != nil {
return ctxerr.Wrap(ctx, err, "encrypt secret value with server private key")
names = append(names, secretVariable.Name)
}
existingVariables, err := ds.GetSecretVariables(ctx, names)
if err != nil {
return ctxerr.Wrap(ctx, err, "get existing secret variables")
}
existingVariableMap := make(map[string]string, len(existingVariables))
for _, existingVariable := range existingVariables {
existingVariableMap[existingVariable.Name] = existingVariable.Value
}
var variablesToInsert []fleet.SecretVariable
var variablesToUpdate []fleet.SecretVariable
for _, secretVariable := range secretVariables {
existingValue, ok := existingVariableMap[secretVariable.Name]
switch {
case !ok:
variablesToInsert = append(variablesToInsert, secretVariable)
case existingValue != secretVariable.Value:
variablesToUpdate = append(variablesToUpdate, secretVariable)
default:
// No change -- the variable value is the same
}
args = append(args, secretVariable.Name, valueEncrypted)
}
if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
return ctxerr.Wrap(ctx, err, "upsert secret variables")
if len(variablesToInsert) > 0 {
values := strings.TrimSuffix(strings.Repeat("(?,?),", len(variablesToInsert)), ",")
stmt := fmt.Sprintf(`
INSERT INTO secret_variables (name, value)
VALUES %s`, values)
args := make([]interface{}, 0, len(variablesToInsert)*2)
for _, secretVariable := range variablesToInsert {
valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
if err != nil {
return ctxerr.Wrap(ctx, err, "encrypt secret value for insert with server private key")
}
args = append(args, secretVariable.Name, valueEncrypted)
}
if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
return ctxerr.Wrap(ctx, err, "insert secret variables")
}
}
if len(variablesToUpdate) > 0 {
stmt := `
UPDATE secret_variables
SET value = ?
WHERE name = ?`
for _, secretVariable := range variablesToUpdate {
valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
if err != nil {
return ctxerr.Wrap(ctx, err, "encrypt secret value for update with server private key")
}
if _, err := ds.writer(ctx).ExecContext(ctx, stmt, valueEncrypted, secretVariable.Name); err != nil {
return ctxerr.Wrap(ctx, err, "update secret variables")
}
}
}
return nil
@ -44,7 +90,7 @@ func (ds *Datastore) GetSecretVariables(ctx context.Context, names []string) ([]
}
stmt, args, err := sqlx.In(`
SELECT name, value
SELECT name, value, updated_at
FROM secret_variables
WHERE name IN (?)`, names)
if err != nil {
@ -70,14 +116,19 @@ func (ds *Datastore) GetSecretVariables(ctx context.Context, names []string) ([]
}
func (ds *Datastore) ExpandEmbeddedSecrets(ctx context.Context, document string) (string, error) {
expanded, _, err := ds.expandEmbeddedSecrets(ctx, document)
return expanded, err
}
func (ds *Datastore) expandEmbeddedSecrets(ctx context.Context, document string) (string, []fleet.SecretVariable, error) {
embeddedSecrets := fleet.ContainsPrefixVars(document, fleet.ServerSecretPrefix)
if len(embeddedSecrets) == 0 {
return document, nil
return document, nil, nil
}
secrets, err := ds.GetSecretVariables(ctx, embeddedSecrets)
if err != nil {
return "", ctxerr.Wrap(ctx, err, "expanding embedded secrets")
return "", nil, ctxerr.Wrap(ctx, err, "expanding embedded secrets")
}
secretMap := make(map[string]string, len(secrets))
@ -95,7 +146,7 @@ func (ds *Datastore) ExpandEmbeddedSecrets(ctx context.Context, document string)
}
if len(missingSecrets) > 0 {
return "", fleet.MissingSecretsError{MissingSecrets: missingSecrets}
return "", nil, fleet.MissingSecretsError{MissingSecrets: missingSecrets}
}
expanded := fleet.MaybeExpand(document, func(s string) (string, bool) {
@ -106,7 +157,25 @@ func (ds *Datastore) ExpandEmbeddedSecrets(ctx context.Context, document string)
return val, ok
})
return expanded, nil
return expanded, secrets, nil
}
func (ds *Datastore) ExpandEmbeddedSecretsAndUpdatedAt(ctx context.Context, document string) (string, *time.Time, error) {
expanded, secrets, err := ds.expandEmbeddedSecrets(ctx, document)
if err != nil {
return "", nil, ctxerr.Wrap(ctx, err, "expanding embedded secrets and updated at")
}
if len(secrets) == 0 {
return expanded, nil, nil
}
// Find the most recent updated_at timestamp
var updatedAt time.Time
for _, secret := range secrets {
if secret.UpdatedAt.After(updatedAt) {
updatedAt = secret.UpdatedAt
}
}
return expanded, &updatedAt, err
}
func (ds *Datastore) ValidateEmbeddedSecrets(ctx context.Context, documents []string) error {

View file

@ -59,18 +59,33 @@ func testUpsertSecretVariables(t *testing.T, ds *Datastore) {
assert.Equal(t, secretMap[result.Name], result.Value)
}
// Update a secret
// Update a secret and insert a new one
secretMap["test2"] = "newTestValue2"
secretMap["test4"] = "testValue4"
err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{
{Name: "test2", Value: secretMap["test2"]},
{Name: "test4", Value: secretMap["test4"]},
})
assert.NoError(t, err)
results, err = ds.GetSecretVariables(ctx, []string{"test2"})
results, err = ds.GetSecretVariables(ctx, []string{"test2", "test4"})
assert.NoError(t, err)
require.Len(t, results, 1)
assert.Equal(t, "test2", results[0].Name)
assert.Equal(t, secretMap[results[0].Name], results[0].Value)
require.Len(t, results, 2)
for _, result := range results {
assert.Equal(t, secretMap[result.Name], result.Value)
}
// Make sure updated_at timestamp does not change when we update a secret with the same value
original, err := ds.GetSecretVariables(ctx, []string{"test1"})
require.NoError(t, err)
require.Len(t, original, 1)
err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{
{Name: "test1", Value: secretMap["test1"]},
})
require.NoError(t, err)
updated, err := ds.GetSecretVariables(ctx, []string{"test1"})
require.NoError(t, err)
require.Len(t, original, 1)
assert.Equal(t, original[0], updated[0])
}
func testValidateEmbeddedSecrets(t *testing.T, ds *Datastore) {
@ -122,7 +137,7 @@ Hello doc${FLEET_SECRET_INVALID}. $FLEET_SECRET_ALSO_INVALID
func testExpandEmbeddedSecrets(t *testing.T, ds *Datastore) {
noSecrets := `
This document contains to fleet secrets.
This document contains no fleet secrets.
$FLEET_VAR_XX $HOSTNAME ${SOMETHING_ELSE}
`
@ -157,10 +172,18 @@ Hello doc${FLEET_SECRET_INVALID}. $FLEET_SECRET_ALSO_INVALID
expanded, err := ds.ExpandEmbeddedSecrets(ctx, noSecrets)
require.NoError(t, err)
require.Equal(t, noSecrets, expanded)
expanded, secretsUpdatedAt, err := ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, noSecrets)
require.NoError(t, err)
require.Equal(t, noSecrets, expanded)
assert.Nil(t, secretsUpdatedAt)
expanded, err = ds.ExpandEmbeddedSecrets(ctx, validSecret)
require.NoError(t, err)
require.Equal(t, validSecretExpanded, expanded)
expanded, secretsUpdatedAt, err = ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, validSecret)
require.NoError(t, err)
require.Equal(t, validSecretExpanded, expanded)
assert.NotNil(t, secretsUpdatedAt)
_, err = ds.ExpandEmbeddedSecrets(ctx, invalidSecret)
require.ErrorContains(t, err, "$FLEET_SECRET_INVALID")

View file

@ -1,7 +1,6 @@
package fleet
import (
"bytes"
"context"
"crypto/md5" // nolint: gosec
"encoding/hex"
@ -205,6 +204,7 @@ type MDMAppleConfigProfile struct {
LabelsExcludeAny []ConfigurationProfileLabel `db:"-" json:"labels_exclude_any,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UploadedAt time.Time `db:"uploaded_at" json:"updated_at"` // NOTE: JSON field is still `updated_at` for historical reasons, would be an API breaking change
SecretsUpdatedAt *time.Time `db:"secrets_updated_at" json:"-"`
}
// MDMProfilesUpdates flags updates that were done during batch processing of profiles.
@ -321,6 +321,7 @@ type MDMAppleProfilePayload struct {
HostUUID string `db:"host_uuid"`
HostPlatform string `db:"host_platform"`
Checksum []byte `db:"checksum"`
SecretsUpdatedAt *time.Time `db:"secrets_updated_at"`
Status *MDMDeliveryStatus `db:"status" json:"status"`
OperationType MDMOperationType `db:"operation_type"`
Detail string `db:"detail"`
@ -333,20 +334,6 @@ func (p *MDMAppleProfilePayload) DidNotInstallOnHost() bool {
return p.Status != nil && (*p.Status == MDMDeliveryFailed || *p.Status == MDMDeliveryPending) && p.OperationType == MDMOperationTypeInstall
}
func (p MDMAppleProfilePayload) Equal(other MDMAppleProfilePayload) bool {
statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status
return p.ProfileUUID == other.ProfileUUID &&
p.ProfileIdentifier == other.ProfileIdentifier &&
p.ProfileName == other.ProfileName &&
p.HostUUID == other.HostUUID &&
p.HostPlatform == other.HostPlatform &&
bytes.Equal(p.Checksum, other.Checksum) &&
statusEqual &&
p.OperationType == other.OperationType &&
p.Detail == other.Detail &&
p.CommandUUID == other.CommandUUID
}
type MDMAppleBulkUpsertHostProfilePayload struct {
ProfileUUID string
ProfileIdentifier string
@ -357,6 +344,7 @@ type MDMAppleBulkUpsertHostProfilePayload struct {
Status *MDMDeliveryStatus
Detail string
Checksum []byte
SecretsUpdatedAt *time.Time
}
// MDMAppleFileVaultSummary reports the number of macOS hosts being managed with Apples disk
@ -603,13 +591,18 @@ type MDMAppleDeclaration struct {
// Checksum is a checksum of the JSON contents
Checksum string `db:"checksum" json:"-"`
// Token is used to identify if declaration needs to be re-applied.
// It contains the checksum of the JSON contents and secrets updated timestamp (if secret variables are present).
Token string `db:"token" json:"-"`
// labels associated with this Declaration
LabelsIncludeAll []ConfigurationProfileLabel `db:"-" json:"labels_include_all,omitempty"`
LabelsIncludeAny []ConfigurationProfileLabel `db:"-" json:"labels_include_any,omitempty"`
LabelsExcludeAny []ConfigurationProfileLabel `db:"-" json:"labels_exclude_any,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"`
SecretsUpdatedAt *time.Time `db:"secrets_updated_at" json:"-"`
}
type MDMAppleRawDeclaration struct {
@ -697,10 +690,14 @@ type MDMAppleHostDeclaration struct {
// Checksum contains the MD5 checksum of the declaration JSON uploaded
// by the IT admin. Fleet uses this value as the ServerToken.
Checksum string `db:"checksum" json:"-"`
// SecretsUpdatedAt is the timestamp when the secrets were last updated or when this declaration was uploaded.
SecretsUpdatedAt *time.Time `db:"secrets_updated_at" json:"-"`
}
func (p MDMAppleHostDeclaration) Equal(other MDMAppleHostDeclaration) bool {
statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status
secretsEqual := p.SecretsUpdatedAt == nil && other.SecretsUpdatedAt == nil || p.SecretsUpdatedAt != nil && other.SecretsUpdatedAt != nil && p.SecretsUpdatedAt.Equal(*other.SecretsUpdatedAt)
return statusEqual &&
p.HostUUID == other.HostUUID &&
p.DeclarationUUID == other.DeclarationUUID &&
@ -708,7 +705,8 @@ func (p MDMAppleHostDeclaration) Equal(other MDMAppleHostDeclaration) bool {
p.Identifier == other.Identifier &&
p.OperationType == other.OperationType &&
p.Detail == other.Detail &&
p.Checksum == other.Checksum
p.Checksum == other.Checksum &&
secretsEqual
}
func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration {
@ -733,8 +731,9 @@ type MDMAppleDDMTokensResponse struct {
//
// https://developer.apple.com/documentation/devicemanagement/synchronizationtokens
type MDMAppleDDMDeclarationsToken struct {
DeclarationsToken string `db:"checksum"`
Timestamp time.Time `db:"latest_created_timestamp"`
DeclarationsToken string `db:"token"`
// Timestamp must JSON marshal to format YYYY-mm-ddTHH:MM:SSZ
Timestamp time.Time `db:"latest_created_timestamp"`
}
// MDMAppleDDMDeclarationItemsResponse is the response from the DDM declaration items endpoint.
@ -770,7 +769,7 @@ type MDMAppleDDMManifest struct {
// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse
type MDMAppleDDMDeclarationItem struct {
Identifier string `db:"identifier"`
ServerToken string `db:"checksum"`
ServerToken string `db:"token"`
}
// MDMAppleDDMDeclarationResponse represents a declaration in the datastore. It is used for the DDM

View file

@ -472,6 +472,8 @@ func TestMDMAppleHostDeclarationEqual(t *testing.T) {
fieldsInEqualMethod++
items[1].Status = &status1
fieldsInEqualMethod++
items[1].SecretsUpdatedAt = items[0].SecretsUpdatedAt
fieldsInEqualMethod++
assert.Equal(t, fieldsInEqualMethod, numberOfFields, "MDMAppleHostDeclaration.Equal needs to be updated for new/updated field(s)")
assert.True(t, items[0].Equal(items[1]))
@ -481,85 +483,6 @@ func TestMDMAppleHostDeclarationEqual(t *testing.T) {
assert.True(t, items[0].Equal(items[1]))
}
func TestMDMAppleProfilePayloadEqual(t *testing.T) {
t.Parallel()
// This test is intended to ensure that the Equal method on MDMAppleProfilePayload is updated when new fields are added.
// The Equal method is used to identify whether database update is needed.
items := [...]MDMAppleProfilePayload{{}, {}}
numberOfFields := 0
for i := 0; i < len(items); i++ {
rValue := reflect.ValueOf(&items[i]).Elem()
numberOfFields = rValue.NumField()
for j := 0; j < numberOfFields; j++ {
field := rValue.Field(j)
switch field.Kind() {
case reflect.String:
valueToSet := fmt.Sprintf("test %d", i)
field.SetString(valueToSet)
case reflect.Int:
field.SetInt(int64(i))
case reflect.Bool:
field.SetBool(i%2 == 0)
case reflect.Pointer:
field.Set(reflect.New(field.Type().Elem()))
case reflect.Slice:
switch field.Type().Elem().Kind() {
case reflect.Uint8:
valueToSet := []byte("test")
field.Set(reflect.ValueOf(valueToSet))
default:
t.Fatalf("unhandled slice type %s", field.Type().Elem().Kind())
}
default:
t.Fatalf("unhandled field type %s", field.Kind())
}
}
}
status0 := MDMDeliveryStatus("status")
status1 := MDMDeliveryStatus("status")
items[0].Status = &status0
checksum0 := []byte("checksum")
checksum1 := []byte("checksum")
items[0].Checksum = checksum0
assert.False(t, items[0].Equal(items[1]))
// Set known fields to be equal
fieldsInEqualMethod := 0
items[1].ProfileUUID = items[0].ProfileUUID
fieldsInEqualMethod++
items[1].ProfileIdentifier = items[0].ProfileIdentifier
fieldsInEqualMethod++
items[1].ProfileName = items[0].ProfileName
fieldsInEqualMethod++
items[1].HostUUID = items[0].HostUUID
fieldsInEqualMethod++
items[1].HostPlatform = items[0].HostPlatform
fieldsInEqualMethod++
items[1].Checksum = checksum1
fieldsInEqualMethod++
items[1].Status = &status1
fieldsInEqualMethod++
items[1].OperationType = items[0].OperationType
fieldsInEqualMethod++
items[1].Detail = items[0].Detail
fieldsInEqualMethod++
items[1].CommandUUID = items[0].CommandUUID
fieldsInEqualMethod++
assert.Equal(t, fieldsInEqualMethod, numberOfFields, "MDMAppleProfilePayload.Equal needs to be updated for new/updated field(s)")
assert.True(t, items[0].Equal(items[1]))
// Set pointers and slices to nil
items[0].Status = nil
items[1].Status = nil
items[0].Checksum = nil
items[1].Checksum = nil
assert.True(t, items[0].Equal(items[1]))
}
func TestConfigurationProfileLabelEqual(t *testing.T) {
t.Parallel()

View file

@ -1899,6 +1899,10 @@ type Datastore interface {
// ExpandEmbeddedSecrets expands the fleet secrets in a
// document using the secrets stored in the datastore.
ExpandEmbeddedSecrets(ctx context.Context, document string) (string, error)
// ExpandEmbeddedSecretsAndUpdatedAt is like ExpandEmbeddedSecrets but also
// returns the latest updated_at time of the secrets used in the expansion.
ExpandEmbeddedSecretsAndUpdatedAt(ctx context.Context, document string) (string, *time.Time, error)
}
// MDMAppleStore wraps nanomdm's storage and adds methods to deal with

View file

@ -437,10 +437,11 @@ type MDMProfileBatchPayload struct {
// Deprecated: Labels is the backwards-compatible way of specifying
// LabelsIncludeAll.
Labels []string `json:"labels,omitempty"`
LabelsIncludeAll []string `json:"labels_include_all,omitempty"`
LabelsIncludeAny []string `json:"labels_include_any,omitempty"`
LabelsExcludeAny []string `json:"labels_exclude_any,omitempty"`
Labels []string `json:"labels,omitempty"`
LabelsIncludeAll []string `json:"labels_include_all,omitempty"`
LabelsIncludeAny []string `json:"labels_include_any,omitempty"`
LabelsExcludeAny []string `json:"labels_exclude_any,omitempty"`
SecretsUpdatedAt *time.Time `json:"-"`
}
func NewMDMConfigProfilePayloadFromWindows(cp *MDMWindowsConfigProfile) *MDMConfigProfilePayload {

View file

@ -1,8 +1,11 @@
package fleet
import "time"
type SecretVariable struct {
Name string `json:"name" db:"name"`
Value string `json:"value" db:"value"`
Name string `json:"name" db:"name"`
Value string `json:"value" db:"value"`
UpdatedAt time.Time `json:"-" db:"updated_at"`
}
func (h SecretVariable) AuthzType() string {

View file

@ -1185,6 +1185,8 @@ type ValidateEmbeddedSecretsFunc func(ctx context.Context, documents []string) e
type ExpandEmbeddedSecretsFunc func(ctx context.Context, document string) (string, error)
type ExpandEmbeddedSecretsAndUpdatedAtFunc func(ctx context.Context, document string) (string, *time.Time, error)
type DataStore struct {
HealthCheckFunc HealthCheckFunc
HealthCheckFuncInvoked bool
@ -2932,6 +2934,9 @@ type DataStore struct {
ExpandEmbeddedSecretsFunc ExpandEmbeddedSecretsFunc
ExpandEmbeddedSecretsFuncInvoked bool
ExpandEmbeddedSecretsAndUpdatedAtFunc ExpandEmbeddedSecretsAndUpdatedAtFunc
ExpandEmbeddedSecretsAndUpdatedAtFuncInvoked bool
mu sync.Mutex
}
@ -4906,7 +4911,7 @@ func (s *DataStore) UpdateCronStats(ctx context.Context, id int, status fleet.Cr
s.mu.Lock()
s.UpdateCronStatsFuncInvoked = true
s.mu.Unlock()
return s.UpdateCronStatsFunc(ctx, id, status, &fleet.CronScheduleErrors{})
return s.UpdateCronStatsFunc(ctx, id, status, cronErrors)
}
func (s *DataStore) UpdateAllCronStatsForInstance(ctx context.Context, instance string, fromStatus fleet.CronStatsStatus, toStatus fleet.CronStatsStatus) error {
@ -7008,3 +7013,10 @@ func (s *DataStore) ExpandEmbeddedSecrets(ctx context.Context, document string)
s.mu.Unlock()
return s.ExpandEmbeddedSecretsFunc(ctx, document)
}
func (s *DataStore) ExpandEmbeddedSecretsAndUpdatedAt(ctx context.Context, document string) (string, *time.Time, error) {
s.mu.Lock()
s.ExpandEmbeddedSecretsAndUpdatedAtFuncInvoked = true
s.mu.Unlock()
return s.ExpandEmbeddedSecretsAndUpdatedAtFunc(ctx, document)
}

View file

@ -381,7 +381,7 @@ func (svc *Service) NewMDMAppleConfigProfile(ctx context.Context, teamID uint, r
}
// Expand and validate secrets in profile
expanded, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(b))
expanded, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(b))
if err != nil {
return nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError("profile", err.Error()))
}
@ -403,6 +403,7 @@ func (svc *Service) NewMDMAppleConfigProfile(ctx context.Context, teamID uint, r
// Save the original unexpanded profile
cp.Mobileconfig = b
cp.SecretsUpdatedAt = secretsUpdatedAt
labelMap, err := svc.validateProfileLabels(ctx, labels)
if err != nil {
@ -512,7 +513,7 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i
return nil, err
}
dataWithSecrets, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(data))
dataWithSecrets, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(data))
if err != nil {
return nil, fleet.NewInvalidArgumentError("profile", err.Error())
}
@ -533,6 +534,7 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i
}
d := fleet.NewMDMAppleDeclaration(data, tmID, name, rawDecl.Type, rawDecl.Identifier)
d.SecretsUpdatedAt = secretsUpdatedAt
switch labelsMembershipMode {
case fleet.LabelsIncludeAny:
@ -1989,7 +1991,7 @@ func (svc *Service) BatchSetMDMAppleProfiles(ctx context.Context, tmID *uint, tm
)
}
// Expand profile for validation
expanded, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(prof))
expanded, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(prof))
if err != nil {
return ctxerr.Wrap(ctx,
fleet.NewInvalidArgumentError(fmt.Sprintf("profiles[%d]", i), err.Error()),
@ -2009,6 +2011,7 @@ func (svc *Service) BatchSetMDMAppleProfiles(ctx context.Context, tmID *uint, tm
// Store original unexpanded profile
mdmProf.Mobileconfig = prof
mdmProf.SecretsUpdatedAt = secretsUpdatedAt
if byName[mdmProf.Name] {
return ctxerr.Wrap(ctx,
@ -3422,6 +3425,7 @@ func ReconcileAppleProfiles(
ProfileIdentifier: p.ProfileIdentifier,
ProfileName: p.ProfileName,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
OperationType: pp.OperationType,
Status: pp.Status,
CommandUUID: pp.CommandUUID,
@ -3453,6 +3457,7 @@ func ReconcileAppleProfiles(
ProfileIdentifier: p.ProfileIdentifier,
ProfileName: p.ProfileName,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
}
hostProfiles = append(hostProfiles, hostProfile)
hostProfilesToInstallMap[hostProfileUUID{HostUUID: p.HostUUID, ProfileUUID: p.ProfileUUID}] = hostProfile
@ -3490,6 +3495,7 @@ func ReconcileAppleProfiles(
ProfileIdentifier: p.ProfileIdentifier,
ProfileName: p.ProfileName,
Checksum: p.Checksum,
SecretsUpdatedAt: p.SecretsUpdatedAt,
})
}
@ -4177,6 +4183,9 @@ func (svc *MDMAppleDDMService) handleTokens(ctx context.Context, hostUUID string
return nil, ctxerr.Wrap(ctx, err, "getting synchronization tokens")
}
// Important: Timestamp must use format YYYY-mm-ddTHH:MM:SSZ (no milliseconds)
// Source: https://developer.apple.com/documentation/devicemanagement/synchronizationtokens?language=objc
tok.Timestamp = tok.Timestamp.Truncate(time.Second)
b, err := json.Marshal(fleet.MDMAppleDDMTokensResponse{
SyncTokens: *tok,
})
@ -4262,7 +4271,7 @@ func (svc *MDMAppleDDMService) handleActivationDeclaration(ctx context.Context,
},
"ServerToken": "%s",
"Type": "com.apple.activation.simple"
}`, parts[2], references, d.Checksum)
}`, parts[2], references, d.Token)
return []byte(response), nil
}
@ -4285,7 +4294,7 @@ func (svc *MDMAppleDDMService) handleConfigurationDeclaration(ctx context.Contex
if err := json.Unmarshal([]byte(expanded), &tempd); err != nil {
return nil, ctxerr.Wrap(ctx, err, "unmarshaling stored declaration")
}
tempd["ServerToken"] = d.Checksum
tempd["ServerToken"] = d.Token
b, err := json.Marshal(tempd)
if err != nil {

View file

@ -215,6 +215,9 @@ func setupAppleMDMService(t *testing.T, license *fleet.LicenseInfo) (fleet.Servi
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
}
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
apnsCert, apnsKey, err := mysql.GenerateTestCertBytes()
require.NoError(t, err)
crt, key, err := apple_mdm.NewSCEPCACertKey()

View file

@ -472,22 +472,23 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() {
_, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string,
expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) {
expectedDeclsByToken map[string]fleet.MDMAppleDeclaration) {
require.Equal(t, expectedDeclTok, r.DeclarationsToken)
require.NotEmpty(t, r.Declarations.Activations)
require.Empty(t, r.Declarations.Assets)
require.Empty(t, r.Declarations.Management)
require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum))
require.Len(t, r.Declarations.Configurations, len(expectedDeclsByToken))
for _, m := range r.Declarations.Configurations {
d, ok := expectedDeclsByChecksum[m.ServerToken]
require.True(t, ok)
d, ok := expectedDeclsByToken[m.ServerToken]
if !ok {
for k := range expectedDeclsByToken {
t.Logf("expected token: %x", k)
}
}
require.True(t, ok, "server token %x not found for %s", m.ServerToken, m.Identifier)
require.Equal(t, d.Identifier, m.Identifier)
}
}
calcChecksum := func(source []byte) string {
csum := fmt.Sprintf("%x", md5.Sum(source)) //nolint:gosec
return strings.ToUpper(csum)
}
tmpl := `
{
@ -516,25 +517,15 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() {
decls[1] = []byte(strings.ReplaceAll(string(decls[1]), myBash, "$"+fleet.ServerSecretPrefix+"BASH"))
secretProfile := decls[2]
decls[2] = []byte("${" + fleet.ServerSecretPrefix + "PROFILE}")
declsByChecksum := map[string]fleet.MDMAppleDeclaration{
calcChecksum(decls[0]): {
Identifier: "com.fleet.config0",
},
calcChecksum(decls[1]): {
Identifier: "com.fleet.config1",
},
calcChecksum(decls[2]): {
Identifier: "com.fleet.config2",
},
}
// Create declarations
// First dry run
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{
profilesReq := batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{
{Name: "N0", Contents: decls[0]},
{Name: "N1", Contents: decls[1]},
{Name: "N2", Contents: decls[2]},
}}, http.StatusNoContent, "dry_run", "true")
}}
// First dry run
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent, "dry_run", "true")
var resp listMDMConfigProfilesResponse
s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp)
@ -557,11 +548,7 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() {
s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp)
// Now real run
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: []fleet.MDMProfileBatchPayload{
{Name: "N0", Contents: decls[0]},
{Name: "N1", Contents: decls[1]},
{Name: "N2", Contents: decls[2]},
}}, http.StatusNoContent)
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent)
s.DoJSON("GET", "/api/latest/fleet/mdm/profiles", &listMDMConfigProfilesRequest{}, http.StatusOK, &resp)
require.Len(t, resp.Profiles, len(decls))
@ -585,9 +572,11 @@ SELECT
identifier,
name,
raw_json,
checksum,
HEX(checksum) as checksum,
HEX(token) as token,
created_at,
uploaded_at
uploaded_at,
secrets_updated_at
FROM mdm_apple_declarations
WHERE name = ?`
@ -599,19 +588,29 @@ WHERE name = ?`
}
nameToIdentifier := make(map[string]string, 3)
nameToUUID := make(map[string]string, 3)
declsByToken := map[string]fleet.MDMAppleDeclaration{}
decl := getDeclaration(t, "N0")
nameToIdentifier["N0"] = decl.Identifier
nameToUUID["N0"] = decl.DeclarationUUID
declsByToken[decl.Token] = fleet.MDMAppleDeclaration{
Identifier: "com.fleet.config0",
}
decl = getDeclaration(t, "N1")
assert.NotContains(t, string(decl.RawJSON), myBash)
assert.Contains(t, string(decl.RawJSON), "$"+fleet.ServerSecretPrefix+"BASH")
nameToIdentifier["N1"] = decl.Identifier
nameToUUID["N1"] = decl.DeclarationUUID
n1Token := decl.Token
declsByToken[decl.Token] = fleet.MDMAppleDeclaration{
Identifier: "com.fleet.config1",
}
decl = getDeclaration(t, "N2")
assert.Equal(t, string(decl.RawJSON), "${"+fleet.ServerSecretPrefix+"PROFILE}")
nameToIdentifier["N2"] = decl.Identifier
nameToUUID["N2"] = decl.DeclarationUUID
declsByToken[decl.Token] = fleet.MDMAppleDeclaration{
Identifier: "com.fleet.config2",
}
// trigger a profile sync
s.awaitTriggerProfileSchedule(t)
@ -624,7 +623,7 @@ WHERE name = ?`
r, err = mdmDevice.DeclarativeManagement("declaration-items")
require.NoError(t, err)
itemsResp := parseDeclarationItemsResp(t, r)
checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByChecksum)
checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByToken)
// Now, retrieve the declaration configuration profiles
declarationPath := fmt.Sprintf("declaration/configuration/%s", nameToIdentifier["N0"])
@ -646,6 +645,82 @@ WHERE name = ?`
require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed))
assert.EqualValues(t, `{"DataAssetReference":"com.fleet.asset.bash","ServiceType":"com.apple.bash2"}`, gotParsed.Payload)
// Upload the same profiles again -- nothing should change
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent, "dry_run", "true")
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent)
s.awaitTriggerProfileSchedule(t)
// Get tokens again
r, err = mdmDevice.DeclarativeManagement("tokens")
require.NoError(t, err)
tokens = parseTokensResp(t, r)
currDeclToken = tokens.SyncTokens.DeclarationsToken
// Get declaration items -- the checksums should be the same as before
r, err = mdmDevice.DeclarativeManagement("declaration-items")
require.NoError(t, err)
itemsResp = parseDeclarationItemsResp(t, r)
checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByToken)
// Change the secrets.
myBash = "my.new.bash"
req = secretVariablesRequest{
SecretVariables: []fleet.SecretVariable{
{
Name: "FLEET_SECRET_BASH",
Value: myBash, // changed
},
{
Name: "FLEET_SECRET_PROFILE",
Value: string(secretProfile), // did not change
},
},
}
s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp)
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent, "dry_run", "true")
s.Do("POST", "/api/latest/fleet/mdm/profiles/batch", profilesReq, http.StatusNoContent)
// The token of the declaration with the updated secret should have changed.
decl = getDeclaration(t, "N1")
assert.NotContains(t, string(decl.RawJSON), myBash)
assert.Contains(t, string(decl.RawJSON), "$"+fleet.ServerSecretPrefix+"BASH")
nameToIdentifier["N1"] = decl.Identifier
nameToUUID["N1"] = decl.DeclarationUUID
assert.NotEqual(t, n1Token, decl.Token)
// Update expected token
delete(declsByToken, n1Token)
declsByToken[decl.Token] = fleet.MDMAppleDeclaration{
Identifier: "com.fleet.config1",
}
s.awaitTriggerProfileSchedule(t)
// Get tokens again
r, err = mdmDevice.DeclarativeManagement("tokens")
require.NoError(t, err)
tokens = parseTokensResp(t, r)
currDeclToken = tokens.SyncTokens.DeclarationsToken
// Only N1 should have changed
r, err = mdmDevice.DeclarativeManagement("declaration-items")
require.NoError(t, err)
itemsResp = parseDeclarationItemsResp(t, r)
checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByToken)
// Now, retrieve the declaration configuration profiles
declarationPath = fmt.Sprintf("declaration/configuration/%s", nameToIdentifier["N0"])
r, err = mdmDevice.DeclarativeManagement(declarationPath)
require.NoError(t, err)
require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed))
assert.EqualValues(t, `{"DataAssetReference":"com.fleet.asset.bash","ServiceType":"com.apple.bash0"}`, gotParsed.Payload)
declarationPath = fmt.Sprintf("declaration/configuration/%s", nameToIdentifier["N1"])
r, err = mdmDevice.DeclarativeManagement(declarationPath)
require.NoError(t, err)
require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed))
assert.EqualValues(t, `{"DataAssetReference":"com.fleet.asset.bash","ServiceType":"my.new.bash"}`, gotParsed.Payload)
declarationPath = fmt.Sprintf("declaration/configuration/%s", nameToIdentifier["N2"])
r, err = mdmDevice.DeclarativeManagement(declarationPath)
require.NoError(t, err)
require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed))
assert.EqualValues(t, `{"DataAssetReference":"com.fleet.asset.bash","ServiceType":"com.apple.bash2"}`, gotParsed.Payload)
// Delete the profiles
s.Do("DELETE", "/api/latest/fleet/configuration_profiles/"+nameToUUID["N0"], nil, http.StatusOK)
s.Do("DELETE", "/api/latest/fleet/configuration_profiles/"+nameToUUID["N1"], nil, http.StatusOK)

View file

@ -259,12 +259,47 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() {
s.checkMDMProfilesSummaries(t, nil, expectedNoTeamSummary, &expectedNoTeamSummary) // empty because host was transferred
s.checkMDMProfilesSummaries(t, &tm.ID, expectedTeamSummary, &expectedTeamSummary) // host still verifying team profiles
// with no changes
// Upload the same profiles again. No changes expected.
s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent,
"team_id", fmt.Sprint(tm.ID))
s.awaitTriggerProfileSchedule(t)
installs, removes = checkNextPayloads(t, mdmDevice, false)
require.Empty(t, installs)
require.Empty(t, removes)
// Change the secret variable and upload the profiles again. We should see the profile with updated secret installed.
secretName = "newSecretName"
req = secretVariablesRequest{
SecretVariables: []fleet.SecretVariable{
{
Name: "FLEET_SECRET_NAME",
Value: secretName, // changed
},
{
Name: "FLEET_SECRET_PROFILE",
Value: secretProfile, // did not change
},
},
}
s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp)
s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent,
"team_id", fmt.Sprint(tm.ID))
s.awaitTriggerProfileSchedule(t)
installs, removes = checkNextPayloads(t, mdmDevice, false)
// Manually replace the expected secret variables in the profile
wantTeamProfilesChanged := [][]byte{
teamProfiles[1],
}
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_IDENTIFIER",
secretIdentifier))
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "${FLEET_SECRET_TYPE}", secretType))
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_NAME", secretName))
// verify that we should install the team profiles
s.signedProfilesMatch(wantTeamProfilesChanged, installs)
wantTeamProfiles[1] = wantTeamProfilesChanged[0]
// No profiles should be deleted
assert.Empty(t, removes)
// Clear the profiles using the new (non-deprecated) endpoint.
s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: nil}, http.StatusNoContent, "team_id",
fmt.Sprint(tm.ID), "dry_run", "true")
@ -295,6 +330,47 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() {
// verify that we should install the team profiles
s.signedProfilesMatch(wantTeamProfiles, installs)
// Upload the same profiles again. No changes expected.
s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID), "dry_run", "true")
s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID))
s.awaitTriggerProfileSchedule(t)
installs, removes = checkNextPayloads(t, mdmDevice, false)
require.Empty(t, installs)
require.Empty(t, removes)
// Change the secret variable and upload the profiles again. We should see the profile with updated secret installed.
secretName = "new2SecretName"
req = secretVariablesRequest{
SecretVariables: []fleet.SecretVariable{
{
Name: "FLEET_SECRET_NAME",
Value: secretName, // changed
},
{
Name: "FLEET_SECRET_PROFILE",
Value: secretProfile, // did not change
},
},
}
s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp)
s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID), "dry_run", "true")
s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID))
s.awaitTriggerProfileSchedule(t)
installs, removes = checkNextPayloads(t, mdmDevice, false)
// Manually replace the expected secret variables in the profile
wantTeamProfilesChanged = [][]byte{
teamProfiles[1],
}
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_IDENTIFIER",
secretIdentifier))
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "${FLEET_SECRET_TYPE}", secretType))
wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_NAME", secretName))
// verify that we should install the team profiles
s.signedProfilesMatch(wantTeamProfilesChanged, installs)
wantTeamProfiles[1] = wantTeamProfilesChanged[0]
// No profiles should be deleted
assert.Empty(t, removes)
var hostResp getHostResponse
s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), getHostRequest{}, http.StatusOK, &hostResp)
require.NotEmpty(t, hostResp.Host.MDM.Profiles)
@ -5291,6 +5367,9 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariablesUpload() {
getProfileContents := func(profileUUID string) string {
profile, err := s.ds.GetMDMAppleDeclaration(context.Background(), profileUUID)
require.NoError(s.T(), err)
// Since our DDM profiles contain secrets, the checksum and token should be different
assert.NotNil(s.T(), profile.SecretsUpdatedAt)
assert.NotEqual(s.T(), profile.Checksum, profile.Token)
return string(profile.RawJSON)
}
@ -5424,6 +5503,7 @@ func (s *integrationMDMTestSuite) TestAppleConfigSecretVariablesUpload() {
getProfileContents := func(profileUUID string) string {
profile, err := s.ds.GetMDMAppleConfigProfile(context.Background(), profileUUID)
require.NoError(s.T(), err)
assert.NotNil(s.T(), profile.SecretsUpdatedAt)
return string(profile.Mobileconfig)
}

View file

@ -1636,10 +1636,11 @@ func (svc *Service) BatchSetMDMProfiles(
// In order to map the expanded profiles back to the original profiles, we will use the index.
profilesWithSecrets := make(map[int]fleet.MDMProfileBatchPayload, len(profiles))
for i, p := range profiles {
expanded, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(p.Contents))
expanded, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(p.Contents))
if err != nil {
return err
}
p.SecretsUpdatedAt = secretsUpdatedAt
pCopy := p
// If the profile does not contain secrets, then expanded and original content point to the same slice/memory location.
pCopy.Contents = []byte(expanded)
@ -1912,6 +1913,7 @@ func getAppleProfiles(
}
mdmDecl := fleet.NewMDMAppleDeclaration(prof.Contents, tmID, prof.Name, rawDecl.Type, rawDecl.Identifier)
mdmDecl.SecretsUpdatedAt = prof.SecretsUpdatedAt
for _, labelName := range prof.LabelsIncludeAll {
if lbl, ok := labelMap[labelName]; ok {
declLabel := fleet.ConfigurationProfileLabel{
@ -1967,6 +1969,7 @@ func getAppleProfiles(
}
mdmProf, err := fleet.NewMDMAppleConfigProfile(prof.Contents, tmID)
mdmProf.SecretsUpdatedAt = prof.SecretsUpdatedAt
if err != nil {
return nil, nil, ctxerr.Wrap(ctx,
fleet.NewInvalidArgumentError(prof.Name, err.Error()),

View file

@ -1294,8 +1294,8 @@ func TestMDMBatchSetProfiles(t *testing.T) {
ds.ValidateEmbeddedSecretsFunc = func(ctx context.Context, documents []string) error {
return nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
testCases := []struct {
@ -2113,8 +2113,8 @@ func TestBatchSetMDMProfilesLabels(t *testing.T) {
ds.ValidateEmbeddedSecretsFunc = func(ctx context.Context, documents []string) error {
return nil
}
ds.ExpandEmbeddedSecretsFunc = func(ctx context.Context, document string) (string, error) {
return document, nil
ds.ExpandEmbeddedSecretsAndUpdatedAtFunc = func(ctx context.Context, document string) (string, *time.Time, error) {
return document, nil, nil
}
profiles := []fleet.MDMProfileBatchPayload{