From 1c311b73bed4ad15ed8181ca1b7994f95dedaf60 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Wed, 13 Mar 2024 10:05:46 -0500 Subject: [PATCH 01/30] Fleet in your calendar configs (#17462) Sub-task for #17230 # Configuration changes App configuration: ```yaml integrations: google_calendar: - email: name@service-account.com private_key: *** domain: fleetdm.com ``` Team configuration: ```yaml integrations: google_calendar: email: name@service-account.com enable_calendar_events: true policies: - name: My policy id: 12 webhook_url: https://example.com/policy-remediation ``` Note: Policy is looked up by name when configuration is set. The policy id is set/updated by the server for internal use. # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- cmd/fleetctl/apply_test.go | 116 ++++++++++- cmd/fleetctl/gitops_test.go | 22 +++ .../expectedGetConfigAppConfigJson.json | 3 +- .../expectedGetConfigAppConfigYaml.yml | 1 + ...ectedGetConfigIncludeServerConfigJson.json | 3 +- ...pectedGetConfigIncludeServerConfigYaml.yml | 1 + .../testdata/expectedGetTeamsJson.json | 6 +- .../testdata/expectedGetTeamsYaml.yml | 4 + .../gitops/global_config_no_paths.yml | 4 + .../testdata/gitops/team_config_no_paths.yml | 8 + .../macosSetupExpectedAppConfigEmpty.yml | 1 + .../macosSetupExpectedAppConfigSet.yml | 1 + .../macosSetupExpectedTeam1And2Empty.yml | 4 + .../macosSetupExpectedTeam1And2Set.yml | 4 + .../testdata/macosSetupExpectedTeam1Empty.yml | 2 + ee/server/service/teams.go | 72 ++++++- server/datastore/mysql/policies.go | 37 ++++ server/datastore/mysql/policies_test.go | 42 ++++ server/datastore/mysql/schema.sql | 2 +- server/fleet/app.go | 3 + server/fleet/datastore.go | 1 + server/fleet/integrations.go | 47 ++++- server/fleet/teams.go | 8 + server/mock/datastore_mock.go | 12 ++ server/service/appconfig.go | 1 + server/service/appconfig_test.go | 4 + server/service/client.go | 64 +++++- server/service/integration_core_test.go | 184 ++++++++++++++++++ server/service/integration_enterprise_test.go | 58 ++++++ .../generated_files/appconfig.txt | 4 + 30 files changed, 706 insertions(+), 13 deletions(-) diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index cd9340a19c..f53a78313c 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -143,8 +143,19 @@ func TestApplyTeamSpecs(t *testing.T) { } agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) + googleCalEmail := "service-valid@example.com" ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { - return &fleet.AppConfig{AgentOptions: &agentOpts, MDM: fleet.MDM{EnabledAndConfigured: true}}, nil + return &fleet.AppConfig{ + AgentOptions: &agentOpts, + MDM: fleet.MDM{EnabledAndConfigured: true}, + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Email: googleCalEmail, + }, + }, + }, + }, nil } ds.SaveTeamFunc = func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { @@ -439,6 +450,109 @@ spec: HostPercentage: 25, }, *teamsByName["team1"].Config.WebhookSettings.HostStatusWebhook, ) + + // Apply calendar integration + validPolicyID := uint(10) + validPolicyName := "validPolicy" + ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + var policies = make(map[string]*fleet.Policy) + for _, name := range names { + if name != validPolicyName { + return nil, ¬FoundError{} + } + policies[name] = &fleet.Policy{ + PolicyData: fleet.PolicyData{ID: validPolicyID, TeamID: &teamsByName["team1"].ID, Name: validPolicyName}, + } + } + return policies, nil + } + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: https://example.com/webhook +`, + ) + require.Equal(t, "[+] applied 1 teams\n", runAppForTest(t, []string{"apply", "-f", filename})) + require.NotNil(t, teamsByName["team1"].Config.Integrations.GoogleCalendar) + assert.Equal( + t, fleet.TeamGoogleCalendarIntegration{ + Email: googleCalEmail, + Enable: true, + Policies: []*fleet.PolicyRef{{Name: validPolicyName, ID: validPolicyID}}, + WebhookURL: "https://example.com/webhook", + }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, + ) + + // Apply calendar integration -- invalid email + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: not_present_globally@example.com + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: https://example.com/webhook +`, + ) + + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") + + // Apply calendar integration -- invalid policy name + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: invalidPolicy + webhook_url: https://example.com/webhook +`, + ) + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "name is invalid") + + // Apply calendar integration -- invalid webhook destination + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: bozo +`, + ) + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "invalid URI for request") } func writeTmpYml(t *testing.T, contents string) string { diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 52fdba57ef..048c1750a1 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -360,6 +360,8 @@ func TestFullGlobalGitOps(t *testing.T) { assert.Len(t, appliedScripts, 1) assert.Len(t, appliedMacProfiles, 1) assert.Len(t, appliedWinProfiles, 1) + require.Len(t, savedAppConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].Email) } func TestFullTeamGitOps(t *testing.T) { @@ -389,6 +391,13 @@ func TestFullTeamGitOps(t *testing.T) { EnabledAndConfigured: true, WindowsEnabledAndConfigured: true, }, + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Email: "service@example.com", + }, + }, + }, }, nil } @@ -457,6 +466,12 @@ func TestFullTeamGitOps(t *testing.T) { } return nil, nil } + ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + if slices.Contains(names, "policy1") && slices.Contains(names, "policy2") { + return map[string]*fleet.Policy{"policy1": &policy, "policy2": &policy}, nil + } + return nil, nil + } ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, IDs []uint) ([]uint, error) { policyDeleted = true assert.Equal(t, []uint{policy.ID}, IDs) @@ -536,6 +551,10 @@ func TestFullTeamGitOps(t *testing.T) { assert.Len(t, appliedWinProfiles, 1) assert.True(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "https://example.com/host_status_webhook", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) + require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) + assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) + assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) + assert.Len(t, savedTeam.Config.Integrations.GoogleCalendar.Policies, 2) // Now clear the settings tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml") @@ -569,6 +588,9 @@ team_settings: assert.Equal(t, secret, enrolledSecrets[0].Secret) assert.False(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) + assert.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) + assert.False(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) + assert.Empty(t, savedTeam.Config.Integrations.GoogleCalendar) assert.Empty(t, savedTeam.Config.MDM.MacOSSettings.CustomSettings) assert.Empty(t, savedTeam.Config.MDM.WindowsSettings.CustomSettings.Value) assert.Empty(t, savedTeam.Config.MDM.MacOSUpdates.Deadline.Value) diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json index cf01e91342..985bd03f3e 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json @@ -79,7 +79,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "mdm": { "apple_bm_terms_expired": false, diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml index 7597345851..1e517e952d 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml @@ -11,6 +11,7 @@ spec: enable_host_users: true enable_software_inventory: false integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json index 216dbd75b3..25550bf507 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json @@ -119,7 +119,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "update_interval": { "osquery_detail": "1h0m0s", diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml index 3d614097fa..4ed7f3ed7a 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml @@ -11,6 +11,7 @@ spec: enable_host_users: true enable_software_inventory: false integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/expectedGetTeamsJson.json b/cmd/fleetctl/testdata/expectedGetTeamsJson.json index d19784f2fd..08f5fcbf3e 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsJson.json +++ b/cmd/fleetctl/testdata/expectedGetTeamsJson.json @@ -22,7 +22,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "features": { "enable_host_users": true, @@ -92,7 +93,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "features": { "enable_host_users": false, diff --git a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml index 1249b3e5fd..c0e3ff6dc6 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_updates: @@ -49,6 +51,8 @@ spec: host_expiry_settings: host_expiry_enabled: true host_expiry_window: 15 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_updates: diff --git a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml index 7d0b81e96d..cf2eeece60 100644 --- a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml @@ -137,6 +137,10 @@ org_settings: integrations: jira: [] zendesk: [] + google_calendar: + - email: service@example.com + private_key: google_calendar_private_key + domain: example.com mdm: apple_bm_default_team: "" end_user_authentication: diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 3295e75bbb..608bafcf79 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -15,6 +15,14 @@ team_settings: host_expiry_settings: host_expiry_enabled: true host_expiry_window: 30 + integrations: + google_calendar: + email: service@example.com + enable_calendar_events: true + policies: + - name: policy1 + - name: policy2 + webhook_url: https://example.com/google_calendar_webhook agent_options: command_line_flags: distributed_denylist_duration: 0 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml index a453a2f7f5..cf831eac67 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml @@ -11,6 +11,7 @@ spec: host_expiry_enabled: false host_expiry_window: 0 integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml index 5826922729..7fed6f8836 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml @@ -11,6 +11,7 @@ spec: host_expiry_enabled: false host_expiry_window: 0 integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml index 7315325b48..036b0320ec 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: @@ -40,6 +42,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml index 1cce56630c..d281e9089b 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: @@ -40,6 +42,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml index c6e8b1653b..b685d3488d 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml @@ -9,6 +9,8 @@ spec: features: enable_host_users: false enable_software_inventory: false + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index 95588cd466..f0ad454df4 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/authz" @@ -1068,6 +1070,15 @@ func (svc *Service) editTeamFromSpec( fleet.ValidateEnabledHostStatusIntegrations(*spec.WebhookSettings.HostStatusWebhook, invalid) team.Config.WebhookSettings.HostStatusWebhook = spec.WebhookSettings.HostStatusWebhook } + + if spec.Integrations.GoogleCalendar != nil { + err = svc.validateTeamCalendarIntegrations(ctx, team, spec.Integrations.GoogleCalendar, appCfg, invalid) + if err != nil { + return ctxerr.Wrap(ctx, err, "validate team calendar integrations") + } + team.Config.Integrations.GoogleCalendar = spec.Integrations.GoogleCalendar + } + if invalid.HasErrors() { return ctxerr.Wrap(ctx, invalid) } @@ -1124,7 +1135,9 @@ func (svc *Service) editTeamFromSpec( } if didUpdateMacOSEndUserAuth { - if err := svc.updateMacOSSetupEnableEndUserAuth(ctx, spec.MDM.MacOSSetup.EnableEndUserAuthentication, &team.ID, &team.Name); err != nil { + if err := svc.updateMacOSSetupEnableEndUserAuth( + ctx, spec.MDM.MacOSSetup.EnableEndUserAuthentication, &team.ID, &team.Name, + ); err != nil { return err } } @@ -1132,6 +1145,63 @@ func (svc *Service) editTeamFromSpec( return nil } +func (svc *Service) validateTeamCalendarIntegrations( + ctx context.Context, team *fleet.Team, calendarIntegration *fleet.TeamGoogleCalendarIntegration, + appCfg *fleet.AppConfig, invalid *fleet.InvalidArgumentError, +) error { + if !calendarIntegration.Enable { + return nil + } + // Validate email + emailValid := false + calendarIntegration.Email = strings.TrimSpace(calendarIntegration.Email) + for _, globalCals := range appCfg.Integrations.GoogleCalendar { + if globalCals.Email == calendarIntegration.Email { + emailValid = true + break + } + } + if !emailValid { + invalid.Append("integrations.google_calendar.email", "email must match a global Google Calendar integration email") + } + // Validate URL + if u, err := url.ParseRequestURI(calendarIntegration.WebhookURL); err != nil { + invalid.Append("integrations.google_calendar.webhook_url", err.Error()) + } else if u.Scheme != "https" && u.Scheme != "http" { + invalid.Append("integrations.google_calendar.webhook_url", "webhook_url must be https or http") + } + // Validate policy ids + if len(calendarIntegration.Policies) == 0 { + invalid.Append("integrations.google_calendar.policies", "policies are required") + } + if len(calendarIntegration.Policies) > 0 { + for _, policy := range calendarIntegration.Policies { + policy.Name = strings.TrimSpace(policy.Name) + } + calendarIntegration.Policies = server.RemoveDuplicatesFromSlice(calendarIntegration.Policies) + policyNames := make([]string, 0, len(calendarIntegration.Policies)) + for _, policy := range calendarIntegration.Policies { + policyNames = append(policyNames, policy.Name) + } + // Policies must be team policies. Global policies are not allowed. + policyMap, err := svc.ds.PoliciesByName(ctx, policyNames, team.ID) + if err != nil { + level.Error(svc.logger).Log("msg", "error getting policies by name", "names", policyNames, "err", err) + if fleet.IsNotFound(err) { + invalid.Append("integrations.google_calendar.policies[].name", "name is invalid") + } else { + return err + } + } else { + // PoliciesByName guarantees that all policies are present + for _, policy := range calendarIntegration.Policies { + policy.ID = policyMap[policy.Name].ID + } + } + } + return nil +} + func (svc *Service) applyTeamMacOSSettings(ctx context.Context, spec *fleet.TeamSpec, applyUpon *fleet.MacOSSettings) error { oldCustomSettings := applyUpon.CustomSettings setFields, err := applyUpon.FromMap(spec.MDM.MacOSSettings) diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 69567ec8c5..39c7d3e056 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "golang.org/x/text/unicode/norm" "sort" @@ -444,6 +445,42 @@ func (ds *Datastore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fl return policiesByID, nil } +func (ds *Datastore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + sqlQuery := `SELECT ` + policyCols + ` + FROM policies p + WHERE p.team_id = ? AND p.name IN (?)` + query, args, err := sqlx.In(sqlQuery, teamID, names) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "building query to get policies by name") + } + + var policies []*fleet.Policy + err = sqlx.SelectContext( + ctx, + ds.reader(ctx), + &policies, + query, args..., + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(fmt.Sprintf("%v", names))) + } + return nil, ctxerr.Wrap(ctx, err, "getting policies by name") + } + + policiesByName := make(map[string]*fleet.Policy, len(names)) + for _, p := range policies { + policiesByName[p.Name] = p + } + for _, name := range names { + if policiesByName[name] == nil { + return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(name)) + } + } + + return policiesByName, nil +} + func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { return deletePolicyDB(ctx, ds.writer(ctx), ids, nil) } diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 5e16c9a402..52da13348a 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -38,6 +38,7 @@ func TestPolicies(t *testing.T) { {"PolicyQueriesForHost", testPolicyQueriesForHost}, {"PolicyQueriesForHostPlatforms", testPolicyQueriesForHostPlatforms}, {"PoliciesByID", testPoliciesByID}, + {"PoliciesByName", testPoliciesByName}, {"TeamPolicyTransfer", testTeamPolicyTransfer}, {"ApplyPolicySpec", testApplyPolicySpec}, {"Save", testPoliciesSave}, @@ -1114,6 +1115,47 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { require.ErrorAs(t, err, &nfe) } +func testPoliciesByName(t *testing.T, ds *Datastore) { + ctx := context.Background() + user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) + policyName1 := "policy1" + policyName2 := "policy2" + _ = newTestPolicy(t, ds, user1, policyName1, "darwin", nil) + _ = newTestPolicy(t, ds, user1, policyName2, "darwin", nil) + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) + require.NoError(t, err) + + // No names provided + _, err = ds.PoliciesByName(context.Background(), []string{}, team1.ID) + require.Error(t, err) + + // Policies don't belong to a team + _, err = ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) + require.Error(t, err) + var nfe fleet.NotFoundError + require.ErrorAs(t, err, &nfe) + + policy1 := newTestPolicy(t, ds, user1, policyName1, "darwin", &team1.ID) + policy2 := newTestPolicy(t, ds, user1, policyName2, "darwin", &team1.ID) + + policiesByName, err := ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) + require.NoError(t, err) + require.Len(t, policiesByName, 2) + assert.Equal(t, policiesByName[policyName1].ID, policy1.ID) + assert.Equal(t, policiesByName[policyName2].ID, policy2.ID) + assert.Equal(t, policiesByName[policyName1].Name, policy1.Name) + assert.Equal(t, policiesByName[policyName2].Name, policy2.Name) + + // Policy does not exist + _, err = ds.PoliciesByName(context.Background(), []string{"doesn't exist"}, team1.ID) + assert.ErrorAs(t, err, &nfe) + + // One exists and one doesn't + _, err = ds.PoliciesByName(context.Background(), []string{policyName1, "doesn't exist"}, team1.ID) + assert.ErrorAs(t, err, &nfe) + +} + func testTeamPolicyTransfer(t *testing.T, ds *Datastore) { ctx := context.Background() user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a35608e669..0040d49902 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -41,7 +41,7 @@ CREATE TABLE `app_config_json` ( UNIQUE KEY `id` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); +INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `carve_blocks` ( diff --git a/server/fleet/app.go b/server/fleet/app.go index 778f6fe7eb..4b936063a0 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -488,6 +488,9 @@ func (c *AppConfig) Obfuscate() { for _, zdIntegration := range c.Integrations.Zendesk { zdIntegration.APIToken = MaskedPassword } + for _, calIntegration := range c.Integrations.GoogleCalendar { + calIntegration.PrivateKey = MaskedPassword + } } // Clone implements cloner. diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 4081db8af0..69af794e71 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -588,6 +588,7 @@ type Datastore interface { ListGlobalPolicies(ctx context.Context, opts ListOptions) ([]*Policy, error) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error) + PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*Policy, error) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) CountPolicies(ctx context.Context, teamID *uint, matchQuery string) (int, error) UpdateHostPolicyCounts(ctx context.Context) error diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 131af88806..5f3c5e440a 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -13,8 +13,9 @@ import ( // TeamIntegrations contains the configuration for external services' // integrations for a specific team. type TeamIntegrations struct { - Jira []*TeamJiraIntegration `json:"jira"` - Zendesk []*TeamZendeskIntegration `json:"zendesk"` + Jira []*TeamJiraIntegration `json:"jira"` + Zendesk []*TeamZendeskIntegration `json:"zendesk"` + GoogleCalendar *TeamGoogleCalendarIntegration `json:"google_calendar"` } // MatchWithIntegrations matches the team integrations to their corresponding @@ -110,6 +111,17 @@ func (z TeamZendeskIntegration) UniqueKey() string { return z.URL + "\n" + strconv.FormatInt(z.GroupID, 10) } +type TeamGoogleCalendarIntegration struct { + Email string `json:"email"` + Enable bool `json:"enable_calendar_events"` + Policies []*PolicyRef `json:"policies"` + WebhookURL string `json:"webhook_url"` +} +type PolicyRef struct { + Name string `json:"name"` + ID uint `json:"id"` +} + // JiraIntegration configures an instance of an integration with the Jira // system. type JiraIntegration struct { @@ -335,10 +347,17 @@ func makeTestZendeskRequest(ctx context.Context, intg *ZendeskIntegration) error return nil } +type GoogleCalendarIntegration struct { + Email string `json:"email"` + PrivateKey string `json:"private_key"` + Domain string `json:"domain"` +} + // Integrations configures the integrations with external systems. type Integrations struct { - Jira []*JiraIntegration `json:"jira"` - Zendesk []*ZendeskIntegration `json:"zendesk"` + Jira []*JiraIntegration `json:"jira"` + Zendesk []*ZendeskIntegration `json:"zendesk"` + GoogleCalendar []*GoogleCalendarIntegration `json:"google_calendar"` } // ValidateEnabledHostStatusIntegrations checks that the host status integrations @@ -359,6 +378,26 @@ func ValidateEnabledHostStatusIntegrations(webhook HostStatusWebhookSettings, in } } +func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, invalid *InvalidArgumentError) { + if len(intgs) > 1 { + invalid.Append("integrations.google_calendar", "only one Google Calendar integration is allowed at this time") + } + for _, intg := range intgs { + intg.Email = strings.TrimSpace(intg.Email) + if intg.Email == "" { + invalid.Append("integrations.google_calendar.email", "email is required") + } + intg.PrivateKey = strings.TrimSpace(intg.PrivateKey) + if intg.PrivateKey == "" || intg.PrivateKey == MaskedPassword { + invalid.Append("integrations.google_calendar.private_key", "private_key is required") + } + intg.Domain = strings.TrimSpace(intg.Domain) + if intg.Domain == "" { + invalid.Append("integrations.google_calendar.domain", "domain is required") + } + } +} + // ValidateEnabledVulnerabilitiesIntegrations checks that a single integration // is enabled for vulnerabilities. It adds any error it finds to the invalid // argument error, that can then be checked after the call for errors using diff --git a/server/fleet/teams.go b/server/fleet/teams.go index d1e3a86e98..aee5c7ab95 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -404,12 +404,20 @@ type TeamSpec struct { MDM TeamSpecMDM `json:"mdm"` Scripts optjson.Slice[string] `json:"scripts"` WebhookSettings TeamSpecWebhookSettings `json:"webhook_settings"` + Integrations TeamSpecIntegrations `json:"integrations"` } type TeamSpecWebhookSettings struct { HostStatusWebhook *HostStatusWebhookSettings `json:"host_status_webhook"` } +// TeamSpecIntegrations contains the configuration for external services' +// integrations for a specific team. +type TeamSpecIntegrations struct { + // If value is nil, we don't want to change the existing value. + GoogleCalendar *TeamGoogleCalendarIntegration `json:"google_calendar"` +} + // TeamSpecFromTeam returns a TeamSpec constructed from the given Team. func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { features, err := json.Marshal(t.Config.Features) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 1469826979..603d0616ff 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -432,6 +432,8 @@ type ListGlobalPoliciesFunc func(ctx context.Context, opts fleet.ListOptions) ([ type PoliciesByIDFunc func(ctx context.Context, ids []uint) (map[uint]*fleet.Policy, error) +type PoliciesByNameFunc func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) + type DeleteGlobalPoliciesFunc func(ctx context.Context, ids []uint) ([]uint, error) type CountPoliciesFunc func(ctx context.Context, teamID *uint, matchQuery string) (int, error) @@ -1480,6 +1482,9 @@ type DataStore struct { PoliciesByIDFunc PoliciesByIDFunc PoliciesByIDFuncInvoked bool + PoliciesByNameFunc PoliciesByNameFunc + PoliciesByNameFuncInvoked bool + DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFuncInvoked bool @@ -3571,6 +3576,13 @@ func (s *DataStore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fle return s.PoliciesByIDFunc(ctx, ids) } +func (s *DataStore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + s.mu.Lock() + s.PoliciesByNameFuncInvoked = true + s.mu.Unlock() + return s.PoliciesByNameFunc(ctx, names, teamID) +} + func (s *DataStore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { s.mu.Lock() s.DeleteGlobalPoliciesFuncInvoked = true diff --git a/server/service/appconfig.go b/server/service/appconfig.go index c92cd344cd..88ee842e33 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -385,6 +385,7 @@ func (svc *Service) ModifyAppConfig(ctx context.Context, p []byte, applyOpts fle appConfig.ServerSettings.EnableAnalytics = true } + fleet.ValidateGoogleCalendarIntegrations(appConfig.Integrations.GoogleCalendar, invalid) fleet.ValidateEnabledVulnerabilitiesIntegrations(appConfig.WebhookSettings.VulnerabilitiesWebhook, appConfig.Integrations, invalid) fleet.ValidateEnabledFailingPoliciesIntegrations(appConfig.WebhookSettings.FailingPoliciesWebhook, appConfig.Integrations, invalid) fleet.ValidateEnabledHostStatusIntegrations(appConfig.WebhookSettings.HostStatusWebhook, invalid) diff --git a/server/service/appconfig_test.go b/server/service/appconfig_test.go index 884700bd7e..e23f3b18bd 100644 --- a/server/service/appconfig_test.go +++ b/server/service/appconfig_test.go @@ -489,6 +489,9 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { Zendesk: []*fleet.ZendeskIntegration{ {APIToken: "zendesktoken"}, }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + {PrivateKey: "google-calendar-private-key"}, + }, }, }, nil } @@ -566,6 +569,7 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { require.Equal(t, ac.SMTPSettings.SMTPPassword, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Jira[0].APIToken, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Zendesk[0].APIToken, fleet.MaskedPassword) + require.Equal(t, ac.Integrations.GoogleCalendar[0].PrivateKey, fleet.MaskedPassword) } }) } diff --git a/server/service/client.go b/server/service/client.go index f26bfcc051..b1eb35d694 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -880,14 +880,30 @@ func (c *Client) DoGitOps( } var mdmAppConfig map[string]interface{} var team map[string]interface{} + var teamCalendarIntegration map[string]interface{} if config.TeamName == nil { group.AppConfig = config.OrgSettings group.EnrollSecret = &fleet.EnrollSecretSpec{Secrets: config.OrgSettings["secrets"].([]*fleet.EnrollSecret)} group.AppConfig.(map[string]interface{})["agent_options"] = config.AgentOptions delete(config.OrgSettings, "secrets") // secrets are applied separately in Client.ApplyGroup - if _, ok := group.AppConfig.(map[string]interface{})["mdm"]; !ok { - group.AppConfig.(map[string]interface{})["mdm"] = map[string]interface{}{} + + // Integrations + var integrations interface{} + var ok bool + if integrations, ok = group.AppConfig.(map[string]interface{})["integrations"]; !ok || integrations == nil { + integrations = map[string]interface{}{} + group.AppConfig.(map[string]interface{})["integrations"] = integrations } + if jira, ok := integrations.(map[string]interface{})["jira"]; !ok || jira == nil { + integrations.(map[string]interface{})["jira"] = []interface{}{} + } + if zendesk, ok := integrations.(map[string]interface{})["zendesk"]; !ok || zendesk == nil { + integrations.(map[string]interface{})["zendesk"] = []interface{}{} + } + if googleCal, ok := integrations.(map[string]interface{})["google_calendar"]; !ok || googleCal == nil { + integrations.(map[string]interface{})["google_calendar"] = []interface{}{} + } + // Ensure mdm config exists mdmConfig, ok := group.AppConfig.(map[string]interface{})["mdm"] if !ok || mdmConfig == nil { @@ -941,6 +957,32 @@ func (c *Client) DoGitOps( // Clear out any existing host_status_webhook settings team["webhook_settings"].(map[string]interface{})["host_status_webhook"] = map[string]interface{}{} } + // Integrations + var integrations interface{} + var ok bool + if integrations, ok = config.TeamSettings["integrations"]; !ok || integrations == nil { + integrations = map[string]interface{}{} + } + team["integrations"] = integrations + _, ok = integrations.(map[string]interface{}) + if !ok { + return errors.New("team_settings.integrations config is not a map") + } + if calendar, ok := integrations.(map[string]interface{})["google_calendar"]; ok { + if calendar == nil { + calendar = map[string]interface{}{} + integrations.(map[string]interface{})["google_calendar"] = calendar + } + teamCalendarIntegration, ok = calendar.(map[string]interface{}) + if !ok { + return errors.New("team_settings.integrations.google_calendar config is not a map") + } + } + // We clear the calendar integration and re-apply it after updating policies. + // This is needed because the calendar integration may be referencing policies that need to be + // created/updated. + integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} + team["mdm"] = map[string]interface{}{} mdmAppConfig = team["mdm"].(map[string]interface{}) } @@ -1044,6 +1086,24 @@ func (c *Client) DoGitOps( if err != nil { return err } + + // Apply calendar integration + if len(teamCalendarIntegration) > 0 { + group = spec.Group{} + team = make(map[string]interface{}) + team["name"] = *config.TeamName + team["integrations"] = map[string]interface{}{"google_calendar": teamCalendarIntegration} + rawTeam, err := json.Marshal(team) + if err != nil { + return fmt.Errorf("error marshalling team spec: %w", err) + } + group.Teams = []json.RawMessage{rawTeam} + _, err = c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplySpecOptions{DryRun: dryRun}) + if err != nil { + return err + } + } + err = c.doGitOpsQueries(config, logFn, dryRun) if err != nil { return err diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 2367b7529e..a02913a4e8 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -5171,6 +5171,190 @@ func (s *integrationTestSuite) TestExternalIntegrationsConfig() { require.Len(t, config.Integrations.Zendesk, 0) } +func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { + t := s.T() + email := "service-account@example.com" + privateKey := "-----BEGIN PRIVATE KEY-----\nXXXXX\n-----END" + domain := "example.com" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, privateKey, domain, + )), http.StatusOK, + ) + + appConfig := s.getConfig() + require.Len(t, appConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) + + // Add 2nd config -- not allowed at this time + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }, + { + "email": "bozo@example.com"", + "private_key": "abc", + "domain": "example.com" + }] + } + }`, email, privateKey, domain, + )), http.StatusBadRequest, + ) + + // Make an unrelated config change, should not remove the integrations + var appCfgResp appConfigResponse + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "org_info": { + "org_name": "test-google-calendar-integrations" + } + }`, + ), http.StatusOK, &appCfgResp, + ) + require.Equal(t, "test-google-calendar-integrations", appCfgResp.OrgInfo.OrgName) + require.Len(t, appCfgResp.Integrations.GoogleCalendar, 1) + + // Update calendar config + domain = "new.com" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, privateKey, domain, + )), http.StatusOK, + ) + appConfig = s.getConfig() + require.Len(t, appConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) + + // Clearing other integrations does not clear Google Calendar integration + appCfgResp = appConfigResponse{} + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "integrations": { + "jira": [], + "zendesk": [] + } + }`, + ), http.StatusOK, &appCfgResp, + ) + require.Len(t, appCfgResp.Integrations.GoogleCalendar, 1) + + // Clearing Google Calendar integration + appCfgResp = appConfigResponse{} + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "integrations": { + "google_calendar": [] + } + }`, + ), http.StatusOK, &appCfgResp, + ) + assert.Empty(t, appCfgResp.Integrations.GoogleCalendar) + + // Try adding Google Calendar integration without sending private key -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "domain": %q + }] + } + }`, email, domain, + )), http.StatusUnprocessableEntity, + ) + + // Try adding Google Calendar integration with masked private key -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, fleet.MaskedPassword, domain, + )), http.StatusUnprocessableEntity, + ) + + // Empty email -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": " ", + "private_key": %q, + "domain": %q + }] + } + }`, privateKey, domain, + )), http.StatusUnprocessableEntity, + ) + + // Empty domain -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": "" + }] + } + }`, email, privateKey, + )), http.StatusUnprocessableEntity, + ) + + // Unknown fields fails as bad request + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + "foo": "bar" + }] + } + }`, email, privateKey, domain, + )), http.StatusBadRequest, + ) + +} + func (s *integrationTestSuite) TestQueriesBadRequests() { t := s.T() diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 787207fb23..c9dfed2560 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -97,6 +97,23 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { s.Do("POST", "/api/latest/fleet/teams", team, http.StatusOK) + // Create global calendar integration + calendarEmail := "service@example.com" + calendarWebhookUrl := "https://example.com/webhook" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": "testKey", + "domain": "example.com" + }] + } + }`, calendarEmail, + )), http.StatusOK, + ) + // updates a team, no secret is provided so it will keep the one generated // automatically when the team was created. agentOpts := json.RawMessage(`{"config": {"views": {"foo": "bar"}}, "overrides": {"platforms": {"darwin": {"views": {"bar": "qux"}}}}}`) @@ -163,6 +180,47 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { // an activity was created for team spec applied s.lastActivityMatches(fleet.ActivityTypeAppliedSpecTeam{}.ActivityName(), fmt.Sprintf(`{"teams": [{"id": %d, "name": %q}]}`, team.ID, team.Name), 0) + // Create team policy + teamPolicy, err := s.ds.NewTeamPolicy( + context.Background(), team.ID, nil, fleet.PolicyPayload{Name: "TestSpecTeamPolicy", Query: "SELECT 1"}, + ) + require.NoError(t, err) + defer func() { + _, err = s.ds.DeleteTeamPolicies(context.Background(), team.ID, []uint{teamPolicy.ID}) + require.NoError(t, err) + }() + + // Apply calendar integration + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "integrations": map[string]any{ + "google_calendar": map[string]any{ + "email": calendarEmail, + "enable_calendar_events": true, + "policies": []any{ + map[string]any{ + "name": teamPolicy.Name, + }, + }, + "webhook_url": calendarWebhookUrl, + }, + }, + }, + }, + } + s.DoJSON("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK, &applyResp) + require.Len(t, applyResp.TeamIDsByName, 1) + + team, err = s.ds.TeamByName(context.Background(), teamName) + require.NotNil(t, team.Config.Integrations.GoogleCalendar) + assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) + assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) + assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) + require.Len(t, team.Config.Integrations.GoogleCalendar.Policies, 1) + assert.Equal(t, teamPolicy.ID, team.Config.Integrations.GoogleCalendar.Policies[0].ID) + // dry-run with invalid windows updates teamSpecs = map[string]any{ "specs": []any{ diff --git a/tools/cloner-check/generated_files/appconfig.txt b/tools/cloner-check/generated_files/appconfig.txt index 5480e2432f..7deb8d10fc 100644 --- a/tools/cloner-check/generated_files/appconfig.txt +++ b/tools/cloner-check/generated_files/appconfig.txt @@ -84,6 +84,10 @@ github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration APIToken string github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration GroupID int64 github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableFailingPolicies bool github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableSoftwareVulnerabilities bool +github.com/fleetdm/fleet/v4/server/fleet/Integrations GoogleCalendar []*fleet.GoogleCalendarIntegration +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Email string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration PrivateKey string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Domain string github.com/fleetdm/fleet/v4/server/fleet/AppConfig MDM fleet.MDM github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMDefaultTeam string github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMEnabledAndConfigured bool From be0e89142f69a8b6ce636ec8b286be063667ecb9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Wed, 13 Mar 2024 13:51:21 -0300 Subject: [PATCH 02/30] Add migrations for calendar events (#17585) #17230 --- server/datastore/mysql/hosts.go | 1 + server/datastore/mysql/hosts_test.go | 19 ++++++- .../20240313085226_AddCalendarEventTables.go | 52 ++++++++++++++++++ ...40313085226_AddCalendarEventTables_test.go | 53 +++++++++++++++++++ server/fleet/calendar_events.go | 29 ++++++++++ 5 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go create mode 100644 server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go create mode 100644 server/fleet/calendar_events.go diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index df2f4395bc..ca8986e2e5 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -502,6 +502,7 @@ var hostRefs = []string{ "query_results", "host_activities", "host_mdm_actions", + "host_calendar_events", } // NOTE: The following tables are explicity excluded from hostRefs list and accordingly are not diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index d1dedf0617..b57784b1b3 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -2554,7 +2554,6 @@ func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { h, err = ds.HostLiteByID(context.Background(), 0) assert.ErrorIs(t, err, sql.ErrNoRows) assert.Nil(t, h) - } func testHostsAddToTeam(t *testing.T, ds *Datastore) { @@ -2795,7 +2794,6 @@ func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { assert.Equal(t, 2, total) require.Len(t, unseen, 1) assert.Equal(t, host3.ID, unseen[0]) - } func testHostsListByPolicy(t *testing.T, ds *Datastore) { @@ -6577,6 +6575,23 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) { `, host.ID) require.NoError(t, err) + // Add a calendar event for the host. + _, err = ds.writer(context.Background()).Exec(` + INSERT INTO calendar_events (email, start_time, end_time, event) + VALUES ('foobar@example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, '{}'); + `) + require.NoError(t, err) + var calendarEventID int + err = ds.writer(context.Background()).Get(&calendarEventID, ` + SELECT id FROM calendar_events WHERE email = 'foobar@example.com'; + `) + require.NoError(t, err) + _, err = ds.writer(context.Background()).Exec(` + INSERT INTO host_calendar_events (host_id, calendar_event_id, webhook_status) + VALUES (?, ?, 1); + `, host.ID, calendarEventID) + require.NoError(t, err) + // Check there's an entry for the host in all the associated tables. for _, hostRef := range hostRefs { var ok bool diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go new file mode 100644 index 0000000000..242a11ecbb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go @@ -0,0 +1,52 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240313085226, Down_20240313085226) +} + +func Up_20240313085226(tx *sql.Tx) error { + // TODO(lucas): Check if we need more indexes. + + if _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS calendar_events ( + id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + email VARCHAR(255) NOT NULL, + start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + end_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + event JSON NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ); +`); err != nil { + return fmt.Errorf("create calendar_events table: %w", err) + } + + if _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS host_calendar_events ( + id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + host_id INT(10) UNSIGNED NOT NULL, + calendar_event_id INT(10) UNSIGNED NOT NULL, + webhook_status TINYINT NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + UNIQUE KEY idx_one_calendar_event_per_host (host_id), + FOREIGN KEY (calendar_event_id) REFERENCES calendar_events(id) ON DELETE CASCADE + ); +`); err != nil { + return fmt.Errorf("create host_calendar_events table: %w", err) + } + + return nil +} + +func Down_20240313085226(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go new file mode 100644 index 0000000000..216a3e468b --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go @@ -0,0 +1,53 @@ +package tables + +import ( + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestUp_20240313085226(t *testing.T) { + db := applyUpToPrev(t) + applyNext(t, db) + + sampleEvent := fleet.CalendarEvent{ + Email: "foo@example.com", + StartTime: time.Now().UTC(), + EndTime: time.Now().UTC().Add(30 * time.Minute), + Data: []byte("{\"foo\": \"bar\"}"), + } + sampleEvent.ID = uint(execNoErrLastID(t, db, + `INSERT INTO calendar_events (email, start_time, end_time, event) VALUES (?, ?, ?, ?);`, + sampleEvent.Email, sampleEvent.StartTime, sampleEvent.EndTime, sampleEvent.Data, + )) + + sampleHostEvent := fleet.HostCalendarEvent{ + HostID: 1, + CalendarEventID: sampleEvent.ID, + WebhookStatus: fleet.CalendarWebhookStatusPending, + } + sampleHostEvent.ID = uint(execNoErrLastID(t, db, + `INSERT INTO host_calendar_events (host_id, calendar_event_id, webhook_status) VALUES (?, ?, ?);`, + sampleHostEvent.HostID, sampleHostEvent.CalendarEventID, sampleHostEvent.WebhookStatus, + )) + + var event fleet.CalendarEvent + err := db.Get(&event, `SELECT * FROM calendar_events WHERE id = ?;`, sampleEvent.ID) + require.NoError(t, err) + sampleEvent.CreatedAt = event.CreatedAt // sampleEvent doesn't have this set. + sampleEvent.UpdatedAt = event.UpdatedAt // sampleEvent doesn't have this set. + sampleEvent.StartTime = sampleEvent.StartTime.Round(time.Second) + sampleEvent.EndTime = sampleEvent.EndTime.Round(time.Second) + event.StartTime = event.StartTime.Round(time.Second) + event.EndTime = event.EndTime.Round(time.Second) + require.Equal(t, sampleEvent, event) + + var hostEvent fleet.HostCalendarEvent + err = db.Get(&hostEvent, `SELECT * FROM host_calendar_events WHERE id = ?;`, sampleHostEvent.ID) + require.NoError(t, err) + sampleHostEvent.CreatedAt = hostEvent.CreatedAt // sampleHostEvent doesn't have this set. + sampleHostEvent.UpdatedAt = hostEvent.UpdatedAt // sampleHostEvent doesn't have this set. + require.Equal(t, sampleHostEvent, hostEvent) +} diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go new file mode 100644 index 0000000000..7671b4aba5 --- /dev/null +++ b/server/fleet/calendar_events.go @@ -0,0 +1,29 @@ +package fleet + +import "time" + +type CalendarEvent struct { + ID uint `db:"id"` + Email string `db:"email"` + StartTime time.Time `db:"start_time"` + EndTime time.Time `db:"end_time"` + Data []byte `db:"event"` + + UpdateCreateTimestamps +} + +type CalendarWebhookStatus int + +const ( + CalendarWebhookStatusPending CalendarWebhookStatus = iota + CalendarWebhookStatusSent +) + +type HostCalendarEvent struct { + ID uint `db:"id"` + HostID uint `db:"host_id"` + CalendarEventID uint `db:"calendar_event_id"` + WebhookStatus CalendarWebhookStatus `db:"webhook_status"` + + UpdateCreateTimestamps +} From c9b917a49105110964ef35bc7777f9e6d908aed5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 14:07:13 -0500 Subject: [PATCH 03/30] Calendar interface (#17633) # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features. - [ ] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/calendar/calendar.go | 16 ++ ee/server/calendar/google_calendar.go | 393 ++++++++++++++++++++++++++ server/fleet/calendar.go | 24 ++ 3 files changed, 433 insertions(+) create mode 100644 ee/server/calendar/calendar.go create mode 100644 ee/server/calendar/google_calendar.go create mode 100644 server/fleet/calendar.go diff --git a/ee/server/calendar/calendar.go b/ee/server/calendar/calendar.go new file mode 100644 index 0000000000..469437f3a0 --- /dev/null +++ b/ee/server/calendar/calendar.go @@ -0,0 +1,16 @@ +package calendar + +import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" +) + +type GoogleCalendarConfig struct { + Context context.Context + IntegrationConfig *fleet.GoogleCalendarIntegration + UserEmail string + Logger log.Logger + // Should be nil for production + API GoogleCalendarAPI +} diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go new file mode 100644 index 0000000000..ff3d2dfb80 --- /dev/null +++ b/ee/server/calendar/google_calendar.go @@ -0,0 +1,393 @@ +package calendar + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/log/level" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + "net/http" + "time" +) + +const ( + eventTitle = "💻🚫Downtime" + startHour = 9 + endHour = 17 + eventLength = 30 * time.Minute + calendarID = "primary" +) + +var calendarScopes = []string{ + "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/calendar.settings.readonly", +} + +// GoogleCalendar is an implementation of the Calendar interface that uses the +// Google Calendar API to manage events. +type GoogleCalendar struct { + config *GoogleCalendarConfig + timezoneOffset *int +} + +type GoogleCalendarAPI interface { + Connect(ctx context.Context, email, privateKey, subject string) error + GetSetting(name string) (*calendar.Setting, error) + ListEvents(timeMin, timeMax string) (*calendar.Events, error) + CreateEvent(event *calendar.Event) (*calendar.Event, error) + GetEvent(id, eTag string) (*calendar.Event, error) + DeleteEvent(id string) error +} + +type eventDetails struct { + ID string `json:"id"` + ETag string `json:"etag"` +} + +type GoogleCalendarLowLevelAPI struct { + service *calendar.Service +} + +// Connect creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarLowLevelAPI) Connect(ctx context.Context, email, privateKey, subject string) error { + // Create a new calendar service + conf := &jwt.Config{ + Email: email, + Scopes: calendarScopes, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: subject, + } + client := conf.Client(ctx) + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + return err + } + lowLevelAPI.service = service + return nil +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { + return lowLevelAPI.service.Settings.Get(name).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + return lowLevelAPI.service.Events.Insert(calendarID, event).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { + return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { + // Default maximum number of events returned is 250, which should be sufficient for most calendars. + return lowLevelAPI.service.Events.List(calendarID).EventTypes("default").OrderBy("startTime").SingleEvents(true).TimeMin(timeMin).TimeMax(timeMax).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { + return lowLevelAPI.service.Events.Delete(calendarID, id).Do() +} + +func (c *GoogleCalendar) Connect(config any) (fleet.Calendar, error) { + gConfig, ok := config.(*GoogleCalendarConfig) + if !ok { + return nil, errors.New("invalid Google calendar config") + } + if gConfig.API == nil { + var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} + gConfig.API = lowLevelAPI + } + err := gConfig.API.Connect( + gConfig.Context, gConfig.IntegrationConfig.Email, gConfig.IntegrationConfig.PrivateKey, gConfig.UserEmail, + ) + if err != nil { + return nil, ctxerr.Wrap(gConfig.Context, err, "creating Google calendar service") + } + + gCal := &GoogleCalendar{ + config: gConfig, + } + + return gCal, nil +} + +func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { + if c.config == nil { + return nil, false, errors.New("the Google calendar is not connected. Please call Connect first") + } + if event.EndTime.Before(time.Now()) { + return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) + } + details, err := c.unmarshalDetails(event) + if err != nil { + return nil, false, err + } + gEvent, err := c.config.API.GetEvent(details.ID, details.ETag) + var deleted bool + switch { + // http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later + case googleapi.IsNotModified(err): + return event, false, nil + case isNotFound(err): + deleted = true + case err != nil: + return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event") + } + if !deleted && gEvent.Status != "cancelled" { + if details.ETag == gEvent.Etag { + // Event was not modified + return event, false, nil + } + endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + if err != nil { + return nil, false, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), + ) + } + // If event already ended, it is effectively deleted + if endTime.After(time.Now()) { + startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + if err != nil { + return nil, false, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), + ) + } + fleetEvent, err := c.googleEventToFleetEvent(startTime, endTime, gEvent) + if err != nil { + return nil, false, err + } + return fleetEvent, true, nil + } + } + + newStartDate := event.StartTime.Add(24 * time.Hour) + if newStartDate.Weekday() == time.Saturday { + newStartDate = newStartDate.Add(48 * time.Hour) + } else if newStartDate.Weekday() == time.Sunday { + newStartDate = newStartDate.Add(24 * time.Hour) + } + + fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) + if err != nil { + return nil, false, err + } + return fleetEvent, true, nil +} + +func isNotFound(err error) bool { + if err == nil { + return false + } + var ae *googleapi.Error + ok := errors.As(err, &ae) + return ok && ae.Code == http.StatusNotFound +} + +func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { + var details eventDetails + err := json.Unmarshal(event.Data, &details) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "unmarshaling Google calendar event details") + } + if details.ID == "" { + return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ID") + } + if details.ETag == "" { + return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ETag") + } + return &details, nil +} + +func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { + if c.config == nil { + return nil, errors.New("the Google calendar is not connected. Please call Connect first") + } + if c.timezoneOffset == nil { + err := getTimezone(c) + if err != nil { + return nil, err + } + } + + location := time.FixedZone("", *c.timezoneOffset) + dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location) + dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location) + + now := time.Now().In(location) + if dayEnd.Before(now) { + // The workday has already ended. + return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"}) + } + + // Adjust day start if workday already started + if dayStart.Before(now) { + dayStart = now.Truncate(eventLength) + if dayStart.Before(now) { + dayStart = dayStart.Add(eventLength) + } + if dayStart.Equal(dayEnd) { + return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "no time available for event"}) + } + } + eventStart := dayStart + eventEnd := dayStart.Add(eventLength) + + searchStart := dayStart.Add(-24 * time.Hour) + events, err := c.config.API.ListEvents(searchStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") + } + for _, gEvent := range events.Items { + // Ignore cancelled events + if gEvent.Status == "cancelled" { + continue + } + + // Ignore events that the user has declined + var attending bool + if len(gEvent.Attendees) == 0 { + // No attendees, so we assume the user is attending + attending = true + } else { + for _, attendee := range gEvent.Attendees { + if attendee.Email == c.config.UserEmail { + if attendee.ResponseStatus != "declined" { + attending = true + } + break + } + } + } + if !attending { + continue + } + + // Ignore events that will end before our event + endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), + ) + } + if endTime.Before(eventStart) || endTime.Equal(eventStart) { + continue + } + + startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), + ) + } + + if startTime.Before(eventEnd) { + // Event occurs during our event, so we need to adjust. + fmt.Printf("VICTOR Adjusting event times due to %s: %s - %s\n", gEvent.Summary, eventStart, eventEnd) + var isLastSlot bool + eventStart, eventEnd, isLastSlot = adjustEventTimes(endTime, dayEnd) + if isLastSlot { + break + } + continue + } + // Since events are sorted by startTime, all subsequent events are after our event, so we can stop processing + break + } + + event := &calendar.Event{} + event.Start = &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)} + event.End = &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)} + event.Summary = eventTitle + event.Description = body + event, err = c.config.API.CreateEvent(event) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "creating Google calendar event") + } + + // Convert Google event to Fleet event + fleetEvent, err := c.googleEventToFleetEvent(eventStart, eventEnd, event) + if err != nil { + return nil, err + } + level.Debug(c.config.Logger).Log("msg", "created Google calendar events", "user", c.config.UserEmail, "startTime", eventStart) + fmt.Printf("VICTOR Created event with id:%s and ETag:%s\n", event.Id, event.Etag) + + return fleetEvent, nil +} + +func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool) { + eventStart = endTime.Truncate(eventLength) + if eventStart.Before(endTime) { + eventStart = eventStart.Add(eventLength) + } + eventEnd = eventStart.Add(eventLength) + // If we are at the end of the day, pick the last slot + if eventEnd.After(dayEnd) { + eventEnd = dayEnd + eventStart = eventEnd.Add(-eventLength) + isLastSlot = true + } + if eventEnd.Equal(dayEnd) { + isLastSlot = true + } + return eventStart, eventEnd, isLastSlot +} + +func getTimezone(gCal *GoogleCalendar) error { + config := gCal.config + setting, err := config.API.GetSetting("timezone") + if err != nil { + return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone") + } + + loc, err := time.LoadLocation(setting.Value) + if err != nil { + // Could not load location, use EST + level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", setting.Value, "err", err) + loc, _ = time.LoadLocation("America/New_York") + } + _, timezoneOffset := time.Now().In(loc).Zone() + gCal.timezoneOffset = &timezoneOffset + return nil +} + +func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime time.Time, event *calendar.Event) ( + *fleet.CalendarEvent, error, +) { + fleetEvent := &fleet.CalendarEvent{} + fleetEvent.StartTime = startTime + fleetEvent.EndTime = endTime + fleetEvent.Email = c.config.UserEmail + details := &eventDetails{ + ID: event.Id, + ETag: event.Etag, + } + detailsJson, err := json.Marshal(details) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "marshaling Google calendar event details") + } + fleetEvent.Data = detailsJson + return fleetEvent, nil +} + +func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { + if c.config == nil { + return errors.New("the Google calendar is not connected. Please call Connect first") + } + details, err := c.unmarshalDetails(event) + if err != nil { + return err + } + err = c.config.API.DeleteEvent(details.ID) + if err != nil { + return ctxerr.Wrap(c.config.Context, err, "deleting Google calendar event") + } + return nil +} diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go new file mode 100644 index 0000000000..dc682e24ea --- /dev/null +++ b/server/fleet/calendar.go @@ -0,0 +1,24 @@ +package fleet + +import "time" + +type DayEndedError struct { + Msg string +} + +func (e DayEndedError) Error() string { + return e.Msg +} + +type Calendar interface { + // Connect to calendar. This method must be called first. Currently, config must be a *GoogleCalendarConfig + Connect(config any) (Calendar, error) + // GetAndUpdateEvent retrieves the event from the calendar. + // If the event has been modified, it returns the updated event. + // If the event has been deleted, it schedules a new event with given body callback and returns the new event. + GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) + // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. + CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) + // DeleteEvent deletes the event with the given ID. + DeleteEvent(event *CalendarEvent) error +} From d3e1716572f8432627649d3735b80246b1e869b5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 15:01:52 -0500 Subject: [PATCH 04/30] Calendar config API endpoints bug fixes. (#17640) Bug fixes for frontend - google_calendar can be nil for global config to indicate that it should not change - `fleet/teams/:id` endpoint now working --- ee/server/service/teams.go | 9 +++++++++ server/service/appconfig.go | 4 ++++ server/service/integration_enterprise_test.go | 15 +++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index f0ad454df4..f573c89faa 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -209,6 +209,15 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T team.Config.Integrations.Jira = payload.Integrations.Jira team.Config.Integrations.Zendesk = payload.Integrations.Zendesk + // Only update the google calendar integration if it's not nil + if payload.Integrations.GoogleCalendar != nil { + invalid := &fleet.InvalidArgumentError{} + _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) + if invalid.HasErrors() { + return nil, ctxerr.Wrap(ctx, invalid) + } + team.Config.Integrations.GoogleCalendar = payload.Integrations.GoogleCalendar + } } if payload.WebhookSettings != nil || payload.Integrations != nil { diff --git a/server/service/appconfig.go b/server/service/appconfig.go index 88ee842e33..8a346183de 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -478,6 +478,10 @@ func (svc *Service) ModifyAppConfig(ctx context.Context, p []byte, applyOpts fle } } } + // If google_calendar is null, we keep the existing setting. If it's not null, we update. + if newAppConfig.Integrations.GoogleCalendar == nil { + appConfig.Integrations.GoogleCalendar = oldAppConfig.Integrations.GoogleCalendar + } if !license.IsPremium() { // reset transparency url to empty for downgraded licenses diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index c9dfed2560..2484ce0d11 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -1029,6 +1029,21 @@ func (s *integrationEnterpriseTestSuite) TestTeamEndpoints() { modifyExpiry.HostExpirySettings.HostExpiryWindow = 0 s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyExpiry, http.StatusUnprocessableEntity, &tmResp) + // Modify team's calendar config + modifyCalendar := fleet.TeamPayload{ + Integrations: &fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Email: "calendar@example.com", + }, + }, + } + s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyCalendar, http.StatusOK, &tmResp) + assert.Equal(t, modifyCalendar.Integrations.GoogleCalendar, tmResp.Team.Config.Integrations.GoogleCalendar) + + // Illegal team calendar config + modifyCalendar.Integrations.GoogleCalendar.Enable = true + s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyCalendar, http.StatusUnprocessableEntity, &tmResp) + // list team users var usersResp listUsersResponse s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/users", tm1ID), nil, http.StatusOK, &usersResp) From 63e9d49dfc687741376ec9a4c1e3e4a1db60146c Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 19:00:51 -0500 Subject: [PATCH 05/30] Calendar config updates -- policy table now has calendar_events_enabled (#17645) # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Added/updated tests - [x] If database migrations are included, checked table schema to confirm autoupdate - For database migrations: - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration. - [x] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects. - [x] Manual QA for all new/changed functionality --- cmd/fleetctl/apply_test.go | 41 ------- cmd/fleetctl/get_test.go | 38 ++++--- cmd/fleetctl/gitops_test.go | 7 -- .../expectedHostDetailResponseJson.json | 6 +- .../expectedHostDetailResponseYaml.yml | 2 + .../testdata/gitops/team_config_no_paths.yml | 4 +- ee/server/service/teams.go | 53 +++------ ...40314151747_AddCalendarEventsToPolicies.go | 22 ++++ ...151747_AddCalendarEventsToPolicies_test.go | 42 +++++++ server/datastore/mysql/policies.go | 55 ++------- server/datastore/mysql/policies_test.go | 107 +++++++----------- server/fleet/datastore.go | 1 - server/fleet/integrations.go | 13 +-- server/fleet/policies.go | 8 ++ server/fleet/teams.go | 6 + server/mock/datastore_mock.go | 12 -- server/service/client.go | 32 +----- server/service/integration_enterprise_test.go | 63 +++++------ server/service/team_policies.go | 35 +++--- server/webhooks/failing_policies_test.go | 27 +++-- 20 files changed, 241 insertions(+), 333 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go create mode 100644 server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index f53a78313c..655e2e57b0 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -452,20 +452,6 @@ spec: ) // Apply calendar integration - validPolicyID := uint(10) - validPolicyName := "validPolicy" - ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - var policies = make(map[string]*fleet.Policy) - for _, name := range names { - if name != validPolicyName { - return nil, ¬FoundError{} - } - policies[name] = &fleet.Policy{ - PolicyData: fleet.PolicyData{ID: validPolicyID, TeamID: &teamsByName["team1"].ID, Name: validPolicyName}, - } - } - return policies, nil - } filename = writeTmpYml( t, ` apiVersion: v1 @@ -477,8 +463,6 @@ spec: google_calendar: email: `+googleCalEmail+` enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: https://example.com/webhook `, ) @@ -488,7 +472,6 @@ spec: t, fleet.TeamGoogleCalendarIntegration{ Email: googleCalEmail, Enable: true, - Policies: []*fleet.PolicyRef{{Name: validPolicyName, ID: validPolicyID}}, WebhookURL: "https://example.com/webhook", }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, ) @@ -505,8 +488,6 @@ spec: google_calendar: email: not_present_globally@example.com enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: https://example.com/webhook `, ) @@ -514,26 +495,6 @@ spec: _, err = runAppNoChecks([]string{"apply", "-f", filename}) assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") - // Apply calendar integration -- invalid policy name - filename = writeTmpYml( - t, ` -apiVersion: v1 -kind: team -spec: - team: - name: team1 - integrations: - google_calendar: - email: `+googleCalEmail+` - enable_calendar_events: true - policies: - - name: invalidPolicy - webhook_url: https://example.com/webhook -`, - ) - _, err = runAppNoChecks([]string{"apply", "-f", filename}) - assert.ErrorContains(t, err, "name is invalid") - // Apply calendar integration -- invalid webhook destination filename = writeTmpYml( t, ` @@ -546,8 +507,6 @@ spec: google_calendar: email: `+googleCalEmail+` enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: bozo `, ) diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 8bf99d7742..07bb5b3e1b 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -331,29 +331,31 @@ func TestGetHosts(t *testing.T) { return []*fleet.HostPolicy{ { PolicyData: fleet.PolicyData{ - ID: 1, - Name: "query1", - Query: defaultPolicyQuery, - Description: "Some description", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - Resolution: ptr.String("Some resolution"), - TeamID: ptr.Uint(1), + ID: 1, + Name: "query1", + Query: defaultPolicyQuery, + Description: "Some description", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + Resolution: ptr.String("Some resolution"), + TeamID: ptr.Uint(1), + CalendarEventsEnabled: true, }, Response: "passes", }, { PolicyData: fleet.PolicyData{ - ID: 2, - Name: "query2", - Query: defaultPolicyQuery, - Description: "", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - Resolution: nil, - TeamID: nil, + ID: 2, + Name: "query2", + Query: defaultPolicyQuery, + Description: "", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + Resolution: nil, + TeamID: nil, + CalendarEventsEnabled: false, }, Response: "fails", }, diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 048c1750a1..c6013aa959 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -466,12 +466,6 @@ func TestFullTeamGitOps(t *testing.T) { } return nil, nil } - ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - if slices.Contains(names, "policy1") && slices.Contains(names, "policy2") { - return map[string]*fleet.Policy{"policy1": &policy, "policy2": &policy}, nil - } - return nil, nil - } ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, IDs []uint) ([]uint, error) { policyDeleted = true assert.Equal(t, []uint{policy.ID}, IDs) @@ -554,7 +548,6 @@ func TestFullTeamGitOps(t *testing.T) { require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) - assert.Len(t, savedTeam.Config.Integrations.GoogleCalendar.Policies, 2) // Now clear the settings tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml") diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index 8c6e8dc3a7..0a05fb8b08 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -76,7 +76,8 @@ "team_id": 1, "updated_at": "0001-01-01T00:00:00Z", "created_at": "0001-01-01T00:00:00Z", - "critical": false + "critical": false, + "calendar_events_enabled": true }, { "id": 2, @@ -91,7 +92,8 @@ "team_id": null, "updated_at": "0001-01-01T00:00:00Z", "created_at": "0001-01-01T00:00:00Z", - "critical": false + "critical": false, + "calendar_events_enabled": false } ], "status": "offline", diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index fc4431c9e1..5b1f81d4a1 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -62,6 +62,7 @@ spec: created_at: "0001-01-01T00:00:00Z" updated_at: "0001-01-01T00:00:00Z" critical: false + calendar_events_enabled: true - author_email: "alice@example.com" author_id: 1 author_name: Alice @@ -75,6 +76,7 @@ spec: created_at: "0001-01-01T00:00:00Z" updated_at: "0001-01-01T00:00:00Z" critical: false + calendar_events_enabled: false policy_updated_at: "0001-01-01T00:00:00Z" public_ip: "" primary_ip: "" diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 608bafcf79..58564a7bc2 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -19,9 +19,6 @@ team_settings: google_calendar: email: service@example.com enable_calendar_events: true - policies: - - name: policy1 - - name: policy2 webhook_url: https://example.com/google_calendar_webhook agent_options: command_line_flags: @@ -97,6 +94,7 @@ policies: description: This policy should always fail. resolution: There is no resolution for this policy. query: SELECT 1 FROM osquery_info WHERE start_time < 0; + calendar_events_enabled: true - name: Passing policy platform: linux,windows,darwin,chrome description: This policy should always pass. diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index f573c89faa..e0f503b381 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -197,19 +197,21 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T } if payload.Integrations != nil { - // the team integrations must reference an existing global config integration. - if _, err := payload.Integrations.MatchWithIntegrations(appCfg.Integrations); err != nil { - return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) - } + if payload.Integrations.Jira != nil || payload.Integrations.Zendesk != nil { + // the team integrations must reference an existing global config integration. + if _, err := payload.Integrations.MatchWithIntegrations(appCfg.Integrations); err != nil { + return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) + } - // integrations must be unique - if err := payload.Integrations.Validate(); err != nil { - return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) - } + // integrations must be unique + if err := payload.Integrations.Validate(); err != nil { + return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) + } - team.Config.Integrations.Jira = payload.Integrations.Jira - team.Config.Integrations.Zendesk = payload.Integrations.Zendesk - // Only update the google calendar integration if it's not nil + team.Config.Integrations.Jira = payload.Integrations.Jira + team.Config.Integrations.Zendesk = payload.Integrations.Zendesk + } + // Only update the calendar integration if it's not nil if payload.Integrations.GoogleCalendar != nil { invalid := &fleet.InvalidArgumentError{} _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) @@ -1179,35 +1181,6 @@ func (svc *Service) validateTeamCalendarIntegrations( } else if u.Scheme != "https" && u.Scheme != "http" { invalid.Append("integrations.google_calendar.webhook_url", "webhook_url must be https or http") } - // Validate policy ids - if len(calendarIntegration.Policies) == 0 { - invalid.Append("integrations.google_calendar.policies", "policies are required") - } - if len(calendarIntegration.Policies) > 0 { - for _, policy := range calendarIntegration.Policies { - policy.Name = strings.TrimSpace(policy.Name) - } - calendarIntegration.Policies = server.RemoveDuplicatesFromSlice(calendarIntegration.Policies) - policyNames := make([]string, 0, len(calendarIntegration.Policies)) - for _, policy := range calendarIntegration.Policies { - policyNames = append(policyNames, policy.Name) - } - // Policies must be team policies. Global policies are not allowed. - policyMap, err := svc.ds.PoliciesByName(ctx, policyNames, team.ID) - if err != nil { - level.Error(svc.logger).Log("msg", "error getting policies by name", "names", policyNames, "err", err) - if fleet.IsNotFound(err) { - invalid.Append("integrations.google_calendar.policies[].name", "name is invalid") - } else { - return err - } - } else { - // PoliciesByName guarantees that all policies are present - for _, policy := range calendarIntegration.Policies { - policy.ID = policyMap[policy.Name].ID - } - } - } return nil } diff --git a/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go new file mode 100644 index 0000000000..485993bf75 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go @@ -0,0 +1,22 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240314151747, Down_20240314151747) +} + +func Up_20240314151747(tx *sql.Tx) error { + _, err := tx.Exec(`ALTER TABLE policies ADD COLUMN calendar_events_enabled TINYINT(1) UNSIGNED NOT NULL DEFAULT '0'`) + if err != nil { + return fmt.Errorf("failed to add calendar_events_enabled to policies: %w", err) + } + return nil +} + +func Down_20240314151747(_ *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go new file mode 100644 index 0000000000..2deb81a9f1 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go @@ -0,0 +1,42 @@ +package tables + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestUp_20240314151747(t *testing.T) { + db := applyUpToPrev(t) + + policy1 := execNoErrLastID( + t, db, "INSERT INTO policies (name, query, description, checksum) VALUES (?,?,?,?)", "policy", "", "", "checksum", + ) + + // Apply current migration. + applyNext(t, db) + + var policyCheck []struct { + ID int64 `db:"id"` + CalEnabled bool `db:"calendar_events_enabled"` + } + err := db.SelectContext(context.Background(), &policyCheck, `SELECT id, calendar_events_enabled FROM policies ORDER BY id`) + require.NoError(t, err) + require.Len(t, policyCheck, 1) + assert.Equal(t, policy1, policyCheck[0].ID) + assert.Equal(t, false, policyCheck[0].CalEnabled) + + policy2 := execNoErrLastID( + t, db, "INSERT INTO policies (name, query, description, checksum, calendar_events_enabled) VALUES (?,?,?,?,?)", "policy2", "", "", + "checksum2", 1, + ) + + policyCheck = nil + err = db.SelectContext(context.Background(), &policyCheck, `SELECT id, calendar_events_enabled FROM policies WHERE id = ?`, policy2) + require.NoError(t, err) + require.Len(t, policyCheck, 1) + assert.Equal(t, policy2, policyCheck[0].ID) + assert.Equal(t, true, policyCheck[0].CalEnabled) + +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 39c7d3e056..d2f2424072 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" "golang.org/x/text/unicode/norm" "sort" @@ -20,7 +19,7 @@ import ( const policyCols = ` p.id, p.team_id, p.resolution, p.name, p.query, p.description, - p.author_id, p.platforms, p.created_at, p.updated_at, p.critical + p.author_id, p.platforms, p.created_at, p.updated_at, p.critical, p.calendar_events_enabled ` var policySearchColumns = []string{"p.name"} @@ -116,10 +115,12 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo p.Name = norm.NFC.String(p.Name) sql := ` UPDATE policies - SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, checksum = ` + policiesChecksumComputedColumn() + ` + SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, calendar_events_enabled = ?, checksum = ` + policiesChecksumComputedColumn() + ` WHERE id = ? ` - result, err := ds.writer(ctx).ExecContext(ctx, sql, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.ID) + result, err := ds.writer(ctx).ExecContext( + ctx, sql, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.ID, + ) if err != nil { return ctxerr.Wrap(ctx, err, "updating policy") } @@ -445,42 +446,6 @@ func (ds *Datastore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fl return policiesByID, nil } -func (ds *Datastore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - sqlQuery := `SELECT ` + policyCols + ` - FROM policies p - WHERE p.team_id = ? AND p.name IN (?)` - query, args, err := sqlx.In(sqlQuery, teamID, names) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building query to get policies by name") - } - - var policies []*fleet.Policy - err = sqlx.SelectContext( - ctx, - ds.reader(ctx), - &policies, - query, args..., - ) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(fmt.Sprintf("%v", names))) - } - return nil, ctxerr.Wrap(ctx, err, "getting policies by name") - } - - policiesByName := make(map[string]*fleet.Policy, len(names)) - for _, p := range policies { - policiesByName[p.Name] = p - } - for _, name := range names { - if policiesByName[name] == nil { - return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(name)) - } - } - - return policiesByName, nil -} - func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { return deletePolicyDB(ctx, ds.writer(ctx), ids, nil) } @@ -562,10 +527,11 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u nameUnicode := norm.NFC.String(args.Name) res, err := ds.writer(ctx).ExecContext(ctx, fmt.Sprintf( - `INSERT INTO policies (name, query, description, team_id, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)`, + `INSERT INTO policies (name, query, description, team_id, resolution, author_id, platforms, critical, calendar_events_enabled, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, %s)`, policiesChecksumComputedColumn(), ), nameUnicode, args.Query, args.Description, teamID, args.Resolution, authorID, args.Platform, args.Critical, + args.CalendarEventsEnabled, ) switch { case err == nil: @@ -623,15 +589,17 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs team_id, platforms, critical, + calendar_events_enabled, checksum - ) VALUES ( ?, ?, ?, ?, ?, (SELECT IFNULL(MIN(id), NULL) FROM teams WHERE name = ?), ?, ?, %s) + ) VALUES ( ?, ?, ?, ?, ?, (SELECT IFNULL(MIN(id), NULL) FROM teams WHERE name = ?), ?, ?, ?, %s) ON DUPLICATE KEY UPDATE query = VALUES(query), description = VALUES(description), author_id = VALUES(author_id), resolution = VALUES(resolution), platforms = VALUES(platforms), - critical = VALUES(critical) + critical = VALUES(critical), + calendar_events_enabled = VALUES(calendar_events_enabled) `, policiesChecksumComputedColumn(), ) for _, spec := range specs { @@ -640,6 +608,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs spec.Name = norm.NFC.String(spec.Name) res, err := tx.ExecContext(ctx, query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, spec.Team, spec.Platform, spec.Critical, + spec.CalendarEventsEnabled, ) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert") diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 52da13348a..b0ef3b1bcc 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -38,7 +38,6 @@ func TestPolicies(t *testing.T) { {"PolicyQueriesForHost", testPolicyQueriesForHost}, {"PolicyQueriesForHostPlatforms", testPolicyQueriesForHostPlatforms}, {"PoliciesByID", testPoliciesByID}, - {"PoliciesByName", testPoliciesByName}, {"TeamPolicyTransfer", testTeamPolicyTransfer}, {"ApplyPolicySpec", testApplyPolicySpec}, {"Save", testPoliciesSave}, @@ -583,10 +582,11 @@ func testTeamPolicyProprietary(t *testing.T, ds *Datastore) { require.Error(t, err) p, err := ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, fleet.PolicyPayload{ - Name: "query1", - Query: "select 1;", - Description: "query1 desc", - Resolution: "query1 resolution", + Name: "query1", + Query: "select 1;", + Description: "query1 desc", + Resolution: "query1 resolution", + CalendarEventsEnabled: true, }) require.NoError(t, err) @@ -616,6 +616,7 @@ func testTeamPolicyProprietary(t *testing.T, ds *Datastore) { assert.Equal(t, "query1 resolution", *p.Resolution) require.NotNil(t, p.AuthorID) assert.Equal(t, user1.ID, *p.AuthorID) + assert.True(t, p.CalendarEventsEnabled) globalPolicies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) require.NoError(t, err) @@ -1115,47 +1116,6 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { require.ErrorAs(t, err, &nfe) } -func testPoliciesByName(t *testing.T, ds *Datastore) { - ctx := context.Background() - user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) - policyName1 := "policy1" - policyName2 := "policy2" - _ = newTestPolicy(t, ds, user1, policyName1, "darwin", nil) - _ = newTestPolicy(t, ds, user1, policyName2, "darwin", nil) - team1, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) - require.NoError(t, err) - - // No names provided - _, err = ds.PoliciesByName(context.Background(), []string{}, team1.ID) - require.Error(t, err) - - // Policies don't belong to a team - _, err = ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) - require.Error(t, err) - var nfe fleet.NotFoundError - require.ErrorAs(t, err, &nfe) - - policy1 := newTestPolicy(t, ds, user1, policyName1, "darwin", &team1.ID) - policy2 := newTestPolicy(t, ds, user1, policyName2, "darwin", &team1.ID) - - policiesByName, err := ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) - require.NoError(t, err) - require.Len(t, policiesByName, 2) - assert.Equal(t, policiesByName[policyName1].ID, policy1.ID) - assert.Equal(t, policiesByName[policyName2].ID, policy2.ID) - assert.Equal(t, policiesByName[policyName1].Name, policy1.Name) - assert.Equal(t, policiesByName[policyName2].Name, policy2.Name) - - // Policy does not exist - _, err = ds.PoliciesByName(context.Background(), []string{"doesn't exist"}, team1.ID) - assert.ErrorAs(t, err, &nfe) - - // One exists and one doesn't - _, err = ds.PoliciesByName(context.Background(), []string{policyName1, "doesn't exist"}, team1.ID) - assert.ErrorAs(t, err, &nfe) - -} - func testTeamPolicyTransfer(t *testing.T, ds *Datastore) { ctx := context.Background() user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) @@ -1286,12 +1246,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2;", - Description: "query2 desc", - Resolution: "some other resolution", - Team: "team1", - Platform: "darwin", + Name: "query2", + Query: "select 2;", + Description: "query2 desc", + Resolution: "some other resolution", + Team: "team1", + Platform: "darwin", + CalendarEventsEnabled: true, }, { Name: "query3", @@ -1326,6 +1287,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, teamPolicies[0].Resolution) assert.Equal(t, "some other resolution", *teamPolicies[0].Resolution) assert.Equal(t, "darwin", teamPolicies[0].Platform) + assert.True(t, teamPolicies[0].CalendarEventsEnabled) assert.Equal(t, "query3", teamPolicies[1].Name) assert.Equal(t, "select 3;", teamPolicies[1].Query) @@ -1335,6 +1297,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, teamPolicies[1].Resolution) assert.Equal(t, "some other good resolution", *teamPolicies[1].Resolution) assert.Equal(t, "windows,linux", teamPolicies[1].Platform) + assert.False(t, teamPolicies[1].CalendarEventsEnabled) // Make sure apply is idempotent require.NoError(t, ds.ApplyPolicySpecs(ctx, user1.ID, []*fleet.PolicySpec{ @@ -1347,12 +1310,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2;", - Description: "query2 desc", - Resolution: "some other resolution", - Team: "team1", - Platform: "darwin", + Name: "query2", + Query: "select 2;", + Description: "query2 desc", + Resolution: "some other resolution", + Team: "team1", + Platform: "darwin", + CalendarEventsEnabled: true, }, { Name: "query3", @@ -1382,12 +1346,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2 from updated;", - Description: "query2 desc updated", - Resolution: "some other resolution updated", - Team: "team1", // No error, team did not change - Platform: "windows", + Name: "query2", + Query: "select 2 from updated;", + Description: "query2 desc updated", + Resolution: "some other resolution updated", + Team: "team1", // No error, team did not change + Platform: "windows", + CalendarEventsEnabled: false, }, })) policies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) @@ -1402,6 +1367,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, policies[0].Resolution) assert.Equal(t, "some resolution updated", *policies[0].Resolution) assert.Equal(t, "", policies[0].Platform) + assert.False(t, policies[0].CalendarEventsEnabled) teamPolicies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) require.NoError(t, err) @@ -1481,11 +1447,12 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { assert.Equal(t, computeChecksum(*gp), hex.EncodeToString(globalChecksum)) payload = fleet.PolicyPayload{ - Name: "team1 query", - Query: "select 2;", - Description: "team1 query desc", - Resolution: "team1 query resolution", - Critical: true, + Name: "team1 query", + Query: "select 2;", + Description: "team1 query desc", + Resolution: "team1 query resolution", + Critical: true, + CalendarEventsEnabled: true, } tp1, err := ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, payload) require.NoError(t, err) @@ -1494,6 +1461,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { require.Equal(t, tp1.Description, payload.Description) require.Equal(t, *tp1.Resolution, payload.Resolution) require.Equal(t, tp1.Critical, payload.Critical) + assert.Equal(t, tp1.CalendarEventsEnabled, payload.CalendarEventsEnabled) var teamChecksum []uint8 err = ds.writer(context.Background()).Get(&teamChecksum, `SELECT checksum FROM policies WHERE id = ?`, tp1.ID) require.NoError(t, err) @@ -1522,6 +1490,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { tp2.Description = "team1 query desc updated" tp2.Resolution = ptr.String("team1 query resolution updated") tp2.Critical = false + tp2.CalendarEventsEnabled = false err = ds.SavePolicy(ctx, &tp2, true) require.NoError(t, err) tp1, err = ds.Policy(ctx, tp1.ID) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 69af794e71..4081db8af0 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -588,7 +588,6 @@ type Datastore interface { ListGlobalPolicies(ctx context.Context, opts ListOptions) ([]*Policy, error) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error) - PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*Policy, error) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) CountPolicies(ctx context.Context, teamID *uint, matchQuery string) (int, error) UpdateHostPolicyCounts(ctx context.Context) error diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 5f3c5e440a..8c34509485 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -112,14 +112,9 @@ func (z TeamZendeskIntegration) UniqueKey() string { } type TeamGoogleCalendarIntegration struct { - Email string `json:"email"` - Enable bool `json:"enable_calendar_events"` - Policies []*PolicyRef `json:"policies"` - WebhookURL string `json:"webhook_url"` -} -type PolicyRef struct { - Name string `json:"name"` - ID uint `json:"id"` + Email string `json:"email"` + Enable bool `json:"enable_calendar_events"` + WebhookURL string `json:"webhook_url"` } // JiraIntegration configures an instance of an integration with the Jira @@ -380,7 +375,7 @@ func ValidateEnabledHostStatusIntegrations(webhook HostStatusWebhookSettings, in func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, invalid *InvalidArgumentError) { if len(intgs) > 1 { - invalid.Append("integrations.google_calendar", "only one Google Calendar integration is allowed at this time") + invalid.Append("integrations.google_calendar", "integrating with >1 Google Workspace service account is not yet supported.") } for _, intg := range intgs { intg.Email = strings.TrimSpace(intg.Email) diff --git a/server/fleet/policies.go b/server/fleet/policies.go index 78d57f86c4..52a6109b2a 100644 --- a/server/fleet/policies.go +++ b/server/fleet/policies.go @@ -30,6 +30,8 @@ type PolicyPayload struct { // // Empty string targets all platforms. Platform string + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled bool } var ( @@ -107,6 +109,8 @@ type ModifyPolicyPayload struct { Platform *string `json:"platform"` // Critical marks the policy as high impact. Critical *bool `json:"critical" premium:"true"` + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled *bool `json:"calendar_events_enabled" premium:"true"` } // Verify verifies the policy payload is valid. @@ -159,6 +163,8 @@ type PolicyData struct { // Empty string targets all platforms. Platform string `json:"platform" db:"platforms"` + CalendarEventsEnabled bool `json:"calendar_events_enabled" db:"calendar_events_enabled"` + UpdateCreateTimestamps } @@ -212,6 +218,8 @@ type PolicySpec struct { // // Empty string targets all platforms. Platform string `json:"platform,omitempty"` + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled bool `json:"calendar_events_enabled"` } // Verify verifies the policy data is valid. diff --git a/server/fleet/teams.go b/server/fleet/teams.go index aee5c7ab95..9c9acbf4c0 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -451,6 +451,11 @@ func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { webhookSettings.HostStatusWebhook = t.Config.WebhookSettings.HostStatusWebhook } + var integrations TeamSpecIntegrations + if t.Config.Integrations.GoogleCalendar != nil { + integrations.GoogleCalendar = t.Config.Integrations.GoogleCalendar + } + return &TeamSpec{ Name: t.Name, AgentOptions: agentOptions, @@ -459,5 +464,6 @@ func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { MDM: mdmSpec, HostExpirySettings: &t.Config.HostExpirySettings, WebhookSettings: webhookSettings, + Integrations: integrations, }, nil } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 603d0616ff..1469826979 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -432,8 +432,6 @@ type ListGlobalPoliciesFunc func(ctx context.Context, opts fleet.ListOptions) ([ type PoliciesByIDFunc func(ctx context.Context, ids []uint) (map[uint]*fleet.Policy, error) -type PoliciesByNameFunc func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) - type DeleteGlobalPoliciesFunc func(ctx context.Context, ids []uint) ([]uint, error) type CountPoliciesFunc func(ctx context.Context, teamID *uint, matchQuery string) (int, error) @@ -1482,9 +1480,6 @@ type DataStore struct { PoliciesByIDFunc PoliciesByIDFunc PoliciesByIDFuncInvoked bool - PoliciesByNameFunc PoliciesByNameFunc - PoliciesByNameFuncInvoked bool - DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFuncInvoked bool @@ -3576,13 +3571,6 @@ func (s *DataStore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fle return s.PoliciesByIDFunc(ctx, ids) } -func (s *DataStore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - s.mu.Lock() - s.PoliciesByNameFuncInvoked = true - s.mu.Unlock() - return s.PoliciesByNameFunc(ctx, names, teamID) -} - func (s *DataStore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { s.mu.Lock() s.DeleteGlobalPoliciesFuncInvoked = true diff --git a/server/service/client.go b/server/service/client.go index b1eb35d694..8384e3e54d 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -880,7 +880,6 @@ func (c *Client) DoGitOps( } var mdmAppConfig map[string]interface{} var team map[string]interface{} - var teamCalendarIntegration map[string]interface{} if config.TeamName == nil { group.AppConfig = config.OrgSettings group.EnrollSecret = &fleet.EnrollSecretSpec{Secrets: config.OrgSettings["secrets"].([]*fleet.EnrollSecret)} @@ -968,20 +967,14 @@ func (c *Client) DoGitOps( if !ok { return errors.New("team_settings.integrations config is not a map") } - if calendar, ok := integrations.(map[string]interface{})["google_calendar"]; ok { - if calendar == nil { - calendar = map[string]interface{}{} - integrations.(map[string]interface{})["google_calendar"] = calendar - } - teamCalendarIntegration, ok = calendar.(map[string]interface{}) + if googleCal, ok := integrations.(map[string]interface{})["google_calendar"]; !ok || googleCal == nil { + integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} + } else { + _, ok = googleCal.(map[string]interface{}) if !ok { return errors.New("team_settings.integrations.google_calendar config is not a map") } } - // We clear the calendar integration and re-apply it after updating policies. - // This is needed because the calendar integration may be referencing policies that need to be - // created/updated. - integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} team["mdm"] = map[string]interface{}{} mdmAppConfig = team["mdm"].(map[string]interface{}) @@ -1087,23 +1080,6 @@ func (c *Client) DoGitOps( return err } - // Apply calendar integration - if len(teamCalendarIntegration) > 0 { - group = spec.Group{} - team = make(map[string]interface{}) - team["name"] = *config.TeamName - team["integrations"] = map[string]interface{}{"google_calendar": teamCalendarIntegration} - rawTeam, err := json.Marshal(team) - if err != nil { - return fmt.Errorf("error marshalling team spec: %w", err) - } - group.Teams = []json.RawMessage{rawTeam} - _, err = c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplySpecOptions{DryRun: dryRun}) - if err != nil { - return err - } - } - err = c.doGitOpsQueries(config, logFn, dryRun) if err != nil { return err diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 2484ce0d11..22c2779ca0 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -199,12 +199,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { "google_calendar": map[string]any{ "email": calendarEmail, "enable_calendar_events": true, - "policies": []any{ - map[string]any{ - "name": teamPolicy.Name, - }, - }, - "webhook_url": calendarWebhookUrl, + "webhook_url": calendarWebhookUrl, }, }, }, @@ -218,8 +213,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) - require.Len(t, team.Config.Integrations.GoogleCalendar.Policies, 1) - assert.Equal(t, teamPolicy.ID, team.Config.Integrations.GoogleCalendar.Policies[0].ID) // dry-run with invalid windows updates teamSpecs = map[string]any{ @@ -3686,7 +3679,7 @@ func (s *integrationEnterpriseTestSuite) TestGlobalPolicyCreateReadPatch() { } func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { - fields := []string{"Query", "Name", "Description", "Resolution", "Platform", "Critical"} + fields := []string{"Query", "Name", "Description", "Resolution", "Platform", "Critical", "CalendarEventsEnabled"} team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ ID: 42, @@ -3697,24 +3690,26 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { createPol1 := &teamPolicyResponse{} createPol1Req := &teamPolicyRequest{ - Query: "query", - Name: "name1", - Description: "description", - Resolution: "resolution", - Platform: "linux", - Critical: true, + Query: "query", + Name: "name1", + Description: "description", + Resolution: "resolution", + Platform: "linux", + Critical: true, + CalendarEventsEnabled: true, } s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), createPol1Req, http.StatusOK, &createPol1) allEqual(s.T(), createPol1Req, createPol1.Policy, fields...) createPol2 := &teamPolicyResponse{} createPol2Req := &teamPolicyRequest{ - Query: "query", - Name: "name2", - Description: "description", - Resolution: "resolution", - Platform: "linux", - Critical: false, + Query: "query", + Name: "name2", + Description: "description", + Resolution: "resolution", + Platform: "linux", + Critical: false, + CalendarEventsEnabled: false, } s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), createPol2Req, http.StatusOK, &createPol2) allEqual(s.T(), createPol2Req, createPol2.Policy, fields...) @@ -3730,12 +3725,13 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { patchPol1Req := &modifyTeamPolicyRequest{ ModifyPolicyPayload: fleet.ModifyPolicyPayload{ - Name: ptr.String("newName1"), - Query: ptr.String("newQuery"), - Description: ptr.String("newDescription"), - Resolution: ptr.String("newResolution"), - Platform: ptr.String("windows"), - Critical: ptr.Bool(false), + Name: ptr.String("newName1"), + Query: ptr.String("newQuery"), + Description: ptr.String("newDescription"), + Resolution: ptr.String("newResolution"), + Platform: ptr.String("windows"), + Critical: ptr.Bool(false), + CalendarEventsEnabled: ptr.Bool(false), }, } patchPol1 := &modifyTeamPolicyResponse{} @@ -3744,12 +3740,13 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { patchPol2Req := &modifyTeamPolicyRequest{ ModifyPolicyPayload: fleet.ModifyPolicyPayload{ - Name: ptr.String("newName2"), - Query: ptr.String("newQuery"), - Description: ptr.String("newDescription"), - Resolution: ptr.String("newResolution"), - Platform: ptr.String("windows"), - Critical: ptr.Bool(true), + Name: ptr.String("newName2"), + Query: ptr.String("newQuery"), + Description: ptr.String("newDescription"), + Resolution: ptr.String("newResolution"), + Platform: ptr.String("windows"), + Critical: ptr.Bool(true), + CalendarEventsEnabled: ptr.Bool(true), }, } patchPol2 := &modifyTeamPolicyResponse{} diff --git a/server/service/team_policies.go b/server/service/team_policies.go index 40187769ba..15a7a90acd 100644 --- a/server/service/team_policies.go +++ b/server/service/team_policies.go @@ -20,14 +20,15 @@ import ( ///////////////////////////////////////////////////////////////////////////////// type teamPolicyRequest struct { - TeamID uint `url:"team_id"` - QueryID *uint `json:"query_id"` - Query string `json:"query"` - Name string `json:"name"` - Description string `json:"description"` - Resolution string `json:"resolution"` - Platform string `json:"platform"` - Critical bool `json:"critical" premium:"true"` + TeamID uint `url:"team_id"` + QueryID *uint `json:"query_id"` + Query string `json:"query"` + Name string `json:"name"` + Description string `json:"description"` + Resolution string `json:"resolution"` + Platform string `json:"platform"` + Critical bool `json:"critical" premium:"true"` + CalendarEventsEnabled bool `json:"calendar_events_enabled"` } type teamPolicyResponse struct { @@ -40,13 +41,14 @@ func (r teamPolicyResponse) error() error { return r.Err } func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*teamPolicyRequest) resp, err := svc.NewTeamPolicy(ctx, req.TeamID, fleet.PolicyPayload{ - QueryID: req.QueryID, - Name: req.Name, - Query: req.Query, - Description: req.Description, - Resolution: req.Resolution, - Platform: req.Platform, - Critical: req.Critical, + QueryID: req.QueryID, + Name: req.Name, + Query: req.Query, + Description: req.Description, + Resolution: req.Resolution, + Platform: req.Platform, + Critical: req.Critical, + CalendarEventsEnabled: req.CalendarEventsEnabled, }) if err != nil { return teamPolicyResponse{Err: err}, nil @@ -390,6 +392,9 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f if p.Critical != nil { policy.Critical = *p.Critical } + if p.CalendarEventsEnabled != nil { + policy.CalendarEventsEnabled = *p.CalendarEventsEnabled + } logging.WithExtras(ctx, "name", policy.Name, "sql", policy.Query) err = svc.ds.SavePolicy(ctx, policy, shouldRemoveAll) diff --git a/server/webhooks/failing_policies_test.go b/server/webhooks/failing_policies_test.go index 71b890cfe9..1c2edda535 100644 --- a/server/webhooks/failing_policies_test.go +++ b/server/webhooks/failing_policies_test.go @@ -124,7 +124,8 @@ func TestTriggerFailingPoliciesWebhookBasic(t *testing.T) { "passing_host_count": 0, "failing_host_count": 0, "host_count_updated_at": null, - "critical": true + "critical": true, + "calendar_events_enabled": false }, "hosts": [ { @@ -183,16 +184,17 @@ func TestTriggerFailingPoliciesWebhookTeam(t *testing.T) { policiesByID := map[uint]*fleet.Policy{ 1: { PolicyData: fleet.PolicyData{ - ID: 1, - Name: "policy1", - Query: "select 1", - Description: "policy1 description", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - TeamID: &teamID, - Resolution: ptr.String("policy1 resolution"), - Platform: "darwin", + ID: 1, + Name: "policy1", + Query: "select 1", + Description: "policy1 description", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + TeamID: &teamID, + Resolution: ptr.String("policy1 resolution"), + Platform: "darwin", + CalendarEventsEnabled: true, }, }, 2: { @@ -309,7 +311,8 @@ func TestTriggerFailingPoliciesWebhookTeam(t *testing.T) { "passing_host_count": 0, "failing_host_count": 0, "host_count_updated_at": null, - "critical": false + "critical": false, + "calendar_events_enabled": true }, "hosts": [ { From 2db8eb3c80ed1de49b91dc4b539926cf9b094431 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 19:15:35 -0500 Subject: [PATCH 06/30] Update migrations for main rebase. --- ... 20240314085226_AddCalendarEventTables.go} | 6 ++-- ...0314085226_AddCalendarEventTables_test.go} | 2 +- server/datastore/mysql/schema.sql | 33 +++++++++++++++++-- 3 files changed, 35 insertions(+), 6 deletions(-) rename server/datastore/mysql/migrations/tables/{20240313085226_AddCalendarEventTables.go => 20240314085226_AddCalendarEventTables.go} (89%) rename server/datastore/mysql/migrations/tables/{20240313085226_AddCalendarEventTables_test.go => 20240314085226_AddCalendarEventTables_test.go} (97%) diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go similarity index 89% rename from server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go rename to server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go index 242a11ecbb..e9222e9d91 100644 --- a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go +++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go @@ -6,10 +6,10 @@ import ( ) func init() { - MigrationClient.AddMigration(Up_20240313085226, Down_20240313085226) + MigrationClient.AddMigration(Up_20240314085226, Down_20240314085226) } -func Up_20240313085226(tx *sql.Tx) error { +func Up_20240314085226(tx *sql.Tx) error { // TODO(lucas): Check if we need more indexes. if _, err := tx.Exec(` @@ -47,6 +47,6 @@ func Up_20240313085226(tx *sql.Tx) error { return nil } -func Down_20240313085226(tx *sql.Tx) error { +func Down_20240314085226(tx *sql.Tx) error { return nil } diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go similarity index 97% rename from server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go rename to server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go index 216a3e468b..85f61b342b 100644 --- a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go +++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestUp_20240313085226(t *testing.T) { +func TestUp_20240314085226(t *testing.T) { db := applyUpToPrev(t) applyNext(t, db) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 0040d49902..a256591e25 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -44,6 +44,19 @@ CREATE TABLE `app_config_json` ( INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `calendar_events` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `email` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `end_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `event` json NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `carve_blocks` ( `metadata_id` int(10) unsigned NOT NULL, `block_id` int(11) NOT NULL, @@ -192,6 +205,21 @@ CREATE TABLE `host_batteries` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `host_calendar_events` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `host_id` int(10) unsigned NOT NULL, + `calendar_event_id` int(10) unsigned NOT NULL, + `webhook_status` tinyint(4) NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_one_calendar_event_per_host` (`host_id`), + KEY `calendar_event_id` (`calendar_event_id`), + CONSTRAINT `host_calendar_events_ibfk_1` FOREIGN KEY (`calendar_event_id`) REFERENCES `calendar_events` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `host_dep_assignments` ( `host_id` int(10) unsigned NOT NULL, `added_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -779,9 +807,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=257 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=259 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1098,6 +1126,7 @@ CREATE TABLE `policies` ( `platforms` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `critical` tinyint(1) NOT NULL DEFAULT '0', `checksum` binary(16) NOT NULL, + `calendar_events_enabled` tinyint(1) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `idx_policies_checksum` (`checksum`), KEY `idx_policies_author_id` (`author_id`), From 21f95d8b5d2f3530eb46e35b7d02ba36328f98e5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 15 Mar 2024 10:26:58 -0500 Subject: [PATCH 07/30] Calendar interface fixes from code review and refactoring. (#17658) Calendar interface fixes from code review and manual merge with @lucasmrod changes. --- ee/server/calendar/calendar.go | 16 ----- ee/server/calendar/google_calendar.go | 86 ++++++++++++++------------- server/fleet/calendar.go | 11 ++-- 3 files changed, 50 insertions(+), 63 deletions(-) delete mode 100644 ee/server/calendar/calendar.go diff --git a/ee/server/calendar/calendar.go b/ee/server/calendar/calendar.go deleted file mode 100644 index 469437f3a0..0000000000 --- a/ee/server/calendar/calendar.go +++ /dev/null @@ -1,16 +0,0 @@ -package calendar - -import ( - "context" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/go-kit/kit/log" -) - -type GoogleCalendarConfig struct { - Context context.Context - IntegrationConfig *fleet.GoogleCalendarIntegration - UserEmail string - Logger log.Logger - // Should be nil for production - API GoogleCalendarAPI -} diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ff3d2dfb80..ce1e9cfb9a 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -5,16 +5,18 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "time" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" "google.golang.org/api/calendar/v3" "google.golang.org/api/googleapi" "google.golang.org/api/option" - "net/http" - "time" ) const ( @@ -30,15 +32,34 @@ var calendarScopes = []string{ "https://www.googleapis.com/auth/calendar.settings.readonly", } -// GoogleCalendar is an implementation of the Calendar interface that uses the +type GoogleCalendarConfig struct { + Context context.Context + IntegrationConfig *fleet.GoogleCalendarIntegration + Logger kitlog.Logger + // Should be nil for production + API GoogleCalendarAPI +} + +// GoogleCalendar is an implementation of the UserCalendar interface that uses the // Google Calendar API to manage events. type GoogleCalendar struct { - config *GoogleCalendarConfig - timezoneOffset *int + config *GoogleCalendarConfig + currentUserEmail string + timezoneOffset *int +} + +func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { + if config.API == nil { + var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} + config.API = lowLevelAPI + } + return &GoogleCalendar{ + config: config, + } } type GoogleCalendarAPI interface { - Connect(ctx context.Context, email, privateKey, subject string) error + Configure(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error GetSetting(name string) (*calendar.Setting, error) ListEvents(timeMin, timeMax string) (*calendar.Events, error) CreateEvent(event *calendar.Event) (*calendar.Event, error) @@ -55,15 +76,17 @@ type GoogleCalendarLowLevelAPI struct { service *calendar.Service } -// Connect creates a new Google Calendar service using the provided credentials. -func (lowLevelAPI *GoogleCalendarLowLevelAPI) Connect(ctx context.Context, email, privateKey, subject string) error { +// Configure creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarLowLevelAPI) Configure( + ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string, +) error { // Create a new calendar service conf := &jwt.Config{ - Email: email, + Email: serviceAccountEmail, Scopes: calendarScopes, PrivateKey: []byte(privateKey), TokenURL: google.JWTTokenURL, - Subject: subject, + Subject: userToImpersonateEmail, } client := conf.Client(ctx) service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) @@ -95,33 +118,18 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { return lowLevelAPI.service.Events.Delete(calendarID, id).Do() } -func (c *GoogleCalendar) Connect(config any) (fleet.Calendar, error) { - gConfig, ok := config.(*GoogleCalendarConfig) - if !ok { - return nil, errors.New("invalid Google calendar config") - } - if gConfig.API == nil { - var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} - gConfig.API = lowLevelAPI - } - err := gConfig.API.Connect( - gConfig.Context, gConfig.IntegrationConfig.Email, gConfig.IntegrationConfig.PrivateKey, gConfig.UserEmail, +func (c *GoogleCalendar) Configure(userEmail string) error { + err := c.config.API.Configure( + c.config.Context, c.config.IntegrationConfig.Email, c.config.IntegrationConfig.PrivateKey, userEmail, ) if err != nil { - return nil, ctxerr.Wrap(gConfig.Context, err, "creating Google calendar service") + return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service") } - - gCal := &GoogleCalendar{ - config: gConfig, - } - - return gCal, nil + c.currentUserEmail = userEmail + return nil } func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { - if c.config == nil { - return nil, false, errors.New("the Google calendar is not connected. Please call Connect first") - } if event.EndTime.Before(time.Now()) { return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) } @@ -199,16 +207,11 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet if details.ID == "" { return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ID") } - if details.ETag == "" { - return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ETag") - } + // ETag is optional, but we need it to check if the event was modified return &details, nil } func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { - if c.config == nil { - return nil, errors.New("the Google calendar is not connected. Please call Connect first") - } if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -257,7 +260,7 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. attending = true } else { for _, attendee := range gEvent.Attendees { - if attendee.Email == c.config.UserEmail { + if attendee.Email == c.currentUserEmail { if attendee.ResponseStatus != "declined" { attending = true } @@ -316,8 +319,7 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. if err != nil { return nil, err } - level.Debug(c.config.Logger).Log("msg", "created Google calendar events", "user", c.config.UserEmail, "startTime", eventStart) - fmt.Printf("VICTOR Created event with id:%s and ETag:%s\n", event.Id, event.Etag) + level.Debug(c.config.Logger).Log("msg", "created Google calendar event", "user", c.currentUserEmail, "startTime", eventStart) return fleetEvent, nil } @@ -364,7 +366,7 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti fleetEvent := &fleet.CalendarEvent{} fleetEvent.StartTime = startTime fleetEvent.EndTime = endTime - fleetEvent.Email = c.config.UserEmail + fleetEvent.Email = c.currentUserEmail details := &eventDetails{ ID: event.Id, ETag: event.Etag, @@ -379,7 +381,7 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { if c.config == nil { - return errors.New("the Google calendar is not connected. Please call Connect first") + return errors.New("the Google calendar is not connected. Please call Configure first") } details, err := c.unmarshalDetails(event) if err != nil { diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index dc682e24ea..db2bbbc45e 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -10,15 +10,16 @@ func (e DayEndedError) Error() string { return e.Msg } -type Calendar interface { - // Connect to calendar. This method must be called first. Currently, config must be a *GoogleCalendarConfig - Connect(config any) (Calendar, error) +type UserCalendar interface { + // Configure configures the connection to a user's calendar. Once configured, + // CreateEvent, GetAndUpdateEvent and DeleteEvent reference the user's calendar. + Configure(userEmail string) error + // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. + CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) // GetAndUpdateEvent retrieves the event from the calendar. // If the event has been modified, it returns the updated event. // If the event has been deleted, it schedules a new event with given body callback and returns the new event. GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) - // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. - CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } From 712d776be14d688742af4a7ea6f11abf21f7c27e Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 18 Mar 2024 14:44:07 -0500 Subject: [PATCH 08/30] Calendar interface (tests and associated fixes) (#17665) Completed unit tests for Google calendar interface, along with bug fixes. # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/calendar/google_calendar.go | 174 ++++-- ee/server/calendar/google_calendar_test.go | 589 +++++++++++++++++++++ 2 files changed, 708 insertions(+), 55 deletions(-) create mode 100644 ee/server/calendar/google_calendar_test.go diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ce1e9cfb9a..53e189e24b 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -111,7 +111,14 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calend func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { // Default maximum number of events returned is 250, which should be sufficient for most calendars. - return lowLevelAPI.service.Events.List(calendarID).EventTypes("default").OrderBy("startTime").SingleEvents(true).TimeMin(timeMin).TimeMax(timeMax).Do() + return lowLevelAPI.service.Events.List(calendarID). + EventTypes("default"). + OrderBy("startTime"). + SingleEvents(true). + TimeMin(timeMin). + TimeMax(timeMax). + ShowDeleted(false). + Do() } func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { @@ -130,9 +137,7 @@ func (c *GoogleCalendar) Configure(userEmail string) error { } func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { - if event.EndTime.Before(time.Now()) { - return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) - } + // We assume that the Fleet event has not already ended. We will simply return it if it has not been modified. details, err := c.unmarshalDetails(event) if err != nil { return nil, false, err @@ -143,6 +148,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn // http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later case googleapi.IsNotModified(err): return event, false, nil + // http.StatusNotFound should be very rare -- Google keeps events for a while after they are deleted case isNotFound(err): deleted = true case err != nil: @@ -153,21 +159,50 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn // Event was not modified return event, false, nil } - endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) - if err != nil { - return nil, false, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), - ) + if gEvent.End == nil || (gEvent.End.DateTime == "" && gEvent.End.Date == "") { + // We should not see this error. If we do, we can work around by treating event as deleted. + return nil, false, ctxerr.Errorf(c.config.Context, "missing end date/time for Google calendar event: %s", gEvent.Id) } - // If event already ended, it is effectively deleted - if endTime.After(time.Now()) { - startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + + if gEvent.End.DateTime == "" { + // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. + // We won't handle all-day events at this time, and treat the event as deleted. + deleted = true + } + + var endTime *time.Time + if !deleted { + endTime, err = c.parseDateTime(gEvent.End) if err != nil { - return nil, false, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), - ) + return nil, false, err } - fleetEvent, err := c.googleEventToFleetEvent(startTime, endTime, gEvent) + if !endTime.After(time.Now()) { + // If event already ended, it is effectively deleted + // Delete this event to prevent confusion. This operation should be rare. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which is in the past", "err", err) + } + deleted = true + } + } + if !deleted { + if gEvent.Start == nil || (gEvent.Start.DateTime == "" && gEvent.Start.Date == "") { + // We should not see this error. If we do, we can work around by treating event as deleted. + return nil, false, ctxerr.Errorf(c.config.Context, "missing start date/time for Google calendar event: %s", gEvent.Id) + } + if gEvent.Start.DateTime == "" { + // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. + // We won't handle all-day events at this time, and treat the event as deleted. + deleted = true + } + } + if !deleted { + startTime, err := c.parseDateTime(gEvent.Start) + if err != nil { + return nil, false, err + } + fleetEvent, err := c.googleEventToFleetEvent(*startTime, *endTime, gEvent) if err != nil { return nil, false, err } @@ -175,12 +210,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn } } - newStartDate := event.StartTime.Add(24 * time.Hour) - if newStartDate.Weekday() == time.Saturday { - newStartDate = newStartDate.Add(48 * time.Hour) - } else if newStartDate.Weekday() == time.Sunday { - newStartDate = newStartDate.Add(24 * time.Hour) - } + newStartDate := calculateNewEventDate(event.StartTime) fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) if err != nil { @@ -189,6 +219,34 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn return fleetEvent, true, nil } +func calculateNewEventDate(oldStartDate time.Time) time.Time { + // Note: we do not handle time changes (daylight savings time, etc.) -- assuming 1 day is always 24 hours. + newStartDate := oldStartDate.Add(24 * time.Hour) + if newStartDate.Weekday() == time.Saturday { + newStartDate = newStartDate.Add(48 * time.Hour) + } else if newStartDate.Weekday() == time.Sunday { + newStartDate = newStartDate.Add(24 * time.Hour) + } + return newStartDate +} + +func (c *GoogleCalendar) parseDateTime(eventDateTime *calendar.EventDateTime) (*time.Time, error) { + var endTime time.Time + var err error + if eventDateTime.TimeZone != "" { + loc := getLocation(eventDateTime.TimeZone, c.config) + endTime, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc) + } else { + endTime, err = time.Parse(time.RFC3339, eventDateTime.DateTime) + } + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event time: %s", eventDateTime.DateTime), + ) + } + return &endTime, nil +} + func isNotFound(err error) bool { if err == nil { return false @@ -212,6 +270,12 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet } func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { + return c.createEvent(dayOfEvent, body, time.Now) +} + +// createEvent creates a new event on the calendar on the given date. timeNow is a function that returns the current time. +// timeNow can be overwritten for testing +func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow func() time.Time) (*fleet.CalendarEvent, error) { if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -223,27 +287,26 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location) dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location) - now := time.Now().In(location) + now := timeNow().In(location) if dayEnd.Before(now) { // The workday has already ended. return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"}) } // Adjust day start if workday already started - if dayStart.Before(now) { + if !dayStart.After(now) { dayStart = now.Truncate(eventLength) if dayStart.Before(now) { dayStart = dayStart.Add(eventLength) } - if dayStart.Equal(dayEnd) { + if !dayStart.Before(dayEnd) { return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "no time available for event"}) } } eventStart := dayStart eventEnd := dayStart.Add(eventLength) - searchStart := dayStart.Add(-24 * time.Hour) - events, err := c.config.API.ListEvents(searchStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) + events, err := c.config.API.ListEvents(dayStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") } @@ -253,48 +316,44 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. continue } + // Ignore all day events + if gEvent.Start == nil || gEvent.Start.DateTime == "" || gEvent.End == nil || gEvent.End.DateTime == "" { + continue + } + // Ignore events that the user has declined - var attending bool - if len(gEvent.Attendees) == 0 { - // No attendees, so we assume the user is attending - attending = true - } else { - for _, attendee := range gEvent.Attendees { - if attendee.Email == c.currentUserEmail { - if attendee.ResponseStatus != "declined" { - attending = true - } + var declined bool + for _, attendee := range gEvent.Attendees { + if attendee.Email == c.currentUserEmail { + // The user has declined the event, so this time is open for scheduling + if attendee.ResponseStatus == "declined" { + declined = true break } } } - if !attending { + if declined { continue } // Ignore events that will end before our event - endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + endTime, err := c.parseDateTime(gEvent.End) if err != nil { - return nil, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), - ) + return nil, err } - if endTime.Before(eventStart) || endTime.Equal(eventStart) { + if !endTime.After(eventStart) { continue } - startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + startTime, err := c.parseDateTime(gEvent.Start) if err != nil { - return nil, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), - ) + return nil, err } if startTime.Before(eventEnd) { // Event occurs during our event, so we need to adjust. - fmt.Printf("VICTOR Adjusting event times due to %s: %s - %s\n", gEvent.Summary, eventStart, eventEnd) var isLastSlot bool - eventStart, eventEnd, isLastSlot = adjustEventTimes(endTime, dayEnd) + eventStart, eventEnd, isLastSlot = adjustEventTimes(*endTime, dayEnd) if isLastSlot { break } @@ -349,17 +408,22 @@ func getTimezone(gCal *GoogleCalendar) error { return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone") } - loc, err := time.LoadLocation(setting.Value) - if err != nil { - // Could not load location, use EST - level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", setting.Value, "err", err) - loc, _ = time.LoadLocation("America/New_York") - } + loc := getLocation(setting.Value, config) _, timezoneOffset := time.Now().In(loc).Zone() gCal.timezoneOffset = &timezoneOffset return nil } +func getLocation(name string, config *GoogleCalendarConfig) *time.Location { + loc, err := time.LoadLocation(name) + if err != nil { + // Could not load location, use EST + level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", name, "err", err) + loc, _ = time.LoadLocation("America/New_York") + } + return loc +} + func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime time.Time, event *calendar.Event) ( *fleet.CalendarEvent, error, ) { diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go new file mode 100644 index 0000000000..4c3e2db092 --- /dev/null +++ b/ee/server/calendar/google_calendar_test.go @@ -0,0 +1,589 @@ +package calendar + +import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "net/http" + "os" + "testing" + "time" +) + +const ( + baseServiceEmail = "service@example.com" + basePrivateKey = "private-key" + baseUserEmail = "user@example.com" +) + +var ( + baseCtx = context.Background() + logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) +) + +type MockGoogleCalendarLowLevelAPI struct { + ConfigureFunc func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error + GetSettingFunc func(name string) (*calendar.Setting, error) + ListEventsFunc func(timeMin, timeMax string) (*calendar.Events, error) + CreateEventFunc func(event *calendar.Event) (*calendar.Event, error) + GetEventFunc func(id, eTag string) (*calendar.Event, error) + DeleteEventFunc func(id string) error +} + +func (m *MockGoogleCalendarLowLevelAPI) Configure( + ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string, +) error { + return m.ConfigureFunc(ctx, serviceAccountEmail, privateKey, userToImpersonateEmail) +} + +func (m *MockGoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { + return m.GetSettingFunc(name) +} + +func (m *MockGoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { + return m.ListEventsFunc(timeMin, timeMax) +} + +func (m *MockGoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + return m.CreateEventFunc(event) +} + +func (m *MockGoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { + return m.GetEventFunc(id, eTag) +} + +func (m *MockGoogleCalendarLowLevelAPI) DeleteEvent(id string) error { + return m.DeleteEventFunc(id) +} + +func TestGoogleCalendar_Configure(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + assert.Equal(t, baseCtx, ctx) + assert.Equal(t, baseServiceEmail, serviceAccountEmail) + assert.Equal(t, basePrivateKey, privateKey) + assert.Equal(t, baseUserEmail, userToImpersonateEmail) + return nil + } + + // Happy path test + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + // Configure error test + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + return assert.AnError + } + err = cal.Configure(baseUserEmail) + assert.ErrorIs(t, err, assert.AnError) +} + +func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig { + if mockAPI != nil && mockAPI.ConfigureFunc == nil { + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + return nil + } + } + config := &GoogleCalendarConfig{ + Context: context.Background(), + IntegrationConfig: &fleet.GoogleCalendarIntegration{ + Email: baseServiceEmail, + PrivateKey: basePrivateKey, + }, + Logger: logger, + API: mockAPI, + } + return config +} + +func TestGoogleCalendar_DeleteEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, "event-id", id) + return nil + } + + // Happy path test + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)}) + assert.NoError(t, err) + + // API error test + mockAPI.DeleteEventFunc = func(id string) error { + return assert.AnError + } + err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)}) + assert.ErrorIs(t, err, assert.AnError) +} + +func TestGoogleCalendar_unmarshalDetails(t *testing.T) { + t.Parallel() + var gCal = NewGoogleCalendar(makeConfig(&MockGoogleCalendarLowLevelAPI{})) + err := gCal.Configure(baseUserEmail) + assert.NoError(t, err) + details, err := gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"id":"event-id","etag":"event-eTag"}`)}) + assert.NoError(t, err) + assert.Equal(t, "event-id", details.ID) + assert.Equal(t, "event-eTag", details.ETag) + + // Missing ETag is OK + details, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"id":"event-id"}`)}) + assert.NoError(t, err) + assert.Equal(t, "event-id", details.ID) + assert.Equal(t, "", details.ETag) + + // Bad JSON + _, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"bozo`)}) + assert.Error(t, err) + + // Missing id + _, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"myId":"event-id","etag":"event-eTag"}`)}) + assert.Error(t, err) +} + +func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + const baseETag = "event-eTag" + const baseEventID = "event-id" + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + assert.Equal(t, baseEventID, id) + assert.Equal(t, baseETag, eTag) + return &calendar.Event{ + Etag: baseETag, // ETag matches -- no modifications to event + }, nil + } + genBodyFn := func() string { + t.Error("genBodyFn should not be called") + return "event-body" + } + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + eventStartTime := time.Now().UTC() + event := &fleet.CalendarEvent{ + StartTime: eventStartTime, + EndTime: time.Now().Add(time.Hour), + Data: []byte(`{"ID":"` + baseEventID + `","ETag":"` + baseETag + `"}`), + } + + // ETag matches + retrievedEvent, updated, err := cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.False(t, updated) + assert.Equal(t, event, retrievedEvent) + + // http.StatusNotModified response (ETag matches) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, &googleapi.Error{Code: http.StatusNotModified} + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.False(t, updated) + assert.Equal(t, event, retrievedEvent) + + // Cannot unmarshal details + eventBadDetails := &fleet.CalendarEvent{ + StartTime: time.Now(), + EndTime: time.Now().Add(time.Hour), + Data: []byte(`{"bozo`), + } + _, _, err = cal.GetAndUpdateEvent(eventBadDetails, genBodyFn) + assert.Error(t, err) + + // API error test + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, assert.AnError + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.ErrorIs(t, err, assert.AnError) + + // Event has been modified + startTime := time.Now().Add(time.Minute).Truncate(time.Second) + endTime := time.Now().Add(time.Hour).Truncate(time.Second) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, startTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, endTime.UTC(), retrievedEvent.EndTime.UTC()) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + gCal, _ := cal.(*GoogleCalendar) + details, err := gCal.unmarshalDetails(retrievedEvent) + require.NoError(t, err) + assert.Equal(t, "new-eTag", details.ETag) + assert.Equal(t, baseEventID, details.ID) + + // missing end time + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: ""}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // missing start time + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // Bad time format + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: "bozo"}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // Event has been modified, with custom timezone. + tzId := "Africa/Kinshasa" + location, _ := time.LoadLocation(tzId) + startTime = time.Now().Add(time.Minute).Truncate(time.Second).In(location) + endTime = time.Now().Add(time.Hour).Truncate(time.Second).In(location) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.UTC().Format(time.RFC3339), TimeZone: tzId}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339), TimeZone: tzId}, + }, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, startTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, endTime.UTC(), retrievedEvent.EndTime.UTC()) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + + // 404 response (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, &googleapi.Error{Code: http.StatusNotFound} + } + mockAPI.GetSettingFunc = func(name string) (*calendar.Setting, error) { + return &calendar.Setting{Value: "UTC"}, nil + } + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + genBodyFn = func() string { + return "event-body" + } + eventCreated := false + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + assert.Equal(t, eventTitle, event.Summary) + assert.Equal(t, genBodyFn(), event.Description) + event.Id = baseEventID + event.Etag = baseETag + eventCreated = true + return event, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + newEventDate := calculateNewEventDate(eventStartTime) + expectedStartTime := time.Date(newEventDate.Year(), newEventDate.Month(), newEventDate.Day(), startHour, 0, 0, 0, time.UTC) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // cancelled (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + Status: "cancelled", + }, nil + } + eventCreated = false + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // all day event (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{Date: startTime.Format("2006-01-02")}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + eventCreated = false + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // moved in the past event (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Add(-2 * time.Hour).Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Add(-2 * time.Hour).Format(time.RFC3339)}, + }, nil + } + eventCreated = false + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, baseEventID, id) + return nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) +} + +func TestGoogleCalendar_CreateEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + const baseEventID = "event-id" + const baseETag = "event-eTag" + const eventBody = "event-body" + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + tzId := "Africa/Kinshasa" + mockAPI.GetSettingFunc = func(name string) (*calendar.Setting, error) { + return &calendar.Setting{Value: tzId}, nil + } + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + assert.Equal(t, eventTitle, event.Summary) + assert.Equal(t, eventBody, event.Description) + event.Id = baseEventID + event.Etag = baseETag + return event, nil + } + + // Happy path test -- empty calendar + date := time.Now().Add(48 * time.Hour) + location, _ := time.LoadLocation(tzId) + expectedStartTime := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + _, expectedOffset := expectedStartTime.Zone() + event, err := cal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, baseUserEmail, event.Email) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + _, offset := event.StartTime.Zone() + assert.Equal(t, expectedOffset, offset) + _, offset = event.EndTime.Zone() + assert.Equal(t, expectedOffset, offset) + gCal, _ := cal.(*GoogleCalendar) + details, err := gCal.unmarshalDetails(event) + require.NoError(t, err) + assert.Equal(t, baseETag, details.ETag) + assert.Equal(t, baseEventID, details.ID) + + // Workday already ended + date = time.Now().Add(-48 * time.Hour) + _, err = cal.CreateEvent(date, eventBody) + assert.ErrorAs(t, err, &fleet.DayEndedError{}) + + // There is no time left in the day to schedule an event + date = time.Now().Add(48 * time.Hour) + timeNow := func() time.Time { + now := time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 45, 0, 0, location) + return now + } + _, err = gCal.createEvent(date, eventBody, timeNow) + assert.ErrorAs(t, err, &fleet.DayEndedError{}) + + // Workday already started + date = time.Now().Add(48 * time.Hour) + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + timeNow = func() time.Time { + return expectedStartTime + } + event, err = gCal.createEvent(date, eventBody, timeNow) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Busy calendar + date = time.Now().Add(48 * time.Hour) + dayStart := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd := time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + gEvents := &calendar.Events{} + // Cancelled event + gEvent := &calendar.Event{ + Id: "cancelled-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + Status: "cancelled", + } + gEvents.Items = append(gEvents.Items, gEvent) + // All day events + gEvent = &calendar.Event{ + Id: "all-day-event-id", + Start: &calendar.EventDateTime{Date: dayStart.Format(time.DateOnly)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + gEvent = &calendar.Event{ + Id: "all-day2-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{Date: dayEnd.Format(time.DateOnly)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // User-declined event + gEvent = &calendar.Event{ + Id: "user-declined-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: baseUserEmail, ResponseStatus: "declined"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // Event before day + gEvent = &calendar.Event{ + Id: "before-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Add(-time.Hour).Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayStart.Add(-30 * time.Minute).Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event from 6am to 11am + eventStart := time.Date(date.Year(), date.Month(), date.Day(), 6, 0, 0, 0, location) + eventEnd := time.Date(date.Year(), date.Month(), date.Day(), 11, 0, 0, 0, location) + gEvent = &calendar.Event{ + Id: "6-to-11-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: baseUserEmail, ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event from 10am to 10:30am + eventStart = time.Date(date.Year(), date.Month(), date.Day(), 10, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), 10, 30, 0, 0, location) + gEvent = &calendar.Event{ + Id: "10-to-10-30-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // Event from 11am to 11:45am + eventStart = time.Date(date.Year(), date.Month(), date.Day(), 11, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), 11, 45, 0, 0, location) + gEvent = &calendar.Event{ + Id: "11-to-11-45-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event after day + eventStart = time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour, 45, 0, 0, location) + gEvent = &calendar.Event{ + Id: "after-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), 12, 0, 0, 0, location) + event, err = gCal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Full schedule -- pick the last slot + date = time.Now().Add(48 * time.Hour) + dayStart = time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + gEvents = &calendar.Events{} + gEvent = &calendar.Event{ + Id: "9-to-5-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + event, err = gCal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // API error in ListEvents + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return nil, assert.AnError + } + _, err = gCal.CreateEvent(date, eventBody) + assert.ErrorIs(t, err, assert.AnError) + + // API error in CreateEvent + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + return nil, assert.AnError + } + _, err = gCal.CreateEvent(date, eventBody) + assert.ErrorIs(t, err, assert.AnError) +} From 9a8ac02bc13d83ed93fb37ea2fffdc0f39289a77 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Tue, 19 Mar 2024 13:05:48 -0300 Subject: [PATCH 09/30] Happy path implementation of the calendar cron job (#17713) Happy path for #17441. --- cmd/fleet/calendar_cron.go | 454 ++++++++++++++++++ cmd/fleet/calendar_cron_test.go | 57 +++ cmd/fleet/serve.go | 12 + server/datastore/mysql/calendar_events.go | 150 ++++++ .../datastore/mysql/calendar_events_test.go | 6 + server/datastore/mysql/policies.go | 52 +- server/datastore/mysql/policies_test.go | 56 ++- server/fleet/app.go | 7 + server/fleet/calendar.go | 38 +- server/fleet/calendar_events.go | 9 + server/fleet/cron_schedules.go | 1 + server/fleet/datastore.go | 13 + server/fleet/policies.go | 5 + server/mock/datastore_mock.go | 96 ++++ server/service/osquery.go | 97 ++++ 15 files changed, 1050 insertions(+), 3 deletions(-) create mode 100644 cmd/fleet/calendar_cron.go create mode 100644 cmd/fleet/calendar_cron_test.go create mode 100644 server/datastore/mysql/calendar_events.go create mode 100644 server/datastore/mysql/calendar_events_test.go diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go new file mode 100644 index 0000000000..099a938b2c --- /dev/null +++ b/cmd/fleet/calendar_cron.go @@ -0,0 +1,454 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/ee/server/calendar" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/service/schedule" + "github.com/go-kit/log" + kitlog "github.com/go-kit/log" + "github.com/go-kit/log/level" +) + +func newCalendarSchedule( + ctx context.Context, + instanceID string, + ds fleet.Datastore, + logger kitlog.Logger, +) (*schedule.Schedule, error) { + const ( + name = string(fleet.CronCalendar) + defaultInterval = 5 * time.Minute + ) + logger = kitlog.With(logger, "cron", name) + s := schedule.New( + ctx, name, instanceID, defaultInterval, ds, ds, + schedule.WithAltLockID("calendar"), + schedule.WithLogger(logger), + schedule.WithJob( + "calendar_events", + func(ctx context.Context) error { + return cronCalendarEvents(ctx, ds, logger) + }, + ), + ) + + return s, nil +} + +func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error { + appConfig, err := ds.AppConfig(ctx) + if err != nil { + return fmt.Errorf("load app config: %w", err) + } + + if len(appConfig.Integrations.GoogleCalendar) == 0 { + return nil + } + googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] + googleCalendarConfig := calendar.GoogleCalendarConfig{ + Context: ctx, + IntegrationConfig: googleCalendarIntegrationConfig, + Logger: log.With(logger, "component", "google_calendar"), + } + calendar := calendar.NewGoogleCalendar(&googleCalendarConfig) + domain := googleCalendarIntegrationConfig.Domain + + teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ + User: &fleet.User{ + GlobalRole: ptr.String(fleet.RoleAdmin), + }, + }, fleet.ListOptions{}) + if err != nil { + return fmt.Errorf("list teams: %w", err) + } + + for _, team := range teams { + if err := cronCalendarEventsForTeam( + ctx, ds, calendar, *team, appConfig.OrgInfo.OrgName, domain, logger, + ); err != nil { + level.Info(logger).Log("msg", "events calendar cron", "team_id", team.ID, "err", err) + } + } + + return nil +} + +func cronCalendarEventsForTeam( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + team fleet.Team, + orgName string, + domain string, + logger kitlog.Logger, +) error { + if team.Config.Integrations.GoogleCalendar == nil || + !team.Config.Integrations.GoogleCalendar.Enable { + return nil + } + + policies, err := ds.GetCalendarPolicies(ctx, team.ID) + if err != nil { + return fmt.Errorf("get calendar policy ids: %w", err) + } + + if len(policies) == 0 { + return nil + } + + logger = kitlog.With(logger, "team_id", team.ID) + + // + // NOTEs: + // - We ignore hosts that are passing all policies and do not have an associated email. + // - We get only one host per email that's failing policies (the one with lower host id). + // - On every host, we get only the first email that matches the domain (sorted lexicographically). + // + // TODOs(lucas): + // - We need to rate limit calendar requests. + // + + policyIDs := make([]uint, 0, len(policies)) + for _, policy := range policies { + policyIDs = append(policyIDs, policy.ID) + } + hosts, err := ds.GetHostsPolicyMemberships(ctx, domain, policyIDs) + if err != nil { + return fmt.Errorf("get team hosts failing policies: %w", err) + } + + var ( + passingHosts []fleet.HostPolicyMembershipData + failingHosts []fleet.HostPolicyMembershipData + failingHostsWithoutAssociatedEmail []fleet.HostPolicyMembershipData + ) + for _, host := range hosts { + if host.Passing { // host is passing all configured policies + if host.Email != "" { + passingHosts = append(passingHosts, host) + } + } else { // host is failing some of the configured policies + if host.Email == "" { + failingHostsWithoutAssociatedEmail = append(failingHostsWithoutAssociatedEmail, host) + } else { + failingHosts = append(failingHosts, host) + } + } + } + level.Debug(logger).Log( + "msg", "summary", + "passing_hosts", len(passingHosts), + "failing_hosts", len(failingHosts), + "failing_hosts_without_associated_email", len(failingHostsWithoutAssociatedEmail), + ) + + if err := processCalendarFailingHosts( + ctx, ds, calendar, orgName, failingHosts, logger, + ); err != nil { + level.Info(logger).Log("msg", "processing failing hosts", "err", err) + } + + // Remove calendar events from hosts that are passing the policies. + if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil { + level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) + } + + // At last we want to notify the hosts that are failing and don't have an associated email. + if err := fireWebhookForHostsWithoutAssociatedEmail( + team.Config.Integrations.GoogleCalendar.WebhookURL, + domain, + failingHostsWithoutAssociatedEmail, + logger, + ); err != nil { + level.Info(logger).Log("msg", "webhook for hosts without associated email", "err", err) + } + + return nil +} + +func processCalendarFailingHosts( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + orgName string, + hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, +) error { + for _, host := range hosts { + logger := log.With(logger, "host_id", host.HostID) + if err := userCalendar.Configure(host.Email); err != nil { + return fmt.Errorf("configure user calendar: %w", err) + } + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID) + + deletedExpiredEvent := false + if err == nil { + if calendarEvent.EndTime.Before(time.Now()) { + if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { + level.Info(logger).Log("msg", "deleting existing expired calendar event", "err", err) + continue // continue with next host + } + deletedExpiredEvent = true + } + } + + switch { + case err == nil && !deletedExpiredEvent: + if err := processFailingHostExistingCalendarEvent( + ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) + continue // continue with next host + } + case fleet.IsNotFound(err) || deletedExpiredEvent: + if err := processFailingHostCreateCalendarEvent( + ctx, ds, userCalendar, orgName, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) + continue // continue with next host + } + default: + return fmt.Errorf("get calendar event: %w", err) + } + } + + return nil +} + +func processFailingHostExistingCalendarEvent( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + orgName string, + hostCalendarEvent *fleet.HostCalendarEvent, + calendarEvent *fleet.CalendarEvent, + host fleet.HostPolicyMembershipData, +) error { + updatedEvent, updated, err := calendar.GetAndUpdateEvent(calendarEvent, func() string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }) + if err != nil { + return fmt.Errorf("get event calendar on db: %w", err) + } + if updated { + if err := ds.UpdateCalendarEvent(ctx, + calendarEvent.ID, + updatedEvent.StartTime, + updatedEvent.EndTime, + updatedEvent.Data, + ); err != nil { + return fmt.Errorf("updating event calendar on db: %w", err) + } + } + now := time.Now() + eventInFuture := now.Before(updatedEvent.StartTime) + if eventInFuture { + // If the webhook status was sent and event was moved to the future we set the status to pending. + // This can happen if the admin wants to retry a remediation. + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent { + if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil { + return fmt.Errorf("update host calendar webhook status: %w", err) + } + } + // Nothing else to do as event is in the future. + return nil + } + if now.After(updatedEvent.EndTime) { + return fmt.Errorf( + "unexpected event in the past: now=%s, start_time=%s, end_time=%s", + now, updatedEvent.StartTime, updatedEvent.EndTime, + ) + } + + // + // Event happening now. + // + + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent { + return nil + } + + online, err := isHostOnline(ctx, ds, host.HostID) + if err != nil { + return fmt.Errorf("host online check: %w", err) + } + if !online { + // If host is offline then there's nothing to do. + return nil + } + + if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil { + return fmt.Errorf("update host calendar webhook status: %w", err) + } + + // TODO(lucas): If this doesn't work at scale, then implement a special refetch + // for policies only. + if err := ds.UpdateHostRefetchRequested(ctx, host.HostID, true); err != nil { + return fmt.Errorf("refetch host: %w", err) + } + return nil +} + +func processFailingHostCreateCalendarEvent( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + orgName string, + host fleet.HostPolicyMembershipData, +) error { + calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar) + if err != nil { + return fmt.Errorf("create event on user calendar: %w", err) + } + if _, err := ds.NewCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID); err != nil { + return fmt.Errorf("create calendar event on db: %w", err) + } + return nil +} + +func attemptCreatingEventOnUserCalendar( + orgName string, + host fleet.HostPolicyMembershipData, + userCalendar fleet.UserCalendar, +) (*fleet.CalendarEvent, error) { + // TODO(lucas): Where do we handle the following case (it seems CreateEvent needs to return no slot available for the requested day if there are none or too late): + // + // - If it’s the 3rd Tuesday of the month, create an event in the upcoming slot (if available). + // For example, if it’s the 3rd Tuesday of the month at 10:07a, Fleet will look for an open slot starting at 10:30a. + // - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day. + year, month, today := time.Now().Date() + preferredDate := getPreferredCalendarEventDate(year, month, today) + body := generateCalendarEventBody(orgName, host.HostDisplayName) + for { + calendarEvent, err := userCalendar.CreateEvent(preferredDate, body) + var dee fleet.DayEndedError + switch { + case err == nil: + return calendarEvent, nil + case errors.As(err, &dee): + preferredDate = addBusinessDay(preferredDate) + continue + default: + return nil, fmt.Errorf("create event on user calendar: %w", err) + } + } +} + +func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time { + const ( + // 3rd Tuesday of Month + preferredWeekDay = time.Tuesday + preferredOrdinal = 3 + ) + + firstDayOfMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC) + offset := int(preferredWeekDay - firstDayOfMonth.Weekday()) + if offset < 0 { + offset += 7 + } + preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1))) + if today > preferredDate.Day() { + today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC) + preferredDate = addBusinessDay(today_) + } + return preferredDate +} + +func addBusinessDay(date time.Time) time.Time { + nextBusinessDay := 1 + switch weekday := date.Weekday(); weekday { + case time.Friday: + nextBusinessDay += 2 + case time.Saturday: + nextBusinessDay += 1 + } + return date.AddDate(0, 0, nextBusinessDay) +} + +func removeCalendarEventsFromPassingHosts( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + hosts []fleet.HostPolicyMembershipData, +) error { + for _, host := range hosts { + calendarEvent, err := ds.GetCalendarEvent(ctx, host.Email) + switch { + case err == nil: + // OK + case fleet.IsNotFound(err): + continue + default: + return fmt.Errorf("get calendar event from DB: %w", err) + } + + if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { + return fmt.Errorf("delete db calendar event: %w", err) + } + if err := calendar.Configure(host.Email); err != nil { + return fmt.Errorf("connect to user calendar: %w", err) + } + if err := calendar.DeleteEvent(calendarEvent); err != nil { + return fmt.Errorf("delete calendar event: %w", err) + } + } + return nil +} + +func fireWebhookForHostsWithoutAssociatedEmail( + webhookURL string, + domain string, + hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, +) error { + // TODO(lucas): We are firing these every 5 minutes... + for _, host := range hosts { + if err := fleet.FireCalendarWebhook( + webhookURL, + host.HostID, host.HostHardwareSerial, host.HostDisplayName, nil, + fmt.Sprintf("No %s Google account associated with this host.", domain), + ); err != nil { + level.Error(logger).Log( + "msg", "fire webhook for hosts without associated email", "err", err, + ) + } + } + return nil +} + +func generateCalendarEventBody(orgName, hostDisplayName string) string { + return fmt.Sprintf(`Please leave your computer on and connected to power. + +Expect an automated restart. + +%s reserved this time to fix %s.`, orgName, hostDisplayName, + ) +} + +func isHostOnline(ctx context.Context, ds fleet.Datastore, hostID uint) (bool, error) { + hostLite, err := ds.HostLiteByID(ctx, hostID) + if err != nil { + return false, fmt.Errorf("get host lite: %w", err) + } + status := (&fleet.Host{ + DistributedInterval: hostLite.DistributedInterval, + ConfigTLSRefresh: hostLite.ConfigTLSRefresh, + SeenTime: hostLite.SeenTime, + }).Status(time.Now()) + + switch status { + case fleet.StatusOnline, fleet.StatusNew: + return true, nil + case fleet.StatusOffline, fleet.StatusMIA, fleet.StatusMissing: + return false, nil + default: + return false, fmt.Errorf("unknown host status: %s", status) + } +} diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go new file mode 100644 index 0000000000..680cf50d95 --- /dev/null +++ b/cmd/fleet/calendar_cron_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGetPreferredCalendarEventDate(t *testing.T) { + date := func(year int, month time.Month, day int) time.Time { + return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) + } + for _, tc := range []struct { + name string + year int + month time.Month + days int + + expected time.Time + }{ + { + year: 2024, + month: 3, + days: 31, + name: "March 2024", + expected: date(2024, 3, 19), + }, + { + year: 2024, + month: 4, + days: 30, + name: "April 2024", + expected: date(2024, 4, 16), + }, + } { + t.Run(tc.name, func(t *testing.T) { + for day := 1; day <= tc.days; day++ { + actual := getPreferredCalendarEventDate(tc.year, tc.month, day) + require.NotEqual(t, actual.Weekday(), time.Saturday) + require.NotEqual(t, actual.Weekday(), time.Sunday) + if day <= tc.expected.Day() { + require.Equal(t, tc.expected, actual) + } else { + today := date(tc.year, tc.month, day) + if weekday := today.Weekday(); weekday == time.Friday { + require.Equal(t, today.AddDate(0, 0, +3), actual) + } else if weekday == time.Saturday { + require.Equal(t, today.AddDate(0, 0, +2), actual) + } else { + require.Equal(t, today.AddDate(0, 0, +1), actual) + } + } + } + }) + } +} diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 729cc31801..0971c3e39a 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -768,6 +768,18 @@ the way that the Fleet server works. } } + if license.IsPremium() { + if err := cronSchedules.StartCronSchedule( + func() (fleet.CronSchedule, error) { + return newCalendarSchedule( + ctx, instanceID, ds, logger, + ) + }, + ); err != nil { + initFatal(err, "failed to register calendar schedule") + } + } + level.Info(logger).Log("msg", fmt.Sprintf("started cron schedules: %s", strings.Join(cronSchedules.ScheduleNames(), ", "))) // StartCollectors starts a goroutine per collector, using ctx to cancel. diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go new file mode 100644 index 0000000000..3990915976 --- /dev/null +++ b/server/datastore/mysql/calendar_events.go @@ -0,0 +1,150 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +func (ds *Datastore) NewCalendarEvent( + ctx context.Context, + email string, + startTime time.Time, + endTime time.Time, + data []byte, + hostID uint, +) (*fleet.CalendarEvent, error) { + var calendarEvent *fleet.CalendarEvent + if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + const calendarEventsQuery = ` + INSERT INTO calendar_events ( + email, + start_time, + end_time, + event + ) VALUES (?, ?, ?, ?); + ` + result, err := tx.ExecContext( + ctx, + calendarEventsQuery, + email, + startTime, + endTime, + data, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert calendar event") + } + + id, _ := result.LastInsertId() + calendarEvent = &fleet.CalendarEvent{ + ID: uint(id), + Email: email, + StartTime: startTime, + EndTime: endTime, + Data: data, + } + + const hostCalendarEventsQuery = ` + INSERT INTO host_calendar_events ( + host_id, + calendar_event_id, + webhook_status + ) VALUES (?, ?, ?); + ` + result, err = tx.ExecContext( + ctx, + hostCalendarEventsQuery, + hostID, + calendarEvent.ID, + fleet.CalendarWebhookStatusPending, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert host calendar event") + } + return nil + }); err != nil { + return nil, ctxerr.Wrap(ctx, err) + } + return calendarEvent, nil +} + +func (ds *Datastore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { + const calendarEventsQuery = ` + SELECT * FROM calendar_events WHERE email = ?; + ` + var calendarEvent fleet.CalendarEvent + err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, email) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithMessage(fmt.Sprintf("email: %s", email))) + } + return nil, ctxerr.Wrap(ctx, err, "get calendar event") + } + return &calendarEvent, nil +} + +func (ds *Datastore) UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error { + const calendarEventsQuery = ` + UPDATE calendar_events SET + start_time = ?, + end_time = ?, + event = ? + WHERE id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, startTime, endTime, data, calendarEventID); err != nil { + return ctxerr.Wrap(ctx, err, "update calendar event") + } + return nil +} + +func (ds *Datastore) DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error { + const calendarEventsQuery = ` + DELETE FROM calendar_events WHERE id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, calendarEventID); err != nil { + return ctxerr.Wrap(ctx, err, "delete calendar event") + } + return nil +} + +func (ds *Datastore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + const hostCalendarEventsQuery = ` + SELECT * FROM host_calendar_events WHERE host_id = ? + ` + var hostCalendarEvent fleet.HostCalendarEvent + if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostCalendarEvent, hostCalendarEventsQuery, hostID); err != nil { + if err == sql.ErrNoRows { + return nil, nil, ctxerr.Wrap(ctx, notFound("HostCalendarEvent").WithMessage(fmt.Sprintf("host_id: %d", hostID))) + } + return nil, nil, ctxerr.Wrap(ctx, err, "get host calendar event") + } + const calendarEventsQuery = ` + SELECT * FROM calendar_events WHERE id = ? + ` + var calendarEvent fleet.CalendarEvent + if err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, hostCalendarEvent.CalendarEventID); err != nil { + if err == sql.ErrNoRows { + return nil, nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithID(hostCalendarEvent.CalendarEventID)) + } + return nil, nil, ctxerr.Wrap(ctx, err, "get calendar event") + } + return &hostCalendarEvent, &calendarEvent, nil +} + +func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error { + const calendarEventsQuery = ` + UPDATE host_calendar_events SET + webhook_status = ? + WHERE host_id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, status, hostID); err != nil { + return ctxerr.Wrap(ctx, err, "update host calendar event webhook status") + } + return nil +} diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go new file mode 100644 index 0000000000..ccf07b3c7b --- /dev/null +++ b/server/datastore/mysql/calendar_events_test.go @@ -0,0 +1,6 @@ +package mysql + +import "testing" + +func TestCalendarEvents(t *testing.T) { +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index d2f2424072..0b3498319d 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -5,11 +5,12 @@ import ( "database/sql" "encoding/json" "fmt" - "golang.org/x/text/unicode/norm" "sort" "strings" "time" + "golang.org/x/text/unicode/norm" + "github.com/doug-martin/goqu/v9" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" @@ -1159,3 +1160,52 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { return nil } + +func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + query := `SELECT id, name FROM policies WHERE team_id = ? AND calendar_events_enabled;` + var policies []fleet.PolicyCalendarData + err := sqlx.SelectContext(ctx, ds.reader(ctx), &policies, query, teamID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get calendar policies") + } + return policies, nil +} + +// TODO(lucas): Must be tested at scale. +func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) { + query := ` + SELECT + COALESCE(sh.email, '') AS email, + pm.passing AS passing, + h.id AS host_id, + hdn.display_name AS host_display_name, + h.hardware_serial AS host_hardware_serial + FROM ( + SELECT host_id, BIT_AND(COALESCE(passes, 0)) AS passing + FROM policy_membership + WHERE policy_id IN (?) + GROUP BY host_id + ) pm + LEFT JOIN ( + SELECT MIN(h.host_id) as host_id, h.email as email + FROM ( + SELECT host_id, MIN(email) AS email + FROM host_emails WHERE email LIKE CONCAT('%@', ?) + GROUP BY host_id + ) h GROUP BY h.email + ) sh ON sh.host_id = pm.host_id + JOIN hosts h ON h.id = pm.host_id + LEFT JOIN host_display_names hdn ON hdn.host_id = pm.host_id; +` + + query, args, err := sqlx.In(query, policyIDs, domain) + if err != nil { + return nil, ctxerr.Wrapf(ctx, err, "build select get team hosts policy memberships query") + } + var hosts []fleet.HostPolicyMembershipData + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hosts, query, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "listing policies") + } + + return hosts, nil +} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index b0ef3b1bcc..514de6dd38 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -59,6 +59,7 @@ func TestPolicies(t *testing.T) { {"TestPoliciesNameUnicode", testPoliciesNameUnicode}, {"TestPoliciesNameEmoji", testPoliciesNameEmoji}, {"TestPoliciesNameSort", testPoliciesNameSort}, + {"TestGetCalendarPolicies", testGetCalendarPolicies}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -2784,7 +2785,6 @@ func testPoliciesNameEmoji(t *testing.T, ds *Datastore) { assert.NoError(t, err) require.Len(t, policies, 1) assert.Equal(t, emoji1, policies[0].Name) - } // Ensure case-insensitive sort order for policy names @@ -2806,3 +2806,57 @@ func testPoliciesNameSort(t *testing.T, ds *Datastore) { assert.Equal(t, policy.Name, policiesResult[i].Name) } } + +func testGetCalendarPolicies(t *testing.T, ds *Datastore) { + ctx := context.Background() + + // Test with non-existent team. + _, err := ds.GetCalendarPolicies(ctx, 999) + require.NoError(t, err) + + team, err := ds.NewTeam(ctx, &fleet.Team{ + Name: "Foobar", + }) + require.NoError(t, err) + + // Test when the team has no policies. + _, err = ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + + // Create a global query to test that only team policies are returned. + _, err = ds.NewGlobalPolicy(ctx, nil, fleet.PolicyPayload{ + Name: "Global Policy", + Query: "SELECT * FROM time;", + }) + require.NoError(t, err) + + _, err = ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 1", + Query: "SELECT * FROM system_info;", + CalendarEventsEnabled: false, + }) + require.NoError(t, err) + + // Test when the team has policies, but none is configured for calendar. + _, err = ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + + teamPolicy2, err := ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 2", + Query: "SELECT * FROM osquery_info;", + CalendarEventsEnabled: true, + }) + require.NoError(t, err) + teamPolicy3, err := ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 3", + Query: "SELECT * FROM os_version;", + CalendarEventsEnabled: true, + }) + require.NoError(t, err) + + calendarPolicies, err := ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + require.Len(t, calendarPolicies, 2) + require.Equal(t, calendarPolicies[0].ID, teamPolicy2.ID) + require.Equal(t, calendarPolicies[1].ID, teamPolicy3.ID) +} diff --git a/server/fleet/app.go b/server/fleet/app.go index 4b936063a0..560b8bb345 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -571,6 +571,13 @@ func (c *AppConfig) Copy() *AppConfig { clone.Integrations.Zendesk[i] = &zd } } + if len(c.Integrations.GoogleCalendar) > 0 { + clone.Integrations.GoogleCalendar = make([]*GoogleCalendarIntegration, len(c.Integrations.GoogleCalendar)) + for i, g := range c.Integrations.GoogleCalendar { + gc := *g + clone.Integrations.GoogleCalendar[i] = &gc + } + } if c.MDM.MacOSSettings.CustomSettings != nil { clone.MDM.MacOSSettings.CustomSettings = make([]MDMProfileSpec, len(c.MDM.MacOSSettings.CustomSettings)) diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index db2bbbc45e..9b45c2c8a5 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -1,6 +1,12 @@ package fleet -import "time" +import ( + "context" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/server" +) type DayEndedError struct { Msg string @@ -23,3 +29,33 @@ type UserCalendar interface { // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } + +type CalendarWebhookPayload struct { + Timestamp time.Time `json:"timestamp"` + HostID uint `json:"host_id"` + HostDisplayName string `json:"host_display_name"` + HostSerialNumber string `json:"host_serial_number"` + FailingPolicies []PolicyCalendarData `json:"failing_policies,omitempty"` + Error string `json:"error,omitempty"` +} + +func FireCalendarWebhook( + webhookURL string, + hostID uint, + hostHardwareSerial string, + hostDisplayName string, + failingCalendarPolicies []PolicyCalendarData, + err string, +) error { + if err := server.PostJSONWithTimeout(context.Background(), webhookURL, &CalendarWebhookPayload{ + Timestamp: time.Now(), + HostID: hostID, + HostDisplayName: hostDisplayName, + HostSerialNumber: hostHardwareSerial, + FailingPolicies: failingCalendarPolicies, + Error: err, + }); err != nil { + return fmt.Errorf("POST to %q: %w", server.MaskSecretURLParams(webhookURL), server.MaskURLError(err)) + } + return nil +} diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go index 7671b4aba5..348cb074a4 100644 --- a/server/fleet/calendar_events.go +++ b/server/fleet/calendar_events.go @@ -27,3 +27,12 @@ type HostCalendarEvent struct { UpdateCreateTimestamps } + +type HostPolicyMembershipData struct { + Email string `db:"email"` + Passing bool `db:"passing"` + + HostID uint `db:"host_id"` + HostDisplayName string `db:"host_display_name"` + HostHardwareSerial string `db:"host_hardware_serial"` +} diff --git a/server/fleet/cron_schedules.go b/server/fleet/cron_schedules.go index 607f15f85c..6b16734fd4 100644 --- a/server/fleet/cron_schedules.go +++ b/server/fleet/cron_schedules.go @@ -21,6 +21,7 @@ const ( CronWorkerIntegrations CronScheduleName = "integrations" CronActivitiesStreaming CronScheduleName = "activities_streaming" CronMDMAppleProfileManager CronScheduleName = "mdm_apple_profile_manager" + CronCalendar CronScheduleName = "calendar" ) type CronSchedulesService interface { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 4081db8af0..f2178bf32b 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -594,6 +594,9 @@ type Datastore interface { PolicyQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) + GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]HostPolicyMembershipData, error) + GetCalendarPolicies(ctx context.Context, teamID uint) ([]PolicyCalendarData, error) + // Methods used for async processing of host policy query results. AsyncBatchInsertPolicyMembership(ctx context.Context, batch []PolicyMembershipResult) error AsyncBatchUpdatePolicyTimestamp(ctx context.Context, ids []uint, ts time.Time) error @@ -613,6 +616,16 @@ type Datastore interface { // the updated_at timestamp is older than the provided duration DeleteOutOfDateVulnerabilities(ctx context.Context, source VulnerabilitySource, duration time.Duration) error + /////////////////////////////////////////////////////////////////////////////// + // Calendar events + + NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*CalendarEvent, error) + GetCalendarEvent(ctx context.Context, email string) (*CalendarEvent, error) + DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error + UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error + GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error) + UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error + /////////////////////////////////////////////////////////////////////////////// // Team Policies diff --git a/server/fleet/policies.go b/server/fleet/policies.go index 52a6109b2a..dda2ec047d 100644 --- a/server/fleet/policies.go +++ b/server/fleet/policies.go @@ -179,6 +179,11 @@ type Policy struct { HostCountUpdatedAt *time.Time `json:"host_count_updated_at" db:"host_count_updated_at"` } +type PolicyCalendarData struct { + ID uint `db:"id" json:"id"` + Name string `db:"name" json:"name"` +} + func (p Policy) AuthzType() string { return "policy" } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 1469826979..4e35d1eef3 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -440,6 +440,10 @@ type UpdateHostPolicyCountsFunc func(ctx context.Context) error type PolicyQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) +type GetHostsPolicyMembershipsFunc func(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) + +type GetCalendarPoliciesFunc func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) + type AsyncBatchInsertPolicyMembershipFunc func(ctx context.Context, batch []fleet.PolicyMembershipResult) error type AsyncBatchUpdatePolicyTimestampFunc func(ctx context.Context, ids []uint, ts time.Time) error @@ -458,6 +462,18 @@ type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error +type NewCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) + +type GetCalendarEventFunc func(ctx context.Context, email string) (*fleet.CalendarEvent, error) + +type DeleteCalendarEventFunc func(ctx context.Context, calendarEventID uint) error + +type UpdateCalendarEventFunc func(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error + +type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) + +type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error + type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error) @@ -1492,6 +1508,12 @@ type DataStore struct { PolicyQueriesForHostFunc PolicyQueriesForHostFunc PolicyQueriesForHostFuncInvoked bool + GetHostsPolicyMembershipsFunc GetHostsPolicyMembershipsFunc + GetHostsPolicyMembershipsFuncInvoked bool + + GetCalendarPoliciesFunc GetCalendarPoliciesFunc + GetCalendarPoliciesFuncInvoked bool + AsyncBatchInsertPolicyMembershipFunc AsyncBatchInsertPolicyMembershipFunc AsyncBatchInsertPolicyMembershipFuncInvoked bool @@ -1519,6 +1541,24 @@ type DataStore struct { DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFuncInvoked bool + NewCalendarEventFunc NewCalendarEventFunc + NewCalendarEventFuncInvoked bool + + GetCalendarEventFunc GetCalendarEventFunc + GetCalendarEventFuncInvoked bool + + DeleteCalendarEventFunc DeleteCalendarEventFunc + DeleteCalendarEventFuncInvoked bool + + UpdateCalendarEventFunc UpdateCalendarEventFunc + UpdateCalendarEventFuncInvoked bool + + GetHostCalendarEventFunc GetHostCalendarEventFunc + GetHostCalendarEventFuncInvoked bool + + UpdateHostCalendarWebhookStatusFunc UpdateHostCalendarWebhookStatusFunc + UpdateHostCalendarWebhookStatusFuncInvoked bool + NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFuncInvoked bool @@ -3599,6 +3639,20 @@ func (s *DataStore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) return s.PolicyQueriesForHostFunc(ctx, host) } +func (s *DataStore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) { + s.mu.Lock() + s.GetHostsPolicyMembershipsFuncInvoked = true + s.mu.Unlock() + return s.GetHostsPolicyMembershipsFunc(ctx, domain, policyIDs) +} + +func (s *DataStore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + s.mu.Lock() + s.GetCalendarPoliciesFuncInvoked = true + s.mu.Unlock() + return s.GetCalendarPoliciesFunc(ctx, teamID) +} + func (s *DataStore) AsyncBatchInsertPolicyMembership(ctx context.Context, batch []fleet.PolicyMembershipResult) error { s.mu.Lock() s.AsyncBatchInsertPolicyMembershipFuncInvoked = true @@ -3662,6 +3716,48 @@ func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source f return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration) } +func (s *DataStore) NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) { + s.mu.Lock() + s.NewCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.NewCalendarEventFunc(ctx, email, startTime, endTime, data, hostID) +} + +func (s *DataStore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { + s.mu.Lock() + s.GetCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.GetCalendarEventFunc(ctx, email) +} + +func (s *DataStore) DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error { + s.mu.Lock() + s.DeleteCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.DeleteCalendarEventFunc(ctx, calendarEventID) +} + +func (s *DataStore) UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error { + s.mu.Lock() + s.UpdateCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.UpdateCalendarEventFunc(ctx, calendarEventID, startTime, endTime, data) +} + +func (s *DataStore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + s.mu.Lock() + s.GetHostCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.GetHostCalendarEventFunc(ctx, hostID) +} + +func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error { + s.mu.Lock() + s.UpdateHostCalendarWebhookStatusFuncInvoked = true + s.mu.Unlock() + return s.UpdateHostCalendarWebhookStatusFunc(ctx, hostID, status) +} + func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { s.mu.Lock() s.NewTeamPolicyFuncInvoked = true diff --git a/server/service/osquery.go b/server/service/osquery.go index 8a77903a88..379afafd6a 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1001,6 +1001,10 @@ func (svc *Service) SubmitDistributedQueryResults( if len(policyResults) > 0 { + if err := processCalendarPolicies(ctx, svc.ds, ac, host, policyResults, svc.logger); err != nil { + logging.WithErr(ctx, err) + } + // filter policy results for webhooks var policyIDs []uint if globalPolicyAutomationsEnabled(ac.WebhookSettings, ac.Integrations) { @@ -1093,6 +1097,99 @@ func (svc *Service) SubmitDistributedQueryResults( return nil } +func processCalendarPolicies( + ctx context.Context, + ds fleet.Datastore, + appConfig *fleet.AppConfig, + host *fleet.Host, + policyResults map[uint]*bool, + logger log.Logger, +) error { + if len(appConfig.Integrations.GoogleCalendar) == 0 || host.TeamID == nil { + return nil + } + + team, err := ds.Team(ctx, *host.TeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "load host team") + } + + if team.Config.Integrations.GoogleCalendar == nil || !team.Config.Integrations.GoogleCalendar.Enable { + return nil + } + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.ID) + switch { + case err == nil: + if hostCalendarEvent.WebhookStatus != fleet.CalendarWebhookStatusPending { + return nil + } + case fleet.IsNotFound(err): + return nil + default: + return ctxerr.Wrap(ctx, err, "get host calendar event") + } + + now := time.Now() + if now.Before(calendarEvent.StartTime) { + level.Warn(logger).Log("msg", "results came too early", "now", now, "start_time", calendarEvent.StartTime) + return nil + } + + // + // TODO(lucas): Discuss. + // + const allowedTimeBeforeEndTime = 5 * time.Minute // up to 5 minutes before the end_time + + if now.After(calendarEvent.EndTime.Add(-allowedTimeBeforeEndTime)) { + level.Warn(logger).Log("msg", "results came too late", "now", now, "end_time", calendarEvent.EndTime) + return nil + } + + calendarPolicies, err := ds.GetCalendarPolicies(ctx, *host.TeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "get calendar policy ids") + } + if len(calendarPolicies) == 0 { + return nil + } + + failingCalendarPolicies := getFailingCalendarPolicies(policyResults, calendarPolicies) + if len(failingCalendarPolicies) == 0 { + return nil + } + + go func() { + if err := fleet.FireCalendarWebhook( + team.Config.Integrations.GoogleCalendar.WebhookURL, + host.ID, host.HardwareSerial, host.DisplayName(), failingCalendarPolicies, "", + ); err != nil { + level.Error(logger).Log("msg", "fire webhook", "err", err) + return + } + if err := ds.UpdateHostCalendarWebhookStatus(context.Background(), host.ID, fleet.CalendarWebhookStatusSent); err != nil { + level.Error(logger).Log("msg", "mark fired webhook as sent", "err", err) + } + }() + + return nil +} + +func getFailingCalendarPolicies(policyResults map[uint]*bool, calendarPolicies []fleet.PolicyCalendarData) []fleet.PolicyCalendarData { + var failingPolicies []fleet.PolicyCalendarData + for _, calendarPolicy := range calendarPolicies { + result, ok := policyResults[calendarPolicy.ID] + if !ok || // ignore result of a policy that's not configured for calendar. + result == nil { // ignore policies that failed to execute. + continue + } + if !*result { + failingPolicies = append(failingPolicies, calendarPolicy) + } + } + return failingPolicies +} + // preProcessSoftwareResults will run pre-processing on the responses of the software queries. // It will move the results from the software extra queries (e.g. software_vscode_extensions) // into the main software query results (software_{macos|linux|windows}). From 196d8ce5b75c69f5ab12a60cfc8d5e2f0e8e3073 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 16:19:38 -0500 Subject: [PATCH 10/30] Calendar interface updates and mock calendar (#17701) - Updated calendar interface to use updated `genBodyFn` - The mock calendar is enabled by specifying `calendar-mock@example.com` as the service account email. --- cmd/fleet/calendar_cron.go | 14 ++-- ee/server/calendar/google_calendar.go | 46 +++++++----- ee/server/calendar/google_calendar_mock.go | 81 ++++++++++++++++++++++ ee/server/calendar/google_calendar_test.go | 59 ++++++++++++---- server/fleet/calendar.go | 4 +- 5 files changed, 166 insertions(+), 38 deletions(-) create mode 100644 ee/server/calendar/google_calendar_mock.go diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 099a938b2c..e8ec7685d7 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -231,9 +231,10 @@ func processFailingHostExistingCalendarEvent( calendarEvent *fleet.CalendarEvent, host fleet.HostPolicyMembershipData, ) error { - updatedEvent, updated, err := calendar.GetAndUpdateEvent(calendarEvent, func() string { - return generateCalendarEventBody(orgName, host.HostDisplayName) - }) + updatedEvent, updated, err := calendar.GetAndUpdateEvent( + calendarEvent, func(bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }) if err != nil { return fmt.Errorf("get event calendar on db: %w", err) } @@ -325,9 +326,12 @@ func attemptCreatingEventOnUserCalendar( // - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day. year, month, today := time.Now().Date() preferredDate := getPreferredCalendarEventDate(year, month, today) - body := generateCalendarEventBody(orgName, host.HostDisplayName) for { - calendarEvent, err := userCalendar.CreateEvent(preferredDate, body) + calendarEvent, err := userCalendar.CreateEvent( + preferredDate, func(bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }, + ) var dee fleet.DayEndedError switch { case err == nil: diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 53e189e24b..91e1661a84 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -50,8 +50,12 @@ type GoogleCalendar struct { func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { if config.API == nil { - var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} - config.API = lowLevelAPI + if config.IntegrationConfig.Email == "calendar-mock@example.com" { + // Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory + config.API = &GoogleCalendarMockAPI{} + } else { + config.API = &GoogleCalendarLowLevelAPI{} + } } return &GoogleCalendar{ config: config, @@ -136,7 +140,9 @@ func (c *GoogleCalendar) Configure(userEmail string) error { return nil } -func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { +func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func(conflict bool) string) ( + *fleet.CalendarEvent, bool, error, +) { // We assume that the Fleet event has not already ended. We will simply return it if it has not been modified. details, err := c.unmarshalDetails(event) if err != nil { @@ -167,6 +173,10 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn if gEvent.End.DateTime == "" { // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. // We won't handle all-day events at this time, and treat the event as deleted. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err) + } deleted = true } @@ -194,6 +204,10 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn if gEvent.Start.DateTime == "" { // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. // We won't handle all-day events at this time, and treat the event as deleted. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err) + } deleted = true } } @@ -212,7 +226,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn newStartDate := calculateNewEventDate(event.StartTime) - fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) + fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn) if err != nil { return nil, false, err } @@ -269,13 +283,15 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet return &details, nil } -func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { - return c.createEvent(dayOfEvent, body, time.Now) +func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, genBodyFn func(conflict bool) string) (*fleet.CalendarEvent, error) { + return c.createEvent(dayOfEvent, genBodyFn, time.Now) } // createEvent creates a new event on the calendar on the given date. timeNow is a function that returns the current time. // timeNow can be overwritten for testing -func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow func() time.Time) (*fleet.CalendarEvent, error) { +func (c *GoogleCalendar) createEvent( + dayOfEvent time.Time, genBodyFn func(conflict bool) string, timeNow func() time.Time, +) (*fleet.CalendarEvent, error) { if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -310,6 +326,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") } + var conflict bool for _, gEvent := range events.Items { // Ignore cancelled events if gEvent.Status == "cancelled" { @@ -353,7 +370,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow if startTime.Before(eventEnd) { // Event occurs during our event, so we need to adjust. var isLastSlot bool - eventStart, eventEnd, isLastSlot = adjustEventTimes(*endTime, dayEnd) + eventStart, eventEnd, isLastSlot, conflict = adjustEventTimes(*endTime, dayEnd) if isLastSlot { break } @@ -367,7 +384,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow event.Start = &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)} event.End = &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)} event.Summary = eventTitle - event.Description = body + event.Description = genBodyFn(conflict) event, err = c.config.API.CreateEvent(event) if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "creating Google calendar event") @@ -383,7 +400,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow return fleetEvent, nil } -func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool) { +func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool, conflict bool) { eventStart = endTime.Truncate(eventLength) if eventStart.Before(endTime) { eventStart = eventStart.Add(eventLength) @@ -394,11 +411,11 @@ func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time eventEnd = dayEnd eventStart = eventEnd.Add(-eventLength) isLastSlot = true - } - if eventEnd.Equal(dayEnd) { + conflict = true + } else if eventEnd.Equal(dayEnd) { isLastSlot = true } - return eventStart, eventEnd, isLastSlot + return eventStart, eventEnd, isLastSlot, conflict } func getTimezone(gCal *GoogleCalendar) error { @@ -444,9 +461,6 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti } func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { - if c.config == nil { - return errors.New("the Google calendar is not connected. Please call Configure first") - } details, err := c.unmarshalDetails(event) if err != nil { return err diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go new file mode 100644 index 0000000000..a6d7d6040a --- /dev/null +++ b/ee/server/calendar/google_calendar_mock.go @@ -0,0 +1,81 @@ +package calendar + +import ( + "context" + "errors" + kitlog "github.com/go-kit/log" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +type GoogleCalendarMockAPI struct { + logger kitlog.Logger +} + +var events = make(map[string]*calendar.Event) +var mu sync.Mutex +var id uint64 + +const latency = 500 * time.Millisecond + +// Configure creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarMockAPI) Configure(_ context.Context, _ string, _ string, userToImpersonate string) error { + if lowLevelAPI.logger == nil { + lowLevelAPI.logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stderr), "mock", "GoogleCalendarMockAPI", "user", userToImpersonate) + } + return nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) GetSetting(name string) (*calendar.Setting, error) { + time.Sleep(latency) + lowLevelAPI.logger.Log("msg", "GetSetting", "name", name) + if name == "timezone" { + return &calendar.Setting{ + Id: "timezone", + Value: "America/Chicago", + }, nil + } + return nil, errors.New("setting not supported") +} + +func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + mu.Lock() + defer mu.Unlock() + id += 1 + event.Id = strconv.FormatUint(id, 10) + lowLevelAPI.logger.Log("msg", "CreateEvent", "id", event.Id, "start", event.Start.DateTime) + events[event.Id] = event + return event, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) GetEvent(id, _ string) (*calendar.Event, error) { + time.Sleep(latency) + mu.Lock() + defer mu.Unlock() + event, ok := events[id] + if !ok { + return nil, &googleapi.Error{Code: http.StatusNotFound} + } + lowLevelAPI.logger.Log("msg", "GetEvent", "id", id, "start", event.Start.DateTime) + return event, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) ListEvents(string, string) (*calendar.Events, error) { + time.Sleep(latency) + lowLevelAPI.logger.Log("msg", "ListEvents") + return &calendar.Events{}, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error { + time.Sleep(latency) + mu.Lock() + defer mu.Unlock() + lowLevelAPI.logger.Log("msg", "DeleteEvent", "id", id) + delete(events, id) + return nil +} diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go index 4c3e2db092..cd36242751 100644 --- a/ee/server/calendar/google_calendar_test.go +++ b/ee/server/calendar/google_calendar_test.go @@ -162,7 +162,7 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { Etag: baseETag, // ETag matches -- no modifications to event }, nil } - genBodyFn := func() string { + genBodyFn := func(bool) string { t.Error("genBodyFn should not be called") return "event-body" } @@ -300,13 +300,14 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { return &calendar.Events{}, nil } - genBodyFn = func() string { + genBodyFn = func(conflict bool) string { + assert.False(t, conflict) return "event-body" } eventCreated := false mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { assert.Equal(t, eventTitle, event.Summary) - assert.Equal(t, genBodyFn(), event.Description) + assert.Equal(t, genBodyFn(false), event.Description) event.Id = baseEventID event.Etag = baseETag eventCreated = true @@ -345,6 +346,10 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { assert.True(t, eventCreated) // all day event (deleted) + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, baseEventID, id) + return nil + } mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { return &calendar.Event{ Id: baseEventID, @@ -373,10 +378,6 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { }, nil } eventCreated = false - mockAPI.DeleteEventFunc = func(id string) error { - assert.Equal(t, baseEventID, id) - return nil - } retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) require.NoError(t, err) assert.True(t, updated) @@ -411,13 +412,21 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { event.Etag = baseETag return event, nil } + genBodyFn := func(conflict bool) string { + assert.False(t, conflict) + return eventBody + } + genBodyConflictFn := func(conflict bool) string { + assert.True(t, conflict) + return eventBody + } // Happy path test -- empty calendar date := time.Now().Add(48 * time.Hour) location, _ := time.LoadLocation(tzId) expectedStartTime := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) _, expectedOffset := expectedStartTime.Zone() - event, err := cal.CreateEvent(date, eventBody) + event, err := cal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, baseUserEmail, event.Email) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) @@ -434,7 +443,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { // Workday already ended date = time.Now().Add(-48 * time.Hour) - _, err = cal.CreateEvent(date, eventBody) + _, err = cal.CreateEvent(date, genBodyFn) assert.ErrorAs(t, err, &fleet.DayEndedError{}) // There is no time left in the day to schedule an event @@ -443,7 +452,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { now := time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 45, 0, 0, location) return now } - _, err = gCal.createEvent(date, eventBody, timeNow) + _, err = gCal.createEvent(date, genBodyFn, timeNow) assert.ErrorAs(t, err, &fleet.DayEndedError{}) // Workday already started @@ -452,7 +461,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { timeNow = func() time.Time { return expectedStartTime } - event, err = gCal.createEvent(date, eventBody, timeNow) + event, err = gCal.createEvent(date, genBodyFn, timeNow) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -545,7 +554,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { return gEvents, nil } expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), 12, 0, 0, 0, location) - event, err = gCal.CreateEvent(date, eventBody) + event, err = gCal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -565,7 +574,27 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { return gEvents, nil } expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) - event, err = gCal.CreateEvent(date, eventBody) + event, err = gCal.CreateEvent(date, genBodyConflictFn) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Almost full schedule -- pick the last slot + date = time.Now().Add(48 * time.Hour) + dayStart = time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + gEvents = &calendar.Events{} + gEvent = &calendar.Event{ + Id: "9-to-4-30-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = dayEnd + event, err = gCal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -574,7 +603,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { return nil, assert.AnError } - _, err = gCal.CreateEvent(date, eventBody) + _, err = gCal.CreateEvent(date, genBodyFn) assert.ErrorIs(t, err, assert.AnError) // API error in CreateEvent @@ -584,6 +613,6 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { return nil, assert.AnError } - _, err = gCal.CreateEvent(date, eventBody) + _, err = gCal.CreateEvent(date, genBodyFn) assert.ErrorIs(t, err, assert.AnError) } diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index 9b45c2c8a5..592fff430f 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -21,11 +21,11 @@ type UserCalendar interface { // CreateEvent, GetAndUpdateEvent and DeleteEvent reference the user's calendar. Configure(userEmail string) error // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. - CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) + CreateEvent(dateOfEvent time.Time, genBodyFn func(conflict bool) string) (event *CalendarEvent, err error) // GetAndUpdateEvent retrieves the event from the calendar. // If the event has been modified, it returns the updated event. // If the event has been deleted, it schedules a new event with given body callback and returns the new event. - GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) + GetAndUpdateEvent(event *CalendarEvent, genBodyFn func(conflict bool) string) (updatedEvent *CalendarEvent, updated bool, err error) // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } From e4ba41ac85f8e2160e567f47cc141bc32a3f7784 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 19:46:55 -0500 Subject: [PATCH 11/30] Latest changes to configs (#17724) - Remove email from team configs - Accept api_key_json for global config --- cmd/fleetctl/apply_test.go | 29 +------ cmd/fleetctl/gitops_test.go | 9 +- .../gitops/global_config_no_paths.yml | 8 +- .../testdata/gitops/team_config_no_paths.yml | 1 - ee/server/calendar/google_calendar.go | 3 +- ee/server/calendar/google_calendar_test.go | 6 +- ee/server/service/teams.go | 21 ++--- server/fleet/app.go | 10 +-- server/fleet/integrations.go | 45 +++++++--- server/service/appconfig_test.go | 5 +- server/service/integration_core_test.go | 87 +++++++++++-------- server/service/integration_enterprise_test.go | 10 +-- .../generated_files/appconfig.txt | 3 +- 13 files changed, 120 insertions(+), 117 deletions(-) diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 655e2e57b0..e30bd3939d 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -143,17 +143,12 @@ func TestApplyTeamSpecs(t *testing.T) { } agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) - googleCalEmail := "service-valid@example.com" ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{ AgentOptions: &agentOpts, MDM: fleet.MDM{EnabledAndConfigured: true}, Integrations: fleet.Integrations{ - GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - { - Email: googleCalEmail, - }, - }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{{}}, }, }, nil } @@ -461,7 +456,6 @@ spec: name: team1 integrations: google_calendar: - email: `+googleCalEmail+` enable_calendar_events: true webhook_url: https://example.com/webhook `, @@ -470,31 +464,11 @@ spec: require.NotNil(t, teamsByName["team1"].Config.Integrations.GoogleCalendar) assert.Equal( t, fleet.TeamGoogleCalendarIntegration{ - Email: googleCalEmail, Enable: true, WebhookURL: "https://example.com/webhook", }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, ) - // Apply calendar integration -- invalid email - filename = writeTmpYml( - t, ` -apiVersion: v1 -kind: team -spec: - team: - name: team1 - integrations: - google_calendar: - email: not_present_globally@example.com - enable_calendar_events: true - webhook_url: https://example.com/webhook -`, - ) - - _, err = runAppNoChecks([]string{"apply", "-f", filename}) - assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") - // Apply calendar integration -- invalid webhook destination filename = writeTmpYml( t, ` @@ -505,7 +479,6 @@ spec: name: team1 integrations: google_calendar: - email: `+googleCalEmail+` enable_calendar_events: true webhook_url: bozo `, diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index c6013aa959..6a8fab6482 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -361,7 +361,7 @@ func TestFullGlobalGitOps(t *testing.T) { assert.Len(t, appliedMacProfiles, 1) assert.Len(t, appliedWinProfiles, 1) require.Len(t, savedAppConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].ApiKey["client_email"]) } func TestFullTeamGitOps(t *testing.T) { @@ -392,11 +392,7 @@ func TestFullTeamGitOps(t *testing.T) { WindowsEnabledAndConfigured: true, }, Integrations: fleet.Integrations{ - GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - { - Email: "service@example.com", - }, - }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{{}}, }, }, nil } @@ -546,7 +542,6 @@ func TestFullTeamGitOps(t *testing.T) { assert.True(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "https://example.com/host_status_webhook", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) - assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) // Now clear the settings diff --git a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml index cf2eeece60..b487bf46e7 100644 --- a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml @@ -138,9 +138,11 @@ org_settings: jira: [] zendesk: [] google_calendar: - - email: service@example.com - private_key: google_calendar_private_key - domain: example.com + - domain: example.com + api_key_json: { + "client_email": "service@example.com", + "private_key": "google_calendar_private_key", + } mdm: apple_bm_default_team: "" end_user_authentication: diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 58564a7bc2..4785c72a73 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -17,7 +17,6 @@ team_settings: host_expiry_window: 30 integrations: google_calendar: - email: service@example.com enable_calendar_events: true webhook_url: https://example.com/google_calendar_webhook agent_options: diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 91e1661a84..9c0c14707f 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -131,7 +131,8 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { func (c *GoogleCalendar) Configure(userEmail string) error { err := c.config.API.Configure( - c.config.Context, c.config.IntegrationConfig.Email, c.config.IntegrationConfig.PrivateKey, userEmail, + c.config.Context, c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail], + c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], userEmail, ) if err != nil { return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service") diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go index cd36242751..ad5e1c89ca 100644 --- a/ee/server/calendar/google_calendar_test.go +++ b/ee/server/calendar/google_calendar_test.go @@ -93,8 +93,10 @@ func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig { config := &GoogleCalendarConfig{ Context: context.Background(), IntegrationConfig: &fleet.GoogleCalendarIntegration{ - Email: baseServiceEmail, - PrivateKey: basePrivateKey, + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: baseServiceEmail, + fleet.GoogleCalendarPrivateKey: basePrivateKey, + }, }, Logger: logger, API: mockAPI, diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index e0f503b381..fe81963ab5 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/authz" @@ -214,7 +213,7 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T // Only update the calendar integration if it's not nil if payload.Integrations.GoogleCalendar != nil { invalid := &fleet.InvalidArgumentError{} - _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) + _ = svc.validateTeamCalendarIntegrations(payload.Integrations.GoogleCalendar, appCfg, invalid) if invalid.HasErrors() { return nil, ctxerr.Wrap(ctx, invalid) } @@ -1083,7 +1082,7 @@ func (svc *Service) editTeamFromSpec( } if spec.Integrations.GoogleCalendar != nil { - err = svc.validateTeamCalendarIntegrations(ctx, team, spec.Integrations.GoogleCalendar, appCfg, invalid) + err = svc.validateTeamCalendarIntegrations(spec.Integrations.GoogleCalendar, appCfg, invalid) if err != nil { return ctxerr.Wrap(ctx, err, "validate team calendar integrations") } @@ -1157,23 +1156,15 @@ func (svc *Service) editTeamFromSpec( } func (svc *Service) validateTeamCalendarIntegrations( - ctx context.Context, team *fleet.Team, calendarIntegration *fleet.TeamGoogleCalendarIntegration, + calendarIntegration *fleet.TeamGoogleCalendarIntegration, appCfg *fleet.AppConfig, invalid *fleet.InvalidArgumentError, ) error { if !calendarIntegration.Enable { return nil } - // Validate email - emailValid := false - calendarIntegration.Email = strings.TrimSpace(calendarIntegration.Email) - for _, globalCals := range appCfg.Integrations.GoogleCalendar { - if globalCals.Email == calendarIntegration.Email { - emailValid = true - break - } - } - if !emailValid { - invalid.Append("integrations.google_calendar.email", "email must match a global Google Calendar integration email") + // Check that global configs exist + if len(appCfg.Integrations.GoogleCalendar) == 0 { + invalid.Append("integrations.google_calendar.enable_calendar_events", "global Google Calendar integration is not configured") } // Validate URL if u, err := url.ParseRequestURI(calendarIntegration.WebhookURL); err != nil { diff --git a/server/fleet/app.go b/server/fleet/app.go index 560b8bb345..e1720e5963 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "net/url" "reflect" "regexp" @@ -488,9 +489,6 @@ func (c *AppConfig) Obfuscate() { for _, zdIntegration := range c.Integrations.Zendesk { zdIntegration.APIToken = MaskedPassword } - for _, calIntegration := range c.Integrations.GoogleCalendar { - calIntegration.PrivateKey = MaskedPassword - } } // Clone implements cloner. @@ -574,8 +572,10 @@ func (c *AppConfig) Copy() *AppConfig { if len(c.Integrations.GoogleCalendar) > 0 { clone.Integrations.GoogleCalendar = make([]*GoogleCalendarIntegration, len(c.Integrations.GoogleCalendar)) for i, g := range c.Integrations.GoogleCalendar { - gc := *g - clone.Integrations.GoogleCalendar[i] = &gc + gCal := *g + clone.Integrations.GoogleCalendar[i] = &gCal + clone.Integrations.GoogleCalendar[i].ApiKey = make(map[string]string, len(g.ApiKey)) + maps.Copy(clone.Integrations.GoogleCalendar[i].ApiKey, g.ApiKey) } } diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 8c34509485..18cce71825 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -112,7 +112,6 @@ func (z TeamZendeskIntegration) UniqueKey() string { } type TeamGoogleCalendarIntegration struct { - Email string `json:"email"` Enable bool `json:"enable_calendar_events"` WebhookURL string `json:"webhook_url"` } @@ -342,10 +341,14 @@ func makeTestZendeskRequest(ctx context.Context, intg *ZendeskIntegration) error return nil } +const ( + GoogleCalendarEmail = "client_email" + GoogleCalendarPrivateKey = "private_key" +) + type GoogleCalendarIntegration struct { - Email string `json:"email"` - PrivateKey string `json:"private_key"` - Domain string `json:"domain"` + Domain string `json:"domain"` + ApiKey map[string]string `json:"api_key_json"` } // Integrations configures the integrations with external systems. @@ -378,13 +381,35 @@ func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, inva invalid.Append("integrations.google_calendar", "integrating with >1 Google Workspace service account is not yet supported.") } for _, intg := range intgs { - intg.Email = strings.TrimSpace(intg.Email) - if intg.Email == "" { - invalid.Append("integrations.google_calendar.email", "email is required") + if email, ok := intg.ApiKey[GoogleCalendarEmail]; !ok { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarEmail), + fmt.Sprintf("%s is required", GoogleCalendarEmail), + ) + } else { + email = strings.TrimSpace(email) + intg.ApiKey[GoogleCalendarEmail] = email + if email == "" { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarEmail), + fmt.Sprintf("%s cannot be blank", GoogleCalendarEmail), + ) + } } - intg.PrivateKey = strings.TrimSpace(intg.PrivateKey) - if intg.PrivateKey == "" || intg.PrivateKey == MaskedPassword { - invalid.Append("integrations.google_calendar.private_key", "private_key is required") + if privateKey, ok := intg.ApiKey["private_key"]; !ok { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarPrivateKey), + fmt.Sprintf("%s is required", GoogleCalendarPrivateKey), + ) + } else { + privateKey = strings.TrimSpace(privateKey) + intg.ApiKey[GoogleCalendarPrivateKey] = privateKey + if privateKey == "" { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarPrivateKey), + fmt.Sprintf("%s cannot be blank", GoogleCalendarPrivateKey), + ) + } } intg.Domain = strings.TrimSpace(intg.Domain) if intg.Domain == "" { diff --git a/server/service/appconfig_test.go b/server/service/appconfig_test.go index e23f3b18bd..8881b19651 100644 --- a/server/service/appconfig_test.go +++ b/server/service/appconfig_test.go @@ -490,7 +490,7 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { {APIToken: "zendesktoken"}, }, GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - {PrivateKey: "google-calendar-private-key"}, + {ApiKey: map[string]string{fleet.GoogleCalendarPrivateKey: "google-calendar-private-key"}}, }, }, }, nil @@ -569,7 +569,8 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { require.Equal(t, ac.SMTPSettings.SMTPPassword, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Jira[0].APIToken, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Zendesk[0].APIToken, fleet.MaskedPassword) - require.Equal(t, ac.Integrations.GoogleCalendar[0].PrivateKey, fleet.MaskedPassword) + // Google Calendar private key is not obfuscated + require.Equal(t, ac.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey], "google-calendar-private-key") } }) } diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index a02913a4e8..b531fd90bb 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -5181,8 +5181,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }] } @@ -5192,8 +5194,8 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { appConfig := s.getConfig() require.Len(t, appConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) - assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarEmail]) + assert.Equal(t, privateKey, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey]) assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) // Add 2nd config -- not allowed at this time @@ -5202,18 +5204,22 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }, { - "email": "bozo@example.com"", - "private_key": "abc", + "api_key_json": { + "client_email": "bozo@example.com", + "private_key": "abc" + }, "domain": "example.com" }] } }`, email, privateKey, domain, - )), http.StatusBadRequest, + )), http.StatusUnprocessableEntity, ) // Make an unrelated config change, should not remove the integrations @@ -5237,8 +5243,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }] } @@ -5247,8 +5255,8 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { ) appConfig = s.getConfig() require.Len(t, appConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) - assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarEmail]) + assert.Equal(t, privateKey, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey]) assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) // Clearing other integrations does not clear Google Calendar integration @@ -5284,7 +5292,9 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, + "api_key_json": { + "client_email": %q + }, "domain": %q }] } @@ -5292,29 +5302,16 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { )), http.StatusUnprocessableEntity, ) - // Try adding Google Calendar integration with masked private key -- not allowed - s.DoRaw( - "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( - `{ - "integrations": { - "google_calendar": [{ - "email": %q, - "private_key": %q, - "domain": %q - }] - } - }`, email, fleet.MaskedPassword, domain, - )), http.StatusUnprocessableEntity, - ) - // Empty email -- not allowed s.DoRaw( "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( `{ "integrations": { "google_calendar": [{ - "email": " ", - "private_key": %q, + "api_key_json": { + "client_email": " ", + "private_key": %q + }, "domain": %q }] } @@ -5328,8 +5325,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": "" }] } @@ -5343,9 +5342,11 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, - "domain": %q + "api_key_json": { + "client_email": %q, + "private_key": %q + }, + "domain": %q, "foo": "bar" }] } @@ -5353,6 +5354,20 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { )), http.StatusBadRequest, ) + // Null api_key_json -- fails validation + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "api_key_json": null, + "domain": %q + }] + } + }`, domain, + )), http.StatusUnprocessableEntity, + ) + } func (s *integrationTestSuite) TestQueriesBadRequests() { diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 22c2779ca0..7466d78d14 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -105,8 +105,10 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": "testKey", + "api_key_json": { + "client_email": %q, + "private_key": "testKey" + }, "domain": "example.com" }] } @@ -197,7 +199,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { "name": teamName, "integrations": map[string]any{ "google_calendar": map[string]any{ - "email": calendarEmail, "enable_calendar_events": true, "webhook_url": calendarWebhookUrl, }, @@ -210,7 +211,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { team, err = s.ds.TeamByName(context.Background(), teamName) require.NotNil(t, team.Config.Integrations.GoogleCalendar) - assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) @@ -1026,7 +1026,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamEndpoints() { modifyCalendar := fleet.TeamPayload{ Integrations: &fleet.TeamIntegrations{ GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ - Email: "calendar@example.com", + WebhookURL: "https://example.com/modified", }, }, } diff --git a/tools/cloner-check/generated_files/appconfig.txt b/tools/cloner-check/generated_files/appconfig.txt index 7deb8d10fc..9a03bf15b6 100644 --- a/tools/cloner-check/generated_files/appconfig.txt +++ b/tools/cloner-check/generated_files/appconfig.txt @@ -85,9 +85,8 @@ github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration GroupID int64 github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableFailingPolicies bool github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableSoftwareVulnerabilities bool github.com/fleetdm/fleet/v4/server/fleet/Integrations GoogleCalendar []*fleet.GoogleCalendarIntegration -github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Email string -github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration PrivateKey string github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Domain string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration ApiKey map[string]string github.com/fleetdm/fleet/v4/server/fleet/AppConfig MDM fleet.MDM github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMDefaultTeam string github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMEnabledAndConfigured bool From d97e32fc219775c5450b6bc084d12098c4ce234f Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 19:58:48 -0500 Subject: [PATCH 12/30] Fix compile issue due to merge. --- ee/server/calendar/google_calendar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 9c0c14707f..26d1ba1e66 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -50,7 +50,7 @@ type GoogleCalendar struct { func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { if config.API == nil { - if config.IntegrationConfig.Email == "calendar-mock@example.com" { + if config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == "calendar-mock@example.com" { // Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory config.API = &GoogleCalendarMockAPI{} } else { From 4db06f2cbb1734e3da79f9253babb97e9fc4e49a Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:07:27 -0400 Subject: [PATCH 13/30] Fleet Calendar feature: Updates to manage automations modal (#17652) --- .../ManagePoliciesPage/ManagePoliciesPage.tsx | 11 +- .../ExamplePayload/ExamplePayload.tsx | 64 +++++++ .../components/ExamplePayload/_styles.scss | 9 + .../components/ExamplePayload/index.ts | 1 + .../ExampleTicket.tsx} | 42 +---- .../components/ExampleTicket/_styles.scss | 10 ++ .../components/ExampleTicket/index.ts | 1 + .../ManagePolicyAutomationsModal/index.ts | 1 - .../OtherWorkflowsModal.tsx} | 161 +++++++++--------- .../_styles.scss | 9 +- .../components/OtherWorkflowsModal/index.ts | 1 + .../PreviewPayloadModal.tsx | 97 ----------- .../PreviewPayloadModal/_styles.scss | 54 ------ .../components/PreviewPayloadModal/index.ts | 1 - .../PreviewTicketModal/_styles.scss | 13 -- .../components/PreviewTicketModal/index.ts | 1 - 16 files changed, 185 insertions(+), 291 deletions(-) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts rename frontend/pages/policies/ManagePoliciesPage/components/{PreviewTicketModal/PreviewTicketModal.tsx => ExampleTicket/ExampleTicket.tsx} (52%) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts rename frontend/pages/policies/ManagePoliciesPage/components/{ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx => OtherWorkflowsModal/OtherWorkflowsModal.tsx} (82%) rename frontend/pages/policies/ManagePoliciesPage/components/{ManagePolicyAutomationsModal => OtherWorkflowsModal}/_styles.scss (69%) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/PreviewPayloadModal.tsx delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index be95e222d7..ce359b5778 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -41,7 +41,7 @@ import TableDataError from "components/DataError"; import MainContent from "components/MainContent"; import PoliciesTable from "./components/PoliciesTable"; -import ManagePolicyAutomationsModal from "./components/ManagePolicyAutomationsModal"; +import OtherWorkflowsModal from "./components/OtherWorkflowsModal"; import AddPolicyModal from "./components/AddPolicyModal"; import DeletePolicyModal from "./components/DeletePolicyModal"; @@ -129,7 +129,6 @@ const ManagePolicyPage = ({ const [showManageAutomationsModal, setShowManageAutomationsModal] = useState( false ); - const [showPreviewPayloadModal, setShowPreviewPayloadModal] = useState(false); const [showAddPolicyModal, setShowAddPolicyModal] = useState(false); const [showDeletePolicyModal, setShowDeletePolicyModal] = useState(false); @@ -477,10 +476,6 @@ const ManagePolicyPage = ({ const toggleManageAutomationsModal = () => setShowManageAutomationsModal(!showManageAutomationsModal); - const togglePreviewPayloadModal = useCallback(() => { - setShowPreviewPayloadModal(!showPreviewPayloadModal); - }, [setShowPreviewPayloadModal, showPreviewPayloadModal]); - const toggleAddPolicyModal = () => setShowAddPolicyModal(!showAddPolicyModal); const toggleDeletePolicyModal = () => @@ -796,15 +791,13 @@ const ManagePolicyPage = ({ )} {config && automationsConfig && showManageAutomationsModal && ( - )} {showAddPolicyModal && ( diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx new file mode 100644 index 0000000000..4dc1c2462c --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx @@ -0,0 +1,64 @@ +import React, { useContext } from "react"; +import { syntaxHighlight } from "utilities/helpers"; + +import { AppContext } from "context/app"; +import { IPolicyWebhookPreviewPayload } from "interfaces/policy"; + +const baseClass = "example-payload"; + +interface IHostPreview { + id: number; + display_name: string; + url: string; +} + +interface IExamplePayload { + timestamp: string; + policy: IPolicyWebhookPreviewPayload; + hosts: IHostPreview[]; +} + +const ExamplePayload = (): JSX.Element => { + const { isFreeTier } = useContext(AppContext); + + const json: IExamplePayload = { + timestamp: "0000-00-00T00:00:00Z", + policy: { + id: 1, + name: "Is Gatekeeper enabled?", + query: "SELECT 1 FROM gatekeeper WHERE assessments_enabled = 1;", + description: "Checks if gatekeeper is enabled on macOS devices.", + author_id: 1, + author_name: "John", + author_email: "john@example.com", + resolution: "Turn on Gatekeeper feature in System Preferences.", + passing_host_count: 2000, + failing_host_count: 300, + critical: false, + }, + hosts: [ + { + id: 1, + display_name: "macbook-1", + url: "https://fleet.example.com/hosts/1", + }, + { + id: 2, + display_name: "macbbook-2", + url: "https://fleet.example.com/hosts/2", + }, + ], + }; + if (isFreeTier) { + delete json.policy.critical; + } + + return ( +
+ 
POST https://server.com/example
+ 
+
+ ); +}; + +export default ExamplePayload; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss new file mode 100644 index 0000000000..2297445b92 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss @@ -0,0 +1,9 @@ +.example-payload { + display: flex; + flex-direction: column; + gap: $pad-large; + + pre { + margin: 0; + } +} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts new file mode 100644 index 0000000000..a9ab7d050d --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts @@ -0,0 +1 @@ +export { default } from "./ExamplePayload"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx similarity index 52% rename from frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx rename to frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx index c46d2e4e54..1ff58279f6 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx @@ -1,28 +1,24 @@ import React, { useContext } from "react"; import { AppContext } from "context/app"; -import Modal from "components/Modal"; -import Button from "components/buttons/Button"; -import CustomLink from "components/CustomLink"; import { IIntegrationType } from "interfaces/integration"; +import Card from "components/Card"; import JiraPreview from "../../../../../../assets/images/jira-policy-automation-preview-400x419@2x.png"; import ZendeskPreview from "../../../../../../assets/images/zendesk-policy-automation-preview-400x515@2x.png"; import JiraPreviewPremium from "../../../../../../assets/images/jira-policy-automation-preview-premium-400x316@2x.png"; import ZendeskPreviewPremium from "../../../../../../assets/images/zendesk-policy-automation-preview-premium-400x483@2x.png"; -const baseClass = "preview-ticket-modal"; +const baseClass = "example-ticket"; -interface IPreviewTicketModalProps { +interface IExampleTicketProps { integrationType?: IIntegrationType; - onCancel: () => void; } -const PreviewTicketModal = ({ +const ExampleTicket = ({ integrationType, - onCancel, -}: IPreviewTicketModalProps): JSX.Element => { +}: IExampleTicketProps): JSX.Element => { const { isPremiumTier } = useContext(AppContext); const screenshot = @@ -41,30 +37,10 @@ const PreviewTicketModal = ({ ); return ( - -
-

- Want to learn more about how automations in Fleet work?{" "} - -

-
{screenshot}
-
- -
-
- + + {screenshot} + ); }; -export default PreviewTicketModal; +export default ExampleTicket; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss new file mode 100644 index 0000000000..4212f33fa5 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss @@ -0,0 +1,10 @@ +.example-ticket { + display: flex; + flex-direction: column; + align-items: center; + box-sizing: border-box; + + &__screenshot { + max-width: 400px; + } +} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts new file mode 100644 index 0000000000..3557097088 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts @@ -0,0 +1 @@ +export { default } from "./ExampleTicket"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts deleted file mode 100644 index d8e2cefbc9..0000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./ManagePolicyAutomationsModal"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx similarity index 82% rename from frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx rename to frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx index ae71c22cfe..907f4edb4f 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx @@ -19,22 +19,21 @@ import Dropdown from "components/forms/fields/Dropdown"; import InputField from "components/forms/fields/InputField"; import Radio from "components/forms/fields/Radio"; import validUrl from "components/forms/validators/valid_url"; +import RevealButton from "components/buttons/RevealButton"; +import CustomLink from "components/CustomLink"; +import ExampleTicket from "../ExampleTicket"; +import ExamplePayload from "../ExamplePayload"; -import PreviewPayloadModal from "../PreviewPayloadModal"; -import PreviewTicketModal from "../PreviewTicketModal"; - -interface IManagePolicyAutomationsModalProps { +interface IOtherWorkflowsModalProps { automationsConfig: IAutomationsConfig | ITeamAutomationsConfig; availableIntegrations: IIntegrations; availablePolicies: IPolicy[]; isUpdatingAutomations: boolean; - showPreviewPayloadModal: boolean; onExit: () => void; handleSubmit: (formData: { webhook_settings: Pick; integrations: IIntegrations; }) => void; - togglePreviewPayloadModal: () => void; } interface ICheckedPolicy { @@ -83,18 +82,16 @@ const useCheckboxListStateManagement = ( return { policyItems, updatePolicyItems }; }; -const baseClass = "manage-policy-automations-modal"; +const baseClass = "other-workflows-modal"; -const ManagePolicyAutomationsModal = ({ +const OtherWorkflowsModal = ({ automationsConfig, availableIntegrations, availablePolicies, isUpdatingAutomations, - showPreviewPayloadModal, onExit, handleSubmit, - togglePreviewPayloadModal: togglePreviewModal, -}: IManagePolicyAutomationsModalProps): JSX.Element => { +}: IOtherWorkflowsModalProps): JSX.Element => { const { webhook_settings: { failing_policies_webhook: webhook }, } = automationsConfig; @@ -131,6 +128,9 @@ const ManagePolicyAutomationsModal = ({ IIntegration | undefined >(serverEnabledIntegration); + const [showExamplePayload, setShowExamplePayload] = useState(false); + const [showExampleTicket, setShowExampleTicket] = useState(false); + const [errors, setErrors] = useState<{ [key: string]: string }>({}); const { policyItems, updatePolicyItems } = useCheckboxListStateManagement( @@ -218,13 +218,6 @@ const ManagePolicyAutomationsModal = ({ z.group_id === selectedIntegration?.group_id, })) || null; - // if ( - // !isPolicyAutomationsEnabled || - // (!isWebhookEnabled && !selectedIntegration) - // ) { - // newPolicyIds = []; - // } - const updatedEnabledPoliciesAcrossPages = () => { if (webhook.policy_ids) { // Array of policy ids on the page @@ -297,34 +290,52 @@ const ManagePolicyAutomationsModal = ({ placeholder="https://server.com/example" tooltip="Provide a URL to deliver a webhook request to." /> - + setShowExamplePayload(!showExamplePayload)} + /> + {showExamplePayload && } ); }; const renderIntegrations = () => { return jira?.length || zendesk?.length ? ( -
- +
+ +
+ setShowExampleTicket(!showExampleTicket)} /> - -
+ {showExampleTicket && ( + + )} + ) : (
You have no integrations.
@@ -338,22 +349,10 @@ const ManagePolicyAutomationsModal = ({ ); }; - const renderPreview = () => - !isWebhookEnabled ? ( - - ) : ( - - ); - - return showPreviewPayloadModal ? ( - renderPreview() - ) : ( + return ( @@ -372,12 +371,32 @@ const ManagePolicyAutomationsModal = ({ isPolicyAutomationsEnabled ? "enabled" : "disabled" }`} > +
+
Workflow
+ + +
+ {isWebhookEnabled ? renderWebhook() : renderIntegrations()}
{availablePolicies?.length ? ( <> -
- Choose which policies you would like to listen to: -
+
Policies:
{policyItems && policyItems.map((policyItem) => { const { isChecked, name, id } = policyItem; @@ -405,28 +424,14 @@ const ManagePolicyAutomationsModal = ({ )}
-
-
Workflow
- + The workflow will be triggered when hosts fail these policies.{" "} + - -
- {isWebhookEnabled ? renderWebhook() : renderIntegrations()} +

-
- - - ); -}; - -export default PreviewPayloadModal; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss deleted file mode 100644 index 0ff66f034f..0000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss +++ /dev/null @@ -1,54 +0,0 @@ -.preview-payload-modal { - &__sandbox-info { - margin-top: $pad-medium; - - p { - margin: 0; - margin-bottom: $pad-medium; - } - - p:last-child { - margin-bottom: 0; - } - } - - &__info-header { - font-weight: $bold; - } - - &__advanced-options-button { - margin: $pad-medium 0; - color: $core-vibrant-blue; - font-weight: $bold; - font-size: $x-small; - } - - .downcaret { - &::after { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - transform: scale(0.5); - width: 16px; - border-radius: 0px; - padding: 0px; - padding-left: 2px; - margin-bottom: 2px; - } - } - - .upcaret { - &::after { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - transform: scale(0.5) rotate(180deg); - width: 16px; - border-radius: 0px; - padding: 0px; - padding-left: 2px; - margin-bottom: 4px; - margin-left: 14px; - } - } - - .Select-value-label { - font-size: $small; - } -} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts deleted file mode 100644 index bc08b4723d..0000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./PreviewPayloadModal"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss deleted file mode 100644 index 9024a73203..0000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss +++ /dev/null @@ -1,13 +0,0 @@ -.preview-ticket-modal { - &__example { - display: flex; - justify-content: center; - } - - &__screenshot { - width: 400px; - height: auto; - border-radius: 8px; - filter: drop-shadow(0px 4px 16px rgba(0, 0, 0, 0.1)); - } -} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts deleted file mode 100644 index 4d8716d447..0000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./PreviewTicketModal"; From 5137fe380c07c3ed89c5e1b0f3b6f01b54b27490 Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:53:34 -0700 Subject: [PATCH 14/30] 17445 calendar events modal (#17717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses #17445 Follow-up iteration: - Finalize styling of dropdown tooltips - All `//TODO`s Screenshot 2024-03-20 at 1 43 54 PM Screenshot 2024-03-20 at 1 44 01 PM Screenshot 2024-03-20 at 1 44 21 PM Screenshot 2024-03-20 at 1 44 26 PM Screenshot 2024-03-20 at 1 44 33 PM Screenshot 2024-03-20 at 1 44 43 PM Screenshot 2024-03-20 at 1 44 47 PM Screenshot 2024-03-20 at 1 45 02 PM Screenshot 2024-03-20 at 1 45 10 PM --------- Co-authored-by: Jacob Shandling --- frontend/__mocks__/configMock.ts | 1 + frontend/__mocks__/policyMock.ts | 1 + .../components/forms/FormField/FormField.tsx | 8 +- .../forms/fields/InputField/InputField.jsx | 3 + .../components/forms/fields/Slider/Slider.tsx | 5 +- .../graphics/CalendarEventPreview.tsx | 1184 +++++++++++++++++ frontend/components/graphics/index.ts | 2 + .../hooks/useCheckboxListStateManagement.tsx | 36 + frontend/interfaces/config.ts | 4 +- frontend/interfaces/integration.ts | 24 + frontend/interfaces/policy.ts | 2 + frontend/interfaces/team.ts | 4 +- .../HostActionsDropdown/_styles.scss | 105 +- .../HostDetailsPage/HostDetailsPage.tsx | 1 - .../ManagePoliciesPage/ManagePoliciesPage.tsx | 181 ++- .../policies/ManagePoliciesPage/_styles.scss | 28 +- .../CalendarEventsModal.tsx | 318 +++++ .../CalendarEventsModal/_styles.scss | 35 + .../components/CalendarEventsModal/index.ts | 1 + .../PoliciesTable/PoliciesTableConfig.tsx | 10 - frontend/services/entities/team_policies.ts | 2 + frontend/services/entities/teams.ts | 7 +- frontend/styles/var/mixins.scss | 100 ++ 23 files changed, 1917 insertions(+), 145 deletions(-) create mode 100644 frontend/components/graphics/CalendarEventPreview.tsx create mode 100644 frontend/hooks/useCheckboxListStateManagement.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts diff --git a/frontend/__mocks__/configMock.ts b/frontend/__mocks__/configMock.ts index 26b996662a..03e5554851 100644 --- a/frontend/__mocks__/configMock.ts +++ b/frontend/__mocks__/configMock.ts @@ -76,6 +76,7 @@ const DEFAULT_CONFIG_MOCK: IConfig = { integrations: { jira: [], zendesk: [], + google_calendar: [], }, logging: { debug: false, diff --git a/frontend/__mocks__/policyMock.ts b/frontend/__mocks__/policyMock.ts index 048dd6d496..c66c58a0bc 100644 --- a/frontend/__mocks__/policyMock.ts +++ b/frontend/__mocks__/policyMock.ts @@ -22,6 +22,7 @@ const DEFAULT_POLICY_MOCK: IPolicyStats = { webhook: "Off", has_run: true, next_update_ms: 3600000, + calendar_events_enabled: true, }; const createMockPolicy = (overrides?: Partial): IPolicyStats => { diff --git a/frontend/components/forms/FormField/FormField.tsx b/frontend/components/forms/FormField/FormField.tsx index 68d07507a7..80bf5833a2 100644 --- a/frontend/components/forms/FormField/FormField.tsx +++ b/frontend/components/forms/FormField/FormField.tsx @@ -3,6 +3,7 @@ import classnames from "classnames"; import { isEmpty } from "lodash"; import TooltipWrapper from "components/TooltipWrapper"; +import { PlacesType } from "react-tooltip-5"; // all form-field styles are defined in _global.scss, which apply here and elsewhere const baseClass = "form-field"; @@ -16,6 +17,7 @@ export interface IFormFieldProps { name: string; type: string; tooltip?: React.ReactNode; + labelTooltipPosition?: PlacesType; } const FormField = ({ @@ -27,6 +29,7 @@ const FormField = ({ name, type, tooltip, + labelTooltipPosition, }: IFormFieldProps): JSX.Element => { const renderLabel = () => { const labelWrapperClasses = classnames(`${baseClass}__label`, { @@ -45,7 +48,10 @@ const FormField = ({ > {error || (tooltip ? ( - + {label as string} ) : ( diff --git a/frontend/components/forms/fields/InputField/InputField.jsx b/frontend/components/forms/fields/InputField/InputField.jsx index 83c892eb16..72969c256b 100644 --- a/frontend/components/forms/fields/InputField/InputField.jsx +++ b/frontend/components/forms/fields/InputField/InputField.jsx @@ -33,6 +33,7 @@ class InputField extends Component { ]).isRequired, parseTarget: PropTypes.bool, tooltip: PropTypes.string, + labelTooltipPosition: PropTypes.string, helpText: PropTypes.oneOfType([ PropTypes.string, PropTypes.arrayOf(PropTypes.string), @@ -55,6 +56,7 @@ class InputField extends Component { value: "", parseTarget: false, tooltip: "", + labelTooltipPosition: "", helpText: "", enableCopy: false, ignore1password: false, @@ -124,6 +126,7 @@ class InputField extends Component { "error", "name", "tooltip", + "labelTooltipPosition", ]); const copyValue = (e) => { diff --git a/frontend/components/forms/fields/Slider/Slider.tsx b/frontend/components/forms/fields/Slider/Slider.tsx index 2b368275f1..db21fe6c9a 100644 --- a/frontend/components/forms/fields/Slider/Slider.tsx +++ b/frontend/components/forms/fields/Slider/Slider.tsx @@ -6,7 +6,10 @@ import FormField from "components/forms/FormField"; import { IFormFieldProps } from "components/forms/FormField/FormField"; interface ISliderProps { - onChange: () => void; + onChange: (newValue?: { + name: string; + value: string | number | boolean; + }) => void; value: boolean; inactiveText: string; activeText: string; diff --git a/frontend/components/graphics/CalendarEventPreview.tsx b/frontend/components/graphics/CalendarEventPreview.tsx new file mode 100644 index 0000000000..4c29770a68 --- /dev/null +++ b/frontend/components/graphics/CalendarEventPreview.tsx @@ -0,0 +1,1184 @@ +import React from "react"; + +const CalendarEventPreview = () => { + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 💻 🚫  + + + + + + + + + + + + + + + + + 💻 🚫  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export default CalendarEventPreview; diff --git a/frontend/components/graphics/index.ts b/frontend/components/graphics/index.ts index 84e10a3711..fb3b0c5fd4 100644 --- a/frontend/components/graphics/index.ts +++ b/frontend/components/graphics/index.ts @@ -17,6 +17,7 @@ import EmptyTeams from "./EmptyTeams"; import EmptyPacks from "./EmptyPacks"; import EmptySchedule from "./EmptySchedule"; import CollectingResults from "./CollectingResults"; +import CalendarEventPreview from "./CalendarEventPreview"; export const GRAPHIC_MAP = { // Empty state graphics @@ -41,6 +42,7 @@ export const GRAPHIC_MAP = { "file-pem": FilePem, // Other graphics "collecting-results": CollectingResults, + "calendar-event-preview": CalendarEventPreview, }; export type GraphicNames = keyof typeof GRAPHIC_MAP; diff --git a/frontend/hooks/useCheckboxListStateManagement.tsx b/frontend/hooks/useCheckboxListStateManagement.tsx new file mode 100644 index 0000000000..4c1cf9d88d --- /dev/null +++ b/frontend/hooks/useCheckboxListStateManagement.tsx @@ -0,0 +1,36 @@ +import { useState } from "react"; + +import { IPolicy } from "interfaces/policy"; + +interface ICheckedPolicy { + name?: string; + id: number; + isChecked: boolean; +} + +const useCheckboxListStateManagement = ( + allPolicies: IPolicy[], + automatedPolicies: number[] | undefined +) => { + const [policyItems, setPolicyItems] = useState(() => { + return allPolicies.map(({ name, id }) => ({ + name, + id, + isChecked: !!automatedPolicies?.includes(id), + })); + }); + + const updatePolicyItems = (policyId: number) => { + setPolicyItems((prevItems) => + prevItems.map((policy) => + policy.id !== policyId + ? policy + : { ...policy, isChecked: !policy.isChecked } + ) + ); + }; + + return { policyItems, updatePolicyItems }; +}; + +export default useCheckboxListStateManagement; diff --git a/frontend/interfaces/config.ts b/frontend/interfaces/config.ts index 1df44de33d..8eee167f12 100644 --- a/frontend/interfaces/config.ts +++ b/frontend/interfaces/config.ts @@ -4,7 +4,7 @@ import { IWebhookFailingPolicies, IWebhookSoftwareVulnerabilities, } from "interfaces/webhook"; -import { IIntegrations } from "./integration"; +import { IGlobalIntegrations } from "./integration"; export interface ILicense { tier: string; @@ -175,7 +175,7 @@ export interface IConfig { // databases_path: string; // }; webhook_settings: IWebhookSettings; - integrations: IIntegrations; + integrations: IGlobalIntegrations; logging: { debug: boolean; json: boolean; diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts index adcbeeb7e7..49156d6277 100644 --- a/frontend/interfaces/integration.ts +++ b/frontend/interfaces/integration.ts @@ -60,7 +60,31 @@ export interface IIntegrationFormErrors { enableSoftwareVulnerabilities?: boolean; } +export interface IGlobalCalendarIntegration { + email: string; + private_key: string; + domain: string; +} + +interface ITeamCalendarSettings { + enable_calendar_events: boolean; + webhook_url: string; +} + +// zendesk and jira fields are coupled – if one is present, the other needs to be present. If +// one is present and the other is null/missing, the other will be nullified. google_calendar is +// separated – it can be present without the other 2 without nullifying them. +// TODO: Update these types to reflect this. + export interface IIntegrations { zendesk: IZendeskIntegration[]; jira: IJiraIntegration[]; } + +export interface IGlobalIntegrations extends IIntegrations { + google_calendar?: IGlobalCalendarIntegration[] | null; +} + +export interface ITeamIntegrations extends IIntegrations { + google_calendar?: ITeamCalendarSettings | null; +} diff --git a/frontend/interfaces/policy.ts b/frontend/interfaces/policy.ts index 4858de5f37..056ab70406 100644 --- a/frontend/interfaces/policy.ts +++ b/frontend/interfaces/policy.ts @@ -40,6 +40,7 @@ export interface IPolicy { created_at: string; updated_at: string; critical: boolean; + calendar_events_enabled: boolean; } // Used on the manage hosts page and other places where aggregate stats are displayed @@ -90,6 +91,7 @@ export interface IPolicyFormData { query?: string | number | boolean | undefined; team_id?: number; id?: number; + calendar_events_enabled?: boolean; } export interface IPolicyNew { diff --git a/frontend/interfaces/team.ts b/frontend/interfaces/team.ts index 435075902a..8fa4726022 100644 --- a/frontend/interfaces/team.ts +++ b/frontend/interfaces/team.ts @@ -1,7 +1,7 @@ import PropTypes from "prop-types"; import { IConfigFeatures, IWebhookSettings } from "./config"; import enrollSecretInterface, { IEnrollSecret } from "./enroll_secret"; -import { IIntegrations } from "./integration"; +import { ITeamIntegrations } from "./integration"; import { UserRole } from "./user"; export default PropTypes.shape({ @@ -82,7 +82,7 @@ export type ITeamWebhookSettings = Pick< */ export interface ITeamAutomationsConfig { webhook_settings: ITeamWebhookSettings; - integrations: IIntegrations; + integrations: ITeamIntegrations; } /** diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss index 1152dcfb5f..06bd48a653 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss +++ b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss @@ -1,104 +1,9 @@ .host-actions-dropdown { - .form-field { - margin: 0; + @include button-dropdown; + .Select-multi-value-wrapper { + width: 55px; } - - .Select { - position: relative; - border: 0; - height: auto; - - &.is-focused, - &:hover { - border: 0; - } - - &.is-focused:not(.is-open) { - .Select-control { - background-color: initial; - } - } - - .Select-control { - display: flex; - background-color: initial; - height: auto; - justify-content: space-between; - border: 0; - cursor: pointer; - - &:hover { - box-shadow: none; - } - - &:hover .Select-placeholder { - color: $core-vibrant-blue; - } - - .Select-placeholder { - color: $core-fleet-black; - font-size: 14px; - line-height: normal; - padding-left: 0; - margin-top: 1px; - } - - .Select-input { - height: auto; - } - - .Select-arrow-zone { - display: flex; - } - } - - .Select-multi-value-wrapper { - width: 55px; - } - - .Select-placeholder { - display: flex; - align-items: center; - } - - .Select-menu-outer { - margin-top: $pad-xsmall; - box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); - border-radius: $border-radius; - z-index: 6; - overflow: hidden; - border: 0; - width: 188px; - left: unset; - top: unset; - max-height: none; - padding: $pad-small; - position: absolute; - left: -120px; - - .Select-menu { - max-height: none; - } - } - - .Select-arrow { - transition: transform 0.25s ease; - } - - &:not(.is-open) { - .Select-control:hover .Select-arrow { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - } - } - - &.is-open { - .Select-control .Select-placeholder { - color: $core-vibrant-blue; - } - - .Select-arrow { - transform: rotate(180deg); - } - } + .Select > .Select-menu-outer { + left: -120px; } } diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 99faef93dd..a117974fa2 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -13,7 +13,6 @@ import { QueryContext } from "context/query"; import { NotificationContext } from "context/notification"; import activitiesAPI, { - IActivitiesResponse, IPastActivitiesResponse, IUpcomingActivitiesResponse, } from "services/entities/activities"; diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index ce359b5778..80a832026b 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -2,7 +2,9 @@ import React, { useCallback, useContext, useEffect, useState } from "react"; import { useQuery } from "react-query"; import { InjectedRouter } from "react-router/lib/Router"; import PATHS from "router/paths"; -import { noop, isEqual } from "lodash"; +import { noop, isEqual, uniqueId } from "lodash"; + +import { Tooltip as ReactTooltip5 } from "react-tooltip-5"; import { getNextLocationPath } from "utilities/helpers"; @@ -34,6 +36,8 @@ import teamsAPI, { ILoadTeamResponse } from "services/entities/teams"; import { ITableQueryData } from "components/TableContainer/TableContainer"; import Button from "components/buttons/Button"; +// @ts-ignore +import Dropdown from "components/forms/fields/Dropdown"; import RevealButton from "components/buttons/RevealButton"; import Spinner from "components/Spinner"; import TeamsDropdown from "components/TeamsDropdown"; @@ -44,6 +48,8 @@ import PoliciesTable from "./components/PoliciesTable"; import OtherWorkflowsModal from "./components/OtherWorkflowsModal"; import AddPolicyModal from "./components/AddPolicyModal"; import DeletePolicyModal from "./components/DeletePolicyModal"; +import CalendarEventsModal from "./components/CalendarEventsModal"; +import { ICalendarEventsFormData } from "./components/CalendarEventsModal/CalendarEventsModal"; interface IManagePoliciesPageProps { router: InjectedRouter; @@ -125,12 +131,15 @@ const ManagePolicyPage = ({ const [isUpdatingAutomations, setIsUpdatingAutomations] = useState(false); const [isUpdatingPolicies, setIsUpdatingPolicies] = useState(false); + const [ + updatingPolicyEnabledCalendarEvents, + setUpdatingPolicyEnabledCalendarEvents, + ] = useState(false); const [selectedPolicyIds, setSelectedPolicyIds] = useState([]); - const [showManageAutomationsModal, setShowManageAutomationsModal] = useState( - false - ); + const [showOtherWorkflowsModal, setShowOtherWorkflowsModal] = useState(false); const [showAddPolicyModal, setShowAddPolicyModal] = useState(false); const [showDeletePolicyModal, setShowDeletePolicyModal] = useState(false); + const [showCalendarEventsModal, setShowCalendarEventsModal] = useState(false); const [teamPolicies, setTeamPolicies] = useState(); const [inheritedPolicies, setInheritedPolicies] = useState(); @@ -473,14 +482,30 @@ const ManagePolicyPage = ({ ] // Other dependencies can cause infinite re-renders as URL is source of truth ); - const toggleManageAutomationsModal = () => - setShowManageAutomationsModal(!showManageAutomationsModal); + const toggleOtherWorkflowsModal = () => + setShowOtherWorkflowsModal(!showOtherWorkflowsModal); const toggleAddPolicyModal = () => setShowAddPolicyModal(!showAddPolicyModal); const toggleDeletePolicyModal = () => setShowDeletePolicyModal(!showDeletePolicyModal); + const toggleCalendarEventsModal = () => { + setShowCalendarEventsModal(!showCalendarEventsModal); + }; + + const onSelectAutomationOption = (option: string) => { + switch (option) { + case "calendar_events": + toggleCalendarEventsModal(); + break; + case "other_workflows": + toggleOtherWorkflowsModal(); + break; + default: + } + }; + const toggleShowInheritedPolicies = () => { // URL source of truth const locationPath = getNextLocationPath({ @@ -496,6 +521,7 @@ const ManagePolicyPage = ({ const handleUpdateAutomations = async (requestBody: { webhook_settings: Pick; + // TODO - update below type to specify team integration integrations: IIntegrations; }) => { setIsUpdatingAutomations(true); @@ -510,13 +536,59 @@ const ManagePolicyPage = ({ "Could not update policy automations. Please try again." ); } finally { - toggleManageAutomationsModal(); + toggleOtherWorkflowsModal(); setIsUpdatingAutomations(false); refetchConfig(); isAnyTeamSelected && refetchTeamConfig(); } }; + const updatePolicyEnabledCalendarEvents = async ( + formData: ICalendarEventsFormData + ) => { + setUpdatingPolicyEnabledCalendarEvents(true); + + try { + // update enabled and URL in config + const configResponse = teamsAPI.update( + { + integrations: { + google_calendar: { + enable_calendar_events: formData.enabled, + webhook_url: formData.url, + }, + // TODO - can omit these? + zendesk: teamConfig?.integrations.zendesk || [], + jira: teamConfig?.integrations.jira || [], + }, + }, + teamIdForApi + ); + + // update policies calendar events enabled + // TODO - only update changed policies + const policyResponses = formData.policies.map((formPolicy) => + teamPoliciesAPI.update(formPolicy.id, { + calendar_events_enabled: formPolicy.isChecked, + team_id: teamIdForApi, + }) + ); + + await Promise.all([configResponse, ...policyResponses]); + renderFlash("success", "Successfully updated policy automations."); + } catch { + renderFlash( + "error", + "Could not update policy automations. Please try again." + ); + } finally { + toggleCalendarEventsModal(); + setUpdatingPolicyEnabledCalendarEvents(false); + refetchTeamPolicies(); + refetchTeamConfig(); + } + }; + const onAddPolicyClick = () => { setLastEditedQueryName(""); setLastEditedQueryDescription(""); @@ -682,6 +754,60 @@ const ManagePolicyPage = ({ ); }; + const getAutomationsDropdownOptions = () => { + const isAllTeams = teamIdForApi === undefined || teamIdForApi === -1; + let calEventsLabel: React.ReactNode = "Calendar events"; + if (!isPremiumTier) { + const tipId = uniqueId(); + calEventsLabel = ( + +
Calendar events
+ + Available in Fleet Premium + + + ); + } else if (isAllTeams) { + const tipId = uniqueId(); + calEventsLabel = ( + +
Calendar events
+ + Select a team to manage +
+ calendar events. + + + ); + } + + return [ + { + label: calEventsLabel, + value: "calendar_events", + disabled: !isPremiumTier || isAllTeams, + helpText: "Automatically reserve time to resolve failing policies.", + }, + { + label: "Other workflows", + value: "other_workflows", + disabled: false, + helpText: "Create tickets or fire webhooks for failing policies.", + }, + ]; + }; + + const isCalEventsConfigured = + (config?.integrations.google_calendar && + config?.integrations.google_calendar.length > 0) ?? + false; + return (
@@ -709,18 +835,15 @@ const ManagePolicyPage = ({ {showCtaButtons && (
{canManageAutomations && automationsConfig && ( - +
+ +
)} {canAddOrDeletePolicy && (
@@ -790,13 +913,13 @@ const ManagePolicyPage = ({ )}
)} - {config && automationsConfig && showManageAutomationsModal && ( + {config && automationsConfig && showOtherWorkflowsModal && ( )} @@ -815,6 +938,22 @@ const ManagePolicyPage = ({ onSubmit={onDeletePolicySubmit} /> )} + {showCalendarEventsModal && ( + + )}
); diff --git a/frontend/pages/policies/ManagePoliciesPage/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/_styles.scss index 62c29c1d5b..ed99ad0137 100644 --- a/frontend/pages/policies/ManagePoliciesPage/_styles.scss +++ b/frontend/pages/policies/ManagePoliciesPage/_styles.scss @@ -8,13 +8,33 @@ .button-wrap { display: flex; justify-content: flex-end; - min-width: 266px; + align-items: center; + gap: 8px; } } - &__manage-automations { - padding: $pad-small; - margin-right: $pad-small; + &__manage-automations-wrapper { + @include button-dropdown; + .Select-multi-value-wrapper { + width: 146px; + } + .Select > .Select-menu-outer { + left: -186px; + width: 360px; + .is-disabled * { + color: $ui-fleet-black-25; + .react-tooltip { + @include tooltip-text; + } + } + } + .Select-control { + margin-top: 0; + gap: 6px; + } + .Select-placeholder { + font-weight: $bold; + } } &__header { diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx new file mode 100644 index 0000000000..eba5abb4e0 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx @@ -0,0 +1,318 @@ +import React, { useCallback, useState } from "react"; + +import { IPolicy } from "interfaces/policy"; + +import validURL from "components/forms/validators/valid_url"; + +import Button from "components/buttons/Button"; +import RevealButton from "components/buttons/RevealButton"; +import CustomLink from "components/CustomLink"; +import Slider from "components/forms/fields/Slider"; +// @ts-ignore +import InputField from "components/forms/fields/InputField"; +import Graphic from "components/Graphic"; +import Modal from "components/Modal"; +import Checkbox from "components/forms/fields/Checkbox"; +import { syntaxHighlight } from "utilities/helpers"; + +const baseClass = "calendar-events-modal"; + +interface IFormPolicy { + name: string; + id: number; + isChecked: boolean; +} +export interface ICalendarEventsFormData { + enabled: boolean; + url: string; + policies: IFormPolicy[]; +} + +interface ICalendarEventsModal { + onExit: () => void; + updatePolicyEnabledCalendarEvents: ( + formData: ICalendarEventsFormData + ) => void; + isUpdating: boolean; + configured: boolean; + enabled: boolean; + url: string; + policies: IPolicy[]; +} + +// allows any policy name to be the name of a form field, one of the checkboxes +type FormNames = string; + +const CalendarEventsModal = ({ + onExit, + updatePolicyEnabledCalendarEvents, + isUpdating, + configured, + enabled, + url, + policies, +}: ICalendarEventsModal) => { + const [formData, setFormData] = useState({ + enabled, + url, + // TODO - stay udpdated on state of backend approach to syncing policies in the policies table + // and in the new calendar table + // id may change if policy was deleted + // name could change if policy was renamed + policies: policies.map((policy) => ({ + name: policy.name, + id: policy.id, + isChecked: policy.calendar_events_enabled || false, + })), + }); + const [formErrors, setFormErrors] = useState>( + {} + ); + const [showPreviewCalendarEvent, setShowPreviewCalendarEvent] = useState( + false + ); + const [showExamplePayload, setShowExamplePayload] = useState(false); + + const validateCalendarEventsFormData = ( + curFormData: ICalendarEventsFormData + ) => { + const errors: Record = {}; + if (curFormData.enabled) { + const { url: curUrl } = curFormData; + if (!validURL({ url: curUrl })) { + const errorPrefix = curUrl ? `${curUrl} is not` : "Please enter"; + errors.url = `${errorPrefix} a valid resolution webhook URL`; + } + } + return errors; + }; + + // TODO - separate change handlers for checkboxes: + // const onPolicyUpdate = ... + // const onTextFieldUpdate = ... + + const onInputChange = useCallback( + (newVal: { name: FormNames; value: string | number | boolean }) => { + const { name, value } = newVal; + let newFormData: ICalendarEventsFormData; + // for the first two fields, set the new value directly + if (["enabled", "url"].includes(name)) { + newFormData = { ...formData, [name]: value }; + } else if (typeof value === "boolean") { + // otherwise, set the value for a nested policy + const newFormPolicies = formData.policies.map((formPolicy) => { + if (formPolicy.name === name) { + return { ...formPolicy, isChecked: value }; + } + return formPolicy; + }); + newFormData = { ...formData, policies: newFormPolicies }; + } else { + throw TypeError("Unexpected value type for policy checkbox"); + } + setFormData(newFormData); + setFormErrors(validateCalendarEventsFormData(newFormData)); + }, + [formData] + ); + + const togglePreviewCalendarEvent = () => { + setShowPreviewCalendarEvent(!showPreviewCalendarEvent); + }; + + const renderExamplePayload = () => { + return ( + <> + 
POST https://server.com/example
+ 
+      
+    );
+  };
+
+  const renderPolicies = () => {
+    return (
+      

+        
Policies:

+        {formData.policies.map((policy) => {
+          const { isChecked, name, id } = policy;
+          return (
+            

+               {
+                  onInputChange({ name, value: !isChecked });
+                }}
+              >
+                {name}
+              
+            

+          );
+        })}
+        
+          A calendar event will be created for end users if one of their hosts
+          fail any of these policies.{" "}
+          
+        
+      

+    );
+  };
+  const renderPreviewCalendarEventModal = () => {
+    return (
+      
+        <>
+          
A similar event will appear in the end user's calendar:

+          
+          

+            
+          

+        
+      
+    );
+  };
+
+  const renderPlaceholderModal = () => {
+    return (
+      

+        
+          
+        
+        

+          To create calendar events for end users if their hosts fail policies,
+          you must first connect Fleet to your Google Workspace service account.
+        

+        

+          This can be configured in{" "}
+          Settings > Integrations > Calendars.
+        

+        
+        

+          
+        

+      

+    );
+  };
+
+  const renderConfiguredModal = () => (
+    

+      

+         {
+            onInputChange({ name: "enabled", value: !formData.enabled });
+          }}
+          inactiveText="Disabled"
+          activeText="Enabled"
+        />
+        
+      

+      

+        
+         {
+            setShowExamplePayload(!showExamplePayload);
+          }}
+        />
+        {showExamplePayload && renderExamplePayload()}
+        {renderPolicies()}
+      

+      

+        
+        
+      

+    

+  );
+
+  if (showPreviewCalendarEvent) {
+    return renderPreviewCalendarEventModal();
+  }
+  return (
+     {
+              updatePolicyEnabledCalendarEvents(formData);
+            }
+          : onExit
+      }
+      className={baseClass}
+      width="large"
+    >
+      {configured ? renderConfiguredModal() : renderPlaceholderModal()}
+    
+  );
+};
+
+export default CalendarEventsModal;
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss
new file mode 100644
index 0000000000..3b1952a8c3
--- /dev/null
+++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss
@@ -0,0 +1,35 @@
+.calendar-events-modal {
+  .placeholder {
+    display: flex;
+    flex-direction: column;
+    gap: 24px;
+    line-height: 150%;
+    .modal-cta-wrap {
+      margin-top: 0;
+    }
+  }
+  .form-header {
+    display: flex;
+    justify-content: space-between;
+    .button--text-link {
+      white-space: nowrap;
+    }
+  }
+
+  .form-fields {
+    &--disabled {
+      @include disabled;
+    }
+  }
+
+  pre {
+    box-sizing: border-box;
+    margin: 0;
+  }
+}
+
+.calendar-event-preview {
+  p {
+    margin: 24px 0;
+  }
+}
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts
new file mode 100644
index 0000000000..b08ecf1063
--- /dev/null
+++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts
@@ -0,0 +1 @@
+export { default } from "./CalendarEventsModal";
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx b/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
index 4fc8f7973d..c2c1c505a9 100644
--- a/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
@@ -284,16 +284,6 @@ const generateTableHeaders = (
   ];
 
   if (tableType !== "inheritedPolicies") {
-    tableHeaders.push({
-      title: "Automations",
-      Header: "Automations",
-      disableSortBy: true,
-      accessor: "webhook",
-      Cell: (cellProps: ICellProps): JSX.Element => (
-        
-      ),
-    });
-
     if (!canAddOrDeletePolicy) {
       return tableHeaders;
     }
diff --git a/frontend/services/entities/team_policies.ts b/frontend/services/entities/team_policies.ts
index d7a03f1f1b..2858938dc7 100644
--- a/frontend/services/entities/team_policies.ts
+++ b/frontend/services/entities/team_policies.ts
@@ -87,6 +87,7 @@ export default {
       resolution,
       platform,
       critical,
+      calendar_events_enabled,
     } = data;
     const { TEAMS } = endpoints;
     const path = `${TEAMS}/${team_id}/policies/${id}`;
@@ -98,6 +99,7 @@ export default {
       resolution,
       platform,
       critical,
+      calendar_events_enabled,
     });
   },
   destroy: (teamId: number | undefined, ids: number[]) => {
diff --git a/frontend/services/entities/teams.ts b/frontend/services/entities/teams.ts
index 3c7e1a2619..1495445827 100644
--- a/frontend/services/entities/teams.ts
+++ b/frontend/services/entities/teams.ts
@@ -5,7 +5,7 @@ import { pick } from "lodash";
 
 import { buildQueryStringFromParams } from "utilities/url";
 import { IEnrollSecret } from "interfaces/enroll_secret";
-import { IIntegrations } from "interfaces/integration";
+import { ITeamIntegrations } from "interfaces/integration";
 import {
   API_NO_TEAM_ID,
   INewTeamUsersBody,
@@ -39,7 +39,7 @@ export interface ITeamFormData {
 export interface IUpdateTeamFormData {
   name: string;
   webhook_settings: Partial;
-  integrations: IIntegrations;
+  integrations: ITeamIntegrations;
   mdm: {
     macos_updates?: {
       minimum_version: string;
@@ -118,7 +118,7 @@ export default {
       requestBody.webhook_settings = webhook_settings;
     }
     if (integrations) {
-      const { jira, zendesk } = integrations;
+      const { jira, zendesk, google_calendar } = integrations;
       const teamIntegrationProps = [
         "enable_failing_policies",
         "group_id",
@@ -128,6 +128,7 @@ export default {
       requestBody.integrations = {
         jira: jira?.map((j) => pick(j, teamIntegrationProps)),
         zendesk: zendesk?.map((z) => pick(z, teamIntegrationProps)),
+        google_calendar,
       };
     }
     if (mdm) {
diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss
index e786ab7696..a6ef009b68 100644
--- a/frontend/styles/var/mixins.scss
+++ b/frontend/styles/var/mixins.scss
@@ -227,3 +227,103 @@ $max-width: 2560px;
   // compensate in layout for extra clickable area button height
   margin: -8px 0;
 }
+
+@mixin button-dropdown {
+  .form-field {
+    margin: 0;
+  }
+
+  .Select {
+    position: relative;
+    border: 0;
+    height: auto;
+
+    &.is-focused,
+    &:hover {
+      border: 0;
+    }
+
+    &.is-focused:not(.is-open) {
+      .Select-control {
+        background-color: initial;
+      }
+    }
+
+    .Select-control {
+      display: flex;
+      background-color: initial;
+      height: auto;
+      justify-content: space-between;
+      border: 0;
+      cursor: pointer;
+
+      &:hover {
+        box-shadow: none;
+      }
+
+      &:hover .Select-placeholder {
+        color: $core-vibrant-blue;
+      }
+
+      .Select-placeholder {
+        color: $core-fleet-black;
+        font-size: 14px;
+        line-height: normal;
+        padding-left: 0;
+        margin-top: 1px;
+      }
+
+      .Select-input {
+        height: auto;
+      }
+
+      .Select-arrow-zone {
+        display: flex;
+      }
+    }
+
+    .Select-placeholder {
+      display: flex;
+      align-items: center;
+    }
+
+    .Select-menu-outer {
+      margin-top: $pad-xsmall;
+      box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1);
+      border-radius: $border-radius;
+      z-index: 6;
+      overflow: hidden;
+      border: 0;
+      width: 188px;
+      left: unset;
+      top: unset;
+      max-height: none;
+      padding: $pad-small;
+      position: absolute;
+
+      .Select-menu {
+        max-height: none;
+      }
+    }
+
+    .Select-arrow {
+      transition: transform 0.25s ease;
+    }
+
+    &:not(.is-open) {
+      .Select-control:hover .Select-arrow {
+        content: url("../assets/images/icon-chevron-blue-16x16@2x.png");
+      }
+    }
+
+    &.is-open {
+      .Select-control .Select-placeholder {
+        color: $core-vibrant-blue;
+      }
+
+      .Select-arrow {
+        transform: rotate(180deg);
+      }
+    }
+  }
+}

From e8f177dd4368697dd5f7864cda0a07ccb2121b60 Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Thu, 21 Mar 2024 12:23:59 -0300
Subject: [PATCH 15/30] Additional changes to happy path and cleanup cron job
 (#17757)

#17441 & #17442
---
 cmd/fleet/calendar_cron.go                    | 301 +++++++++++++-----
 cmd/fleet/calendar_cron_test.go               |  57 +++-
 server/datastore/mysql/calendar_events.go     |  92 +++++-
 .../datastore/mysql/calendar_events_test.go   | 124 +++++++-
 .../20240314085226_AddCalendarEventTables.go  |   4 +-
 server/datastore/mysql/policies.go            |   1 +
 server/datastore/mysql/schema.sql             |   3 +-
 server/fleet/calendar_events.go               |   3 +-
 server/fleet/datastore.go                     |   4 +-
 server/mock/datastore_mock.go                 |  36 ++-
 tools/webhook/README.md                       |  16 +
 tools/webhook/main.go                         |  39 +++
 12 files changed, 558 insertions(+), 122 deletions(-)
 create mode 100644 tools/webhook/README.md
 create mode 100644 tools/webhook/main.go

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index e8ec7685d7..fa63b487dc 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -30,6 +30,12 @@ func newCalendarSchedule(
 		ctx, name, instanceID, defaultInterval, ds, ds,
 		schedule.WithAltLockID("calendar"),
 		schedule.WithLogger(logger),
+		schedule.WithJob(
+			"calendar_events_cleanup",
+			func(ctx context.Context) error {
+				return cronCalendarEventsCleanup(ctx, ds, logger)
+			},
+		),
 		schedule.WithJob(
 			"calendar_events",
 			func(ctx context.Context) error {
@@ -51,12 +57,7 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L
 		return nil
 	}
 	googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0]
-	googleCalendarConfig := calendar.GoogleCalendarConfig{
-		Context:           ctx,
-		IntegrationConfig: googleCalendarIntegrationConfig,
-		Logger:            log.With(logger, "component", "google_calendar"),
-	}
-	calendar := calendar.NewGoogleCalendar(&googleCalendarConfig)
+	calendar := createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger)
 	domain := googleCalendarIntegrationConfig.Domain
 
 	teams, err := ds.ListTeams(ctx, fleet.TeamFilter{
@@ -79,6 +80,15 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L
 	return nil
 }
 
+func createUserCalendarFromConfig(ctx context.Context, config *fleet.GoogleCalendarIntegration, logger kitlog.Logger) fleet.UserCalendar {
+	googleCalendarConfig := calendar.GoogleCalendarConfig{
+		Context:           ctx,
+		IntegrationConfig: config,
+		Logger:            log.With(logger, "component", "google_calendar"),
+	}
+	return calendar.NewGoogleCalendar(&googleCalendarConfig)
+}
+
 func cronCalendarEventsForTeam(
 	ctx context.Context,
 	ds fleet.Datastore,
@@ -110,9 +120,6 @@ func cronCalendarEventsForTeam(
 	//	- We get only one host per email that's failing policies (the one with lower host id).
 	//	- On every host, we get only the first email that matches the domain (sorted lexicographically).
 	//
-	// TODOs(lucas):
-	//	- We need to rate limit calendar requests.
-	//
 
 	policyIDs := make([]uint, 0, len(policies))
 	for _, policy := range policies {
@@ -159,15 +166,12 @@ func cronCalendarEventsForTeam(
 		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
 	}
 
-	// At last we want to notify the hosts that are failing and don't have an associated email.
-	if err := fireWebhookForHostsWithoutAssociatedEmail(
-		team.Config.Integrations.GoogleCalendar.WebhookURL,
+	// At last we want to log the hosts that are failing and don't have an associated email.
+	logHostsWithoutAssociatedEmail(
 		domain,
 		failingHostsWithoutAssociatedEmail,
 		logger,
-	); err != nil {
-		level.Info(logger).Log("msg", "webhook for hosts without associated email", "err", err)
-	}
+	)
 
 	return nil
 }
@@ -182,34 +186,40 @@ func processCalendarFailingHosts(
 ) error {
 	for _, host := range hosts {
 		logger := log.With(logger, "host_id", host.HostID)
+
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
+
+		expiredEvent := false
+		webhookAlreadyFiredThisMonth := false
+		if err == nil {
+			now := time.Now()
+			webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent
+			if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) {
+				// If the webhook already fired today and the policies are still failing
+				// we give a grace period of one day for the host before we schedule a new event.
+				continue // continue with next host
+			}
+			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
+			if calendarEvent.EndTime.Before(time.Now()) {
+				expiredEvent = true
+			}
+		}
+
 		if err := userCalendar.Configure(host.Email); err != nil {
 			return fmt.Errorf("configure user calendar: %w", err)
 		}
 
-		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
-
-		deletedExpiredEvent := false
-		if err == nil {
-			if calendarEvent.EndTime.Before(time.Now()) {
-				if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
-					level.Info(logger).Log("msg", "deleting existing expired calendar event", "err", err)
-					continue // continue with next host
-				}
-				deletedExpiredEvent = true
-			}
-		}
-
 		switch {
-		case err == nil && !deletedExpiredEvent:
+		case err == nil && !expiredEvent:
 			if err := processFailingHostExistingCalendarEvent(
 				ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err)
 				continue // continue with next host
 			}
-		case fleet.IsNotFound(err) || deletedExpiredEvent:
+		case fleet.IsNotFound(err) || expiredEvent:
 			if err := processFailingHostCreateCalendarEvent(
-				ctx, ds, userCalendar, orgName, host,
+				ctx, ds, userCalendar, orgName, host, webhookAlreadyFiredThisMonth,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host create calendar event", "err", err)
 				continue // continue with next host
@@ -231,13 +241,27 @@ func processFailingHostExistingCalendarEvent(
 	calendarEvent *fleet.CalendarEvent,
 	host fleet.HostPolicyMembershipData,
 ) error {
-	updatedEvent, updated, err := calendar.GetAndUpdateEvent(
-		calendarEvent, func(bool) string {
-			return generateCalendarEventBody(orgName, host.HostDisplayName)
+	updatedEvent := calendarEvent
+	updated := false
+	now := time.Now()
+
+	// Check the user calendar every 30 minutes (and not every time)
+	// to reduce load on both Fleet and the calendar service.
+	if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute {
+		var err error
+		updatedEvent, _, err = calendar.GetAndUpdateEvent(calendarEvent, func(conflict bool) string {
+			return generateCalendarEventBody(orgName, host.HostDisplayName, conflict)
 		})
-	if err != nil {
-		return fmt.Errorf("get event calendar on db: %w", err)
+		if err != nil {
+			return fmt.Errorf("get event calendar on db: %w", err)
+		}
+		// Even if fields haven't changed we want to update the calendar_events.updated_at below.
+		updated = true
+		//
+		// TODO(lucas): Check changing updatedEvent to UTC before consuming.
+		//
 	}
+
 	if updated {
 		if err := ds.UpdateCalendarEvent(ctx,
 			calendarEvent.ID,
@@ -248,16 +272,9 @@ func processFailingHostExistingCalendarEvent(
 			return fmt.Errorf("updating event calendar on db: %w", err)
 		}
 	}
-	now := time.Now()
+
 	eventInFuture := now.Before(updatedEvent.StartTime)
 	if eventInFuture {
-		// If the webhook status was sent and event was moved to the future we set the status to pending.
-		// This can happen if the admin wants to retry a remediation.
-		if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent {
-			if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil {
-				return fmt.Errorf("update host calendar webhook status: %w", err)
-			}
-		}
 		// Nothing else to do as event is in the future.
 		return nil
 	}
@@ -297,18 +314,31 @@ func processFailingHostExistingCalendarEvent(
 	return nil
 }
 
+func sameDate(t1 time.Time, t2 time.Time) bool {
+	y1, m1, d1 := t1.Date()
+	y2, m2, d2 := t2.Date()
+	return y1 == y2 && m1 == m2 && d1 == d2
+}
+
+func sameMonth(t1 time.Time, t2 time.Time) bool {
+	y1, m1, _ := t1.Date()
+	y2, m2, _ := t2.Date()
+	return y1 == y2 && m1 == m2
+}
+
 func processFailingHostCreateCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
 	userCalendar fleet.UserCalendar,
 	orgName string,
 	host fleet.HostPolicyMembershipData,
+	webhookAlreadyFiredThisMonth bool,
 ) error {
-	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar)
+	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar, webhookAlreadyFiredThisMonth)
 	if err != nil {
 		return fmt.Errorf("create event on user calendar: %w", err)
 	}
-	if _, err := ds.NewCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID); err != nil {
+	if _, err := ds.CreateOrUpdateCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID, fleet.CalendarWebhookStatusNone); err != nil {
 		return fmt.Errorf("create calendar event on db: %w", err)
 	}
 	return nil
@@ -318,18 +348,14 @@ func attemptCreatingEventOnUserCalendar(
 	orgName string,
 	host fleet.HostPolicyMembershipData,
 	userCalendar fleet.UserCalendar,
+	webhookAlreadyFiredThisMonth bool,
 ) (*fleet.CalendarEvent, error) {
-	// TODO(lucas): Where do we handle the following case (it seems CreateEvent needs to return no slot available for the requested day if there are none or too late):
-	//
-	// - If it’s the 3rd Tuesday of the month, create an event in the upcoming slot (if available).
-	// For example, if it’s the 3rd Tuesday of the month at 10:07a, Fleet will look for an open slot starting at 10:30a.
-	// - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day.
 	year, month, today := time.Now().Date()
-	preferredDate := getPreferredCalendarEventDate(year, month, today)
+	preferredDate := getPreferredCalendarEventDate(year, month, today, webhookAlreadyFiredThisMonth)
 	for {
 		calendarEvent, err := userCalendar.CreateEvent(
-			preferredDate, func(bool) string {
-				return generateCalendarEventBody(orgName, host.HostDisplayName)
+			preferredDate, func(conflict bool) string {
+				return generateCalendarEventBody(orgName, host.HostDisplayName, conflict)
 			},
 		)
 		var dee fleet.DayEndedError
@@ -345,7 +371,10 @@ func attemptCreatingEventOnUserCalendar(
 	}
 }
 
-func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time {
+func getPreferredCalendarEventDate(
+	year int, month time.Month, today int,
+	webhookAlreadyFired bool,
+) time.Time {
 	const (
 		// 3rd Tuesday of Month
 		preferredWeekDay = time.Tuesday
@@ -360,6 +389,10 @@ func getPreferredCalendarEventDate(year int, month time.Month, today int) time.T
 	preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1)))
 	if today > preferredDate.Day() {
 		today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC)
+		if webhookAlreadyFired {
+			nextMonth := today_.AddDate(0, 1, 0) // move to next month
+			return getPreferredCalendarEventDate(nextMonth.Year(), nextMonth.Month(), 1, false)
+		}
 		preferredDate = addBusinessDay(today_)
 	}
 	return preferredDate
@@ -379,7 +412,7 @@ func addBusinessDay(date time.Time) time.Time {
 func removeCalendarEventsFromPassingHosts(
 	ctx context.Context,
 	ds fleet.Datastore,
-	calendar fleet.UserCalendar,
+	userCalendar fleet.UserCalendar,
 	hosts []fleet.HostPolicyMembershipData,
 ) error {
 	for _, host := range hosts {
@@ -392,47 +425,42 @@ func removeCalendarEventsFromPassingHosts(
 		default:
 			return fmt.Errorf("get calendar event from DB: %w", err)
 		}
-
-		if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
-			return fmt.Errorf("delete db calendar event: %w", err)
-		}
-		if err := calendar.Configure(host.Email); err != nil {
-			return fmt.Errorf("connect to user calendar: %w", err)
-		}
-		if err := calendar.DeleteEvent(calendarEvent); err != nil {
-			return fmt.Errorf("delete calendar event: %w", err)
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
 		}
 	}
 	return nil
 }
 
-func fireWebhookForHostsWithoutAssociatedEmail(
-	webhookURL string,
+func logHostsWithoutAssociatedEmail(
 	domain string,
 	hosts []fleet.HostPolicyMembershipData,
 	logger kitlog.Logger,
-) error {
-	// TODO(lucas): We are firing these every 5 minutes...
-	for _, host := range hosts {
-		if err := fleet.FireCalendarWebhook(
-			webhookURL,
-			host.HostID, host.HostHardwareSerial, host.HostDisplayName, nil,
-			fmt.Sprintf("No %s Google account associated with this host.", domain),
-		); err != nil {
-			level.Error(logger).Log(
-				"msg", "fire webhook for hosts without associated email", "err", err,
-			)
-		}
+) {
+	if len(hosts) == 0 {
+		return
 	}
-	return nil
+	var hostIDs []uint
+	for _, host := range hosts {
+		hostIDs = append(hostIDs, host.HostID)
+	}
+	// Logging as debug because this might get logged every 5 minutes.
+	level.Debug(logger).Log(
+		"msg", fmt.Sprintf("no %s Google account associated with the hosts", domain),
+		"host_ids", fmt.Sprintf("%+v", hostIDs),
+	)
 }
 
-func generateCalendarEventBody(orgName, hostDisplayName string) string {
+func generateCalendarEventBody(orgName, hostDisplayName string, conflict bool) string {
+	conflictStr := ""
+	if conflict {
+		conflictStr = " because there was no remaining availability"
+	}
 	return fmt.Sprintf(`Please leave your computer on and connected to power.
 
 Expect an automated restart.
 
-%s reserved this time to fix %s.`, orgName, hostDisplayName,
+%s reserved this time to fix %s%s.`, orgName, hostDisplayName, conflictStr,
 	)
 }
 
@@ -456,3 +484,112 @@ func isHostOnline(ctx context.Context, ds fleet.Datastore, hostID uint) (bool, e
 		return false, fmt.Errorf("unknown host status: %s", status)
 	}
 }
+
+func cronCalendarEventsCleanup(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error {
+	appConfig, err := ds.AppConfig(ctx)
+	if err != nil {
+		return fmt.Errorf("load app config: %w", err)
+	}
+
+	var userCalendar fleet.UserCalendar
+	if len(appConfig.Integrations.GoogleCalendar) > 0 {
+		googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0]
+		userCalendar = createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger)
+	}
+
+	// If global setting is disabled, we remove all calendar events from the DB
+	// (we cannot delete the events from the user calendar because there's no configuration anymore).
+	if userCalendar == nil {
+		if err := deleteAllCalendarEvents(ctx, ds, nil, nil); err != nil {
+			return fmt.Errorf("delete all calendar events: %w", err)
+		}
+		// We've deleted all calendar events, nothing else to do.
+		return nil
+	}
+
+	//
+	// Feature is configured globally, but now we have to check team by team.
+	//
+
+	teams, err := ds.ListTeams(ctx, fleet.TeamFilter{
+		User: &fleet.User{
+			GlobalRole: ptr.String(fleet.RoleAdmin),
+		},
+	}, fleet.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("list teams: %w", err)
+	}
+
+	for _, team := range teams {
+		if err := deleteTeamCalendarEvents(ctx, ds, userCalendar, *team); err != nil {
+			level.Info(logger).Log("msg", "delete team calendar events", "team_id", team.ID, "err", err)
+		}
+	}
+
+	//
+	// Delete calendar events from DB that haven't been updated for a while
+	// (e.g. host was transferred to another team or global).
+	//
+
+	outOfDateCalendarEvents, err := ds.ListOutOfDateCalendarEvents(ctx, time.Now().Add(-48*time.Hour))
+	if err != nil {
+		return fmt.Errorf("list out of date calendar events: %w", err)
+	}
+	for _, outOfDateCalendarEvent := range outOfDateCalendarEvents {
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, outOfDateCalendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func deleteAllCalendarEvents(
+	ctx context.Context,
+	ds fleet.Datastore,
+	userCalendar fleet.UserCalendar,
+	teamID *uint,
+) error {
+	calendarEvents, err := ds.ListCalendarEvents(ctx, teamID)
+	if err != nil {
+		return fmt.Errorf("list calendar events: %w", err)
+	}
+	for _, calendarEvent := range calendarEvents {
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
+		}
+	}
+	return nil
+}
+
+func deleteTeamCalendarEvents(
+	ctx context.Context,
+	ds fleet.Datastore,
+	userCalendar fleet.UserCalendar,
+	team fleet.Team,
+) error {
+	if team.Config.Integrations.GoogleCalendar != nil &&
+		team.Config.Integrations.GoogleCalendar.Enable {
+		// Feature is enabled, nothing to cleanup.
+		return nil
+	}
+	return deleteAllCalendarEvents(ctx, ds, userCalendar, &team.ID)
+}
+
+func deleteCalendarEvent(ctx context.Context, ds fleet.Datastore, userCalendar fleet.UserCalendar, calendarEvent *fleet.CalendarEvent) error {
+	if userCalendar != nil {
+		// Only delete events from the user's calendar if the event is in the future.
+		if eventInFuture := time.Now().Before(calendarEvent.StartTime); eventInFuture {
+			if err := userCalendar.Configure(calendarEvent.Email); err != nil {
+				return fmt.Errorf("connect to user calendar: %w", err)
+			}
+			if err := userCalendar.DeleteEvent(calendarEvent); err != nil {
+				return fmt.Errorf("delete calendar event: %w", err)
+			}
+		}
+	}
+	if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
+		return fmt.Errorf("delete db calendar event: %w", err)
+	}
+	return nil
+}
diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go
index 680cf50d95..905b79c879 100644
--- a/cmd/fleet/calendar_cron_test.go
+++ b/cmd/fleet/calendar_cron_test.go
@@ -12,34 +12,61 @@ func TestGetPreferredCalendarEventDate(t *testing.T) {
 		return time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
 	}
 	for _, tc := range []struct {
-		name  string
-		year  int
-		month time.Month
-		days  int
+		name                  string
+		year                  int
+		month                 time.Month
+		daysStart             int
+		daysEnd               int
+		webhookFiredThisMonth bool
 
 		expected time.Time
 	}{
 		{
-			year:     2024,
-			month:    3,
-			days:     31,
-			name:     "March 2024",
+			name:                  "March 2024 (webhook hasn't fired)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             1,
+			daysEnd:               31,
+			webhookFiredThisMonth: false,
+
 			expected: date(2024, 3, 19),
 		},
 		{
-			year:     2024,
-			month:    4,
-			days:     30,
-			name:     "April 2024",
+			name:                  "March 2024 (webhook has fired, days before 3rd Tuesday)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             1,
+			daysEnd:               18,
+			webhookFiredThisMonth: true,
+
+			expected: date(2024, 3, 19),
+		},
+		{
+			name:                  "March 2024 (webhook has fired, days after 3rd Tuesday)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             20,
+			daysEnd:               30,
+			webhookFiredThisMonth: true,
+
+			expected: date(2024, 4, 16),
+		},
+		{
+			name:                  "April 2024 (webhook hasn't fired)",
+			year:                  2024,
+			month:                 4,
+			daysEnd:               30,
+			webhookFiredThisMonth: false,
+
 			expected: date(2024, 4, 16),
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			for day := 1; day <= tc.days; day++ {
-				actual := getPreferredCalendarEventDate(tc.year, tc.month, day)
+			for day := tc.daysStart; day <= tc.daysEnd; day++ {
+				actual := getPreferredCalendarEventDate(tc.year, tc.month, day, tc.webhookFiredThisMonth)
 				require.NotEqual(t, actual.Weekday(), time.Saturday)
 				require.NotEqual(t, actual.Weekday(), time.Sunday)
-				if day <= tc.expected.Day() {
+				if day <= tc.expected.Day() || tc.webhookFiredThisMonth {
 					require.Equal(t, tc.expected, actual)
 				} else {
 					today := date(tc.year, tc.month, day)
diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go
index 3990915976..5ffc0f77f3 100644
--- a/server/datastore/mysql/calendar_events.go
+++ b/server/datastore/mysql/calendar_events.go
@@ -11,15 +11,16 @@ import (
 	"github.com/jmoiron/sqlx"
 )
 
-func (ds *Datastore) NewCalendarEvent(
+func (ds *Datastore) CreateOrUpdateCalendarEvent(
 	ctx context.Context,
 	email string,
 	startTime time.Time,
 	endTime time.Time,
 	data []byte,
 	hostID uint,
+	webhookStatus fleet.CalendarWebhookStatus,
 ) (*fleet.CalendarEvent, error) {
-	var calendarEvent *fleet.CalendarEvent
+	var id int64
 	if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
 		const calendarEventsQuery = `
 			INSERT INTO calendar_events (
@@ -27,7 +28,12 @@ func (ds *Datastore) NewCalendarEvent(
 				start_time,
 				end_time,
 				event
-			) VALUES (?, ?, ?, ?);
+			) VALUES (?, ?, ?, ?)
+			ON DUPLICATE KEY UPDATE
+				start_time = VALUES(start_time),
+				end_time = VALUES(end_time),
+				event = VALUES(event),
+				updated_at = CURRENT_TIMESTAMP;
 		`
 		result, err := tx.ExecContext(
 			ctx,
@@ -41,13 +47,13 @@ func (ds *Datastore) NewCalendarEvent(
 			return ctxerr.Wrap(ctx, err, "insert calendar event")
 		}
 
-		id, _ := result.LastInsertId()
-		calendarEvent = &fleet.CalendarEvent{
-			ID:        uint(id),
-			Email:     email,
-			StartTime: startTime,
-			EndTime:   endTime,
-			Data:      data,
+		if insertOnDuplicateDidInsert(result) {
+			id, _ = result.LastInsertId()
+		} else {
+			stmt := `SELECT id FROM calendar_events WHERE email = ?`
+			if err := sqlx.GetContext(ctx, tx, &id, stmt, email); err != nil {
+				return ctxerr.Wrap(ctx, err, "query mdm solution id")
+			}
 		}
 
 		const hostCalendarEventsQuery = `
@@ -55,14 +61,17 @@ func (ds *Datastore) NewCalendarEvent(
 				host_id,
 				calendar_event_id,
 				webhook_status
-			) VALUES (?, ?, ?);
+			) VALUES (?, ?, ?)
+			ON DUPLICATE KEY UPDATE
+				webhook_status = VALUES(webhook_status),
+				calendar_event_id = VALUES(calendar_event_id);
 		`
 		result, err = tx.ExecContext(
 			ctx,
 			hostCalendarEventsQuery,
 			hostID,
-			calendarEvent.ID,
-			fleet.CalendarWebhookStatusPending,
+			id,
+			webhookStatus,
 		)
 		if err != nil {
 			return ctxerr.Wrap(ctx, err, "insert host calendar event")
@@ -71,9 +80,29 @@ func (ds *Datastore) NewCalendarEvent(
 	}); err != nil {
 		return nil, ctxerr.Wrap(ctx, err)
 	}
+
+	calendarEvent, err := getCalendarEventByID(ctx, ds.writer(ctx), uint(id))
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get created calendar event by id")
+	}
 	return calendarEvent, nil
 }
 
+func getCalendarEventByID(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.CalendarEvent, error) {
+	const calendarEventsQuery = `
+		SELECT * FROM calendar_events WHERE id = ?;
+	`
+	var calendarEvent fleet.CalendarEvent
+	err := sqlx.GetContext(ctx, q, &calendarEvent, calendarEventsQuery, id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithID(id))
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get calendar event")
+	}
+	return &calendarEvent, nil
+}
+
 func (ds *Datastore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) {
 	const calendarEventsQuery = `
 		SELECT * FROM calendar_events WHERE email = ?;
@@ -94,7 +123,8 @@ func (ds *Datastore) UpdateCalendarEvent(ctx context.Context, calendarEventID ui
 		UPDATE calendar_events SET
 			start_time = ?,
 			end_time = ?,
-			event = ?
+			event = ?,
+			updated_at = CURRENT_TIMESTAMP
 		WHERE id = ?;
 	`
 	if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, startTime, endTime, data, calendarEventID); err != nil {
@@ -148,3 +178,37 @@ func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID
 	}
 	return nil
 }
+
+func (ds *Datastore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) {
+	calendarEventsQuery := `
+		SELECT ce.* FROM calendar_events ce
+	`
+
+	var args []interface{}
+	if teamID != nil {
+		// TODO(lucas): Should we add a team_id column to calendar_events?
+		calendarEventsQuery += ` JOIN host_calendar_events hce ON ce.id=hce.calendar_event_id
+								 JOIN hosts h ON h.id=hce.host_id WHERE h.team_id = ?`
+		args = append(args, *teamID)
+	}
+
+	var calendarEvents []*fleet.CalendarEvent
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, args...); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get all calendar events")
+	}
+	return calendarEvents, nil
+}
+
+func (ds *Datastore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) {
+	calendarEventsQuery := `
+		SELECT ce.* FROM calendar_events ce WHERE updated_at < ?
+	`
+	var calendarEvents []*fleet.CalendarEvent
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, t); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get all calendar events")
+	}
+	return calendarEvents, nil
+}
diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go
index ccf07b3c7b..3c6030adfa 100644
--- a/server/datastore/mysql/calendar_events_test.go
+++ b/server/datastore/mysql/calendar_events_test.go
@@ -1,6 +1,128 @@
 package mysql
 
-import "testing"
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/ptr"
+	"github.com/stretchr/testify/require"
+)
 
 func TestCalendarEvents(t *testing.T) {
+	ds := CreateMySQLDS(t)
+
+	cases := []struct {
+		name string
+		fn   func(t *testing.T, ds *Datastore)
+	}{
+		{"UpdateCalendarEvent", testUpdateCalendarEvent},
+		{"CreateOrUpdateCalendarEvent", testCreateOrUpdateCalendarEvent},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			defer TruncateTables(t, ds)
+
+			c.fn(t, ds)
+		})
+	}
+}
+
+func testUpdateCalendarEvent(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	host, err := ds.NewHost(context.Background(), &fleet.Host{
+		DetailUpdatedAt: time.Now(),
+		LabelUpdatedAt:  time.Now(),
+		PolicyUpdatedAt: time.Now(),
+		SeenTime:        time.Now(),
+		NodeKey:         ptr.String("1"),
+		UUID:            "1",
+		Hostname:        "foo.local",
+		PrimaryIP:       "192.168.1.1",
+		PrimaryMac:      "30-65-EC-6F-C4-58",
+	})
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{
+		{
+			HostID: host.ID,
+			Email:  "foo@example.com",
+			Source: "google_chrome_profiles",
+		},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	startTime1 := time.Now()
+	endTime1 := startTime1.Add(30 * time.Minute)
+	calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Second)
+
+	err = ds.UpdateCalendarEvent(ctx, calendarEvent.ID, startTime1, endTime1, []byte(`{}`))
+	require.NoError(t, err)
+
+	calendarEvent2, err := ds.GetCalendarEvent(ctx, "foo@example.com")
+	require.NoError(t, err)
+	require.NotEqual(t, *calendarEvent, *calendarEvent2)
+	calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt
+	require.Equal(t, *calendarEvent, *calendarEvent2)
+
+	// TODO(lucas): Add more tests here.
+}
+
+func testCreateOrUpdateCalendarEvent(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	host, err := ds.NewHost(context.Background(), &fleet.Host{
+		DetailUpdatedAt: time.Now(),
+		LabelUpdatedAt:  time.Now(),
+		PolicyUpdatedAt: time.Now(),
+		SeenTime:        time.Now(),
+		NodeKey:         ptr.String("1"),
+		UUID:            "1",
+		Hostname:        "foo.local",
+		PrimaryIP:       "192.168.1.1",
+		PrimaryMac:      "30-65-EC-6F-C4-58",
+	})
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{
+		{
+			HostID: host.ID,
+			Email:  "foo@example.com",
+			Source: "google_chrome_profiles",
+		},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	startTime1 := time.Now()
+	endTime1 := startTime1.Add(30 * time.Minute)
+	calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Second)
+
+	calendarEvent2, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+	require.Greater(t, calendarEvent2.UpdatedAt, calendarEvent.UpdatedAt)
+	calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt
+	require.Equal(t, *calendarEvent, *calendarEvent2)
+
+	time.Sleep(1 * time.Second)
+
+	startTime2 := startTime1.Add(1 * time.Hour)
+	endTime2 := startTime1.Add(30 * time.Minute)
+	calendarEvent3, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime2, endTime2, []byte(`{"foo": "bar"}`), host.ID, fleet.CalendarWebhookStatusPending)
+	require.NoError(t, err)
+	require.Greater(t, calendarEvent3.UpdatedAt, calendarEvent2.UpdatedAt)
+	require.WithinDuration(t, startTime2, calendarEvent3.StartTime, 1*time.Second)
+	require.WithinDuration(t, endTime2, calendarEvent3.EndTime, 1*time.Second)
+	require.Equal(t, string(calendarEvent3.Data), `{"foo": "bar"}`)
+
+	calendarEvent3b, err := ds.GetCalendarEvent(ctx, "foo@example.com")
+	require.NoError(t, err)
+	require.Equal(t, calendarEvent3, calendarEvent3b)
+
+	// TODO(lucas): Add more tests here.
 }
diff --git a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
index e9222e9d91..a385a1da88 100644
--- a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
+++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
@@ -21,7 +21,9 @@ func Up_20240314085226(tx *sql.Tx) error {
 		event JSON NOT NULL,
 
 		created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-		updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+		updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+		UNIQUE KEY idx_one_calendar_event_per_email (email)
 	);
 `); err != nil {
 		return fmt.Errorf("create calendar_events table: %w", err)
diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go
index 0b3498319d..b711c8b932 100644
--- a/server/datastore/mysql/policies.go
+++ b/server/datastore/mysql/policies.go
@@ -1172,6 +1172,7 @@ func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fl
 }
 
 // TODO(lucas): Must be tested at scale.
+// TODO(lucas): Filter out hosts with team_id == NULL
 func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
 	query := `
 	SELECT 
diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql
index a256591e25..cc3c88c7ed 100644
--- a/server/datastore/mysql/schema.sql
+++ b/server/datastore/mysql/schema.sql
@@ -52,7 +52,8 @@ CREATE TABLE `calendar_events` (
   `event` json NOT NULL,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `idx_one_calendar_event_per_email` (`email`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go
index 348cb074a4..3152ee65bb 100644
--- a/server/fleet/calendar_events.go
+++ b/server/fleet/calendar_events.go
@@ -15,7 +15,8 @@ type CalendarEvent struct {
 type CalendarWebhookStatus int
 
 const (
-	CalendarWebhookStatusPending CalendarWebhookStatus = iota
+	CalendarWebhookStatusNone CalendarWebhookStatus = iota
+	CalendarWebhookStatusPending
 	CalendarWebhookStatusSent
 )
 
diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go
index f2178bf32b..65098efdfc 100644
--- a/server/fleet/datastore.go
+++ b/server/fleet/datastore.go
@@ -619,12 +619,14 @@ type Datastore interface {
 	///////////////////////////////////////////////////////////////////////////////
 	// Calendar events
 
-	NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*CalendarEvent, error)
+	CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus CalendarWebhookStatus) (*CalendarEvent, error)
 	GetCalendarEvent(ctx context.Context, email string) (*CalendarEvent, error)
 	DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error
 	UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error
 	GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error)
 	UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error
+	ListCalendarEvents(ctx context.Context, teamID *uint) ([]*CalendarEvent, error)
+	ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*CalendarEvent, error)
 
 	///////////////////////////////////////////////////////////////////////////////
 	// Team Policies
diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go
index 4e35d1eef3..1b77b29cbe 100644
--- a/server/mock/datastore_mock.go
+++ b/server/mock/datastore_mock.go
@@ -462,7 +462,7 @@ type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities
 
 type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error
 
-type NewCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error)
+type CreateOrUpdateCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error)
 
 type GetCalendarEventFunc func(ctx context.Context, email string) (*fleet.CalendarEvent, error)
 
@@ -474,6 +474,10 @@ type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.Hos
 
 type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error
 
+type ListCalendarEventsFunc func(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error)
+
+type ListOutOfDateCalendarEventsFunc func(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error)
+
 type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error)
 
 type ListTeamPoliciesFunc func(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error)
@@ -1541,8 +1545,8 @@ type DataStore struct {
 	DeleteOutOfDateVulnerabilitiesFunc        DeleteOutOfDateVulnerabilitiesFunc
 	DeleteOutOfDateVulnerabilitiesFuncInvoked bool
 
-	NewCalendarEventFunc        NewCalendarEventFunc
-	NewCalendarEventFuncInvoked bool
+	CreateOrUpdateCalendarEventFunc        CreateOrUpdateCalendarEventFunc
+	CreateOrUpdateCalendarEventFuncInvoked bool
 
 	GetCalendarEventFunc        GetCalendarEventFunc
 	GetCalendarEventFuncInvoked bool
@@ -1559,6 +1563,12 @@ type DataStore struct {
 	UpdateHostCalendarWebhookStatusFunc        UpdateHostCalendarWebhookStatusFunc
 	UpdateHostCalendarWebhookStatusFuncInvoked bool
 
+	ListCalendarEventsFunc        ListCalendarEventsFunc
+	ListCalendarEventsFuncInvoked bool
+
+	ListOutOfDateCalendarEventsFunc        ListOutOfDateCalendarEventsFunc
+	ListOutOfDateCalendarEventsFuncInvoked bool
+
 	NewTeamPolicyFunc        NewTeamPolicyFunc
 	NewTeamPolicyFuncInvoked bool
 
@@ -3716,11 +3726,11 @@ func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source f
 	return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration)
 }
 
-func (s *DataStore) NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) {
+func (s *DataStore) CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error) {
 	s.mu.Lock()
-	s.NewCalendarEventFuncInvoked = true
+	s.CreateOrUpdateCalendarEventFuncInvoked = true
 	s.mu.Unlock()
-	return s.NewCalendarEventFunc(ctx, email, startTime, endTime, data, hostID)
+	return s.CreateOrUpdateCalendarEventFunc(ctx, email, startTime, endTime, data, hostID, webhookStatus)
 }
 
 func (s *DataStore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) {
@@ -3758,6 +3768,20 @@ func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID
 	return s.UpdateHostCalendarWebhookStatusFunc(ctx, hostID, status)
 }
 
+func (s *DataStore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.ListCalendarEventsFuncInvoked = true
+	s.mu.Unlock()
+	return s.ListCalendarEventsFunc(ctx, teamID)
+}
+
+func (s *DataStore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.ListOutOfDateCalendarEventsFuncInvoked = true
+	s.mu.Unlock()
+	return s.ListOutOfDateCalendarEventsFunc(ctx, t)
+}
+
 func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) {
 	s.mu.Lock()
 	s.NewTeamPolicyFuncInvoked = true
diff --git a/tools/webhook/README.md b/tools/webhook/README.md
new file mode 100644
index 0000000000..6bdb2416a7
--- /dev/null
+++ b/tools/webhook/README.md
@@ -0,0 +1,16 @@
+# webhook
+
+Test tool for Fleet features that use webhook URLs.
+It reads and parses the request a JSON body and prints the JSON to standard output (with indentation).
+
+```sh
+go run ./tools/webhook 8082
+2024/03/20 09:10:00 {
+  "error": "No fleetdm.com Google account associated with this host.",
+  "host_display_name": "dChYnk.uxURT",
+  "host_id": 2,
+  "host_serial_number": "",
+  "timestamp": "2024-03-20T09:10:00.129982-03:00"
+}
+...
+```
diff --git a/tools/webhook/main.go b/tools/webhook/main.go
new file mode 100644
index 0000000000..452e0a15ca
--- /dev/null
+++ b/tools/webhook/main.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags)
+
+	http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			log.Printf("failed to read body: %s", err)
+			return
+		}
+
+		var v interface{}
+		if err := json.Unmarshal(body, &v); err != nil {
+			log.Printf("failed to parse JSON body: %s", err)
+			return
+		}
+		b, err := json.MarshalIndent(v, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+		log.Printf("%s", b)
+
+		w.WriteHeader(http.StatusOK)
+	}))
+	//nolint:gosec // G114: file server used for testing purposes only.
+	err := http.ListenAndServe("0.0.0.0:"+os.Args[1], nil)
+	if err != nil {
+		panic(err)
+	}
+}

From 2940b32a06ec1c26049f7447e5f3efff476a59d1 Mon Sep 17 00:00:00 2001
From: RachelElysia <71795832+RachelElysia@users.noreply.github.com>
Date: Thu, 21 Mar 2024 16:22:47 -0400
Subject: [PATCH 16/30] Fleet UI: Calendar settings page (#17593)

---
 .../PlatformWrapper/_styles.scss              |  12 +-
 .../EnrollSecretRow/_styles.scss              |   5 +-
 .../InputFieldHiddenContent/_styles.scss      |   5 +-
 frontend/interfaces/integration.ts            |   3 +-
 .../TokenSecretField/_styles.scss             |   7 +-
 .../ManageSoftwareAutomationsModal.tsx        |   4 +-
 .../IntegrationsPage/IntegrationNavItems.tsx  |  56 +--
 .../IntegrationsPage/IntegrationsPage.tsx     |   8 +-
 .../cards/Calendars/Calendars.tsx             | 401 ++++++++++++++++++
 .../cards/Calendars/_styles.scss              |  57 +++
 .../IntegrationsPage/cards/Calendars/index.ts |   1 +
 .../cards/Integrations/Integrations.tsx       |  14 +-
 .../admin/components/SideNav/SideNav.tsx      |   1 -
 .../OtherWorkflowsModal.tsx                   |  12 +-
 frontend/router/paths.ts                      |   1 +
 frontend/styles/var/mixins.scss               |   9 +
 16 files changed, 528 insertions(+), 68 deletions(-)
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts

diff --git a/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss b/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
index 7182a6968b..6af47c2b72 100644
--- a/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
+++ b/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
@@ -51,12 +51,7 @@
   }
 
   &__copy-message {
-    font-weight: $regular;
-    vertical-align: top;
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   .buttons {
@@ -122,9 +117,6 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 }
diff --git a/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss b/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
index c400a9f658..1b5d033a57 100644
--- a/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
+++ b/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
@@ -40,10 +40,7 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   &__action-overlay {
diff --git a/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss b/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
index 89b07fdaf2..4ac1e32a41 100644
--- a/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
+++ b/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
@@ -43,9 +43,6 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 }
diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts
index 49156d6277..aea79f99ee 100644
--- a/frontend/interfaces/integration.ts
+++ b/frontend/interfaces/integration.ts
@@ -61,9 +61,8 @@ export interface IIntegrationFormErrors {
 }
 
 export interface IGlobalCalendarIntegration {
-  email: string;
-  private_key: string;
   domain: string;
+  api_key_json: string;
 }
 
 interface ITeamCalendarSettings {
diff --git a/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss b/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
index fee0ee67cc..bca40979df 100644
--- a/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
+++ b/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
@@ -31,12 +31,7 @@
   }
 
   &__copy-message {
-    font-weight: $regular;
-    vertical-align: top;
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   &__secret-download-icon {
diff --git a/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx b/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
index 002bca8924..2869dd799e 100644
--- a/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
+++ b/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
@@ -8,7 +8,7 @@ import {
   IJiraIntegration,
   IZendeskIntegration,
   IIntegration,
-  IIntegrations,
+  IGlobalIntegrations,
   IIntegrationType,
 } from "interfaces/integration";
 import {
@@ -124,7 +124,7 @@ const ManageAutomationsModal = ({
     }
   }, [destinationUrl]);
 
-  const { data: integrations } = useQuery(
+  const { data: integrations } = useQuery(
     ["integrations"],
     () => configAPI.loadAll(),
     {
diff --git a/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx b/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
index 870444974b..a3f8734da8 100644
--- a/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
+++ b/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
@@ -4,32 +4,34 @@ import { ISideNavItem } from "../components/SideNav/SideNav";
 import Integrations from "./cards/Integrations";
 import Mdm from "./cards/MdmSettings/MdmSettings";
 import AutomaticEnrollment from "./cards/AutomaticEnrollment/AutomaticEnrollment";
+import Calendars from "./cards/Calendars/Calendars";
 
-const getFilteredIntegrationSettingsNavItems = (
-  isSandboxMode = false
-): ISideNavItem[] => {
-  return [
-    // TODO: types
-    {
-      title: "Ticket destinations",
-      urlSection: "ticket-destinations",
-      path: PATHS.ADMIN_INTEGRATIONS_TICKET_DESTINATIONS,
-      Card: Integrations,
-    },
-    {
-      title: "Mobile device management (MDM)",
-      urlSection: "mdm",
-      path: PATHS.ADMIN_INTEGRATIONS_MDM,
-      Card: Mdm,
-      exclude: isSandboxMode,
-    },
-    {
-      title: "Automatic enrollment",
-      urlSection: "automatic-enrollment",
-      path: PATHS.ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT,
-      Card: AutomaticEnrollment,
-    },
-  ].filter((navItem) => !navItem.exclude);
-};
+const integrationSettingsNavItems: ISideNavItem[] = [
+  // TODO: types
+  {
+    title: "Ticket destinations",
+    urlSection: "ticket-destinations",
+    path: PATHS.ADMIN_INTEGRATIONS_TICKET_DESTINATIONS,
+    Card: Integrations,
+  },
+  {
+    title: "Mobile device management (MDM)",
+    urlSection: "mdm",
+    path: PATHS.ADMIN_INTEGRATIONS_MDM,
+    Card: Mdm,
+  },
+  {
+    title: "Automatic enrollment",
+    urlSection: "automatic-enrollment",
+    path: PATHS.ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT,
+    Card: AutomaticEnrollment,
+  },
+  {
+    title: "Calendars",
+    urlSection: "calendars",
+    path: PATHS.ADMIN_INTEGRATIONS_CALENDARS,
+    Card: Calendars,
+  },
+];
 
-export default getFilteredIntegrationSettingsNavItems;
+export default integrationSettingsNavItems;
diff --git a/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx b/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
index 019a219d51..bae02c33ca 100644
--- a/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
+++ b/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
@@ -1,9 +1,8 @@
-import { AppContext } from "context/app";
-import React, { useContext } from "react";
+import React from "react";
 import { InjectedRouter, Params } from "react-router/lib/Router";
 
 import SideNav from "../components/SideNav";
-import getFilteredIntegrationSettingsNavItems from "./IntegrationNavItems";
+import integrationSettingsNavItems from "./IntegrationNavItems";
 
 const baseClass = "integrations";
 
@@ -16,9 +15,8 @@ const IntegrationsPage = ({
   router,
   params,
 }: IIntegrationSettingsPageProps) => {
-  const { isSandboxMode } = useContext(AppContext);
   const { section } = params;
-  const navItems = getFilteredIntegrationSettingsNavItems(isSandboxMode);
+  const navItems = integrationSettingsNavItems;
   const DEFAULT_SETTINGS_SECTION = navItems[0];
   const currentSection =
     navItems.find((item) => item.urlSection === section) ??
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
new file mode 100644
index 0000000000..de7c79a139
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
@@ -0,0 +1,401 @@
+import React, { useState, useContext, useCallback } from "react";
+import { useQuery } from "react-query";
+
+import { IConfig } from "interfaces/config";
+import { NotificationContext } from "context/notification";
+import { AppContext } from "context/app";
+import configAPI from "services/entities/config";
+// @ts-ignore
+import { stringToClipboard } from "utilities/copy_text";
+
+// @ts-ignore
+import InputField from "components/forms/fields/InputField";
+import Button from "components/buttons/Button";
+import SectionHeader from "components/SectionHeader";
+import CustomLink from "components/CustomLink";
+import Spinner from "components/Spinner";
+import DataError from "components/DataError";
+import PremiumFeatureMessage from "components/PremiumFeatureMessage/PremiumFeatureMessage";
+import Icon from "components/Icon";
+
+const CREATING_SERVICE_ACCOUNT =
+  "https://www.fleetdm.com/learn-more-about/creating-service-accounts";
+const GOOGLE_WORKSPACE_DOMAINS =
+  "https://www.fleetdm.com/learn-more-about/google-workspace-domains";
+const DOMAIN_WIDE_DELEGATION =
+  "https://www.fleetdm.com/learn-more-about/domain-wide-delegation";
+const ENABLING_CALENDAR_API =
+  "fleetdm.com/learn-more-about/enabling-calendar-api";
+const OAUTH_SCOPES =
+  "https://www.googleapis.com/auth/calendar.events,https://www.googleapis.com/auth/calendar.settings.readonly";
+
+const API_KEY_JSON_PLACEHOLDER = `{
+  "type": "service_account",
+  "project_id": "fleet-in-your-calendar",
+  "private_key_id": "",
+  "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
+  "client_email": "fleet-calendar-events@fleet-in-your-calendar.iam.gserviceaccount.com",
+  "client_id": "",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/fleet-calendar-events%40fleet-in-your-calendar.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}`;
+
+interface IFormField {
+  name: string;
+  value: string | boolean | number;
+}
+
+interface ICalendarsFormErrors {
+  domain?: string | null;
+  apiKeyJson?: string | null;
+}
+
+interface ICalendarsFormData {
+  domain?: string;
+  apiKeyJson?: string;
+}
+
+const baseClass = "calendars-integration";
+
+const Calendars = (): JSX.Element => {
+  const { renderFlash } = useContext(NotificationContext);
+  const { isPremiumTier } = useContext(AppContext);
+
+  const [formData, setFormData] = useState({
+    domain: "",
+    apiKeyJson: "",
+  });
+  const [isUpdatingSettings, setIsUpdatingSettings] = useState(false);
+  const [formErrors, setFormErrors] = useState({});
+  const [copyMessage, setCopyMessage] = useState("");
+
+  const {
+    isLoading: isLoadingAppConfig,
+    refetch: refetchConfig,
+    error: errorAppConfig,
+  } = useQuery(["config"], () => configAPI.loadAll(), {
+    select: (data: IConfig) => data,
+    onSuccess: (data) => {
+      if (data.integrations.google_calendar) {
+        setFormData({
+          domain: data.integrations.google_calendar[0].domain,
+          // Formats string for better UI readability
+          apiKeyJson: JSON.stringify(
+            data.integrations.google_calendar[0].api_key_json,
+            null,
+            "\t"
+          ),
+        });
+      }
+    },
+  });
+
+  const { apiKeyJson, domain } = formData;
+
+  const validateForm = (curFormData: ICalendarsFormData) => {
+    const errors: ICalendarsFormErrors = {};
+
+    // Must set all keys or no keys at all
+    if (!curFormData.apiKeyJson && !!curFormData.domain) {
+      errors.apiKeyJson = "API key JSON must be present";
+    }
+    if (!curFormData.domain && !!curFormData.apiKeyJson) {
+      errors.apiKeyJson = "Domain must be present";
+    }
+    return errors;
+  };
+
+  const onInputChange = useCallback(
+    ({ name, value }: IFormField) => {
+      const newFormData = { ...formData, [name]: value };
+      setFormData(newFormData);
+      setFormErrors(validateForm(newFormData));
+    },
+    [formData]
+  );
+
+  const onFormSubmit = async (evt: React.MouseEvent) => {
+    setIsUpdatingSettings(true);
+
+    evt.preventDefault();
+
+    // Format for API
+    const formDataToSubmit =
+      formData.apiKeyJson === "" && formData.domain === ""
+        ? [] // Send empty array if no keys are set
+        : [
+            {
+              domain: formData.domain,
+              api_key_json:
+                (formData.apiKeyJson && JSON.parse(formData.apiKeyJson)) ||
+                null,
+            },
+          ];
+
+    // Update integrations.google_calendar only
+    const destination = {
+      google_calendar: formDataToSubmit,
+    };
+
+    try {
+      await configAPI.update({ integrations: destination });
+      renderFlash(
+        "success",
+        "Successfully saved calendar integration settings"
+      );
+      refetchConfig();
+    } catch (e) {
+      renderFlash("error", "Could not save calendar integration settings");
+    } finally {
+      setIsUpdatingSettings(false);
+    }
+  };
+
+  const renderOauthLabel = () => {
+    const onCopyOauthScopes = (evt: React.MouseEvent) => {
+      evt.preventDefault();
+
+      stringToClipboard(OAUTH_SCOPES)
+        .then(() => setCopyMessage(() => "Copied!"))
+        .catch(() => setCopyMessage(() => "Copy failed"));
+
+      // Clear message after 1 second
+      setTimeout(() => setCopyMessage(() => ""), 1000);
+
+      return false;
+    };
+
+    return (
+      
+        
+        {copyMessage && (
+          {copyMessage}
+        )}
+      
+    );
+  };
+
+  const renderForm = () => {
+    return (
+      <>
+        
+        

+          To create calendar events for end users with failing policies,
+          you'll need to configure a dedicated Google Workspace service
+          account.
+        

+        

+          

+            1. Go to the Service Accounts page in Google Cloud Platform.{" "}
+            
+          

+          

+            2. Create a new project for your service account.
+            
    
+              
  • 
+                Click Create project.
+              
    • 
+              
  • 
+                Enter "Fleet calendar events" as the project name.
+              
    • 
+              
  • 
+                For "Organization" and "Location", select
+                your calendar's organization.
+              
    • 
+            

+          

+
+          

+            3. Create the service account.
+            
    
+              
  • 
+                Click Create service account.
+              
    • 
+              
  • 
+                Set the service account name to "Fleet calendar
+                events".
+              
    • 
+              
  • 
+                Set the service account ID to "fleet-calendar-events".
+              
    • 
+              
  • 
+                Click Create and continue.
+              
    • 
+              
  • 
+                Click Done at the bottom of the form. (No need to
+                complete the optional steps.)
+              
    • 
+            

+          

+          

+            4. Create an API key.{" "}
+            
    
+              
  • 
+                Click the Actions menu for your new service account.
+              
    • 
+              
  • 
+                Select Manage keys.
+              
    • 
+              
  • 
+                Click Add key > Create new key.
+              
    • 
+              
  • Select the JSON key type.
    • 
+              
  • 
+                Click Create to create the key & download a JSON file.
+              
    • 
+              
  • 
+                Configure your service account integration in Fleet using the
+                form below:
+                
    
+                  
+                        Paste the full contents of the JSON file downloaded{" "}
+                        
    
+                        when creating your service account API key.
+                      
+                    }
+                    placeholder={API_KEY_JSON_PLACEHOLDER}
+                    ignore1password
+                    inputClassName={`${baseClass}__api-key-json`}
+                  />
+                  
+                        If the end user is signed into multiple Google accounts,
+                        this will be used to identify their work calendar.
+                      
+                    }
+                    placeholder="example.com"
+                    helpText={
+                      <>
+                        You can find your primary domain in Google Workspace{" "}
+                        
+                      
+                    }
+                  />
+                  
+                
+              
    • 
+            

+          

+          

+            5. Authorize the service account via domain-wide delegation.
+            
    
+              
  • 
+                In Google Workspace, go to{" "}
+                
+                  Security > Access and data control > API controls >
+                  Manage Domain Wide Delegation
+                
+                .{" "}
+                
+              
    • 
+              
  • 
+                Under API clients, click Add new.
+              
    • 
+              
  • 
+                Enter the client ID for the service account. You can find this
+                in your downloaded API key JSON file (
+                client_id
+                ), or under Advanced Settings when viewing the service
+                account.
+              
    • 
+              
  • 
+                For the OAuth scopes, paste the following value:
+                
+              
    • 
+              
  • 
+                Click Authorize.
+              
    • 
+            

+          

+          

+            6. Enable the Google Calendar API.
+            
    
+              
  • 
+                In the Google Cloud console API library, go to the Google
+                Calendar API.{" "}
+                
+              
    • 
+              
  • 
+                Make sure the "Fleet calendar events" project is
+                selected at the top of the page.
+              
    • 
+              
  • 
+                Click Enable.
+              
    • 
+            

+          

+        

+      
+    );
+  };
+
+  if (!isPremiumTier) return ;
+
+  if (isLoadingAppConfig) {
+    

+      
+    
;
+  }
+
+  if (errorAppConfig) {
+    return ;
+  }
+
+  return 
{renderForm()}
;
+};
+
+export default Calendars;
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
new file mode 100644
index 0000000000..7045a443ef
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
@@ -0,0 +1,57 @@
+.calendars-integration {
+  &__page-description {
+    font-size: $x-small;
+    color: $core-fleet-black;
+  }
+
+  ui {
+    margin-block-start: $pad-small;
+  }
+
+  li {
+    margin: $pad-small 0;
+  }
+
+  form {
+    margin-top: $pad-large;
+  }
+
+  &__configuration {
+    button {
+      align-self: flex-end;
+    }
+  }
+
+  &__api-key-json {
+    min-width: 100%; // resize vertically only
+    height: 294px;
+    font-size: $x-small;
+  }
+
+  #oauth-scopes {
+    font-family: "SourceCodePro", $monospace;
+    min-height: 80px;
+    padding: $pad-medium;
+    padding-right: $pad-xxlarge;
+    resize: none;
+  }
+
+  &__oauth-scopes-copy-icon-wrapper {
+    display: flex;
+    flex-direction: row-reverse;
+    align-items: center;
+    position: relative;
+    top: 36px;
+    right: 16px;
+    height: 0;
+    gap: 0.5rem;
+  }
+
+  &__copy-message {
+    @include copy-message;
+  }
+
+  &__code {
+    font-family: "SourceCodePro", $monospace;
+  }
+}
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts b/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts
new file mode 100644
index 0000000000..99dcd737cf
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts
@@ -0,0 +1 @@
+export { default } from "./Calendars";
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
index 75f95f836b..abe8aef4d6 100644
--- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
+++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
@@ -8,7 +8,7 @@ import {
   IZendeskIntegration,
   IIntegration,
   IIntegrationTableData,
-  IIntegrations,
+  IGlobalIntegrations,
 } from "interfaces/integration";
 import { IApiError } from "interfaces/errors";
 
@@ -69,7 +69,7 @@ const Integrations = (): JSX.Element => {
     isLoading: isLoadingIntegrations,
     error: loadingIntegrationsError,
     refetch: refetchIntegrations,
-  } = useQuery(
+  } = useQuery(
     ["integrations"],
     () => configAPI.loadAll(),
     {
@@ -133,9 +133,15 @@ const Integrations = (): JSX.Element => {
       // Updates either integrations.jira or integrations.zendesk
       const destination = () => {
         if (integrationDestination === "jira") {
-          return { jira: integrationSubmitData, zendesk: zendeskIntegrations };
+          return {
+            jira: integrationSubmitData,
+            zendesk: zendeskIntegrations,
+          };
         }
-        return { zendesk: integrationSubmitData, jira: jiraIntegrations };
+        return {
+          zendesk: integrationSubmitData,
+          jira: jiraIntegrations,
+        };
       };
 
       setTestingConnection(true);
diff --git a/frontend/pages/admin/components/SideNav/SideNav.tsx b/frontend/pages/admin/components/SideNav/SideNav.tsx
index 1242cdcba9..333f3f3f93 100644
--- a/frontend/pages/admin/components/SideNav/SideNav.tsx
+++ b/frontend/pages/admin/components/SideNav/SideNav.tsx
@@ -12,7 +12,6 @@ export interface ISideNavItem {
   urlSection: string;
   path: string;
   Card: (props: T) => JSX.Element;
-  exclude?: boolean;
 }
 
 interface ISideNavProps {
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
index 907f4edb4f..54622a6927 100644
--- a/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
@@ -3,7 +3,12 @@ import { Link } from "react-router";
 import { isEmpty, noop, omit } from "lodash";
 
 import { IAutomationsConfig, IWebhookSettings } from "interfaces/config";
-import { IIntegration, IIntegrations } from "interfaces/integration";
+import {
+  IGlobalIntegrations,
+  IIntegration,
+  IIntegrations,
+  ITeamIntegrations,
+} from "interfaces/integration";
 import { IPolicy } from "interfaces/policy";
 import { ITeamAutomationsConfig } from "interfaces/team";
 import PATHS from "router/paths";
@@ -26,13 +31,13 @@ import ExamplePayload from "../ExamplePayload";
 
 interface IOtherWorkflowsModalProps {
   automationsConfig: IAutomationsConfig | ITeamAutomationsConfig;
-  availableIntegrations: IIntegrations;
+  availableIntegrations: IGlobalIntegrations | ITeamIntegrations;
   availablePolicies: IPolicy[];
   isUpdatingAutomations: boolean;
   onExit: () => void;
   handleSubmit: (formData: {
     webhook_settings: Pick;
-    integrations: IIntegrations;
+    integrations: IGlobalIntegrations | ITeamIntegrations;
   }) => void;
 }
 
@@ -256,6 +261,7 @@ const OtherWorkflowsModal = ({
       integrations: {
         jira: newJira,
         zendesk: newZendesk,
+        google_calendar: null, // When null, the backend does not update google_calendar
       },
     });
 
diff --git a/frontend/router/paths.ts b/frontend/router/paths.ts
index 114e42dd8e..4a1752c652 100644
--- a/frontend/router/paths.ts
+++ b/frontend/router/paths.ts
@@ -32,6 +32,7 @@ export default {
   ADMIN_INTEGRATIONS_MDM_WINDOWS: `${URL_PREFIX}/settings/integrations/mdm/windows`,
   ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT: `${URL_PREFIX}/settings/integrations/automatic-enrollment`,
   ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT_WINDOWS: `${URL_PREFIX}/settings/integrations/automatic-enrollment/windows`,
+  ADMIN_INTEGRATIONS_CALENDARS: `${URL_PREFIX}/settings/integrations/calendars`,
   ADMIN_TEAMS: `${URL_PREFIX}/settings/teams`,
   ADMIN_ORGANIZATION: `${URL_PREFIX}/settings/organization`,
   ADMIN_ORGANIZATION_INFO: `${URL_PREFIX}/settings/organization/info`,
diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss
index a6ef009b68..3751ffadf6 100644
--- a/frontend/styles/var/mixins.scss
+++ b/frontend/styles/var/mixins.scss
@@ -182,6 +182,15 @@ $max-width: 2560px;
   }
 }
 
+@mixin copy-message {
+  font-weight: $regular;
+  vertical-align: top;
+  background-color: $ui-light-grey;
+  border: solid 1px #e2e4ea;
+  border-radius: 10px;
+  padding: 2px 6px;
+}
+
 @mixin color-contrasted-sections {
   background-color: $ui-off-white;
   .section {

From 16f122f02a697b3d974395c97d3fbf6eb68aafdc Mon Sep 17 00:00:00 2001
From: Victor Lyuboslavsky 
Date: Fri, 22 Mar 2024 09:19:55 -0500
Subject: [PATCH 17/30] Adding calendar test server and other fixes. (#17751)

- Added a calendar server that can be used for load testing at
/tools/calendar
- Fixed minor calendar bugs

# Checklist for submitter
- [ ] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality
---
 ee/server/calendar/google_calendar.go         | 109 ++++--
 .../google_calendar_integration_test.go       | 132 +++++++
 ee/server/calendar/google_calendar_load.go    | 234 ++++++++++++
 ee/server/calendar/google_calendar_mock.go    |   8 +-
 ee/server/calendar/google_calendar_test.go    |  30 ++
 .../load_test/calendar_http_handler.go        | 343 ++++++++++++++++++
 tools/calendar/README.md                      |  26 ++
 tools/calendar/calendar.go                    |  39 ++
 8 files changed, 882 insertions(+), 39 deletions(-)
 create mode 100644 ee/server/calendar/google_calendar_integration_test.go
 create mode 100644 ee/server/calendar/google_calendar_load.go
 create mode 100644 ee/server/calendar/load_test/calendar_http_handler.go
 create mode 100644 tools/calendar/README.md
 create mode 100644 tools/calendar/calendar.go

diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go
index 26d1ba1e66..42f8b7b0d8 100644
--- a/ee/server/calendar/google_calendar.go
+++ b/ee/server/calendar/google_calendar.go
@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
+	"regexp"
 	"time"
 
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
@@ -19,18 +21,30 @@ import (
 	"google.golang.org/api/option"
 )
 
+// The calendar package has the following features for testing:
+// 1. High level UserCalendar interface and Low level GoogleCalendarAPI interface can have a custom implementations.
+// 2. Setting "client_email" to "calendar-mock@example.com" in the API key will use a mock in-memory implementation GoogleCalendarMockAPI of GoogleCalendarAPI.
+// 3. Setting FLEET_GOOGLE_CALENDAR_PLUS_ADDRESSING environment variable to "1" will strip the "plus addressing" from the user email, effectively allowing a single user
+//    to create multiple events in the same calendar. This is useful for load testing. For example: john+test@example.com becomes john@example.com
+
 const (
 	eventTitle  = "💻🚫Downtime"
 	startHour   = 9
 	endHour     = 17
 	eventLength = 30 * time.Minute
 	calendarID  = "primary"
+	mockEmail   = "calendar-mock@example.com"
+	loadEmail   = "calendar-load@example.com"
 )
 
-var calendarScopes = []string{
-	"https://www.googleapis.com/auth/calendar.events",
-	"https://www.googleapis.com/auth/calendar.settings.readonly",
-}
+var (
+	calendarScopes = []string{
+		"https://www.googleapis.com/auth/calendar.events",
+		"https://www.googleapis.com/auth/calendar.settings.readonly",
+	}
+	plusAddressing      = os.Getenv("FLEET_GOOGLE_CALENDAR_PLUS_ADDRESSING") == "1"
+	plusAddressingRegex = regexp.MustCompile(`\+.*@`)
+)
 
 type GoogleCalendarConfig struct {
 	Context           context.Context
@@ -43,19 +57,22 @@ type GoogleCalendarConfig struct {
 // GoogleCalendar is an implementation of the UserCalendar interface that uses the
 // Google Calendar API to manage events.
 type GoogleCalendar struct {
-	config           *GoogleCalendarConfig
-	currentUserEmail string
-	timezoneOffset   *int
+	config            *GoogleCalendarConfig
+	currentUserEmail  string
+	adjustedUserEmail string
+	location          *time.Location
 }
 
 func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar {
-	if config.API == nil {
-		if config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == "calendar-mock@example.com" {
-			// Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory
-			config.API = &GoogleCalendarMockAPI{}
-		} else {
-			config.API = &GoogleCalendarLowLevelAPI{}
-		}
+	switch {
+	case config.API != nil:
+		// Use the provided API.
+	case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == loadEmail:
+		config.API = &GoogleCalendarLoadAPI{Logger: config.Logger}
+	case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == mockEmail:
+		config.API = &GoogleCalendarMockAPI{config.Logger}
+	default:
+		config.API = &GoogleCalendarLowLevelAPI{}
 	}
 	return &GoogleCalendar{
 		config: config,
@@ -101,6 +118,13 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) Configure(
 	return nil
 }
 
+func adjustEmail(email string) string {
+	if plusAddressing {
+		return plusAddressingRegex.ReplaceAllString(email, "@")
+	}
+	return email
+}
+
 func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) {
 	return lowLevelAPI.service.Settings.Get(name).Do()
 }
@@ -130,14 +154,18 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error {
 }
 
 func (c *GoogleCalendar) Configure(userEmail string) error {
+	adjustedUserEmail := adjustEmail(userEmail)
 	err := c.config.API.Configure(
 		c.config.Context, c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail],
-		c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], userEmail,
+		c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], adjustedUserEmail,
 	)
 	if err != nil {
 		return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service")
 	}
 	c.currentUserEmail = userEmail
+	c.adjustedUserEmail = adjustedUserEmail
+	// Clear the timezone offset so that it will be recalculated
+	c.location = nil
 	return nil
 }
 
@@ -162,7 +190,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 		return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event")
 	}
 	if !deleted && gEvent.Status != "cancelled" {
-		if details.ETag == gEvent.Etag {
+		if details.ETag != "" && details.ETag == gEvent.Etag {
 			// Event was not modified
 			return event, false, nil
 		}
@@ -246,20 +274,20 @@ func calculateNewEventDate(oldStartDate time.Time) time.Time {
 }
 
 func (c *GoogleCalendar) parseDateTime(eventDateTime *calendar.EventDateTime) (*time.Time, error) {
-	var endTime time.Time
+	var t time.Time
 	var err error
 	if eventDateTime.TimeZone != "" {
 		loc := getLocation(eventDateTime.TimeZone, c.config)
-		endTime, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
+		t, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
 	} else {
-		endTime, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
+		t, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
 	}
 	if err != nil {
 		return nil, ctxerr.Wrap(
 			c.config.Context, err, fmt.Sprintf("parsing Google calendar event time: %s", eventDateTime.DateTime),
 		)
 	}
-	return &endTime, nil
+	return &t, nil
 }
 
 func isNotFound(err error) bool {
@@ -271,6 +299,15 @@ func isNotFound(err error) bool {
 	return ok && ae.Code == http.StatusNotFound
 }
 
+func isAlreadyDeleted(err error) bool {
+	if err == nil {
+		return false
+	}
+	var ae *googleapi.Error
+	ok := errors.As(err, &ae)
+	return ok && ae.Code == http.StatusGone
+}
+
 func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) {
 	var details eventDetails
 	err := json.Unmarshal(event.Data, &details)
@@ -293,18 +330,18 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, genBodyFn func(confli
 func (c *GoogleCalendar) createEvent(
 	dayOfEvent time.Time, genBodyFn func(conflict bool) string, timeNow func() time.Time,
 ) (*fleet.CalendarEvent, error) {
-	if c.timezoneOffset == nil {
-		err := getTimezone(c)
+	var err error
+	if c.location == nil {
+		c.location, err = getTimezone(c)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	location := time.FixedZone("", *c.timezoneOffset)
-	dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location)
-	dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location)
+	dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, c.location)
+	dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, c.location)
 
-	now := timeNow().In(location)
+	now := timeNow().In(c.location)
 	if dayEnd.Before(now) {
 		// The workday has already ended.
 		return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"})
@@ -342,7 +379,7 @@ func (c *GoogleCalendar) createEvent(
 		// Ignore events that the user has declined
 		var declined bool
 		for _, attendee := range gEvent.Attendees {
-			if attendee.Email == c.currentUserEmail {
+			if attendee.Email == c.adjustedUserEmail {
 				// The user has declined the event, so this time is open for scheduling
 				if attendee.ResponseStatus == "declined" {
 					declined = true
@@ -396,7 +433,9 @@ func (c *GoogleCalendar) createEvent(
 	if err != nil {
 		return nil, err
 	}
-	level.Debug(c.config.Logger).Log("msg", "created Google calendar event", "user", c.currentUserEmail, "startTime", eventStart)
+	level.Debug(c.config.Logger).Log(
+		"msg", "created Google calendar event", "user", c.adjustedUserEmail, "startTime", eventStart, "timezone", c.location.String(),
+	)
 
 	return fleetEvent, nil
 }
@@ -419,17 +458,14 @@ func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time
 	return eventStart, eventEnd, isLastSlot, conflict
 }
 
-func getTimezone(gCal *GoogleCalendar) error {
+func getTimezone(gCal *GoogleCalendar) (*time.Location, error) {
 	config := gCal.config
 	setting, err := config.API.GetSetting("timezone")
 	if err != nil {
-		return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone")
+		return nil, ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone")
 	}
 
-	loc := getLocation(setting.Value, config)
-	_, timezoneOffset := time.Now().In(loc).Zone()
-	gCal.timezoneOffset = &timezoneOffset
-	return nil
+	return getLocation(setting.Value, config), nil
 }
 
 func getLocation(name string, config *GoogleCalendarConfig) *time.Location {
@@ -467,7 +503,10 @@ func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error {
 		return err
 	}
 	err = c.config.API.DeleteEvent(details.ID)
-	if err != nil {
+	switch {
+	case isAlreadyDeleted(err):
+		return nil
+	case err != nil:
 		return ctxerr.Wrap(c.config.Context, err, "deleting Google calendar event")
 	}
 	return nil
diff --git a/ee/server/calendar/google_calendar_integration_test.go b/ee/server/calendar/google_calendar_integration_test.go
new file mode 100644
index 0000000000..7f42b23b22
--- /dev/null
+++ b/ee/server/calendar/google_calendar_integration_test.go
@@ -0,0 +1,132 @@
+package calendar
+
+import (
+	"context"
+	"github.com/fleetdm/fleet/v4/ee/server/calendar/load_test"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	kitlog "github.com/go-kit/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+)
+
+type googleCalendarIntegrationTestSuite struct {
+	suite.Suite
+	server *httptest.Server
+	dbFile *os.File
+}
+
+func (s *googleCalendarIntegrationTestSuite) SetupSuite() {
+	dbFile, err := os.CreateTemp("", "calendar.db")
+	s.Require().NoError(err)
+	handler, err := calendartest.Configure(dbFile.Name())
+	s.Require().NoError(err)
+	server := httptest.NewUnstartedServer(handler)
+	server.Listener.Addr()
+	server.Start()
+	s.server = server
+}
+
+func (s *googleCalendarIntegrationTestSuite) TearDownSuite() {
+	if s.dbFile != nil {
+		s.dbFile.Close()
+		_ = os.Remove(s.dbFile.Name())
+	}
+	if s.server != nil {
+		s.server.Close()
+	}
+	calendartest.Close()
+}
+
+// TestGoogleCalendarIntegration tests should be able to be run in parallel, but this is not natively supported by suites: https://github.com/stretchr/testify/issues/187
+// There are workarounds that can be explored.
+func TestGoogleCalendarIntegration(t *testing.T) {
+	testingSuite := new(googleCalendarIntegrationTestSuite)
+	suite.Run(t, testingSuite)
+}
+
+func (s *googleCalendarIntegrationTestSuite) TestCreateGetDeleteEvent() {
+	t := s.T()
+	userEmail := "user1@example.com"
+	config := &GoogleCalendarConfig{
+		Context: context.Background(),
+		IntegrationConfig: &fleet.GoogleCalendarIntegration{
+			Domain: "example.com",
+			ApiKey: map[string]string{
+				"client_email": loadEmail,
+				"private_key":  s.server.URL,
+			},
+		},
+		Logger: kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stdout)),
+	}
+	gCal := NewGoogleCalendar(config)
+	err := gCal.Configure(userEmail)
+	require.NoError(t, err)
+	genBodyFn := func(bool) string {
+		return "Test event"
+	}
+	eventDate := time.Now().Add(48 * time.Hour)
+	event, err := gCal.CreateEvent(eventDate, genBodyFn)
+	require.NoError(t, err)
+	assert.Equal(t, startHour, event.StartTime.Hour())
+	assert.Equal(t, 0, event.StartTime.Minute())
+
+	eventRsp, updated, err := gCal.GetAndUpdateEvent(event, genBodyFn)
+	require.NoError(t, err)
+	assert.False(t, updated)
+	assert.Equal(t, event, eventRsp)
+
+	err = gCal.DeleteEvent(event)
+	assert.NoError(t, err)
+	// delete again
+	err = gCal.DeleteEvent(event)
+	assert.NoError(t, err)
+
+	// Try to get deleted event
+	eventRsp, updated, err = gCal.GetAndUpdateEvent(event, genBodyFn)
+	require.NoError(t, err)
+	assert.True(t, updated)
+	assert.NotEqual(t, event.StartTime.UTC().Truncate(24*time.Hour), eventRsp.StartTime.UTC().Truncate(24*time.Hour))
+}
+
+func (s *googleCalendarIntegrationTestSuite) TestFillUpCalendar() {
+	t := s.T()
+	userEmail := "user2@example.com"
+	config := &GoogleCalendarConfig{
+		Context: context.Background(),
+		IntegrationConfig: &fleet.GoogleCalendarIntegration{
+			Domain: "example.com",
+			ApiKey: map[string]string{
+				"client_email": loadEmail,
+				"private_key":  s.server.URL,
+			},
+		},
+		Logger: kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stdout)),
+	}
+	gCal := NewGoogleCalendar(config)
+	err := gCal.Configure(userEmail)
+	require.NoError(t, err)
+	genBodyFn := func(bool) string {
+		return "Test event"
+	}
+	eventDate := time.Now().Add(48 * time.Hour)
+	event, err := gCal.CreateEvent(eventDate, genBodyFn)
+	require.NoError(t, err)
+	assert.Equal(t, startHour, event.StartTime.Hour())
+	assert.Equal(t, 0, event.StartTime.Minute())
+
+	currentEventTime := event.StartTime
+	for i := 0; i < 20; i++ {
+		if !(currentEventTime.Hour() == endHour-1 && currentEventTime.Minute() == 30) {
+			currentEventTime = currentEventTime.Add(30 * time.Minute)
+		}
+		event, err = gCal.CreateEvent(eventDate, genBodyFn)
+		require.NoError(t, err)
+		assert.Equal(t, currentEventTime.UTC(), event.StartTime.UTC())
+	}
+
+}
diff --git a/ee/server/calendar/google_calendar_load.go b/ee/server/calendar/google_calendar_load.go
new file mode 100644
index 0000000000..8446af20c5
--- /dev/null
+++ b/ee/server/calendar/google_calendar_load.go
@@ -0,0 +1,234 @@
+package calendar
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
+	kitlog "github.com/go-kit/log"
+	"google.golang.org/api/calendar/v3"
+	"google.golang.org/api/googleapi"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+// GoogleCalendarLoadAPI is used for load testing.
+type GoogleCalendarLoadAPI struct {
+	Logger            kitlog.Logger
+	baseUrl           string
+	userToImpersonate string
+	ctx               context.Context
+	client            *http.Client
+}
+
+// Configure creates a new Google Calendar service using the provided credentials.
+func (lowLevelAPI *GoogleCalendarLoadAPI) Configure(ctx context.Context, _ string, privateKey string, userToImpersonate string) error {
+	if lowLevelAPI.Logger == nil {
+		lowLevelAPI.Logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stderr), "mock", "GoogleCalendarLoadAPI", "user", userToImpersonate)
+	}
+	lowLevelAPI.baseUrl = privateKey
+	lowLevelAPI.userToImpersonate = userToImpersonate
+	lowLevelAPI.ctx = ctx
+	if lowLevelAPI.client == nil {
+		lowLevelAPI.client = fleethttp.NewClient()
+	}
+	return nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) GetSetting(name string) (*calendar.Setting, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/settings")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("name", name)
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var setting calendar.Setting
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &setting)
+	if err != nil {
+		return nil, err
+	}
+	return &setting, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) {
+	body, err := json.Marshal(event)
+	if err != nil {
+		return nil, err
+	}
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/add")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "POST", reqUrl.String(), bytes.NewBuffer(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusCreated {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var rspEvent calendar.Event
+	body, err = io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &rspEvent)
+	if err != nil {
+		return nil, err
+	}
+	return &rspEvent, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) GetEvent(id, _ string) (*calendar.Event, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("id", id)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode == http.StatusNotFound {
+		return nil, &googleapi.Error{Code: http.StatusNotFound}
+	}
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var rspEvent calendar.Event
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &rspEvent)
+	if err != nil {
+		return nil, err
+	}
+	return &rspEvent, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) ListEvents(timeMin string, timeMax string) (*calendar.Events, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/list")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("timemin", timeMin)
+	query.Set("timemax", timeMax)
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var events calendar.Events
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &events)
+	if err != nil {
+		return nil, err
+	}
+	return &events, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) DeleteEvent(id string) error {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/delete")
+	if err != nil {
+		return err
+	}
+	query := reqUrl.Query()
+	query.Set("id", id)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "DELETE", reqUrl.String(), nil)
+	if err != nil {
+		return err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode == http.StatusGone {
+		return &googleapi.Error{Code: http.StatusGone}
+	}
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	return nil
+}
diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go
index a6d7d6040a..255f8d87c7 100644
--- a/ee/server/calendar/google_calendar_mock.go
+++ b/ee/server/calendar/google_calendar_mock.go
@@ -17,7 +17,7 @@ type GoogleCalendarMockAPI struct {
 	logger kitlog.Logger
 }
 
-var events = make(map[string]*calendar.Event)
+var mockEvents = make(map[string]*calendar.Event)
 var mu sync.Mutex
 var id uint64
 
@@ -49,7 +49,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*c
 	id += 1
 	event.Id = strconv.FormatUint(id, 10)
 	lowLevelAPI.logger.Log("msg", "CreateEvent", "id", event.Id, "start", event.Start.DateTime)
-	events[event.Id] = event
+	mockEvents[event.Id] = event
 	return event, nil
 }
 
@@ -57,7 +57,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) GetEvent(id, _ string) (*calendar.Even
 	time.Sleep(latency)
 	mu.Lock()
 	defer mu.Unlock()
-	event, ok := events[id]
+	event, ok := mockEvents[id]
 	if !ok {
 		return nil, &googleapi.Error{Code: http.StatusNotFound}
 	}
@@ -76,6 +76,6 @@ func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error {
 	mu.Lock()
 	defer mu.Unlock()
 	lowLevelAPI.logger.Log("msg", "DeleteEvent", "id", id)
-	delete(events, id)
+	delete(mockEvents, id)
 	return nil
 }
diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go
index ad5e1c89ca..02d024792e 100644
--- a/ee/server/calendar/google_calendar_test.go
+++ b/ee/server/calendar/google_calendar_test.go
@@ -84,6 +84,29 @@ func TestGoogleCalendar_Configure(t *testing.T) {
 	assert.ErrorIs(t, err, assert.AnError)
 }
 
+func TestGoogleCalendar_ConfigurePlusAddressing(t *testing.T) {
+	// Do not run this test in t.Parallel(), since it involves modifying a global variable
+	plusAddressing = true
+	t.Cleanup(
+		func() {
+			plusAddressing = false
+		},
+	)
+	email := "user+my_test+email@example.com"
+	mockAPI := &MockGoogleCalendarLowLevelAPI{}
+	mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error {
+		assert.Equal(t, baseCtx, ctx)
+		assert.Equal(t, baseServiceEmail, serviceAccountEmail)
+		assert.Equal(t, basePrivateKey, privateKey)
+		assert.Equal(t, "user@example.com", userToImpersonateEmail)
+		return nil
+	}
+
+	var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI))
+	err := cal.Configure(email)
+	assert.NoError(t, err)
+}
+
 func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig {
 	if mockAPI != nil && mockAPI.ConfigureFunc == nil {
 		mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error {
@@ -125,6 +148,13 @@ func TestGoogleCalendar_DeleteEvent(t *testing.T) {
 	}
 	err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)})
 	assert.ErrorIs(t, err, assert.AnError)
+
+	// Event already deleted
+	mockAPI.DeleteEventFunc = func(id string) error {
+		return &googleapi.Error{Code: http.StatusGone}
+	}
+	err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)})
+	assert.NoError(t, err)
 }
 
 func TestGoogleCalendar_unmarshalDetails(t *testing.T) {
diff --git a/ee/server/calendar/load_test/calendar_http_handler.go b/ee/server/calendar/load_test/calendar_http_handler.go
new file mode 100644
index 0000000000..e69c945523
--- /dev/null
+++ b/ee/server/calendar/load_test/calendar_http_handler.go
@@ -0,0 +1,343 @@
+// Package calendartest is not imported in production code, so it will not be compiled for Fleet server.
+package calendartest
+
+import (
+	"context"
+	"crypto/md5" //nolint:gosec // (only used in testing)
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	_ "github.com/mattn/go-sqlite3"
+	"google.golang.org/api/calendar/v3"
+	"hash/fnv"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+// This calendar does not support all-day events.
+
+var db *sql.DB
+var timezones = []string{
+	"America/Chicago",
+	"America/New_York",
+	"America/Los_Angeles",
+	"America/Anchorage",
+	"Pacific/Honolulu",
+	"America/Argentina/Buenos_Aires",
+	"Asia/Kolkata",
+	"Europe/London",
+	"Europe/Paris",
+	"Australia/Sydney",
+}
+
+func Configure(dbPath string) (http.Handler, error) {
+	var err error
+	db, err = sql.Open("sqlite3", dbPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	logger := log.New(os.Stdout, "", log.LstdFlags)
+	logger.Println("Server is starting...")
+
+	// Initialize the database schema if needed
+	err = initializeSchema()
+	if err != nil {
+		return nil, err
+	}
+
+	router := http.NewServeMux()
+	router.HandleFunc("/settings", getSetting)
+	router.HandleFunc("/events", getEvent)
+	router.HandleFunc("/events/list", getEvents)
+	router.HandleFunc("/events/add", addEvent)
+	router.HandleFunc("/events/delete", deleteEvent)
+	return logging(logger)(router), nil
+}
+
+func logging(logger *log.Logger) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(
+			func(w http.ResponseWriter, r *http.Request) {
+				defer func() {
+					logger.Println(r.Method, r.URL.String(), r.RemoteAddr)
+				}()
+				next.ServeHTTP(w, r)
+			},
+		)
+	}
+}
+
+func Close() {
+	_ = db.Close()
+}
+
+func getSetting(w http.ResponseWriter, r *http.Request) {
+	name := r.URL.Query().Get("name")
+	if name == "" {
+		http.Error(w, "missing name", http.StatusBadRequest)
+		return
+	}
+	if name != "timezone" {
+		http.Error(w, "unsupported setting", http.StatusNotFound)
+		return
+	}
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	timezone := getTimezone(email)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	setting := calendar.Setting{Value: timezone}
+	err := json.NewEncoder(w).Encode(setting)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// The timezone is determined by the user's email address
+func getTimezone(email string) string {
+	index := hash(email) % uint32(len(timezones))
+	timezone := timezones[index]
+	return timezone
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// getEvent handles GET /events?id=123
+func getEvent(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("id")
+	if id == "" {
+		http.Error(w, "missing id", http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "SELECT email, start, end, summary, description, status FROM events WHERE id = ?"
+	var start, end int64
+	var email, summary, description, status string
+	err := db.QueryRow(sqlStmt, id).Scan(&email, &start, &end, &summary, &description, &status)
+	if errors.Is(err, sql.ErrNoRows) {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	timezone := getTimezone(email)
+	loc, err := time.LoadLocation(timezone)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	calEvent := calendar.Event{}
+	calEvent.Id = id
+	calEvent.Start = &calendar.EventDateTime{DateTime: time.Unix(start, 0).In(loc).Format(time.RFC3339)}
+	calEvent.End = &calendar.EventDateTime{DateTime: time.Unix(end, 0).In(loc).Format(time.RFC3339)}
+	calEvent.Summary = summary
+	calEvent.Description = description
+	calEvent.Status = status
+	calEvent.Etag = computeETag(start, end, summary, description, status)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(w).Encode(calEvent)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func getEvents(w http.ResponseWriter, r *http.Request) {
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	timeMin := r.URL.Query().Get("timemin")
+	if email == "" {
+		http.Error(w, "missing timemin", http.StatusBadRequest)
+		return
+	}
+	timeMax := r.URL.Query().Get("timemax")
+	if email == "" {
+		http.Error(w, "missing timemax", http.StatusBadRequest)
+		return
+	}
+	minTime, err := parseDateTime(r.Context(), &calendar.EventDateTime{DateTime: timeMin})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	maxTime, err := parseDateTime(r.Context(), &calendar.EventDateTime{DateTime: timeMax})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "SELECT id, start, end, summary, description, status FROM events WHERE email = ? AND end > ? AND start < ?"
+	rows, err := db.Query(sqlStmt, email, minTime.Unix(), maxTime.Unix())
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer rows.Close()
+	timezone := getTimezone(email)
+	loc, err := time.LoadLocation(timezone)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	events := calendar.Events{}
+	events.Items = make([]*calendar.Event, 0)
+	for rows.Next() {
+		var id, start, end int64
+		var summary, description, status string
+		err = rows.Scan(&id, &start, &end, &summary, &description, &status)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		calEvent := calendar.Event{}
+		calEvent.Id = fmt.Sprintf("%d", id)
+		calEvent.Start = &calendar.EventDateTime{DateTime: time.Unix(start, 0).In(loc).Format(time.RFC3339)}
+		calEvent.End = &calendar.EventDateTime{DateTime: time.Unix(end, 0).In(loc).Format(time.RFC3339)}
+		calEvent.Summary = summary
+		calEvent.Description = description
+		calEvent.Status = status
+		calEvent.Etag = computeETag(start, end, summary, description, status)
+		events.Items = append(events.Items, &calEvent)
+	}
+	if err = rows.Err(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(w).Encode(events)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// addEvent handles POST /events/add?email=user@example.com
+func addEvent(w http.ResponseWriter, r *http.Request) {
+	var event calendar.Event
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	err = json.Unmarshal(body, &event)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	start, err := parseDateTime(r.Context(), event.Start)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	end, err := parseDateTime(r.Context(), event.End)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	status := "confirmed"
+	sqlStmt := `INSERT INTO events (email, start, end, summary, description, status) VALUES (?, ?, ?, ?, ?, ?)`
+	result, err := db.Exec(sqlStmt, email, start.Unix(), end.Unix(), event.Summary, event.Description, status)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	event.Id = fmt.Sprintf("%d", id)
+	event.Etag = computeETag(start.Unix(), end.Unix(), event.Summary, event.Description, status)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusCreated)
+	err = json.NewEncoder(w).Encode(event)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func computeETag(args ...any) string {
+	h := md5.New() //nolint:gosec // (only used for tests)
+	_, _ = fmt.Fprint(h, args...)
+	checksum := h.Sum(nil)
+	return hex.EncodeToString(checksum)
+}
+
+// deleteEvent handles DELETE /events/delete?id=123
+func deleteEvent(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("id")
+	if id == "" {
+		http.Error(w, "missing id", http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "DELETE FROM events WHERE id = ?"
+	_, err := db.Exec(sqlStmt, id)
+	if errors.Is(err, sql.ErrNoRows) {
+		http.Error(w, "not found", http.StatusGone)
+		return
+	}
+}
+
+func initializeSchema() error {
+	createTableSQL := `CREATE TABLE IF NOT EXISTS events (
+		"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+		"email" TEXT NOT NULL,
+		"start" INTEGER NOT NULL,
+		"end" INTEGER NOT NULL,
+		"summary" TEXT NOT NULL,
+		"description" TEXT NOT NULL,
+		"status" TEXT NOT NULL
+	);`
+	_, err := db.Exec(createTableSQL)
+	if err != nil {
+		return fmt.Errorf("failed to create table: %w", err)
+	}
+	return nil
+}
+
+func parseDateTime(ctx context.Context, eventDateTime *calendar.EventDateTime) (*time.Time, error) {
+	var t time.Time
+	var err error
+	if eventDateTime.TimeZone != "" {
+		var loc *time.Location
+		loc, err = time.LoadLocation(eventDateTime.TimeZone)
+		if err == nil {
+			t, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
+		}
+	} else {
+		t, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
+	}
+	if err != nil {
+		return nil, ctxerr.Wrap(
+			ctx, err, fmt.Sprintf("parsing calendar event time: %s", eventDateTime.DateTime),
+		)
+	}
+	return &t, nil
+}
diff --git a/tools/calendar/README.md b/tools/calendar/README.md
new file mode 100644
index 0000000000..5d222e45de
--- /dev/null
+++ b/tools/calendar/README.md
@@ -0,0 +1,26 @@
+# Calendar server for load testing
+
+Test calendar server that provides a REST API for managing events.
+Since we may not have access to a real calendar server (such as Google Calendar API), this server will be used to test the calendar feature during load testing.
+
+Start the server like:
+```shell
+go run calendar.go --port 8083 --db ./calendar.db
+```
+
+The server uses a SQLite database to store events. This database can be modified during testing.
+
+On the fleet server, configure Google Calendar API key where `client_email` is the specified value and the `private_key` is the base URL of the calendar server:
+```json
+{
+    "client_email": "calendar-load@example.com",
+    "private_key": "http://localhost:8083"
+}
+```
+
+## Useful tricks
+
+To update all the events in SQLite database to start at the current time, do SQL query:
+```sql
+UPDATE events SET start = unixepoch('now'), end = unixepoch('now', '+30 minutes');
+```
diff --git a/tools/calendar/calendar.go b/tools/calendar/calendar.go
new file mode 100644
index 0000000000..64b70c7a94
--- /dev/null
+++ b/tools/calendar/calendar.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	calendartest "github.com/fleetdm/fleet/v4/ee/server/calendar/load_test"
+	_ "github.com/mattn/go-sqlite3"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+func main() {
+	port := flag.Uint("port", 8083, "Port to listen on")
+	dbFileName := flag.String("db", "./calendar.db", "SQLite db file name")
+	flag.Parse()
+
+	handler, err := calendartest.Configure(*dbFileName)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer calendartest.Close()
+
+	listenAddr := fmt.Sprintf(":%d", *port)
+	errLogger := log.New(os.Stderr, "", log.LstdFlags)
+
+	server := &http.Server{
+		Addr:         listenAddr,
+		Handler:      handler,
+		ErrorLog:     errLogger,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  15 * time.Second,
+	}
+
+	// Start the HTTP server
+	log.Fatal(server.ListenAndServe())
+}

From 31fe9d17b97c407f41e295876b4831475494c9ba Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Fri, 22 Mar 2024 11:20:18 -0300
Subject: [PATCH 18/30] More fixes to support users with hosts in same team and
 hosts in different teams (#17789)

#17441
---
 cmd/fleet/calendar_cron.go                |  60 +++++--
 server/datastore/mysql/calendar_events.go |  24 +++
 server/datastore/mysql/policies.go        |  21 ++-
 server/datastore/mysql/policies_test.go   | 200 ++++++++++++++++++++++
 server/fleet/datastore.go                 |   3 +-
 server/mock/datastore_mock.go             |  24 ++-
 6 files changed, 306 insertions(+), 26 deletions(-)

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index fa63b487dc..17962f1847 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -125,7 +125,7 @@ func cronCalendarEventsForTeam(
 	for _, policy := range policies {
 		policyIDs = append(policyIDs, policy.ID)
 	}
-	hosts, err := ds.GetHostsPolicyMemberships(ctx, domain, policyIDs)
+	hosts, err := ds.GetTeamHostsPolicyMemberships(ctx, domain, team.ID, policyIDs)
 	if err != nil {
 		return fmt.Errorf("get team hosts failing policies: %w", err)
 	}
@@ -150,22 +150,28 @@ func cronCalendarEventsForTeam(
 	}
 	level.Debug(logger).Log(
 		"msg", "summary",
+		"team_id", team.ID,
 		"passing_hosts", len(passingHosts),
 		"failing_hosts", len(failingHosts),
 		"failing_hosts_without_associated_email", len(failingHostsWithoutAssociatedEmail),
 	)
 
+	// Remove calendar events from hosts that are passing the calendar policies.
+	//
+	// We execute this first to remove any calendar events for a user that is now passing
+	// policies on one of its hosts, and possibly create a new calendar event if they have
+	// another failing host on the same team.
+	if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil {
+		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
+	}
+
+	// Process hosts that are failing calendar policies.
 	if err := processCalendarFailingHosts(
 		ctx, ds, calendar, orgName, failingHosts, logger,
 	); err != nil {
 		level.Info(logger).Log("msg", "processing failing hosts", "err", err)
 	}
 
-	// Remove calendar events from hosts that are passing the policies.
-	if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil {
-		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
-	}
-
 	// At last we want to log the hosts that are failing and don't have an associated email.
 	logHostsWithoutAssociatedEmail(
 		domain,
@@ -184,14 +190,26 @@ func processCalendarFailingHosts(
 	hosts []fleet.HostPolicyMembershipData,
 	logger kitlog.Logger,
 ) error {
+	hosts = filterHostsWithSameEmail(hosts)
+
 	for _, host := range hosts {
 		logger := log.With(logger, "host_id", host.HostID)
 
-		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 
 		expiredEvent := false
 		webhookAlreadyFiredThisMonth := false
 		if err == nil {
+			if hostCalendarEvent.HostID != host.HostID {
+				// This calendar event belongs to another host with this associated email,
+				// thus we skip this entry.
+				continue // continue with next host
+			}
+			if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending {
+				// This can happen if the host went offline (and never returned results)
+				// after setting the webhook as pending.
+				continue // continue with next host
+			}
 			now := time.Now()
 			webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent
 			if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) {
@@ -200,7 +218,7 @@ func processCalendarFailingHosts(
 				continue // continue with next host
 			}
 			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
-			if calendarEvent.EndTime.Before(time.Now()) {
+			if calendarEvent.EndTime.Before(now) {
 				expiredEvent = true
 			}
 		}
@@ -232,6 +250,25 @@ func processCalendarFailingHosts(
 	return nil
 }
 
+func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData {
+	minHostPerEmail := make(map[string]fleet.HostPolicyMembershipData)
+	for _, host := range hosts {
+		minHost, ok := minHostPerEmail[host.Email]
+		if !ok {
+			minHostPerEmail[host.Email] = host
+			continue
+		}
+		if host.HostID < minHost.HostID {
+			minHostPerEmail[host.Email] = host
+		}
+	}
+	filtered := make([]fleet.HostPolicyMembershipData, 0, len(minHostPerEmail))
+	for _, host := range minHostPerEmail {
+		filtered = append(filtered, host)
+	}
+	return filtered
+}
+
 func processFailingHostExistingCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
@@ -416,10 +453,13 @@ func removeCalendarEventsFromPassingHosts(
 	hosts []fleet.HostPolicyMembershipData,
 ) error {
 	for _, host := range hosts {
-		calendarEvent, err := ds.GetCalendarEvent(ctx, host.Email)
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 		switch {
 		case err == nil:
-			// OK
+			if hostCalendarEvent.HostID != host.HostID {
+				// This calendar event belongs to another host, thus we skip this entry.
+				continue
+			}
 		case fleet.IsNotFound(err):
 			continue
 		default:
diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go
index 5ffc0f77f3..45d8d88331 100644
--- a/server/datastore/mysql/calendar_events.go
+++ b/server/datastore/mysql/calendar_events.go
@@ -167,6 +167,30 @@ func (ds *Datastore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fl
 	return &hostCalendarEvent, &calendarEvent, nil
 }
 
+func (ds *Datastore) GetHostCalendarEventByEmail(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) {
+	const calendarEventsQuery = `
+		SELECT * FROM calendar_events WHERE email = ?
+	`
+	var calendarEvent fleet.CalendarEvent
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, email); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithMessage(fmt.Sprintf("email: %s", email)))
+		}
+		return nil, nil, ctxerr.Wrap(ctx, err, "get calendar event")
+	}
+	const hostCalendarEventsQuery = `
+		SELECT * FROM host_calendar_events WHERE calendar_event_id = ?
+	`
+	var hostCalendarEvent fleet.HostCalendarEvent
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostCalendarEvent, hostCalendarEventsQuery, calendarEvent.ID); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, ctxerr.Wrap(ctx, notFound("HostCalendarEvent").WithID(calendarEvent.ID))
+		}
+		return nil, nil, ctxerr.Wrap(ctx, err, "get host calendar event")
+	}
+	return &hostCalendarEvent, &calendarEvent, nil
+}
+
 func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error {
 	const calendarEventsQuery = `
 		UPDATE host_calendar_events SET
diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go
index b711c8b932..71530961de 100644
--- a/server/datastore/mysql/policies.go
+++ b/server/datastore/mysql/policies.go
@@ -1172,8 +1172,12 @@ func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fl
 }
 
 // TODO(lucas): Must be tested at scale.
-// TODO(lucas): Filter out hosts with team_id == NULL
-func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
+func (ds *Datastore) GetTeamHostsPolicyMemberships(
+	ctx context.Context,
+	domain string,
+	teamID uint,
+	policyIDs []uint,
+) ([]fleet.HostPolicyMembershipData, error) {
 	query := `
 	SELECT 
 		COALESCE(sh.email, '') AS email,
@@ -1188,18 +1192,17 @@ func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain strin
 		GROUP BY host_id
 	) pm
 	LEFT JOIN (
-		SELECT MIN(h.host_id) as host_id, h.email as email
-		FROM (
-			SELECT host_id, MIN(email) AS email
-			FROM host_emails WHERE email LIKE CONCAT('%@', ?)
-			GROUP BY host_id
-		) h GROUP BY h.email
+		SELECT host_id, MIN(email) AS email
+		FROM host_emails
+		JOIN hosts ON host_emails.host_id=hosts.id
+		WHERE email LIKE CONCAT('%@', ?) AND team_id = ? 
+		GROUP BY host_id
 	) sh ON sh.host_id = pm.host_id
 	JOIN hosts h ON h.id = pm.host_id
 	LEFT JOIN host_display_names hdn ON hdn.host_id = pm.host_id;
 `
 
-	query, args, err := sqlx.In(query, policyIDs, domain)
+	query, args, err := sqlx.In(query, policyIDs, domain, teamID)
 	if err != nil {
 		return nil, ctxerr.Wrapf(ctx, err, "build select get team hosts policy memberships query")
 	}
diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go
index 514de6dd38..15ebeee171 100644
--- a/server/datastore/mysql/policies_test.go
+++ b/server/datastore/mysql/policies_test.go
@@ -60,6 +60,7 @@ func TestPolicies(t *testing.T) {
 		{"TestPoliciesNameEmoji", testPoliciesNameEmoji},
 		{"TestPoliciesNameSort", testPoliciesNameSort},
 		{"TestGetCalendarPolicies", testGetCalendarPolicies},
+		{"GetTeamHostsPolicyMemberships", testGetTeamHostsPolicyMemberships},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -2860,3 +2861,202 @@ func testGetCalendarPolicies(t *testing.T, ds *Datastore) {
 	require.Equal(t, calendarPolicies[0].ID, teamPolicy2.ID)
 	require.Equal(t, calendarPolicies[1].ID, teamPolicy3.ID)
 }
+
+func testGetTeamHostsPolicyMemberships(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1"})
+	require.NoError(t, err)
+	team2, err := ds.NewTeam(ctx, &fleet.Team{Name: "team2"})
+	require.NoError(t, err)
+
+	team1Policy1, err := ds.NewTeamPolicy(ctx, team1.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 1 Policy 1",
+		Query:                 "SELECT * FROM osquery_info;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+	team1Policy2, err := ds.NewTeamPolicy(ctx, team1.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 1 Policy 2",
+		Query:                 "SELECT * FROM system_info;",
+		CalendarEventsEnabled: false,
+	})
+	require.NoError(t, err)
+	team2Policy1, err := ds.NewTeamPolicy(ctx, team2.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 2 Policy 1",
+		Query:                 "SELECT * FROM os_version;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+	team2Policy2, err := ds.NewTeamPolicy(ctx, team2.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 2 Policy 2",
+		Query:                 "SELECT * FROM processes;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+
+	// Empty teams.
+	hostsTeam1, err := ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policy1.ID, team1Policy2.ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 0)
+
+	host1, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host1"),
+		NodeKey:        ptr.String("host1"),
+		HardwareSerial: "serial1",
+		ComputerName:   "display_name1",
+		TeamID:         &team1.ID,
+	})
+	require.NoError(t, err)
+	host2, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host2"),
+		NodeKey:        ptr.String("host2"),
+		HardwareSerial: "serial2",
+		ComputerName:   "display_name2",
+		TeamID:         &team2.ID,
+	})
+	require.NoError(t, err)
+	host3, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host3"),
+		NodeKey:        ptr.String("host3"),
+		HardwareSerial: "serial3",
+		ComputerName:   "display_name3",
+		TeamID:         &team2.ID,
+	})
+	require.NoError(t, err)
+	host4, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host4"),
+		NodeKey:        ptr.String("host4"),
+		HardwareSerial: "serial4",
+		ComputerName:   "display_name4",
+	})
+	require.NoError(t, err)
+	host5, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host5"),
+		NodeKey:        ptr.String("host5"),
+		HardwareSerial: "serial5",
+		ComputerName:   "display_name5",
+		TeamID:         &team1.ID,
+	})
+	require.NoError(t, err)
+
+	// No policy results yet.
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policy1.ID, team1Policy2.ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 0)
+
+	err = ds.ReplaceHostDeviceMapping(ctx, host1.ID, []*fleet.HostDeviceMapping{
+		{HostID: host1.ID, Email: "foo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host1.ID, []*fleet.HostDeviceMapping{
+		{HostID: host1.ID, Email: "zoo@example.com", Source: "custom"},
+	}, "custom")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host2.ID, []*fleet.HostDeviceMapping{
+		{HostID: host2.ID, Email: "foo@example.com", Source: "custom"},
+	}, "custom")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host2.ID, []*fleet.HostDeviceMapping{
+		{HostID: host2.ID, Email: "foo@other.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host3.ID, []*fleet.HostDeviceMapping{
+		{HostID: host3.ID, Email: "zoo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host4.ID, []*fleet.HostDeviceMapping{
+		{HostID: host4.ID, Email: "foo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host5.ID, []*fleet.HostDeviceMapping{
+		{HostID: host5.ID, Email: "foo@other.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host1, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(true),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{
+		team2Policy1.ID: ptr.Bool(false),
+		team2Policy2.ID: ptr.Bool(true),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host3, map[uint]*bool{
+		team2Policy1.ID: ptr.Bool(true),
+		team2Policy2.ID: ptr.Bool(true),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host5, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(false),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	team1Policies, err := ds.GetCalendarPolicies(ctx, team1.ID)
+	require.NoError(t, err)
+	require.Len(t, team1Policies, 1)
+	team2Policies, err := ds.GetCalendarPolicies(ctx, team2.ID)
+	require.NoError(t, err)
+	require.Len(t, team2Policies, 2)
+
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policies[0].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 2)
+	require.Equal(t, host1.ID, hostsTeam1[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[0].Email)
+	require.True(t, hostsTeam1[0].Passing)
+	require.Equal(t, "serial1", hostsTeam1[0].HostHardwareSerial)
+	require.Equal(t, "display_name1", hostsTeam1[0].HostDisplayName)
+	require.Equal(t, host5.ID, hostsTeam1[1].HostID)
+	require.Empty(t, hostsTeam1[1].Email)
+	require.False(t, hostsTeam1[1].Passing)
+	require.Equal(t, "serial5", hostsTeam1[1].HostHardwareSerial)
+	require.Equal(t, "display_name5", hostsTeam1[1].HostDisplayName)
+
+	err = ds.AddHostsToTeam(ctx, &team1.ID, []uint{host4.ID})
+	require.NoError(t, err)
+	err = ds.RecordPolicyQueryExecutions(ctx, host4, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(false),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policies[0].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 3)
+	require.Equal(t, host1.ID, hostsTeam1[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[0].Email)
+	require.True(t, hostsTeam1[0].Passing)
+	require.Equal(t, "serial1", hostsTeam1[0].HostHardwareSerial)
+	require.Equal(t, "display_name1", hostsTeam1[0].HostDisplayName)
+	require.Equal(t, host4.ID, hostsTeam1[1].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[1].Email)
+	require.False(t, hostsTeam1[1].Passing)
+	require.Equal(t, "serial4", hostsTeam1[1].HostHardwareSerial)
+	require.Equal(t, "display_name4", hostsTeam1[1].HostDisplayName)
+	require.Equal(t, host5.ID, hostsTeam1[2].HostID)
+	require.Empty(t, hostsTeam1[2].Email)
+	require.False(t, hostsTeam1[2].Passing)
+	require.Equal(t, "serial5", hostsTeam1[2].HostHardwareSerial)
+	require.Equal(t, "display_name5", hostsTeam1[2].HostDisplayName)
+
+	hostsTeam2, err := ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team2.ID, []uint{team2Policies[0].ID, team2Policies[1].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam2, 2)
+	require.Equal(t, host2.ID, hostsTeam2[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam2[0].Email)
+	require.False(t, hostsTeam2[0].Passing)
+	require.Equal(t, "serial2", hostsTeam2[0].HostHardwareSerial)
+	require.Equal(t, "display_name2", hostsTeam2[0].HostDisplayName)
+	require.Equal(t, host3.ID, hostsTeam2[1].HostID)
+	require.Equal(t, "zoo@example.com", hostsTeam2[1].Email)
+	require.True(t, hostsTeam2[1].Passing)
+	require.Equal(t, "serial3", hostsTeam2[1].HostHardwareSerial)
+	require.Equal(t, "display_name3", hostsTeam2[1].HostDisplayName)
+}
diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go
index 65098efdfc..a2f8bf6cdd 100644
--- a/server/fleet/datastore.go
+++ b/server/fleet/datastore.go
@@ -594,7 +594,7 @@ type Datastore interface {
 
 	PolicyQueriesForHost(ctx context.Context, host *Host) (map[string]string, error)
 
-	GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]HostPolicyMembershipData, error)
+	GetTeamHostsPolicyMemberships(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]HostPolicyMembershipData, error)
 	GetCalendarPolicies(ctx context.Context, teamID uint) ([]PolicyCalendarData, error)
 
 	// Methods used for async processing of host policy query results.
@@ -624,6 +624,7 @@ type Datastore interface {
 	DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error
 	UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error
 	GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error)
+	GetHostCalendarEventByEmail(ctx context.Context, email string) (*HostCalendarEvent, *CalendarEvent, error)
 	UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error
 	ListCalendarEvents(ctx context.Context, teamID *uint) ([]*CalendarEvent, error)
 	ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*CalendarEvent, error)
diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go
index 1b77b29cbe..425e0945e7 100644
--- a/server/mock/datastore_mock.go
+++ b/server/mock/datastore_mock.go
@@ -440,7 +440,7 @@ type UpdateHostPolicyCountsFunc func(ctx context.Context) error
 
 type PolicyQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error)
 
-type GetHostsPolicyMembershipsFunc func(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error)
+type GetTeamHostsPolicyMembershipsFunc func(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error)
 
 type GetCalendarPoliciesFunc func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error)
 
@@ -472,6 +472,8 @@ type UpdateCalendarEventFunc func(ctx context.Context, calendarEventID uint, sta
 
 type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error)
 
+type GetHostCalendarEventByEmailFunc func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error)
+
 type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error
 
 type ListCalendarEventsFunc func(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error)
@@ -1512,8 +1514,8 @@ type DataStore struct {
 	PolicyQueriesForHostFunc        PolicyQueriesForHostFunc
 	PolicyQueriesForHostFuncInvoked bool
 
-	GetHostsPolicyMembershipsFunc        GetHostsPolicyMembershipsFunc
-	GetHostsPolicyMembershipsFuncInvoked bool
+	GetTeamHostsPolicyMembershipsFunc        GetTeamHostsPolicyMembershipsFunc
+	GetTeamHostsPolicyMembershipsFuncInvoked bool
 
 	GetCalendarPoliciesFunc        GetCalendarPoliciesFunc
 	GetCalendarPoliciesFuncInvoked bool
@@ -1560,6 +1562,9 @@ type DataStore struct {
 	GetHostCalendarEventFunc        GetHostCalendarEventFunc
 	GetHostCalendarEventFuncInvoked bool
 
+	GetHostCalendarEventByEmailFunc        GetHostCalendarEventByEmailFunc
+	GetHostCalendarEventByEmailFuncInvoked bool
+
 	UpdateHostCalendarWebhookStatusFunc        UpdateHostCalendarWebhookStatusFunc
 	UpdateHostCalendarWebhookStatusFuncInvoked bool
 
@@ -3649,11 +3654,11 @@ func (s *DataStore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host)
 	return s.PolicyQueriesForHostFunc(ctx, host)
 }
 
-func (s *DataStore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
+func (s *DataStore) GetTeamHostsPolicyMemberships(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
 	s.mu.Lock()
-	s.GetHostsPolicyMembershipsFuncInvoked = true
+	s.GetTeamHostsPolicyMembershipsFuncInvoked = true
 	s.mu.Unlock()
-	return s.GetHostsPolicyMembershipsFunc(ctx, domain, policyIDs)
+	return s.GetTeamHostsPolicyMembershipsFunc(ctx, domain, teamID, policyIDs)
 }
 
 func (s *DataStore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) {
@@ -3761,6 +3766,13 @@ func (s *DataStore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fle
 	return s.GetHostCalendarEventFunc(ctx, hostID)
 }
 
+func (s *DataStore) GetHostCalendarEventByEmail(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.GetHostCalendarEventByEmailFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetHostCalendarEventByEmailFunc(ctx, email)
+}
+
 func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error {
 	s.mu.Lock()
 	s.UpdateHostCalendarWebhookStatusFuncInvoked = true

From c6e2e8d6c42493ab985fc5a2aac07c9609cf1f75 Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Fri, 22 Mar 2024 14:16:08 -0300
Subject: [PATCH 19/30] Always create event next 3rd Tuesday (#17799)

Fix to always create events for next 3rd Tuesday #17441
---
 cmd/fleet/calendar_cron.go      |  32 +++------
 cmd/fleet/calendar_cron_test.go | 113 +++++++++++++++++++-------------
 2 files changed, 77 insertions(+), 68 deletions(-)

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index 17962f1847..238260ea3a 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -198,7 +198,6 @@ func processCalendarFailingHosts(
 		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 
 		expiredEvent := false
-		webhookAlreadyFiredThisMonth := false
 		if err == nil {
 			if hostCalendarEvent.HostID != host.HostID {
 				// This calendar event belongs to another host with this associated email,
@@ -217,7 +216,6 @@ func processCalendarFailingHosts(
 				// we give a grace period of one day for the host before we schedule a new event.
 				continue // continue with next host
 			}
-			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
 			if calendarEvent.EndTime.Before(now) {
 				expiredEvent = true
 			}
@@ -237,7 +235,7 @@ func processCalendarFailingHosts(
 			}
 		case fleet.IsNotFound(err) || expiredEvent:
 			if err := processFailingHostCreateCalendarEvent(
-				ctx, ds, userCalendar, orgName, host, webhookAlreadyFiredThisMonth,
+				ctx, ds, userCalendar, orgName, host,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host create calendar event", "err", err)
 				continue // continue with next host
@@ -357,21 +355,14 @@ func sameDate(t1 time.Time, t2 time.Time) bool {
 	return y1 == y2 && m1 == m2 && d1 == d2
 }
 
-func sameMonth(t1 time.Time, t2 time.Time) bool {
-	y1, m1, _ := t1.Date()
-	y2, m2, _ := t2.Date()
-	return y1 == y2 && m1 == m2
-}
-
 func processFailingHostCreateCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
 	userCalendar fleet.UserCalendar,
 	orgName string,
 	host fleet.HostPolicyMembershipData,
-	webhookAlreadyFiredThisMonth bool,
 ) error {
-	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar, webhookAlreadyFiredThisMonth)
+	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar)
 	if err != nil {
 		return fmt.Errorf("create event on user calendar: %w", err)
 	}
@@ -385,10 +376,9 @@ func attemptCreatingEventOnUserCalendar(
 	orgName string,
 	host fleet.HostPolicyMembershipData,
 	userCalendar fleet.UserCalendar,
-	webhookAlreadyFiredThisMonth bool,
 ) (*fleet.CalendarEvent, error) {
 	year, month, today := time.Now().Date()
-	preferredDate := getPreferredCalendarEventDate(year, month, today, webhookAlreadyFiredThisMonth)
+	preferredDate := getPreferredCalendarEventDate(year, month, today)
 	for {
 		calendarEvent, err := userCalendar.CreateEvent(
 			preferredDate, func(conflict bool) string {
@@ -408,10 +398,7 @@ func attemptCreatingEventOnUserCalendar(
 	}
 }
 
-func getPreferredCalendarEventDate(
-	year int, month time.Month, today int,
-	webhookAlreadyFired bool,
-) time.Time {
+func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time {
 	const (
 		// 3rd Tuesday of Month
 		preferredWeekDay = time.Tuesday
@@ -425,12 +412,13 @@ func getPreferredCalendarEventDate(
 	}
 	preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1)))
 	if today > preferredDate.Day() {
-		today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC)
-		if webhookAlreadyFired {
-			nextMonth := today_.AddDate(0, 1, 0) // move to next month
-			return getPreferredCalendarEventDate(nextMonth.Year(), nextMonth.Month(), 1, false)
+		// We are past the preferred date, so we move to next month and calculate again.
+		month := month + 1
+		if month == 13 {
+			month = 1
+			year += 1
 		}
-		preferredDate = addBusinessDay(today_)
+		return getPreferredCalendarEventDate(year, month, 1)
 	}
 	return preferredDate
 }
diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go
index 905b79c879..84f5ece527 100644
--- a/cmd/fleet/calendar_cron_test.go
+++ b/cmd/fleet/calendar_cron_test.go
@@ -12,72 +12,93 @@ func TestGetPreferredCalendarEventDate(t *testing.T) {
 		return time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
 	}
 	for _, tc := range []struct {
-		name                  string
-		year                  int
-		month                 time.Month
-		daysStart             int
-		daysEnd               int
-		webhookFiredThisMonth bool
+		name      string
+		year      int
+		month     time.Month
+		daysStart int
+		daysEnd   int
 
 		expected time.Time
 	}{
 		{
-			name:                  "March 2024 (webhook hasn't fired)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             1,
-			daysEnd:               31,
-			webhookFiredThisMonth: false,
+			name:      "March 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     3,
+			daysStart: 1,
+			daysEnd:   19,
 
 			expected: date(2024, 3, 19),
 		},
 		{
-			name:                  "March 2024 (webhook has fired, days before 3rd Tuesday)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             1,
-			daysEnd:               18,
-			webhookFiredThisMonth: true,
-
-			expected: date(2024, 3, 19),
-		},
-		{
-			name:                  "March 2024 (webhook has fired, days after 3rd Tuesday)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             20,
-			daysEnd:               30,
-			webhookFiredThisMonth: true,
+			name:      "March 2024 (past 3rd Tuesday)",
+			year:      2024,
+			month:     3,
+			daysStart: 20,
+			daysEnd:   31,
 
 			expected: date(2024, 4, 16),
 		},
 		{
-			name:                  "April 2024 (webhook hasn't fired)",
-			year:                  2024,
-			month:                 4,
-			daysEnd:               30,
-			webhookFiredThisMonth: false,
+			name:      "April 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     4,
+			daysStart: 1,
+			daysEnd:   16,
 
 			expected: date(2024, 4, 16),
 		},
+		{
+			name:      "April 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     4,
+			daysStart: 17,
+			daysEnd:   30,
+
+			expected: date(2024, 5, 21),
+		},
+		{
+			name:      "May 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     5,
+			daysStart: 1,
+			daysEnd:   21,
+
+			expected: date(2024, 5, 21),
+		},
+		{
+			name:      "May 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     5,
+			daysStart: 22,
+			daysEnd:   31,
+
+			expected: date(2024, 6, 18),
+		},
+		{
+			name:      "Dec 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     12,
+			daysStart: 1,
+			daysEnd:   17,
+
+			expected: date(2024, 12, 17),
+		},
+		{
+			name:      "Dec 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     12,
+			daysStart: 18,
+			daysEnd:   31,
+
+			expected: date(2025, 1, 21),
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			for day := tc.daysStart; day <= tc.daysEnd; day++ {
-				actual := getPreferredCalendarEventDate(tc.year, tc.month, day, tc.webhookFiredThisMonth)
+				actual := getPreferredCalendarEventDate(tc.year, tc.month, day)
 				require.NotEqual(t, actual.Weekday(), time.Saturday)
 				require.NotEqual(t, actual.Weekday(), time.Sunday)
-				if day <= tc.expected.Day() || tc.webhookFiredThisMonth {
-					require.Equal(t, tc.expected, actual)
-				} else {
-					today := date(tc.year, tc.month, day)
-					if weekday := today.Weekday(); weekday == time.Friday {
-						require.Equal(t, today.AddDate(0, 0, +3), actual)
-					} else if weekday == time.Saturday {
-						require.Equal(t, today.AddDate(0, 0, +2), actual)
-					} else {
-						require.Equal(t, today.AddDate(0, 0, +1), actual)
-					}
-				}
+				require.Equal(t, tc.expected, actual)
 			}
 		})
 	}

From fbb271caeefb242c182bad2d51163adbedc174b1 Mon Sep 17 00:00:00 2001
From: RachelElysia <71795832+RachelElysia@users.noreply.github.com>
Date: Fri, 22 Mar 2024 14:52:54 -0400
Subject: [PATCH 20/30] Fleet UI: Calendar settings iterations (#17779)

---
 .../cards/Calendars/Calendars.tsx             | 33 +++++++++++++++++--
 .../cards/Calendars/_styles.scss              |  5 +++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
index de7c79a139..da22ea4801 100644
--- a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
@@ -33,7 +33,7 @@ const API_KEY_JSON_PLACEHOLDER = `{
   "type": "service_account",
   "project_id": "fleet-in-your-calendar",
   "private_key_id": "",
-  "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
+  "private_key": "-----BEGIN PRIVATE KEY----\\n\\n-----END PRIVATE KEY-----\\n",
   "client_email": "fleet-calendar-events@fleet-in-your-calendar.iam.gserviceaccount.com",
   "client_id": "",
   "auth_uri": "https://accounts.google.com/o/oauth2/auth",
@@ -58,6 +58,16 @@ interface ICalendarsFormData {
   apiKeyJson?: string;
 }
 
+// Used to surface error.message in UI of unknown error type
+type ErrorWithMessage = {
+  message: string;
+  [key: string]: unknown;
+};
+
+const isErrorWithMessage = (error: unknown): error is ErrorWithMessage => {
+  return (error as ErrorWithMessage).message !== undefined;
+};
+
 const baseClass = "calendars-integration";
 
 const Calendars = (): JSX.Element => {
@@ -103,7 +113,18 @@ const Calendars = (): JSX.Element => {
       errors.apiKeyJson = "API key JSON must be present";
     }
     if (!curFormData.domain && !!curFormData.apiKeyJson) {
-      errors.apiKeyJson = "Domain must be present";
+      errors.domain = "Domain must be present";
+    }
+    if (curFormData.apiKeyJson) {
+      try {
+        JSON.parse(curFormData.apiKeyJson);
+      } catch (e: unknown) {
+        if (isErrorWithMessage(e)) {
+          errors.apiKeyJson = e.message.toString();
+        } else {
+          throw e;
+        }
+      }
     }
     return errors;
   };
@@ -277,6 +298,7 @@ const Calendars = (): JSX.Element => {
                     placeholder={API_KEY_JSON_PLACEHOLDER}
                     ignore1password
                     inputClassName={`${baseClass}__api-key-json`}
+                    error={formErrors.apiKeyJson}
                   />
                    {
                         />
                       
                     }
+                    error={formErrors.domain}
                   />
); diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss index 7045a443ef..01db771e00 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss +++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss @@ -4,6 +4,10 @@ color: $core-fleet-black; } + p { + margin: $pad-large 0; + } + ui { margin-block-start: $pad-small; } @@ -30,6 +34,7 @@ #oauth-scopes { font-family: "SourceCodePro", $monospace; + color: $core-fleet-black; min-height: 80px; padding: $pad-medium; padding-right: $pad-xxlarge; From a10aac29c609b4c930a7474bfdb2a67c45943d1b Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:54:32 -0700 Subject: [PATCH 21/30] =?UTF-8?q?UI=20=E2=80=93=20Calendar=20events=20moda?= =?UTF-8?q?l=20follow=20up=20(#17788)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Follow-up work to #17717 **Finalize disabled options and tooltips:** Screenshot 2024-03-21 at 5 14 40 PM Screenshot 2024-03-21 at 5 15 13 PM **Only update policies and settings when there's a diff:** ![1(1)](https://github.com/fleetdm/fleet/assets/61553566/183d1834-3c54-4fef-a208-dfbb0354e507) **Reorganize onChange handlers, types** - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Jacob Shandling --- frontend/interfaces/integration.ts | 8 +- frontend/pages/SoftwarePage/SoftwarePage.tsx | 6 +- .../AddIntegrationModal.tsx | 4 +- .../EditIntegrationModal.tsx | 4 +- .../IntegrationForm/IntegrationForm.tsx | 4 +- .../HostActionsDropdown/_styles.scss | 1 + .../ManagePoliciesPage/ManagePoliciesPage.tsx | 89 ++++++++++++------- .../policies/ManagePoliciesPage/_styles.scss | 30 ++++++- .../CalendarEventsModal.tsx | 54 ++++++----- .../OtherWorkflowsModal.tsx | 7 +- frontend/styles/var/mixins.scss | 1 - 11 files changed, 132 insertions(+), 76 deletions(-) diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts index aea79f99ee..f6302a67b8 100644 --- a/frontend/interfaces/integration.ts +++ b/frontend/interfaces/integration.ts @@ -75,15 +75,17 @@ interface ITeamCalendarSettings { // separated – it can be present without the other 2 without nullifying them. // TODO: Update these types to reflect this. -export interface IIntegrations { +export interface IZendeskJiraIntegrations { zendesk: IZendeskIntegration[]; jira: IJiraIntegration[]; } -export interface IGlobalIntegrations extends IIntegrations { +// reality is that IZendeskJiraIntegrations are optional – should be something like `extends +// Partial`, but that leads to a mess of types to resolve. +export interface IGlobalIntegrations extends IZendeskJiraIntegrations { google_calendar?: IGlobalCalendarIntegration[] | null; } -export interface ITeamIntegrations extends IIntegrations { +export interface ITeamIntegrations extends IZendeskJiraIntegrations { google_calendar?: ITeamCalendarSettings | null; } diff --git a/frontend/pages/SoftwarePage/SoftwarePage.tsx b/frontend/pages/SoftwarePage/SoftwarePage.tsx index 0d513c3481..20ccb325e4 100644 --- a/frontend/pages/SoftwarePage/SoftwarePage.tsx +++ b/frontend/pages/SoftwarePage/SoftwarePage.tsx @@ -11,7 +11,7 @@ import { import { IJiraIntegration, IZendeskIntegration, - IIntegrations, + IZendeskJiraIntegrations, } from "interfaces/integration"; import { ITeamConfig } from "interfaces/team"; import { IWebhookSoftwareVulnerabilities } from "interfaces/webhook"; @@ -186,7 +186,9 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { const vulnWebhookSettings = softwareConfig?.webhook_settings?.vulnerabilities_webhook; const isVulnWebhookEnabled = !!vulnWebhookSettings?.enable_vulnerabilities_webhook; - const isVulnIntegrationEnabled = (integrations?: IIntegrations) => { + const isVulnIntegrationEnabled = ( + integrations?: IZendeskJiraIntegrations + ) => { return ( !!integrations?.jira?.some((j) => j.enable_software_vulnerabilities) || !!integrations?.zendesk?.some((z) => z.enable_software_vulnerabilities) diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx index 0dc4dc630a..ef3a693220 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx @@ -4,7 +4,7 @@ import Modal from "components/Modal"; // @ts-ignore import Dropdown from "components/forms/fields/Dropdown"; import CustomLink from "components/CustomLink"; -import { IIntegration, IIntegrations } from "interfaces/integration"; +import { IIntegration, IZendeskJiraIntegrations } from "interfaces/integration"; import IntegrationForm from "../IntegrationForm"; const baseClass = "add-integration-modal"; @@ -17,7 +17,7 @@ interface IAddIntegrationModalProps { ) => void; serverErrors?: { base: string; email: string }; backendValidators: { [key: string]: string }; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; testingConnection: boolean; } diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx index 83d99a14fb..e5219f2708 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx @@ -4,7 +4,7 @@ import Modal from "components/Modal"; import Spinner from "components/Spinner"; import { IIntegration, - IIntegrations, + IZendeskJiraIntegrations, IIntegrationTableData, } from "interfaces/integration"; import IntegrationForm from "../IntegrationForm"; @@ -15,7 +15,7 @@ interface IEditIntegrationModalProps { onCancel: () => void; onSubmit: (jiraIntegrationSubmitData: IIntegration[]) => void; backendValidators: { [key: string]: string }; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; integrationEditing?: IIntegrationTableData; testingConnection: boolean; } diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx index 0cce5bb5a1..1d4bad9950 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx @@ -5,7 +5,7 @@ import { IIntegrationFormData, IIntegrationTableData, IIntegration, - IIntegrations, + IZendeskJiraIntegrations, IIntegrationType, } from "interfaces/integration"; @@ -26,7 +26,7 @@ interface IIntegrationFormProps { integrationDestination: string ) => void; integrationEditing?: IIntegrationTableData; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; integrationEditingUrl?: string; integrationEditingUsername?: string; integrationEditingEmail?: string; diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss index 06bd48a653..04d394a3b5 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss +++ b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss @@ -1,5 +1,6 @@ .host-actions-dropdown { @include button-dropdown; + color: $core-fleet-black; .Select-multi-value-wrapper { width: 55px; } diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index 80a832026b..a49d276128 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -14,7 +14,7 @@ import { TableContext } from "context/table"; import { NotificationContext } from "context/notification"; import useTeamIdParam from "hooks/useTeamIdParam"; import { IConfig, IWebhookSettings } from "interfaces/config"; -import { IIntegrations } from "interfaces/integration"; +import { IZendeskJiraIntegrations } from "interfaces/integration"; import { IPolicyStats, ILoadAllPoliciesResponse, @@ -519,10 +519,9 @@ const ManagePolicyPage = ({ router?.replace(locationPath); }; - const handleUpdateAutomations = async (requestBody: { + const handleUpdateOtherWorkflows = async (requestBody: { webhook_settings: Pick; - // TODO - update below type to specify team integration - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; }) => { setIsUpdatingAutomations(true); try { @@ -549,32 +548,52 @@ const ManagePolicyPage = ({ setUpdatingPolicyEnabledCalendarEvents(true); try { - // update enabled and URL in config - const configResponse = teamsAPI.update( - { - integrations: { - google_calendar: { - enable_calendar_events: formData.enabled, - webhook_url: formData.url, + // update team config if either field has been changed + const responses: Promise[] = []; + if ( + formData.enabled !== + teamConfig?.integrations.google_calendar?.enable_calendar_events || + formData.url !== teamConfig?.integrations.google_calendar?.webhook_url + ) { + responses.push( + teamsAPI.update( + { + integrations: { + google_calendar: { + enable_calendar_events: formData.enabled, + webhook_url: formData.url, + }, + // These fields will never actually be changed here. See comment above + // IGlobalIntegrations definition. + zendesk: teamConfig?.integrations.zendesk || [], + jira: teamConfig?.integrations.jira || [], + }, }, - // TODO - can omit these? - zendesk: teamConfig?.integrations.zendesk || [], - jira: teamConfig?.integrations.jira || [], - }, - }, - teamIdForApi - ); + teamIdForApi + ) + ); + } - // update policies calendar events enabled - // TODO - only update changed policies - const policyResponses = formData.policies.map((formPolicy) => - teamPoliciesAPI.update(formPolicy.id, { - calendar_events_enabled: formPolicy.isChecked, - team_id: teamIdForApi, + // update changed policies calendar events enabled + const changedPolicies = formData.policies.filter((formPolicy) => { + const prevPolicyState = teamPolicies?.find( + (policy) => policy.id === formPolicy.id + ); + return ( + formPolicy.isChecked !== prevPolicyState?.calendar_events_enabled + ); + }); + + responses.concat( + changedPolicies.map((changedPolicy) => { + return teamPoliciesAPI.update(changedPolicy.id, { + calendar_events_enabled: changedPolicy.isChecked, + team_id: teamIdForApi, + }); }) ); - await Promise.all([configResponse, ...policyResponses]); + await Promise.all(responses); renderFlash("success", "Successfully updated policy automations."); } catch { renderFlash( @@ -761,8 +780,16 @@ const ManagePolicyPage = ({ const tipId = uniqueId(); calEventsLabel = ( -
Calendar events
- +
+ Calendar events +
+ Available in Fleet Premium @@ -771,13 +798,15 @@ const ManagePolicyPage = ({ const tipId = uniqueId(); calEventsLabel = ( -
Calendar events
+
+ Calendar events +
Select a team to manage
@@ -920,7 +949,7 @@ const ManagePolicyPage = ({ availablePolicies={availablePoliciesForAutomation} isUpdatingAutomations={isUpdatingAutomations} onExit={toggleOtherWorkflowsModal} - handleSubmit={handleUpdateAutomations} + handleSubmit={handleUpdateOtherWorkflows} /> )} {showAddPolicyModal && ( diff --git a/frontend/pages/policies/ManagePoliciesPage/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/_styles.scss index ed99ad0137..3c88a2db1c 100644 --- a/frontend/pages/policies/ManagePoliciesPage/_styles.scss +++ b/frontend/pages/policies/ManagePoliciesPage/_styles.scss @@ -21,19 +21,43 @@ .Select > .Select-menu-outer { left: -186px; width: 360px; + .dropdown__help-text { + color: $ui-fleet-black-50; + } .is-disabled * { color: $ui-fleet-black-25; + .label-text { + font-style: normal; + // increase height to allow for broader tooltip activation area + position: absolute; + height: 34px; + width: 100%; + } + .dropdown__help-text { + // compensate for absolute label-text height + margin-top: 20px; + } .react-tooltip { @include tooltip-text; + font-style: normal; + text-align: center; } } } .Select-control { margin-top: 0; gap: 6px; - } - .Select-placeholder { - font-weight: $bold; + .Select-placeholder { + color: $core-vibrant-blue; + font-weight: $bold; + } + .dropdown__custom-arrow .dropdown__icon { + svg { + path { + stroke: $core-vibrant-blue-over; + } + } + } } } diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx index eba5abb4e0..93847411eb 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx @@ -55,10 +55,6 @@ const CalendarEventsModal = ({ const [formData, setFormData] = useState({ enabled, url, - // TODO - stay udpdated on state of backend approach to syncing policies in the policies table - // and in the new calendar table - // id may change if policy was deleted - // name could change if policy was renamed policies: policies.map((policy) => ({ name: policy.name, id: policy.id, @@ -87,29 +83,26 @@ const CalendarEventsModal = ({ return errors; }; - // TODO - separate change handlers for checkboxes: - // const onPolicyUpdate = ... - // const onTextFieldUpdate = ... - - const onInputChange = useCallback( - (newVal: { name: FormNames; value: string | number | boolean }) => { + // two onChange handlers to handle different levels of nesting in the form data + const onFeatureEnabledOrUrlChange = useCallback( + (newVal: { name: "enabled" | "url"; value: string | boolean }) => { const { name, value } = newVal; - let newFormData: ICalendarEventsFormData; - // for the first two fields, set the new value directly - if (["enabled", "url"].includes(name)) { - newFormData = { ...formData, [name]: value }; - } else if (typeof value === "boolean") { - // otherwise, set the value for a nested policy - const newFormPolicies = formData.policies.map((formPolicy) => { - if (formPolicy.name === name) { - return { ...formPolicy, isChecked: value }; - } - return formPolicy; - }); - newFormData = { ...formData, policies: newFormPolicies }; - } else { - throw TypeError("Unexpected value type for policy checkbox"); - } + const newFormData = { ...formData, [name]: value }; + setFormData(newFormData); + setFormErrors(validateCalendarEventsFormData(newFormData)); + }, + [formData] + ); + const onPolicyEnabledChange = useCallback( + (newVal: { name: FormNames; value: boolean }) => { + const { name, value } = newVal; + const newFormPolicies = formData.policies.map((formPolicy) => { + if (formPolicy.name === name) { + return { ...formPolicy, isChecked: value }; + } + return formPolicy; + }); + const newFormData = { ...formData, policies: newFormPolicies }; setFormData(newFormData); setFormErrors(validateCalendarEventsFormData(newFormData)); }, @@ -157,7 +150,7 @@ const CalendarEventsModal = ({ name={name} // can't use parseTarget as value needs to be set to !currentValue onChange={() => { - onInputChange({ name, value: !isChecked }); + onPolicyEnabledChange({ name, value: !isChecked }); }} > {name} @@ -232,7 +225,10 @@ const CalendarEventsModal = ({ { - onInputChange({ name: "enabled", value: !formData.enabled }); + onFeatureEnabledOrUrlChange({ + name: "enabled", + value: !formData.enabled, + }); }} inactiveText="Disabled" activeText="Enabled" @@ -251,7 +247,7 @@ const CalendarEventsModal = ({ { +const findEnabledIntegration = ({ + jira, + zendesk, +}: IZendeskJiraIntegrations) => { return ( jira?.find((j) => j.enable_failing_policies) || zendesk?.find((z) => z.enable_failing_policies) diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss index 3751ffadf6..fab5ee9ec2 100644 --- a/frontend/styles/var/mixins.scss +++ b/frontend/styles/var/mixins.scss @@ -275,7 +275,6 @@ $max-width: 2560px; } .Select-placeholder { - color: $core-fleet-black; font-size: 14px; line-height: normal; padding-left: 0; From 62049b04bd79ef16a97f6e38e43bf30fe01f6cd9 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 14:25:03 -0500 Subject: [PATCH 22/30] Added TestEventForDifferentHost for calendar_cron. (#17802) Added TestEventForDifferentHost for calendar_cron. --- cmd/fleet/calendar_cron_test.go | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go index 84f5ece527..c6db483c55 100644 --- a/cmd/fleet/calendar_cron_test.go +++ b/cmd/fleet/calendar_cron_test.go @@ -1,6 +1,11 @@ package main import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + kitlog "github.com/go-kit/log" + "os" "testing" "time" @@ -8,6 +13,7 @@ import ( ) func TestGetPreferredCalendarEventDate(t *testing.T) { + t.Parallel() date := func(year int, month time.Month, day int) time.Time { return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) } @@ -103,3 +109,84 @@ func TestGetPreferredCalendarEventDate(t *testing.T) { }) } } + +// TestEventForDifferentHost tests case when event exists, but for a different host. Nothing should happen. +// The old event will eventually be cleaned up by the cleanup job, and afterward a new event will be created. +func TestEventForDifferentHost(t *testing.T) { + t.Parallel() + ds := new(mock.Store) + ctx := context.Background() + logger := kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + {}, + }, + }, + }, nil + } + teamID1 := uint(1) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + }, + }, + }, + }, + }, nil + } + policyID1 := uint(10) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + require.Equal(t, teamID1, teamID) + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + }, nil + } + hostID1 := uint(100) + hostID2 := uint(101) + userEmail1 := "user@example.com" + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + require.Equal(t, teamID1, teamID) + require.Equal(t, []uint{policyID1}, policyIDs) + return []fleet.HostPolicyMembershipData{ + { + HostID: hostID1, + Email: userEmail1, + Passing: false, + }, + }, nil + } + // Return an existing event, but for a different host + eventTime := time.Now().Add(time.Hour) + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + require.Equal(t, userEmail1, email) + calEvent := &fleet.CalendarEvent{ + ID: 1, + Email: email, + StartTime: eventTime, + EndTime: eventTime, + } + hcEvent := &fleet.HostCalendarEvent{ + ID: 1, + HostID: hostID2, + CalendarEventID: 1, + WebhookStatus: fleet.CalendarWebhookStatusNone, + } + return hcEvent, calEvent, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + +} From 355379aa0b9050f5588c10693d0a88920f9b88ab Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Fri, 22 Mar 2024 16:26:11 -0300 Subject: [PATCH 23/30] Fleet calendar process 100 hosts at a time (#17806) Add concurrency for #17441. --- cmd/fleet/calendar_cron.go | 190 +++++++++++++++++++++++-------------- 1 file changed, 120 insertions(+), 70 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 238260ea3a..a909add0b6 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "time" "github.com/fleetdm/fleet/v4/ee/server/calendar" @@ -13,6 +14,7 @@ import ( "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" + "golang.org/x/sync/errgroup" ) func newCalendarSchedule( @@ -57,7 +59,6 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] - calendar := createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger) domain := googleCalendarIntegrationConfig.Domain teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ @@ -71,7 +72,7 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L for _, team := range teams { if err := cronCalendarEventsForTeam( - ctx, ds, calendar, *team, appConfig.OrgInfo.OrgName, domain, logger, + ctx, ds, googleCalendarIntegrationConfig, *team, appConfig.OrgInfo.OrgName, domain, logger, ); err != nil { level.Info(logger).Log("msg", "events calendar cron", "team_id", team.ID, "err", err) } @@ -92,7 +93,7 @@ func createUserCalendarFromConfig(ctx context.Context, config *fleet.GoogleCalen func cronCalendarEventsForTeam( ctx context.Context, ds fleet.Datastore, - calendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, team fleet.Team, orgName string, domain string, @@ -161,13 +162,13 @@ func cronCalendarEventsForTeam( // We execute this first to remove any calendar events for a user that is now passing // policies on one of its hosts, and possibly create a new calendar event if they have // another failing host on the same team. - if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil { + if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger); err != nil { level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) } // Process hosts that are failing calendar policies. if err := processCalendarFailingHosts( - ctx, ds, calendar, orgName, failingHosts, logger, + ctx, ds, calendarConfig, orgName, failingHosts, logger, ); err != nil { level.Info(logger).Log("msg", "processing failing hosts", "err", err) } @@ -185,67 +186,82 @@ func cronCalendarEventsForTeam( func processCalendarFailingHosts( ctx context.Context, ds fleet.Datastore, - userCalendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, orgName string, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, ) error { hosts = filterHostsWithSameEmail(hosts) - for _, host := range hosts { - logger := log.With(logger, "host_id", host.HostID) + const consumers = 100 + hostsCh := make(chan fleet.HostPolicyMembershipData) + g, ctx := errgroup.WithContext(ctx) - hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) + for i := 0; i < consumers; i++ { + g.Go(func() error { + for host := range hostsCh { + logger := log.With(logger, "host_id", host.HostID) - expiredEvent := false - if err == nil { - if hostCalendarEvent.HostID != host.HostID { - // This calendar event belongs to another host with this associated email, - // thus we skip this entry. - continue // continue with next host - } - if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending { - // This can happen if the host went offline (and never returned results) - // after setting the webhook as pending. - continue // continue with next host - } - now := time.Now() - webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent - if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) { - // If the webhook already fired today and the policies are still failing - // we give a grace period of one day for the host before we schedule a new event. - continue // continue with next host - } - if calendarEvent.EndTime.Before(now) { - expiredEvent = true - } - } + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) - if err := userCalendar.Configure(host.Email); err != nil { - return fmt.Errorf("configure user calendar: %w", err) - } + expiredEvent := false + if err == nil { + if hostCalendarEvent.HostID != host.HostID { + // This calendar event belongs to another host with this associated email, + // thus we skip this entry. + continue // continue with next host + } + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending { + // This can happen if the host went offline (and never returned results) + // after setting the webhook as pending. + continue // continue with next host + } + now := time.Now() + webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent + if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) { + // If the webhook already fired today and the policies are still failing + // we give a grace period of one day for the host before we schedule a new event. + continue // continue with next host + } + if calendarEvent.EndTime.Before(now) { + expiredEvent = true + } + } - switch { - case err == nil && !expiredEvent: - if err := processFailingHostExistingCalendarEvent( - ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, - ); err != nil { - level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) - continue // continue with next host + userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) + if err := userCalendar.Configure(host.Email); err != nil { + return fmt.Errorf("configure user calendar: %w", err) + } + + switch { + case err == nil && !expiredEvent: + if err := processFailingHostExistingCalendarEvent( + ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) + continue // continue with next host + } + case fleet.IsNotFound(err) || expiredEvent: + if err := processFailingHostCreateCalendarEvent( + ctx, ds, userCalendar, orgName, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) + continue // continue with next host + } + default: + return fmt.Errorf("get calendar event: %w", err) + } } - case fleet.IsNotFound(err) || expiredEvent: - if err := processFailingHostCreateCalendarEvent( - ctx, ds, userCalendar, orgName, host, - ); err != nil { - level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) - continue // continue with next host - } - default: - return fmt.Errorf("get calendar event: %w", err) - } + return nil + }) } - return nil + for _, host := range hosts { + hostsCh <- host + } + close(hostsCh) + + return g.Wait() } func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData { @@ -437,27 +453,61 @@ func addBusinessDay(date time.Time) time.Time { func removeCalendarEventsFromPassingHosts( ctx context.Context, ds fleet.Datastore, - userCalendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, ) error { + hostIDsByEmail := make(map[string][]uint) for _, host := range hosts { - hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) - switch { - case err == nil: - if hostCalendarEvent.HostID != host.HostID { - // This calendar event belongs to another host, thus we skip this entry. - continue - } - case fleet.IsNotFound(err): - continue - default: - return fmt.Errorf("get calendar event from DB: %w", err) - } - if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { - return fmt.Errorf("delete user calendar event: %w", err) - } + hostIDsByEmail[host.Email] = append(hostIDsByEmail[host.Email], host.HostID) } - return nil + type emailWithHosts struct { + email string + hostIDs []uint + } + emails := make([]emailWithHosts, 0, len(hostIDsByEmail)) + for email, hostIDs := range hostIDsByEmail { + emails = append(emails, emailWithHosts{ + email: email, + hostIDs: hostIDs, + }) + } + + const consumers = 100 + emailsCh := make(chan emailWithHosts) + g, ctx := errgroup.WithContext(ctx) + + for i := 0; i < consumers; i++ { + g.Go(func() error { + for email := range emailsCh { + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, email.email) + switch { + case err == nil: + if ok := slices.Contains(email.hostIDs, hostCalendarEvent.HostID); !ok { + // None of the hosts belong to this calendar event. + continue + } + case fleet.IsNotFound(err): + continue + default: + return fmt.Errorf("get calendar event from DB: %w", err) + } + userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) + if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { + return fmt.Errorf("delete user calendar event: %w", err) + } + } + return nil + }) + } + + for _, emailWithHostIDs := range emails { + emailsCh <- emailWithHostIDs + } + close(emailsCh) + + return g.Wait() } func logHostsWithoutAssociatedEmail( From 35a21d5f0c8339c9d98bd2b5efa9f2a2028f3c2f Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 14:29:05 -0500 Subject: [PATCH 24/30] Calendar helper scripts for testing (#17798) Calendar helper scripts for testing --- tools/calendar/README.md | 6 + tools/calendar/delete-events/delete-events.go | 74 ++++++++++++ tools/calendar/move-events/move-events.go | 111 ++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 tools/calendar/delete-events/delete-events.go create mode 100644 tools/calendar/move-events/move-events.go diff --git a/tools/calendar/README.md b/tools/calendar/README.md index 5d222e45de..bab2f481b4 100644 --- a/tools/calendar/README.md +++ b/tools/calendar/README.md @@ -1,3 +1,9 @@ +# Helper methods for Google calendar + +To delete all downtime events from a Google Calendar, use `delete-events/delete-events.go` + +To move all downtime events from multiple Google Calendars to a specific time, use `move-events/move-events.go` + # Calendar server for load testing Test calendar server that provides a REST API for managing events. diff --git a/tools/calendar/delete-events/delete-events.go b/tools/calendar/delete-events/delete-events.go new file mode 100644 index 0000000000..cfa0b2ce07 --- /dev/null +++ b/tools/calendar/delete-events/delete-events.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "flag" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/option" + "log" + "os" +) + +// Delete all events with eventTitle from the primary calendar of the user. + +var ( + serviceEmail = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL") + privateKey = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY") +) + +const ( + eventTitle = "💻🚫Downtime" +) + +func main() { + if serviceEmail == "" || privateKey == "" { + log.Fatal("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL and FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY must be set") + } + userEmail := flag.String("user", "", "User email to impersonate") + flag.Parse() + if *userEmail == "" { + log.Fatal("--user is required") + } + + ctx := context.Background() + conf := &jwt.Config{ + Email: serviceEmail, + Scopes: []string{ + "https://www.googleapis.com/auth/calendar.events", "https://www.googleapis.com/auth/calendar.settings.readonly", + }, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: *userEmail, + } + client := conf.Client(ctx) + // Create a new calendar service + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + log.Fatalf("Unable to create Calendar service: %v", err) + } + numberDeleted := 0 + for { + list, err := service.Events.List("primary").EventTypes("default").MaxResults(1000).OrderBy("startTime").SingleEvents(true).ShowDeleted(false).Q(eventTitle).Do() + if err != nil { + log.Fatalf("Unable to retrieve list of events: %v", err) + } + if len(list.Items) == 0 { + break + } + for _, item := range list.Items { + if item.Summary == eventTitle { + err = service.Events.Delete("primary", item.Id).Do() + if err != nil { + log.Fatalf("Unable to delete event: %v", err) + } + numberDeleted++ + if numberDeleted%10 == 0 { + log.Printf("Deleted %d events", numberDeleted) + } + } + } + } + log.Printf("DONE. Deleted %d events total", numberDeleted) +} diff --git a/tools/calendar/move-events/move-events.go b/tools/calendar/move-events/move-events.go new file mode 100644 index 0000000000..2e66b56e64 --- /dev/null +++ b/tools/calendar/move-events/move-events.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "flag" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/option" + "log" + "os" + "strings" + "sync" + "time" +) + +// Move all events with eventTitle from the primary calendar of the user to the new time. +// Only events in the future relative to the new event time are moved. In other words, if the current event time is in the past, it is not moved. + +var ( + serviceEmail = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL") + privateKey = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY") +) + +const ( + eventTitle = "💻🚫Downtime" +) + +func main() { + if serviceEmail == "" || privateKey == "" { + log.Fatal("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL and FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY must be set") + } + userEmails := flag.String("users", "", "Comma-separated list of user emails to impersonate") + dateTimeStr := flag.String("datetime", "", "Event time in "+time.RFC3339+" format") + flag.Parse() + if *userEmails == "" { + log.Fatal("--users are required") + } + if *dateTimeStr == "" { + log.Fatal("--datetime is required") + } + dateTime, err := time.Parse(time.RFC3339, *dateTimeStr) + if err != nil { + log.Fatalf("Unable to parse datetime: %v", err) + } + dateTimeEndStr := dateTime.Add(30 * time.Minute).Format(time.RFC3339) + userEmailList := strings.Split(*userEmails, ",") + if len(userEmailList) == 0 { + log.Fatal("No user emails provided") + } + + ctx := context.Background() + + var wg sync.WaitGroup + + for _, userEmail := range userEmailList { + wg.Add(1) + go func(userEmail string) { + defer wg.Done() + conf := &jwt.Config{ + Email: serviceEmail, + Scopes: []string{ + "https://www.googleapis.com/auth/calendar.events", "https://www.googleapis.com/auth/calendar.settings.readonly", + }, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: userEmail, + } + client := conf.Client(ctx) + // Create a new calendar service + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + log.Fatalf("Unable to create Calendar service: %v", err) + } + + numberMoved := 0 + for { + list, err := service.Events.List("primary").EventTypes("default"). + MaxResults(1000). + OrderBy("startTime"). + SingleEvents(true). + ShowDeleted(false). + TimeMin(dateTimeEndStr). + Q(eventTitle). + Do() + if err != nil { + log.Fatalf("Unable to retrieve list of events: %v", err) + } + if len(list.Items) == 0 { + break + } + for _, item := range list.Items { + if item.Summary == eventTitle { + item.Start.DateTime = dateTime.Format(time.RFC3339) + item.End.DateTime = dateTime.Add(30 * time.Minute).Format(time.RFC3339) + _, err := service.Events.Update("primary", item.Id, item).Do() + if err != nil { + log.Fatalf("Unable to update event: %v", err) + } + numberMoved++ + } + } + } + log.Printf("Moved %d events for %s", numberMoved, userEmail) + }(userEmail) + } + + // Wait for all goroutines to finish + wg.Wait() + +} From 2e5656328013c591484a91624c51823b46fb98d7 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 15:53:51 -0500 Subject: [PATCH 25/30] Adding retry logic when rate limited by Google Calendar API. (#17810) Adding retry logic when rate limited by Google Calendar API. --- cmd/fleet/calendar_cron.go | 4 +- ee/server/calendar/google_calendar.go | 87 ++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index a909add0b6..2813a01603 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -193,7 +193,7 @@ func processCalendarFailingHosts( ) error { hosts = filterHostsWithSameEmail(hosts) - const consumers = 100 + const consumers = 20 hostsCh := make(chan fleet.HostPolicyMembershipData) g, ctx := errgroup.WithContext(ctx) @@ -473,7 +473,7 @@ func removeCalendarEventsFromPassingHosts( }) } - const consumers = 100 + const consumers = 20 emailsCh := make(chan emailWithHosts) g, ctx := errgroup.WithContext(ctx) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 42f8b7b0d8..ea7376b16a 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -10,6 +10,7 @@ import ( "regexp" "time" + "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/log" @@ -72,7 +73,7 @@ func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == mockEmail: config.API = &GoogleCalendarMockAPI{config.Logger} default: - config.API = &GoogleCalendarLowLevelAPI{} + config.API = &GoogleCalendarLowLevelAPI{logger: config.Logger} } return &GoogleCalendar{ config: config, @@ -95,6 +96,7 @@ type eventDetails struct { type GoogleCalendarLowLevelAPI struct { service *calendar.Service + logger kitlog.Logger } // Configure creates a new Google Calendar service using the provided credentials. @@ -126,31 +128,77 @@ func adjustEmail(email string) string { } func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { - return lowLevelAPI.service.Settings.Get(name).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Settings.Get(name).Do() + }, + ) + return result.(*calendar.Setting), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { - return lowLevelAPI.service.Events.Insert(calendarID, event).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Events.Insert(calendarID, event).Do() + }, + ) + return result.(*calendar.Event), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { - return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() + }, + ) + return result.(*calendar.Event), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { - // Default maximum number of events returned is 250, which should be sufficient for most calendars. - return lowLevelAPI.service.Events.List(calendarID). - EventTypes("default"). - OrderBy("startTime"). - SingleEvents(true). - TimeMin(timeMin). - TimeMax(timeMax). - ShowDeleted(false). - Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + // Default maximum number of events returned is 250, which should be sufficient for most calendars. + return lowLevelAPI.service.Events.List(calendarID). + EventTypes("default"). + OrderBy("startTime"). + SingleEvents(true). + TimeMin(timeMin). + TimeMax(timeMax). + ShowDeleted(false). + Do() + }, + ) + return result.(*calendar.Events), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { - return lowLevelAPI.service.Events.Delete(calendarID, id).Do() + _, err := lowLevelAPI.withRetry( + func() (any, error) { + return nil, lowLevelAPI.service.Events.Delete(calendarID, id).Do() + }, + ) + return err +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) withRetry(fn func() (any, error)) (any, error) { + retryStrategy := backoff.NewExponentialBackOff() + retryStrategy.MaxElapsedTime = 10 * time.Minute + var result any + err := backoff.Retry( + func() error { + var err error + result, err = fn() + if err != nil { + if isRateLimited(err) { + level.Debug(lowLevelAPI.logger).Log("msg", "rate limited by Google calendar API", "err", err) + return err + } + return backoff.Permanent(err) + } + return nil + }, retryStrategy, + ) + return result, err } func (c *GoogleCalendar) Configure(userEmail string) error { @@ -308,6 +356,17 @@ func isAlreadyDeleted(err error) bool { return ok && ae.Code == http.StatusGone } +func isRateLimited(err error) bool { + if err == nil { + return false + } + var ae *googleapi.Error + ok := errors.As(err, &ae) + return ok && (ae.Code == http.StatusTooManyRequests || + (ae.Code == http.StatusForbidden && + (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded."))) +} + func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { var details eventDetails err := json.Unmarshal(event.Data, &details) From 9090d8541f5192f49bebd859f3fc35409081d1c9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 25 Mar 2024 07:21:56 -0300 Subject: [PATCH 26/30] Calendar update event if meeting occurring now (#17815) #17441 --------- Co-authored-by: Victor Lyuboslavsky --- cmd/fleet/calendar_cron.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 2813a01603..c6113d7181 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -296,9 +296,7 @@ func processFailingHostExistingCalendarEvent( updated := false now := time.Now() - // Check the user calendar every 30 minutes (and not every time) - // to reduce load on both Fleet and the calendar service. - if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute { + if shouldReloadCalendarEvent(now, calendarEvent, hostCalendarEvent) { var err error updatedEvent, _, err = calendar.GetAndUpdateEvent(calendarEvent, func(conflict bool) string { return generateCalendarEventBody(orgName, host.HostDisplayName, conflict) @@ -365,6 +363,24 @@ func processFailingHostExistingCalendarEvent( return nil } +func shouldReloadCalendarEvent(now time.Time, calendarEvent *fleet.CalendarEvent, hostCalendarEvent *fleet.HostCalendarEvent) bool { + // Check the user calendar every 30 minutes (and not every cron run) + // to reduce load on both Fleet and the calendar service. + if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute { + return true + } + // If the event is supposed to be happening now, we want to check if the user moved/deleted the + // event on the last minute. + if eventHappeningNow(now, calendarEvent) && hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusNone { + return true + } + return false +} + +func eventHappeningNow(now time.Time, calendarEvent *fleet.CalendarEvent) bool { + return !now.Before(calendarEvent.StartTime) && now.Before(calendarEvent.EndTime) +} + func sameDate(t1 time.Time, t2 time.Time) bool { y1, m1, d1 := t1.Date() y2, m2, d2 := t2.Date() From 51cd71f46423d1e9595ce940696e954dfbf4c268 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 25 Mar 2024 15:15:13 -0300 Subject: [PATCH 27/30] Fix concurrency bug in calendar cron (#17832) #17441 --- cmd/fleet/calendar_cron.go | 60 +-- cmd/fleet/calendar_cron_test.go | 453 ++++++++++++++++++++- ee/server/calendar/google_calendar_mock.go | 26 +- 3 files changed, 504 insertions(+), 35 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index c6113d7181..e4b1927d55 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "slices" + "sync" "time" "github.com/fleetdm/fleet/v4/ee/server/calendar" @@ -14,7 +15,6 @@ import ( "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" - "golang.org/x/sync/errgroup" ) func newCalendarSchedule( @@ -162,16 +162,18 @@ func cronCalendarEventsForTeam( // We execute this first to remove any calendar events for a user that is now passing // policies on one of its hosts, and possibly create a new calendar event if they have // another failing host on the same team. - if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger); err != nil { - level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) - } + start := time.Now() + removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger) + level.Debug(logger).Log( + "msg", "passing_hosts", "took", time.Since(start), + ) // Process hosts that are failing calendar policies. - if err := processCalendarFailingHosts( - ctx, ds, calendarConfig, orgName, failingHosts, logger, - ); err != nil { - level.Info(logger).Log("msg", "processing failing hosts", "err", err) - } + start = time.Now() + processCalendarFailingHosts(ctx, ds, calendarConfig, orgName, failingHosts, logger) + level.Debug(logger).Log( + "msg", "failing_hosts", "took", time.Since(start), + ) // At last we want to log the hosts that are failing and don't have an associated email. logHostsWithoutAssociatedEmail( @@ -190,15 +192,18 @@ func processCalendarFailingHosts( orgName string, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, -) error { +) { hosts = filterHostsWithSameEmail(hosts) const consumers = 20 hostsCh := make(chan fleet.HostPolicyMembershipData) - g, ctx := errgroup.WithContext(ctx) + var wg sync.WaitGroup for i := 0; i < consumers; i++ { - g.Go(func() error { + wg.Add(+1) + go func() { + defer wg.Done() + for host := range hostsCh { logger := log.With(logger, "host_id", host.HostID) @@ -230,7 +235,8 @@ func processCalendarFailingHosts( userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) if err := userCalendar.Configure(host.Email); err != nil { - return fmt.Errorf("configure user calendar: %w", err) + level.Error(logger).Log("msg", "configure user calendar", "err", err) + continue // continue with next host } switch { @@ -249,11 +255,11 @@ func processCalendarFailingHosts( continue // continue with next host } default: - return fmt.Errorf("get calendar event: %w", err) + level.Error(logger).Log("msg", "get calendar event from db", "err", err) + continue // continue with next host } } - return nil - }) + }() } for _, host := range hosts { @@ -261,7 +267,7 @@ func processCalendarFailingHosts( } close(hostsCh) - return g.Wait() + wg.Wait() } func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData { @@ -472,7 +478,7 @@ func removeCalendarEventsFromPassingHosts( calendarConfig *fleet.GoogleCalendarIntegration, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, -) error { +) { hostIDsByEmail := make(map[string][]uint) for _, host := range hosts { hostIDsByEmail[host.Email] = append(hostIDsByEmail[host.Email], host.HostID) @@ -491,10 +497,13 @@ func removeCalendarEventsFromPassingHosts( const consumers = 20 emailsCh := make(chan emailWithHosts) - g, ctx := errgroup.WithContext(ctx) + var wg sync.WaitGroup for i := 0; i < consumers; i++ { - g.Go(func() error { + wg.Add(+1) + go func() { + defer wg.Done() + for email := range emailsCh { hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, email.email) @@ -507,15 +516,16 @@ func removeCalendarEventsFromPassingHosts( case fleet.IsNotFound(err): continue default: - return fmt.Errorf("get calendar event from DB: %w", err) + level.Error(logger).Log("msg", "get calendar event from DB", "err", err) + continue } userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { - return fmt.Errorf("delete user calendar event: %w", err) + level.Error(logger).Log("msg", "delete user calendar event", "err", err) + continue } } - return nil - }) + }() } for _, emailWithHostIDs := range emails { @@ -523,7 +533,7 @@ func removeCalendarEventsFromPassingHosts( } close(emailsCh) - return g.Wait() + wg.Wait() } func logHostsWithoutAssociatedEmail( diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go index c6db483c55..4d9133377c 100644 --- a/cmd/fleet/calendar_cron_test.go +++ b/cmd/fleet/calendar_cron_test.go @@ -2,12 +2,21 @@ package main import ( "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/ee/server/calendar" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" kitlog "github.com/go-kit/log" - "os" - "testing" - "time" "github.com/stretchr/testify/require" ) @@ -188,5 +197,441 @@ func TestEventForDifferentHost(t *testing.T) { err := cronCalendarEvents(ctx, ds, logger) require.NoError(t, err) - +} + +func TestCalendarEventsMultipleHosts(t *testing.T) { + ds := new(mock.Store) + ctx := context.Background() + logger := kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + t.Cleanup(func() { + calendar.ClearMockEvents() + }) + + // TODO(lucas): Test! + webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + requestBodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("webhook request: %s\n", requestBodyBytes) + })) + t.Cleanup(func() { + webhookServer.Close() + }) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Domain: "example.com", + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: "calendar-mock@example.com", + }, + }, + }, + }, + }, nil + } + + teamID1 := uint(1) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + }, nil + } + + policyID1 := uint(10) + policyID2 := uint(11) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + require.Equal(t, teamID1, teamID) + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + { + ID: policyID2, + Name: "Policy 2", + }, + }, nil + } + + hostID1, userEmail1 := uint(100), "user1@example.com" + hostID2, userEmail2 := uint(101), "user2@example.com" + hostID3, userEmail3 := uint(102), "user3@other.com" + hostID4, userEmail4 := uint(103), "user4@other.com" + + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + require.Equal(t, teamID1, teamID) + require.Equal(t, []uint{policyID1, policyID2}, policyIDs) + return []fleet.HostPolicyMembershipData{ + { + HostID: hostID1, + Email: userEmail1, + Passing: false, + }, + { + HostID: hostID2, + Email: userEmail2, + Passing: true, + }, + { + HostID: hostID3, + Email: userEmail3, + Passing: false, + }, + { + HostID: hostID4, + Email: userEmail4, + Passing: true, + }, + }, nil + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + return nil, nil, notFoundErr{} + } + + ds.CreateOrUpdateCalendarEventFunc = func(ctx context.Context, + email string, + startTime, endTime time.Time, + data []byte, + hostID uint, + webhookStatus fleet.CalendarWebhookStatus, + ) (*fleet.CalendarEvent, error) { + switch email { + case userEmail1: + require.Equal(t, hostID1, hostID) + case userEmail2: + require.Equal(t, hostID2, hostID) + case userEmail3: + require.Equal(t, hostID3, hostID) + case userEmail4: + require.Equal(t, hostID4, hostID) + } + require.Equal(t, fleet.CalendarWebhookStatusNone, webhookStatus) + require.NotEmpty(t, data) + require.NotZero(t, startTime) + require.NotZero(t, endTime) + // Currently, the returned calendar event is unused. + return nil, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) +} + +type notFoundErr struct{} + +func (n notFoundErr) IsNotFound() bool { + return true +} + +func (n notFoundErr) Error() string { + return "not found" +} + +func TestCalendarEvents1KHosts(t *testing.T) { + ds := new(mock.Store) + ctx := context.Background() + var logger kitlog.Logger + if os.Getenv("CALENDAR_TEST_LOGGING") != "" { + logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + } else { + logger = kitlog.NewNopLogger() + } + t.Cleanup(func() { + calendar.ClearMockEvents() + }) + + // TODO(lucas): Use for the test. + webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + requestBodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("webhook request: %s\n", requestBodyBytes) + })) + t.Cleanup(func() { + webhookServer.Close() + }) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Domain: "example.com", + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: "calendar-mock@example.com", + }, + }, + }, + }, + }, nil + } + + teamID1 := uint(1) + teamID2 := uint(2) + teamID3 := uint(3) + teamID4 := uint(4) + teamID5 := uint(5) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID2, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID3, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID4, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID5, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + }, nil + } + + policyID1 := uint(10) + policyID2 := uint(11) + policyID3 := uint(12) + policyID4 := uint(13) + policyID5 := uint(14) + policyID6 := uint(15) + policyID7 := uint(16) + policyID8 := uint(17) + policyID9 := uint(18) + policyID10 := uint(19) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + switch teamID { + case teamID1: + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + { + ID: policyID2, + Name: "Policy 2", + }, + }, nil + case teamID2: + return []fleet.PolicyCalendarData{ + { + ID: policyID3, + Name: "Policy 3", + }, + { + ID: policyID4, + Name: "Policy 4", + }, + }, nil + case teamID3: + return []fleet.PolicyCalendarData{ + { + ID: policyID5, + Name: "Policy 5", + }, + { + ID: policyID6, + Name: "Policy 6", + }, + }, nil + case teamID4: + return []fleet.PolicyCalendarData{ + { + ID: policyID7, + Name: "Policy 7", + }, + { + ID: policyID8, + Name: "Policy 8", + }, + }, nil + case teamID5: + return []fleet.PolicyCalendarData{ + { + ID: policyID9, + Name: "Policy 9", + }, + { + ID: policyID10, + Name: "Policy 10", + }, + }, nil + default: + return nil, notFoundErr{} + } + } + + hosts := make([]fleet.HostPolicyMembershipData, 0, 1000) + for i := 0; i < 1000; i++ { + hosts = append(hosts, fleet.HostPolicyMembershipData{ + Email: fmt.Sprintf("user%d@example.com", i), + Passing: i%2 == 0, + HostID: uint(i), + HostDisplayName: fmt.Sprintf("display_name%d", i), + HostHardwareSerial: fmt.Sprintf("serial%d", i), + }) + } + + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + var start, end int + switch teamID { + case teamID1: + start, end = 0, 200 + case teamID2: + start, end = 200, 400 + case teamID3: + start, end = 400, 600 + case teamID4: + start, end = 600, 800 + case teamID5: + start, end = 800, 1000 + } + return hosts[start:end], nil + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + return nil, nil, notFoundErr{} + } + + eventsCreated := 0 + var eventsCreatedMu sync.Mutex + + eventPerHost := make(map[uint]*fleet.CalendarEvent) + + ds.CreateOrUpdateCalendarEventFunc = func(ctx context.Context, + email string, + startTime, endTime time.Time, + data []byte, + hostID uint, + webhookStatus fleet.CalendarWebhookStatus, + ) (*fleet.CalendarEvent, error) { + require.Equal(t, fmt.Sprintf("user%d@example.com", hostID), email) + eventsCreatedMu.Lock() + eventsCreated += 1 + eventPerHost[hostID] = &fleet.CalendarEvent{ + ID: hostID, + Email: email, + StartTime: startTime, + EndTime: endTime, + Data: data, + UpdateCreateTimestamps: fleet.UpdateCreateTimestamps{ + CreateTimestamp: fleet.CreateTimestamp{ + CreatedAt: time.Now(), + }, + UpdateTimestamp: fleet.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, + } + eventsCreatedMu.Unlock() + require.Equal(t, fleet.CalendarWebhookStatusNone, webhookStatus) + require.NotEmpty(t, data) + require.NotZero(t, startTime) + require.NotZero(t, endTime) + // Currently, the returned calendar event is unused. + return nil, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + + createdCalendarEvents := calendar.ListGoogleMockEvents() + require.Equal(t, eventsCreated, 500) + require.Len(t, createdCalendarEvents, 500) + + hosts = make([]fleet.HostPolicyMembershipData, 0, 1000) + for i := 0; i < 1000; i++ { + hosts = append(hosts, fleet.HostPolicyMembershipData{ + Email: fmt.Sprintf("user%d@example.com", i), + Passing: true, + HostID: uint(i), + HostDisplayName: fmt.Sprintf("display_name%d", i), + HostHardwareSerial: fmt.Sprintf("serial%d", i), + }) + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + hostID, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(email, "user"), "@example.com")) + require.NoError(t, err) + if hostID%2 == 0 { + return nil, nil, notFoundErr{} + } + require.Contains(t, eventPerHost, uint(hostID)) + return &fleet.HostCalendarEvent{ + ID: uint(hostID), + HostID: uint(hostID), + CalendarEventID: uint(hostID), + WebhookStatus: fleet.CalendarWebhookStatusNone, + }, eventPerHost[uint(hostID)], nil + } + + ds.DeleteCalendarEventFunc = func(ctx context.Context, calendarEventID uint) error { + return nil + } + + err = cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + + createdCalendarEvents = calendar.ListGoogleMockEvents() + require.Len(t, createdCalendarEvents, 0) } diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go index 255f8d87c7..08e3a72e20 100644 --- a/ee/server/calendar/google_calendar_mock.go +++ b/ee/server/calendar/google_calendar_mock.go @@ -3,23 +3,26 @@ package calendar import ( "context" "errors" - kitlog "github.com/go-kit/log" - "google.golang.org/api/calendar/v3" - "google.golang.org/api/googleapi" "net/http" "os" "strconv" "sync" "time" + + kitlog "github.com/go-kit/log" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" ) type GoogleCalendarMockAPI struct { logger kitlog.Logger } -var mockEvents = make(map[string]*calendar.Event) -var mu sync.Mutex -var id uint64 +var ( + mockEvents = make(map[string]*calendar.Event) + mu sync.Mutex + id uint64 +) const latency = 500 * time.Millisecond @@ -44,6 +47,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) GetSetting(name string) (*calendar.Set } func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + time.Sleep(latency) mu.Lock() defer mu.Unlock() id += 1 @@ -79,3 +83,13 @@ func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error { delete(mockEvents, id) return nil } + +func ListGoogleMockEvents() map[string]*calendar.Event { + return mockEvents +} + +func ClearMockEvents() { + mu.Lock() + defer mu.Unlock() + mockEvents = make(map[string]*calendar.Event) +} From 72662291b0c39e99373ca911d5a570542f79c0c4 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 25 Mar 2024 16:11:46 -0500 Subject: [PATCH 28/30] Adding embedded timezone database. --- server/fleet/calendar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index 592fff430f..5eb4597f44 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "time" + _ "time/tzdata" // embed timezone information in the program "github.com/fleetdm/fleet/v4/server" ) From b92733b0e3eba5af1a79fd65ae0ef7b998f1fea4 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 25 Mar 2024 16:59:02 -0500 Subject: [PATCH 29/30] Adding another error message for rate limiting. --- ee/server/calendar/google_calendar.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ea7376b16a..7283269dfe 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "regexp" + "strings" "time" "github.com/cenkalti/backoff/v4" @@ -364,7 +365,7 @@ func isRateLimited(err error) bool { ok := errors.As(err, &ae) return ok && (ae.Code == http.StatusTooManyRequests || (ae.Code == http.StatusForbidden && - (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded."))) + (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded." || strings.HasPrefix(ae.Message, "Quota exceeded")))) } func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { From 9861f6eca633b0f7577a7af96b090a988e31c050 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 26 Mar 2024 13:47:20 -0500 Subject: [PATCH 30/30] Added Fleet in your calendar changes file. --- changes/17230-fleet-in-your-calendar | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/17230-fleet-in-your-calendar diff --git a/changes/17230-fleet-in-your-calendar b/changes/17230-fleet-in-your-calendar new file mode 100644 index 0000000000..299239a074 --- /dev/null +++ b/changes/17230-fleet-in-your-calendar @@ -0,0 +1,5 @@ +Added integration with Google Calendar. +- Fleet admins can enable Google Calendar integration by using a Google service account with domain-wide delegation. +- Calendar integration is enabled at the team level for specific team policies. +- If the policy is failing, a calendar event will be put on the host user's calendar for the 3rd Tuesday of the month. +- During the event, Fleet will fire a webhook. IT admins should use this webhook to trigger a script or MDM command that will remediate the issue.