Stop channel bug fix and reload frequency (#20425)

Unreleased bug fix for #19352
This commit is contained in:
Victor Lyuboslavsky 2024-07-15 09:25:44 -05:00 committed by GitHub
parent 7ae1fe9527
commit dc7a3cd827
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 20 additions and 15 deletions

View file

@ -270,7 +270,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
return nil, false, err
}
gEvent, err := c.config.API.GetEvent(details.ID, details.ETag)
var deleted bool
var deleted, channelStopped bool
switch {
// http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later
case googleapi.IsNotModified(err):
@ -278,11 +278,6 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
// http.StatusNotFound should be very rare -- Google keeps events for a while after they are deleted
case isNotFound(err):
deleted = true
// If event was deleted, we need to stop watching it
err = c.config.API.Stop(details.ChannelID, details.ResourceID)
if err != nil {
level.Warn(c.config.Logger).Log("msg", "stopping Google calendar event watch", "err", err)
}
case err != nil:
return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event")
}
@ -304,6 +299,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err)
}
deleted = true
channelStopped = true
}
var endTime *time.Time
@ -320,6 +316,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which is in the past", "err", err)
}
deleted = true
channelStopped = true
}
}
if !deleted {
@ -335,6 +332,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err)
}
deleted = true
channelStopped = true
}
}
if !deleted {
@ -350,6 +348,14 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
}
}
// If event was deleted/cancelled, we need to stop watching it
if !channelStopped {
err = c.config.API.Stop(details.ChannelID, details.ResourceID)
if err != nil {
level.Warn(c.config.Logger).Log("msg", "stopping Google calendar event watch", "err", err)
}
}
newStartDate := calculateNewEventDate(event.StartTime)
fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn)

View file

@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"slices"
"sync"
"time"
@ -17,11 +16,12 @@ import (
"github.com/go-kit/log"
kitlog "github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/google/uuid"
)
const (
calendarConsumers = 18
reloadFrequency = 12 * time.Hour
reloadFrequency = 30 * time.Minute
)
func NewCalendarSchedule(
@ -385,13 +385,6 @@ func processFailingHostExistingCalendarEvent(
}
}()
// Remove event from the queue so that we don't process this event again.
// Note: This item can be added back to the queue while we are processing it.
err = distributedLock.RemoveFromSet(ctx, calendar.QueueKey, eventUUID)
if err != nil {
return fmt.Errorf("remove calendar event from queue: %w", err)
}
updatedEvent := calendarEvent
updated := false
now := time.Now()
@ -422,6 +415,12 @@ func processFailingHostExistingCalendarEvent(
); err != nil {
return fmt.Errorf("updating event calendar on db: %w", err)
}
// Remove event from the queue so that we don't process this event again.
err = distributedLock.RemoveFromSet(ctx, calendar.QueueKey, eventUUID)
if err != nil {
return fmt.Errorf("remove calendar event from queue: %w", err)
}
}
eventInFuture := now.Before(updatedEvent.StartTime)