diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index e8ec7685d7..fa63b487dc 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -30,6 +30,12 @@ func newCalendarSchedule( ctx, name, instanceID, defaultInterval, ds, ds, schedule.WithAltLockID("calendar"), schedule.WithLogger(logger), + schedule.WithJob( + "calendar_events_cleanup", + func(ctx context.Context) error { + return cronCalendarEventsCleanup(ctx, ds, logger) + }, + ), schedule.WithJob( "calendar_events", func(ctx context.Context) error { @@ -51,12 +57,7 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] - googleCalendarConfig := calendar.GoogleCalendarConfig{ - Context: ctx, - IntegrationConfig: googleCalendarIntegrationConfig, - Logger: log.With(logger, "component", "google_calendar"), - } - calendar := calendar.NewGoogleCalendar(&googleCalendarConfig) + calendar := createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger) domain := googleCalendarIntegrationConfig.Domain teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ @@ -79,6 +80,15 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } +func createUserCalendarFromConfig(ctx context.Context, config *fleet.GoogleCalendarIntegration, logger kitlog.Logger) fleet.UserCalendar { + googleCalendarConfig := calendar.GoogleCalendarConfig{ + Context: ctx, + IntegrationConfig: config, + Logger: log.With(logger, "component", "google_calendar"), + } + return calendar.NewGoogleCalendar(&googleCalendarConfig) +} + func cronCalendarEventsForTeam( ctx context.Context, ds fleet.Datastore, @@ -110,9 +120,6 @@ func cronCalendarEventsForTeam( // - We get only one host per email that's failing policies (the one with lower host id). // - On every host, we get only the first email that matches the domain (sorted lexicographically). // - // TODOs(lucas): - // - We need to rate limit calendar requests. - // policyIDs := make([]uint, 0, len(policies)) for _, policy := range policies { @@ -159,15 +166,12 @@ func cronCalendarEventsForTeam( level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) } - // At last we want to notify the hosts that are failing and don't have an associated email. - if err := fireWebhookForHostsWithoutAssociatedEmail( - team.Config.Integrations.GoogleCalendar.WebhookURL, + // At last we want to log the hosts that are failing and don't have an associated email. + logHostsWithoutAssociatedEmail( domain, failingHostsWithoutAssociatedEmail, logger, - ); err != nil { - level.Info(logger).Log("msg", "webhook for hosts without associated email", "err", err) - } + ) return nil } @@ -182,34 +186,40 @@ func processCalendarFailingHosts( ) error { for _, host := range hosts { logger := log.With(logger, "host_id", host.HostID) + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID) + + expiredEvent := false + webhookAlreadyFiredThisMonth := false + if err == nil { + now := time.Now() + webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent + if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) { + // If the webhook already fired today and the policies are still failing + // we give a grace period of one day for the host before we schedule a new event. + continue // continue with next host + } + webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime) + if calendarEvent.EndTime.Before(time.Now()) { + expiredEvent = true + } + } + if err := userCalendar.Configure(host.Email); err != nil { return fmt.Errorf("configure user calendar: %w", err) } - hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID) - - deletedExpiredEvent := false - if err == nil { - if calendarEvent.EndTime.Before(time.Now()) { - if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { - level.Info(logger).Log("msg", "deleting existing expired calendar event", "err", err) - continue // continue with next host - } - deletedExpiredEvent = true - } - } - switch { - case err == nil && !deletedExpiredEvent: + case err == nil && !expiredEvent: if err := processFailingHostExistingCalendarEvent( ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, ); err != nil { level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) continue // continue with next host } - case fleet.IsNotFound(err) || deletedExpiredEvent: + case fleet.IsNotFound(err) || expiredEvent: if err := processFailingHostCreateCalendarEvent( - ctx, ds, userCalendar, orgName, host, + ctx, ds, userCalendar, orgName, host, webhookAlreadyFiredThisMonth, ); err != nil { level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) continue // continue with next host @@ -231,13 +241,27 @@ func processFailingHostExistingCalendarEvent( calendarEvent *fleet.CalendarEvent, host fleet.HostPolicyMembershipData, ) error { - updatedEvent, updated, err := calendar.GetAndUpdateEvent( - calendarEvent, func(bool) string { - return generateCalendarEventBody(orgName, host.HostDisplayName) + updatedEvent := calendarEvent + updated := false + now := time.Now() + + // Check the user calendar every 30 minutes (and not every time) + // to reduce load on both Fleet and the calendar service. + if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute { + var err error + updatedEvent, _, err = calendar.GetAndUpdateEvent(calendarEvent, func(conflict bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName, conflict) }) - if err != nil { - return fmt.Errorf("get event calendar on db: %w", err) + if err != nil { + return fmt.Errorf("get event calendar on db: %w", err) + } + // Even if fields haven't changed we want to update the calendar_events.updated_at below. + updated = true + // + // TODO(lucas): Check changing updatedEvent to UTC before consuming. + // } + if updated { if err := ds.UpdateCalendarEvent(ctx, calendarEvent.ID, @@ -248,16 +272,9 @@ func processFailingHostExistingCalendarEvent( return fmt.Errorf("updating event calendar on db: %w", err) } } - now := time.Now() + eventInFuture := now.Before(updatedEvent.StartTime) if eventInFuture { - // If the webhook status was sent and event was moved to the future we set the status to pending. - // This can happen if the admin wants to retry a remediation. - if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent { - if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil { - return fmt.Errorf("update host calendar webhook status: %w", err) - } - } // Nothing else to do as event is in the future. return nil } @@ -297,18 +314,31 @@ func processFailingHostExistingCalendarEvent( return nil } +func sameDate(t1 time.Time, t2 time.Time) bool { + y1, m1, d1 := t1.Date() + y2, m2, d2 := t2.Date() + return y1 == y2 && m1 == m2 && d1 == d2 +} + +func sameMonth(t1 time.Time, t2 time.Time) bool { + y1, m1, _ := t1.Date() + y2, m2, _ := t2.Date() + return y1 == y2 && m1 == m2 +} + func processFailingHostCreateCalendarEvent( ctx context.Context, ds fleet.Datastore, userCalendar fleet.UserCalendar, orgName string, host fleet.HostPolicyMembershipData, + webhookAlreadyFiredThisMonth bool, ) error { - calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar) + calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar, webhookAlreadyFiredThisMonth) if err != nil { return fmt.Errorf("create event on user calendar: %w", err) } - if _, err := ds.NewCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID); err != nil { + if _, err := ds.CreateOrUpdateCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID, fleet.CalendarWebhookStatusNone); err != nil { return fmt.Errorf("create calendar event on db: %w", err) } return nil @@ -318,18 +348,14 @@ func attemptCreatingEventOnUserCalendar( orgName string, host fleet.HostPolicyMembershipData, userCalendar fleet.UserCalendar, + webhookAlreadyFiredThisMonth bool, ) (*fleet.CalendarEvent, error) { - // TODO(lucas): Where do we handle the following case (it seems CreateEvent needs to return no slot available for the requested day if there are none or too late): - // - // - If it’s the 3rd Tuesday of the month, create an event in the upcoming slot (if available). - // For example, if it’s the 3rd Tuesday of the month at 10:07a, Fleet will look for an open slot starting at 10:30a. - // - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day. year, month, today := time.Now().Date() - preferredDate := getPreferredCalendarEventDate(year, month, today) + preferredDate := getPreferredCalendarEventDate(year, month, today, webhookAlreadyFiredThisMonth) for { calendarEvent, err := userCalendar.CreateEvent( - preferredDate, func(bool) string { - return generateCalendarEventBody(orgName, host.HostDisplayName) + preferredDate, func(conflict bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName, conflict) }, ) var dee fleet.DayEndedError @@ -345,7 +371,10 @@ func attemptCreatingEventOnUserCalendar( } } -func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time { +func getPreferredCalendarEventDate( + year int, month time.Month, today int, + webhookAlreadyFired bool, +) time.Time { const ( // 3rd Tuesday of Month preferredWeekDay = time.Tuesday @@ -360,6 +389,10 @@ func getPreferredCalendarEventDate(year int, month time.Month, today int) time.T preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1))) if today > preferredDate.Day() { today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC) + if webhookAlreadyFired { + nextMonth := today_.AddDate(0, 1, 0) // move to next month + return getPreferredCalendarEventDate(nextMonth.Year(), nextMonth.Month(), 1, false) + } preferredDate = addBusinessDay(today_) } return preferredDate @@ -379,7 +412,7 @@ func addBusinessDay(date time.Time) time.Time { func removeCalendarEventsFromPassingHosts( ctx context.Context, ds fleet.Datastore, - calendar fleet.UserCalendar, + userCalendar fleet.UserCalendar, hosts []fleet.HostPolicyMembershipData, ) error { for _, host := range hosts { @@ -392,47 +425,42 @@ func removeCalendarEventsFromPassingHosts( default: return fmt.Errorf("get calendar event from DB: %w", err) } - - if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { - return fmt.Errorf("delete db calendar event: %w", err) - } - if err := calendar.Configure(host.Email); err != nil { - return fmt.Errorf("connect to user calendar: %w", err) - } - if err := calendar.DeleteEvent(calendarEvent); err != nil { - return fmt.Errorf("delete calendar event: %w", err) + if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { + return fmt.Errorf("delete user calendar event: %w", err) } } return nil } -func fireWebhookForHostsWithoutAssociatedEmail( - webhookURL string, +func logHostsWithoutAssociatedEmail( domain string, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, -) error { - // TODO(lucas): We are firing these every 5 minutes... - for _, host := range hosts { - if err := fleet.FireCalendarWebhook( - webhookURL, - host.HostID, host.HostHardwareSerial, host.HostDisplayName, nil, - fmt.Sprintf("No %s Google account associated with this host.", domain), - ); err != nil { - level.Error(logger).Log( - "msg", "fire webhook for hosts without associated email", "err", err, - ) - } +) { + if len(hosts) == 0 { + return } - return nil + var hostIDs []uint + for _, host := range hosts { + hostIDs = append(hostIDs, host.HostID) + } + // Logging as debug because this might get logged every 5 minutes. + level.Debug(logger).Log( + "msg", fmt.Sprintf("no %s Google account associated with the hosts", domain), + "host_ids", fmt.Sprintf("%+v", hostIDs), + ) } -func generateCalendarEventBody(orgName, hostDisplayName string) string { +func generateCalendarEventBody(orgName, hostDisplayName string, conflict bool) string { + conflictStr := "" + if conflict { + conflictStr = " because there was no remaining availability" + } return fmt.Sprintf(`Please leave your computer on and connected to power. Expect an automated restart. -%s reserved this time to fix %s.`, orgName, hostDisplayName, +%s reserved this time to fix %s%s.`, orgName, hostDisplayName, conflictStr, ) } @@ -456,3 +484,112 @@ func isHostOnline(ctx context.Context, ds fleet.Datastore, hostID uint) (bool, e return false, fmt.Errorf("unknown host status: %s", status) } } + +func cronCalendarEventsCleanup(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error { + appConfig, err := ds.AppConfig(ctx) + if err != nil { + return fmt.Errorf("load app config: %w", err) + } + + var userCalendar fleet.UserCalendar + if len(appConfig.Integrations.GoogleCalendar) > 0 { + googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] + userCalendar = createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger) + } + + // If global setting is disabled, we remove all calendar events from the DB + // (we cannot delete the events from the user calendar because there's no configuration anymore). + if userCalendar == nil { + if err := deleteAllCalendarEvents(ctx, ds, nil, nil); err != nil { + return fmt.Errorf("delete all calendar events: %w", err) + } + // We've deleted all calendar events, nothing else to do. + return nil + } + + // + // Feature is configured globally, but now we have to check team by team. + // + + teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ + User: &fleet.User{ + GlobalRole: ptr.String(fleet.RoleAdmin), + }, + }, fleet.ListOptions{}) + if err != nil { + return fmt.Errorf("list teams: %w", err) + } + + for _, team := range teams { + if err := deleteTeamCalendarEvents(ctx, ds, userCalendar, *team); err != nil { + level.Info(logger).Log("msg", "delete team calendar events", "team_id", team.ID, "err", err) + } + } + + // + // Delete calendar events from DB that haven't been updated for a while + // (e.g. host was transferred to another team or global). + // + + outOfDateCalendarEvents, err := ds.ListOutOfDateCalendarEvents(ctx, time.Now().Add(-48*time.Hour)) + if err != nil { + return fmt.Errorf("list out of date calendar events: %w", err) + } + for _, outOfDateCalendarEvent := range outOfDateCalendarEvents { + if err := deleteCalendarEvent(ctx, ds, userCalendar, outOfDateCalendarEvent); err != nil { + return fmt.Errorf("delete user calendar event: %w", err) + } + } + + return nil +} + +func deleteAllCalendarEvents( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + teamID *uint, +) error { + calendarEvents, err := ds.ListCalendarEvents(ctx, teamID) + if err != nil { + return fmt.Errorf("list calendar events: %w", err) + } + for _, calendarEvent := range calendarEvents { + if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { + return fmt.Errorf("delete user calendar event: %w", err) + } + } + return nil +} + +func deleteTeamCalendarEvents( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + team fleet.Team, +) error { + if team.Config.Integrations.GoogleCalendar != nil && + team.Config.Integrations.GoogleCalendar.Enable { + // Feature is enabled, nothing to cleanup. + return nil + } + return deleteAllCalendarEvents(ctx, ds, userCalendar, &team.ID) +} + +func deleteCalendarEvent(ctx context.Context, ds fleet.Datastore, userCalendar fleet.UserCalendar, calendarEvent *fleet.CalendarEvent) error { + if userCalendar != nil { + // Only delete events from the user's calendar if the event is in the future. + if eventInFuture := time.Now().Before(calendarEvent.StartTime); eventInFuture { + if err := userCalendar.Configure(calendarEvent.Email); err != nil { + return fmt.Errorf("connect to user calendar: %w", err) + } + if err := userCalendar.DeleteEvent(calendarEvent); err != nil { + return fmt.Errorf("delete calendar event: %w", err) + } + } + } + if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { + return fmt.Errorf("delete db calendar event: %w", err) + } + return nil +} diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go index 680cf50d95..905b79c879 100644 --- a/cmd/fleet/calendar_cron_test.go +++ b/cmd/fleet/calendar_cron_test.go @@ -12,34 +12,61 @@ func TestGetPreferredCalendarEventDate(t *testing.T) { return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) } for _, tc := range []struct { - name string - year int - month time.Month - days int + name string + year int + month time.Month + daysStart int + daysEnd int + webhookFiredThisMonth bool expected time.Time }{ { - year: 2024, - month: 3, - days: 31, - name: "March 2024", + name: "March 2024 (webhook hasn't fired)", + year: 2024, + month: 3, + daysStart: 1, + daysEnd: 31, + webhookFiredThisMonth: false, + expected: date(2024, 3, 19), }, { - year: 2024, - month: 4, - days: 30, - name: "April 2024", + name: "March 2024 (webhook has fired, days before 3rd Tuesday)", + year: 2024, + month: 3, + daysStart: 1, + daysEnd: 18, + webhookFiredThisMonth: true, + + expected: date(2024, 3, 19), + }, + { + name: "March 2024 (webhook has fired, days after 3rd Tuesday)", + year: 2024, + month: 3, + daysStart: 20, + daysEnd: 30, + webhookFiredThisMonth: true, + + expected: date(2024, 4, 16), + }, + { + name: "April 2024 (webhook hasn't fired)", + year: 2024, + month: 4, + daysEnd: 30, + webhookFiredThisMonth: false, + expected: date(2024, 4, 16), }, } { t.Run(tc.name, func(t *testing.T) { - for day := 1; day <= tc.days; day++ { - actual := getPreferredCalendarEventDate(tc.year, tc.month, day) + for day := tc.daysStart; day <= tc.daysEnd; day++ { + actual := getPreferredCalendarEventDate(tc.year, tc.month, day, tc.webhookFiredThisMonth) require.NotEqual(t, actual.Weekday(), time.Saturday) require.NotEqual(t, actual.Weekday(), time.Sunday) - if day <= tc.expected.Day() { + if day <= tc.expected.Day() || tc.webhookFiredThisMonth { require.Equal(t, tc.expected, actual) } else { today := date(tc.year, tc.month, day) diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go index 3990915976..5ffc0f77f3 100644 --- a/server/datastore/mysql/calendar_events.go +++ b/server/datastore/mysql/calendar_events.go @@ -11,15 +11,16 @@ import ( "github.com/jmoiron/sqlx" ) -func (ds *Datastore) NewCalendarEvent( +func (ds *Datastore) CreateOrUpdateCalendarEvent( ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, + webhookStatus fleet.CalendarWebhookStatus, ) (*fleet.CalendarEvent, error) { - var calendarEvent *fleet.CalendarEvent + var id int64 if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { const calendarEventsQuery = ` INSERT INTO calendar_events ( @@ -27,7 +28,12 @@ func (ds *Datastore) NewCalendarEvent( start_time, end_time, event - ) VALUES (?, ?, ?, ?); + ) VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + start_time = VALUES(start_time), + end_time = VALUES(end_time), + event = VALUES(event), + updated_at = CURRENT_TIMESTAMP; ` result, err := tx.ExecContext( ctx, @@ -41,13 +47,13 @@ func (ds *Datastore) NewCalendarEvent( return ctxerr.Wrap(ctx, err, "insert calendar event") } - id, _ := result.LastInsertId() - calendarEvent = &fleet.CalendarEvent{ - ID: uint(id), - Email: email, - StartTime: startTime, - EndTime: endTime, - Data: data, + if insertOnDuplicateDidInsert(result) { + id, _ = result.LastInsertId() + } else { + stmt := `SELECT id FROM calendar_events WHERE email = ?` + if err := sqlx.GetContext(ctx, tx, &id, stmt, email); err != nil { + return ctxerr.Wrap(ctx, err, "query mdm solution id") + } } const hostCalendarEventsQuery = ` @@ -55,14 +61,17 @@ func (ds *Datastore) NewCalendarEvent( host_id, calendar_event_id, webhook_status - ) VALUES (?, ?, ?); + ) VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE + webhook_status = VALUES(webhook_status), + calendar_event_id = VALUES(calendar_event_id); ` result, err = tx.ExecContext( ctx, hostCalendarEventsQuery, hostID, - calendarEvent.ID, - fleet.CalendarWebhookStatusPending, + id, + webhookStatus, ) if err != nil { return ctxerr.Wrap(ctx, err, "insert host calendar event") @@ -71,9 +80,29 @@ func (ds *Datastore) NewCalendarEvent( }); err != nil { return nil, ctxerr.Wrap(ctx, err) } + + calendarEvent, err := getCalendarEventByID(ctx, ds.writer(ctx), uint(id)) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get created calendar event by id") + } return calendarEvent, nil } +func getCalendarEventByID(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.CalendarEvent, error) { + const calendarEventsQuery = ` + SELECT * FROM calendar_events WHERE id = ?; + ` + var calendarEvent fleet.CalendarEvent + err := sqlx.GetContext(ctx, q, &calendarEvent, calendarEventsQuery, id) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithID(id)) + } + return nil, ctxerr.Wrap(ctx, err, "get calendar event") + } + return &calendarEvent, nil +} + func (ds *Datastore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { const calendarEventsQuery = ` SELECT * FROM calendar_events WHERE email = ?; @@ -94,7 +123,8 @@ func (ds *Datastore) UpdateCalendarEvent(ctx context.Context, calendarEventID ui UPDATE calendar_events SET start_time = ?, end_time = ?, - event = ? + event = ?, + updated_at = CURRENT_TIMESTAMP WHERE id = ?; ` if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, startTime, endTime, data, calendarEventID); err != nil { @@ -148,3 +178,37 @@ func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID } return nil } + +func (ds *Datastore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) { + calendarEventsQuery := ` + SELECT ce.* FROM calendar_events ce + ` + + var args []interface{} + if teamID != nil { + // TODO(lucas): Should we add a team_id column to calendar_events? + calendarEventsQuery += ` JOIN host_calendar_events hce ON ce.id=hce.calendar_event_id + JOIN hosts h ON h.id=hce.host_id WHERE h.team_id = ?` + args = append(args, *teamID) + } + + var calendarEvents []*fleet.CalendarEvent + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, args...); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, ctxerr.Wrap(ctx, err, "get all calendar events") + } + return calendarEvents, nil +} + +func (ds *Datastore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) { + calendarEventsQuery := ` + SELECT ce.* FROM calendar_events ce WHERE updated_at < ? + ` + var calendarEvents []*fleet.CalendarEvent + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, t); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get all calendar events") + } + return calendarEvents, nil +} diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go index ccf07b3c7b..3c6030adfa 100644 --- a/server/datastore/mysql/calendar_events_test.go +++ b/server/datastore/mysql/calendar_events_test.go @@ -1,6 +1,128 @@ package mysql -import "testing" +import ( + "context" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/require" +) func TestCalendarEvents(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"UpdateCalendarEvent", testUpdateCalendarEvent}, + {"CreateOrUpdateCalendarEvent", testCreateOrUpdateCalendarEvent}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + + c.fn(t, ds) + }) + } +} + +func testUpdateCalendarEvent(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: ptr.String("1"), + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{ + { + HostID: host.ID, + Email: "foo@example.com", + Source: "google_chrome_profiles", + }, + }, "google_chrome_profiles") + require.NoError(t, err) + + startTime1 := time.Now() + endTime1 := startTime1.Add(30 * time.Minute) + calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone) + require.NoError(t, err) + + time.Sleep(1 * time.Second) + + err = ds.UpdateCalendarEvent(ctx, calendarEvent.ID, startTime1, endTime1, []byte(`{}`)) + require.NoError(t, err) + + calendarEvent2, err := ds.GetCalendarEvent(ctx, "foo@example.com") + require.NoError(t, err) + require.NotEqual(t, *calendarEvent, *calendarEvent2) + calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt + require.Equal(t, *calendarEvent, *calendarEvent2) + + // TODO(lucas): Add more tests here. +} + +func testCreateOrUpdateCalendarEvent(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: ptr.String("1"), + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{ + { + HostID: host.ID, + Email: "foo@example.com", + Source: "google_chrome_profiles", + }, + }, "google_chrome_profiles") + require.NoError(t, err) + + startTime1 := time.Now() + endTime1 := startTime1.Add(30 * time.Minute) + calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone) + require.NoError(t, err) + + time.Sleep(1 * time.Second) + + calendarEvent2, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone) + require.NoError(t, err) + require.Greater(t, calendarEvent2.UpdatedAt, calendarEvent.UpdatedAt) + calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt + require.Equal(t, *calendarEvent, *calendarEvent2) + + time.Sleep(1 * time.Second) + + startTime2 := startTime1.Add(1 * time.Hour) + endTime2 := startTime1.Add(30 * time.Minute) + calendarEvent3, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime2, endTime2, []byte(`{"foo": "bar"}`), host.ID, fleet.CalendarWebhookStatusPending) + require.NoError(t, err) + require.Greater(t, calendarEvent3.UpdatedAt, calendarEvent2.UpdatedAt) + require.WithinDuration(t, startTime2, calendarEvent3.StartTime, 1*time.Second) + require.WithinDuration(t, endTime2, calendarEvent3.EndTime, 1*time.Second) + require.Equal(t, string(calendarEvent3.Data), `{"foo": "bar"}`) + + calendarEvent3b, err := ds.GetCalendarEvent(ctx, "foo@example.com") + require.NoError(t, err) + require.Equal(t, calendarEvent3, calendarEvent3b) + + // TODO(lucas): Add more tests here. } diff --git a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go index e9222e9d91..a385a1da88 100644 --- a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go +++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go @@ -21,7 +21,9 @@ func Up_20240314085226(tx *sql.Tx) error { event JSON NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + UNIQUE KEY idx_one_calendar_event_per_email (email) ); `); err != nil { return fmt.Errorf("create calendar_events table: %w", err) diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 0b3498319d..b711c8b932 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -1172,6 +1172,7 @@ func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fl } // TODO(lucas): Must be tested at scale. +// TODO(lucas): Filter out hosts with team_id == NULL func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) { query := ` SELECT diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a256591e25..cc3c88c7ed 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -52,7 +52,8 @@ CREATE TABLE `calendar_events` ( `event` json NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) + PRIMARY KEY (`id`), + UNIQUE KEY `idx_one_calendar_event_per_email` (`email`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go index 348cb074a4..3152ee65bb 100644 --- a/server/fleet/calendar_events.go +++ b/server/fleet/calendar_events.go @@ -15,7 +15,8 @@ type CalendarEvent struct { type CalendarWebhookStatus int const ( - CalendarWebhookStatusPending CalendarWebhookStatus = iota + CalendarWebhookStatusNone CalendarWebhookStatus = iota + CalendarWebhookStatusPending CalendarWebhookStatusSent ) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index f2178bf32b..65098efdfc 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -619,12 +619,14 @@ type Datastore interface { /////////////////////////////////////////////////////////////////////////////// // Calendar events - NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*CalendarEvent, error) + CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus CalendarWebhookStatus) (*CalendarEvent, error) GetCalendarEvent(ctx context.Context, email string) (*CalendarEvent, error) DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error + ListCalendarEvents(ctx context.Context, teamID *uint) ([]*CalendarEvent, error) + ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*CalendarEvent, error) /////////////////////////////////////////////////////////////////////////////// // Team Policies diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 4e35d1eef3..1b77b29cbe 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -462,7 +462,7 @@ type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error -type NewCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) +type CreateOrUpdateCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error) type GetCalendarEventFunc func(ctx context.Context, email string) (*fleet.CalendarEvent, error) @@ -474,6 +474,10 @@ type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.Hos type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error +type ListCalendarEventsFunc func(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) + +type ListOutOfDateCalendarEventsFunc func(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) + type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error) @@ -1541,8 +1545,8 @@ type DataStore struct { DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFuncInvoked bool - NewCalendarEventFunc NewCalendarEventFunc - NewCalendarEventFuncInvoked bool + CreateOrUpdateCalendarEventFunc CreateOrUpdateCalendarEventFunc + CreateOrUpdateCalendarEventFuncInvoked bool GetCalendarEventFunc GetCalendarEventFunc GetCalendarEventFuncInvoked bool @@ -1559,6 +1563,12 @@ type DataStore struct { UpdateHostCalendarWebhookStatusFunc UpdateHostCalendarWebhookStatusFunc UpdateHostCalendarWebhookStatusFuncInvoked bool + ListCalendarEventsFunc ListCalendarEventsFunc + ListCalendarEventsFuncInvoked bool + + ListOutOfDateCalendarEventsFunc ListOutOfDateCalendarEventsFunc + ListOutOfDateCalendarEventsFuncInvoked bool + NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFuncInvoked bool @@ -3716,11 +3726,11 @@ func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source f return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration) } -func (s *DataStore) NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) { +func (s *DataStore) CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error) { s.mu.Lock() - s.NewCalendarEventFuncInvoked = true + s.CreateOrUpdateCalendarEventFuncInvoked = true s.mu.Unlock() - return s.NewCalendarEventFunc(ctx, email, startTime, endTime, data, hostID) + return s.CreateOrUpdateCalendarEventFunc(ctx, email, startTime, endTime, data, hostID, webhookStatus) } func (s *DataStore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { @@ -3758,6 +3768,20 @@ func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID return s.UpdateHostCalendarWebhookStatusFunc(ctx, hostID, status) } +func (s *DataStore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) { + s.mu.Lock() + s.ListCalendarEventsFuncInvoked = true + s.mu.Unlock() + return s.ListCalendarEventsFunc(ctx, teamID) +} + +func (s *DataStore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) { + s.mu.Lock() + s.ListOutOfDateCalendarEventsFuncInvoked = true + s.mu.Unlock() + return s.ListOutOfDateCalendarEventsFunc(ctx, t) +} + func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { s.mu.Lock() s.NewTeamPolicyFuncInvoked = true diff --git a/tools/webhook/README.md b/tools/webhook/README.md new file mode 100644 index 0000000000..6bdb2416a7 --- /dev/null +++ b/tools/webhook/README.md @@ -0,0 +1,16 @@ +# webhook + +Test tool for Fleet features that use webhook URLs. +It reads and parses the request a JSON body and prints the JSON to standard output (with indentation). + +```sh +go run ./tools/webhook 8082 +2024/03/20 09:10:00 { + "error": "No fleetdm.com Google account associated with this host.", + "host_display_name": "dChYnk.uxURT", + "host_id": 2, + "host_serial_number": "", + "timestamp": "2024-03-20T09:10:00.129982-03:00" +} +... +``` diff --git a/tools/webhook/main.go b/tools/webhook/main.go new file mode 100644 index 0000000000..452e0a15ca --- /dev/null +++ b/tools/webhook/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "io" + "log" + "net/http" + "os" +) + +func main() { + log.SetFlags(log.LstdFlags) + + http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("failed to read body: %s", err) + return + } + + var v interface{} + if err := json.Unmarshal(body, &v); err != nil { + log.Printf("failed to parse JSON body: %s", err) + return + } + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + panic(err) + } + log.Printf("%s", b) + + w.WriteHeader(http.StatusOK) + })) + //nolint:gosec // G114: file server used for testing purposes only. + err := http.ListenAndServe("0.0.0.0:"+os.Args[1], nil) + if err != nil { + panic(err) + } +}