| | | 1 | | using System.Collections.Concurrent; |
| | | 2 | | using Hangfire.Common; |
| | | 3 | | using Hangfire.Server; |
| | | 4 | | using Services.ApiServices.TelegramBotApi; |
| | | 5 | | |
| | | 6 | | namespace OzonOrdersWeb.Observability; |
| | | 7 | | |
| | | 8 | | /// <summary> |
| | | 9 | | /// Отправляет в Telegram необработанные исключения Hangfire-джобов. Отмена и остановка сервера ошибкой не считаются. |
| | | 10 | | /// Повторы одной и той же джобы глушатся на <see cref="Cooldown"/>, чтобы падающий каждую минуту recurring не заспамил |
| | | 11 | | /// </summary> |
| | 0 | 12 | | public sealed class HangfireFailureTelegramFilter(IServiceProvider services, TimeSpan? cooldown = null) : JobFilterAttri |
| | | 13 | | { |
| | 0 | 14 | | private static readonly ConcurrentDictionary<string, DateTime> LastSentUtc = new(); |
| | 0 | 15 | | private readonly TimeSpan Cooldown = cooldown ?? TimeSpan.FromMinutes(30); |
| | | 16 | | |
| | | 17 | | public void OnPerforming(PerformingContext filterContext) |
| | | 18 | | { |
| | 0 | 19 | | } |
| | | 20 | | |
| | | 21 | | public void OnPerformed(PerformedContext filterContext) |
| | | 22 | | { |
| | 0 | 23 | | var ex = filterContext.Exception; |
| | 0 | 24 | | if (ex is null || filterContext.ExceptionHandled) return; |
| | 0 | 25 | | if (ex is OperationCanceledException || |
| | 0 | 26 | | (ex is AggregateException agg && agg.InnerExceptions.All(e => e is OperationCanceledException))) return; |
| | | 27 | | |
| | 0 | 28 | | var job = filterContext.BackgroundJob.Job; |
| | 0 | 29 | | var jobName = $"{job.Type.Name}.{job.Method.Name}"; |
| | 0 | 30 | | if (!ShouldSend(jobName, DateTime.UtcNow)) return; |
| | | 31 | | |
| | | 32 | | try |
| | | 33 | | { |
| | 0 | 34 | | var notifier = services.GetService<ITelegramNotifier>(); |
| | 0 | 35 | | if (notifier is not { IsConfigured: true }) return; |
| | | 36 | | |
| | 0 | 37 | | var retry = filterContext.GetJobParameter<int>("RetryCount"); |
| | 0 | 38 | | var message = TelegramMessageFactory.Exception($"Hangfire: {jobName}", ex, |
| | 0 | 39 | | new Dictionary<string, string?> |
| | 0 | 40 | | { |
| | 0 | 41 | | ["Job Id"] = filterContext.BackgroundJob.Id, |
| | 0 | 42 | | ["Попытка"] = retry > 0 ? (retry + 1).ToString() : null |
| | 0 | 43 | | }); |
| | | 44 | | // Не ждём Telegram: воркер освобождается сразу. SendAsync исключений не бросает. |
| | 0 | 45 | | _ = notifier.SendAsync(message); |
| | 0 | 46 | | } |
| | 0 | 47 | | catch |
| | | 48 | | { |
| | | 49 | | // Уведомление не должно влиять на выполнение джобов. |
| | 0 | 50 | | } |
| | 0 | 51 | | } |
| | | 52 | | |
| | | 53 | | private bool ShouldSend(string jobName, DateTime nowUtc) |
| | | 54 | | { |
| | | 55 | | while (true) |
| | | 56 | | { |
| | 0 | 57 | | if (!LastSentUtc.TryGetValue(jobName, out var last)) |
| | | 58 | | { |
| | 0 | 59 | | if (LastSentUtc.TryAdd(jobName, nowUtc)) return true; |
| | | 60 | | continue; |
| | | 61 | | } |
| | | 62 | | |
| | 0 | 63 | | if (nowUtc - last < Cooldown) return false; |
| | 0 | 64 | | if (LastSentUtc.TryUpdate(jobName, nowUtc, last)) return true; |
| | | 65 | | } |
| | | 66 | | } |
| | | 67 | | } |