| | | 1 | | using Serilog; |
| | | 2 | | using Services.HangfireService; |
| | | 3 | | |
| | | 4 | | namespace OzonOrdersWeb.HostedServices; |
| | | 5 | | |
| | | 6 | | /// <summary> |
| | | 7 | | /// Вотчдог планировщика Hangfire: замечает, что recurring-джобы не ставятся при живых серверах, |
| | | 8 | | /// откатывает осиротевшие XA PREPARED-транзакции (они держат блокировки таблиц Hangfire), |
| | | 9 | | /// чистит джобы без состояния и пишет Log.Error (лог-вотчер заведёт issue). |
| | | 10 | | /// </summary> |
| | | 11 | | /// <remarks> |
| | | 12 | | /// Healthcheck и heartbeat серверов этот сбой не показывают: воркеры живы, просто планировщику |
| | | 13 | | /// нечего им отдать. Инцидент 09–14.09.2026 — пять дней без загрузки заказов. |
| | | 14 | | /// Конфиг: <c>Hangfire:Watchdog:{Enabled,IntervalMinutes,StuckThresholdMinutes}</c>. |
| | | 15 | | /// </remarks> |
| | | 16 | | public sealed class HangfireSchedulerWatchdogHostedService : BackgroundService |
| | | 17 | | { |
| | 0 | 18 | | private static readonly TimeSpan StatelessJobAge = TimeSpan.FromMinutes(5); |
| | 0 | 19 | | private static readonly TimeSpan StartupDelay = TimeSpan.FromMinutes(3); |
| | | 20 | | |
| | | 21 | | private readonly IServiceProvider _services; |
| | | 22 | | private readonly IConfiguration _configuration; |
| | | 23 | | |
| | 0 | 24 | | private HashSet<string> _xaSeenOnPreviousPass = new(StringComparer.Ordinal); |
| | | 25 | | private bool _alertActive; |
| | | 26 | | |
| | 0 | 27 | | public HangfireSchedulerWatchdogHostedService(IServiceProvider services, IConfiguration configuration) |
| | | 28 | | { |
| | 0 | 29 | | _services = services; |
| | 0 | 30 | | _configuration = configuration; |
| | 0 | 31 | | } |
| | | 32 | | |
| | | 33 | | protected override async Task ExecuteAsync(CancellationToken stoppingToken) |
| | | 34 | | { |
| | 0 | 35 | | if (!_configuration.GetValue("Hangfire:ServerEnabled", true) |
| | 0 | 36 | | || !_configuration.GetValue("Hangfire:Watchdog:Enabled", true)) |
| | | 37 | | { |
| | 0 | 38 | | Log.Information("Hangfire watchdog: отключён конфигурацией"); |
| | 0 | 39 | | return; |
| | | 40 | | } |
| | | 41 | | |
| | 0 | 42 | | var interval = TimeSpan.FromMinutes(Math.Max(1, _configuration.GetValue("Hangfire:Watchdog:IntervalMinutes", 2)) |
| | 0 | 43 | | var stuckThreshold = TimeSpan.FromMinutes(Math.Max(1, _configuration.GetValue("Hangfire:Watchdog:StuckThresholdM |
| | | 44 | | |
| | | 45 | | // После деплоя планировщик сам догоняет просроченное — не поднимать тревогу на старте. |
| | | 46 | | try |
| | | 47 | | { |
| | 0 | 48 | | await Task.Delay(StartupDelay, stoppingToken); |
| | 0 | 49 | | } |
| | 0 | 50 | | catch (OperationCanceledException) |
| | | 51 | | { |
| | 0 | 52 | | return; |
| | | 53 | | } |
| | | 54 | | |
| | 0 | 55 | | using var timer = new PeriodicTimer(interval); |
| | | 56 | | do |
| | | 57 | | { |
| | | 58 | | try |
| | | 59 | | { |
| | 0 | 60 | | await RunPassAsync(stuckThreshold, stoppingToken); |
| | 0 | 61 | | } |
| | 0 | 62 | | catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) |
| | | 63 | | { |
| | 0 | 64 | | return; |
| | | 65 | | } |
| | 0 | 66 | | catch (Exception ex) |
| | | 67 | | { |
| | 0 | 68 | | Log.Warning(ex, "Hangfire watchdog: проход не удался"); |
| | 0 | 69 | | } |
| | 0 | 70 | | } while (await timer.WaitForNextTickAsync(stoppingToken)); |
| | 0 | 71 | | } |
| | | 72 | | |
| | | 73 | | private async Task RunPassAsync(TimeSpan stuckThreshold, CancellationToken ct) |
| | | 74 | | { |
| | 0 | 75 | | using var scope = _services.CreateScope(); |
| | 0 | 76 | | var storage = scope.ServiceProvider.GetRequiredService<HangfireSchedulerWatchdogStorage>(); |
| | | 77 | | |
| | 0 | 78 | | var nowUtc = DateTime.UtcNow; |
| | 0 | 79 | | var observation = await storage.ObserveAsync(nowUtc, StatelessJobAge, ct); |
| | 0 | 80 | | var verdict = HangfireSchedulerWatchdogAnalyzer.Analyze(observation, _xaSeenOnPreviousPass, stuckThreshold); |
| | 0 | 81 | | _xaSeenOnPreviousPass = observation.PreparedXaTransactions.Select(x => x.Xid).ToHashSet(StringComparer.Ordinal); |
| | | 82 | | |
| | 0 | 83 | | foreach (var xa in verdict.XaToRollback) |
| | | 84 | | { |
| | 0 | 85 | | var error = await storage.RollbackXaAsync(xa, ct); |
| | 0 | 86 | | if (error is null) |
| | | 87 | | { |
| | 0 | 88 | | Log.Warning("Hangfire watchdog: откатил осиротевшую XA-транзакцию {Xid}", xa.Xid); |
| | | 89 | | } |
| | | 90 | | else |
| | | 91 | | { |
| | 0 | 92 | | Log.Error("Hangfire watchdog: не удалось откатить XA-транзакцию {Xid}: {Error}", xa.Xid, error); |
| | | 93 | | } |
| | 0 | 94 | | } |
| | | 95 | | |
| | 0 | 96 | | if (verdict.OrphanedStatelessJobs > 0) |
| | | 97 | | { |
| | 0 | 98 | | var deleted = await storage.DeleteStatelessJobsAsync(nowUtc - StatelessJobAge, ct); |
| | 0 | 99 | | Log.Warning("Hangfire watchdog: удалено {Count} джобов без состояния", deleted); |
| | | 100 | | } |
| | | 101 | | |
| | 0 | 102 | | if (verdict.SchedulerStuck) |
| | | 103 | | { |
| | 0 | 104 | | if (!_alertActive) |
| | | 105 | | { |
| | 0 | 106 | | _alertActive = true; |
| | 0 | 107 | | var oldest = verdict.StuckJobs[0]; |
| | 0 | 108 | | Log.Error( |
| | 0 | 109 | | "Hangfire watchdog: планировщик стоит — {Count} recurring-джобов просрочены при живых серверах, самы |
| | 0 | 110 | | verdict.StuckJobs.Count, oldest.Id, oldest.NextExecutionUtc, observation.PreparedXaTransactions.Coun |
| | | 111 | | } |
| | | 112 | | } |
| | 0 | 113 | | else if (_alertActive) |
| | | 114 | | { |
| | 0 | 115 | | _alertActive = false; |
| | 0 | 116 | | Log.Information("Hangfire watchdog: планировщик снова ставит recurring-джобы"); |
| | | 117 | | } |
| | 0 | 118 | | } |
| | | 119 | | } |