| | | 1 | | using Microsoft.EntityFrameworkCore; |
| | | 2 | | using Microsoft.Extensions.Caching.Memory; |
| | | 3 | | using Microsoft.Extensions.Options; |
| | | 4 | | using OzonDomains.Models; |
| | | 5 | | using OzonRepositories.Context; |
| | | 6 | | |
| | | 7 | | namespace OzonOrdersWeb.WebServices.SystemAlerts; |
| | | 8 | | |
| | | 9 | | public sealed class SystemAlertIncidentStore( |
| | | 10 | | ApplicationDbContext context, |
| | | 11 | | IOptions<SystemAlertsOptions>? options = null, |
| | | 12 | | IMemoryCache? cache = null, |
| | | 13 | | SystemAlertOrderIndex? orderIndex = null) |
| | | 14 | | { |
| | | 15 | | /// <summary>Сколько заказов хранится в инциденте: столько же умещается в ссылку на сетку заказов.</summary> |
| | | 16 | | private const int StoredOrdersLimit = SystemAlertLinks.MaxOrdersInLink; |
| | | 17 | | |
| | | 18 | | private const string CountsCacheKey = "system-alert-counts"; |
| | | 19 | | private const string IdsCacheKey = "system-alert-ids"; |
| | | 20 | | |
| | | 21 | | private const int TitleMaxLength = 500; |
| | | 22 | | private const int DetailsMaxLength = 1000; |
| | | 23 | | private const int UrlMaxLength = 2000; |
| | | 24 | | |
| | | 25 | | private readonly SystemAlertsOptions _options = options?.Value ?? new SystemAlertsOptions(); |
| | | 26 | | |
| | | 27 | | /// <summary> |
| | | 28 | | /// Активные подтверждённые инциденты, которые видны админам: без «наблюдаемых» (debounce), |
| | | 29 | | /// принятых к сведению и заглушённых. |
| | | 30 | | /// </summary> |
| | | 31 | | public Task<List<SystemAlertIncident>> GetActiveAsync(CancellationToken cancellationToken) |
| | | 32 | | { |
| | | 33 | | var now = DateTime.UtcNow; |
| | | 34 | | return context.SystemAlertIncidents.AsNoTracking() |
| | | 35 | | .Where(x => x.IsActive && x.ConfirmedAtUtc != null && x.AcknowledgedAtUtc == null |
| | | 36 | | && (x.MutedUntilUtc == null || x.MutedUntilUtc <= now)) |
| | | 37 | | .OrderBy(x => x.Severity == "danger" ? 0 : x.Severity == "warning" ? 1 : 2) |
| | | 38 | | .ThenByDescending(x => x.LastObservedAtUtc).ToListAsync(cancellationToken); |
| | | 39 | | } |
| | | 40 | | |
| | | 41 | | /// <summary> |
| | | 42 | | /// Счётчики видимых инцидентов по severity для чипов в шапке: один сгруппированный запрос и короткий кэш, |
| | | 43 | | /// который сбрасывается при любом изменении инцидентов. |
| | | 44 | | /// </summary> |
| | | 45 | | public async Task<SystemAlertCounts> GetActiveCountsAsync(CancellationToken cancellationToken) |
| | | 46 | | { |
| | | 47 | | if (cache != null && cache.TryGetValue(CountsCacheKey, out SystemAlertCounts? cached) && cached != null) |
| | | 48 | | return cached; |
| | | 49 | | |
| | | 50 | | var now = DateTime.UtcNow; |
| | | 51 | | var rows = await context.SystemAlertIncidents.AsNoTracking() |
| | | 52 | | .Where(x => x.IsActive && x.ConfirmedAtUtc != null && x.AcknowledgedAtUtc == null |
| | | 53 | | && (x.MutedUntilUtc == null || x.MutedUntilUtc <= now)) |
| | | 54 | | .GroupBy(x => x.Severity) |
| | | 55 | | .Select(g => new { Severity = g.Key, Count = g.Count() }) |
| | | 56 | | .ToListAsync(cancellationToken); |
| | | 57 | | int Count(string severity) => rows.Where(x => x.Severity == severity).Sum(x => x.Count); |
| | | 58 | | var counts = new SystemAlertCounts(Count("danger"), Count("warning"), Count("info")); |
| | | 59 | | cache?.Set(CountsCacheKey, counts, TimeSpan.FromSeconds(Math.Max(1, _options.CountsCacheSeconds))); |
| | | 60 | | return counts; |
| | | 61 | | } |
| | | 62 | | |
| | | 63 | | /// <summary> |
| | | 64 | | /// Id видимых инцидентов по severity: по ним браузер отличает новые алерты (ещё не просмотренные пользователем) |
| | | 65 | | /// от уже знакомых. Тот же фильтр и тот же кэш-срок, что у счётчиков. |
| | | 66 | | /// </summary> |
| | | 67 | | public async Task<SystemAlertIds> GetActiveIdsAsync(CancellationToken cancellationToken) |
| | | 68 | | { |
| | | 69 | | if (cache != null && cache.TryGetValue(IdsCacheKey, out SystemAlertIds? cached) && cached != null) |
| | | 70 | | return cached; |
| | | 71 | | |
| | | 72 | | var now = DateTime.UtcNow; |
| | | 73 | | var rows = await context.SystemAlertIncidents.AsNoTracking() |
| | | 74 | | .Where(x => x.IsActive && x.ConfirmedAtUtc != null && x.AcknowledgedAtUtc == null |
| | | 75 | | && (x.MutedUntilUtc == null || x.MutedUntilUtc <= now)) |
| | | 76 | | .Select(x => new { x.Id, x.Severity }) |
| | | 77 | | .ToListAsync(cancellationToken); |
| | | 78 | | List<long> Ids(string severity) => rows.Where(x => x.Severity == severity).Select(x => x.Id).OrderBy(x => x).ToL |
| | | 79 | | var ids = new SystemAlertIds(Ids("danger"), Ids("warning"), Ids("info")); |
| | | 80 | | cache?.Set(IdsCacheKey, ids, TimeSpan.FromSeconds(Math.Max(1, _options.CountsCacheSeconds))); |
| | | 81 | | return ids; |
| | | 82 | | } |
| | | 83 | | |
| | | 84 | | private void InvalidateCounts() |
| | | 85 | | { |
| | | 86 | | cache?.Remove(CountsCacheKey); |
| | | 87 | | cache?.Remove(IdsCacheKey); |
| | | 88 | | } |
| | | 89 | | |
| | | 90 | | public async Task<IReadOnlyList<SystemAlertTransition>> SyncAsync(IEnumerable<SystemAlertFinding> findings, Cancella |
| | | 91 | | { |
| | | 92 | | var now = DateTime.UtcNow; |
| | | 93 | | var confirmationChecks = Math.Max(1, _options.ConfirmationChecks); |
| | | 94 | | var list = findings.GroupBy(x => x.Key, StringComparer.Ordinal).Select(x => x.First()).ToList(); |
| | | 95 | | var active = await context.SystemAlertIncidents.Where(x => x.IsActive).OrderBy(x => x.Id).ToListAsync(cancellati |
| | | 96 | | var transitions = new List<SystemAlertTransition>(); |
| | | 97 | | var current = new Dictionary<string, SystemAlertIncident>(StringComparer.Ordinal); |
| | | 98 | | |
| | | 99 | | // Уникального индекса на (Key, IsActive) нет: при дубле оставляем самый старый, остальные закрываем. |
| | | 100 | | var byKey = new Dictionary<string, SystemAlertIncident>(StringComparer.Ordinal); |
| | | 101 | | foreach (var incident in active) |
| | | 102 | | { |
| | | 103 | | if (byKey.TryAdd(incident.Key, incident)) continue; |
| | | 104 | | incident.IsActive = false; |
| | | 105 | | incident.ResolvedAtUtc = now; |
| | | 106 | | } |
| | | 107 | | |
| | | 108 | | foreach (var finding in list) |
| | | 109 | | { |
| | | 110 | | var title = Truncate(finding.Title, TitleMaxLength); |
| | | 111 | | var details = Truncate(string.Join("\n", finding.Lines), DetailsMaxLength); |
| | | 112 | | var url = Truncate(finding.Url, UrlMaxLength); |
| | | 113 | | if (!byKey.TryGetValue(finding.Key, out var incident)) |
| | | 114 | | { |
| | | 115 | | var confirmed = !finding.Debounced || confirmationChecks <= 1; |
| | | 116 | | var created = new SystemAlertIncident |
| | | 117 | | { |
| | | 118 | | Key = finding.Key, Domain = finding.Domain, Severity = finding.Severity, Title = title, |
| | | 119 | | Details = details, Url = url, FirstObservedAtUtc = now, LastObservedAtUtc = now, |
| | | 120 | | OccurrenceCount = 1, ConsecutiveObservations = 1, IsActive = true, |
| | | 121 | | ConfirmedAtUtc = confirmed ? now : null |
| | | 122 | | }; |
| | | 123 | | ApplyOrders(created, finding); |
| | | 124 | | context.SystemAlertIncidents.Add(created); |
| | | 125 | | current[finding.Key] = created; |
| | | 126 | | if (confirmed) |
| | | 127 | | transitions.Add(new SystemAlertTransition(finding.Key, finding.Severity, title, finding.Lines, false |
| | | 128 | | finding.Domain, finding.Url, finding.Orders?.Count ?? 0)); |
| | | 129 | | continue; |
| | | 130 | | } |
| | | 131 | | |
| | | 132 | | // Проверки чаще, чем раз в ConfirmationMinGapSeconds (ручные обновления) не приближают подтверждение: |
| | | 133 | | // debounce должен отсекать сбой во времени, а не по числу кликов. |
| | | 134 | | if (incident.ConfirmedAtUtc == null && (now - incident.LastObservedAtUtc).TotalSeconds < _options.Confirmati |
| | | 135 | | continue; |
| | | 136 | | |
| | | 137 | | incident.ConsecutiveObservations = incident.ConsecutiveObservations < int.MaxValue ? incident.ConsecutiveObs |
| | | 138 | | var notify = false; |
| | | 139 | | if (incident.ConfirmedAtUtc == null) |
| | | 140 | | { |
| | | 141 | | // Проблема «на наблюдении»: открываем только после N проверок подряд, единичный сбой не мигает. |
| | | 142 | | if (incident.ConsecutiveObservations >= confirmationChecks) |
| | | 143 | | { |
| | | 144 | | incident.ConfirmedAtUtc = now; |
| | | 145 | | // Время начала считаем с первой проверки, а не с подтверждения. |
| | | 146 | | notify = true; |
| | | 147 | | } |
| | | 148 | | } |
| | | 149 | | else if (incident.Severity != finding.Severity) |
| | | 150 | | { |
| | | 151 | | // OccurrenceCount считает смены состояния (открытие, изменение severity), а не проверки джоба. |
| | | 152 | | incident.OccurrenceCount++; |
| | | 153 | | if (SeverityRank(finding.Severity) < SeverityRank(incident.Severity)) |
| | | 154 | | ClearSuppression(incident); // эскалация возвращает инцидент в поле зрения |
| | | 155 | | notify = true; |
| | | 156 | | } |
| | | 157 | | else if (incident.MutedUntilUtc <= now) |
| | | 158 | | { |
| | | 159 | | // Заглушка истекла, а проблема осталась — напоминаем один раз. |
| | | 160 | | notify = true; |
| | | 161 | | } |
| | | 162 | | |
| | | 163 | | if (incident.MutedUntilUtc <= now) |
| | | 164 | | { |
| | | 165 | | incident.MutedUntilUtc = null; |
| | | 166 | | incident.MutedBy = null; |
| | | 167 | | } |
| | | 168 | | |
| | | 169 | | incident.Domain = finding.Domain; incident.Severity = finding.Severity; incident.Title = title; |
| | | 170 | | incident.Details = details; incident.Url = url; |
| | | 171 | | incident.LastObservedAtUtc = now; |
| | | 172 | | ApplyOrders(incident, finding); |
| | | 173 | | current[finding.Key] = incident; |
| | | 174 | | if (notify && !IsSuppressed(incident, now)) |
| | | 175 | | transitions.Add(new SystemAlertTransition(finding.Key, finding.Severity, title, finding.Lines, false, |
| | | 176 | | finding.Domain, finding.Url, finding.Orders?.Count ?? 0)); |
| | | 177 | | } |
| | | 178 | | |
| | | 179 | | var keys = list.Select(x => x.Key).ToHashSet(StringComparer.Ordinal); |
| | | 180 | | foreach (var incident in byKey.Values.Where(x => !keys.Contains(x.Key))) |
| | | 181 | | { |
| | | 182 | | if (incident.ConfirmedAtUtc == null) |
| | | 183 | | { |
| | | 184 | | // Не дожили до подтверждения: это был единичный сбой, в историю не пишем. |
| | | 185 | | context.SystemAlertIncidents.Remove(incident); |
| | | 186 | | continue; |
| | | 187 | | } |
| | | 188 | | |
| | | 189 | | var suppressed = IsSuppressed(incident, now); |
| | | 190 | | incident.IsActive = false; incident.ResolvedAtUtc = now; |
| | | 191 | | if (!suppressed) |
| | | 192 | | transitions.Add(new SystemAlertTransition(incident.Key, incident.Severity, incident.Title, [], true, |
| | | 193 | | incident.Domain, incident.Url, incident.OrderCount)); |
| | | 194 | | ClearSuppression(incident); |
| | | 195 | | } |
| | | 196 | | await context.SaveChangesAsync(cancellationToken); |
| | | 197 | | InvalidateCounts(); |
| | | 198 | | PublishOrderIndex(list, current); |
| | | 199 | | return transitions; |
| | | 200 | | } |
| | | 201 | | |
| | | 202 | | private static void ApplyOrders(SystemAlertIncident incident, SystemAlertFinding finding) |
| | | 203 | | { |
| | | 204 | | incident.OrderCount = finding.Orders?.Count ?? 0; |
| | | 205 | | incident.OrdersJson = finding.Orders is { Count: > 0 } ? SystemAlertOrderIndex.Serialize(finding.Orders, StoredO |
| | | 206 | | } |
| | | 207 | | |
| | | 208 | | /// <summary> |
| | | 209 | | /// Пересобирает индекс «заказ → алерты» из подтверждённых находок. Ack и mute индекс не скрывают: |
| | | 210 | | /// заказ по-прежнему проблемный, а «тишина» относится к баннеру и чипам. |
| | | 211 | | /// </summary> |
| | | 212 | | private void PublishOrderIndex(IReadOnlyList<SystemAlertFinding> findings, Dictionary<string, SystemAlertIncident> c |
| | | 213 | | { |
| | | 214 | | if (orderIndex == null) return; |
| | | 215 | | orderIndex.Replace(findings |
| | | 216 | | .Where(f => f.Orders is { Count: > 0 } && current.TryGetValue(f.Key, out var incident) && incident.Confirmed |
| | | 217 | | .SelectMany(f => f.Orders!.Select(o => (new OrderAlertEntry(f.Key, o.Severity ?? f.Severity, f.Domain, f.Tit |
| | | 218 | | } |
| | | 219 | | |
| | | 220 | | /// <summary>После рестарта индекса нет до первой проверки алертов: достраиваем его из сохранённых инцидентов.</summ |
| | | 221 | | public async Task EnsureOrderIndexAsync(CancellationToken cancellationToken) |
| | | 222 | | { |
| | | 223 | | if (orderIndex == null || orderIndex.IsLoaded) return; |
| | | 224 | | |
| | | 225 | | var incidents = await context.SystemAlertIncidents.AsNoTracking() |
| | | 226 | | .Where(x => x.IsActive && x.ConfirmedAtUtc != null && x.OrdersJson != null) |
| | | 227 | | .Select(x => new { x.Key, x.Severity, x.Domain, x.Title, x.OrdersJson }) |
| | | 228 | | .ToListAsync(cancellationToken); |
| | | 229 | | orderIndex.Replace(incidents.SelectMany(x => SystemAlertOrderIndex.Deserialize(x.OrdersJson) |
| | | 230 | | .Select(o => (new OrderAlertEntry(x.Key, o.Severity ?? x.Severity, x.Domain, x.Title, o.Reason), o.Id)))); |
| | | 231 | | } |
| | | 232 | | |
| | | 233 | | /// <summary>Принять инцидент к сведению: не шумит до восстановления или повышения severity.</summary> |
| | | 234 | | public async Task<bool> AcknowledgeAsync(long id, string? userName, CancellationToken cancellationToken) |
| | | 235 | | { |
| | | 236 | | var incident = await FindActiveAsync(id, cancellationToken); |
| | | 237 | | if (incident == null) return false; |
| | | 238 | | incident.AcknowledgedAtUtc = DateTime.UtcNow; |
| | | 239 | | incident.AcknowledgedBy = Truncate(userName, 100); |
| | | 240 | | await context.SaveChangesAsync(cancellationToken); |
| | | 241 | | InvalidateCounts(); |
| | | 242 | | return true; |
| | | 243 | | } |
| | | 244 | | |
| | | 245 | | /// <summary>Заглушить инцидент на несколько часов (1..<see cref="SystemAlertsOptions.MaxMuteHours"/>).</summary> |
| | | 246 | | public async Task<bool> MuteAsync(long id, int hours, string? userName, CancellationToken cancellationToken) |
| | | 247 | | { |
| | | 248 | | var incident = await FindActiveAsync(id, cancellationToken); |
| | | 249 | | if (incident == null) return false; |
| | | 250 | | var clamped = Math.Clamp(hours, 1, Math.Max(1, _options.MaxMuteHours)); |
| | | 251 | | incident.MutedUntilUtc = DateTime.UtcNow.AddHours(clamped); |
| | | 252 | | incident.MutedBy = Truncate(userName, 100); |
| | | 253 | | await context.SaveChangesAsync(cancellationToken); |
| | | 254 | | InvalidateCounts(); |
| | | 255 | | return true; |
| | | 256 | | } |
| | | 257 | | |
| | | 258 | | /// <summary>Снять подтверждение и заглушку.</summary> |
| | | 259 | | public async Task<bool> RestoreAsync(long id, CancellationToken cancellationToken) |
| | | 260 | | { |
| | | 261 | | var incident = await FindActiveAsync(id, cancellationToken); |
| | | 262 | | if (incident == null) return false; |
| | | 263 | | ClearSuppression(incident); |
| | | 264 | | await context.SaveChangesAsync(cancellationToken); |
| | | 265 | | InvalidateCounts(); |
| | | 266 | | return true; |
| | | 267 | | } |
| | | 268 | | |
| | | 269 | | private Task<SystemAlertIncident?> FindActiveAsync(long id, CancellationToken cancellationToken) => |
| | | 270 | | context.SystemAlertIncidents.FirstOrDefaultAsync(x => x.Id == id && x.IsActive && x.ConfirmedAtUtc != null, canc |
| | | 271 | | |
| | | 272 | | public static bool IsSuppressed(SystemAlertIncident incident, DateTime nowUtc) => |
| | | 273 | | incident.AcknowledgedAtUtc != null || incident.MutedUntilUtc > nowUtc; |
| | | 274 | | |
| | | 275 | | private static void ClearSuppression(SystemAlertIncident incident) |
| | | 276 | | { |
| | | 277 | | incident.AcknowledgedAtUtc = null; incident.AcknowledgedBy = null; |
| | | 278 | | incident.MutedUntilUtc = null; incident.MutedBy = null; |
| | | 279 | | } |
| | | 280 | | |
| | | 281 | | private static int SeverityRank(string severity) => severity switch { "danger" => 0, "warning" => 1, _ => 2 }; |
| | | 282 | | |
| | | 283 | | /// <summary>Поля инцидента ограничены схемой: превышение в strict-режиме MariaDB уронило бы весь SaveChanges.</summ |
| | | 284 | | private static string Truncate(string? value, int maxLength) => |
| | | 285 | | string.IsNullOrEmpty(value) || value.Length <= maxLength ? value ?? string.Empty : string.Concat(value.AsSpan(0, |
| | | 286 | | } |
| | | 287 | | |
| | 7 | 288 | | public sealed record SystemAlertCounts(int Danger, int Warning, int Info); |
| | | 289 | | |
| | | 290 | | public sealed record SystemAlertIds(List<long> Danger, List<long> Warning, List<long> Info); |
| | | 291 | | |
| | | 292 | | public sealed record SystemAlertTransition(string Key, string Severity, string Title, IReadOnlyList<string> Lines, bool |
| | | 293 | | string Domain = "", string? Url = null, int OrderCount = 0); |