| | | 1 | | using Microsoft.EntityFrameworkCore; |
| | | 2 | | using OzonDomains.Models; |
| | | 3 | | using OzonOrdersWeb.WebServices.JobLogs; |
| | | 4 | | using OzonOrdersWeb.WebServices.Orders; |
| | | 5 | | using OzonOrdersWeb.Observability; |
| | | 6 | | using OzonRepositories.Context; |
| | | 7 | | using Services.OzonWebhookServices.Monitoring; |
| | | 8 | | using Services.SearchServices.OrderSearch; |
| | | 9 | | using Microsoft.Extensions.Options; |
| | | 10 | | using System.Security.Cryptography; |
| | | 11 | | using System.Text; |
| | | 12 | | |
| | | 13 | | namespace OzonOrdersWeb.WebServices.SystemAlerts; |
| | | 14 | | |
| | | 15 | | public sealed record SystemAlertFinding(string Key, string Domain, string Severity, string Title, IReadOnlyList<string> |
| | | 16 | | bool Debounced = false, IReadOnlyList<SystemAlertOrderRef>? Orders = null); |
| | | 17 | | |
| | | 18 | | /// <summary>Заказ, затронутый находкой, и короткая причина — для значка в сетках заказов и колонки «Заказы» на вкладке |
| | | 19 | | /// <param name="Severity"> |
| | | 20 | | /// Серьёзность для значка именно этого заказа; null — как у находки. Просрочки (дедлайн отгрузки, срок поставки, |
| | | 21 | | /// «статус не менялся») показываем жёлтым, даже если сама находка критична. |
| | | 22 | | /// </param> |
| | | 23 | | public sealed record SystemAlertOrderRef(int OrderId, string Reason, string? Severity = null) |
| | | 24 | | { |
| | | 25 | | public const int MaxReasonLength = 120; |
| | | 26 | | |
| | | 27 | | /// <summary>Склеивает причины по одному заказу, убирает дубли и обрезает текст.</summary> |
| | | 28 | | public static IReadOnlyList<SystemAlertOrderRef> Build(IEnumerable<(int OrderId, string Reason)> items, string? seve |
| | | 29 | | .GroupBy(x => x.OrderId).OrderByDescending(g => g.Key) |
| | | 30 | | .Select(g => new SystemAlertOrderRef(g.Key, Trim(string.Join("; ", g.Select(x => x.Reason).Where(x => !string.Is |
| | | 31 | | .ToList(); |
| | | 32 | | |
| | | 33 | | /// <summary>Причина «1С не провела документ»: текст ошибки 1С часто уже начинается со слов «Не проведено» — не повт |
| | | 34 | | public static string OneCReason(string? errorSummary) => |
| | | 35 | | string.IsNullOrWhiteSpace(errorSummary) ? "1С: не проведён" |
| | | 36 | | : errorSummary.TrimStart().StartsWith("Не проведено", StringComparison.OrdinalIgnoreCase) ? "1С: " + errorSummar |
| | | 37 | | : "1С: не проведён — " + errorSummary.Trim(); |
| | | 38 | | |
| | | 39 | | public static string Trim(string? text) => |
| | | 40 | | string.IsNullOrWhiteSpace(text) ? string.Empty : text.Length <= MaxReasonLength ? text.Trim() : text[..(MaxReaso |
| | | 41 | | } |
| | | 42 | | |
| | | 43 | | public interface ISystemAlertSource |
| | | 44 | | { |
| | | 45 | | Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default); |
| | | 46 | | } |
| | | 47 | | |
| | | 48 | | /// <summary>Единая точка агрегации системных проблем. Источники не знают о UI и не возвращают секреты.</summary> |
| | | 49 | | public sealed class SystemAlertFactory( |
| | | 50 | | IEnumerable<ISystemAlertSource> sources, |
| | | 51 | | SystemAlertIncidentStore incidentStore, |
| | | 52 | | SensitiveDataRedactor redactor, |
| | | 53 | | ILogger<SystemAlertFactory>? logger = null) |
| | | 54 | | { |
| | | 55 | | public async Task<IReadOnlyList<SystemAlertTransition>> EvaluateAsync(CancellationToken cancellationToken = default) |
| | | 56 | | await incidentStore.SyncAsync(await CollectAsync(cancellationToken), cancellationToken); |
| | | 57 | | |
| | | 58 | | public async Task<List<JobLogsSystemAlertDto>> GetAlertsAsync(CancellationToken cancellationToken = default) |
| | | 59 | | { |
| | | 60 | | // UI читает только сохранённые инциденты (их обновляет джоб раз в 5 минут): опрос админов не нагружает журналы. |
| | | 61 | | var persisted = await incidentStore.GetActiveAsync(cancellationToken); |
| | | 62 | | return persisted.Select(x => new JobLogsSystemAlertDto |
| | | 63 | | { |
| | | 64 | | Key = x.Key, Domain = x.Domain, Severity = x.Severity, Title = x.Title, |
| | | 65 | | Lines = string.IsNullOrWhiteSpace(x.Details) ? [] : x.Details.Split('\n').ToList(), Url = x.Url ?? string.Em |
| | | 66 | | }).ToList(); |
| | | 67 | | } |
| | | 68 | | |
| | | 69 | | private async Task<List<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken) |
| | | 70 | | { |
| | | 71 | | // Источники разделяют scoped DbContext. Параллельные EF-запросы на одном экземпляре запрещены. |
| | | 72 | | var findings = new List<SystemAlertFinding>(); |
| | | 73 | | foreach (var source in sources) |
| | | 74 | | { |
| | | 75 | | try |
| | | 76 | | { |
| | | 77 | | findings.AddRange(await source.CollectAsync(cancellationToken)); |
| | | 78 | | } |
| | | 79 | | catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) |
| | | 80 | | { |
| | | 81 | | throw; |
| | | 82 | | } |
| | | 83 | | catch (Exception ex) |
| | | 84 | | { |
| | | 85 | | // Падение одного источника не должно скрывать остальные проблемы. |
| | | 86 | | var name = source.GetType().Name; |
| | | 87 | | logger?.LogWarning(ex, "Источник системных алертов {Source} не отработал", name); |
| | | 88 | | findings.Add(new SystemAlertFinding("alert-source-error-" + name, "monitoring", "warning", |
| | | 89 | | "Источник системных алертов не отработал: " + name, [ex.GetType().Name + ": " + ex.Message], |
| | | 90 | | "/Studio2/JobLogs?tab=alerts", Debounced: true)); |
| | | 91 | | } |
| | | 92 | | } |
| | | 93 | | return findings.Select(Redact).ToList(); |
| | | 94 | | } |
| | | 95 | | |
| | | 96 | | private SystemAlertFinding Redact(SystemAlertFinding finding) => finding with |
| | | 97 | | { |
| | | 98 | | Title = redactor.Redact(finding.Title, 500), |
| | | 99 | | Lines = finding.Lines.Select(x => redactor.Redact(x, 500)).ToList(), |
| | | 100 | | Url = redactor.Redact(finding.Url, 2000), |
| | | 101 | | Orders = finding.Orders?.Select(x => x with { Reason = SystemAlertOrderRef.Trim(redactor.Redact(x.Reason, System |
| | | 102 | | }; |
| | | 103 | | } |
| | | 104 | | |
| | | 105 | | public sealed class OzonSystemAlertSource( |
| | | 106 | | OzonWebhookAvailabilityStore webhookAvailability, |
| | | 107 | | OrdersStatusSyncHealthStore statusSyncHealth) : ISystemAlertSource |
| | | 108 | | { |
| | | 109 | | public Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 110 | | { |
| | | 111 | | var findings = new List<SystemAlertFinding>(); |
| | | 112 | | var webhooks = webhookAvailability.Current; |
| | | 113 | | if (webhooks.HasChecking) |
| | | 114 | | findings.Add(new("ozon-webhook-checking", "ozon", "warning", |
| | | 115 | | "Пуш-уведомления Ozon: адрес на проверке — статус появится после ближайшей проверки Ozon", |
| | | 116 | | webhooks.DescribeChecking(), "/Studio2/JobLogs?tab=webhook")); |
| | | 117 | | if (webhooks.HasProblem) |
| | | 118 | | findings.Add(new("ozon-webhook-availability", "ozon", "danger", |
| | | 119 | | "Пуш-уведомления Ozon недоступны — статусы заказов не приходят автоматически", |
| | | 120 | | webhooks.DescribeProblems(), "/Studio2/JobLogs?tab=webhook", Debounced: true)); |
| | | 121 | | if (statusSyncHealth.HasProblem) |
| | | 122 | | findings.Add(new("ozon-status-sync", "ozon", "danger", |
| | | 123 | | "Опрос статусов Ozon не получает ответов — статусы заказов могут устареть", |
| | | 124 | | [statusSyncHealth.Details ?? "Ozon API не отвечает"], "/Studio2/JobLogs?tab=webhook", Debounced: true)); |
| | | 125 | | return Task.FromResult<IReadOnlyList<SystemAlertFinding>>(findings); |
| | | 126 | | } |
| | | 127 | | } |
| | | 128 | | |
| | | 129 | | /// <summary>Контроль существующих журналов: пороги не скрывают единичный сбой, но не создают алерт на каждую строку.</s |
| | | 130 | | public sealed class JobLogsSystemAlertSource(ApplicationDbContext context, IOrderSearchDiagnosticsService orderSearch, |
| | | 131 | | IOptions<SystemAlertsOptions> options) : ISystemAlertSource |
| | | 132 | | { |
| | | 133 | | public async Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 134 | | { |
| | | 135 | | var settings = options.Value; |
| | | 136 | | var nowUtc = DateTime.UtcNow; |
| | | 137 | | // Актуальность: журнал считается только внутри окна своего алерта (SourceWindowHours), а записи без |
| | | 138 | | // верхней границы по времени (зависшие Running/Received, просрочки) — не старше RecordMaxAgeDays. |
| | | 139 | | DateTime Since(string key) => nowUtc.AddHours(-settings.WindowHoursFor(key)); |
| | | 140 | | var recordsNotOlderThanUtc = nowUtc.AddDays(-Math.Max(1, settings.RecordMaxAgeDays)); |
| | | 141 | | var findings = new List<SystemAlertFinding>(); |
| | | 142 | | var minDateUtc = Since("transactions-failed"); |
| | | 143 | | var txRuns = context.BackgroundJobExecutions.AsNoTracking().Where(x => x.StartedAt >= minDateUtc); |
| | | 144 | | var txFailures = await CountUnrecoveredAsync( |
| | | 145 | | txRuns.Where(x => x.Status == "Failed" || x.Status == "PartialSuccess").Select(x => x.StartedAt), |
| | | 146 | | txRuns.Where(x => x.Status == "Completed").Select(x => x.StartedAt), cancellationToken); |
| | | 147 | | var txFailureId = await txRuns.Where(x => x.Status == "Failed" || x.Status == "PartialSuccess") |
| | | 148 | | .OrderByDescending(x => x.StartedAt).Select(x => (long?)x.Id).FirstOrDefaultAsync(cancellationToken); |
| | | 149 | | AddFailures(findings, "transactions-failed", "transactions", txFailures, |
| | | 150 | | "Фоновые транзакции завершаются с ошибкой", JobLogUrl("transactions", txFailureId)); |
| | | 151 | | |
| | | 152 | | // Ошибка проведения неактуальна, если по тому же заказу позже проведение прошло успешно. |
| | | 153 | | var oneCSince = Since("onec-posting"); |
| | | 154 | | var unpostedQuery = context.OrderHistoryOneCPostings.AsNoTracking() |
| | | 155 | | .Where(x => !x.IsPosted && !string.IsNullOrEmpty(x.ErrorSummary) && x.PostedAt >= oneCSince) |
| | | 156 | | .Where(f => !context.OrderHistoryOneCPostings.Any(s => s.OrderId == f.OrderId && s.IsPosted && s.PostedAt > |
| | | 157 | | var unposted = await StatsAsync(unpostedQuery.Select(x => x.PostedAt), cancellationToken); |
| | | 158 | | var unpostedOrders = unposted.Count == 0 |
| | | 159 | | ? null |
| | | 160 | | : SystemAlertOrderRef.Build((await unpostedQuery.OrderByDescending(x => x.PostedAt).Take(500) |
| | | 161 | | .Select(x => new { x.OrderId, x.ErrorSummary }).ToListAsync(cancellationToken)) |
| | | 162 | | .Select(x => (x.OrderId, SystemAlertOrderRef.OneCReason(x.ErrorSummary)))); |
| | | 163 | | AddFailures(findings, "onec-posting", "onec", unposted, |
| | | 164 | | "Документы 1С не проведены", SystemAlertLinks.OneCPostingFailures, orders: unpostedOrders); |
| | | 165 | | |
| | | 166 | | var emailSince = Since("email-routing"); |
| | | 167 | | var emailRuns = context.EmailRoutingJobExecutions.AsNoTracking().Where(x => x.StartedAtUtc >= emailSince); |
| | | 168 | | var emailFailures = await CountUnrecoveredAsync( |
| | | 169 | | emailRuns.Where(x => x.Status == EmailRoutingJobExecutionStatus.Failed).Select(x => x.StartedAtUtc), |
| | | 170 | | emailRuns.Where(x => x.Status == EmailRoutingJobExecutionStatus.Success).Select(x => x.StartedAtUtc), cancel |
| | | 171 | | AddFailures(findings, "email-routing", "email", emailFailures, |
| | | 172 | | "Загрузка заказов из почты завершается с ошибкой", "/Studio2/JobLogs?tab=email"); |
| | | 173 | | |
| | | 174 | | var stockSince = Since("warehouse-stock-sync"); |
| | | 175 | | var stockRuns = context.WarehouseStockSyncRuns.AsNoTracking().Where(x => x.StartedAtUtc >= stockSince); |
| | | 176 | | var stockFailures = await CountUnrecoveredAsync( |
| | | 177 | | stockRuns.Where(x => x.Status == WarehouseStockSyncRunStatus.Failed).Select(x => x.StartedAtUtc), |
| | | 178 | | stockRuns.Where(x => x.Status == WarehouseStockSyncRunStatus.Success).Select(x => x.StartedAtUtc), cancellat |
| | | 179 | | var stockFailureId = await stockRuns.Where(x => x.Status == WarehouseStockSyncRunStatus.Failed) |
| | | 180 | | .OrderByDescending(x => x.StartedAtUtc).Select(x => (long?)x.Id).FirstOrDefaultAsync(cancellationToken); |
| | | 181 | | AddFailures(findings, "warehouse-stock-sync", "stock", stockFailures, |
| | | 182 | | "Синхронизация складских остатков завершается с ошибкой", JobLogUrl("stocksync", stockFailureId)); |
| | | 183 | | |
| | | 184 | | var webhookSince = Since("ozon-webhook-processing"); |
| | | 185 | | var webhookErrors = await StatsAsync(context.OzonPushNotifications.AsNoTracking() |
| | | 186 | | .Where(x => x.ProcessingStatus == "Error" && x.ReceivedAt >= webhookSince).Select(x => x.ReceivedAt), cancel |
| | | 187 | | var webhookErrorId = await context.OzonPushNotifications.AsNoTracking() |
| | | 188 | | .Where(x => x.ProcessingStatus == "Error" && x.ReceivedAt >= webhookSince) |
| | | 189 | | .OrderByDescending(x => x.ReceivedAt).Select(x => (long?)x.Id).FirstOrDefaultAsync(cancellationToken); |
| | | 190 | | AddFailures(findings, "ozon-webhook-processing", "ozon", webhookErrors, |
| | | 191 | | "Webhook Ozon обрабатываются с ошибкой", JobLogUrl("webhook", webhookErrorId)); |
| | | 192 | | var pendingWebhook = await context.OzonPushNotifications.AsNoTracking() |
| | | 193 | | .CountAsync(x => x.ProcessingStatus == "Received" && x.ReceivedAt >= recordsNotOlderThanUtc |
| | | 194 | | && x.ReceivedAt < nowUtc.AddMinutes(-Math.Max(1, settings.PendingWebhookMaxAgeMinutes)), ca |
| | | 195 | | if (pendingWebhook >= Math.Max(1, settings.MaxPendingWebhooks)) |
| | | 196 | | AddFailures(findings, "ozon-webhook-pending", "ozon", pendingWebhook, |
| | | 197 | | "Webhook Ozon ожидают обработки более 15 минут", "/Studio2/JobLogs?tab=webhook", debounced: true); |
| | | 198 | | |
| | | 199 | | if (await context.OzonClients.AsNoTracking().AnyAsync(cancellationToken)) |
| | | 200 | | { |
| | | 201 | | var lastNewOrderAt = await context.OzonPushNotifications.AsNoTracking() |
| | | 202 | | .Where(x => x.MessageType == "TYPE_NEW_POSTING") |
| | | 203 | | .MaxAsync(x => (DateTime?)x.ReceivedAt, cancellationToken); |
| | | 204 | | var warningAge = Math.Max(1, settings.OzonNewOrdersMaxAgeHours); |
| | | 205 | | if (!lastNewOrderAt.HasValue || lastNewOrderAt.Value < DateTime.UtcNow.AddHours(-warningAge)) |
| | | 206 | | { |
| | | 207 | | var dangerAge = Math.Max(warningAge, settings.OzonNewOrdersDangerAgeHours); |
| | | 208 | | var severity = !lastNewOrderAt.HasValue || lastNewOrderAt.Value < DateTime.UtcNow.AddHours(-dangerAge) |
| | | 209 | | ? "danger" |
| | | 210 | | : "warning"; |
| | | 211 | | var detail = lastNewOrderAt.HasValue |
| | | 212 | | ? $"Последний новый заказ получен: {lastNewOrderAt.Value:O}" |
| | | 213 | | : "Новые заказы Ozon ещё не были получены."; |
| | | 214 | | findings.Add(new SystemAlertFinding("ozon-new-orders-stale", "ozon", severity, |
| | | 215 | | "Давно не поступали новые заказы Ozon", [detail], "/Studio2/JobLogs?tab=webhook")); |
| | | 216 | | } |
| | | 217 | | } |
| | | 218 | | |
| | | 219 | | var fileSince = Since("file-upload"); |
| | | 220 | | var fileFailures = await StatsAsync(context.FileUploadRecords.AsNoTracking() |
| | | 221 | | .Where(x => x.Date >= fileSince && !x.Result.Contains("успешно")).Select(x => x.Date), cancellationToken); |
| | | 222 | | var fileFailureId = await context.FileUploadRecords.AsNoTracking() |
| | | 223 | | .Where(x => x.Date >= fileSince && !x.Result.Contains("успешно")) |
| | | 224 | | .OrderByDescending(x => x.Date).Select(x => (long?)x.Id).FirstOrDefaultAsync(cancellationToken); |
| | | 225 | | AddFailures(findings, "file-upload", "fileupload", fileFailures, |
| | | 226 | | "Импорт файлов заказов завершился с ошибкой", JobLogUrl("fileupload", fileFailureId)); |
| | | 227 | | |
| | | 228 | | var clearSince = Since("bitrix-stock-clear"); |
| | | 229 | | var clearFailures = await StatsAsync(context.BitrixStockClearExecutions.AsNoTracking() |
| | | 230 | | .Where(x => (x.Status == "Failed" || x.Status == "PartialSuccess") && x.StartedAt >= clearSince).Select(x => |
| | | 231 | | AddFailures(findings, "bitrix-stock-clear", "stock", clearFailures, |
| | | 232 | | "Очистка остатков BitrixStock завершилась с ошибкой", "/Studio2/BitrixStock?tab=clearHistory"); |
| | | 233 | | var pushSince = Since("bitrix-stock-push"); |
| | | 234 | | var pushFailures = await StatsAsync(context.BitrixStockPushExecutions.AsNoTracking() |
| | | 235 | | .Where(x => (x.Status == "Failed" || x.Status == "PartialSuccess") && x.StartedAt >= pushSince).Select(x => |
| | | 236 | | AddFailures(findings, "bitrix-stock-push", "stock", pushFailures, |
| | | 237 | | "Выгрузка остатков BitrixStock завершилась с ошибкой", "/Studio2/BitrixStock?tab=pushHistory"); |
| | | 238 | | |
| | | 239 | | var stuckJobs = await context.BackgroundJobExecutions.AsNoTracking() |
| | | 240 | | .CountAsync(x => x.Status == "Running" && x.CompletedAt == null && x.StartedAt >= recordsNotOlderThanUtc |
| | | 241 | | && x.StartedAt < nowUtc.AddMinutes(-Math.Max(1, settings.RunningJobMaxAgeMinutes)), cancell |
| | | 242 | | var stuckJobId = await context.BackgroundJobExecutions.AsNoTracking() |
| | | 243 | | .Where(x => x.Status == "Running" && x.CompletedAt == null && x.StartedAt >= recordsNotOlderThanUtc |
| | | 244 | | && x.StartedAt < nowUtc.AddMinutes(-Math.Max(1, settings.RunningJobMaxAgeMinutes))) |
| | | 245 | | .OrderBy(x => x.StartedAt).Select(x => (long?)x.Id).FirstOrDefaultAsync(cancellationToken); |
| | | 246 | | if (stuckJobs >= Math.Max(1, settings.MaxStuckJobs)) |
| | | 247 | | AddFailures(findings, "transactions-stuck", "hangfire", stuckJobs, |
| | | 248 | | "Фоновые транзакции выполняются более двух часов", JobLogUrl("transactions", stuckJobId), debounced: tru |
| | | 249 | | |
| | | 250 | | var overdueControl = await context.OrderControlTasks.AsNoTracking() |
| | | 251 | | .CountAsync(x => x.Status == OrderControlTaskStatus.Overdue && x.DueAtUtc >= recordsNotOlderThanUtc, cancell |
| | | 252 | | if (overdueControl >= Math.Max(1, settings.MaxOverdueOrderControlTasks)) |
| | | 253 | | AddFailures(findings, "order-control-overdue", "ordercontrol", overdueControl, |
| | | 254 | | "Есть просроченные задачи контроля заказов", "/Studio2/OrderControl", debounced: true); |
| | | 255 | | |
| | | 256 | | var search = await orderSearch.GetSnapshotAsync(cancellationToken); |
| | | 257 | | var driftThreshold = Math.Max(1, search.DriftAlertThreshold); |
| | | 258 | | if (search.IsDriftAlert) |
| | | 259 | | { |
| | | 260 | | var danger = Math.Abs(search.CountDrift) >= (long)driftThreshold * Math.Max(1, settings.OrderSearchDriftDang |
| | | 261 | | findings.Add(new SystemAlertFinding("order-search-drift", "search", danger ? "danger" : "warning", |
| | | 262 | | "Elasticsearch расходится с базой заказов", |
| | | 263 | | [$"Расхождение: {search.CountDrift}; порог: {search.DriftAlertThreshold}"], "/Studio2/JobLogs?tab=orders |
| | | 264 | | } |
| | | 265 | | else if (search.CountDrift != 0) |
| | | 266 | | findings.Add(new SystemAlertFinding("order-search-drift-minor", "search", "info", "Небольшое расхождение Ela |
| | | 267 | | [$"Расхождение: {search.CountDrift}; порог алерта: {search.DriftAlertThreshold}"], "/Studio2/JobLogs?tab |
| | | 268 | | |
| | | 269 | | if (search.EsEnabled && !search.IndexExists) |
| | | 270 | | findings.Add(new SystemAlertFinding("order-search-index-missing", "search", "danger", |
| | | 271 | | "Индекс заказов Elasticsearch отсутствует", [$"Индекс: {search.IndexName}"], "/Studio2/JobLogs?tab=order |
| | | 272 | | |
| | | 273 | | // LastDriftCheckAt живёт в памяти процесса: null после рестарта — не повод для тревоги. |
| | | 274 | | if (search.EsEnabled && search.LastDriftCheckAt is { } lastCheck && |
| | | 275 | | lastCheck < DateTime.UtcNow.AddHours(-Math.Max(1, settings.OrderSearchDriftCheckStaleHours))) |
| | | 276 | | findings.Add(new SystemAlertFinding("order-search-drift-stale", "search", "warning", |
| | | 277 | | "Проверка расхождения Elasticsearch давно не выполнялась", |
| | | 278 | | [$"Последняя проверка: {lastCheck:O}"], "/Studio2/JobLogs?tab=ordersearch", Debounced: true)); |
| | | 279 | | return findings; |
| | | 280 | | } |
| | | 281 | | |
| | | 282 | | /// <summary>Число сбоев и время самого свежего.</summary> |
| | | 283 | | private readonly record struct FailureStats(int Count, DateTime? Last); |
| | | 284 | | |
| | | 285 | | private static async Task<FailureStats> StatsAsync(IQueryable<DateTime> failureTimes, CancellationToken cancellation |
| | | 286 | | { |
| | | 287 | | var count = await failureTimes.CountAsync(cancellationToken); |
| | | 288 | | if (count == 0) return default; |
| | | 289 | | var last = await failureTimes.Select(x => (DateTime?)x).MaxAsync(cancellationToken); |
| | | 290 | | return new FailureStats(count, last); |
| | | 291 | | } |
| | | 292 | | |
| | | 293 | | /// <summary>Сбои, после которых не было успешного запуска: починенная проблема не висит до конца окна анализа.</sum |
| | | 294 | | private static async Task<FailureStats> CountUnrecoveredAsync(IQueryable<DateTime> failures, IQueryable<DateTime> su |
| | | 295 | | CancellationToken cancellationToken) |
| | | 296 | | { |
| | | 297 | | var lastSuccess = await successes.OrderByDescending(x => x).Select(x => (DateTime?)x).FirstOrDefaultAsync(cancel |
| | | 298 | | return await StatsAsync(lastSuccess.HasValue ? failures.Where(x => x > lastSuccess.Value) : failures, cancellati |
| | | 299 | | } |
| | | 300 | | |
| | | 301 | | private void AddFailures(List<SystemAlertFinding> findings, string key, string domain, FailureStats stats, string ti |
| | | 302 | | bool debounced = false, IReadOnlyList<SystemAlertOrderRef>? orders = null) |
| | | 303 | | { |
| | | 304 | | if (stats.Count == 0) return; |
| | | 305 | | var settings = options.Value; |
| | | 306 | | var danger = Math.Max(1, settings.DangerFailureCount); |
| | | 307 | | var severity = stats.Count >= danger ? "danger" : "warning"; |
| | | 308 | | var lines = new List<string> { $"Проблем за окно анализа: {stats.Count}" }; |
| | | 309 | | if (stats.Last is { } last) |
| | | 310 | | { |
| | | 311 | | // Затихающая проблема: свежих ошибок давно нет — не кричим «критично», пока их окно не истекло. |
| | | 312 | | if (severity == "danger" && settings.DangerFreshnessHours > 0 && last < DateTime.UtcNow.AddHours(-settings.D |
| | | 313 | | severity = "warning"; |
| | | 314 | | lines.Add($"Последняя ошибка: {last:yyyy-MM-dd HH:mm} UTC ({FormatAge(DateTime.UtcNow - last)} назад)"); |
| | | 315 | | } |
| | | 316 | | findings.Add(new SystemAlertFinding(key, domain, severity, title, lines, url, debounced, orders)); |
| | | 317 | | } |
| | | 318 | | |
| | | 319 | | private static string FormatAge(TimeSpan age) => age.TotalHours >= 48 ? $"{(int)age.TotalDays} д" |
| | | 320 | | : age.TotalMinutes >= 90 ? $"{(int)age.TotalHours} ч" : $"{Math.Max(1, (int)age.TotalMinutes)} мин"; |
| | | 321 | | |
| | | 322 | | private void AddFailures(List<SystemAlertFinding> findings, string key, string domain, int count, string title, stri |
| | | 323 | | bool debounced = false) => |
| | | 324 | | AddFailures(findings, key, domain, new FailureStats(count, null), title, url, debounced); |
| | | 325 | | |
| | | 326 | | private static string JobLogUrl(string tab, long? id) => |
| | | 327 | | id is > 0 ? $"/Studio2/JobLogs?tab={tab}&executionId={id.Value}" : $"/Studio2/JobLogs?tab={tab}"; |
| | | 328 | | } |
| | | 329 | | |
| | | 330 | | /// <summary>Пассивная диагностика интеграций: операции сами пишут success/failure, опросов внешних API нет.</summary> |
| | 10 | 331 | | public sealed class PassiveSystemAlertSource(ApplicationDbContext context, IOptions<SystemAlertsOptions> options) : ISys |
| | | 332 | | { |
| | | 333 | | public async Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 334 | | { |
| | 10 | 335 | | var settings = options.Value; |
| | 10 | 336 | | var since = DateTime.UtcNow.AddHours(-Math.Max(1, settings.AnalysisWindowHours)); |
| | 10 | 337 | | var staleBefore = DateTime.UtcNow.AddMinutes(-Math.Max(1, settings.SuccessMaxAgeMinutes)); |
| | 10 | 338 | | var sources = await context.SystemMonitoringSignals.AsNoTracking().Where(x => x.OccurredAtUtc >= since) |
| | 10 | 339 | | .GroupBy(x => new { x.SourceKey, x.ScopeKey }) |
| | 10 | 340 | | .Select(x => new |
| | 10 | 341 | | { |
| | 10 | 342 | | Source = x.Key.SourceKey, |
| | 10 | 343 | | Scope = x.Key.ScopeKey, |
| | 10 | 344 | | Failed = x.Count(s => s.Outcome == "failure"), |
| | 10 | 345 | | LastSuccess = x.Where(s => s.Outcome == "success").Max(s => (DateTime?)s.OccurredAtUtc), |
| | 10 | 346 | | LastFailure = x.Where(s => s.Outcome == "failure").Max(s => (DateTime?)s.OccurredAtUtc), |
| | 10 | 347 | | LastFailureMessage = x.Where(s => s.Outcome == "failure").OrderByDescending(s => s.OccurredAtUtc).Select |
| | 10 | 348 | | }) |
| | 10 | 349 | | .ToListAsync(cancellationToken); |
| | | 350 | | // Сигнал и запись аудита API-поставщика пишутся одной операцией. Берём именно |
| | | 351 | | // последнюю ошибку, чтобы ссылка из инцидента открывала первоисточник, а не |
| | | 352 | | // общий список алертов. |
| | 10 | 353 | | var supplierFailureIds = await context.ApiSupplierRequestAuditEntries.AsNoTracking() |
| | 10 | 354 | | .Where(x => !x.IsSuccess && x.OccurredAtUtc >= since) |
| | 10 | 355 | | .GroupBy(x => x.ProviderKey) |
| | 10 | 356 | | .Select(x => new { Provider = x.Key, Id = x.OrderByDescending(y => y.OccurredAtUtc).ThenByDescending(y => y. |
| | 10 | 357 | | .ToDictionaryAsync(x => x.Provider, x => x.Id, StringComparer.OrdinalIgnoreCase, cancellationToken); |
| | 10 | 358 | | var findings = new List<SystemAlertFinding>(); |
| | 54 | 359 | | foreach (var source in sources) |
| | | 360 | | { |
| | | 361 | | // События 1С (номенклатура, поставщик) приходят только при проблемах — «нет успешных операций» для них не п |
| | 17 | 362 | | var eventOnly = source.Source.StartsWith("onec-", StringComparison.Ordinal); |
| | 17 | 363 | | var stale = !eventOnly && !IsUserDriven(source.Source) && |
| | 17 | 364 | | (!source.LastSuccess.HasValue || source.LastSuccess < StaleBefore(source.Source, staleBefore)); |
| | | 365 | | // Ошибки, после которых был успех, — вылеченная проблема: не держим алерт до конца окна анализа. |
| | | 366 | | // События 1С (eventOnly) успехов не пишут, для них восстановления нет. |
| | 17 | 367 | | var recovered = !eventOnly && source.LastSuccess.HasValue && source.LastFailure.HasValue |
| | 17 | 368 | | && source.LastSuccess.Value > source.LastFailure.Value; |
| | 17 | 369 | | var failed = recovered ? 0 : source.Failed; |
| | 17 | 370 | | if (failed == 0 && !stale) continue; |
| | 9 | 371 | | var severity = failed >= Math.Max(1, settings.DangerFailureCount) || stale ? "danger" : "warning"; |
| | 9 | 372 | | var detail = eventOnly ? $"Событий за окно анализа: {source.Failed}" |
| | 9 | 373 | | : source.LastSuccess.HasValue ? $"Последний успех: {source.LastSuccess.Value:O}; ошибок: {source.Failed} |
| | 9 | 374 | | var scope = string.IsNullOrWhiteSpace(source.Scope) ? string.Empty : $" ({source.Scope})"; |
| | 9 | 375 | | findings.Add(new SystemAlertFinding(BuildFindingKey(source.Source, source.Scope), "integration", severity, |
| | 9 | 376 | | "Интеграция требует внимания: " + GetSourceTitle(source.Source, source.Scope) + (eventOnly ? string.Empt |
| | 9 | 377 | | string.IsNullOrWhiteSpace(source.LastFailureMessage) ? [detail] : [detail, "Последнее: " + source.LastFa |
| | 9 | 378 | | GetSourceUrl(source.Source, source.Scope, supplierFailureIds))); |
| | | 379 | | } |
| | 10 | 380 | | return findings; |
| | 10 | 381 | | } |
| | | 382 | | |
| | | 383 | | /// <summary> |
| | | 384 | | /// Сигналы пишутся только при действиях пользователей (проценка, корзина, открытие фида), поэтому |
| | | 385 | | /// отсутствие успеха — это тишина (ночь, выходной), а не сбой. Для них тревожат только ошибки. |
| | | 386 | | /// </summary> |
| | 1 | 387 | | private static readonly HashSet<string> UserDrivenSources = new(StringComparer.Ordinal) |
| | 1 | 388 | | { |
| | 1 | 389 | | "supplier-quotation", "supplier-cart", "supplier-api", "supplier-order-placement", |
| | 1 | 390 | | "feed-price-stock", "fid-suppliers-load", "fid-suppliers-mapping" |
| | 1 | 391 | | }; |
| | | 392 | | |
| | | 393 | | /// <summary>Суточные задачи: допустимый возраст последнего успеха больше общего порога.</summary> |
| | 1 | 394 | | private static readonly Dictionary<string, TimeSpan> DailySourceMaxAge = new(StringComparer.Ordinal) |
| | 1 | 395 | | { |
| | 1 | 396 | | ["ozon-report-import"] = TimeSpan.FromHours(26) |
| | 1 | 397 | | }; |
| | | 398 | | |
| | 13 | 399 | | private static bool IsUserDriven(string source) => UserDrivenSources.Contains(source); |
| | | 400 | | |
| | | 401 | | private static DateTime StaleBefore(string source, DateTime defaultStaleBefore) => |
| | 3 | 402 | | DailySourceMaxAge.TryGetValue(source, out var maxAge) ? DateTime.UtcNow - maxAge : defaultStaleBefore; |
| | | 403 | | |
| | | 404 | | private static string BuildFindingKey(string source, string? scope) |
| | | 405 | | { |
| | 9 | 406 | | var hash = SHA256.HashData(Encoding.UTF8.GetBytes(source + "\n" + scope)); |
| | 9 | 407 | | return "signal-" + Convert.ToHexString(hash).ToLowerInvariant(); |
| | | 408 | | } |
| | | 409 | | |
| | 9 | 410 | | private static string GetSourceTitle(string source, string? scope = null) => source switch |
| | 9 | 411 | | { |
| | 4 | 412 | | "onec-nomenclature" => scope switch |
| | 4 | 413 | | { |
| | 1 | 414 | | "create-failed" => "1С: не удалось создать номенклатуру или производителя", |
| | 1 | 415 | | "duplicate" => "1С: найдены дубли карточек номенклатуры", |
| | 0 | 416 | | "missing" => "1С: документ создан черновиком — не найдена номенклатура", |
| | 1 | 417 | | "ambiguous" => "1С: артикул у нескольких брендов — документ создан черновиком", |
| | 1 | 418 | | "brand-mismatch" => "1С: у артикула карточка другого бренда", |
| | 0 | 419 | | "input-conflict" => "1С: артикул заказа расходится с ProductKey — документ создан черновиком", |
| | 0 | 420 | | "producer-unknown" => "1С: бренд товара не распознан — документ создан черновиком", |
| | 0 | 421 | | _ => "1С: номенклатура" |
| | 4 | 422 | | }, |
| | 0 | 423 | | "onec-stock" => scope switch |
| | 0 | 424 | | { |
| | 0 | 425 | | "shortage" => "1С: нехватка остатка перед проведением документа", |
| | 0 | 426 | | _ => "1С: остатки" |
| | 0 | 427 | | }, |
| | 0 | 428 | | "onec-supplier" => "1С: поставщик не найден в контрагентах", |
| | 2 | 429 | | "supplier-quotation" => "проценка API-поставщика", |
| | 0 | 430 | | "supplier-cart" => "корзина API-поставщика", |
| | 0 | 431 | | "supplier-order-placement" => "оформление заказа у поставщика", |
| | 1 | 432 | | "supplier-api" => "API-поставщик", |
| | 2 | 433 | | _ => source |
| | 9 | 434 | | }; |
| | | 435 | | |
| | | 436 | | private static string GetSourceUrl(string source, string? scope, IReadOnlyDictionary<string, long> supplierFailureId |
| | | 437 | | { |
| | 9 | 438 | | if (source.StartsWith("supplier-", StringComparison.Ordinal) && |
| | 9 | 439 | | !string.IsNullOrWhiteSpace(scope) && supplierFailureIds.TryGetValue(scope, out var auditId)) |
| | 0 | 440 | | return $"/Studio2/ApiSupplierRequestAudit?highlightId={auditId}"; |
| | | 441 | | |
| | 9 | 442 | | return source switch |
| | 9 | 443 | | { |
| | 1 | 444 | | "email-routing" or "dropbox-routing" => "/Studio2/JobLogs?tab=email", |
| | 0 | 445 | | "warehouse-stock-import" => "/Studio2/JobLogs?tab=stocksync", |
| | 1 | 446 | | "ozon-report-import" => "/Studio2/JobLogs?tab=webhook", |
| | 0 | 447 | | "feed-price-stock" or "fid-suppliers-load" or "fid-suppliers-mapping" => "/Studio2/FidSupplierLink", |
| | 7 | 448 | | _ => "/Studio2/JobLogs?tab=alerts" |
| | 9 | 449 | | }; |
| | | 450 | | } |
| | | 451 | | } |