| | | 1 | | using System.Diagnostics; |
| | | 2 | | using System.Globalization; |
| | | 3 | | using System.Text.Json; |
| | | 4 | | using Microsoft.EntityFrameworkCore; |
| | | 5 | | using Microsoft.Extensions.Options; |
| | | 6 | | using OzonRepositories.Context; |
| | | 7 | | using Services.HangfireService; |
| | | 8 | | |
| | | 9 | | namespace OzonOrdersWeb.WebServices.SystemAlerts; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// Планировщик Hangfire: просроченные recurring-джобы, отсутствие живых серверов, осиротевшие XA-транзакции. |
| | | 13 | | /// Только читает: откат XA и чистку делает <c>HangfireSchedulerWatchdogHostedService</c>. |
| | | 14 | | /// </summary> |
| | | 15 | | /// <remarks>Инцидент 09–14.09.2026: планировщик стоял пять дней при зелёном healthcheck.</remarks> |
| | | 16 | | public sealed class HangfireSystemAlertSource(HangfireSchedulerWatchdogStorage storage, IOptions<SystemAlertsOptions> op |
| | | 17 | | { |
| | | 18 | | public async Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 19 | | { |
| | | 20 | | var settings = options.Value; |
| | | 21 | | if (!settings.HangfireChecksEnabled) return []; |
| | | 22 | | |
| | | 23 | | var overdue = TimeSpan.FromMinutes(Math.Max(1, settings.HangfireRecurringOverdueMinutes)); |
| | | 24 | | var observation = await storage.ObserveAsync(DateTime.UtcNow, TimeSpan.FromMinutes(5), cancellationToken); |
| | | 25 | | var verdict = HangfireSchedulerWatchdogAnalyzer.Analyze(observation, [], overdue); |
| | | 26 | | var findings = new List<SystemAlertFinding>(); |
| | | 27 | | |
| | | 28 | | if (observation.ServerHeartbeatsUtc.Count == 0 || !verdict.ServersAlive) |
| | | 29 | | findings.Add(new SystemAlertFinding("hangfire-no-servers", "hangfire", "danger", |
| | | 30 | | "Нет живых серверов Hangfire — фоновые задачи не выполняются", |
| | | 31 | | ["Heartbeat серверов отсутствует или устарел."], SystemAlertLinks.HangfireServers, Debounced: true)); |
| | | 32 | | |
| | | 33 | | if (verdict.StuckJobs.Count > 0) |
| | | 34 | | { |
| | | 35 | | var oldest = verdict.StuckJobs[0]; |
| | | 36 | | findings.Add(new SystemAlertFinding("hangfire-scheduler-stuck", "hangfire", "danger", |
| | | 37 | | "Планировщик Hangfire не запускает recurring-джобы", |
| | | 38 | | [$"Просрочено джобов: {verdict.StuckJobs.Count}; самый старый: {oldest.Id} (ожидался {oldest.NextExecuti |
| | | 39 | | SystemAlertLinks.HangfireRecurring, Debounced: true)); |
| | | 40 | | } |
| | | 41 | | |
| | | 42 | | if (observation.PreparedXaTransactions.Count > 0) |
| | | 43 | | findings.Add(new SystemAlertFinding("hangfire-xa-prepared", "hangfire", "warning", |
| | | 44 | | "В БД Hangfire висят XA PREPARED транзакции — они могут блокировать планировщик", |
| | | 45 | | [$"Транзакций: {observation.PreparedXaTransactions.Count}; вотчдог откатит их при повторном обнаружении" |
| | | 46 | | SystemAlertLinks.HangfireRecurring, Debounced: true)); |
| | | 47 | | |
| | | 48 | | if (observation.OrphanedStatelessJobs > 0) |
| | | 49 | | findings.Add(new SystemAlertFinding("hangfire-stateless-jobs", "hangfire", "warning", |
| | | 50 | | "В БД Hangfire есть джобы без состояния", |
| | | 51 | | [$"Джобов: {observation.OrphanedStatelessJobs}"], SystemAlertLinks.HangfireRecurring, Debounced: true)); |
| | | 52 | | |
| | | 53 | | return findings; |
| | | 54 | | } |
| | | 55 | | } |
| | | 56 | | |
| | | 57 | | /// <summary>Свободное место на диске приложения.</summary> |
| | | 58 | | public sealed class DiskSystemAlertSource(IOptions<SystemAlertsOptions> options) : ISystemAlertSource |
| | | 59 | | { |
| | | 60 | | public Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 61 | | { |
| | | 62 | | var settings = options.Value; |
| | | 63 | | if (!settings.DiskChecksEnabled) return Task.FromResult<IReadOnlyList<SystemAlertFinding>>([]); |
| | | 64 | | |
| | | 65 | | var drive = DriveInfo.GetDrives() |
| | | 66 | | .Where(x => x.IsReady && AppContext.BaseDirectory.StartsWith(x.RootDirectory.FullName, StringComparison.Ordi |
| | | 67 | | .OrderByDescending(x => x.RootDirectory.FullName.Length) |
| | | 68 | | .FirstOrDefault(); |
| | | 69 | | if (drive == null || drive.TotalSize <= 0) return Task.FromResult<IReadOnlyList<SystemAlertFinding>>([]); |
| | | 70 | | |
| | | 71 | | var severity = EvaluateSeverity(drive.AvailableFreeSpace, drive.TotalSize, settings); |
| | | 72 | | if (severity == null) return Task.FromResult<IReadOnlyList<SystemAlertFinding>>([]); |
| | | 73 | | |
| | | 74 | | var percent = drive.AvailableFreeSpace * 100d / drive.TotalSize; |
| | | 75 | | IReadOnlyList<SystemAlertFinding> result = |
| | | 76 | | [ |
| | | 77 | | new("disk-space", "server", severity, "На диске приложения заканчивается место", |
| | | 78 | | [$"Свободно {FormatGb(drive.AvailableFreeSpace)} из {FormatGb(drive.TotalSize)} ({percent:0.#}%)"], |
| | | 79 | | "/Studio2/JobLogs?tab=server", Debounced: true) |
| | | 80 | | ]; |
| | | 81 | | return Task.FromResult(result); |
| | | 82 | | } |
| | | 83 | | |
| | | 84 | | public static string? EvaluateSeverity(long freeBytes, long totalBytes, SystemAlertsOptions settings) |
| | | 85 | | { |
| | | 86 | | var percent = freeBytes * 100d / totalBytes; |
| | | 87 | | var freeGb = freeBytes / 1024d / 1024d / 1024d; |
| | | 88 | | if (percent < settings.DiskFreePercentDanger || freeGb < settings.DiskFreeGbDanger) return "danger"; |
| | | 89 | | return percent < settings.DiskFreePercentWarning ? "warning" : null; |
| | | 90 | | } |
| | | 91 | | |
| | | 92 | | private static string FormatGb(long bytes) => (bytes / 1024d / 1024d / 1024d).ToString("0.#", CultureInfo.InvariantC |
| | | 93 | | } |
| | | 94 | | |
| | | 95 | | /// <summary>Состояние кластера Elasticsearch и свободное место на его узлах (серия деплоев уже забивала диск и переводи |
| | 0 | 96 | | public sealed class ElasticsearchSystemAlertSource( |
| | 0 | 97 | | IHttpClientFactory httpClientFactory, |
| | 0 | 98 | | IConfiguration configuration, |
| | 0 | 99 | | IOptions<SystemAlertsOptions> options) : ISystemAlertSource |
| | | 100 | | { |
| | | 101 | | public const string HttpClientName = "SystemAlertsElasticsearch"; |
| | | 102 | | private const string Url = "/Studio2/JobLogs?tab=ordersearch"; |
| | | 103 | | |
| | | 104 | | public async Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 105 | | { |
| | 0 | 106 | | var settings = options.Value; |
| | 0 | 107 | | if (!settings.ElasticsearchChecksEnabled || !configuration.GetValue("OrdersSearch:UseElasticsearch", false)) ret |
| | | 108 | | |
| | 0 | 109 | | var baseUrl = (configuration["OrdersSearch:Url"] ?? "http://localhost:9201").TrimEnd('/'); |
| | 0 | 110 | | using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 0 | 111 | | timeout.CancelAfter(TimeSpan.FromSeconds(Math.Max(1, settings.ElasticsearchTimeoutSeconds))); |
| | 0 | 112 | | var client = httpClientFactory.CreateClient(HttpClientName); |
| | 0 | 113 | | var findings = new List<SystemAlertFinding>(); |
| | | 114 | | |
| | | 115 | | string health; |
| | | 116 | | try |
| | | 117 | | { |
| | 0 | 118 | | health = await client.GetStringAsync(baseUrl + "/_cluster/health", timeout.Token); |
| | 0 | 119 | | } |
| | 0 | 120 | | catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException && |
| | | 121 | | { |
| | 0 | 122 | | return |
| | 0 | 123 | | [ |
| | 0 | 124 | | new SystemAlertFinding("elasticsearch-unavailable", "search", "danger", |
| | 0 | 125 | | "Elasticsearch недоступен — поиск заказов не работает", [ex.Message], Url, Debounced: true) |
| | 0 | 126 | | ]; |
| | | 127 | | } |
| | | 128 | | |
| | 0 | 129 | | var status = ReadClusterStatus(health); |
| | 0 | 130 | | if (status == "red") |
| | 0 | 131 | | findings.Add(new SystemAlertFinding("elasticsearch-cluster-red", "search", "danger", |
| | 0 | 132 | | "Кластер Elasticsearch в состоянии red — поиск заказов может не работать", |
| | 0 | 133 | | ["Проверьте свободное место на диске и распределение шардов."], Url, Debounced: true)); |
| | 0 | 134 | | else if (status == "yellow" && settings.ElasticsearchYellowIsWarning) |
| | 0 | 135 | | findings.Add(new SystemAlertFinding("elasticsearch-cluster-yellow", "search", "warning", |
| | 0 | 136 | | "Кластер Elasticsearch в состоянии yellow", ["Часть реплик не размещена."], Url, Debounced: true)); |
| | | 137 | | |
| | | 138 | | try |
| | | 139 | | { |
| | 0 | 140 | | var fs = await client.GetStringAsync(baseUrl + "/_nodes/stats/fs", timeout.Token); |
| | 0 | 141 | | var free = ReadFreeDiskPercent(fs); |
| | 0 | 142 | | if (free is { } percent) |
| | | 143 | | { |
| | 0 | 144 | | var severity = percent < settings.ElasticsearchDiskFreePercentDanger ? "danger" |
| | 0 | 145 | | : percent < settings.ElasticsearchDiskFreePercentWarning ? "warning" : null; |
| | 0 | 146 | | if (severity != null) |
| | 0 | 147 | | findings.Add(new SystemAlertFinding("elasticsearch-disk", "search", severity, |
| | 0 | 148 | | "На диске Elasticsearch заканчивается место", |
| | 0 | 149 | | [$"Свободно {percent:0.#}% на самом заполненном узле"], Url, Debounced: true)); |
| | | 150 | | } |
| | 0 | 151 | | } |
| | 0 | 152 | | catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException && |
| | | 153 | | { |
| | | 154 | | // Кластер ответил на health, но не на stats: сам по себе это не инцидент. |
| | 0 | 155 | | } |
| | | 156 | | |
| | 0 | 157 | | return findings; |
| | 0 | 158 | | } |
| | | 159 | | |
| | | 160 | | public static string? ReadClusterStatus(string json) |
| | | 161 | | { |
| | 1 | 162 | | using var doc = JsonDocument.Parse(json); |
| | 1 | 163 | | return doc.RootElement.TryGetProperty("status", out var status) ? status.GetString() : null; |
| | 1 | 164 | | } |
| | | 165 | | |
| | | 166 | | /// <summary>Минимальный процент свободного места среди узлов; null — данных нет.</summary> |
| | | 167 | | public static double? ReadFreeDiskPercent(string json) |
| | | 168 | | { |
| | 1 | 169 | | using var doc = JsonDocument.Parse(json); |
| | 1 | 170 | | if (!doc.RootElement.TryGetProperty("nodes", out var nodes)) return null; |
| | 1 | 171 | | double? min = null; |
| | 6 | 172 | | foreach (var node in nodes.EnumerateObject()) |
| | | 173 | | { |
| | 2 | 174 | | if (!node.Value.TryGetProperty("fs", out var fs) || !fs.TryGetProperty("total", out var total)) continue; |
| | 2 | 175 | | if (!total.TryGetProperty("total_in_bytes", out var totalBytes) || !total.TryGetProperty("available_in_bytes |
| | 2 | 176 | | if (totalBytes.GetInt64() <= 0) continue; |
| | 2 | 177 | | var percent = available.GetInt64() * 100d / totalBytes.GetInt64(); |
| | 2 | 178 | | min = min is null ? percent : Math.Min(min.Value, percent); |
| | | 179 | | } |
| | 1 | 180 | | return min; |
| | 1 | 181 | | } |
| | | 182 | | } |
| | | 183 | | |
| | | 184 | | /// <summary>MariaDB: задержка простого запроса и заполнение пула соединений. Недоступность БД видна как ошибка самого и |
| | | 185 | | public sealed class DatabaseSystemAlertSource(ApplicationDbContext context, IOptions<SystemAlertsOptions> options) : ISy |
| | | 186 | | { |
| | | 187 | | private const string Url = "/Studio2/JobLogs?tab=server"; |
| | | 188 | | |
| | | 189 | | public async Task<IReadOnlyList<SystemAlertFinding>> CollectAsync(CancellationToken cancellationToken = default) |
| | | 190 | | { |
| | | 191 | | var settings = options.Value; |
| | | 192 | | if (!settings.DatabaseChecksEnabled) return []; |
| | | 193 | | |
| | | 194 | | var connection = context.Database.GetDbConnection(); |
| | | 195 | | var shouldClose = connection.State != System.Data.ConnectionState.Open; |
| | | 196 | | if (shouldClose) await connection.OpenAsync(cancellationToken); |
| | | 197 | | try |
| | | 198 | | { |
| | | 199 | | var startedAt = Stopwatch.GetTimestamp(); |
| | | 200 | | await ScalarAsync(connection, "SELECT 1", cancellationToken); |
| | | 201 | | var latencyMs = (long)Stopwatch.GetElapsedTime(startedAt).TotalMilliseconds; |
| | | 202 | | var connected = ToLong(await StatusAsync(connection, "Threads_connected", cancellationToken)); |
| | | 203 | | var max = ToLong(await ScalarAsync(connection, "SELECT @@max_connections", cancellationToken)); |
| | | 204 | | |
| | | 205 | | var findings = new List<SystemAlertFinding>(); |
| | | 206 | | var latencySeverity = latencyMs >= settings.DatabaseLatencyDangerMs ? "danger" |
| | | 207 | | : latencyMs >= settings.DatabaseLatencyWarningMs ? "warning" : null; |
| | | 208 | | if (latencySeverity != null) |
| | | 209 | | findings.Add(new SystemAlertFinding("mariadb-latency", "database", latencySeverity, |
| | | 210 | | "MariaDB отвечает медленно", [$"Простой запрос занял {latencyMs} мс"], Url, Debounced: true)); |
| | | 211 | | |
| | | 212 | | if (max > 0) |
| | | 213 | | { |
| | | 214 | | var usage = connected * 100d / max; |
| | | 215 | | var connSeverity = usage >= settings.DatabaseConnectionsDangerPercent ? "danger" |
| | | 216 | | : usage >= settings.DatabaseConnectionsWarningPercent ? "warning" : null; |
| | | 217 | | if (connSeverity != null) |
| | | 218 | | findings.Add(new SystemAlertFinding("mariadb-connections", "database", connSeverity, |
| | | 219 | | "MariaDB почти исчерпала лимит соединений", [$"Соединений: {connected} из {max} ({usage:0}%)"], |
| | | 220 | | } |
| | | 221 | | return findings; |
| | | 222 | | } |
| | | 223 | | finally |
| | | 224 | | { |
| | | 225 | | if (shouldClose) await connection.CloseAsync(); |
| | | 226 | | } |
| | | 227 | | } |
| | | 228 | | |
| | | 229 | | private static async Task<string?> ScalarAsync(System.Data.Common.DbConnection connection, string sql, CancellationT |
| | | 230 | | { |
| | | 231 | | await using var command = connection.CreateCommand(); |
| | | 232 | | command.CommandText = sql; |
| | | 233 | | return (await command.ExecuteScalarAsync(cancellationToken))?.ToString(); |
| | | 234 | | } |
| | | 235 | | |
| | | 236 | | private static async Task<string?> StatusAsync(System.Data.Common.DbConnection connection, string name, Cancellation |
| | | 237 | | { |
| | | 238 | | await using var command = connection.CreateCommand(); |
| | | 239 | | command.CommandText = $"SHOW GLOBAL STATUS LIKE '{name}'"; |
| | | 240 | | await using var reader = await command.ExecuteReaderAsync(cancellationToken); |
| | | 241 | | return await reader.ReadAsync(cancellationToken) ? reader.GetValue(1)?.ToString() : null; |
| | | 242 | | } |
| | | 243 | | |
| | | 244 | | private static long ToLong(string? value) => long.TryParse(value, CultureInfo.InvariantCulture, out var result) ? re |
| | | 245 | | } |