| | | 1 | | using Microsoft.EntityFrameworkCore; |
| | | 2 | | using Microsoft.Extensions.Configuration; |
| | | 3 | | using Microsoft.Extensions.Logging; |
| | | 4 | | using OzonRepositories.Context; |
| | | 5 | | |
| | | 6 | | namespace Services.SearchServices.OrderSearch; |
| | | 7 | | |
| | | 8 | | public interface IOrderSearchDriftChecker |
| | | 9 | | { |
| | | 10 | | Task<OrderSearchDriftCheckResult> CheckAsync(string trigger, CancellationToken cancellationToken = default); |
| | | 11 | | } |
| | | 12 | | |
| | | 13 | | public sealed class OrderSearchDriftCheckResult |
| | | 14 | | { |
| | 7 | 15 | | public bool Skipped { get; init; } |
| | | 16 | | |
| | 22 | 17 | | public string Trigger { get; init; } = string.Empty; |
| | | 18 | | |
| | 13 | 19 | | public long IndexCount { get; init; } |
| | | 20 | | |
| | 13 | 21 | | public long DbOrderCount { get; init; } |
| | | 22 | | |
| | 30 | 23 | | public long AbsoluteDrift { get; init; } |
| | | 24 | | |
| | 7 | 25 | | public int AlertThreshold { get; init; } |
| | | 26 | | |
| | 12 | 27 | | public bool IsAlert { get; init; } |
| | | 28 | | |
| | 7 | 29 | | public bool IndexExists { get; init; } |
| | | 30 | | |
| | 25 | 31 | | public string Message { get; init; } = string.Empty; |
| | | 32 | | } |
| | | 33 | | |
| | | 34 | | /// <summary> |
| | | 35 | | /// Сравнивает число документов в индексе ES с числом строк в <c>Orders</c>. |
| | | 36 | | /// При |ES − MySQL| > порога — запись в JobLogs и метрики Prometheus. |
| | | 37 | | /// </summary> |
| | | 38 | | public sealed class OrderSearchDriftChecker : IOrderSearchDriftChecker |
| | | 39 | | { |
| | | 40 | | private readonly IOrderSearchService _searchService; |
| | | 41 | | private readonly IOrderSearchDiagnosticsStore _store; |
| | | 42 | | private readonly ApplicationDbContext _context; |
| | | 43 | | private readonly IConfiguration _configuration; |
| | | 44 | | private readonly ILogger<OrderSearchDriftChecker> _logger; |
| | | 45 | | private readonly IOrderSearchDriftMetricsReporter? _metricsReporter; |
| | | 46 | | |
| | | 47 | | public OrderSearchDriftChecker( |
| | | 48 | | IOrderSearchService searchService, |
| | | 49 | | IOrderSearchDiagnosticsStore store, |
| | | 50 | | ApplicationDbContext context, |
| | | 51 | | IConfiguration configuration, |
| | | 52 | | ILogger<OrderSearchDriftChecker> logger, |
| | | 53 | | IOrderSearchDriftMetricsReporter? metricsReporter = null) |
| | | 54 | | { |
| | | 55 | | _searchService = searchService; |
| | | 56 | | _store = store; |
| | | 57 | | _context = context; |
| | | 58 | | _configuration = configuration; |
| | | 59 | | _logger = logger; |
| | | 60 | | _metricsReporter = metricsReporter; |
| | | 61 | | } |
| | | 62 | | |
| | | 63 | | public async Task<OrderSearchDriftCheckResult> CheckAsync(string trigger, CancellationToken cancellationToken = defa |
| | | 64 | | { |
| | | 65 | | if (!_configuration.GetValue<bool>("OrdersSearch:UseElasticsearch")) |
| | | 66 | | { |
| | | 67 | | return new OrderSearchDriftCheckResult |
| | | 68 | | { |
| | | 69 | | Skipped = true, |
| | | 70 | | Trigger = trigger, |
| | | 71 | | Message = "Elasticsearch выключен (OrdersSearch:UseElasticsearch=false)." |
| | | 72 | | }; |
| | | 73 | | } |
| | | 74 | | |
| | | 75 | | var threshold = GetAlertThreshold(); |
| | | 76 | | var dbOrderCount = await _context.Orders.AsNoTracking().CountAsync(cancellationToken); |
| | | 77 | | var indexExists = await _searchService.IndexExistsAsync(); |
| | | 78 | | var indexCount = indexExists ? await _searchService.GetTotalCountAsync() : 0L; |
| | | 79 | | var absoluteDrift = Math.Abs(indexCount - dbOrderCount); |
| | | 80 | | var isAlert = indexExists && absoluteDrift > threshold; |
| | | 81 | | |
| | | 82 | | var message = isAlert |
| | | 83 | | ? $"Расхождение ES↔MySQL: индекс {indexCount:N0}, БД {dbOrderCount:N0}, |Δ|={absoluteDrift:N0} (порог {thres |
| | | 84 | | : $"ES↔MySQL OK: индекс {indexCount:N0}, БД {dbOrderCount:N0}, |Δ|={absoluteDrift:N0}."; |
| | | 85 | | |
| | | 86 | | _store.RecordDrift(new OrderSearchDriftEvent |
| | | 87 | | { |
| | | 88 | | Trigger = trigger, |
| | | 89 | | IndexCount = indexCount, |
| | | 90 | | DbOrderCount = dbOrderCount, |
| | | 91 | | AbsoluteDrift = absoluteDrift, |
| | | 92 | | AlertThreshold = threshold, |
| | | 93 | | IsAlert = isAlert, |
| | | 94 | | Message = message |
| | | 95 | | }); |
| | | 96 | | |
| | | 97 | | _metricsReporter?.Report(indexCount, dbOrderCount, absoluteDrift, isAlert); |
| | | 98 | | |
| | | 99 | | if (isAlert) |
| | | 100 | | { |
| | | 101 | | _logger.LogWarning( |
| | | 102 | | "OrderSearch drift alert ({Trigger}): index={IndexCount}, db={DbCount}, drift={Drift}, threshold={Thresh |
| | | 103 | | trigger, indexCount, dbOrderCount, absoluteDrift, threshold); |
| | | 104 | | } |
| | | 105 | | else |
| | | 106 | | { |
| | | 107 | | _logger.LogInformation( |
| | | 108 | | "OrderSearch drift check ({Trigger}): index={IndexCount}, db={DbCount}, drift={Drift}", |
| | | 109 | | trigger, indexCount, dbOrderCount, absoluteDrift); |
| | | 110 | | } |
| | | 111 | | |
| | | 112 | | return new OrderSearchDriftCheckResult |
| | | 113 | | { |
| | | 114 | | Trigger = trigger, |
| | | 115 | | IndexCount = indexCount, |
| | | 116 | | DbOrderCount = dbOrderCount, |
| | | 117 | | AbsoluteDrift = absoluteDrift, |
| | | 118 | | AlertThreshold = threshold, |
| | | 119 | | IsAlert = isAlert, |
| | | 120 | | IndexExists = indexExists, |
| | | 121 | | Message = message |
| | | 122 | | }; |
| | | 123 | | } |
| | | 124 | | |
| | | 125 | | public static int GetAlertThreshold(IConfiguration configuration) => |
| | | 126 | | Math.Clamp(configuration.GetValue("OrdersSearch:CountDriftAlertThreshold", 50), 0, 1_000_000); |
| | | 127 | | |
| | | 128 | | private int GetAlertThreshold() => GetAlertThreshold(_configuration); |
| | | 129 | | } |