| | | 1 | | using Microsoft.EntityFrameworkCore; |
| | | 2 | | using Microsoft.Extensions.Logging; |
| | | 3 | | using Microsoft.Extensions.Options; |
| | | 4 | | using OzonDomains.Models; |
| | | 5 | | using OzonRepositories.Context; |
| | | 6 | | using Services.DataServices; |
| | | 7 | | using Services.SystemMonitoringServices; |
| | | 8 | | |
| | | 9 | | namespace Services.EmailRoutingServices; |
| | | 10 | | |
| | | 11 | | public class EmailAttachmentRoutingService : IEmailAttachmentRoutingService |
| | | 12 | | { |
| | | 13 | | private readonly IEmailInboxClient _emailInboxClient; |
| | | 14 | | private readonly IRoutingRuleEvaluator _ruleEvaluator; |
| | | 15 | | private readonly IFileDeduplicationService _deduplicationService; |
| | | 16 | | private readonly IDropboxFileRouter _dropboxFileRouter; |
| | | 17 | | private readonly IOrderEmailImportService _orderEmailImportService; |
| | | 18 | | private readonly EmailRoutingRuleDataServices _rulesDataServices; |
| | | 19 | | private readonly EmailMessageInboxDataServices _messageDataServices; |
| | | 20 | | private readonly EmailAttachmentProcessingResultDataServices _resultDataServices; |
| | | 21 | | private readonly ApplicationDbContext _db; |
| | | 22 | | private readonly EmailRoutingExecutionOptions _executionOptions; |
| | | 23 | | private readonly ILogger<EmailAttachmentRoutingService> _logger; |
| | | 24 | | private readonly ISystemMonitoringSignalWriter? _signals; |
| | | 25 | | |
| | 5 | 26 | | public EmailAttachmentRoutingService( |
| | 5 | 27 | | IEmailInboxClient emailInboxClient, |
| | 5 | 28 | | IRoutingRuleEvaluator ruleEvaluator, |
| | 5 | 29 | | IFileDeduplicationService deduplicationService, |
| | 5 | 30 | | IDropboxFileRouter dropboxFileRouter, |
| | 5 | 31 | | IOrderEmailImportService orderEmailImportService, |
| | 5 | 32 | | EmailRoutingRuleDataServices rulesDataServices, |
| | 5 | 33 | | EmailMessageInboxDataServices messageDataServices, |
| | 5 | 34 | | EmailAttachmentProcessingResultDataServices resultDataServices, |
| | 5 | 35 | | ApplicationDbContext db, |
| | 5 | 36 | | IOptions<EmailRoutingExecutionOptions> executionOptions, |
| | 5 | 37 | | ILogger<EmailAttachmentRoutingService> logger, |
| | 5 | 38 | | ISystemMonitoringSignalWriter? signals = null) |
| | | 39 | | { |
| | 5 | 40 | | _emailInboxClient = emailInboxClient; |
| | 5 | 41 | | _ruleEvaluator = ruleEvaluator; |
| | 5 | 42 | | _deduplicationService = deduplicationService; |
| | 5 | 43 | | _dropboxFileRouter = dropboxFileRouter; |
| | 5 | 44 | | _orderEmailImportService = orderEmailImportService; |
| | 5 | 45 | | _rulesDataServices = rulesDataServices; |
| | 5 | 46 | | _messageDataServices = messageDataServices; |
| | 5 | 47 | | _resultDataServices = resultDataServices; |
| | 5 | 48 | | _db = db; |
| | 5 | 49 | | _executionOptions = executionOptions.Value; |
| | 5 | 50 | | _logger = logger; |
| | 5 | 51 | | _signals = signals; |
| | 5 | 52 | | } |
| | | 53 | | |
| | | 54 | | public async Task<EmailInboxProcessResult> ProcessInboxAsync(bool dryRun = false, CancellationToken cancellationToke |
| | | 55 | | { |
| | | 56 | | try |
| | | 57 | | { |
| | 6 | 58 | | var rules = await _rulesDataServices.GetActiveRules(); |
| | 6 | 59 | | var knownMessages = await _messageDataServices.GetKnownInboxMessagesAsync(); |
| | 6 | 60 | | var messages = await _emailInboxClient.FetchMessagesAsync(knownMessages, cancellationToken); |
| | 6 | 61 | | var batchResult = new EmailInboxProcessResult { BatchesProcessed = 1 }; |
| | | 62 | | |
| | 22 | 63 | | foreach (var message in messages) |
| | | 64 | | { |
| | | 65 | | // В обычном polling обрабатываем только новые письма. |
| | | 66 | | // Письма, которые уже есть в БД, не трогаем (переобработка — только через Reprocess). |
| | 5 | 67 | | batchResult.Add(await ProcessMessageInternal( |
| | 5 | 68 | | message, rules, dryRun, cancellationToken, forceReprocess: false)); |
| | | 69 | | } |
| | | 70 | | |
| | 6 | 71 | | await (_signals?.RecordAsync("email-routing", "success", cancellationToken: cancellationToken) ?? Task.Compl |
| | 6 | 72 | | return batchResult; |
| | | 73 | | } |
| | 0 | 74 | | catch (Exception) when (!cancellationToken.IsCancellationRequested) |
| | | 75 | | { |
| | 0 | 76 | | await (_signals?.RecordAsync("email-routing", "failure", message: "Ошибка обработки входящей почты", cancell |
| | 0 | 77 | | throw; |
| | | 78 | | } |
| | 6 | 79 | | } |
| | | 80 | | |
| | | 81 | | public async Task<int> ReprocessMessageAsync(long messageId, CancellationToken cancellationToken = default) |
| | | 82 | | { |
| | 0 | 83 | | var message = await _db.EmailMessageInboxes |
| | 0 | 84 | | .Include(x => x.Attachments) |
| | 0 | 85 | | .FirstOrDefaultAsync(x => x.Id == messageId, cancellationToken); |
| | 0 | 86 | | if (message == null) |
| | | 87 | | { |
| | 0 | 88 | | return 0; |
| | | 89 | | } |
| | | 90 | | |
| | 0 | 91 | | var dto = new InboxMessageDto |
| | 0 | 92 | | { |
| | 0 | 93 | | MessageExternalId = message.MessageExternalId, |
| | 0 | 94 | | Uid = message.Uid, |
| | 0 | 95 | | SenderEmail = message.SenderEmail, |
| | 0 | 96 | | Subject = message.Subject, |
| | 0 | 97 | | BodyText = message.BodyText, |
| | 0 | 98 | | ReceivedAtUtc = message.ReceivedAtUtc, |
| | 0 | 99 | | Attachments = message.Attachments.Select(x => new InboxAttachmentDto |
| | 0 | 100 | | { |
| | 0 | 101 | | FileName = x.FileName, |
| | 0 | 102 | | Extension = x.Extension, |
| | 0 | 103 | | FileSizeBytes = x.FileSizeBytes, |
| | 0 | 104 | | FileBytes = x.FileBytes |
| | 0 | 105 | | }).ToList() |
| | 0 | 106 | | }; |
| | | 107 | | |
| | 0 | 108 | | var rules = await _rulesDataServices.GetActiveRules(); |
| | | 109 | | // Вручную переобрабатываем письмо, даже если оно уже было обработано ранее. |
| | 0 | 110 | | var result = await ProcessMessageInternal(dto, rules, false, cancellationToken, forceReprocess: true); |
| | 0 | 111 | | return result.MessagesProcessed; |
| | 0 | 112 | | } |
| | | 113 | | |
| | | 114 | | private async Task<EmailInboxProcessResult> ProcessMessageInternal( |
| | | 115 | | InboxMessageDto messageDto, |
| | | 116 | | List<EmailRoutingRule> rules, |
| | | 117 | | bool dryRun, |
| | | 118 | | CancellationToken cancellationToken, |
| | | 119 | | bool forceReprocess) |
| | | 120 | | { |
| | 5 | 121 | | var existing = await _messageDataServices.GetByExternalId(messageDto.MessageExternalId); |
| | 5 | 122 | | if (existing == null && !string.IsNullOrWhiteSpace(messageDto.Uid)) |
| | | 123 | | { |
| | 3 | 124 | | existing = await _messageDataServices.GetByUid(messageDto.Uid); |
| | | 125 | | } |
| | | 126 | | |
| | 5 | 127 | | if (existing != null && !forceReprocess) |
| | | 128 | | { |
| | 3 | 129 | | _logger.LogInformation( |
| | 3 | 130 | | "Skip already known message {MessageExternalId} (uid {Uid}, forceReprocess=false). Current status: {Stat |
| | 3 | 131 | | messageDto.MessageExternalId, |
| | 3 | 132 | | messageDto.Uid, |
| | 3 | 133 | | existing.ProcessingStatus); |
| | 3 | 134 | | return new EmailInboxProcessResult(); |
| | | 135 | | } |
| | | 136 | | |
| | 2 | 137 | | var result = new EmailInboxProcessResult(); |
| | | 138 | | |
| | 2 | 139 | | var messageEntity = existing ?? new EmailMessageInbox |
| | 2 | 140 | | { |
| | 2 | 141 | | MessageExternalId = messageDto.MessageExternalId, |
| | 2 | 142 | | Uid = messageDto.Uid, |
| | 2 | 143 | | SenderEmail = messageDto.SenderEmail, |
| | 2 | 144 | | Subject = messageDto.Subject, |
| | 2 | 145 | | BodyText = messageDto.BodyText, |
| | 2 | 146 | | ReceivedAtUtc = messageDto.ReceivedAtUtc, |
| | 2 | 147 | | CreatedAt = DateTime.UtcNow, |
| | 2 | 148 | | ProcessingStatus = EmailMessageProcessingStatus.New |
| | 2 | 149 | | }; |
| | | 150 | | |
| | 2 | 151 | | if (existing == null) |
| | | 152 | | { |
| | 2 | 153 | | await _messageDataServices.AddInboxMessage(messageEntity); |
| | | 154 | | } |
| | | 155 | | |
| | 2 | 156 | | var hadErrors = false; |
| | 2 | 157 | | var hadWarnings = false; |
| | 2 | 158 | | var matchedCount = 0; |
| | | 159 | | |
| | 10 | 160 | | foreach (var attachmentDto in messageDto.Attachments) |
| | | 161 | | { |
| | 3 | 162 | | var hash = _deduplicationService.ComputeSha256(attachmentDto.FileBytes); |
| | 3 | 163 | | var messageKey = existing?.MessageExternalId ?? messageDto.MessageExternalId; |
| | | 164 | | |
| | 3 | 165 | | if (!forceReprocess && IsAttachmentAlreadyStored(messageEntity, hash, attachmentDto)) |
| | | 166 | | { |
| | 1 | 167 | | _logger.LogInformation( |
| | 1 | 168 | | "Skip duplicate attachment {FileName} for message {MessageExternalId}", |
| | 1 | 169 | | attachmentDto.FileName, |
| | 1 | 170 | | messageKey); |
| | 1 | 171 | | hadWarnings = true; |
| | 1 | 172 | | result.AttachmentsDuplicate++; |
| | 1 | 173 | | continue; |
| | | 174 | | } |
| | | 175 | | |
| | 2 | 176 | | if (!forceReprocess && await _deduplicationService.IsDuplicateAsync( |
| | 2 | 177 | | messageKey, hash, attachmentDto.FileName, attachmentDto.FileSizeBytes)) |
| | | 178 | | { |
| | 0 | 179 | | _logger.LogInformation( |
| | 0 | 180 | | "Skip duplicate attachment {FileName} already stored for message {MessageExternalId}", |
| | 0 | 181 | | attachmentDto.FileName, |
| | 0 | 182 | | messageKey); |
| | 0 | 183 | | hadWarnings = true; |
| | 0 | 184 | | result.AttachmentsDuplicate++; |
| | 0 | 185 | | continue; |
| | | 186 | | } |
| | | 187 | | |
| | 2 | 188 | | var attachmentEntity = new EmailAttachmentInbox |
| | 2 | 189 | | { |
| | 2 | 190 | | EmailMessageInboxId = messageEntity.Id, |
| | 2 | 191 | | FileName = attachmentDto.FileName, |
| | 2 | 192 | | Extension = attachmentDto.Extension, |
| | 2 | 193 | | FileSizeBytes = attachmentDto.FileSizeBytes, |
| | 2 | 194 | | Sha256Hash = hash, |
| | 2 | 195 | | FileBytes = attachmentDto.FileBytes, |
| | 2 | 196 | | CreatedAt = DateTime.UtcNow |
| | 2 | 197 | | }; |
| | 2 | 198 | | _db.EmailAttachmentInboxes.Add(attachmentEntity); |
| | 2 | 199 | | await _db.SaveChangesAsync(cancellationToken); |
| | | 200 | | |
| | 2 | 201 | | var ruleResult = _ruleEvaluator.Evaluate(messageDto, attachmentDto, rules); |
| | 2 | 202 | | if (!ruleResult.IsMatch || ruleResult.Rule == null) |
| | | 203 | | { |
| | 2 | 204 | | hadWarnings = true; |
| | 2 | 205 | | result.AttachmentsRejected++; |
| | 2 | 206 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 2 | 207 | | { |
| | 2 | 208 | | EmailMessageInboxId = messageEntity.Id, |
| | 2 | 209 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 2 | 210 | | Status = EmailAttachmentProcessingStatus.RejectedByRule, |
| | 2 | 211 | | Reason = ruleResult.RejectReason |
| | 2 | 212 | | }); |
| | 2 | 213 | | continue; |
| | | 214 | | } |
| | | 215 | | |
| | 0 | 216 | | matchedCount++; |
| | 0 | 217 | | if (dryRun) |
| | | 218 | | { |
| | 0 | 219 | | result.AttachmentsPending++; |
| | 0 | 220 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 0 | 221 | | { |
| | 0 | 222 | | EmailMessageInboxId = messageEntity.Id, |
| | 0 | 223 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 0 | 224 | | EmailRoutingRuleId = ruleResult.Rule.Id, |
| | 0 | 225 | | Status = EmailAttachmentProcessingStatus.Pending, |
| | 0 | 226 | | Reason = "Dry-run: загрузка не выполнялась." |
| | 0 | 227 | | }); |
| | 0 | 228 | | continue; |
| | | 229 | | } |
| | | 230 | | |
| | 0 | 231 | | var importResult = await _orderEmailImportService.TryImportAsync( |
| | 0 | 232 | | ruleResult.Rule, messageDto, attachmentDto, forceReprocess, cancellationToken); |
| | | 233 | | |
| | 0 | 234 | | if (importResult.Handled) |
| | | 235 | | { |
| | 0 | 236 | | if (!importResult.IsSuccess) |
| | | 237 | | { |
| | 0 | 238 | | hadErrors = true; |
| | 0 | 239 | | result.AttachmentsFailed++; |
| | 0 | 240 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 0 | 241 | | { |
| | 0 | 242 | | EmailMessageInboxId = messageEntity.Id, |
| | 0 | 243 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 0 | 244 | | EmailRoutingRuleId = ruleResult.Rule.Id, |
| | 0 | 245 | | Status = EmailAttachmentProcessingStatus.Failed, |
| | 0 | 246 | | Reason = $"Ошибка импорта заказов ({importResult.MappingName}).", |
| | 0 | 247 | | Error = importResult.Error |
| | 0 | 248 | | }); |
| | 0 | 249 | | continue; |
| | | 250 | | } |
| | | 251 | | |
| | 0 | 252 | | result.AttachmentsOrdersImported++; |
| | 0 | 253 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 0 | 254 | | { |
| | 0 | 255 | | EmailMessageInboxId = messageEntity.Id, |
| | 0 | 256 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 0 | 257 | | EmailRoutingRuleId = ruleResult.Rule.Id, |
| | 0 | 258 | | Status = EmailAttachmentProcessingStatus.OrdersImported, |
| | 0 | 259 | | Reason = BuildOrdersImportedReason(importResult) |
| | 0 | 260 | | }); |
| | | 261 | | } |
| | | 262 | | else |
| | | 263 | | { |
| | 0 | 264 | | var route = await _dropboxFileRouter.RouteAsync(ruleResult.Rule, messageDto, attachmentDto, hash); |
| | 0 | 265 | | if (!route.IsSuccess) |
| | | 266 | | { |
| | 0 | 267 | | hadErrors = true; |
| | 0 | 268 | | result.AttachmentsFailed++; |
| | 0 | 269 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 0 | 270 | | { |
| | 0 | 271 | | EmailMessageInboxId = messageEntity.Id, |
| | 0 | 272 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 0 | 273 | | EmailRoutingRuleId = ruleResult.Rule.Id, |
| | 0 | 274 | | Status = EmailAttachmentProcessingStatus.Failed, |
| | 0 | 275 | | Reason = "Ошибка загрузки в Dropbox.", |
| | 0 | 276 | | Error = route.Error |
| | 0 | 277 | | }); |
| | 0 | 278 | | continue; |
| | | 279 | | } |
| | | 280 | | |
| | 0 | 281 | | result.AttachmentsUploaded++; |
| | 0 | 282 | | await _resultDataServices.AddResult(new EmailAttachmentProcessingResult |
| | 0 | 283 | | { |
| | 0 | 284 | | EmailMessageInboxId = messageEntity.Id, |
| | 0 | 285 | | EmailAttachmentInboxId = attachmentEntity.Id, |
| | 0 | 286 | | EmailRoutingRuleId = ruleResult.Rule.Id, |
| | 0 | 287 | | Status = EmailAttachmentProcessingStatus.Uploaded, |
| | 0 | 288 | | Reason = "Файл успешно загружен в Dropbox.", |
| | 0 | 289 | | DropboxFolderPath = route.FolderPath, |
| | 0 | 290 | | DropboxFileName = route.FileName, |
| | 0 | 291 | | DropboxFilePath = route.FilePath, |
| | 0 | 292 | | DropboxDownloadLink = route.DownloadLink |
| | 0 | 293 | | }); |
| | | 294 | | } |
| | | 295 | | |
| | 0 | 296 | | if (ruleResult.Rule.ProcessOnlyFirstMatchedAttachment && !_executionOptions.AllowMultiMatch) |
| | | 297 | | { |
| | | 298 | | break; |
| | | 299 | | } |
| | 0 | 300 | | } |
| | | 301 | | |
| | 2 | 302 | | messageEntity.ProcessedAtUtc = DateTime.UtcNow; |
| | 2 | 303 | | messageEntity.ProcessingStatus = hadErrors |
| | 2 | 304 | | ? EmailMessageProcessingStatus.Failed |
| | 2 | 305 | | : hadWarnings |
| | 2 | 306 | | ? EmailMessageProcessingStatus.ProcessedWithWarnings |
| | 2 | 307 | | : EmailMessageProcessingStatus.ProcessedSuccessfully; |
| | 2 | 308 | | messageEntity.ProcessingError = hadErrors ? "Есть ошибки обработки вложений." : null; |
| | | 309 | | |
| | 2 | 310 | | await _messageDataServices.UpdateInboxMessage(messageEntity); |
| | 2 | 311 | | _logger.LogInformation( |
| | 2 | 312 | | "Processed message {MessageExternalId}. Matches: {MatchedCount}, status: {Status}", |
| | 2 | 313 | | messageDto.MessageExternalId, |
| | 2 | 314 | | matchedCount, |
| | 2 | 315 | | messageEntity.ProcessingStatus); |
| | | 316 | | |
| | 2 | 317 | | result.MessagesProcessed = 1; |
| | 2 | 318 | | return result; |
| | 5 | 319 | | } |
| | | 320 | | |
| | | 321 | | /// <summary> |
| | | 322 | | /// Reason ограничен MaxLength(1000) на EmailAttachmentProcessingResult — номера обрезаются, чтобы влезть. |
| | | 323 | | /// </summary> |
| | | 324 | | private const int MaxReasonLength = 1000; |
| | | 325 | | |
| | | 326 | | private static string BuildOrdersImportedReason(OrderImportResult importResult) |
| | | 327 | | { |
| | 0 | 328 | | var reason = $"Заказы импортированы напрямую ({importResult.MappingName}/{importResult.Delivery}): " + |
| | 0 | 329 | | $"добавлено {importResult.AddedCount}, уже было в базе (пропущено) {importResult.SkippedExistingCou |
| | | 330 | | |
| | 0 | 331 | | if (importResult.AddedOrderNumbers.Count > 0) |
| | | 332 | | { |
| | 0 | 333 | | reason += $" Номера: {string.Join(", ", importResult.AddedOrderNumbers)}."; |
| | | 334 | | } |
| | | 335 | | |
| | 0 | 336 | | return reason.Length > MaxReasonLength |
| | 0 | 337 | | ? reason[..(MaxReasonLength - 1)] + "…" |
| | 0 | 338 | | : reason; |
| | | 339 | | } |
| | | 340 | | |
| | | 341 | | private static bool IsAttachmentAlreadyStored( |
| | | 342 | | EmailMessageInbox messageEntity, |
| | | 343 | | string hash, |
| | | 344 | | InboxAttachmentDto attachmentDto) => |
| | 3 | 345 | | messageEntity.Attachments.Any(x => |
| | 4 | 346 | | x.Sha256Hash == hash && |
| | 4 | 347 | | x.FileName == attachmentDto.FileName && |
| | 4 | 348 | | x.FileSizeBytes == attachmentDto.FileSizeBytes); |
| | | 349 | | } |