| | | 1 | | using System.Diagnostics; |
| | | 2 | | using Hangfire.Common; |
| | | 3 | | using Hangfire.Server; |
| | | 4 | | using Serilog; |
| | | 5 | | |
| | | 6 | | namespace OzonOrdersWeb.Observability; |
| | | 7 | | |
| | 5 | 8 | | public sealed class HangfireObservabilityFilter(int slowMilliseconds) : JobFilterAttribute, IServerFilter |
| | | 9 | | { |
| | | 10 | | private const string StopwatchKey = "Studio2ObservabilityStopwatch"; |
| | | 11 | | |
| | | 12 | | public void OnPerforming(PerformingContext filterContext) |
| | | 13 | | { |
| | 0 | 14 | | filterContext.Items[StopwatchKey] = Stopwatch.StartNew(); |
| | 0 | 15 | | Logger(filterContext.BackgroundJob.Job) |
| | 0 | 16 | | .ForContext("JobId", filterContext.BackgroundJob.Id) |
| | 0 | 17 | | .Information("Hangfire job {JobType}.{JobMethod} начат", |
| | 0 | 18 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name); |
| | 0 | 19 | | } |
| | | 20 | | |
| | | 21 | | public void OnPerformed(PerformedContext filterContext) |
| | | 22 | | { |
| | 4 | 23 | | var elapsed = filterContext.Items.TryGetValue(StopwatchKey, out var value) && value is Stopwatch timer |
| | 4 | 24 | | ? timer.ElapsedMilliseconds |
| | 4 | 25 | | : 0; |
| | 4 | 26 | | var logger = Logger(filterContext.BackgroundJob.Job) |
| | 4 | 27 | | .ForContext("JobId", filterContext.BackgroundJob.Id) |
| | 4 | 28 | | .ForContext("ElapsedMilliseconds", elapsed); |
| | | 29 | | |
| | 4 | 30 | | var isCanceled = filterContext.Exception is OperationCanceledException || |
| | 4 | 31 | | (filterContext.Exception is AggregateException agg && agg.InnerExceptions.All(e => e is Operati |
| | | 32 | | |
| | 4 | 33 | | if (isCanceled) |
| | | 34 | | { |
| | 2 | 35 | | var isShutdown = filterContext.Canceled || |
| | 2 | 36 | | filterContext.CancellationToken?.ShutdownToken.IsCancellationRequested == true; |
| | | 37 | | |
| | 2 | 38 | | if (isShutdown) |
| | | 39 | | { |
| | 1 | 40 | | logger.Information("Hangfire job {JobType}.{JobMethod} прерван остановкой сервера (re-queued) за {Elapse |
| | 1 | 41 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed); |
| | | 42 | | } |
| | | 43 | | else |
| | | 44 | | { |
| | 1 | 45 | | logger.Warning("Hangfire job {JobType}.{JobMethod} отменён за {ElapsedMilliseconds} мс: {Message}", |
| | 1 | 46 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed, fil |
| | | 47 | | } |
| | 1 | 48 | | return; |
| | | 49 | | } |
| | | 50 | | |
| | 2 | 51 | | var isLockContention = HangfireDistributedLockContentionFilter.IsDistributedLockContentionException(filterContex |
| | | 52 | | |
| | 2 | 53 | | if (filterContext.Exception is not null && !filterContext.ExceptionHandled && !isLockContention) |
| | | 54 | | { |
| | 1 | 55 | | logger.Error(filterContext.Exception, "Hangfire job {JobType}.{JobMethod} завершился ошибкой за {ElapsedMill |
| | 1 | 56 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed); |
| | | 57 | | } |
| | 1 | 58 | | else if (filterContext.ExceptionHandled || isLockContention) |
| | | 59 | | { |
| | 1 | 60 | | logger.Information("Hangfire job {JobType}.{JobMethod} завершён (обработан фильтром) за {ElapsedMilliseconds |
| | 1 | 61 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed); |
| | | 62 | | } |
| | 0 | 63 | | else if (elapsed >= slowMilliseconds) |
| | | 64 | | { |
| | 0 | 65 | | logger.Warning("Медленный Hangfire job {JobType}.{JobMethod} завершён за {ElapsedMilliseconds} мс", |
| | 0 | 66 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed); |
| | | 67 | | } |
| | | 68 | | else |
| | | 69 | | { |
| | 0 | 70 | | logger.Information("Hangfire job {JobType}.{JobMethod} завершён за {ElapsedMilliseconds} мс", |
| | 0 | 71 | | filterContext.BackgroundJob.Job.Type.Name, filterContext.BackgroundJob.Job.Method.Name, elapsed); |
| | | 72 | | } |
| | 0 | 73 | | } |
| | | 74 | | |
| | 4 | 75 | | private static Serilog.ILogger Logger(Job job) => Log.ForContext("SourceContext", typeof(HangfireObservabilityFilter |
| | 4 | 76 | | .ForContext("EventKind", "HangfireJob") |
| | 4 | 77 | | .ForContext("EventCategory", "jobs") |
| | 4 | 78 | | .ForContext("JobType", job.Type.FullName) |
| | 4 | 79 | | .ForContext("JobMethod", job.Method.Name); |
| | | 80 | | } |