Merge pull request 'Updated API to use LiteCharms Security' (#83) from payments into master

Reviewed-on: #83

.gitignore

@@ -362,3 +362,5 @@ MigrationBackup/
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
 /LiteCharms.Features.Tests/http/http-client.env.json
+/LiteCharms.Features.Tests/http/midrandshop-api/http-client.env.json
+/LiteCharms.Features.Tests/http/authentik/http-client.env.json

LiteCharms.Features.MidrandBooks.Seed/appsettings.json

@@ -1,6 +1,6 @@
 {
   "FeatureManagement": {
-    "CategorySeederService": true,
+    "CategorySeederService": false,
     "CustomerSeederService": false,
     "ProductsSeederService": false
   },

LiteCharms.Features.MidrandBooks.Tests/PayfastServiceFeatureTests.cs

@@ -37,7 +37,7 @@ public sealed class PayfastServiceFeatureTests(Fixture fixture) : IClassFixture<
 
         string liveTargetIp = addresses.First().ToString();
 
-        var result = await payfastService.ValidateReferrerIpAsync(liveTargetIp, fixture.CancellationToken);
+        var result = await payfastService.ValidateReferrerIpAsync(liveTargetIp, true, fixture.CancellationToken);
 
         Assert.True(result.IsSuccess);
         Assert.True(result.Value);
@@ -48,7 +48,7 @@ public sealed class PayfastServiceFeatureTests(Fixture fixture) : IClassFixture<
     {
         string rogueIp = "8.8.8.8";
 
-        var result = await payfastService.ValidateReferrerIpAsync(rogueIp, fixture.CancellationToken);
+        var result = await payfastService.ValidateReferrerIpAsync(rogueIp, true, fixture.CancellationToken);
 
         Assert.True(result.IsSuccess);
         Assert.False(result.Value);
@@ -87,7 +87,7 @@ public sealed class PayfastServiceFeatureTests(Fixture fixture) : IClassFixture<
 
         var result = await payfastService.ValidateServerConfirmationAsync(arbitraryParameters, isSandbox: true, fixture.CancellationToken);
 
-        Assert.True(result.IsSuccess); 
+        Assert.True(result.IsSuccess);
         Assert.False(result.Value);   // Handshake data rejected as fraudulent/unrecognized
     }
 

LiteCharms.Features.MidrandBooks/Payments/Events/Handlers/PayfastPaymentConfirmationReceivedEventHandler.cs

@@ -1,17 +1,21 @@
 using LiteCharms.Features.Hasher;
 using LiteCharms.Features.Hasher.Configuration;
+using LiteCharms.Features.Mediator;
 using LiteCharms.Features.MidrandBooks.Orders;
 using LiteCharms.Features.MidrandBooks.Payments.Models;
 
 namespace LiteCharms.Features.MidrandBooks.Payments.Events.Handlers;
 
-public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvider services, IOptions<HasherSettings> hasherOptions, ILogger<PayfastPaymentConfirmationReceivedEvent> logger) : 
+public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvider services, IOptions<HasherSettings> hasherOptions, ILogger<PayfastPaymentConfirmationReceivedEvent> logger) :
     INotificationHandler<PayfastPaymentConfirmationReceivedEvent>
 {
     private readonly HasherSettings hasherSettings = hasherOptions.Value;
 
     public async ValueTask Handle(PayfastPaymentConfirmationReceivedEvent notification, CancellationToken cancellationToken)
     {
+        using var activity = MediatorTelemetry.Source.StartActivity($"Quartz: {typeof(PayfastPaymentConfirmationReceivedEvent).Name}");
+        activity?.SetTag("event.correlation_id", notification.CorrelationId);
+
         await using var scope = services.CreateAsyncScope();
         var hashService = scope.ServiceProvider.GetRequiredService<HashService>();
         var orderService = scope.ServiceProvider.GetRequiredService<OrderService>();
@@ -23,7 +27,7 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
         var dict = payload.ToParamDictionary();
         var localSignature = PayfastService.GenerateSignature(dict, hasherSettings.PayfastPassphrase);
 
-        if(localSignature.IsFailed)
+        if (localSignature.IsFailed)
             throw new Exception("Failed to generate local signature for incoming webhook payload.");
 
         if (!string.Equals(localSignature.Value, payload.Signature, StringComparison.OrdinalIgnoreCase))
@@ -61,9 +65,9 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
 
         if (notification.PerformBackgroundChecks)
         {
-            var isHostValid = await payfastService.ValidateReferrerIpAsync(notification.RemoteIpAddress!, cancellationToken);
+            var isHostValid = await payfastService.ValidateReferrerIpAsync(notification.RemoteIpAddress!, notification.AllowLoopback, cancellationToken);
 
-            if (isHostValid.IsFailed) 
+            if (isHostValid.IsFailed)
                 throw new Exception("Security validation exception: Webhook packet source address failed cluster validation checks.");
 
             if (!isHostValid.Value)
@@ -71,7 +75,7 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
 
             var isAmountValid = payfastService.ValidatePaymentAmount(orderResult.Value.Total, payload.AmountGross);
 
-            if (!isAmountValid.Value) 
+            if (!isAmountValid.Value)
                 throw new Exception("Security validation exception: Transaction cost variance bounds breached.");
 
             var paramList = new List<string>();
@@ -91,8 +95,8 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
 
             var serverConfirmation = await payfastService.ValidateServerConfirmationAsync(rawParamString, isSandbox: true, cancellationToken);
 
-            if (serverConfirmation.IsFailed) 
+            if (serverConfirmation.IsFailed)
-                throw new Exception("Security validation exception: Payfast central handshake server rejected payload legitimacy."); 
+                throw new Exception("Security validation exception: Payfast central handshake server rejected payload legitimacy.");
         }
 
         await payfastService.WriteLedgerEntryAsync(new CreateGatewayLedgerEntry
@@ -105,7 +109,7 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
             AmountFee = fee,
             AmountGross = gross,
             AmountNet = net,
-            PaymentStatus = status,            
+            PaymentStatus = status,
         }, cancellationToken);
 
         if (status.Equals("COMPLETE", StringComparison.OrdinalIgnoreCase))
@@ -154,5 +158,7 @@ public sealed class PayfastPaymentConfirmationReceivedEventHandler(IServiceProvi
 
             logger.LogInformation("Webhook validation pipeline passed checks successfully, logged entry to ledger with status: {Status}", status);
         }
+        activity?.SetStatus(ActivityStatusCode.Ok);
+
     }
 }

LiteCharms.Features.MidrandBooks/Payments/Events/PayfastPaymentConfirmationReceivedEvent.cs

@@ -13,15 +13,18 @@ public sealed class PayfastPaymentConfirmationReceivedEvent : EventBase, IEvent
 
     public bool PerformBackgroundChecks { get; set; }
 
+    public bool AllowLoopback { get; set; }
+
     public PayfastPaymentConfirmationReceivedEvent() { }
 
-    private PayfastPaymentConfirmationReceivedEvent(PayfastWebhookPayload? payload, string paymentId, bool performBackgroundChecks = true)
+    private PayfastPaymentConfirmationReceivedEvent(PayfastWebhookPayload? payload, string paymentId, bool performBackgroundChecks = true, bool allowLoopback = false)
     {
         Payload = payload;
         CorrelationId = paymentId;
         PerformBackgroundChecks = performBackgroundChecks;
+        AllowLoopback = allowLoopback;
     }
 
-    public static PayfastPaymentConfirmationReceivedEvent Create(PayfastWebhookPayload? payload, string paymentId, bool performBackgroundChecks = true) =>
+    public static PayfastPaymentConfirmationReceivedEvent Create(PayfastWebhookPayload? payload, string paymentId, bool performBackgroundChecks = true, bool allowLoopback = false) =>
-        new(payload, paymentId, performBackgroundChecks);
+        new(payload, paymentId, performBackgroundChecks, allowLoopback);
 }

LiteCharms.Features.MidrandBooks/Payments/PayfastService.cs

@@ -49,7 +49,7 @@ public sealed partial class PayfastService(IDbContextFactory<MidrandBooksDbConte
         }
     }
 
-    public async ValueTask<Result<bool>> ValidateReferrerIpAsync(string remoteIpAddress, CancellationToken cancellationToken = default)
+    public async ValueTask<Result<bool>> ValidateReferrerIpAsync(string remoteIpAddress, bool allowLoopback = false, CancellationToken cancellationToken = default)
     {
         if (string.IsNullOrWhiteSpace(remoteIpAddress)) 
             return Result.Fail<bool>("Remote IP address is null or whitespace.");
@@ -74,6 +74,12 @@ public sealed partial class PayfastService(IDbContextFactory<MidrandBooksDbConte
 
             if (IPAddress.TryParse(remoteIpAddress, out var incomingIp))
             {
+                if (allowLoopback && IPAddress.IsLoopback(incomingIp))
+                {
+                    logger.LogInformation("Local development loopback IP '{RemoteIp}' allowed bypassing DNS verification.", remoteIpAddress);
+                    return Result.Ok(true);
+                }
+
                 bool isValid = validIps.Contains(incomingIp);
 
                 if (!isValid)

LiteCharms.Features.Tests/http/litecharms/app.http

@@ -0,0 +1,6 @@
+### Authentik Token Request (Service Account Explicit)
+POST {{authority}}/connect/token
+Content-Type: application/x-www-form-urlencoded
+Accept-Encoding: identity
+
+grant_type={{grantType}}&client_id={{clientId}}&client_secret={{clientSecret}}&scope={{scope}}

LiteCharms.Features.Tests/http/litecharms/http-client.env.json

@@ -0,0 +1,9 @@
+{
+  "uat": {
+    "authority": "https://sts.security.khongisa.co.za",
+    "grantType": "client_credentials",
+    "clientId": "midrandbooks-api-uat",
+    "clientSecret": "secret_5a36d0024980544c875447a4b052938becc3fbbb10b8b2c097310c1a53ba3c0a",
+    "scope": "midrandbooks-api"
+  }
+}

LiteCharms.Features.Tests/http/midrandshop-api/http-client.env.json

@@ -1,16 +0,0 @@
-{
-  "local": {
-    "baseUrl": "https://localhost:7196",
-    "paymentId": "jdPB2zaKM3Z",
-    "signature": "6aeff59bb74f2448ff2c3d81b2ec95de",
-    "item_name": "System Architecture Book",
-    "amount": "350.00"
-  },
-  "uat": {
-    "baseUrl": "https://api.uat.midrandbooks.co.za",
-    "paymentId": "jdPB2zaKM3Z",
-    "signature": "6aeff59bb74f2448ff2c3d81b2ec95de",
-    "item_name": "System Architecture Book",
-    "amount": "350.00"
-  }
-}

LiteCharms.Features/Abstractions/IJobOrchestrator.cs

@@ -0,0 +1,12 @@
+namespace LiteCharms.Features.Abstractions;
+
+public interface IJobOrchestrator
+{
+    ValueTask SendAsync<TNotification>(TNotification notification, CancellationToken cancellationToken = default)
+        where TNotification : IEvent;
+
+    ValueTask ScheduleAsync<TNotification>(TNotification notification, string cronExpression, CancellationToken cancellationToken = default) 
+        where TNotification : IEvent;
+
+    ValueTask<bool> InterruptAsync(string eventName, string? correlationId = null, CancellationToken cancellationToken = default);
+}

LiteCharms.Features/Api/Configuration/LiteCharmsSettings.cs

@@ -0,0 +1,16 @@
+namespace LiteCharms.Features.Api.Configuration;
+
+public sealed class LiteCharmsSettings
+{
+    public string? Authority { get; set; }
+
+    public string? ClientId { get; set; }
+
+    public string? ClientSecret { get; set; }
+
+    public string? Audience { get; set; }
+
+    public string? RequiredClaimName { get; set; }
+
+    public string? RequiredClaimNameValue { get; set; }
+}

LiteCharms.Features/Api/OpenApiBearerSecuritySchemeTransformer.cs

@@ -8,7 +8,7 @@ public sealed class OpenApiBearerSecuritySchemeTransformer : IOpenApiDocumentTra
         {
             Type = SecuritySchemeType.Http,
             Scheme = "bearer",
-            Description = "JWT Authorization header using the Bearer scheme. Example: \"Bearer {token}\"",
+            Description = "JWT Authorization header using the Bearer scheme",
         };
 
         document.AddComponent("Bearer", bearerScheme);

LiteCharms.Features/Extensions/Api.cs

@@ -1,5 +1,7 @@
 using LiteCharms.Features.Abstractions;
 using LiteCharms.Features.Api;
+using LiteCharms.Features.Api.Configuration;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
 
 namespace LiteCharms.Features.Extensions;
 
@@ -8,6 +10,161 @@ public static class Api
     public const string Books = nameof(Books);
     public const string Payments = nameof(Payments);
 
+    public static IServiceCollection AddLiteCharmsUiSecurity(this IServiceCollection services, IConfiguration configuration)
+    {
+        var configSection = configuration.GetSection(nameof(LiteCharmsSettings));
+
+        var authOptions = new LiteCharmsSettings();
+        configSection.Bind(authOptions);
+
+        services.Configure<LiteCharmsSettings>(configSection);
+
+        services.AddAuthentication(options =>
+        {
+            options.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+            options.DefaultChallengeScheme = OpenIdConnectDefaults.AuthenticationScheme;
+        })
+        .AddCookie(CookieAuthenticationDefaults.AuthenticationScheme)
+        .AddOpenIdConnect(OpenIdConnectDefaults.AuthenticationScheme, options =>
+        {
+            options.Authority = authOptions.Authority;           
+
+            options.ClientId = authOptions.ClientId;
+            options.ClientSecret = authOptions.ClientSecret;
+            options.SignedOutCallbackPath = "/signout-callback-oidc";
+
+            options.ResponseType = "code";
+            options.SaveTokens = true;
+            options.GetClaimsFromUserInfoEndpoint = true;
+
+            options.Scope.Clear();
+            options.Scope.Add("openid");
+            options.Scope.Add("profile");
+            options.Scope.Add("email");
+
+            options.CorrelationCookie.SecurePolicy = CookieSecurePolicy.Always;
+            options.CorrelationCookie.SameSite = SameSiteMode.None;
+            options.CorrelationCookie.HttpOnly = true;
+
+            options.NonceCookie.SecurePolicy = CookieSecurePolicy.Always;
+            options.NonceCookie.SameSite = SameSiteMode.None;
+            options.NonceCookie.HttpOnly = true;
+        });
+
+        return services;
+    }
+
+    public static IServiceCollection AddLiteCharmsApiSecurity(this IServiceCollection services, IConfiguration configuration)
+    {
+        var configSection = configuration.GetSection(nameof(LiteCharmsSettings));
+
+        var authOptions = new LiteCharmsSettings();
+        configSection.Bind(authOptions);
+
+        services.Configure<LiteCharmsSettings>(configSection);
+
+        services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
+            .AddJwtBearer(options =>
+            {
+                options.Authority = authOptions.Authority;
+                options.Audience = authOptions.Audience;
+                options.TokenValidationParameters = new TokenValidationParameters
+                {
+                    ValidIssuer = authOptions.Authority,
+                    ValidateAudience = true,
+                    ValidateIssuer = true,
+                };
+            });
+
+        if (!string.IsNullOrWhiteSpace(authOptions.RequiredClaimName) && !string.IsNullOrWhiteSpace(authOptions.RequiredClaimNameValue))
+        {
+            services.AddAuthorizationBuilder()
+                .AddPolicy("RequiredScope", policy =>
+                    policy.RequireClaim(authOptions.RequiredClaimName, authOptions.RequiredClaimNameValue));
+        }
+        else
+            services.AddAuthorization();
+
+        return services;
+    }
+
+    public static WebApplication AddSecurityEndpoints(this WebApplication app)
+    {
+        app.MapGet("/login", async (HttpContext context, string redirectUri = "/") =>
+        {
+            await context.ChallengeAsync(OpenIdConnectDefaults.AuthenticationScheme, new AuthenticationProperties
+            {
+                RedirectUri = redirectUri,                
+            });
+        });
+
+        app.MapGet("/logout", async (HttpContext context, IHttpClientFactory httpClientFactory, IOptions<LiteCharmsSettings> settings) =>
+        {
+            await context.SignOutAsync(CookieAuthenticationDefaults.AuthenticationScheme);
+
+            string currentBaseUrl = $"https://{context.Request.Host}{context.Request.PathBase}/";
+
+            await context.SignOutAsync(OpenIdConnectDefaults.AuthenticationScheme, new AuthenticationProperties
+            {
+                RedirectUri = currentBaseUrl
+            });
+        });
+
+        return app;
+    }
+
+    public static IServiceCollection AddApiServices(this IServiceCollection services, IConfiguration configuration)
+    {
+        services.AddHttpClient();
+
+        services.AddApiVersioning(options =>
+        {
+            options.DefaultApiVersion = new ApiVersion(1);
+            options.ReportApiVersions = true;
+            options.AssumeDefaultVersionWhenUnspecified = true;
+            options.ApiVersionReader = ApiVersionReader.Combine(new UrlSegmentApiVersionReader(),
+                new QueryStringApiVersionReader("version"),
+                new QueryStringApiVersionReader("version"),
+                new MediaTypeApiVersionReader("version"));
+        })
+        .AddApiExplorer(options =>
+        {
+            options.GroupNameFormat = "'v'VVV";
+            options.SubstituteApiVersionInUrl = true;
+        });
+
+        var urls = configuration["ASPNETCORE_URLS"] ?? configuration["Urls"];
+        var healthUrl = "http://localhost:8080/health";
+
+        if (!string.IsNullOrWhiteSpace(urls))
+        {
+            string firstUrl = urls.Split(';').FirstOrDefault(s => s.Contains("http://"))!
+                .Replace("0.0.0.0", "localhost")
+                .Replace("*", "localhost")
+                .Replace("+", "localhost");
+
+            healthUrl = $"{firstUrl.TrimEnd('/')}/health";
+        }
+
+        services.AddHealthChecksUI(setup =>
+        {
+            setup.SetNotifyUnHealthyOneTimeUntilChange();
+            setup.AddHealthCheckEndpoint("primary, heal", healthUrl);
+            setup.SetHeaderText("Midrand Books");
+        })
+        .AddInMemoryStorage();
+
+        services.AddOutputCache(options =>
+        {
+            options.AddBasePolicy(builder => builder.Cache());
+            options.DefaultExpirationTimeSpan = TimeSpan.FromSeconds(10);
+        });
+
+        services.AddOpenApi(options => options.AddDocumentTransformer<OpenApiBearerSecuritySchemeTransformer>());
+
+        return services;
+    }
+
     public static IApplicationBuilder MapEndpoints(this WebApplication app, IDictionary<int, RouteGroupBuilder> versionGroups)
     {
         var endpoints = app.Services.GetRequiredService<IEnumerable<IEndpoint>>();
@@ -43,54 +200,4 @@ public static class Api
 
     public static string ToEndpointName(this Type target, string? annotation = "") =>
         $"{target.Name.Replace("Endpoint", string.Empty)}{annotation}".ToLower(CultureInfo.CurrentCulture);
-
-    public static IServiceCollection AddApiServices(this IServiceCollection services, IConfiguration configuration)
-    {
-        services.AddHttpClient();
-
-        services.AddApiVersioning(options =>
-        {
-            options.DefaultApiVersion = new ApiVersion(1);
-            options.ReportApiVersions = true;
-            options.AssumeDefaultVersionWhenUnspecified = true;
-            options.ApiVersionReader = ApiVersionReader.Combine(new UrlSegmentApiVersionReader(),
-                new QueryStringApiVersionReader("version"),
-                new QueryStringApiVersionReader("version"),
-                new MediaTypeApiVersionReader("version"));
-        })
-        .AddApiExplorer(options =>
-        {
-            options.GroupNameFormat = "'v'VVV";
-            options.SubstituteApiVersionInUrl = true;
-        });
-
-        var urls = configuration["ASPNETCORE_URLS"] ?? configuration["Urls"];
-        var healthUrl = "http://localhost:8080/health";
-
-        if (!string.IsNullOrWhiteSpace(urls))
-        {
-            string firstUrl = urls.Split(';').FirstOrDefault(s => s.Contains("http://"))!
-                .Replace("*", "localhost").Replace("+", "localhost");
-
-            healthUrl = $"{firstUrl.TrimEnd('/')}/health";
-        }
-
-        services.AddHealthChecksUI(setup =>
-        {
-            setup.SetNotifyUnHealthyOneTimeUntilChange();
-            setup.AddHealthCheckEndpoint("primary, heal", healthUrl);
-            setup.SetHeaderText("Midrand Books");
-        })
-        .AddInMemoryStorage();
-
-        services.AddOutputCache(options =>
-        {
-            options.AddBasePolicy(builder => builder.Cache());
-            options.DefaultExpirationTimeSpan = TimeSpan.FromSeconds(10);
-        });
-
-        services.AddOpenApi(options => options.AddDocumentTransformer<OpenApiBearerSecuritySchemeTransformer>());
-
-        return services;
-    }
 }

LiteCharms.Features/Extensions/Quartz.cs

@@ -1,5 +1,5 @@
-using LiteCharms.Features.Quartz;
+using LiteCharms.Features.Abstractions;
-using LiteCharms.Features.Quartz.Abstractions;
+using LiteCharms.Features.Quartz;
 using static LiteCharms.Features.Extensions.Postgres;
 
 namespace LiteCharms.Features.Extensions;

LiteCharms.Features/Hasher/HashService.cs

@@ -4,7 +4,7 @@ namespace LiteCharms.Features.Hasher;
 
 public sealed partial class HashService(IHashids hasher) : IService
 {
-    [GeneratedRegex(@"\A\b[0-9a-fA-F]+\b\Z")]
+    [GeneratedRegex(@"\A\b[0-9a-fA-F]+\b\Z", RegexOptions.None, matchTimeoutMilliseconds: 100)]
     private static partial Regex HexHashRegex { get; }
 
     [GeneratedRegex(@"\A[0-9a-fA-F]{32}\Z", RegexOptions.None, matchTimeoutMilliseconds: 100)]

LiteCharms.Features/LiteCharms.Features.csproj

@@ -29,6 +29,21 @@
     <None Include="..\icon.png" Pack="true" PackagePath="\" />
   </ItemGroup>
 
+  <!-- Security (IODC)-->
+  <ItemGroup>
+    <PackageReference Include="IdentityModel.AspNetCore" Version="4.3.0" />
+    <PackageReference Include="IdentityModel.AspNetCore.OAuth2introspection" Version="6.2.0" />
+    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="3.0.1" />
+    <PackageReference Include="IdentityModel" Version="6.2.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.Certificate" Version="10.0.8" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
+
+    <Using Include="Microsoft.AspNetCore.Authentication"/>
+    <Using Include="Microsoft.AspNetCore.Authentication.OpenIdConnect"/>
+    <Using Include="Microsoft.AspNetCore.Authentication.Cookies"/>
+    <Using Include="IdentityModel.AspNetCore.OAuth2Introspection"/>
+  </ItemGroup>
+
   <!-- API Versioning -->
   <ItemGroup>
     <PackageReference Include="AccessTokenClient.Extensions" Version="5.1.0" />

LiteCharms.Features/Quartz/Abstractions/IJobOrchestrator.cs

@@ -1,12 +0,0 @@
-using LiteCharms.Features.Abstractions;
-
-namespace LiteCharms.Features.Quartz.Abstractions;
-
-public interface IJobOrchestrator
-{
-    Task SendAsync<TNotification>(TNotification notification, CancellationToken cancellationToken = default)
-        where TNotification : IEvent;
-
-    Task ScheduleAsync<TNotification>(TNotification notification, string cronExpression, CancellationToken cancellationToken = default) 
-        where TNotification : IEvent;
-}

LiteCharms.Features/Quartz/JobOrchestrator.cs

@@ -1,11 +1,10 @@
 using LiteCharms.Features.Abstractions;
-using LiteCharms.Features.Quartz.Abstractions;
 
 namespace LiteCharms.Features.Quartz;
 
 public sealed class JobOrchestrator(ISchedulerFactory schedulerFactory) : IJobOrchestrator
 {
-    public async Task SendAsync<TNotification>(TNotification notification, CancellationToken cancellationToken = default)
+    public async ValueTask SendAsync<TNotification>(TNotification notification, CancellationToken cancellationToken = default)
         where TNotification : IEvent
     {
         var chainedJobGroup = "onetime-jobs";
@@ -19,17 +18,18 @@ public sealed class JobOrchestrator(ISchedulerFactory schedulerFactory) : IJobOr
             .WithDescription($"Correlation ID: {notification.CorrelationId}")
             .UsingJobData(new JobDataMap { ["Payload"] = JsonSerializer.Serialize(notification) })
             .DisallowConcurrentExecution()
+            .RequestRecovery()
             .Build();
 
         var trigger = global::Quartz.TriggerBuilder.Create()
             .WithIdentity(triggerKey)
-            .StartNow()            
+            .StartNow()
             .Build();
 
         await scheduler.ScheduleJob(job, new List<ITrigger> { trigger }.AsReadOnly(), replace: true, cancellationToken);
     }
 
-    public async Task ScheduleAsync<TNotification>(TNotification notification, string cronExpression, CancellationToken cancellationToken = default)
+    public async ValueTask ScheduleAsync<TNotification>(TNotification notification, string cronExpression, CancellationToken cancellationToken = default)
         where TNotification : IEvent
     {
         var chainedJobGroup = "scheduled-jobs";
@@ -63,4 +63,25 @@ public sealed class JobOrchestrator(ISchedulerFactory schedulerFactory) : IJobOr
         else
             await scheduler.ScheduleJob(job, new List<ITrigger> { trigger }.AsReadOnly(), replace: true, cancellationToken);
     }
+
+    public async ValueTask<bool> InterruptAsync(string eventName, string? correlationId = null, CancellationToken cancellationToken = default)
+    {
+        var scheduler = await schedulerFactory.GetScheduler(cancellationToken);
+
+        var jobKeyName = string.Empty;
+        var jobGroup = string.Empty;
+
+        if (!string.IsNullOrWhiteSpace(correlationId))
+        {
+            jobKeyName = $"{eventName.ToLower(CultureInfo.InvariantCulture)}-{correlationId.ToLower(CultureInfo.InvariantCulture)}";
+            jobGroup = "onetime-jobs";
+        }
+        else
+        {
+            jobKeyName = eventName.ToLower(CultureInfo.InvariantCulture);
+            jobGroup = "scheduled-jobs";
+        }
+
+        return await scheduler.Interrupt(JobKey.Create(jobKeyName, jobGroup), cancellationToken);
+    }
 }

LiteCharms.Features/Quartz/MediatorJob.cs

@@ -8,6 +8,9 @@ public sealed class MediatorJob<TNotification>(IMediator mediator) : IJob where
 {
     public async Task Execute(IJobExecutionContext context)
     {
+        if (context.Recovering)
+            Trace.WriteLine($"CRITICAL RECOVERY: Resurrecting job '{typeof(TNotification).Name}' after a previous cluster node crashed mid-execution.");
+
         var data = context.MergedJobDataMap["Payload"] as string;
 
         if (string.IsNullOrWhiteSpace(data))
@@ -21,17 +24,28 @@ public sealed class MediatorJob<TNotification>(IMediator mediator) : IJob where
 
         if (notification is null)
         {
-            Trace.WriteLine("Notification could not be JSon converted from data string, job ended");
+            Trace.WriteLine("Notification could not be Json converted from data string, job ended");
 
             return;
         }
-                
+
-        using var activity = MediatorTelemetry.Source.StartActivity($"Quartz: {typeof(TNotification).Name}");
+        using var activity = MediatorTelemetry.Source.StartActivity(typeof(TNotification).Name);
-                
+
         activity?.SetTag("event.correlation_id", notification.CorrelationId);
 
-        await mediator.Publish(notification, context.CancellationToken);
+        try
+        {
+            await mediator.Publish(notification, context.CancellationToken);
 
-        Trace.WriteLine("Job published");
+            Trace.WriteLine("Job published successfully");
+        }
+        catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested)
+        {
+            Trace.WriteLine($"Job '{typeof(TNotification).Name}' was gracefully interrupted by the cluster control plane.");
+            
+            activity?.SetStatus(ActivityStatusCode.Ok);
+
+            return;
+        }
     }
 }

LiteCharms.Features/Quartz/RetryJobListener.cs

@@ -12,6 +12,9 @@ public sealed class RetryJobListener : IJobListener
 
     public async Task JobWasExecuted(IJobExecutionContext context, JobExecutionException? jobException, CancellationToken cancellationToken = default)
     {
+        if (context.CancellationToken.IsCancellationRequested)
+            return;
+
         if (jobException is not null && context.RefireCount < RetryCount)
             jobException.RefireImmediately = true;
     }