Added guardrails around the cluster as well as software level

LiteCharms.Features/Quartz/JobOrchestrator.cs

@@ -18,6 +18,7 @@ public sealed class JobOrchestrator(ISchedulerFactory schedulerFactory) : IJobOr
             .WithDescription($"Correlation ID: {notification.CorrelationId}")
             .UsingJobData(new JobDataMap { ["Payload"] = JsonSerializer.Serialize(notification) })
             .DisallowConcurrentExecution()
+            .RequestRecovery()
             .Build();
 
         var trigger = global::Quartz.TriggerBuilder.Create()

LiteCharms.Features/Quartz/MediatorJob.cs

@@ -8,6 +8,9 @@ public sealed class MediatorJob<TNotification>(IMediator mediator) : IJob where
 {
     public async Task Execute(IJobExecutionContext context)
     {
+        if (context.Recovering)
+            Trace.WriteLine($"CRITICAL RECOVERY: Resurrecting job '{typeof(TNotification).Name}' after a previous cluster node crashed mid-execution.");
+
         var data = context.MergedJobDataMap["Payload"] as string;
 
         if (string.IsNullOrWhiteSpace(data))