Merge pull request 'Refactored the S3 services to properly upload the file' (#34) from s3service into master

Reviewed-on: #34

LiteCharms.Features.Tests/CommonFixture.cs

@@ -14,18 +14,20 @@ public class CommonFixture : IDisposable
     {
         Configuration = new ConfigurationBuilder()
             .SetBasePath(Directory.GetCurrentDirectory())
-            .AddJsonFile("appsettings.json")
             .AddUserSecrets<CommonFixture>()
+            .AddJsonFile(Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json"), optional: true, reloadOnChange: true)
             .AddEnvironmentVariables()
             .Build();
 
-        Services = new ServiceCollection()
+        Services = new ServiceCollection()            
             .AddMediator()
             .AddLogging()
             .AddShopServices()
             .AddEmailServiceBus()
+            .AddGarageS3(Configuration)
             .AddShopDatabase(Configuration)
             .AddEmailServices(Configuration)
+            .AddSingleton(Configuration)
             .BuildServiceProvider();
 
         Mediator = Services.GetRequiredService<IMediator>();

LiteCharms.Features.Tests/LiteCharms.Features.Tests.csproj

@@ -27,10 +27,10 @@
 
   <!-- Global Usings -->
   <ItemGroup>
-    <Using Include="Mediator"/>
-    <Using Include="Xunit.Abstractions"/>
-    <Using Include="Microsoft.Extensions.DependencyInjection"/>
-    <Using Include="Microsoft.Extensions.Configuration"/>
+    <Using Include="Mediator" />
+    <Using Include="Xunit.Abstractions" />
+    <Using Include="Microsoft.Extensions.DependencyInjection" />
+    <Using Include="Microsoft.Extensions.Configuration" />
   </ItemGroup>
 
   <ItemGroup>
@@ -43,7 +43,7 @@
 
   <ItemGroup>
     <None Update="appsettings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>
 

LiteCharms.Features.Tests/NotificationsFeatureTests.cs

@@ -1,4 +1,6 @@
-using LiteCharms.Features.Shop.Notifications;
+using LiteCharms.Features.Models;
+using LiteCharms.Features.Shop.Notifications;
+using LiteCharms.Features.Shop.Notifications.Events;
 
 namespace LiteCharms.Features.Tests;
 
@@ -32,4 +34,31 @@ public class NotificationsFeatureTests(CommonFixture fixture, ITestOutputHelper
 
         foreach (var error in createResult.Errors) output.WriteLine(error.Message);
     }
+
+    [Fact]
+    public async Task GetNotifications_ShouldReturn_AllNotifications()
+    {
+        DateRange range = new()
+        {
+            From = DateOnly.FromDateTime(new DateTime(2026, 04, 01, 0, 0, 0, DateTimeKind.Utc)),
+            To = DateOnly.FromDateTime(DateTime.UtcNow),
+            MaxRecords = 10
+        };
+
+        var getResult = await notificationService.GetNotificationsAsync(range);
+
+        Assert.True(getResult.IsSuccess);
+
+        foreach (var error in getResult.Errors) output.WriteLine(error.Message);
+    }
+
+    [Fact]
+    public async Task ProcessEmailNotificationsEvent_ShouldSucceed()
+    {
+        var notification = ProcessEmailNotificationsEvent.Create();
+
+        await fixture.Mediator.Publish(notification);
+
+        Assert.True(true);
+    }
 }

LiteCharms.Features.Tests/S3ServiceFeatureTests.cs

@@ -0,0 +1,28 @@
+using LiteCharms.Features.S3.Abstractions;
+
+namespace LiteCharms.Features.Tests;
+
+public class S3ServiceFeatureTests(CommonFixture fixture, ITestOutputHelper output) : IClassFixture<CommonFixture>
+{
+    [Fact]
+    public async Task BookshopS3Service_MustReturnUrl()
+    {
+        var service = fixture.Services.GetKeyedService<IS3Service>(S3.Constants.BookshopQuotesBucketName);
+
+        var fileName = "appsettings.json";
+
+        string path = Path.Combine(Directory.GetCurrentDirectory(), fileName);
+
+        Assert.True(File.Exists(path));
+
+        var stream = File.OpenRead(path);
+
+        var result = await service!.UploadFileAsync(fileName, stream, MimeKit.MimeTypes.GetMimeType(fileName));
+
+        Assert.True(result.IsSuccess);
+        Assert.NotNull(result.Value);
+        Assert.NotEmpty(result.Value);
+
+        output.WriteLine(result.Value);        
+    }
+}

LiteCharms.Features.Tests/appsettings.json

@@ -1,4 +1,16 @@
 {
+  "BookshopS3Settings": {
+    "ServiceUrl": "http://192.168.1.177:30900",
+    "Region": "garage",
+    "BucketName": "bookshop",
+    "CdnBaseUrl": "https://bookshop.cdn.khongisa.co.za"
+  },
+  "BookshopQuotesS3Settings": {
+    "ServiceUrl": "http://192.168.1.177:30900",
+    "Region": "garage",
+    "BucketName": "bookshop.quotes",
+    "CdnBaseUrl": "https://bookshop.quotes.cdn.khongisa.co.za"
+  },
   "Email": {
     "Credentials": {
       "Username": "shop@litecharms.co.za"

LiteCharms.Features/Email/EmailService.cs

@@ -35,8 +35,9 @@ public class EmailService(IOptions<SmtpSettings> options) : IDisposable
 
             var bodyBuilder = new BodyBuilder();
 
-            foreach (var attachment in message.Body?.Attachments!)
-                bodyBuilder.Attachments.Add(attachment.Name!, attachment.FileStream!, cancellationToken);
+            if (message.Body!.Properties.HasAttachments)
+                foreach (var attachment in message.Body?.Attachments!)
+                    bodyBuilder.Attachments.Add(attachment.Name!, attachment.FileStream!, cancellationToken);
 
             if (!message.Body.Properties.IsHtml) bodyBuilder.TextBody = message.Body.Message;
             if (message.Body.Properties.IsHtml) bodyBuilder.HtmlBody = message.Body.Message;

LiteCharms.Features/Extensions/Quartz.cs

@@ -34,7 +34,7 @@ public static class Quartz
                 storage.UseClustering(cluster =>
                 {
                     cluster.CheckinInterval = TimeSpan.FromSeconds(30);
-                    cluster.CheckinMisfireThreshold = TimeSpan.FromSeconds(20);
+                    cluster.CheckinMisfireThreshold = TimeSpan.FromSeconds(90);
                 });
             });
         });
@@ -62,6 +62,8 @@ public static class Quartz
             config.UseDefaultThreadPool(options => options.MaxConcurrency = 1);
             config.UseTimeZoneConverter();
 
+            config.SetProperty("quartz.jobStore.misfireThreshold", TimeSpan.FromMinutes(2).TotalMilliseconds.ToString());
+
             config.UsePersistentStore(storage =>
             {
                 storage.PerformSchemaValidation = false;
@@ -74,7 +76,7 @@ public static class Quartz
                 storage.UseClustering(cluster =>
                 {
                     cluster.CheckinInterval = TimeSpan.FromSeconds(30);
-                    cluster.CheckinMisfireThreshold = TimeSpan.FromSeconds(20);
+                    cluster.CheckinMisfireThreshold = TimeSpan.FromSeconds(90);
                 });
             });
         });

LiteCharms.Features/Extensions/S3.cs

@@ -0,0 +1,64 @@
+using LiteCharms.Features.S3;
+using LiteCharms.Features.S3.Abstractions;
+using static LiteCharms.Features.S3.Constants;
+
+namespace LiteCharms.Features.Extensions;
+
+public static class S3
+{
+    public static IServiceCollection AddGarageS3(this IServiceCollection services, IConfiguration configuration)
+    {        
+        if (!string.IsNullOrWhiteSpace(configuration.GetSection($"{BookshopS3SettingsSection}:ServiceUrl").Value))
+        {
+            services.AddKeyedSingleton<IAmazonS3, AmazonS3Client>(BookshopBucketName, (provider, client) =>
+                new AmazonS3Client(new BasicAWSCredentials(configuration.GetSection($"{BookshopS3SettingsSection}:AccessKey").Value,
+                configuration.GetSection($"{BookshopS3SettingsSection}:SecretKey").Value),
+                new AmazonS3Config
+                {
+                    ServiceURL = configuration.GetSection($"{BookshopS3SettingsSection}:ServiceUrl").Value,
+                    AuthenticationRegion = configuration.GetSection($"{BookshopS3SettingsSection}:Region").Value,
+                    ForcePathStyle = true,
+                    EndpointDiscoveryEnabled = true,
+                    UseHttp = configuration.GetSection($"{BookshopS3SettingsSection}:ServiceUrl").Value!.Contains("http://")
+                }));
+
+            services.AddKeyedScoped<IS3Service, BookshopS3Service>(BookshopBucketName); 
+        }
+
+        if (!string.IsNullOrWhiteSpace(configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:ServiceUrl").Value))
+        {
+            services.AddKeyedSingleton<IAmazonS3, AmazonS3Client>(BookshopInvoicesBucketName, (provider, client) =>
+                new AmazonS3Client(new BasicAWSCredentials(configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:AccessKey").Value,
+                configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:SecretKey").Value),
+                new AmazonS3Config
+                {
+                    ServiceURL = configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:ServiceUrl").Value,
+                    AuthenticationRegion = configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:Region").Value,
+                    ForcePathStyle = true,
+                    EndpointDiscoveryEnabled = true,
+                    UseHttp = configuration.GetSection($"{BookshopS3SettingsSection}:ServiceUrl").Value!.Contains("http://")
+                }));
+
+            services.AddKeyedScoped<IS3Service, BookshopInvoicesS3Service>(BookshopInvoicesBucketName);
+        }
+
+        if (!string.IsNullOrWhiteSpace(configuration.GetSection($"{BookshopQuotesS3SettingsSection}:ServiceUrl").Value))
+        {
+            services.AddKeyedSingleton<IAmazonS3, AmazonS3Client>(BookshopQuotesBucketName, (provider, client) =>
+                new AmazonS3Client(new BasicAWSCredentials(configuration.GetSection($"{BookshopQuotesS3SettingsSection}:AccessKey").Value,
+                configuration.GetSection($"{BookshopQuotesS3SettingsSection}:SecretKey").Value),
+                new AmazonS3Config
+                {
+                    ServiceURL = configuration.GetSection($"{BookshopQuotesS3SettingsSection}:ServiceUrl").Value,
+                    AuthenticationRegion = configuration.GetSection($"{BookshopQuotesS3SettingsSection}:Region").Value,
+                    ForcePathStyle = true,
+                    EndpointDiscoveryEnabled = true,
+                    UseHttp = configuration.GetSection($"{BookshopS3SettingsSection}:ServiceUrl").Value!.Contains("http://")
+                }));
+
+            services.AddKeyedScoped<IS3Service, BookshopQuotesS3Service>(BookshopQuotesBucketName);
+        }
+
+        return services;
+    }
+}

LiteCharms.Features/LiteCharms.Features.csproj

@@ -128,6 +128,17 @@
     <Using Include="FluentResults" />
     <Using Include="Mediator" />
   </ItemGroup>
+  
+  <!-- Amazon S3 SDK -->
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.Extensions.NetCore.Setup" Version="4.0.3.40" />
+    <PackageReference Include="AWSSDK.S3" Version="4.0.23.3" />
+    
+    <!-- global Usings -->
+    <Using Include="Amazon.S3" />
+    <Using Include="Amazon.S3.Model" />
+    <Using Include="Amazon.Runtime" />
+  </ItemGroup>
 
   <!-- Shared Usings -->
   <ItemGroup>

LiteCharms.Features/Quartz/JobOrchestrator.cs

@@ -50,9 +50,9 @@ public class JobOrchestrator(ISchedulerFactory schedulerFactory) : IJobOrchestra
 
         var trigger = global::Quartz.TriggerBuilder.Create()
             .WithIdentity(triggerKey)
-            .WithDescription($"Scheduled via Main Job at {now:g} UTC")
+            .WithDescription($"Scheduled via Main Job at {now:g}")
             .WithCronSchedule(cronExpression, cron => cron
-                .WithMisfireHandlingInstructionFireAndProceed())
+                .WithMisfireHandlingInstructionIgnoreMisfires())
             .StartAt(now)
             .Build();
 

LiteCharms.Features/Quartz/MediatorJob.cs

@@ -10,16 +10,28 @@ public class MediatorJob<TNotification>(IMediator mediator) : IJob where TNotifi
     {
         var data = context.MergedJobDataMap["Payload"] as string;
 
-        if (string.IsNullOrWhiteSpace(data)) return;
+        if (string.IsNullOrWhiteSpace(data))
+        {
+            Trace.WriteLine("Job Payload missing, job ended");
+
+            return;
+        }
 
         var notification = JsonSerializer.Deserialize<TNotification>(data);
 
-        if (notification is null) return;
+        if (notification is null)
+        {
+            Trace.WriteLine("Notification could not be JSon converted from data string, job ended");
+
+            return;
+        }
                 
         using var activity = MediatorTelemetry.Source.StartActivity($"Quartz: {typeof(TNotification).Name}");
                 
         activity?.SetTag("event.correlation_id", notification.CorrelationId);
 
         await mediator.Publish(notification, context.CancellationToken);
+
+        Trace.WriteLine("Job published");
     }
 }

LiteCharms.Features/S3/Abstractions/IS3Service.cs

@@ -0,0 +1,6 @@
+namespace LiteCharms.Features.S3.Abstractions;
+
+public interface IS3Service
+{
+    Task<Result<string>> UploadFileAsync(string fileName, Stream fileStream, string contentType, CancellationToken cancellationToken = default);
+}

LiteCharms.Features/S3/Abstractions/S3ServiceBase.cs

@@ -0,0 +1,42 @@
+namespace LiteCharms.Features.S3.Abstractions;
+
+public abstract class S3ServiceBase(IAmazonS3 amazonS3)
+{
+    protected readonly IAmazonS3 Client = amazonS3;
+
+    protected abstract string BucketName { get; }
+    protected abstract string CdnBaseUrl { get; }
+
+    public virtual async Task<Result<string>> UploadFileAsync(string fileName, Stream fileStream, string contentType, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            if (string.IsNullOrWhiteSpace(BucketName))
+                return Result.Fail<string>("Bucket name is not configured.");
+
+            if (string.IsNullOrWhiteSpace(CdnBaseUrl))
+                return Result.Fail<string>("CDN base URL is not configured.");
+
+            var fileKey = $"{Guid.NewGuid():N}{Path.GetExtension(fileName)}";
+
+            var putRequest = new PutObjectRequest
+            {
+                BucketName = BucketName,
+                Key = fileKey,
+                InputStream = fileStream,
+                ContentType = contentType,
+                UseChunkEncoding = false
+            };
+
+            var response = await Client.PutObjectAsync(putRequest, cancellationToken);
+
+            return response.HttpStatusCode != System.Net.HttpStatusCode.OK
+                ? Result.Fail<string>($"Failed to upload {fileName} to S3.")
+                : Result.Ok($"{CdnBaseUrl}/{fileKey}");
+        }
+        catch (Exception ex)
+        {
+            return Result.Fail<string>(new Error($"Error uploading {fileName} to S3: {ex.Message}").CausedBy(ex));
+        }
+    }
+}

LiteCharms.Features/S3/BookshopInvoicesS3Service.cs

@@ -0,0 +1,11 @@
+using LiteCharms.Features.S3.Abstractions;
+using static LiteCharms.Features.S3.Constants;
+
+namespace LiteCharms.Features.S3;
+
+public class BookshopInvoicesS3Service(IConfiguration configuration, [FromKeyedServices(BookshopInvoicesBucketName)] IAmazonS3 amazonS3) :
+    S3ServiceBase(amazonS3), IS3Service
+{
+    protected override string BucketName => configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:BucketName").Value ?? "";
+    protected override string CdnBaseUrl => configuration.GetSection($"{BookshopInvoicesS3SettingsSection}:CdnBaseUrl").Value ?? "";
+}

LiteCharms.Features/S3/BookshopQuotesS3Service.cs

@@ -0,0 +1,11 @@
+using LiteCharms.Features.S3.Abstractions;
+using static LiteCharms.Features.S3.Constants;
+
+namespace LiteCharms.Features.S3;
+
+public class BookshopQuotesS3Service(IConfiguration configuration, [FromKeyedServices(BookshopQuotesBucketName)] IAmazonS3 amazonS3) :
+    S3ServiceBase(amazonS3), IS3Service
+{
+    protected override string BucketName => configuration.GetSection($"{BookshopQuotesS3SettingsSection}:BucketName").Value ?? "";
+    protected override string CdnBaseUrl => configuration.GetSection($"{BookshopQuotesS3SettingsSection}:CdnBaseUrl").Value ?? "";
+}

LiteCharms.Features/S3/BookshopS3Service.cs

@@ -0,0 +1,11 @@
+using LiteCharms.Features.S3.Abstractions;
+using static LiteCharms.Features.S3.Constants;
+
+namespace LiteCharms.Features.S3;
+
+public class BookshopS3Service(IConfiguration configuration, [FromKeyedServices(BookshopBucketName)] IAmazonS3 amazonS3) :
+    S3ServiceBase(amazonS3), IS3Service
+{
+    protected override string BucketName => configuration.GetSection($"{BookshopS3SettingsSection}:BucketName").Value ?? "";
+    protected override string CdnBaseUrl => configuration.GetSection($"{BookshopS3SettingsSection}:CdnBaseUrl").Value ?? "";
+}

LiteCharms.Features/S3/Configuration/S3Settings.cs

@@ -0,0 +1,16 @@
+namespace LiteCharms.Features.S3.Configuration;
+
+public class S3Settings
+{
+    public string? ServiceUrl { get; set; }
+
+    public string? AccessKey { get; set; }
+
+    public string? SecretKey { get; set; }
+
+    public string? BucketName { get; set; }
+
+    public string? Region { get; set; }
+
+    public string? CdnBaseUrl { get; set; }
+}

LiteCharms.Features/S3/Constants.cs

@@ -0,0 +1,12 @@
+namespace LiteCharms.Features.S3;
+
+public static class Constants
+{
+    public const string BookshopS3SettingsSection = "BookshopS3Settings";
+    public const string BookshopInvoicesS3SettingsSection = "BookshopInvoicesS3Settings";
+    public const string BookshopQuotesS3SettingsSection = "BookshopQuotesS3Settings";
+
+    public const string BookshopBucketName = "bookshop";
+    public const string BookshopInvoicesBucketName = "bookshop.invoices";
+    public const string BookshopQuotesBucketName = "bookshop.quotes";
+}

LiteCharms.Features/Shop/Notifications/Events/Handlers/ProcessEmailNotificationsEventHandler.cs

@@ -1,4 +1,5 @@
-using LiteCharms.Features.Email;
+using k8s.KubeConfigModels;
+using LiteCharms.Features.Email;
 using LiteCharms.Features.Shop.Notifications.Models;
 using LiteCharms.Features.Shop.Postgres;
 
@@ -15,17 +16,20 @@ public class ProcessEmailNotificationsEventHandler(IDbContextFactory<ShopDbConte
         {
             using var context = await contextFactory.CreateDbContextAsync(cancellationToken);
 
+            if (emailService.Status != EmailStatuses.Connected)
+                await emailService.ConnectAsync(cancellationToken);
+
             var notifications = await context.Notifications
                 .OrderByDescending(o => o.CreatedAt)
                 .ThenBy(o => o.Priority)
-                .Where(n => n.CorrelationIdType == CorrelationIdTypes.Email)
-                .Where(n => n.Direction == NotificationDirection.Outgoing)
+                .Where(n => n.Platform == NotificationPlatforms.Email && 
+                    n.Direction == NotificationDirection.Outgoing && n.Processed == false)
                 .Take(message.MaxRecords)
                 .ToListAsync(cancellationToken);
 
             foreach (var notification in notifications)
             {
-                if (dropBatch || cancellationToken.IsCancellationRequested) break;
+                if (dropBatch) break;
 
                 var sendResult = await SendEmailAsync(notification,emailService, cancellationToken);
 
@@ -46,12 +50,16 @@ public class ProcessEmailNotificationsEventHandler(IDbContextFactory<ShopDbConte
                 notification.UpdatedAt = DateTime.UtcNow;
             }
 
-            await context.SaveChangesAsync(cancellationToken);
+            await context.SaveChangesAsync(cancellationToken);            
         }
         catch (Exception ex)
         {
             logger.LogError(ex, ex.Message);
         }
+        finally
+        {
+            await emailService.DisconnectAsync(cancellationToken);
+        }
     }
 
     private async Task<Result> SendEmailAsync(Notification notification, EmailService service, CancellationToken cancellationToken = default)

LiteCharms.Features/Shop/Notifications/Events/ProcessEmailNotificationsEvent.cs

@@ -8,6 +8,8 @@ public class ProcessEmailNotificationsEvent : EventBase, IEvent
 
     public int MaxRecords { get; set; }
 
+    public ProcessEmailNotificationsEvent() { MaxRecords = 1000; }
+
     private ProcessEmailNotificationsEvent(int maxRecords = 1000) => MaxRecords = maxRecords;
 
     public static ProcessEmailNotificationsEvent Create(int maxRecords = 1000) => new(maxRecords);

LiteCharms.Features/Shop/Notifications/NotificationService.cs

@@ -63,8 +63,8 @@ public class NotificationService(IDbContextFactory<ShopDbContext> contextFactory
     {
         try
         {
-            var fromDate = range.From.ToDateTime(TimeOnly.MinValue);
-            var toDate = range.To.ToDateTime(TimeOnly.MaxValue);
+            var fromDate = range.From.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc);
+            var toDate = range.To.ToDateTime(TimeOnly.MaxValue, DateTimeKind.Utc);
 
             using var context = await contextFactory.CreateDbContextAsync(cancellationToken);
 
@@ -96,12 +96,9 @@ public class NotificationService(IDbContextFactory<ShopDbContext> contextFactory
                 return Result.Fail(new Error($"Notification with id {request.NotificationId} not found."));
 
             notification.Processed = request.Processed;
-
-            if (request.HasError)
-            {
-                notification.HasError = request.HasError;
-                notification.Errors = request.Errors;
-            }
+            notification.UpdatedAt = DateTime.UtcNow;
+            notification.HasError = request.HasError;
+            notification.Errors = request.Errors;
 
             return await context.SaveChangesAsync(cancellationToken) > 0
                 ? Result.Ok()