Add support for inventory timestamps

Reviewed-on: #1

PartSource.Api/Controllers/InventoryController.cs

@@ -23,7 +23,7 @@ namespace PartSource.Api.Controllers
         [Route("sku/{sku}/storeNumber/{storeNumber}")]
         public async Task<ActionResult> GetInventory(int sku, int storeNumber)
         {
-            PartsAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
+            PartAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
 
             if (inventory == null)
             {
@@ -36,7 +36,8 @@ namespace PartSource.Api.Controllers
                 {
                     StoreNumber = inventory.Store,
                     Sku = sku,
-                    Quantity = inventory.QTY
+                    Quantity = inventory.QTY,
+                    Updated = inventory.Updated ?? System.DateTime.MinValue
                 }
             });
         }

PartSource.Automation/Jobs/BulkUpdateInventory.cs

@@ -14,19 +14,22 @@ using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 
-namespace PartSource.Automation.Jobs.POC
+namespace PartSource.Automation.Jobs
 {
     public class BulkUpdateInventory : IAutomationJob
     {
         private readonly FtpService _ftpService;
         private readonly ILogger<BulkUpdateInventory> _logger;
+		private readonly string _connectionString;
 
         public BulkUpdateInventory(IConfiguration configuration, ILogger<BulkUpdateInventory> logger)
         {
             FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
             _ftpService = new FtpService(ftpConfiguration);
 
-			_logger = logger;
+            _connectionString = configuration.GetConnectionString("PartSourceDatabase");
+
+            _logger = logger;
         }
 
         public async Task Run(CancellationToken token, params string[] arguments)
@@ -46,11 +49,11 @@ namespace PartSource.Automation.Jobs.POC
 
 			DataTable dataTable = GetDataTable(file);
 
-			using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
+			using SqlConnection connection = new SqlConnection(_connectionString);
 			connection.Open();
 
 			using SqlCommand command = new SqlCommand("TRUNCATE TABLE PartAvailability", connection);
-			await command.ExecuteNonQueryAsync();
+			await command.ExecuteNonQueryAsync(token);
 
 			using SqlBulkCopy bulk = new SqlBulkCopy(connection)
 			{
@@ -58,7 +61,7 @@ namespace PartSource.Automation.Jobs.POC
 				BulkCopyTimeout = 14400
 			};
 
-			bulk.WriteToServer(dataTable);
+			await bulk.WriteToServerAsync(dataTable, token);
 
             _ftpService.Delete(lastUploadedFile.Filename);
 
@@ -71,6 +74,7 @@ namespace PartSource.Automation.Jobs.POC
 			dataTable.Columns.Add("Store", typeof(int));
 			dataTable.Columns.Add("SKU", typeof(string));
 			dataTable.Columns.Add("QTY", typeof(int));
+			dataTable.Columns.Add("Updated", typeof(DateTime));
 
 			using StreamReader reader = new StreamReader(filename);
 			string line = reader.ReadLine(); // Burn the header row
@@ -88,9 +92,10 @@ namespace PartSource.Automation.Jobs.POC
 				string sku = columns[1].Trim();
 				if (int.TryParse(columns[0], out int store)
 					&& !string.IsNullOrEmpty(sku)
-					&& int.TryParse(columns[2], out int quantity))
+					&& int.TryParse(columns[2], out int quantity)
+					&& DateTime.TryParse(columns[3], out DateTime updated))
 				{
-					dataTable.Rows.Add(new object[] { store, sku, quantity });
+					dataTable.Rows.Add(new object[] { store, sku, quantity, updated });
 				}
 			}
 

PartSource.Automation/Jobs/ExecuteSsisPackages.cs

@@ -1,9 +1,11 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using PartSource.Automation.Models.Configuration;
+using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -17,11 +19,11 @@ namespace PartSource.Automation.Jobs
 		private readonly ILogger<ExecuteSsisPackages> _logger;
 
 		// TODO: set from config
-		private readonly string[] _ssisPackages = {"Parts Availability" };
+		private readonly string[] _ssisPackages = {"Parts Price" };
 
 		public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
 		{
-			FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
+			FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
 
 			_emailService = emailService;
 			_ftpService = new FtpService(ftpConfiguration);
@@ -36,7 +38,18 @@ namespace PartSource.Automation.Jobs
 				{
 					try
 					{
-					//	_ftpService.Download($"{package}.txt");
+                        FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
+						   .Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
+						   .OrderByDescending(f => f.Modified)
+						   .FirstOrDefault();
+
+                        if (lastUploadedFile == null)
+                        {
+                            _logger.LogInformation($"No {package} file available.");
+                            return;
+                        }
+
+                        _ftpService.Download($"{package}.txt");
 						_ssisService.Execute($"{package}.dtsx");
 
 						_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
@@ -45,7 +58,6 @@ namespace PartSource.Automation.Jobs
 					catch (Exception ex)
 					{
 						_logger.LogError($"Execution of SSIS package {package} failed.", ex);
-
 						throw;
 					}
 				}

PartSource.Automation/Jobs/POC/ImageList.cs

@@ -18,13 +18,13 @@ using Ratermania.Shopify.Resources;
 
 namespace PartSource.Automation.Jobs.POC
 {
-    public class GetImageUrls : IAutomationJob
+    public class ImageList : IAutomationJob
     {
         private readonly NexpartService _nexpartService;
         private readonly PartSourceContext _partSourceContext;
         private readonly FitmentContext _fitmentContext;
 
-        public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
+        public ImageList(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
         {
             _nexpartService = nexpartService;
             _partSourceContext = partSourceContext;

PartSource.Automation/Jobs/PartialInventoryUpdate.cs

@@ -14,21 +14,25 @@ using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 
-namespace PartSource.Automation.Jobs.POC
+namespace PartSource.Automation.Jobs
 {
     public class PartialInventoryUpdate : IAutomationJob
     {
         private readonly FtpService _ftpService;
         private readonly ILogger<PartialInventoryUpdate> _logger;
+        private readonly string _connectionString;
 
         public PartialInventoryUpdate(IConfiguration configuration, ILogger<PartialInventoryUpdate> logger)
         {
             FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
             _ftpService = new FtpService(ftpConfiguration);
 
+            _connectionString = _connectionString = configuration.GetConnectionString("PartSourceDatabase");
+
             _logger = logger;
         }
 
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities")]
         public async Task Run(CancellationToken token, params string[] arguments)
         {
             FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
@@ -46,38 +50,68 @@ namespace PartSource.Automation.Jobs.POC
 
             string file = _ftpService.Download($"{lastUploadedFile.Filename}");
 
-            using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
-			connection.Open();
+            using SqlConnection connection = new SqlConnection(_connectionString);
+            connection.Open();
 
             using StreamReader reader = new StreamReader(file);
             string line = reader.ReadLine(); // Burn the header row
 
+            IDictionary<string, object> parameters = new Dictionary<string, object>();
+            string command = string.Empty;
+            int i = 0;
+
             while (reader.Peek() > 0)
             {
                 line = reader.ReadLine();
 
                 string[] columns = line.Split("|");
-                for (int i = 0; i < columns.Length; i++)
+                for (int j = 0; j < columns.Length; j++)
                 {
-                    columns[i] = columns[i].Replace("\"", string.Empty);
+                    columns[j] = columns[j].Replace("\"", string.Empty);
                 }
 
                 if (int.TryParse(columns[0], out int store)
                     && int.TryParse(columns[1], out int sku)
-                    && int.TryParse(columns[2], out int quantity))
+                    && int.TryParse(columns[2], out int quantity)
+                    && DateTime.TryParse(columns[3], out DateTime updated))
                 {
-                    using SqlCommand sqlCommand = new SqlCommand("UPDATE PartAvailability SET QTY = @qty WHERE SKU = @sku AND Store = @store", connection);
-                    sqlCommand.Parameters.Add(new SqlParameter("qty", quantity));
-                    sqlCommand.Parameters.Add(new SqlParameter("sku", sku));
-                    sqlCommand.Parameters.Add(new SqlParameter("store", store));
+                    command += $"UPDATE PartAvailability SET QTY = @qty_{i}, Updated = @updated_{i} WHERE SKU = @sku_{i} AND Store = @store_{i};";
 
-                    await sqlCommand.ExecuteNonQueryAsync();
+                    parameters.Add($"qty_{i}", quantity);
+                    parameters.Add($"store_{i}", store);
+                    parameters.Add($"sku_{i}", sku);
+                    parameters.Add($"updated_{i}", updated);
+
+                    i++;
+                }
+
+                if (i == 250)
+                {
+                    using SqlCommand nested = new SqlCommand(command, connection);
+                    foreach (KeyValuePair<string, object> parameter in parameters)
+                    {
+                        nested.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
+                    }
+
+                    await nested.ExecuteNonQueryAsync(token);
+
+                    parameters.Clear();
+                    command = string.Empty;
+                    i = 0;
                 }
             }
 
+            using SqlCommand sqlCommand = new SqlCommand(command, connection);
+            foreach (KeyValuePair<string, object> parameter in parameters)
+            {
+                sqlCommand.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
+            }
+
+            await sqlCommand.ExecuteNonQueryAsync(token);
+
             _ftpService.Delete(lastUploadedFile.Filename);
 
             return;
         }
-	}
+    }
 }

PartSource.Automation/Jobs/UpdateFitment.cs

@@ -93,7 +93,7 @@ namespace PartSource.Automation.Jobs
                                 ? string.Empty
                                 : product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
 
-                            IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
+                            IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
                             IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
 
                             if (!vehicleIdFitment.Any())

PartSource.Automation/Program.cs

@@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using PartSource.Automation.Jobs;
-using PartSource.Automation.Jobs.POC;
 using PartSource.Automation.Services;
 using PartSource.Data.AutoMapper;
 using PartSource.Data.Contexts;
@@ -75,28 +74,37 @@ namespace PartSource.Automation
 
                     .AddAutomation(options =>
                     {
-                    options.HasBaseInterval(new TimeSpan(0, 5, 0))
-                        .HasMaxFailures(1)
-                        //.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
-                        //
-                        //.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
-                        //	.HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
-                        //.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
-                        //.HasDependency<SyncronizeProducts>()
-                        //.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
-                        //.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
-                        //		.HasDependency<UpdateFitment>()
-                        //	.HasDependency<ProcessWhiFitment>()
-                        //	.HasDependency<SyncronizeProducts>()
-                        //	.StartsAt(DateTime.Today.AddHours(8))
-                        //) ;
-                        //.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
-                        //	.StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
-                        //)
-                        .HasJob<BulkUpdateInventory>(options =>
-                            options.HasInterval(new TimeSpan(1, 0, 0))
-                                .StartsAt(DateTime.Today.AddHours(-27))
-                            )
+                        options.HasBaseInterval(new TimeSpan(0, 5, 0))
+                            .HasMaxFailures(5)
+                            //.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
+                            //
+                            //.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
+                            //	.HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
+                            //.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
+                            //.HasDependency<SyncronizeProducts>()
+                            //.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
+                            //.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
+                            //		.HasDependency<UpdateFitment>()
+                            //	.HasDependency<ProcessWhiFitment>()
+                            //	.HasDependency<SyncronizeProducts>()
+                            //	.StartsAt(DateTime.Today.AddHours(8))
+                            //) ;
+                            //.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
+                            //	.StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
+                            //)
+                            .HasJob<ExecuteSsisPackages>(options =>
+                                options.HasInterval(new TimeSpan(24, 0, 0))
+                                    .StartsAt(DateTime.Today.AddHours(-24))
+                                )
+                            .HasJob<UpdatePricing>(options =>
+                                options.HasInterval(new TimeSpan(24, 0, 0))
+                                    .StartsAt(DateTime.Today.AddHours(-22))
+                                    .HasDependency<ExecuteSsisPackages>()
+                                )
+                            .HasJob<BulkUpdateInventory>(options =>
+                                options.HasInterval(new TimeSpan(1, 0, 0))
+                                    .StartsAt(DateTime.Today.AddHours(-27))
+                                )
                             .HasJob<PartialInventoryUpdate>(options =>
                                 options.HasInterval(new TimeSpan(1, 0, 0))
                                     .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))

PartSource.Automation/appsettings.json

@@ -11,19 +11,18 @@
 		"SmtpHost": "localhost"
 	},
 	"FtpServers": {
-		"AzureConfiguration": {
-			"Username": "ps-ftp\\$ps-ftp",
-			"Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579",
-			"Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot",
-			"Destination": "C:\\Partsource.Automation\\Downloads",
-			"Port":  21
-		},
 		"WhiConfiguration": {
 			"Username": "ctc_seo",
 			"Password": "YD3gtaQ5kPdtNKs",
 			"Url": "ftp://ftp.whisolutions.com",
 			"Destination": "C:\\Partsource.Automation\\Downloads\\WHI",
-			"Port":  3001
+			"Port": 3001
+		},
+		"AutomationConfiguration": {
+			"Username": "stageuser",
+			"Password": "FXepK^cFYS|[H<",
+			"Url": "ftp://localhost",
+			"Destination": "C:\\Partsource.Automation\\Downloads"
 		}
 	},
 	"ssisConfiguration": {

PartSource.Data/Contexts/PartSourceContext.cs

@@ -32,7 +32,7 @@ namespace PartSource.Data.Contexts
 
         public DbSet<Vehicle> Vehicles { get; set; }
         
-        public DbSet<PartsAvailability> PartAvailabilities { get; set; }
+        public DbSet<PartAvailability> PartAvailabilities { get; set; }
 
         public DbSet<BaseVehicle> BaseVehicles { get; set; }
 
@@ -48,7 +48,7 @@ namespace PartSource.Data.Contexts
         {
             base.OnModelCreating(modelBuilder);
 
-            modelBuilder.Entity<PartsAvailability>().HasKey(p => new { p.Store, p.SKU });
+            modelBuilder.Entity<PartAvailability>().HasKey(p => new { p.Store, p.SKU });
             modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
 
             modelBuilder.Entity<ShopifyChangelog>()

PartSource.Data/Models/PartAvailability.cs

@@ -1,9 +1,10 @@
-using System.ComponentModel.DataAnnotations;
+using System;
+using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
 
 namespace PartSource.Data.Models
 {
-    public class PartsAvailability
+    public class PartAvailability
     {
         [Column(Order = 0)]
         [DatabaseGenerated(DatabaseGeneratedOption.None)]
@@ -13,14 +14,8 @@ namespace PartSource.Data.Models
         [DatabaseGenerated(DatabaseGeneratedOption.None)]
         public int SKU { get; set; }
 
-        [Column("Line Code")]
-        [StringLength(50)]
-        public string Line_Code { get; set; }
-
-        [Column("Part Number")]
-        [StringLength(50)]
-        public string Part_Number { get; set; }
-
         public int? QTY { get; set; }
+
+        public DateTime? Updated { get; set; }
     }
 }

PartSource.Services/PartService.cs

@@ -21,7 +21,7 @@ namespace PartSource.Services
             _fitmentContext = fitmentContext;
         }
 
-        public async Task<PartsAvailability> GetInventory(int sku, int storeNumber)
+        public async Task<PartAvailability> GetInventory(int sku, int storeNumber)
         {
             return await _partSourceContext.PartAvailabilities.FirstOrDefaultAsync(s => s.Store == storeNumber && s.SKU == sku);
         }