Pricing updates

Reviewed-on: #2

PartSource.Api/Controllers/InventoryController.cs

@@ -1,8 +1,7 @@
-using Microsoft.AspNetCore.Mvc;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
 using PartSource.Data.Models;
 using PartSource.Services;
-using System.Net;
-using System.Threading.Tasks;
 
 namespace PartSource.Api.Controllers
 {
@@ -23,7 +22,7 @@ namespace PartSource.Api.Controllers
         [Route("sku/{sku}/storeNumber/{storeNumber}")]
         public async Task<ActionResult> GetInventory(int sku, int storeNumber)
         {
-            PartsAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
+            PartAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
 
             if (inventory == null)
             {
@@ -36,7 +35,8 @@ namespace PartSource.Api.Controllers
                 {
                     StoreNumber = inventory.Store,
                     Sku = sku,
-                    Quantity = inventory.QTY
+                    Quantity = inventory.QTY,
+                    Updated = inventory.Updated
                 }
             });
         }

PartSource.Api/appsettings.json

@@ -1,7 +1,6 @@
 {
     "ConnectionStrings": {
         "PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;",
-        "FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=automation;Password=)6L)XP%m(x-UU#M;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
         //"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true"
         "FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
     },

PartSource.Automation/Jobs/BulkUpdateInventory.cs

@@ -14,18 +14,21 @@ using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 
-namespace PartSource.Automation.Jobs.POC
+namespace PartSource.Automation.Jobs
 {
     public class BulkUpdateInventory : IAutomationJob
     {
         private readonly FtpService _ftpService;
         private readonly ILogger<BulkUpdateInventory> _logger;
+		private readonly string _connectionString;
 
         public BulkUpdateInventory(IConfiguration configuration, ILogger<BulkUpdateInventory> logger)
         {
             FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
             _ftpService = new FtpService(ftpConfiguration);
 
+            _connectionString = configuration.GetConnectionString("PartSourceDatabase");
+
             _logger = logger;
         }
 
@@ -46,11 +49,11 @@ namespace PartSource.Automation.Jobs.POC
 
 			DataTable dataTable = GetDataTable(file);
 
-			using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
+			using SqlConnection connection = new SqlConnection(_connectionString);
 			connection.Open();
 
 			using SqlCommand command = new SqlCommand("TRUNCATE TABLE PartAvailability", connection);
-			await command.ExecuteNonQueryAsync();
+			await command.ExecuteNonQueryAsync(token);
 
 			using SqlBulkCopy bulk = new SqlBulkCopy(connection)
 			{
@@ -58,7 +61,7 @@ namespace PartSource.Automation.Jobs.POC
 				BulkCopyTimeout = 14400
 			};
 
-			bulk.WriteToServer(dataTable);
+			await bulk.WriteToServerAsync(dataTable, token);
 
             _ftpService.Delete(lastUploadedFile.Filename);
 
@@ -71,6 +74,7 @@ namespace PartSource.Automation.Jobs.POC
 			dataTable.Columns.Add("Store", typeof(int));
 			dataTable.Columns.Add("SKU", typeof(string));
 			dataTable.Columns.Add("QTY", typeof(int));
+			dataTable.Columns.Add("Updated", typeof(string));
 
 			using StreamReader reader = new StreamReader(filename);
 			string line = reader.ReadLine(); // Burn the header row
@@ -86,11 +90,14 @@ namespace PartSource.Automation.Jobs.POC
 				}
 
 				string sku = columns[1].Trim();
+				string updated = columns[3].Trim();
+
 				if (int.TryParse(columns[0], out int store)
 					&& !string.IsNullOrEmpty(sku)
-					&& int.TryParse(columns[2], out int quantity))
+					&& int.TryParse(columns[2], out int quantity)
+					&& !string.IsNullOrEmpty(updated))
 				{
-					dataTable.Rows.Add(new object[] { store, sku, quantity });
+					dataTable.Rows.Add(new object[] { store, sku, quantity, updated });
 				}
 			}
 

PartSource.Automation/Jobs/ExecuteSsisPackages.cs

@@ -1,9 +1,11 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using PartSource.Automation.Models.Configuration;
+using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -17,11 +19,11 @@ namespace PartSource.Automation.Jobs
 		private readonly ILogger<ExecuteSsisPackages> _logger;
 
 		// TODO: set from config
-		private readonly string[] _ssisPackages = {"Parts Availability" };
+		private readonly string[] _ssisPackages = {"Parts Price" };
 
 		public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
 		{
-			FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
+			FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
 
 			_emailService = emailService;
 			_ftpService = new FtpService(ftpConfiguration);
@@ -36,7 +38,18 @@ namespace PartSource.Automation.Jobs
 				{
 					try
 					{
-					//	_ftpService.Download($"{package}.txt");
+                        FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
+						   .Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
+						   .OrderByDescending(f => f.Modified)
+						   .FirstOrDefault();
+
+                        if (lastUploadedFile == null)
+                        {
+                            _logger.LogInformation($"No {package} file available.");
+                            return;
+                        }
+
+                        _ftpService.Download($"{package}.txt");
 						_ssisService.Execute($"{package}.dtsx");
 
 						_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
@@ -45,7 +58,6 @@ namespace PartSource.Automation.Jobs
 					catch (Exception ex)
 					{
 						_logger.LogError($"Execution of SSIS package {package} failed.", ex);
-
 						throw;
 					}
 				}

PartSource.Automation/Jobs/POC/ImageList.cs

@@ -18,13 +18,13 @@ using Ratermania.Shopify.Resources;
 
 namespace PartSource.Automation.Jobs.POC
 {
-    public class GetImageUrls : IAutomationJob
+    public class ImageList : IAutomationJob
     {
         private readonly NexpartService _nexpartService;
         private readonly PartSourceContext _partSourceContext;
         private readonly FitmentContext _fitmentContext;
 
-        public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
+        public ImageList(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
         {
             _nexpartService = nexpartService;
             _partSourceContext = partSourceContext;

PartSource.Automation/Jobs/PartialInventoryUpdate.cs

@@ -14,21 +14,25 @@ using PartSource.Automation.Models.Ftp;
 using PartSource.Automation.Services;
 using Ratermania.Automation.Interfaces;
 
-namespace PartSource.Automation.Jobs.POC
+namespace PartSource.Automation.Jobs
 {
     public class PartialInventoryUpdate : IAutomationJob
     {
         private readonly FtpService _ftpService;
         private readonly ILogger<PartialInventoryUpdate> _logger;
+        private readonly string _connectionString;
 
         public PartialInventoryUpdate(IConfiguration configuration, ILogger<PartialInventoryUpdate> logger)
         {
             FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
             _ftpService = new FtpService(ftpConfiguration);
 
+            _connectionString = _connectionString = configuration.GetConnectionString("PartSourceDatabase");
+
             _logger = logger;
         }
 
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities")]
         public async Task Run(CancellationToken token, params string[] arguments)
         {
             FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
@@ -46,35 +50,67 @@ namespace PartSource.Automation.Jobs.POC
 
             string file = _ftpService.Download($"{lastUploadedFile.Filename}");
 
-            using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
+            using SqlConnection connection = new SqlConnection(_connectionString);
             connection.Open();
 
             using StreamReader reader = new StreamReader(file);
             string line = reader.ReadLine(); // Burn the header row
 
+            IDictionary<string, object> parameters = new Dictionary<string, object>();
+            string command = string.Empty;
+            int i = 0;
+
             while (reader.Peek() > 0)
             {
                 line = reader.ReadLine();
 
                 string[] columns = line.Split("|");
-                for (int i = 0; i < columns.Length; i++)
+                for (int j = 0; j < columns.Length; j++)
                 {
-                    columns[i] = columns[i].Replace("\"", string.Empty);
+                    columns[j] = columns[j].Replace("\"", string.Empty);
                 }
+                string sku = columns[1].Trim();
+                string updated = columns[3].Trim();
 
                 if (int.TryParse(columns[0], out int store)
-                    && int.TryParse(columns[1], out int sku)
+                    && !string.IsNullOrEmpty(sku)
-                    && int.TryParse(columns[2], out int quantity))
+                    && int.TryParse(columns[2], out int quantity)
+                    && !string.IsNullOrEmpty(updated))
                 {
-                    using SqlCommand sqlCommand = new SqlCommand("UPDATE PartAvailability SET QTY = @qty WHERE SKU = @sku AND Store = @store", connection);
+                    command += $"UPDATE PartAvailability SET QTY = @qty_{i}, Updated = @updated_{i} WHERE SKU = @sku_{i} AND Store = @store_{i};";
-                    sqlCommand.Parameters.Add(new SqlParameter("qty", quantity));
-                    sqlCommand.Parameters.Add(new SqlParameter("sku", sku));
-                    sqlCommand.Parameters.Add(new SqlParameter("store", store));
 
-                    await sqlCommand.ExecuteNonQueryAsync();
+                    parameters.Add($"qty_{i}", quantity);
+                    parameters.Add($"store_{i}", store);
+                    parameters.Add($"sku_{i}", sku);
+                    parameters.Add($"updated_{i}", updated);
+
+                    i++;
+                }
+
+                if (i == 250)
+                {
+                    using SqlCommand nested = new SqlCommand(command, connection);
+                    foreach (KeyValuePair<string, object> parameter in parameters)
+                    {
+                        nested.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
+                    }
+
+                    await nested.ExecuteNonQueryAsync(token);
+
+                    parameters.Clear();
+                    command = string.Empty;
+                    i = 0;
                 }
             }
 
+            using SqlCommand sqlCommand = new SqlCommand(command, connection);
+            foreach (KeyValuePair<string, object> parameter in parameters)
+            {
+                sqlCommand.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
+            }
+
+            await sqlCommand.ExecuteNonQueryAsync(token);
+
             _ftpService.Delete(lastUploadedFile.Filename);
 
             return;

PartSource.Automation/Jobs/TestJob.cs

@@ -24,7 +24,7 @@ namespace PartSource.Automation.Jobs
 #pragma warning disable CS1998, CA1303
 		public async Task Run(CancellationToken token, params string[] arguments)
 		{
-		//	_emailService.Send("Automation Test Message", "This is a test email from the automation server. If this message was in your spam folder, whitelist the address that sent this email.");
+			_emailService.Send("Automation Test Message", "This is a test email from the automation server. If this message was in your spam folder, whitelist the address that sent this email.");
 
 			_logger.LogInformation("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc scelerisque congue euismod. Curabitur enim eros, sollicitudin ac purus eget, dignissim mattis augue. In quam sapien, tincidunt et elementum vitae, interdum vitae sem.");
 			_logger.LogWarning("Praesent feugiat sapien non suscipit faucibus. Mauris fermentum ut augue a feugiat. Integer felis sem, laoreet et augue at, finibus maximus ex. Fusce sit amet erat non tortor porta condimentum condimentum quis ipsum.");

PartSource.Automation/Jobs/UpdateFitment.cs

@@ -93,7 +93,7 @@ namespace PartSource.Automation.Jobs
                                 ? string.Empty
                                 : product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
 
-                            IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
+                            IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
                             IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
 
                             if (!vehicleIdFitment.Any())

PartSource.Automation/Jobs/UpdatePricing.cs

@@ -76,7 +76,8 @@ namespace PartSource.Automation.Jobs
 
                             try
                             {
-                                _shopifyClient.BulkActions.Add(new Metafield
+
+                                await _shopifyClient.Metafields.Add(new Metafield
                                 {
                                     Namespace = "Pricing",
                                     Key = "CorePrice",
@@ -85,24 +86,16 @@ namespace PartSource.Automation.Jobs
                                     OwnerResource = "product",
                                     OwnerId = product.Id
                                 });
+
+                                await _shopifyClient.Products.Update(product);
+
+                                _logger.LogInformation("Updated product id {productId}", product.Id);
                             }
 
                             catch (Exception ex)
                             {
-                                _logger.LogWarning(ex, $"Failed to update pricing for product ID {product.Id}");
+                                _logger.LogWarning(ex, "Failed to update pricing for product ID {productId}", product.Id);
                             }
-
-
-                        }
-
-                        try
-                        {
-                            _shopifyClient.BulkActions.Update(product);
-                        }
-
-                        catch (Exception ex)
-                        {
-                            _logger.LogWarning(ex, $"Failed to update pricing for product ID {product.Id}");
                         }
                     }
                 }
@@ -119,14 +112,9 @@ namespace PartSource.Automation.Jobs
                     _logger.LogWarning(ex, "Failed to get the next set of products.  Retrying");
                     products = await _shopifyClient.Products.GetPrevious();
                 }
-            }
 
-            while (_shopifyClient.BulkActions.PendingCount() > 0)
+                _emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
-            {
-                await Task.Delay(15 * 1000, token);
-                _logger.LogInformation(_shopifyClient.BulkActions.PendingCount().ToString());
             }
-            //    _emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
         }
     }
 }

PartSource.Automation/Program.cs

@@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using PartSource.Automation.Jobs;
-using PartSource.Automation.Jobs.POC;
 using PartSource.Automation.Services;
 using PartSource.Data.AutoMapper;
 using PartSource.Data.Contexts;
@@ -64,7 +63,7 @@ namespace PartSource.Automation
                     {
                         options.ApiKey = builder.Configuration["Shopify:ApiKey"];
                         options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
-                        options.ApiVersion = "2022-10";
+                        options.ApiVersion = "2024-10";
                         options.ShopDomain = builder.Configuration["Shopify:ShopDomain"];
 
                         //options.ApiKey = "9a533dad460321c6ce8f30bf5b8691ed";
@@ -76,7 +75,7 @@ namespace PartSource.Automation
                     .AddAutomation(options =>
                     {
                         options.HasBaseInterval(new TimeSpan(0, 5, 0))
-                        .HasMaxFailures(1)
+                            .HasMaxFailures(5)
                             //.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
                             //
                             //.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
@@ -93,14 +92,23 @@ namespace PartSource.Automation
                             //.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
                             //	.StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
                             //)
-                        .HasJob<BulkUpdateInventory>(options =>
+                            //.HasJob<ExecuteSsisPackages>(options =>
-                            options.HasInterval(new TimeSpan(1, 0, 0))
+                            //    options.HasInterval(new TimeSpan(24, 0, 0))
-                                .StartsAt(DateTime.Today.AddHours(-27))
+                            //        .StartsAt(DateTime.Today.AddHours(-24))
-                            )
+                            //    )
-                            .HasJob<PartialInventoryUpdate>(options =>
+                            .HasJob<UpdatePricing>(options =>
-                                options.HasInterval(new TimeSpan(1, 0, 0))
+                                options.HasInterval(new TimeSpan(24, 0, 0))
-                                    .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
+                                    .StartsAt(DateTime.Today.AddHours(-22))
+                                    //.HasDependency<ExecuteSsisPackages>()
                                 );
+                            //.HasJob<BulkUpdateInventory>(options =>
+                            //    options.HasInterval(new TimeSpan(1, 0, 0))
+                            //        .StartsAt(DateTime.Today.AddHours(-27))
+                            //    );
+                            //.HasJob<TestJob>(options =>
+                            //    options.HasInterval(new TimeSpan(1, 0, 0))
+                            //        .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
+                            //);
 
                         //.HasJob<PartialInventoryUpdate>(options => options.HasInterval(new TimeSpan(1, 0, 0))
                         //.HasDependency<ExecuteSsisPackages>()

PartSource.Automation/appsettings.json

@@ -6,24 +6,23 @@
 	},
 	"emailConfiguration": {
 		"From": "alerts@ps-automation.eastus2.cloudapp.azure.com",
-		"To": "tom@soundpress.com,Anas.Bajwa@Partsource.ca,josh@soundpress.com,alex.au@partsource.ca,michael.massara@partsource.ca",
+		"To": "tom@soundpress.com,josh@soundpress.com,alex.au@partsource.ca,michael.massara@partsource.ca",
-		//"To": "tom@tomraterman.com",
+		//"To": "tom@tomraterman.com,josh@soundpress.com",
 		"SmtpHost": "localhost"
 	},
 	"FtpServers": {
-		"AzureConfiguration": {
-			"Username": "ps-ftp\\$ps-ftp",
-			"Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579",
-			"Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot",
-			"Destination": "C:\\Partsource.Automation\\Downloads",
-			"Port":  21
-		},
 		"WhiConfiguration": {
 			"Username": "ctc_seo",
 			"Password": "YD3gtaQ5kPdtNKs",
 			"Url": "ftp://ftp.whisolutions.com",
 			"Destination": "C:\\Partsource.Automation\\Downloads\\WHI",
 			"Port": 3001
+		},
+		"AutomationConfiguration": {
+			"Username": "stageuser",
+			"Password": "FXepK^cFYS|[H<",
+			"Url": "ftp://localhost",
+			"Destination": "C:\\Partsource.Automation\\Downloads"
 		}
 	},
 	"ssisConfiguration": {

PartSource.Data/Contexts/PartSourceContext.cs

@@ -32,7 +32,7 @@ namespace PartSource.Data.Contexts
 
         public DbSet<Vehicle> Vehicles { get; set; }
         
-        public DbSet<PartsAvailability> PartAvailabilities { get; set; }
+        public DbSet<PartAvailability> PartAvailabilities { get; set; }
 
         public DbSet<BaseVehicle> BaseVehicles { get; set; }
 
@@ -48,7 +48,7 @@ namespace PartSource.Data.Contexts
         {
             base.OnModelCreating(modelBuilder);
 
-            modelBuilder.Entity<PartsAvailability>().HasKey(p => new { p.Store, p.SKU });
+            modelBuilder.Entity<PartAvailability>().HasKey(p => new { p.Store, p.SKU });
             modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
 
             modelBuilder.Entity<ShopifyChangelog>()

PartSource.Data/Models/PartAvailability.cs

@@ -1,9 +1,10 @@
-using System.ComponentModel.DataAnnotations;
+using System;
+using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
 
 namespace PartSource.Data.Models
 {
-    public class PartsAvailability
+    public class PartAvailability
     {
         [Column(Order = 0)]
         [DatabaseGenerated(DatabaseGeneratedOption.None)]
@@ -13,14 +14,8 @@ namespace PartSource.Data.Models
         [DatabaseGenerated(DatabaseGeneratedOption.None)]
         public int SKU { get; set; }
 
-        [Column("Line Code")]
-        [StringLength(50)]
-        public string Line_Code { get; set; }
-
-        [Column("Part Number")]
-        [StringLength(50)]
-        public string Part_Number { get; set; }
-
         public int? QTY { get; set; }
+
+        public string Updated { get; set; }
     }
 }

PartSource.Services/PartService.cs

@@ -21,7 +21,7 @@ namespace PartSource.Services
             _fitmentContext = fitmentContext;
         }
 
-        public async Task<PartsAvailability> GetInventory(int sku, int storeNumber)
+        public async Task<PartAvailability> GetInventory(int sku, int storeNumber)
         {
             return await _partSourceContext.PartAvailabilities.FirstOrDefaultAsync(s => s.Store == storeNumber && s.SKU == sku);
         }