diff --git a/.editorconfig b/.editorconfig index 136d44f..5676a4a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,3 +8,6 @@ dotnet_analyzer_diagnostic.category-Design.severity = silent # Default severity for analyzer diagnostics with category 'Globalization' dotnet_analyzer_diagnostic.category-Globalization.severity = silent + +# Default severity for analyzer diagnostics with category 'Performance' +dotnet_analyzer_diagnostic.category-Performance.severity = none diff --git a/PartSource.Automation/Factories/JobFactory.cs b/PartSource.Automation/Factories/JobFactory.cs deleted file mode 100644 index 75ad368..0000000 --- a/PartSource.Automation/Factories/JobFactory.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using PartSource.Automation.Jobs; -using PartSource.Automation.Jobs.Interfaces; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace PartSource.Automation.Factories -{ - public class JobFactory - { - private readonly IServiceProvider _serviceProvider; - - public JobFactory(IServiceProvider serviceProvider) - { - _serviceProvider = serviceProvider; - } - - public IAutomationJob Build(string jobName) - { - switch (jobName) - { - case nameof(AddAndUpdateProducts): - return _serviceProvider.GetService(); - - case nameof(DeleteProducts): - return _serviceProvider.GetService(); - - case nameof(StatusCheck): - return _serviceProvider.GetService(); - - //case nameof(TestJob): - // return new TestJob(); - - case nameof(UpdateFitment): - return _serviceProvider.GetService(); - - //case nameof(UpdatePricing): - // return _serviceProvider.GetService(); - - case nameof(UpdatePositioning): - return _serviceProvider.GetService(); - - case nameof(ExecuteSsisPackages): - return _serviceProvider.GetService(); - - default: - throw new Exception($"The job {jobName} could not be found."); - } - } - } -} diff --git a/PartSource.Automation/Jobs/AddAndUpdateProducts.cs b/PartSource.Automation/Jobs/AddAndUpdateProducts.cs deleted file mode 100644 index 50223e9..0000000 --- a/PartSource.Automation/Jobs/AddAndUpdateProducts.cs +++ /dev/null @@ -1,326 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; -using PartSource.Data; -using PartSource.Data.Models; -using Ratermania.Shopify; -using Ratermania.Shopify.Resources; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace PartSource.Automation.Jobs -{ - public class AddAndUpdateProducts : IAutomationJob - { - private readonly PartSourceContext _partSourceContext; - private readonly ShopifyClient _shopifyClient; - - public AddAndUpdateProducts(PartSourceContext partSourceContext, ShopifyClient shopifyClient) - { - _partSourceContext = partSourceContext; - _shopifyClient = shopifyClient; - } - - - public async Task Run() - { - //throw new Exception("You need to add a ProductVariant resource to the Shopify client"); - - await SyncronizeIdsAndSkus(); - //await AddSkus(); - // await AddVariants(); - - return new AutomationJobResult - { - IsSuccess = true - }; - } - - /// - /// Ensures syncronization between Shopify IDs and Partsource SKUs - /// - private async Task SyncronizeIdsAndSkus() - { - IEnumerable products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); - - //_partSourceContext.Database.ExecuteSqlCommand("UPDATE ImportData SET ShopifyId = NULL"); - - while (products != null && products.Any()) - { - - foreach (Product product in products) - { - foreach (Variant variant in product.Variants) - { - ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == variant.Sku); - - if (importData != null) - { - importData.ShopifyId = product.Id; - } - } - } - - try - { - await _partSourceContext.SaveChangesAsync(); - } - - catch - { - Console.WriteLine("Failed to save a batch of products"); - } - - finally - { - products = await _shopifyClient.Products.GetNext(); - } - } - } - - - public async Task AddSkus() - { - IList items = _partSourceContext.ImportData - .Where(i => i.IsVariant == null && i.ShopifyId == null) - // .OrderBy(i => i.Title) - .ToList(); - - // items = items.Where(i => i.Title == items.First().Title).ToList(); - // - foreach (ImportData importData in items) - { - try - { - // Images - IList productImages = new List(); - - if (!string.IsNullOrEmpty(importData.ImageSrc)) - { - foreach (string src in importData?.ImageSrc.Split(',')) - { - productImages.Add(new ProductImage - { - Src = src, - Alt = importData.ImageAltText - }); - } - - if (productImages.Count > 0) - { - productImages.Add(new ProductImage - { - Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg", - Alt = "No Image Found" - }); - } - } - - // Product Tags - List productTags = new List - { - items[0].LineCode, - items[0].PartNumber, - }; - - //List productVariants = new List(); - - ////foreach (ImportData itemVariant in items) - ////{ - //productVariants.Add(new ProductVariant - //{ - // InventoryPolicy = "Deny", - // CompareAtPrice = importData.CompareAt, - // Price = importData.Price, - // Sku = importData.VariantSku, - // Title = importData.VariantTitle ?? importData.Title, - // Option1 = importData.VariantTitle, - // RequiresShipping = false, - //}); - //} - - - - Product requestData = new Product - { - BodyHtml = importData.BodyHtml, - Title = importData.Title, - Vendor = importData.Vendor, - Tags = string.Join(",", productTags), - //ProductType = importData.FINELINE_NM, - Images = productImages.ToArray(), - //Variants = productVariants.ToArray(), - CreatedAt = DateTime.Now, - UpdatedAt = DateTime.Now, - PublishedAt = DateTime.Now - }; - - requestData = await _shopifyClient.Products.Add(requestData); - - if (requestData.Id > 0) - { - //foreach (ImportData variant in items) - //{ - // variant.ShopifyId = requestData.Id; - //} - - importData.ShopifyId = requestData.Id; - _partSourceContext.SaveChanges(); - - - Console.WriteLine($"{importData.VariantSku}"); - } - - else - { - Console.WriteLine($"SHOPIFY ID WAS 0 - {importData.VariantSku}"); - } - } - - catch (Exception ex) - { - Console.WriteLine($"Failed to add SKU {importData.VariantSku}: {ex.StackTrace}"); - } - - //items = _partSourceContext.ImportData - // .Where(i => i.Title == _partSourceContext.ImportData.First(d => d.ShopifyId == null).Title) - // .ToList(); - } - } - - public async Task AddVariants() - { - IList items = _partSourceContext.ImportData - .Where(i => i.IsVariant.Value && i.ShopifyId == null) - .OrderBy(i => i.Title) - .ToList(); - - items = items.Where(i => i.Title == items.First().Title).ToList(); - - while (items != null && items.Count > 0) - { - // Images - IList productImages = new List(); - - if (!string.IsNullOrEmpty(items[0].ImageSrc)) - { - productImages = items.SelectMany(v => - { - IList images = new List(); - - foreach (string src in v.ImageSrc?.Split(',')) - { - images.Add(new ProductImage - { - Src = src, - Alt = v.ImageAltText - }); - } - - return images; - }).ToList(); - - if (productImages.Count > 0) - { - productImages.Add(new ProductImage - { - Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg", - Alt = "No Image Found" - }); - } - } - - - // Product Tags - List productTags = new List - { - items[0].LineCode, - items[0].PartNumber, - }; - - //List productVariants = new List(); - - //foreach (ImportData itemVariant in items) - //{ - // productVariants.Add(new ProductVariant - // { - // InventoryPolicy = "Deny", - // CompareAtPrice = itemVariant.CompareAt, - // Price = itemVariant.Price, - // Sku = itemVariant.VariantSku, - // Title = itemVariant.VariantTitle, - // Option1 = itemVariant.VariantTitle, - // RequiresShipping = false, - // }); - //} - - - Product requestData = new Product - { - BodyHtml = items[0].BodyHtml, - Title = items[0].Title, - Vendor = items[0].Vendor, - Tags = string.Join(",", productTags), - //ProductType = importData.FINELINE_NM, - Images = productImages.ToArray(), - //Variants = productVariants.ToArray(), - CreatedAt = DateTime.Now, - UpdatedAt = DateTime.Now, - PublishedAt = DateTime.Now - }; - - requestData = await _shopifyClient.Products.Add(requestData); - - if (requestData.Id > 0) - { - foreach (ImportData variant in items) - { - variant.ShopifyId = requestData.Id; - } - - _partSourceContext.SaveChanges(); - - Console.WriteLine($"{items[0].VariantSku}"); - } - - else - { - Console.WriteLine($"SHOPIFY ID WAS 0 - {items[0].VariantSku}"); - } - - ImportData next = _partSourceContext.ImportData.FirstOrDefault(i => i.IsVariant != null && i.ShopifyId == null); - - if (next != null) - { - items = _partSourceContext.ImportData - .Where(i => i.Title == next.Title) - .ToList(); - } - - else - { - items = null; - } - } - } - } - - //private void Log(string message) - //{ - // try - // { - // using (FileStream fileStream = File.OpenWrite(@"C:\users\tommy\desktop\log.txt")) - // { - // fileStream.Write(Encoding.UTF8.GetBytes(message + "\n")); - // } - // } - - // catch - // { - // // LOL Fix this - // Log(message); - // } - //} -} diff --git a/PartSource.Automation/Jobs/Archive/AddAndUpdateProducts.cs b/PartSource.Automation/Jobs/Archive/AddAndUpdateProducts.cs new file mode 100644 index 0000000..fca475b --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/AddAndUpdateProducts.cs @@ -0,0 +1,324 @@ +//using PartSource.Automation.Models; +//using PartSource.Data; +//using PartSource.Data.Models; +//using Ratermania.Shopify; +//using Ratermania.Shopify.Resources; +//using System; +//using System.Collections.Generic; +//using System.Linq; +//using System.Threading.Tasks; + +//namespace PartSource.Automation.Jobs +//{ +// public class AddAndUpdateProducts// : IAutomationJob +// { +// private readonly PartSourceContext _partSourceContext; +// private readonly ShopifyClient _shopifyClient; + +// public AddAndUpdateProducts(PartSourceContext partSourceContext, ShopifyClient shopifyClient) +// { +// _partSourceContext = partSourceContext; +// _shopifyClient = shopifyClient; +// } + + +// public async Task Run() +// { +// //throw new Exception("You need to add a ProductVariant resource to the Shopify client"); + +// await SyncronizeIdsAndSkus(); +// //await AddSkus(); +// // await AddVariants(); + +// return new AutomationJobResult +// { +// IsSuccess = true +// }; +// } + +// /// +// /// Ensures syncronization between Shopify IDs and Partsource SKUs +// /// +// private async Task SyncronizeIdsAndSkus() +// { +// IEnumerable products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); + +// //_partSourceContext.Database.ExecuteSqlCommand("UPDATE ImportData SET ShopifyId = NULL"); + +// while (products != null && products.Any()) +// { + +// foreach (Product product in products) +// { +// foreach (Variant variant in product.Variants) +// { +// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == variant.Sku); + +// if (importData != null) +// { +// importData.ShopifyId = product.Id; +// } +// } +// } + +// try +// { +// await _partSourceContext.SaveChangesAsync(); +// } + +// catch +// { +// Console.WriteLine("Failed to save a batch of products"); +// } + +// finally +// { +// products = await _shopifyClient.Products.GetNext(); +// } +// } +// } + + +// public async Task AddSkus() +// { +// IList items = _partSourceContext.ImportData +// .Where(i => i.IsVariant == null && i.ShopifyId == null) +// // .OrderBy(i => i.Title) +// .ToList(); + +// // items = items.Where(i => i.Title == items.First().Title).ToList(); +// // +// foreach (ImportData importData in items) +// { +// try +// { +// // Images +// IList productImages = new List(); + +// if (!string.IsNullOrEmpty(importData.ImageSrc)) +// { +// foreach (string src in importData?.ImageSrc.Split(',')) +// { +// productImages.Add(new ProductImage +// { +// Src = src, +// Alt = importData.ImageAltText +// }); +// } + +// if (productImages.Count > 0) +// { +// productImages.Add(new ProductImage +// { +// Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg", +// Alt = "No Image Found" +// }); +// } +// } + +// // Product Tags +// List productTags = new List +// { +// items[0].LineCode, +// items[0].PartNumber, +// }; + +// //List productVariants = new List(); + +// ////foreach (ImportData itemVariant in items) +// ////{ +// //productVariants.Add(new ProductVariant +// //{ +// // InventoryPolicy = "Deny", +// // CompareAtPrice = importData.CompareAt, +// // Price = importData.Price, +// // Sku = importData.VariantSku, +// // Title = importData.VariantTitle ?? importData.Title, +// // Option1 = importData.VariantTitle, +// // RequiresShipping = false, +// //}); +// //} + + + +// Product requestData = new Product +// { +// BodyHtml = importData.BodyHtml, +// Title = importData.Title, +// Vendor = importData.Vendor, +// Tags = string.Join(",", productTags), +// //ProductType = importData.FINELINE_NM, +// Images = productImages.ToArray(), +// //Variants = productVariants.ToArray(), +// CreatedAt = DateTime.Now, +// UpdatedAt = DateTime.Now, +// PublishedAt = DateTime.Now +// }; + +// requestData = await _shopifyClient.Products.Add(requestData); + +// if (requestData.Id > 0) +// { +// //foreach (ImportData variant in items) +// //{ +// // variant.ShopifyId = requestData.Id; +// //} + +// importData.ShopifyId = requestData.Id; +// _partSourceContext.SaveChanges(); + + +// Console.WriteLine($"{importData.VariantSku}"); +// } + +// else +// { +// Console.WriteLine($"SHOPIFY ID WAS 0 - {importData.VariantSku}"); +// } +// } + +// catch (Exception ex) +// { +// Console.WriteLine($"Failed to add SKU {importData.VariantSku}: {ex.StackTrace}"); +// } + +// //items = _partSourceContext.ImportData +// // .Where(i => i.Title == _partSourceContext.ImportData.First(d => d.ShopifyId == null).Title) +// // .ToList(); +// } +// } + +// public async Task AddVariants() +// { +// IList items = _partSourceContext.ImportData +// .Where(i => i.IsVariant.Value && i.ShopifyId == null) +// .OrderBy(i => i.Title) +// .ToList(); + +// items = items.Where(i => i.Title == items.First().Title).ToList(); + +// while (items != null && items.Count > 0) +// { +// // Images +// IList productImages = new List(); + +// if (!string.IsNullOrEmpty(items[0].ImageSrc)) +// { +// productImages = items.SelectMany(v => +// { +// IList images = new List(); + +// foreach (string src in v.ImageSrc?.Split(',')) +// { +// images.Add(new ProductImage +// { +// Src = src, +// Alt = v.ImageAltText +// }); +// } + +// return images; +// }).ToList(); + +// if (productImages.Count > 0) +// { +// productImages.Add(new ProductImage +// { +// Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg", +// Alt = "No Image Found" +// }); +// } +// } + + +// // Product Tags +// List productTags = new List +// { +// items[0].LineCode, +// items[0].PartNumber, +// }; + +// //List productVariants = new List(); + +// //foreach (ImportData itemVariant in items) +// //{ +// // productVariants.Add(new ProductVariant +// // { +// // InventoryPolicy = "Deny", +// // CompareAtPrice = itemVariant.CompareAt, +// // Price = itemVariant.Price, +// // Sku = itemVariant.VariantSku, +// // Title = itemVariant.VariantTitle, +// // Option1 = itemVariant.VariantTitle, +// // RequiresShipping = false, +// // }); +// //} + + +// Product requestData = new Product +// { +// BodyHtml = items[0].BodyHtml, +// Title = items[0].Title, +// Vendor = items[0].Vendor, +// Tags = string.Join(",", productTags), +// //ProductType = importData.FINELINE_NM, +// Images = productImages.ToArray(), +// //Variants = productVariants.ToArray(), +// CreatedAt = DateTime.Now, +// UpdatedAt = DateTime.Now, +// PublishedAt = DateTime.Now +// }; + +// requestData = await _shopifyClient.Products.Add(requestData); + +// if (requestData.Id > 0) +// { +// foreach (ImportData variant in items) +// { +// variant.ShopifyId = requestData.Id; +// } + +// _partSourceContext.SaveChanges(); + +// Console.WriteLine($"{items[0].VariantSku}"); +// } + +// else +// { +// Console.WriteLine($"SHOPIFY ID WAS 0 - {items[0].VariantSku}"); +// } + +// ImportData next = _partSourceContext.ImportData.FirstOrDefault(i => i.IsVariant != null && i.ShopifyId == null); + +// if (next != null) +// { +// items = _partSourceContext.ImportData +// .Where(i => i.Title == next.Title) +// .ToList(); +// } + +// else +// { +// items = null; +// } +// } +// } +// } + +// //private void Log(string message) +// //{ +// // try +// // { +// // using (FileStream fileStream = File.OpenWrite(@"C:\users\tommy\desktop\log.txt")) +// // { +// // fileStream.Write(Encoding.UTF8.GetBytes(message + "\n")); +// // } +// // } + +// // catch +// // { +// // // LOL Fix this +// // Log(message); +// // } +// //} +//} diff --git a/PartSource.Automation/Jobs/BuildVehicleCache.cs b/PartSource.Automation/Jobs/Archive/BuildVehicleCache.cs similarity index 100% rename from PartSource.Automation/Jobs/BuildVehicleCache.cs rename to PartSource.Automation/Jobs/Archive/BuildVehicleCache.cs diff --git a/PartSource.Automation/Jobs/Archive/DeleteProducts.cs b/PartSource.Automation/Jobs/Archive/DeleteProducts.cs new file mode 100644 index 0000000..a1e5151 --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/DeleteProducts.cs @@ -0,0 +1,72 @@ +//using PartSource.Automation.Models; +//using PartSource.Data; +//using Ratermania.Shopify; +//using Ratermania.Shopify.Resources; +//using System; +//using System.Collections.Generic; +//using System.Linq; +//using System.Threading.Tasks; + +//namespace PartSource.Automation.Jobs +//{ +// public class DeleteProducts //: IAutomationJob +// { +// private readonly ShopifyClient _shopifyClient; +// private readonly PartSourceContext _partSourceContext; + +// public DeleteProducts(ShopifyClient shopifyClient, PartSourceContext partSourceContext) +// { +// _shopifyClient = shopifyClient; +// _partSourceContext = partSourceContext; +// } + +// // If this job fails, oh well. Run it again and again until it works, or use the Shopify UI (LOL) +// public async Task Run() +// { +// Console.WriteLine("This job will delete ALL PRODUCTS from Shopify. If you really want to delete EVERY SINGLE PRODUCT, type 'mechanical keyboard' below."); +// string input = Console.ReadLine(); + +// if (input != "mechanical keyboard") +// { +// return new AutomationJobResult +// { +// IsSuccess = true +// }; +// } + +// IList shopifyIds = _partSourceContext.ImportData +// .Select(i => i.ShopifyId) +// .Distinct() +// .ToList(); + +// foreach (long? id in shopifyIds) +// { +// Product product = await _shopifyClient.Products.GetById((long)id); + +// await _shopifyClient.Products.Delete(product); + +// Console.WriteLine(id); +// } + +// //IEnumerable products = await _shopifyClient.Products.Get(); + +// //while (products != null) +// //{ +// // foreach (Product product in products) +// // { +// // bool result = await _shopifyClient.Products.Delete(product); +// // } + +// // products = await _shopifyClient.Products.GetNext(); + +// // Console.Write('.'); +// //} + +// return new AutomationJobResult +// { +// Message = "All products deleted. Don't forget to truncate the ImportData table", +// IsSuccess = true +// }; +// } +// } +//} \ No newline at end of file diff --git a/PartSource.Automation/Jobs/Archive/StatusCheck.cs b/PartSource.Automation/Jobs/Archive/StatusCheck.cs new file mode 100644 index 0000000..89e2d32 --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/StatusCheck.cs @@ -0,0 +1,33 @@ +//using PartSource.Automation.Models; +//using PartSource.Automation.Services; +//using System.Threading.Tasks; + +//namespace PartSource.Automation.Jobs +//{ +// public class StatusCheck //: IAutomationJob +// { +// private readonly string[] phoneNumbers = { "8593609107", "5134008303" }; +// private readonly EmailService _emailService; + +// public StatusCheck(EmailService emailService) +// { +// _emailService = emailService; +// } + +// public async Task Run() +// { +// foreach (string phoneNumber in phoneNumbers) +// { +// // TODO: One day it won't just be AT&T numbers +// string to = $"{phoneNumber}@txt.att.net"; + +// _emailService.Send(to, string.Empty, "The Partsource automation server is running. Check the server for more details."); +// } + +// return new AutomationJobResult +// { +// IsSuccess = true +// }; +// } +// } +//} diff --git a/PartSource.Automation/Jobs/Archive/UpdateFitment - Copy.cs b/PartSource.Automation/Jobs/Archive/UpdateFitment - Copy.cs new file mode 100644 index 0000000..0aa03ce --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/UpdateFitment - Copy.cs @@ -0,0 +1,246 @@ +//using Microsoft.EntityFrameworkCore; +//using Newtonsoft.Json; +//using PartSource.Automation.Models; +//using PartSource.Data; +//using PartSource.Data.Models; +//using PartSource.Services; +//using Ratermania.Shopify; +//using Ratermania.Shopify.Exceptions; +//using Ratermania.Shopify.Resources; +//using System; +//using System.Collections.Generic; +//using System.Data; +//using System.Linq; +//using System.Text.RegularExpressions; +//using System.Threading.Tasks; + +//namespace PartSource.Automation.Jobs +//{ +// public class UpdateFitment //: IAutomationJob +// { +// private readonly IServiceProvider _serviceProvider; +// private readonly ShopifyClient _shopifyClient; +// private readonly PartSourceContext _partSourceContext; +// private readonly NexpartService _nexpartService; + +// private readonly VehicleService _vehicleService; + +// public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService) +// { +// _partSourceContext = partSourceContext; +// _shopifyClient = shopifyClient; +// _vehicleService = vehicleService; +// } + +// public async Task Run() +// { +// IEnumerable products = null; + +// try +// { +// products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); +// } + +// catch (Exception ex) +// { +// // TODO: Logging +// return new AutomationJobResult +// { +// Message = "Failed to get products from Shopify", +// IsSuccess = false +// }; +// } + +// while (products != null && products.Any()) +// { +// foreach (Product product in products) +// { +// try +// { +// ImportData importData = await _partSourceContext.ImportData +// .Where(i => i.ShopifyId == product.Id && i.UpdatedAt <= DateTime.Now.AddDays(-7)) +// .FirstOrDefaultAsync(); + +// if (importData == null) +// { +// continue; +// } + +// bool isFitment = false; + +// IList vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode, 255); + +// if (vehicles.Count > 250) +// { +// continue; +// } + +// IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); + +// if (vehicleIdFitment.Count > 0 && vehicleIdFitment.Count <= 250) +// { +// isFitment = true; + +// string json = JsonConvert.SerializeObject(vehicleIdFitment); +// if (json.Length >= 100000) +// { +// continue; +// } + +// Metafield vehicleMetafield = new Metafield +// { +// Namespace = "fitment", +// Key = "ids", +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(vehicleMetafield); +// } + +// IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); +// if (ymmFitment.Count > 0) +// { +// isFitment = true; + +// string json = JsonConvert.SerializeObject(ymmFitment); +// if (json.Length >= 100000) +// { +// continue; +// } + +// Metafield ymmMetafield = new Metafield +// { +// Namespace = "fitment", +// Key = "seo", +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(ymmMetafield); +// } + +// Metafield isFitmentMetafield = new Metafield +// { +// Namespace = "Flags", +// Key = "IsFitment", +// Value = isFitment.ToString(), +// ValueType = "string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(isFitmentMetafield); + + +// //Metafield noteTextMetafield = new Metafield +// //{ +// // Namespace = "Flags", +// // Key = "IsFitment", +// // Value = isFitment.ToString(), +// // ValueType = "string", +// // OwnerResource = "product", +// // OwnerId = product.Id +// //}; + +// //await _shopifyClient.Metafields.Add(noteTextMetafield); + + +// List tags = new List +// { +// importData.LineCode, +// importData.PartNumber +// }; + +// for (int j = 0; j < vehicleIdFitment.Count; j += 25) +// { +// tags.Add(string.Join('-', vehicleIdFitment.Skip(j).Take(25).Select(j => $"v{j}"))); +// } + +// tags.AddRange(ymmFitment); + +// if (tags.Count > 250) +// { +// tags = tags.Take(250).ToList(); +// } + +// product.Tags = string.Join(',', tags); + +// await _shopifyClient.Products.Update(product); + +// importData.IsFitment = isFitment; +// importData.UpdatedAt = DateTime.Now; +// importData.UpdateType = "Fitment"; +// } + +// catch (ShopifyClientException ex) +// { +// // TODO: Log +// } + +// catch (Exception ex) +// { +// // TODO: Log +// } +// } + +// try +// { +// _partSourceContext.SaveChanges(); +// products = await _shopifyClient.Products.GetNext(); +// } + +// catch (Exception ex) +// { +// products = await _shopifyClient.Products.GetNext(); +// } +// } + + +// return new AutomationJobResult +// { +// IsSuccess = true +// }; +// } + +// private async Task DeleteFitmentMetafields(long shopifyId) +// { +// IDictionary parameters = new Dictionary +// { +// { "metafield[owner_id]", shopifyId}, +// { "metafield[owner_resource]", "product" }, +// { "namespace", "fitment" }, +// }; + +// IEnumerable metafields = await _shopifyClient.Metafields.Get(parameters); + +// foreach (Metafield metafield in metafields) +// { +// await _shopifyClient.Metafields.Delete(metafield); +// } +// } + +// public IList GetVehicles(string partNumber, string lineCode) +// { +// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); + +// //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')"; +// string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;"; + +//#pragma warning disable EF1000 // Possible SQL injection vulnerability. +// IList vehicles = _partSourceContext.Vehicles.FromSql(sql).ToList(); +//#pragma warning restore EF1000 // Possible SQL injection vulnerability. + +// return vehicles; +// } +// private void Update() +// { + +// } + +// } +//} diff --git a/PartSource.Automation/Jobs/Archive/UpdateFitment.cs b/PartSource.Automation/Jobs/Archive/UpdateFitment.cs new file mode 100644 index 0000000..0cc58aa --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/UpdateFitment.cs @@ -0,0 +1,200 @@ +//using Microsoft.EntityFrameworkCore; +//using Newtonsoft.Json; +//using PartSource.Automation.Jobs.Interfaces; +//using PartSource.Automation.Models; +//using PartSource.Automation.Services; +//using PartSource.Data; +//using PartSource.Data.Models; +//using PartSource.Data.Nexpart; +//using PartSource.Services; +//using PartSource.Services.Integrations; +//using System; +//using System.Collections.Concurrent; +//using System.Collections.Generic; +//using System.Data; +//using System.Data.SqlClient; +//using System.IO; +//using System.Linq; +//using System.Text; +//using System.Text.RegularExpressions; +//using System.Threading.Tasks; +//using Ratermania.Shopify.Entities; + +//namespace PartSource.Automation.Jobs +//{ +// public class UpdateFitment : IAutomationJob +// { +// private readonly IServiceProvider _serviceProvider; +// private readonly ShopifyClient _shopifyClient; +// private readonly PartSourceContext _partSourceContext; +// private readonly NexpartService _nexpartService; + +// private readonly VehicleService _vehicleService; + +// public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService) +// { +// _partSourceContext = partSourceContext; +// _shopifyClient = shopifyClient; +// _vehicleService = vehicleService; +// } + +// public async Task Run() +// { +// IEnumerable products = await _shopifyClient.Products.Get(); + +// int i = 0; + +// while (products != null && products.Any()) +// { +// foreach (Product product in products) +// { +// try +// { +// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku && i.UpdatedAt == null); +// IList vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode); + +// if (vehicles == null || vehicles.Count == 0) +// { +// continue; +// } + +// await DeleteFitmentMetafields(product.Id); + +// bool isFitment = false; + +// IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); +// if (vehicleIdFitment.Count > 0) +// { +// isFitment = true; + +// string json = JsonConvert.SerializeObject(vehicleIdFitment); +// if (json.Length >= 100000) +// { +// continue; +// } + +// Metafield vehicleMetafield = new Metafield +// { +// Namespace = "fitment", +// Key = "ids", +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(vehicleMetafield); +// } + +// IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); +// if (ymmFitment.Count > 0) +// { +// isFitment = true; + +// string json = JsonConvert.SerializeObject(ymmFitment); +// if (json.Length >= 100000) +// { +// continue; +// } + +// Metafield ymmMetafield = new Metafield +// { +// Namespace = "fitment", +// Key = "seo", +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(ymmMetafield); +// } + +// Metafield isFitmentMetafield = new Metafield +// { +// Namespace = "Flags", +// Key = "IsFitment", +// Value = isFitment.ToString(), +// ValueType = "string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(isFitmentMetafield); + +// importData.UpdatedAt = DateTime.Now; +// importData.UpdateType = "Fitment"; +// } + +// catch (Exception ex) +// { +// Console.WriteLine($"{product.Id}: {ex.Message}"); +// } +// } + +// try +// { +// i++; +// Console.WriteLine(i); + +// await _partSourceContext.SaveChangesAsync(); + +// products = await _shopifyClient.Products.GetNext(); +// } + +// catch (Exception ex) +// { +// i++; +// Console.WriteLine(i); + +// Console.WriteLine($"Retrying request: {ex.Message}"); + +// await _partSourceContext.SaveChangesAsync(); + +// products = await _shopifyClient.Products.GetNext(); +// } +// } + +// return new AutomationJobResult +// { +// IsSuccess = true +// }; +// } + +// private async Task DeleteFitmentMetafields(long shopifyId) +// { +// IDictionary parameters = new Dictionary +// { +// { "metafield[owner_id]", shopifyId}, +// { "metafield[owner_resource]", "product" }, +// { "namespace", "fitment" }, +// }; + +// IEnumerable metafields = await _shopifyClient.Metafields.Get(parameters); + +// foreach (Metafield metafield in metafields) +// { +// await _shopifyClient.Metafields.Delete(metafield); +// } +// } + +// public IList GetVehicles(string partNumber, string lineCode) +// { +// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); + +// //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')"; +// string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;"; + +//#pragma warning disable EF1000 // Possible SQL injection vulnerability. +// IList vehicles = _partSourceContext.VehicleData.FromSql(sql).ToList(); +//#pragma warning restore EF1000 // Possible SQL injection vulnerability. + +// return vehicles; +// } +// private void Update() +// { + +// } + +// } +//} diff --git a/PartSource.Automation/Jobs/Archive/UpdatePositioning.cs b/PartSource.Automation/Jobs/Archive/UpdatePositioning.cs new file mode 100644 index 0000000..7eb6636 --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/UpdatePositioning.cs @@ -0,0 +1,209 @@ +//using Microsoft.EntityFrameworkCore; +//using Newtonsoft.Json; +//using PartSource.Automation.Models; +//using PartSource.Data; +//using PartSource.Data.Models; +//using PartSource.Services; +//using Ratermania.Shopify; +//using Ratermania.Shopify.Resources; +//using System; +//using System.Collections.Generic; +//using System.Diagnostics.CodeAnalysis; +//using System.Linq; +//using System.Text.RegularExpressions; +//using System.Threading.Tasks; + +//namespace PartSource.Automation.Jobs +//{ +// public class UpdatePositioning// : IAutomationJob +// { +// private readonly ShopifyClient _shopifyClient; +// private readonly PartSourceContext _partSourceContext; +// private readonly VehicleService _vehicleService; + +// public UpdatePositioning(PartSourceContext partSourceContext, ShopifyClient shopifyClient, VehicleService vehicleService) +// { +// _partSourceContext = partSourceContext; +// _shopifyClient = shopifyClient; +// _vehicleService = vehicleService; +// } + +// public async Task Run() +// { +// IDictionary parameters = new Dictionary +// { +// { "limit", 250 } +// }; + +// IEnumerable products = await _shopifyClient.Products.Get(parameters); + +// while (products != null && products.Any()) +// { +// foreach (Product product in products) +// { +// try +// { +// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku); + +// if (importData == null || importData?.LineCode == "SVG") // Headlights go in front, DUH +// { +// continue; +// } + +// IList fitments = GetPositionOrderedFitments(importData?.PartNumber, importData?.LineCode); +// IList vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode); + +// if (fitments.Count == 0 || vehicles.Count == 0) +// { +// continue; +// } + +// //await DeletePositionMetafields(product.Id); + +// //string currentPosition = fitments[0].Position; +// //List vehicleIds = new List(); + +// //foreach (Fitment fitment in fitments) +// //{ +// // if (fitment.Position != currentPosition) +// // { +// // await SavePositionMetafield(product, vehicleIds, currentPosition); + +// // currentPosition = fitment.Position; +// // vehicleIds = new List(); +// // } + +// // // We don't need to DCF map because these are both sourced from WHI +// // IList fitmentVehicleIds = vehicles.Where(v => v.BaseVehicleId == fitment.BaseVehicleId && v.EngineConfigId == fitment.EngineConfigId) +// // .Select(v => v.VehicleToEngineConfigId) +// // .Distinct() +// // .ToList(); + +// // vehicleIds.AddRange(fitmentVehicleIds); +// //} + +// //await SavePositionMetafield(product, vehicleIds, currentPosition); + + +// IList notes = fitments.Select(f => f.NoteText) +// .Distinct() +// .ToList(); + +// IList vehicleNotes = new List(); + +// foreach (string noteText in notes) +// { +// IList vehicleIds = fitments.Where(f => f.NoteText == noteText) +// .Select(f => new { f.EngineConfigId, f.BaseVehicleId }) +// .SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId)) +// .Select(v => v.VehicleToEngineConfigId) +// .ToList(); + + +// vehicleNotes.Add(new { noteText, vehicleIds }); +// } + +// string json = JsonConvert.SerializeObject(vehicleNotes); +// if (json.Length >= 100000) +// { +// continue; +// } + +// Metafield vehicleMetafield = new Metafield +// { +// Namespace = "fitment", +// Key = "note_text", +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// await _shopifyClient.Metafields.Add(vehicleMetafield); + +// //importData.UpdatedAt = DateTime.Now; +// //importData.UpdateType = "Positioning"; +// ; + +// } + +// catch (Exception ex) +// { +// Console.WriteLine($"{product.Id}: {ex.Message}"); +// } +// } + +// try +// { +// Console.Write('.'); +// // await _partSourceContext.SaveChangesAsync(); +// products = await _shopifyClient.Products.GetNext(); +// } + +// catch (Exception ex) +// { +// Console.WriteLine($"Retrying: {ex.Message}"); + +// products = await _shopifyClient.Products.GetPrevious(); +// } +// } + +// return new AutomationJobResult +// { +// Message = "Positioning updated successfully", +// IsSuccess = true +// }; +// } + +// private IList GetPositionOrderedFitments(string partNumber, string lineCode) +// { +// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); + +// IQueryable whiCodes = _partSourceContext.DcfMappings +// .Where(d => d.LineCode == lineCode) +// .Select(d => d.WhiCode); + +// return _partSourceContext.Fitments +// .Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode) && !string.IsNullOrEmpty(f.Position)) +// .OrderBy(f => f.Position) +// .ToList(); + +// } + +// [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "It's a Shopify metafield key")] +// private async Task SavePositionMetafield(Product product, IList vehicleIds, string position) +// { +// if (vehicleIds.Count == 0) +// { +// return; +// } + +// string json = JsonConvert.SerializeObject(vehicleIds); +// if (json.Length >= 100000) +// { +// // TODO: Logging +// return; +// } + +// string key = position.ToLowerInvariant().Replace(" ", "_"); +// if (key.Length > 20) +// { +// key = key.Substring(0, 20); +// } + +// Metafield vehicleMetafield = new Metafield +// { +// Namespace = "position", +// Key = key, +// Value = json, +// ValueType = "json_string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// System.Diagnostics.Debug.WriteLine(json); + +// await _shopifyClient.Metafields.Add(vehicleMetafield); +// } +// } +//} diff --git a/PartSource.Automation/Jobs/Archive/UpdatePricing.cs b/PartSource.Automation/Jobs/Archive/UpdatePricing.cs new file mode 100644 index 0000000..f9979da --- /dev/null +++ b/PartSource.Automation/Jobs/Archive/UpdatePricing.cs @@ -0,0 +1,113 @@ +//using Microsoft.EntityFrameworkCore; +//using Microsoft.Extensions.Logging.Abstractions; +//using Ratermania.Automation.Interfaces; +//using PartSource.Automation.Models; +//using PartSource.Data; +//using PartSource.Data.Models; +//using Ratermania.Shopify; +//using Ratermania.Shopify.Resources; +//using Ratermania.Shopify.Resources.Enums; +//using System; +//using System.Collections.Generic; +//using System.Linq; +//using System.Threading.Tasks; +//using Microsoft.Extensions.Logging; + +//namespace PartSource.Automation.Jobs +//{ +// public class UpdatePricing : IAutomationJob +// { +// private readonly ILogger _logger; +// private readonly PartSourceContext _partSourceContext; +// private readonly ShopifyClient _shopifyClient; + +// public UpdatePricing(ILogger logger, PartSourceContext partSourceContext, ShopifyClient shopifyClient) +// { +// _logger = logger; +// _partSourceContext = partSourceContext; +// _shopifyClient = shopifyClient; +// } + +// public async Task Run() +// { +// IEnumerable products = null; +// IEnumerable prices = null; +// int updateCount = 0; + +// try +// { +// products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); +// prices = await _partSourceContext.PartPrices.ToListAsync(); +// } + +// catch (Exception ex) +// { +// _logger.LogError(ex, "Failed to get the initial set of products from Shopify."); + +// throw; +// } + +// while (products != null && products.Any()) +// { +// foreach (Product product in products) +// { +// if (product.Variants.Length > 0) +// { +// Variant variant = product.Variants[0]; +// PartPrice partPrice = prices.Where(p => p.SKU == variant.Sku).FirstOrDefault(); + +// if (partPrice == null || !partPrice.Your_Price.HasValue || !partPrice.Compare_Price.HasValue) +// { +// continue; +// } + +// if (product.Variants[0].Price != partPrice.Your_Price.Value || product.Variants[0].CompareAtPrice != partPrice.Compare_Price.Value) +// { +// product.Variants[0].Price = partPrice.Your_Price.Value; +// product.Variants[0].CompareAtPrice = partPrice.Compare_Price.Value; + +// product.PublishedAt = partPrice.Active.Trim().ToUpperInvariant() == "Y" ? (DateTime?)DateTime.Now : null; +// product.PublishedScope = PublishedScope.Global; + +// Metafield metafield = new Metafield +// { +// Namespace = "Pricing", +// Key = "CorePrice", +// Value = partPrice.Core_Price.HasValue ? partPrice.Core_Price.Value.ToString() : "0.00", +// ValueType = "string", +// OwnerResource = "product", +// OwnerId = product.Id +// }; + +// try +// { +// await _shopifyClient.Metafields.Add(metafield); +// await _shopifyClient.Products.Update(product); + +// updateCount++; +// } + +// catch (Exception ex) +// { +// _logger.LogWarning(ex, $"Failed to update pricing for SKU {variant.Sku}"); +// } +// } +// } +// } + +// try +// { +// products = await _shopifyClient.Products.GetNext(); + +// _logger.LogInformation($"Total updated: {updateCount}"); +// } + +// catch (Exception ex) +// { +// _logger.LogWarning(ex, "Failed to get the next set of products. Retrying"); +// products = await _shopifyClient.Products.GetPrevious(); +// } +// } +// } +// } +//} \ No newline at end of file diff --git a/PartSource.Automation/Jobs/DeleteProducts.cs b/PartSource.Automation/Jobs/DeleteProducts.cs deleted file mode 100644 index 9b9c901..0000000 --- a/PartSource.Automation/Jobs/DeleteProducts.cs +++ /dev/null @@ -1,73 +0,0 @@ -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; -using PartSource.Data; -using Ratermania.Shopify; -using Ratermania.Shopify.Resources; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace PartSource.Automation.Jobs -{ - public class DeleteProducts : IAutomationJob - { - private readonly ShopifyClient _shopifyClient; - private readonly PartSourceContext _partSourceContext; - - public DeleteProducts(ShopifyClient shopifyClient, PartSourceContext partSourceContext) - { - _shopifyClient = shopifyClient; - _partSourceContext = partSourceContext; - } - - // If this job fails, oh well. Run it again and again until it works, or use the Shopify UI (LOL) - public async Task Run() - { - Console.WriteLine("This job will delete ALL PRODUCTS from Shopify. If you really want to delete EVERY SINGLE PRODUCT, type 'mechanical keyboard' below."); - string input = Console.ReadLine(); - - if (input != "mechanical keyboard") - { - return new AutomationJobResult - { - IsSuccess = true - }; - } - - IList shopifyIds = _partSourceContext.ImportData - .Select(i => i.ShopifyId) - .Distinct() - .ToList(); - - foreach (long? id in shopifyIds) - { - Product product = await _shopifyClient.Products.GetById((long)id); - - await _shopifyClient.Products.Delete(product); - - Console.WriteLine(id); - } - - //IEnumerable products = await _shopifyClient.Products.Get(); - - //while (products != null) - //{ - // foreach (Product product in products) - // { - // bool result = await _shopifyClient.Products.Delete(product); - // } - - // products = await _shopifyClient.Products.GetNext(); - - // Console.Write('.'); - //} - - return new AutomationJobResult - { - Message = "All products deleted. Don't forget to truncate the ImportData table", - IsSuccess = true - }; - } - } -} \ No newline at end of file diff --git a/PartSource.Automation/Jobs/ExecuteSsisPackages.cs b/PartSource.Automation/Jobs/ExecuteSsisPackages.cs index 7f09c32..8a33ba6 100644 --- a/PartSource.Automation/Jobs/ExecuteSsisPackages.cs +++ b/PartSource.Automation/Jobs/ExecuteSsisPackages.cs @@ -1,8 +1,9 @@ -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using PartSource.Automation.Models.Configuration; using PartSource.Automation.Services; +using Ratermania.Automation.Interfaces; using System; -using System.Collections.Generic; using System.Threading.Tasks; namespace PartSource.Automation.Jobs @@ -11,43 +12,42 @@ namespace PartSource.Automation.Jobs { private readonly FtpService _ftpService; private readonly SsisService _ssisService; + private readonly ILogger _logger; // TODO: set from config private readonly string[] _ssisPackages = { "Parts Availability", "Parts Price" }; - public ExecuteSsisPackages(FtpService ftpService, SsisService ssisService) + public ExecuteSsisPackages(IConfiguration configuration, SsisService ssisService, ILogger logger) { - _ftpService = ftpService; + FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get(); + + _ftpService = new FtpService(ftpConfiguration); _ssisService = ssisService; + _logger = logger; } - public async Task Run() + public async Task Run() { - IList updatedPackages = new List(); - IList failedPackages = new List(); - - foreach (string package in _ssisPackages) + await Task.Run(() => { - try + foreach (string package in _ssisPackages) { - _ftpService.Download($"{package}.txt"); - _ssisService.Execute($"{package}.dtsx"); + try + { + _ftpService.Download($"{package}.txt"); + _ssisService.Execute($"{package}.dtsx"); - updatedPackages.Add(package); + _logger.LogInformation($"Execution of SSIS package {package} completed successfully."); + } + + catch (Exception ex) + { + _logger.LogError($"Execution of SSIS package {package} failed.", ex); + + throw; + } } - - catch (Exception ex) - { - failedPackages.Add(package); - // TODO: Log - } - } - - return new AutomationJobResult - { - Message = $"Updated Packages: {string.Join(',', updatedPackages)} \n Failed Packages: {string.Join(',', failedPackages)}", - IsSuccess = failedPackages.Count == 0 - }; + }); } } } diff --git a/PartSource.Automation/Jobs/Interfaces/IAutomationJob.cs b/PartSource.Automation/Jobs/Interfaces/IAutomationJob.cs deleted file mode 100644 index 05e2229..0000000 --- a/PartSource.Automation/Jobs/Interfaces/IAutomationJob.cs +++ /dev/null @@ -1,14 +0,0 @@ -using PartSource.Automation.Models; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace PartSource.Automation.Jobs.Interfaces -{ - public interface IAutomationJob - { - Task Run(); - } -} diff --git a/PartSource.Automation/Jobs/ProcessWhiFitment.cs b/PartSource.Automation/Jobs/ProcessWhiFitment.cs new file mode 100644 index 0000000..8a72024 --- /dev/null +++ b/PartSource.Automation/Jobs/ProcessWhiFitment.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using PartSource.Automation.Models.Configuration; +using PartSource.Automation.Models.Enums; +using PartSource.Automation.Services; +using Ratermania.Automation.Interfaces; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.IO.Compression; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs +{ + public class ProcessWhiFitment : IAutomationJob + { + private readonly ILogger _logger; + private readonly WhiSeoService _whiSeoService; + private readonly FtpConfiguration _ftpConfiguration; + private readonly SeoDataType _seoDataType; + + public ProcessWhiFitment(IConfiguration configuration, ILogger logger, WhiSeoService whiSeoService) + { + _logger = logger; + _whiSeoService = whiSeoService; + + _seoDataType = SeoDataType.Fitment; + + _ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get(); + } + + public async Task Run() + { + _whiSeoService.Truncate(_seoDataType); + _whiSeoService.GetFiles(_seoDataType); + + string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant()); + DirectoryInfo directoryInfo = new DirectoryInfo(directory); + + foreach (FileInfo fileInfo in directoryInfo.GetFiles()) + { + if (!fileInfo.Name.EndsWith("csv.gz")) + { + continue; + } + + string filename = Decompress(fileInfo); + + using DataTable dataTable = new DataTable(); + dataTable.Columns.Add("LineCode", typeof(string)); + dataTable.Columns.Add("PartNumber", typeof(string)); + dataTable.Columns.Add("BaseVehicleId", typeof(int)); + dataTable.Columns.Add("EngineConfigId", typeof(int)); + dataTable.Columns.Add("Position", typeof(string)); + dataTable.Columns.Add("NoteText", typeof(string)); + + using StreamReader reader = new StreamReader(filename); + string line = reader.ReadLine(); // Burn the header row + + try + { + int skippedLines = 0; + while (reader.Peek() > 0) + { + line = reader.ReadLine(); + + string[] columns = line.Replace("\"", string.Empty).Split(','); + + if (columns.Length != 8) + { + skippedLines++; + continue; + } + + string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim(); + string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim(); + string position = columns[7].Trim(); + string noteText = columns[4].Trim(); + + if (!string.IsNullOrEmpty(lineCode) + && !string.IsNullOrEmpty(partNumber) + && int.TryParse(columns[5], out int baseVehicleId) + && int.TryParse(columns[6], out int engineConfigId)) + { + dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText }); + } + } + + _whiSeoService.BulkCopy(_seoDataType, dataTable); + + if (skippedLines == 0) + { + _logger.LogInformation($"Copied {filename} to the database."); + } + + else + { + _logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed."); + } + } + + catch (Exception ex) + { + _logger.LogError($"Failed to copy {filename} to the database.", ex); + } + + try + { + reader.Close(); + + File.Delete(filename); + File.Delete(fileInfo.FullName); + } + + catch (Exception ex) + { + _logger.LogWarning($"Failed to delete {filename}. This file will need to be deleted manually.", ex); + } + } + } + + public string Decompress(FileInfo fileInfo) + { + string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length); + + using FileStream filestream = File.Create(decompressedFile); + using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress); + + decompressionStream.CopyTo(filestream); + + return decompressedFile; + } + } +} \ No newline at end of file diff --git a/PartSource.Automation/Jobs/StatusCheck.cs b/PartSource.Automation/Jobs/StatusCheck.cs deleted file mode 100644 index 6e95933..0000000 --- a/PartSource.Automation/Jobs/StatusCheck.cs +++ /dev/null @@ -1,37 +0,0 @@ -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; -using PartSource.Automation.Services; -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading.Tasks; - -namespace PartSource.Automation.Jobs -{ - public class StatusCheck : IAutomationJob - { - private readonly string[] phoneNumbers = { "8593609107", "5134008303" }; - private readonly EmailService _emailService; - - public StatusCheck(EmailService emailService) - { - _emailService = emailService; - } - - public async Task Run() - { - foreach (string phoneNumber in phoneNumbers) - { - // TODO: One day it won't just be AT&T numbers - string to = $"{phoneNumber}@txt.att.net"; - - _emailService.Send(to, string.Empty, "The Partsource automation server is running. Check the server for more details."); - } - - return new AutomationJobResult - { - IsSuccess = true - }; - } - } -} diff --git a/PartSource.Automation/Jobs/UpdateFitment - Copy.cs b/PartSource.Automation/Jobs/UpdateFitment - Copy.cs deleted file mode 100644 index 3e5c536..0000000 --- a/PartSource.Automation/Jobs/UpdateFitment - Copy.cs +++ /dev/null @@ -1,253 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Newtonsoft.Json; -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; -using PartSource.Automation.Services; -using PartSource.Data; -using PartSource.Data.Models; -using PartSource.Data.Nexpart; -using PartSource.Services; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Data; -using System.Data.SqlClient; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Ratermania.Shopify; -using Ratermania.Shopify.Resources; -using Ratermania.Shopify.Exceptions; - -namespace PartSource.Automation.Jobs -{ - public class UpdateFitment : IAutomationJob - { - private readonly IServiceProvider _serviceProvider; - private readonly ShopifyClient _shopifyClient; - private readonly PartSourceContext _partSourceContext; - private readonly NexpartService _nexpartService; - - private readonly VehicleService _vehicleService; - - public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService) - { - _partSourceContext = partSourceContext; - _shopifyClient = shopifyClient; - _vehicleService = vehicleService; - } - - public async Task Run() - { - IEnumerable products = null; - - try - { - products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); - } - - catch (Exception ex) - { - // TODO: Logging - return new AutomationJobResult - { - Message = "Failed to get products from Shopify", - IsSuccess = false - }; - } - - while (products != null && products.Any()) - { - foreach (Product product in products) - { - try - { - ImportData importData = await _partSourceContext.ImportData - .Where(i => i.ShopifyId == product.Id && i.UpdatedAt <= DateTime.Now.AddDays(-7)) - .FirstOrDefaultAsync(); - - if (importData == null) - { - continue; - } - - bool isFitment = false; - - IList vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode, 255); - - if (vehicles.Count > 250) - { - continue; - } - - IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); - - if (vehicleIdFitment.Count > 0 && vehicleIdFitment.Count <= 250) - { - isFitment = true; - - string json = JsonConvert.SerializeObject(vehicleIdFitment); - if (json.Length >= 100000) - { - continue; - } - - Metafield vehicleMetafield = new Metafield - { - Namespace = "fitment", - Key = "ids", - Value = json, - ValueType = "json_string", - OwnerResource = "product", - OwnerId = product.Id - }; - - await _shopifyClient.Metafields.Add(vehicleMetafield); - } - - IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); - if (ymmFitment.Count > 0) - { - isFitment = true; - - string json = JsonConvert.SerializeObject(ymmFitment); - if (json.Length >= 100000) - { - continue; - } - - Metafield ymmMetafield = new Metafield - { - Namespace = "fitment", - Key = "seo", - Value = json, - ValueType = "json_string", - OwnerResource = "product", - OwnerId = product.Id - }; - - await _shopifyClient.Metafields.Add(ymmMetafield); - } - - Metafield isFitmentMetafield = new Metafield - { - Namespace = "Flags", - Key = "IsFitment", - Value = isFitment.ToString(), - ValueType = "string", - OwnerResource = "product", - OwnerId = product.Id - }; - - await _shopifyClient.Metafields.Add(isFitmentMetafield); - - - //Metafield noteTextMetafield = new Metafield - //{ - // Namespace = "Flags", - // Key = "IsFitment", - // Value = isFitment.ToString(), - // ValueType = "string", - // OwnerResource = "product", - // OwnerId = product.Id - //}; - - //await _shopifyClient.Metafields.Add(noteTextMetafield); - - - List tags = new List - { - importData.LineCode, - importData.PartNumber - }; - - for (int j = 0; j < vehicleIdFitment.Count; j += 25) - { - tags.Add(string.Join('-', vehicleIdFitment.Skip(j).Take(25).Select(j => $"v{j}"))); - } - - tags.AddRange(ymmFitment); - - if (tags.Count > 250) - { - tags = tags.Take(250).ToList(); - } - - product.Tags = string.Join(',', tags); - - await _shopifyClient.Products.Update(product); - - importData.IsFitment = isFitment; - importData.UpdatedAt = DateTime.Now; - importData.UpdateType = "Fitment"; - } - - catch (ShopifyClientException ex) - { - // TODO: Log - } - - catch (Exception ex) - { - // TODO: Log - } - } - - try - { - _partSourceContext.SaveChanges(); - products = await _shopifyClient.Products.GetNext(); - } - - catch (Exception ex) - { - products = await _shopifyClient.Products.GetNext(); - } - } - - - return new AutomationJobResult - { - IsSuccess = true - }; - } - - private async Task DeleteFitmentMetafields(long shopifyId) - { - IDictionary parameters = new Dictionary - { - { "metafield[owner_id]", shopifyId}, - { "metafield[owner_resource]", "product" }, - { "namespace", "fitment" }, - }; - - IEnumerable metafields = await _shopifyClient.Metafields.Get(parameters); - - foreach (Metafield metafield in metafields) - { - await _shopifyClient.Metafields.Delete(metafield); - } - } - - public IList GetVehicles(string partNumber, string lineCode) - { - partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); - - //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')"; - string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;"; - -#pragma warning disable EF1000 // Possible SQL injection vulnerability. - IList vehicles = _partSourceContext.Vehicles.FromSql(sql).ToList(); -#pragma warning restore EF1000 // Possible SQL injection vulnerability. - - return vehicles; - } - private void Update() - { - - } - - } -} diff --git a/PartSource.Automation/Jobs/UpdateFitment.cs b/PartSource.Automation/Jobs/UpdateFitment.cs index 0cc58aa..366f696 100644 --- a/PartSource.Automation/Jobs/UpdateFitment.cs +++ b/PartSource.Automation/Jobs/UpdateFitment.cs @@ -1,200 +1,196 @@ -//using Microsoft.EntityFrameworkCore; -//using Newtonsoft.Json; -//using PartSource.Automation.Jobs.Interfaces; -//using PartSource.Automation.Models; -//using PartSource.Automation.Services; -//using PartSource.Data; -//using PartSource.Data.Models; -//using PartSource.Data.Nexpart; -//using PartSource.Services; -//using PartSource.Services.Integrations; -//using System; -//using System.Collections.Concurrent; -//using System.Collections.Generic; -//using System.Data; -//using System.Data.SqlClient; -//using System.IO; -//using System.Linq; -//using System.Text; -//using System.Text.RegularExpressions; -//using System.Threading.Tasks; -//using Ratermania.Shopify.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using PartSource.Automation.Models; +using PartSource.Data; +using PartSource.Data.Models; +using PartSource.Services; +using Ratermania.Automation.Interfaces; +using Ratermania.Shopify; +using Ratermania.Shopify.Exceptions; +using Ratermania.Shopify.Resources; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; -//namespace PartSource.Automation.Jobs -//{ -// public class UpdateFitment : IAutomationJob -// { -// private readonly IServiceProvider _serviceProvider; -// private readonly ShopifyClient _shopifyClient; -// private readonly PartSourceContext _partSourceContext; -// private readonly NexpartService _nexpartService; +namespace PartSource.Automation.Jobs +{ + public class UpdateFitment : IAutomationJob + { + private readonly ILogger _logger; + private readonly ShopifyClient _shopifyClient; + private readonly PartSourceContext _partSourceContext; + private readonly VehicleService _vehicleService; -// private readonly VehicleService _vehicleService; + public UpdateFitment(ILogger logger, PartSourceContext partSourceContext, ShopifyClient shopifyClient, VehicleService vehicleService) + { + _logger = logger; + _partSourceContext = partSourceContext; + _shopifyClient = shopifyClient; + _vehicleService = vehicleService; + } -// public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService) -// { -// _partSourceContext = partSourceContext; -// _shopifyClient = shopifyClient; -// _vehicleService = vehicleService; -// } + public async Task Run() + { + IEnumerable products = null; -// public async Task Run() -// { -// IEnumerable products = await _shopifyClient.Products.Get(); + try + { + products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); + } -// int i = 0; + catch (Exception ex) + { + _logger.LogError("Failed to get products from Shopify", ex); + throw; + } -// while (products != null && products.Any()) -// { -// foreach (Product product in products) -// { -// try -// { -// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku && i.UpdatedAt == null); -// IList vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode); + while (products != null && products.Any()) + { + foreach (Product product in products) + { + ImportData importData = null; -// if (vehicles == null || vehicles.Count == 0) -// { -// continue; -// } + try + { + importData = await _partSourceContext.ImportData + .Where(i => i.ShopifyId == product.Id && i.UpdatedAt <= DateTime.Now.AddDays(-7)) + .FirstOrDefaultAsync(); -// await DeleteFitmentMetafields(product.Id); + if (importData == null) + { + continue; + } -// bool isFitment = false; + bool isFitment = false; -// IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); -// if (vehicleIdFitment.Count > 0) -// { -// isFitment = true; + IList vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode, 255); -// string json = JsonConvert.SerializeObject(vehicleIdFitment); -// if (json.Length >= 100000) -// { -// continue; -// } + if (vehicles.Count > 250) + { + vehicles = vehicles.Take(250) + .ToList(); -// Metafield vehicleMetafield = new Metafield -// { -// Namespace = "fitment", -// Key = "ids", -// Value = json, -// ValueType = "json_string", -// OwnerResource = "product", -// OwnerId = product.Id -// }; + _logger.LogInformation($"SKU {importData.VariantSku} fits more than 250 vehicles. Only the first 250 will be used."); + } -// await _shopifyClient.Metafields.Add(vehicleMetafield); -// } + IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); -// IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); -// if (ymmFitment.Count > 0) -// { -// isFitment = true; + if (vehicleIdFitment.Count > 0) + { + isFitment = true; -// string json = JsonConvert.SerializeObject(ymmFitment); -// if (json.Length >= 100000) -// { -// continue; -// } + string json = JsonConvert.SerializeObject(vehicleIdFitment); + if (json.Length < 100000) + { + Metafield vehicleMetafield = new Metafield + { + Namespace = "fitment", + Key = "ids", + Value = json, + ValueType = "json_string", + OwnerResource = "product", + OwnerId = product.Id + }; -// Metafield ymmMetafield = new Metafield -// { -// Namespace = "fitment", -// Key = "seo", -// Value = json, -// ValueType = "json_string", -// OwnerResource = "product", -// OwnerId = product.Id -// }; + await _shopifyClient.Metafields.Add(vehicleMetafield); + } -// await _shopifyClient.Metafields.Add(ymmMetafield); -// } + else + { + _logger.LogWarning($"Vehicle ID fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added."); + continue; + } + } -// Metafield isFitmentMetafield = new Metafield -// { -// Namespace = "Flags", -// Key = "IsFitment", -// Value = isFitment.ToString(), -// ValueType = "string", -// OwnerResource = "product", -// OwnerId = product.Id -// }; + IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); + if (ymmFitment.Count > 0) + { + isFitment = true; -// await _shopifyClient.Metafields.Add(isFitmentMetafield); + string json = JsonConvert.SerializeObject(ymmFitment); + if (json.Length < 100000) + { + Metafield ymmMetafield = new Metafield + { + Namespace = "fitment", + Key = "seo", + Value = json, + ValueType = "json_string", + OwnerResource = "product", + OwnerId = product.Id + }; -// importData.UpdatedAt = DateTime.Now; -// importData.UpdateType = "Fitment"; -// } + await _shopifyClient.Metafields.Add(ymmMetafield); + } -// catch (Exception ex) -// { -// Console.WriteLine($"{product.Id}: {ex.Message}"); -// } -// } + else + { + _logger.LogWarning($"Year/make/model fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added."); + continue; + } + } -// try -// { -// i++; -// Console.WriteLine(i); + Metafield isFitmentMetafield = new Metafield + { + Namespace = "Flags", + Key = "IsFitment", + Value = isFitment.ToString(), + ValueType = "string", + OwnerResource = "product", + OwnerId = product.Id + }; -// await _partSourceContext.SaveChangesAsync(); + await _shopifyClient.Metafields.Add(isFitmentMetafield); -// products = await _shopifyClient.Products.GetNext(); -// } + List tags = new List + { + importData.LineCode, + importData.PartNumber + }; -// catch (Exception ex) -// { -// i++; -// Console.WriteLine(i); + for (int j = 0; j < vehicleIdFitment.Count; j += 25) + { + tags.Add(string.Join('-', vehicleIdFitment.Skip(j).Take(25).Select(j => $"v{j}"))); + } -// Console.WriteLine($"Retrying request: {ex.Message}"); + tags.AddRange(ymmFitment); -// await _partSourceContext.SaveChangesAsync(); + if (tags.Count > 250) + { + tags = tags.Take(250).ToList(); + } -// products = await _shopifyClient.Products.GetNext(); -// } -// } + product.Tags = string.Join(',', tags); -// return new AutomationJobResult -// { -// IsSuccess = true -// }; -// } + await _shopifyClient.Products.Update(product); -// private async Task DeleteFitmentMetafields(long shopifyId) -// { -// IDictionary parameters = new Dictionary -// { -// { "metafield[owner_id]", shopifyId}, -// { "metafield[owner_resource]", "product" }, -// { "namespace", "fitment" }, -// }; + importData.IsFitment = isFitment; + importData.UpdatedAt = DateTime.Now; + importData.UpdateType = "Fitment"; + } -// IEnumerable metafields = await _shopifyClient.Metafields.Get(parameters); + catch (Exception ex) + { + _logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku}", ex); + } + } -// foreach (Metafield metafield in metafields) -// { -// await _shopifyClient.Metafields.Delete(metafield); -// } -// } + try + { + _partSourceContext.SaveChanges(); + products = await _shopifyClient.Products.GetNext(); + } -// public IList GetVehicles(string partNumber, string lineCode) -// { -// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); - -// //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')"; -// string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;"; - -//#pragma warning disable EF1000 // Possible SQL injection vulnerability. -// IList vehicles = _partSourceContext.VehicleData.FromSql(sql).ToList(); -//#pragma warning restore EF1000 // Possible SQL injection vulnerability. - -// return vehicles; -// } -// private void Update() -// { - -// } - -// } -//} + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get the next set of products. Retrying"); + products = await _shopifyClient.Products.GetPrevious(); + } + } + } + } +} diff --git a/PartSource.Automation/Jobs/UpdatePositioning.cs b/PartSource.Automation/Jobs/UpdatePositioning.cs index 1070672..05e9cef 100644 --- a/PartSource.Automation/Jobs/UpdatePositioning.cs +++ b/PartSource.Automation/Jobs/UpdatePositioning.cs @@ -1,6 +1,5 @@ using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; -using PartSource.Automation.Jobs.Interfaces; using PartSource.Automation.Models; using PartSource.Data; using PartSource.Data.Models; @@ -16,7 +15,7 @@ using System.Threading.Tasks; namespace PartSource.Automation.Jobs { - public class UpdatePositioning : IAutomationJob + public class UpdatePositioning// : IAutomationJob { private readonly ShopifyClient _shopifyClient; private readonly PartSourceContext _partSourceContext; diff --git a/PartSource.Automation/Jobs/UpdatePricing.cs b/PartSource.Automation/Jobs/UpdatePricing.cs index c687d1b..df4a0f4 100644 --- a/PartSource.Automation/Jobs/UpdatePricing.cs +++ b/PartSource.Automation/Jobs/UpdatePricing.cs @@ -43,7 +43,8 @@ namespace PartSource.Automation.Jobs catch (Exception ex) { _logger.LogError(ex, "Failed to get the initial set of products from Shopify."); - return; + + throw; } while (products != null && products.Any()) @@ -60,7 +61,6 @@ namespace PartSource.Automation.Jobs continue; } - if (product.Variants[0].Price != partPrice.Your_Price.Value || product.Variants[0].CompareAtPrice != partPrice.Compare_Price.Value) { product.Variants[0].Price = partPrice.Your_Price.Value; diff --git a/PartSource.Automation/Models/Enums/SeoDataType.cs b/PartSource.Automation/Models/Enums/SeoDataType.cs new file mode 100644 index 0000000..2e5305b --- /dev/null +++ b/PartSource.Automation/Models/Enums/SeoDataType.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace PartSource.Automation.Models.Enums +{ + public enum SeoDataType + { + Fitment, + Vehicle, + Image, + Prodfeed + } +} diff --git a/PartSource.Automation/PartSource.Automation.csproj b/PartSource.Automation/PartSource.Automation.csproj index 76850a6..4945950 100644 --- a/PartSource.Automation/PartSource.Automation.csproj +++ b/PartSource.Automation/PartSource.Automation.csproj @@ -18,11 +18,11 @@ + - diff --git a/PartSource.Automation/Program.cs b/PartSource.Automation/Program.cs index 8299c9d..1dbcc33 100644 --- a/PartSource.Automation/Program.cs +++ b/PartSource.Automation/Program.cs @@ -4,19 +4,12 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using PartSource.Automation.Factories; using PartSource.Automation.Jobs; -using PartSource.Automation.Jobs.Interfaces; -using PartSource.Automation.Models; -using PartSource.Automation.Models.Configuration; using PartSource.Automation.Services; using PartSource.Data; using PartSource.Data.AutoMapper; -using PartSource.Services; -using Ratermania.Automation; using Ratermania.Automation.DependencyInjection; using Ratermania.Automation.Logging; -using Ratermania.Shopify; using Ratermania.Shopify.DependencyInjection; using System; using System.IO; @@ -26,7 +19,6 @@ namespace PartSource.Automation { class Program { - static async Task Main(string[] args) { using IHost host = CreateHostBuilder().Build(); @@ -61,22 +53,35 @@ namespace PartSource.Automation .AddAutomation(options => { - options.HasBaseInterval(new TimeSpan(0, 1, 0)) + options.HasBaseInterval(new TimeSpan(0, 15, 0)) .HasMaxFailures(5) - .HasJob(options => - { - options.HasInterval(new TimeSpan(0, 5, 0)); - }) - .AddApiServer(); + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))) + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + // .HasDependency() + // .StartsAt(DateTime.Today.AddHours(8)) + //) + + .HasJob(options => + options.HasInterval(new TimeSpan(24, 0, 0)) + .StartsAt(DateTime.Today.AddHours(26)) + ) + .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + .StartsAt(DateTime.Today.AddHours(27)) + .HasDependency() + ) + .AddApiServer(options => options.HasApiKey(Environment.GetEnvironmentVariable("AUTOMATION_API_KEY"))); }) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddAutoMapper(typeof(PartSourceProfile)); }) .ConfigureLogging((builder, logging) => { - logging.AddEventLog(); - - logging.AddProvider(new AutomationLoggerProvider()); + logging.ClearProviders() + .AddProvider(new AutomationLoggerProvider()); }); } } diff --git a/PartSource.Automation/Services/EmailService.cs b/PartSource.Automation/Services/EmailService.cs index 00d94da..f5fa3d1 100644 --- a/PartSource.Automation/Services/EmailService.cs +++ b/PartSource.Automation/Services/EmailService.cs @@ -1,4 +1,5 @@ -using PartSource.Automation.Models.Configuration; +using Microsoft.Extensions.Configuration; +using PartSource.Automation.Models.Configuration; using System; using System.Collections.Generic; using System.Configuration; @@ -13,9 +14,9 @@ namespace PartSource.Automation.Services { private readonly EmailConfiguration _emailConfiguration; - public EmailService(EmailConfiguration emailConfiguration) + public EmailService(IConfiguration configuration) { - _emailConfiguration = emailConfiguration; + _emailConfiguration = configuration.GetSection("emailConfiguration").Get(); } public void Send(string subject, string body) diff --git a/PartSource.Automation/Services/FtpService.cs b/PartSource.Automation/Services/FtpService.cs index 7945fe5..853f259 100644 --- a/PartSource.Automation/Services/FtpService.cs +++ b/PartSource.Automation/Services/FtpService.cs @@ -1,5 +1,7 @@ -using PartSource.Automation.Models.Configuration; +using Microsoft.Extensions.Configuration; +using PartSource.Automation.Models.Configuration; using System; +using System.Collections.Generic; using System.Configuration; using System.IO; using System.Net; @@ -13,23 +15,38 @@ namespace PartSource.Automation.Services public FtpService(FtpConfiguration ftpConfiguration) { - _ftpConfiguration = ftpConfiguration; + _ftpConfiguration = ftpConfiguration; + } + + public string[] ListFiles(string directory) + { + FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}")); + request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); + request.Method = WebRequestMethods.Ftp.ListDirectory; + + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); + using StreamReader reader = new StreamReader(response.GetResponseStream()); + + string files = reader.ReadToEnd(); + + return files.Length > 0 + ? files.Split("\r\n") + : Array.Empty(); } public void Download(string filename) { - FtpWebRequest request = (FtpWebRequest)WebRequest.Create($"{_ftpConfiguration.Url}/{filename}"); + string file = $"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}"; + + FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}")); request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); request.Method = WebRequestMethods.Ftp.DownloadFile; - using (FtpWebResponse response = (FtpWebResponse)request.GetResponse()) - { - using (Stream responseStream = response.GetResponseStream()) - using (FileStream fileStream = new FileStream($"{_ftpConfiguration.Destination}\\{filename}", FileMode.Create)) - { - responseStream.CopyTo(fileStream); - } - } + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); + using Stream responseStream = response.GetResponseStream(); + using FileStream fileStream = new FileStream($"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}", FileMode.Create); + + responseStream.CopyTo(fileStream); } } } diff --git a/PartSource.Automation/Services/SsisService.cs b/PartSource.Automation/Services/SsisService.cs index b46a2ae..6fccf2a 100644 --- a/PartSource.Automation/Services/SsisService.cs +++ b/PartSource.Automation/Services/SsisService.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using PartSource.Automation.Models.Configuration; using System.Configuration; using System.Diagnostics; +using Microsoft.Extensions.Configuration; namespace PartSource.Automation.Services { @@ -14,9 +15,9 @@ namespace PartSource.Automation.Services { private readonly SsisConfiguration _ssisConfiguration; - public SsisService(SsisConfiguration ssisConfiguration) + public SsisService(IConfiguration configuration) { - _ssisConfiguration = ssisConfiguration; + _ssisConfiguration = configuration.GetSection("ssisConfiguration").Get(); } public bool Execute(string packageName) diff --git a/PartSource.Automation/Services/WhiSeoService.cs b/PartSource.Automation/Services/WhiSeoService.cs new file mode 100644 index 0000000..3e73021 --- /dev/null +++ b/PartSource.Automation/Services/WhiSeoService.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using PartSource.Automation.Models.Configuration; +using PartSource.Automation.Models.Enums; +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Text; + +namespace PartSource.Automation.Services +{ + public class WhiSeoService + { + private readonly FtpService _ftpService; + private readonly string _connectionString; + private readonly ILogger _logger; + + public WhiSeoService(IConfiguration configuration, ILogger logger) + { + FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:WhiConfiguration").Get(); + _ftpService = new FtpService(ftpConfiguration); + + _connectionString = configuration.GetConnectionString("PartSourceDatabase"); + + _logger = logger; + } + + public void GetFiles(SeoDataType seoDataType) + { + string seoDataTypeString = seoDataType.ToString().ToLowerInvariant(); + string[] files = _ftpService.ListFiles(seoDataTypeString); + + foreach (string file in files) + { + if (file.EndsWith("csv.gz")) + { + try + { + _ftpService.Download($"{seoDataTypeString}/{file}"); + _logger.LogInformation($"Finished downloading {file}."); + } + + + catch (Exception ex) + { + _logger.LogWarning($"Failed to download {file}, quitting", ex); + throw; + } + } + } + } + + public void Truncate(SeoDataType seoDataType) + { + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + using SqlCommand command = new SqlCommand($"TRUNCATE TABLE [{seoDataType}]", connection); + command.ExecuteNonQuery(); +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + } + + public void BulkCopy(SeoDataType seoDataType, DataTable dataTable) + { + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + + using SqlBulkCopy bulk = new SqlBulkCopy(connection) + { + DestinationTableName = seoDataType.ToString(), + BulkCopyTimeout = 14400 + }; + + bulk.WriteToServer(dataTable); + } + } +} diff --git a/PartSource.Automation/appsettings.json b/PartSource.Automation/appsettings.json index 12c17aa..0c73dd1 100644 --- a/PartSource.Automation/appsettings.json +++ b/PartSource.Automation/appsettings.json @@ -10,20 +10,23 @@ "To": "tom@tomraterman.com", "SmtpHost": "localhost" }, - "ftpConfiguration": { - "Username": "ps-ftp\\$ps-ftp", - "Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579", - "Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot", - "Destination": "C:\\Partsource.Automation\\downloads" + "FtpServers": { + "AzureConfiguration": { + "Username": "ps-ftp\\$ps-ftp", + "Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579", + "Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot", + "Destination": "C:\\Partsource.Automation\\Downloads" + }, + "WhiConfiguration": { + "Username": "ctc_seo", + "Password": "be34hz64e4", + "Url": "ftp://ftp.whisolutions.com", + "Destination": "C:\\Partsource.Automation\\Downloads\\WHI" + } }, "ssisConfiguration": { "Directory": "c:\\Partsource.Automation\\ssis" }, - //"Shopify": { - // "ApiKey": "9a533dad460321c6ce8f30bf5b8691ed", - // "ApiSecret": "dc9e28365d9858e544d57ac7af43fee7", - // "ShopDomain": "dev-partsource.myshopify.com" - //} "Shopify": { "ApiKey": "88f931933b566ade1fc92c6a39f04b34", "ApiSecret": "527a3b4213c2c7ecb214728a899052df", diff --git a/PartSource.sln b/PartSource.sln index 7faa82e..7f24697 100644 --- a/PartSource.sln +++ b/PartSource.sln @@ -11,13 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PartSource.Services", "Part EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PartSource.Automation", "PartSource.Automation\PartSource.Automation.csproj", "{C85D675B-A76C-4F9C-9C57-1E063211C946}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{1631E7EC-E54D-4F3F-9800-6EE1D5B2CB48}" - ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig - EndProjectSection -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Automation", "..\shopify\Automation\Automation.csproj", "{40E3046C-7B99-4F92-8626-1EF2892DFDCD}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Also Debug|Any CPU = Also Debug|Any CPU @@ -103,24 +96,6 @@ Global {C85D675B-A76C-4F9C-9C57-1E063211C946}.Release|x64.Build.0 = Release|Any CPU {C85D675B-A76C-4F9C-9C57-1E063211C946}.Release|x86.ActiveCfg = Release|Any CPU {C85D675B-A76C-4F9C-9C57-1E063211C946}.Release|x86.Build.0 = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|Any CPU.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|x64.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|x64.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|x86.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Also Debug|x86.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|x64.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|x64.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|x86.ActiveCfg = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Debug|x86.Build.0 = Debug|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|Any CPU.Build.0 = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|x64.ActiveCfg = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|x64.Build.0 = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|x86.ActiveCfg = Release|Any CPU - {40E3046C-7B99-4F92-8626-1EF2892DFDCD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE