From 60edbee0b889dac9d7b1be067d8ba2a6e09e435f Mon Sep 17 00:00:00 2001 From: Tom Raterman Date: Thu, 17 Mar 2022 20:04:12 -0400 Subject: [PATCH] Current state, whatever that means --- PartSource.Api/appsettings.json | 15 +- .../Jobs/ExecuteSsisPackages.cs | 2 +- .../Jobs/GetNexpartMenuItems.cs | 75 +++++++++ .../Jobs/POC/FixMultipleSeoTables.cs | 65 ++++++++ .../Jobs/POC/UpdateFitmentHtml.cs | 145 ++++++++++++++++++ .../Jobs/POC/UpdateFitmentScratchpad.cs | 144 +++++++++++++++++ .../Jobs/ProcessWhiFitment.cs | 94 +++++++++--- .../Jobs/ProcessWhiVehicles.cs | 126 +++++++++++++++ PartSource.Automation/Jobs/UpdateFitment.cs | 95 ++++++++++-- .../Jobs/UpdatePositioning.cs | 68 ++++---- PartSource.Automation/Jobs/UpdatePricing.cs | 60 +++++++- .../Models/Jobs/UpdatePricingResult.cs | 19 +++ PartSource.Automation/Program.cs | 66 +++++--- .../Services/EmailService.cs | 16 +- PartSource.Automation/Services/FtpService.cs | 2 +- .../Services/WhiSeoService.cs | 74 ++++++++- PartSource.Automation/appsettings.json | 4 +- PartSource.Data/Dtos/VehicleFitmentDto.cs | 14 ++ PartSource.Data/Models/Fitment.cs | 2 +- PartSource.Data/Models/Vehicle.cs | 3 + PartSource.Services/NexpartService.cs | 68 ++++---- PartSource.Services/VehicleService.cs | 86 +++++++++-- 22 files changed, 1087 insertions(+), 156 deletions(-) create mode 100644 PartSource.Automation/Jobs/GetNexpartMenuItems.cs create mode 100644 PartSource.Automation/Jobs/POC/FixMultipleSeoTables.cs create mode 100644 PartSource.Automation/Jobs/POC/UpdateFitmentHtml.cs create mode 100644 PartSource.Automation/Jobs/POC/UpdateFitmentScratchpad.cs create mode 100644 PartSource.Automation/Jobs/ProcessWhiVehicles.cs create mode 100644 PartSource.Automation/Models/Jobs/UpdatePricingResult.cs create mode 100644 PartSource.Data/Dtos/VehicleFitmentDto.cs diff --git a/PartSource.Api/appsettings.json b/PartSource.Api/appsettings.json index d92f19a..5d9e49f 100644 --- a/PartSource.Api/appsettings.json +++ b/PartSource.Api/appsettings.json @@ -3,17 +3,22 @@ "PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;", "FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true" }, - "Logging": { - "LogLevel": { - "Default": "Warning" - } - }, + "Logging": { + "LogLevel": { + "Default": "Warning" + } + }, "AllowedHosts": "*", "Nexpart": { "ApiKey": "AFA5CC1431CD43DCA663C17BDAD850BB-C8F5BDF9A09C4464A66731CA4427CBB9", "ApiSecret": "7AB35766-0379-4DF0-9665-DAEE13822F43", "Url": "http://acespssint.nexpart.com:4001/partselect/1.0/services/PartSelectService.PartSelectHttpSoap11Endpoint/" }, + //"Shopify": { + // "ApiKey": "9a533dad460321c6ce8f30bf5b8691ed", + // "ApiSecret": "dc9e28365d9858e544d57ac7af43fee7", + // "ShopDomain": "dev-partsource.myshopify.com" + //} "Shopify": { "ApiKey": "9a533dad460321c6ce8f30bf5b8691ed", "ApiSecret": "dc9e28365d9858e544d57ac7af43fee7", diff --git a/PartSource.Automation/Jobs/ExecuteSsisPackages.cs b/PartSource.Automation/Jobs/ExecuteSsisPackages.cs index 8a33ba6..6472727 100644 --- a/PartSource.Automation/Jobs/ExecuteSsisPackages.cs +++ b/PartSource.Automation/Jobs/ExecuteSsisPackages.cs @@ -15,7 +15,7 @@ namespace PartSource.Automation.Jobs private readonly ILogger _logger; // TODO: set from config - private readonly string[] _ssisPackages = { "Parts Availability", "Parts Price" }; + private readonly string[] _ssisPackages = { "Parts Availability" }; public ExecuteSsisPackages(IConfiguration configuration, SsisService ssisService, ILogger logger) { diff --git a/PartSource.Automation/Jobs/GetNexpartMenuItems.cs b/PartSource.Automation/Jobs/GetNexpartMenuItems.cs new file mode 100644 index 0000000..12b14d7 --- /dev/null +++ b/PartSource.Automation/Jobs/GetNexpartMenuItems.cs @@ -0,0 +1,75 @@ +using PartSource.Data.Nexpart; +using PartSource.Services; +using Ratermania.Automation.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs +{ + public class GetNexpartMenuItems : IAutomationJob + { + private readonly NexpartService _nexpartService; + + public GetNexpartMenuItems(NexpartService nexpartService) + { + _nexpartService = nexpartService; + } + + public async Task Run() + { + IList rows = new List + { + "\"Level 1\", \"Level 2\", \"Level 3\", \"Menu ID\"" + }; + + MenuNodesLookup menuNodesLookup = new MenuNodesLookup + { + MenuId = 1, + NumberOfLevels = 1 + }; + + MenuNodesLookupResponse categoryResponse = await _nexpartService.SendRequest(menuNodesLookup); + + foreach (MenuNode categoryNode in categoryResponse.ResponseBody.MenuNode) + { + rows.Add($"\"{categoryNode.Description}\",\"\",\"\",{categoryNode.Id}"); + + MenuNodesLookup subgroupLookup = new MenuNodesLookup + { + MenuId = 1, + NumberOfLevels = 1, + ParentMenuNodeId = categoryNode.Id + }; + + MenuNodesLookupResponse subgroupResponse = await _nexpartService.SendRequest(subgroupLookup); + + foreach (MenuNode subgroupNode in subgroupResponse.ResponseBody.MenuNode) + { + rows.Add($"\"{categoryNode.Description}\",\"{subgroupNode.Description}\",\"\",{subgroupNode.Id}"); + + MenuNodesLookup thirdLookup = new MenuNodesLookup + { + MenuId = 1, + NumberOfLevels = 1, + ParentMenuNodeId = subgroupNode.Id + }; + + MenuNodesLookupResponse thirdResponse = await _nexpartService.SendRequest(thirdLookup); + + foreach (MenuNode thirdNode in thirdResponse.ResponseBody.MenuNode) + { + rows.Add($"\"{categoryNode.Description}\",\"{subgroupNode.Description}\",\"{thirdNode.Description}\",{thirdNode.Id}"); + } + + } + } + + await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\Partsource Menu Items.csv", rows); + + ; + } + } +} diff --git a/PartSource.Automation/Jobs/POC/FixMultipleSeoTables.cs b/PartSource.Automation/Jobs/POC/FixMultipleSeoTables.cs new file mode 100644 index 0000000..7771d10 --- /dev/null +++ b/PartSource.Automation/Jobs/POC/FixMultipleSeoTables.cs @@ -0,0 +1,65 @@ +using Ratermania.Automation.Interfaces; +using Ratermania.Shopify; +using Ratermania.Shopify.Resources; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs.POC +{ + public class FixMultipleSeoTables : IAutomationJob + { + private readonly ShopifyClient _shopifyClient; + + public FixMultipleSeoTables(ShopifyClient shopifyClient) + { + _shopifyClient = shopifyClient; + } + + public async Task Run() + { + IEnumerable products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 }, { "product_type", "CA112-SC137-FL13750_Intake Manifolds" } }); + + while (products != null && products.Any()) + { + foreach (Product product in products) + { + try + { + string[] parts = product.BodyHtml.Split("
"); + + if (parts.Length > 2) + { + string ul = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("") + "".Length); + string seoData = "
" + parts[1].Substring(0, parts[1].IndexOf("") + "".Length) + "
"; + string vehicleIds = new Regex("
").Match(product.BodyHtml).Value; + + product.BodyHtml = ul + seoData + vehicleIds; + + await _shopifyClient.Products.Update(product); + } + } + + catch + { + Console.WriteLine($"Failed to update {product.Id}"); + } + } + + try + { + Console.WriteLine("Did 250"); + products = await _shopifyClient.Products.GetNext(); + } + + catch (Exception ex) + { + products = await _shopifyClient.Products.GetPrevious(); + } + } + } + } +} diff --git a/PartSource.Automation/Jobs/POC/UpdateFitmentHtml.cs b/PartSource.Automation/Jobs/POC/UpdateFitmentHtml.cs new file mode 100644 index 0000000..67ff0a5 --- /dev/null +++ b/PartSource.Automation/Jobs/POC/UpdateFitmentHtml.cs @@ -0,0 +1,145 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using PartSource.Automation.Models; +using PartSource.Data; +using PartSource.Data.Contexts; +using PartSource.Data.Models; +using PartSource.Services; +using Ratermania.Automation.Interfaces; +using Ratermania.Shopify; +using Ratermania.Shopify.Exceptions; +using Ratermania.Shopify.Resources; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs +{ + public class UpdateFitmentHtml : IAutomationJob + { + private readonly ILogger _logger; + private readonly ShopifyClient _shopifyClient; + private readonly PartSourceContext _partSourceContext; + private readonly FitmentContext _fitmentContext; + private readonly VehicleService _vehicleService; + + public UpdateFitmentHtml(ILogger logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService) + { + _logger = logger; + _partSourceContext = partSourceContext; + _fitmentContext = fitmentContext; + _shopifyClient = shopifyClient; + _vehicleService = vehicleService; + } + + public async Task Run() + { + + IEnumerable products = null; + + try + { + products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 }, { "vendor", "FRAM" } }); + } + + catch (Exception ex) + { + _logger.LogError("Failed to get products from Shopify", ex); + throw; + } + + int i = 1; + + while (products != null && products.Any()) + { + foreach (Product product in products) + { + ImportData importData = null; + + try + { + importData = await _partSourceContext.ImportData.FirstOrDefaultAsync(parts => parts.VariantSku == product.Variants[0].Sku); + + if (importData == null) + { + continue; + } + + product.BodyHtml = importData.BodyHtml; + + IList vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode); + + IList ymmFitment = _vehicleService.GetYmmFitment(vehicles); + if (ymmFitment.Count > 0) + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.AppendLine(""); + + foreach (string fitment in ymmFitment) + { + try + { + string[] parts = fitment.Split(' ', 2); + + stringBuilder.AppendLine($""); + } + + catch + { + // This is still a POC at this point. Oh well... + } + } + + stringBuilder.AppendLine("
This Part Fits
{parts[1]}{parts[0].Replace("-", ", ")}
"); + + product.BodyHtml += $"
{stringBuilder.ToString()}
"; + } + + IList vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles); + + if (vehicleIdFitment.Count > 0) + { + string vehicleIdString = string.Join('-', vehicleIdFitment.Select(j => $"v{j}")); + product.BodyHtml += $"
{vehicleIdString}
"; + } + + List tags = new List + { + importData.LineCode, + importData.PartNumber, + "zzzIsFitment=true" + }; + + product.Tags = string.Join(',', tags); + + await _shopifyClient.Products.Update(product); + } + + catch (Exception ex) + { + _logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku} - {ex.Message}", ex); + } + } + + try + { + Console.WriteLine(i); + products = await _shopifyClient.Products.GetNext(); + + i++; + } + + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get the next set of products. Retrying"); + products = await _shopifyClient.Products.GetPrevious(); + } + } + } + } +} diff --git a/PartSource.Automation/Jobs/POC/UpdateFitmentScratchpad.cs b/PartSource.Automation/Jobs/POC/UpdateFitmentScratchpad.cs new file mode 100644 index 0000000..a1ea5e4 --- /dev/null +++ b/PartSource.Automation/Jobs/POC/UpdateFitmentScratchpad.cs @@ -0,0 +1,144 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using PartSource.Automation.Models; +using PartSource.Data; +using PartSource.Data.Contexts; +using PartSource.Data.Dtos; +using PartSource.Data.Models; +using PartSource.Services; +using Ratermania.Automation.Interfaces; +using Ratermania.Shopify; +using Ratermania.Shopify.Exceptions; +using Ratermania.Shopify.Resources; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs.POC +{ + public class UpdateFitmentScratchpad : IAutomationJob + { + private readonly ILogger _logger; + private readonly ShopifyClient _shopifyClient; + private readonly PartSourceContext _partSourceContext; + private readonly FitmentContext _fitmentContext; + private readonly VehicleService _vehicleService; + + public UpdateFitmentScratchpad(ILogger logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService) + { + _logger = logger; + _partSourceContext = partSourceContext; + _fitmentContext = fitmentContext; + _shopifyClient = shopifyClient; + _vehicleService = vehicleService; + } + + public async Task Run() + { + IList productTypes = new List + { + "C172-S231-F23107_(PS) Wipers - TRICO Neoform", + "CA172-SC231-FL23106_Wiper Blades - Bosch Icon", + "CA172-SC231-FL23107_(PS) Wipers - TRICO Neoform", + "CA172-SC231-FL23109_(PS) Wipers - TRICO Tech/Exact Fit", + "CA172-SC231-FL23110_Wiper Accessories", + "CA172-SC231-FL23114_Wiper Blades - Rear", + "CA172-SC231-FL23116_(PS) Wipers - Bosch Insight (Hybrid)", + "CA172-SC231-FL23117_(PS) Wipers - Bosch Clear Advantage (Beam)", + "CA172-SC231-FL23118_(PS) Wipers - Winter", + "CA172-SC231-FL23125_Wiper Blades - Bosch Areotwin" + }; + + IList csvData = new List + { + "\"Line Code\", \"Part Number\", \"Year\", \"Make\", \"Model\", \"Position\"" + }; + + foreach (string type in productTypes) + { + IEnumerable products = null; + + try + { + products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 }, { "product_type", type } }); + } + + catch (Exception ex) + { + _logger.LogError("Failed to get products from Shopify", ex); + throw; + } + + + + while (products != null && products.Any()) + { + foreach (Product product in products) + { + ImportData importData = null; + + try + { + IEnumerable metafields = await _shopifyClient.Metafields.Get(new Dictionary { { "metafield[owner_id]", product.Id }, { "metafield[owner_resource]", "product" } }); + + importData = new ImportData + { + LineCode = metafields.FirstOrDefault(m => m.Key == "custom_label_0").Value ?? string.Empty, + PartNumber = product.Title.Split(' ')[0], + VariantSku = product.Variants[0].Sku // They know we can't do fitment for variants + }; + + string csvRow = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("") + "".Length); + + IList vehicles = _vehicleService.GetVehicleFitmentForPart(importData.PartNumber, importData.LineCode); + + if (vehicles.Count > 0) + { + foreach (VehicleFitmentDto vehicle in vehicles) + { + try + { + csvData.Add($"\"{importData.LineCode}\", \"{importData.PartNumber}\", \"{vehicle.Vehicle.Year}\", \"{vehicle.Vehicle.MakeName}\", \"{vehicle.Vehicle.ModelName}\", \"{vehicle.Fitment.Position}\""); + } + + catch + { + // This is still a POC at this point. Oh well... + } + } + + + } + } + + catch (Exception ex) + { + _logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku} - {ex.Message}", ex); + } + } + + try + { + Console.WriteLine($"Did an iteration of {type}"); + products = await _shopifyClient.Products.GetNext(); + } + + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get the next set of products. Retrying"); + products = await _shopifyClient.Products.GetPrevious(); + } + } + } + + await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\Wiper Fitment.csv", csvData); + ; + } + } +} diff --git a/PartSource.Automation/Jobs/ProcessWhiFitment.cs b/PartSource.Automation/Jobs/ProcessWhiFitment.cs index 2ab08f0..43bdae6 100644 --- a/PartSource.Automation/Jobs/ProcessWhiFitment.cs +++ b/PartSource.Automation/Jobs/ProcessWhiFitment.cs @@ -11,6 +11,7 @@ using System.Data; using System.IO; using System.IO.Compression; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -24,6 +25,8 @@ namespace PartSource.Automation.Jobs private readonly FtpConfiguration _ftpConfiguration; private readonly SeoDataType _seoDataType; + private readonly IDictionary _noteDictionary; + public ProcessWhiFitment(IConfiguration configuration, ILogger logger, WhiSeoService whiSeoService) { _logger = logger; @@ -32,48 +35,68 @@ namespace PartSource.Automation.Jobs _seoDataType = SeoDataType.Fitment; _ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get(); + + _noteDictionary = new ConcurrentDictionary(); } public async Task Run() { - _whiSeoService.Truncate(); - _whiSeoService.GetFiles(_seoDataType); + _whiSeoService.TruncateFitmentTables(); + // _whiSeoService.GetFiles(_seoDataType); string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant()); DirectoryInfo directoryInfo = new DirectoryInfo(directory); - IEnumerable> fileGroups = directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()); + ConcurrentQueue> fileGroups = new ConcurrentQueue>(); - foreach (IGrouping fileGroup in fileGroups) + foreach (IGrouping fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last())) { - foreach (FileInfo fileInfo in fileGroup) + fileGroups.Enqueue(fileGroup); + } + + while (fileGroups.Count > 0) + { + Parallel.For(0, 2, i => { - try + bool result = fileGroups.TryDequeue(out IGrouping fileGroup); + + if (!result) { - string filename = Decompress(fileInfo); - string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); + return; - DataTable dataTable = GetDataTable(filename); - - _whiSeoService.BulkCopy(_seoDataType, dataTable, tableName); - _logger.LogInformation($"Copied {fileInfo.Name} to the database."); - - File.Delete(filename); } - catch (Exception ex) + foreach (FileInfo fileInfo in fileGroup) { - _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); + try + { + string filename = Decompress(fileInfo); + string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); + + DataTable dataTable = GetDataTable(filename); + + _whiSeoService.BulkCopyFitment(dataTable, tableName); + _logger.LogInformation($"Copied {fileInfo.Name} to the database."); + + File.Delete(filename); + } + + catch (Exception ex) + { + _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); + } } - } - string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.')); - _whiSeoService.CreateFitmentTable(fitmentTable); + string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.')); + _whiSeoService.CreateFitmentTable(fitmentTable); - _logger.LogInformation($"Created fitment table for part group {fitmentTable}."); + _logger.LogInformation($"Created fitment table for part group {fitmentTable}."); + }); } _whiSeoService.CreateFitmentView(); + + _whiSeoService.SaveNotes(_noteDictionary); } public string Decompress(FileInfo fileInfo) @@ -96,7 +119,7 @@ namespace PartSource.Automation.Jobs dataTable.Columns.Add("BaseVehicleId", typeof(int)); dataTable.Columns.Add("EngineConfigId", typeof(int)); dataTable.Columns.Add("Position", typeof(string)); - dataTable.Columns.Add("NoteText", typeof(string)); + dataTable.Columns.Add("FitmentNoteHash", typeof(string)); using StreamReader reader = new StreamReader(filename); string line = reader.ReadLine(); // Burn the header row @@ -112,20 +135,45 @@ namespace PartSource.Automation.Jobs } string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim(); - string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim(); + string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9\\-]", string.Empty).Trim(); string position = columns[7].Trim(); + string noteText = columns[4].Trim(); + string noteTextHash = GetMD5Hash(noteText); + + if (!_noteDictionary.ContainsKey(noteTextHash)) + { + _noteDictionary.Add(noteTextHash, noteText); + } if (!string.IsNullOrEmpty(lineCode) && !string.IsNullOrEmpty(partNumber) && int.TryParse(columns[5], out int baseVehicleId) && int.TryParse(columns[6], out int engineConfigId)) { - dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText }); + dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash }); } } return dataTable; } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")] + private string GetMD5Hash(string input) + { + using MD5 md5 = MD5.Create(); + + byte[] inputBytes = Encoding.UTF8.GetBytes(input); + byte[] hashBytes = md5.ComputeHash(inputBytes); + + StringBuilder stringBuilder = new StringBuilder(); + + for (int i = 0; i < hashBytes.Length; i++) + { + stringBuilder.Append(hashBytes[i].ToString("X2")); + } + + return stringBuilder.ToString(); + } } } \ No newline at end of file diff --git a/PartSource.Automation/Jobs/ProcessWhiVehicles.cs b/PartSource.Automation/Jobs/ProcessWhiVehicles.cs new file mode 100644 index 0000000..83ff005 --- /dev/null +++ b/PartSource.Automation/Jobs/ProcessWhiVehicles.cs @@ -0,0 +1,126 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using PartSource.Automation.Models.Configuration; +using PartSource.Automation.Models.Enums; +using PartSource.Automation.Services; +using Ratermania.Automation.Interfaces; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace PartSource.Automation.Jobs +{ + public class ProcessWhiVehicles : IAutomationJob + { + private readonly ILogger _logger; + private readonly WhiSeoService _whiSeoService; + private readonly FtpConfiguration _ftpConfiguration; + private readonly SeoDataType _seoDataType; + + public ProcessWhiVehicles(IConfiguration configuration, ILogger logger, WhiSeoService whiSeoService) + { + _logger = logger; + _whiSeoService = whiSeoService; + + _seoDataType = SeoDataType.Vehicle; + + _ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get(); + + } + + public async Task Run() + { + _whiSeoService.TruncateVehicleTable(); + _whiSeoService.GetFiles(_seoDataType); + + string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant()); + DirectoryInfo directoryInfo = new DirectoryInfo(directory); + + IEnumerable files = directoryInfo.GetFiles().Where(f => f.Name.StartsWith("seo_aces_vehicle_feed")); + + foreach (FileInfo fileInfo in files) + { + try + { + string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); + + DataTable dataTable = GetDataTable(fileInfo.FullName); + + _whiSeoService.BulkCopyVehicle(dataTable, tableName); + _logger.LogInformation($"Copied {fileInfo.Name} to the database."); + + File.Delete(fileInfo.FullName); + } + + catch (Exception ex) + { + _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); + } + } + + _whiSeoService.CreateVehicleTable(); + + _logger.LogInformation($"Created vehicle table."); + } + + private DataTable GetDataTable(string filename) + { + using DataTable dataTable = new DataTable(); + dataTable.Columns.Add("Year", typeof(int)); + dataTable.Columns.Add("MakeId", typeof(int)); + dataTable.Columns.Add("MakeName", typeof(string)); + dataTable.Columns.Add("ModelId", typeof(int)); + dataTable.Columns.Add("ModelName", typeof(string)); + dataTable.Columns.Add("EngineConfigId", typeof(int)); + dataTable.Columns.Add("EngineDescription", typeof(string)); + dataTable.Columns.Add("BaseVehicleId", typeof(int)); + dataTable.Columns.Add("VehicleToEngineConfigId", typeof(int)); + dataTable.Columns.Add("SubmodelId", typeof(int)); + dataTable.Columns.Add("SubmodelName", typeof(string)); + + using StreamReader reader = new StreamReader(filename); + string line = reader.ReadLine(); // Burn the header row + + while (reader.Peek() > 0) + { + line = reader.ReadLine(); + + string[] columns = line.Split("\",\""); + for (int i = 0; i < columns.Length; i++) + { + columns[i] = columns[i].Replace("\"", string.Empty); + } + + string makeName = columns[4].Trim(); + string modelName = columns[6].Trim(); + string submodelName = columns[34].Trim(); + string engineDescription = columns[51].Trim(); + + if (!string.IsNullOrEmpty(makeName) + && !string.IsNullOrEmpty(modelName) + && !string.IsNullOrEmpty(submodelName) + && !string.IsNullOrEmpty(engineDescription) + && int.TryParse(columns[0], out int baseVehicleId) + && int.TryParse(columns[2], out int year) + && int.TryParse(columns[3], out int makeId) + && int.TryParse(columns[5], out int modelId) + && int.TryParse(columns[33], out int submodelId) + && int.TryParse(columns[35], out int engineConfigId) + && int.TryParse(columns[36], out int vehicleToEngineConfigId)) + { + dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName }); + } + } + + return dataTable; + } + } +} \ No newline at end of file diff --git a/PartSource.Automation/Jobs/UpdateFitment.cs b/PartSource.Automation/Jobs/UpdateFitment.cs index e820fd5..d6c39ee 100644 --- a/PartSource.Automation/Jobs/UpdateFitment.cs +++ b/PartSource.Automation/Jobs/UpdateFitment.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Generic; using System.Data; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -38,14 +39,21 @@ namespace PartSource.Automation.Jobs public async Task Run() { - IEnumerable products = null; - - try + IList productTypes = new List { - products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } });//, {"product_type", "CA142-SC130-FL13029_Certified Brake Pads" } }); + "CA108-SC349-FL34907_CV Shafts, New" + }; + + foreach (string type in productTypes) + { + IEnumerable products = null; + + try + { + products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 }, { "product_type", "CA108-SC349-FL34907_CV Shafts, New" } }); } - catch (Exception ex) + catch (Exception ex) { _logger.LogError("Failed to get products from Shopify", ex); throw; @@ -61,14 +69,25 @@ namespace PartSource.Automation.Jobs try { - importData = await _partSourceContext.ImportData.FirstOrDefaultAsync(parts => parts.ShopifyId == product.Id); + IEnumerable metafields = await _shopifyClient.Metafields.Get(new Dictionary { { "metafield[owner_id]", product.Id }, { "metafield[owner_resource]", "product" } }); - if (importData == null) + //importData = await _partSourceContext.ImportData.FirstOrDefaultAsync(parts => parts.ShopifyId == product.Id); + + //if (importData == null) + //{ + // continue; + importData = new ImportData { - continue; - } + LineCode = metafields.FirstOrDefault(m => m.Key == "custom_label_0").Value ?? string.Empty, + PartNumber = product.Title.Split(' ')[0], + VariantSku = product.Variants[0].Sku // They know we can't do fitment for variants + }; + // } + + //importData.PartNumber = product.Title.Split(' ')[0]; bool isFitment = false; + string bodyHtml = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("") + "".Length); IList vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode); @@ -84,6 +103,10 @@ namespace PartSource.Automation.Jobs if (vehicleIdFitment.Count > 0) { + string vehicleIdString = string.Join('-', vehicleIdFitment.Select(j => $"v{j}")); + + bodyHtml += $"
{vehicleIdString}
"; + isFitment = true; string json = JsonConvert.SerializeObject(vehicleIdFitment); @@ -114,6 +137,28 @@ namespace PartSource.Automation.Jobs { isFitment = true; + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.AppendLine(""); + + foreach (string fitment in ymmFitment) + { + try + { + string[] parts = fitment.Split(' ', 2); + + stringBuilder.AppendLine($""); + } + + catch + { + // This is still a POC at this point. Oh well... + } + } + + stringBuilder.AppendLine("
This Part Fits
{parts[1]}{parts[0].Replace("-", ", ")}
"); + + bodyHtml += $"
{stringBuilder.ToString()}
"; + string json = JsonConvert.SerializeObject(ymmFitment); if (json.Length < 100000) { @@ -149,12 +194,32 @@ namespace PartSource.Automation.Jobs await _shopifyClient.Metafields.Add(isFitmentMetafield); - List tags = new List + Metafield lineCodeMetafield = new Metafield { - importData.LineCode, - importData.PartNumber + Namespace = "google", + Key = "custom_label_0", + Value = importData.LineCode, + ValueType = "string", + OwnerResource = "product", + OwnerId = product.Id }; + //await _shopifyClient.Metafields.Add(lineCodeMetafield); + + Metafield partNumberMetafield = new Metafield + { + Namespace = "google", + Key = "custom_label_1", + Value = importData.PartNumber, + ValueType = "string", + OwnerResource = "product", + OwnerId = product.Id + }; + + // await _shopifyClient.Metafields.Add(partNumberMetafield); + + List tags = new List(); + for (int j = 0; j < vehicleIdFitment.Count; j += 25) { tags.Add(string.Join('-', vehicleIdFitment.Skip(j).Take(25).Select(j => $"v{j}"))); @@ -173,8 +238,8 @@ namespace PartSource.Automation.Jobs tags.Add(zzzIsFitment); - product.Tags = string.Join(',', tags); - + //product.Tags = string.Join(',', tags); + product.BodyHtml = bodyHtml; await _shopifyClient.Products.Update(product); importData.IsFitment = isFitment; @@ -205,5 +270,7 @@ namespace PartSource.Automation.Jobs } } } + ; } } +} diff --git a/PartSource.Automation/Jobs/UpdatePositioning.cs b/PartSource.Automation/Jobs/UpdatePositioning.cs index 6709710..505eddf 100644 --- a/PartSource.Automation/Jobs/UpdatePositioning.cs +++ b/PartSource.Automation/Jobs/UpdatePositioning.cs @@ -47,7 +47,14 @@ namespace PartSource.Automation.Jobs { try { - ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku); + IEnumerable metafields = await _shopifyClient.Metafields.Get(new Dictionary { { "metafield[owner_id]", product.Id }, { "metafield[owner_resource]", "product" } }); + + ImportData importData = new ImportData + { + LineCode = metafields.FirstOrDefault(m => m.Key == "custom_label_0").Value ?? string.Empty, + PartNumber = product.Title.Split(' ')[0], + VariantSku = product.Variants[0].Sku // They know we can't do fitment for variants + }; if (importData == null || importData?.LineCode == "SVG") // Headlights go in front, DUH { @@ -87,40 +94,41 @@ namespace PartSource.Automation.Jobs await SavePositionMetafield(product, vehicleIds, currentPosition); - IList notes = fitments.Select(f => f.NoteText) - .Distinct() - .ToList(); + //IList notes = fitments.Select(f => f.NoteText) - IList vehicleNotes = new List(); + // .Distinct() + // .ToList(); - foreach (string noteText in notes) - { - vehicleIds = fitments.Where(f => f.NoteText == noteText) - .Select(f => new { f.EngineConfigId, f.BaseVehicleId }) - .SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId)) - .Select(v => v.VehicleToEngineConfigId) - .ToList(); + //IList vehicleNotes = new List(); - vehicleNotes.Add(new { noteText, vehicleIds }); - } + //foreach (string noteText in notes) + //{ + // vehicleIds = fitments.Where(f => f.NoteText == noteText) + // .Select(f => new { f.EngineConfigId, f.BaseVehicleId }) + // .SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId)) + // .Select(v => v.VehicleToEngineConfigId) + // .ToList(); - string json = JsonConvert.SerializeObject(vehicleNotes); - if (json.Length >= 100000) - { - continue; - } + // vehicleNotes.Add(new { noteText, vehicleIds }); + //} - Metafield vehicleMetafield = new Metafield - { - Namespace = "fitment", - Key = "note_text", - Value = json, - ValueType = "json_string", - OwnerResource = "product", - OwnerId = product.Id - }; + //string json = JsonConvert.SerializeObject(vehicleNotes); + //if (json.Length >= 100000) + //{ + // continue; + //} - await _shopifyClient.Metafields.Add(vehicleMetafield); + //Metafield vehicleMetafield = new Metafield + //{ + // Namespace = "fitment", + // Key = "note_text", + // Value = json, + // ValueType = "json_string", + // OwnerResource = "product", + // OwnerId = product.Id + //}; + + //await _shopifyClient.Metafields.Add(vehicleMetafield); //importData.UpdatedAt = DateTime.Now; //importData.UpdateType = "Positioning"; @@ -148,7 +156,7 @@ namespace PartSource.Automation.Jobs private IList GetPositionOrderedFitments(string partNumber, string lineCode) { - partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); + partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]\\-", string.Empty); IQueryable whiCodes = _fitmentContext.DcfMappings .Where(d => d.LineCode == lineCode) diff --git a/PartSource.Automation/Jobs/UpdatePricing.cs b/PartSource.Automation/Jobs/UpdatePricing.cs index e38c2b5..442d78a 100644 --- a/PartSource.Automation/Jobs/UpdatePricing.cs +++ b/PartSource.Automation/Jobs/UpdatePricing.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using PartSource.Automation.Models.Jobs; using PartSource.Automation.Services; using PartSource.Data.Contexts; using PartSource.Data.Models; @@ -9,7 +10,9 @@ using Ratermania.Shopify.Resources; using Ratermania.Shopify.Resources.Enums; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Mail; using System.Threading.Tasks; namespace PartSource.Automation.Jobs @@ -31,15 +34,14 @@ namespace PartSource.Automation.Jobs public async Task Run() { + List pricingReport = new List(); IEnumerable products = null; IEnumerable prices = null; - int updateCount = 0; try { products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); - - prices = await _partSourceContext.PartPrices.ToListAsync(); + prices = await _partSourceContext.PartPrices.AsNoTracking().ToListAsync(); } catch (Exception ex) @@ -53,6 +55,8 @@ namespace PartSource.Automation.Jobs { foreach (Product product in products) { + List productPricingUpdate = new List(); + if (product.Variants.Length > 0) { bool hasUpdate = false; @@ -69,6 +73,15 @@ namespace PartSource.Automation.Jobs if (product.Variants[i].Price.ToString("G29") != partPrice.Your_Price.Value.ToString("G29") || product.Variants[i].CompareAtPrice.ToString("G29") != partPrice.Compare_Price.Value.ToString("G29")) { + productPricingUpdate.Add(new UpdatePricingResult + { + Sku = variant.Sku, + OldPrice = product.Variants[i].Price, + NewPrice = partPrice.Your_Price.Value, + OldCompareAt = product.Variants[i].CompareAtPrice, + NewCompareAt = partPrice.Compare_Price.Value + }); + product.Variants[i].Price = partPrice.Your_Price.Value; product.Variants[i].CompareAtPrice = partPrice.Compare_Price.Value; @@ -95,8 +108,8 @@ namespace PartSource.Automation.Jobs { //await _shopifyClient.Metafields.Add(metafield); await _shopifyClient.Products.Update(product); - - updateCount++; + + pricingReport.AddRange(productPricingUpdate); } catch (Exception ex) @@ -111,7 +124,7 @@ namespace PartSource.Automation.Jobs { products = await _shopifyClient.Products.GetNext(); - _logger.LogInformation($"Total updated: {updateCount}"); + _logger.LogInformation($"Total updated: {pricingReport.Count}"); } catch (Exception ex) @@ -121,7 +134,40 @@ namespace PartSource.Automation.Jobs } } - _emailService.Send("Pricing Update Completed", $"The pricing update has completed. Total updated: {updateCount}"); + Attachment attachment = GetPricingReportAttachment(pricingReport); + + _emailService.Send("Pricing Update Completed", $"The pricing update has completed. Total updated: {pricingReport.Count}", attachment); + } + + private Attachment GetPricingReportAttachment(IList pricingReport) + { + string directory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "Pricing Reports"); + string filename = Path.Combine(directory, $"Pricing Update {DateTime.Now.ToString("yyyy-MM-dd")}.csv"); + + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + if (File.Exists(filename)) + { + File.Delete(filename); + } + + using FileStream fileStream = File.OpenWrite(filename); + using StreamWriter streamWriter = new StreamWriter(fileStream, System.Text.Encoding.UTF8); + + streamWriter.WriteLine("SKU, Old Price, New Price, Old Compare At, New Compare At"); + + foreach (UpdatePricingResult pricingResult in pricingReport) + { + streamWriter.WriteLine($"{pricingResult.Sku},{pricingResult.OldPrice},{pricingResult.NewPrice},{pricingResult.OldCompareAt},{pricingResult.NewCompareAt}"); + } + + streamWriter.Close(); + fileStream.Close(); + + return new Attachment(filename); } } } \ No newline at end of file diff --git a/PartSource.Automation/Models/Jobs/UpdatePricingResult.cs b/PartSource.Automation/Models/Jobs/UpdatePricingResult.cs new file mode 100644 index 0000000..2a9101e --- /dev/null +++ b/PartSource.Automation/Models/Jobs/UpdatePricingResult.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace PartSource.Automation.Models.Jobs +{ + public class UpdatePricingResult + { + public string Sku { get; set; } + + public decimal OldPrice { get; set; } + + public decimal NewPrice { get; set; } + + public decimal OldCompareAt { get; set; } + + public decimal NewCompareAt { get; set; } + } +} diff --git a/PartSource.Automation/Program.cs b/PartSource.Automation/Program.cs index 0e06edb..c54bec5 100644 --- a/PartSource.Automation/Program.cs +++ b/PartSource.Automation/Program.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using PartSource.Automation.Jobs; +using PartSource.Automation.Jobs.POC; using PartSource.Automation.Services; using PartSource.Data; using PartSource.Data.AutoMapper; @@ -21,8 +22,7 @@ namespace PartSource.Automation { class Program { - static async Task Main(string[] args) - { + static async Task Main(string[] args){ try { using IHost host = CreateHostBuilder().Build(); @@ -55,7 +55,11 @@ namespace PartSource.Automation ) .AddDbContext(options => - options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts => opts.EnableRetryOnFailure()) + options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts => + { + opts.EnableRetryOnFailure(); + opts.CommandTimeout(600); + }) ) .AddShopify(options => @@ -64,33 +68,44 @@ namespace PartSource.Automation options.ApiSecret = builder.Configuration["Shopify:ApiSecret"]; options.ApiVersion = "2020-01"; options.ShopDomain = builder.Configuration["Shopify:ShopDomain"]; + + //options.ApiKey = "9a533dad460321c6ce8f30bf5b8691ed"; + //options.ApiSecret = "dc9e28365d9858e544d57ac7af43fee7"; + //options.ApiVersion = "2020-01"; + //options.ShopDomain = "dev-partsource.myshopify.com"; }) .AddAutomation(options => { options.HasBaseInterval(new TimeSpan(0, 15, 0)) - .HasMaxFailures(5) - //.HasJob(options => options.HasInterval(new TimeSpan(7, 0, 0, 0))); - // - //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))) - .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - // .HasDependency() - // .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - // .HasDependency() - //.HasDependency() - // .StartsAt(DateTime.Today.AddHours(8)) - //) ; - //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - // .StartsAt(DateTime.Parse("2021-04-01 08:00:00")) - //) - // .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - // .StartsAt(DateTime.Today.AddHours(26)) - // ) - // .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - // .HasDependency() - // .StartsAt(DateTime.Today.AddHours(27) - // ) - ); + .HasMaxFailures(1) + //.HasJob(options => options.HasInterval(new TimeSpan(7, 0, 0, 0))); + // + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))) + // .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + //.HasDependency() + .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + // .HasDependency() + // .HasDependency() + // .HasDependency() + // .StartsAt(DateTime.Today.AddHours(8)) + //) ; + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + // .StartsAt(DateTime.Parse("2021-04-01 08:00:00")) + //) + //.HasJob(options => + // options.HasInterval(new TimeSpan(24, 0, 0)) + // .StartsAt(DateTime.Today.AddHours(5)) + // ); + //.StartsAt(DateTime.Today.AddHours(26)) + //) + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + //.HasDependency() + //.StartsAt(DateTime.Today.AddHours(27) + //) + //); //.AddApiServer(); }) @@ -98,6 +113,7 @@ namespace PartSource.Automation .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddAutoMapper(typeof(PartSourceProfile)); diff --git a/PartSource.Automation/Services/EmailService.cs b/PartSource.Automation/Services/EmailService.cs index 44ae76e..e407793 100644 --- a/PartSource.Automation/Services/EmailService.cs +++ b/PartSource.Automation/Services/EmailService.cs @@ -3,6 +3,7 @@ using PartSource.Automation.Models.Configuration; using System; using System.Collections.Generic; using System.Configuration; +using System.IO; using System.Linq; using System.Net.Mail; using System.Text; @@ -19,7 +20,7 @@ namespace PartSource.Automation.Services _emailConfiguration = configuration.GetSection("emailConfiguration").Get(); } - public void Send(string subject, string body) + public void Send(string subject, string body, Attachment attachment = null) { using SmtpClient smtpClient = new SmtpClient { @@ -31,15 +32,20 @@ namespace PartSource.Automation.Services From = new MailAddress(_emailConfiguration.From), Subject = subject, Body = body, - IsBodyHtml = true + IsBodyHtml = true, }; + if (attachment != null) + { + mailMessage.Attachments.Add(attachment); + } + foreach (string address in _emailConfiguration.To.Split(',')) { mailMessage.To.Add(address); } - // smtpClient.Send(mailMessage); + smtpClient.Send(mailMessage); } public void Send(string to, string subject, string body) @@ -54,7 +60,7 @@ namespace PartSource.Automation.Services From = new MailAddress(_emailConfiguration.From), Subject = subject, Body = body, - IsBodyHtml = false + IsBodyHtml = false, }; foreach (string address in _emailConfiguration.To.Split(',')) @@ -62,7 +68,7 @@ namespace PartSource.Automation.Services mailMessage.To.Add(to); } - // smtpClient.Send(mailMessage); + smtpClient.Send(mailMessage); } } } diff --git a/PartSource.Automation/Services/FtpService.cs b/PartSource.Automation/Services/FtpService.cs index 853f259..25ed55a 100644 --- a/PartSource.Automation/Services/FtpService.cs +++ b/PartSource.Automation/Services/FtpService.cs @@ -41,7 +41,7 @@ namespace PartSource.Automation.Services FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}")); request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); request.Method = WebRequestMethods.Ftp.DownloadFile; - + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); using Stream responseStream = response.GetResponseStream(); using FileStream fileStream = new FileStream($"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}", FileMode.Create); diff --git a/PartSource.Automation/Services/WhiSeoService.cs b/PartSource.Automation/Services/WhiSeoService.cs index b194220..d22dc3e 100644 --- a/PartSource.Automation/Services/WhiSeoService.cs +++ b/PartSource.Automation/Services/WhiSeoService.cs @@ -35,7 +35,7 @@ namespace PartSource.Automation.Services foreach (string file in files) { - if (file.EndsWith("csv.gz")) + if (file.Contains(".csv")) { try { @@ -53,7 +53,16 @@ namespace PartSource.Automation.Services } } - public void Truncate() + public void TruncateVehicleTable() + { + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + + using SqlCommand command = new SqlCommand($"truncate table dbo.Vehicle", connection); + command.ExecuteNonQuery(); + } + + public void TruncateFitmentTables() { using SqlConnection connection = new SqlConnection(_connectionString); connection.Open(); @@ -62,11 +71,36 @@ namespace PartSource.Automation.Services command.ExecuteNonQuery(); } - public void BulkCopy(SeoDataType seoDataType, DataTable dataTable, string tableName) + public void SaveNotes(IDictionary notes) + { + using DataTable dataTable = new DataTable(); + dataTable.Columns.Add("NoteText", typeof(string)); + dataTable.Columns.Add("Hash", typeof(string)); + + foreach (KeyValuePair note in notes) + { + + dataTable.Rows.Add(new string[] { note.Value, note.Key }); + } + + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + + using SqlBulkCopy bulk = new SqlBulkCopy(connection) + { + DestinationTableName = $"FitmentNote", + BulkCopyTimeout = 14400 + }; + + bulk.WriteToServer(dataTable); + } + + public void BulkCopyFitment(DataTable dataTable, string tableName) { using SqlConnection connection = new SqlConnection(_connectionString); connection.Open(); + string sql = string.Empty; using SqlCommand command = new SqlCommand($"EXEC CreateFitmentTempTable @tableName = '{tableName}'", connection); command.ExecuteNonQuery(); @@ -74,6 +108,25 @@ namespace PartSource.Automation.Services using SqlBulkCopy bulk = new SqlBulkCopy(connection) { DestinationTableName = $"FitmentTemp.{tableName}", + BulkCopyTimeout = 1 + }; + + bulk.WriteToServer(dataTable); + } + + public void BulkCopyVehicle(DataTable dataTable, string tableName) + { + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + + string sql = string.Empty; + + using SqlCommand command = new SqlCommand($"EXEC CreateVehicleTempTable @tableName = '{tableName}'", connection); + command.ExecuteNonQuery(); + + using SqlBulkCopy bulk = new SqlBulkCopy(connection) + { + DestinationTableName = $"VehicleTemp.{tableName}", BulkCopyTimeout = 14400 }; @@ -97,6 +150,21 @@ namespace PartSource.Automation.Services using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection); command.ExecuteNonQuery(); + + using SqlCommand command2 = new SqlCommand($"exec CreateFitmentIndexes", connection); + command2.ExecuteNonQuery(); + } + + public void CreateVehicleTable() + { + return; + + using SqlConnection connection = new SqlConnection(_connectionString); + connection.Open(); + + using SqlCommand command = new SqlCommand($"exec CreateVehicleTable", connection); + command.CommandTimeout = 1800; + command.ExecuteNonQuery(); } } } diff --git a/PartSource.Automation/appsettings.json b/PartSource.Automation/appsettings.json index 461dded..47ad194 100644 --- a/PartSource.Automation/appsettings.json +++ b/PartSource.Automation/appsettings.json @@ -6,6 +6,7 @@ "emailConfiguration": { "From": "alerts@ps-automation.eastus2.cloudapp.azure.com", "To": "tom@soundpress.com,Anas.Bajwa@Partsource.ca,josh@soundpress.com,alex.au@partsource.ca,michael.massara@partsource.ca", + //"To": "tom@tomraterman.com", "SmtpHost": "localhost" }, "FtpServers": { @@ -34,7 +35,8 @@ "LogLevel": { "Default": "Information", "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information" + "Microsoft.Hosting.Lifetime": "Information", + // "Microsoft.EntityFrameworkCore.Database.Command": "Information" }, "EventLog": { "LogLevel": { diff --git a/PartSource.Data/Dtos/VehicleFitmentDto.cs b/PartSource.Data/Dtos/VehicleFitmentDto.cs new file mode 100644 index 0000000..56bf211 --- /dev/null +++ b/PartSource.Data/Dtos/VehicleFitmentDto.cs @@ -0,0 +1,14 @@ +using PartSource.Data.Models; +using System; +using System.Collections.Generic; +using System.Text; + +namespace PartSource.Data.Dtos +{ + public class VehicleFitmentDto + { + public Fitment Fitment { get; set; } + + public Vehicle Vehicle { get; set; } + } +} diff --git a/PartSource.Data/Models/Fitment.cs b/PartSource.Data/Models/Fitment.cs index 3ac90a3..05f1ad4 100644 --- a/PartSource.Data/Models/Fitment.cs +++ b/PartSource.Data/Models/Fitment.cs @@ -18,6 +18,6 @@ namespace PartSource.Data.Models public string Position { get; set; } - public string NoteText { get; set; } + public string FitmentNoteHash { get; set; } } } diff --git a/PartSource.Data/Models/Vehicle.cs b/PartSource.Data/Models/Vehicle.cs index 061b16e..fe19524 100644 --- a/PartSource.Data/Models/Vehicle.cs +++ b/PartSource.Data/Models/Vehicle.cs @@ -30,5 +30,8 @@ namespace PartSource.Data.Models [Key] public int VehicleToEngineConfigId { get; set; } + + [NotMapped] + public string Position { get; set; } } } diff --git a/PartSource.Services/NexpartService.cs b/PartSource.Services/NexpartService.cs index e2c7c25..1c96908 100644 --- a/PartSource.Services/NexpartService.cs +++ b/PartSource.Services/NexpartService.cs @@ -10,34 +10,42 @@ using System.Xml.Serialization; namespace PartSource.Services { - public class NexpartService - { - public async Task SendRequest(T requestContent) - { - Envelope envelope = new Envelope(); - envelope.Body.Content = (object)(T)requestContent; - XmlSerializer serializer = new XmlSerializer(typeof(Envelope)); - StringBuilder sb = new StringBuilder(); - using (TextWriter textWriter = (TextWriter)new StringWriter(sb)) - serializer.Serialize(textWriter, (object)envelope); - U content; - using (HttpClient client = new HttpClient()) - { - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", "QUZBNUNDMTQzMUNENDNEQ0E2NjNDMTdCREFEODUwQkItQzhGNUJERjlBMDlDNDQ2NEE2NjczMUNBNDQyN0NCQjk6N0FCMzU3NjYtMDM3OS00REYwLTk2NjUtREFFRTEzODIyRjQz"); - try - { - //HttpResponseMessage response = await client.PostAsync(ConfigurationManager.AppSettings["NexpartUrl"], (HttpContent)new StringContent(sb.ToString(), Encoding.UTF8, "text/xml")); - HttpResponseMessage response = await client.PostAsync("http://acespssint.nexpart.com:4001/partselect/1.0/services/PartSelectService.PartSelectHttpSoap11Endpoint/", (HttpContent)new StringContent(sb.ToString(), Encoding.UTF8, "text/xml")); - Stream result = await response.Content.ReadAsStreamAsync(); - string str = await response.Content.ReadAsStringAsync(); - content = (U)((Envelope)serializer.Deserialize(result)).Body.Content; - } - catch (Exception ex) - { - throw; - } - } - return content; - } - } + public class NexpartService + { + public async Task SendRequest(T requestContent) + { + Envelope envelope = new Envelope(); + envelope.Body.Content = requestContent; + XmlSerializer serializer = new XmlSerializer(typeof(Envelope)); + StringBuilder sb = new StringBuilder(); + + + + using (TextWriter textWriter = new StringWriter(sb)) + { + serializer.Serialize(textWriter, (object)envelope); + U content; + + string x = textWriter.ToString(); + + using (HttpClient client = new HttpClient()) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", "QjM4ODAyMzM3QjQxNEM2QTk4M0RFMjM0Mjk4Rjk4M0UtOUIzNUUxNzNBQUYxNEE2QjhCQjI2RjZDOUY2ODk1NDU6MkMzOUVCOTYtRDBBRS00QkVBLTlCMzItMUYyNTA5MDJGQTE0"); + try + { + //HttpResponseMessage response = await client.PostAsync(ConfigurationManager.AppSettings["NexpartUrl"], (HttpContent)new StringContent(sb.ToString(), Encoding.UTF8, "text/xml")); + HttpResponseMessage response = await client.PostAsync("http://acespssprod.nexpart.com:8081/partselect/1.0/services/PartSelectService.PartSelectHttpSoap11Endpoint/", new StringContent(textWriter.ToString(), Encoding.UTF8)); + Stream result = await response.Content.ReadAsStreamAsync(); + string str = await response.Content.ReadAsStringAsync(); + content = (U)((Envelope)serializer.Deserialize(result)).Body.Content; + } + catch (Exception ex) + { + throw; + } + } + return content; + } + } + } } diff --git a/PartSource.Services/VehicleService.cs b/PartSource.Services/VehicleService.cs index bff3204..cd765e8 100644 --- a/PartSource.Services/VehicleService.cs +++ b/PartSource.Services/VehicleService.cs @@ -244,7 +244,7 @@ namespace PartSource.Services IList fitmentTags = new List(); - IList makeModels = vehicles.Select(v => $"{v.MakeName},{v.ModelName}").Distinct().ToList(); + IList makeModels = vehicles.OrderBy(v => v.MakeName).ThenBy(v => v.ModelName).Select(v => $"{v.MakeName},{v.ModelName}").Distinct().ToList(); foreach (string makeModel in makeModels) { @@ -268,24 +268,55 @@ namespace PartSource.Services return fitmentTags; } + public IList GetYmmFitmentRange(IList vehicles) + { + if (vehicles.Count == 0) + { + return new string[0]; + } + + IList fitmentTags = new List(); + + IList makeModels = vehicles.Select(v => $"{v.MakeName},{v.ModelName}").Distinct().ToList(); + + foreach (string makeModel in makeModels) + { + string make = makeModel.Split(',')[0]; + string model = makeModel.Split(',')[1]; + + int minYear = vehicles + .Where(v => v.MakeName == make && v.ModelName == model) + .Min(v => v.Year); + + int maxYear = vehicles + .Where(v => v.MakeName == make && v.ModelName == model) + .Max(v => v.Year); + + string tag = minYear == maxYear + ? $"{minYear} {make.Trim()} {model.Trim()}" + : $"{minYear}-{maxYear} {make.Trim()} {model.Trim()}"; + + System.Diagnostics.Debug.WriteLine(tag); + + fitmentTags.Add(tag); + } + + return fitmentTags; + } + public IList GetVehicleIdFitment(IList vehicles) { - return vehicles.Select(v => v.VehicleToEngineConfigId).ToArray(); + return vehicles.Select(v => v.VehicleToEngineConfigId).Distinct().ToArray(); } - public IList GetVehiclesForPart(string partNumber, string lineCode) - { - return GetVehiclesForPart(partNumber, lineCode, -1); - } - - public IList GetVehiclesForPart(string partNumber, string lineCode, int maxVehicles) + public IList GetVehiclesForPart(string partNumber, string lineCode, int maxVehicles = 0) { if (string.IsNullOrEmpty(partNumber) || string.IsNullOrEmpty(lineCode)) { return null; } - partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty); + partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9\\-]", string.Empty); IQueryable whiCodes = _fitmentContext.DcfMappings .Where(d => d.LineCode == lineCode) @@ -296,7 +327,42 @@ namespace PartSource.Services .Join(_fitmentContext.Vehicles, f => new { f.BaseVehicleId, f.EngineConfigId }, v => new { v.BaseVehicleId, v.EngineConfigId }, - (f, v) => v); + (f, v) => v) + .Distinct() + .OrderByDescending(x => x.Year); + + if (maxVehicles > 0) + { + vehicles = vehicles.Take(maxVehicles); + } + + return vehicles.ToList(); + } + + public IList GetVehicleFitmentForPart(string partNumber, string lineCode, int maxVehicles = 0) + { + if (string.IsNullOrEmpty(partNumber) || string.IsNullOrEmpty(lineCode)) + { + return null; + } + + partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9\\-]", string.Empty); + + IQueryable whiCodes = _fitmentContext.DcfMappings + .Where(d => d.LineCode == lineCode) + .Select(d => d.WhiCode); + + IQueryable vehicles = _fitmentContext.Fitments + .Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode)) + .Join(_fitmentContext.Vehicles, + f => new { f.BaseVehicleId, f.EngineConfigId }, + v => new { v.BaseVehicleId, v.EngineConfigId }, + (f, v) => new VehicleFitmentDto + { + Fitment = f, + Vehicle = v + }) + .Distinct(); if (maxVehicles > 0) {