soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fitment update

Browse Source
This commit is contained in:
Tom Raterman
2023-01-29 11:48:13 -05:00
parent b259b77967
commit ff20615481
22 changed files with 777 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
PartSource.Automation/Jobs/ExecuteSsisPackages.cs
View File

@@ -9,50 +9,49 @@ using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class ExecuteSsisPackages : IAutomationJob
{
private readonly EmailService _emailService;
private readonly FtpService _ftpService;
private readonly SsisService _ssisService;
private readonly ILogger<ExecuteSsisPackages> _logger;
public class ExecuteSsisPackages : IAutomationJob
{
private readonly EmailService _emailService;
private readonly FtpService _ftpService;
private readonly SsisService _ssisService;
private readonly ILogger<ExecuteSsisPackages> _logger;
// TODO: set from config
private readonly string[] _ssisPackages = { "Parts Price", "Parts Availability" };
// TODO: set from config
private readonly string[] _ssisPackages = {"Parts Price", "Parts Availability" };
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
_emailService = emailService;
_ftpService = new FtpService(ftpConfiguration);
_ssisService = ssisService;
_logger = logger;
}
_emailService = emailService;
_ftpService = new FtpService(ftpConfiguration);
_ssisService = ssisService;
_logger = logger;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
await Task.Run(() =>
{
foreach (string package in _ssisPackages)
{
try
{
_ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
public async Task Run(CancellationToken token, params string[] arguments)
{
await Task.Run(() =>
{
foreach (string package in _ssisPackages)
{
try
{
_ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
}
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
}
catch (Exception ex)
{
_logger.LogError($"Execution of SSIS package {package} failed.", ex);
catch (Exception ex)
{
_logger.LogError($"Execution of SSIS package {package} failed.", ex);
throw;
}
}
throw;
}
}
_emailService.Send("Database Updates Complete", "Database updates to support pricing and availability have completed successfully.");
});
}
}
// _emailService.Send("Database Updates Complete", "Database updates to support pricing and availability have completed successfully.");
});
}
}
}

8
PartSource.Automation/Jobs/GetNexpartMenuItems.cs
View File

@@ -28,7 +28,7 @@ namespace PartSource.Automation.Jobs
MenuNodesLookup menuNodesLookup = new MenuNodesLookup
{
MenuId = 2,
MenuId = 1,
NumberOfLevels = 1
};
@@ -40,7 +40,7 @@ namespace PartSource.Automation.Jobs
MenuNodesLookup subgroupLookup = new MenuNodesLookup
{
MenuId = 2,
MenuId = 1,
NumberOfLevels = 1,
ParentMenuNodeId = categoryNode.Id
};
@@ -53,7 +53,7 @@ namespace PartSource.Automation.Jobs
MenuNodesLookup thirdLookup = new MenuNodesLookup
{
MenuId = 2,
MenuId = 1,
NumberOfLevels = 1,
ParentMenuNodeId = subgroupNode.Id
};
@@ -68,7 +68,7 @@ namespace PartSource.Automation.Jobs
}
}
//await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\Partsource Menu Items.csv", rows);
await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\Partsource Menu Items.csv", rows);
;
}

66
PartSource.Automation/Jobs/POC/FixMultipleSeoTables.cs
View File

@@ -1,66 +0,0 @@
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs.POC
{
public class FixMultipleSeoTables : IAutomationJob
{
private readonly ShopifyClient _shopifyClient;
public FixMultipleSeoTables(ShopifyClient shopifyClient)
{
_shopifyClient = shopifyClient;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "product_type", "CA112-SC137-FL13750_Intake Manifolds" } });
while (products != null && products.Any())
{
foreach (Product product in products)
{
try
{
string[] parts = product.BodyHtml.Split("<div id=\"seoData\">");
if (parts.Length > 2)
{
string ul = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
string seoData = "<div id=\"seoData\">" + parts[1].Substring(0, parts[1].IndexOf("</table>") + "</table>".Length) + "</div>";
string vehicleIds = new Regex("<div id=\"vehicleIDs\".*</div>").Match(product.BodyHtml).Value;
product.BodyHtml = ul + seoData + vehicleIds;
await _shopifyClient.Products.Update(product);
}
}
catch
{
Console.WriteLine($"Failed to update {product.Id}");
}
}
try
{
Console.WriteLine("Did 250");
products = await _shopifyClient.Products.GetNext();
}
catch (Exception ex)
{
products = await _shopifyClient.Products.GetPrevious();
}
}
}
}
}

84
PartSource.Automation/Jobs/POC/ImageList.cs Normal file
View File

@@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly PartSourceContext _partSourceContext;
public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext)
{
_nexpartService = nexpartService;
_partSourceContext = partSourceContext;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> rows = new List<string> {
"\"Line Code\", \"Part Number\", \"Image URL(s)\""
};
IList<ImportData> importData = await _partSourceContext.ImportData
//.Take(5000)
.ToListAsync();
foreach (ImportData item in importData)
{
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = item.LineCode,
PartNumber = item.PartNumber
}
},
DataOption = new[] { "DIST_LINE", "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
rows.Add($"\"{item.LineCode}\", \"{item.PartNumber}\", \"{string.Join(";", urls)}\"");
}
}
}
await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\WHI Images.csv", rows);
}
}
}

146
PartSource.Automation/Jobs/POC/UpdateFitmentHtml.cs
View File

@@ -1,146 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Exceptions;
using Ratermania.Shopify.Resources;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class UpdateFitmentHtml : IAutomationJob
{
private readonly ILogger<UpdateFitmentHtml> _logger;
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
public UpdateFitmentHtml(ILogger<UpdateFitmentHtml> logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
{
_logger = logger;
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = null;
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "vendor", "FRAM" } });
}
catch (Exception ex)
{
_logger.LogError("Failed to get products from Shopify", ex);
throw;
}
int i = 1;
while (products != null && products.Any())
{
foreach (Product product in products)
{
ImportData importData = null;
try
{
importData = await _partSourceContext.ImportData.FirstOrDefaultAsync(parts => parts.VariantSku == product.Variants[0].Sku);
if (importData == null)
{
continue;
}
product.BodyHtml = importData.BodyHtml;
IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<string> ymmFitment = _vehicleService.GetYmmFitment(vehicles);
if (ymmFitment.Count > 0)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendLine("<table><tr><th colspan=\"2\">This Part Fits</th></tr>");
foreach (string fitment in ymmFitment)
{
try
{
string[] parts = fitment.Split(' ', 2);
stringBuilder.AppendLine($"<tr><td>{parts[1]}</td><td>{parts[0].Replace("-", ", ")}</td></tr>");
}
catch
{
// This is still a POC at this point. Oh well...
}
}
stringBuilder.AppendLine("</table>");
product.BodyHtml += $"<div id=\"seoData\">{stringBuilder.ToString()}</div>";
}
IList<int> vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles);
if (vehicleIdFitment.Count > 0)
{
string vehicleIdString = string.Join('-', vehicleIdFitment.Select(j => $"v{j}"));
product.BodyHtml += $"<div id=\"vehicleIDs\" style=\"display:none;\">{vehicleIdString}</div>";
}
List<string> tags = new List<string>
{
importData.LineCode,
importData.PartNumber,
"zzzIsFitment=true"
};
product.Tags = string.Join(',', tags);
await _shopifyClient.Products.Update(product);
}
catch (Exception ex)
{
_logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku} - {ex.Message}", ex);
}
}
try
{
Console.WriteLine(i);
products = await _shopifyClient.Products.GetNext();
i++;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
products = await _shopifyClient.Products.GetPrevious();
}
}
}
}
}

145
PartSource.Automation/Jobs/POC/UpdateFitmentScratchpad.cs
View File

@@ -1,145 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Dtos;
using PartSource.Data.Models;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Exceptions;
using Ratermania.Shopify.Resources;
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs.POC
{
public class UpdateFitmentScratchpad : IAutomationJob
{
private readonly ILogger<UpdateFitmentScratchpad> _logger;
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
public UpdateFitmentScratchpad(ILogger<UpdateFitmentScratchpad> logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
{
_logger = logger;
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> productTypes = new List<string>
{
"C172-S231-F23107_(PS) Wipers - TRICO Neoform",
"CA172-SC231-FL23106_Wiper Blades - Bosch Icon",
"CA172-SC231-FL23107_(PS) Wipers - TRICO Neoform",
"CA172-SC231-FL23109_(PS) Wipers - TRICO Tech/Exact Fit",
"CA172-SC231-FL23110_Wiper Accessories",
"CA172-SC231-FL23114_Wiper Blades - Rear",
"CA172-SC231-FL23116_(PS) Wipers - Bosch Insight (Hybrid)",
"CA172-SC231-FL23117_(PS) Wipers - Bosch Clear Advantage (Beam)",
"CA172-SC231-FL23118_(PS) Wipers - Winter",
"CA172-SC231-FL23125_Wiper Blades - Bosch Areotwin"
};
IList<string> csvData = new List<string>
{
"\"Line Code\", \"Part Number\", \"Year\", \"Make\", \"Model\", \"Position\""
};
foreach (string type in productTypes)
{
IEnumerable<Product> products = null;
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "product_type", type } });
}
catch (Exception ex)
{
_logger.LogError("Failed to get products from Shopify", ex);
throw;
}
while (products != null && products.Any())
{
foreach (Product product in products)
{
ImportData importData = null;
try
{
IEnumerable<Metafield> metafields = await _shopifyClient.Metafields.Get(new Dictionary<string, object> { { "metafield[owner_id]", product.Id }, { "metafield[owner_resource]", "product" } });
importData = new ImportData
{
LineCode = metafields.FirstOrDefault(m => m.Key == "custom_label_0").Value ?? string.Empty,
PartNumber = product.Title.Split(' ')[0],
VariantSku = product.Variants[0].Sku // They know we can't do fitment for variants
};
string csvRow = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
IList<VehicleFitmentDto> vehicles = _vehicleService.GetVehicleFitmentForPart(importData.PartNumber, importData.LineCode);
if (vehicles.Count > 0)
{
foreach (VehicleFitmentDto vehicle in vehicles)
{
try
{
csvData.Add($"\"{importData.LineCode}\", \"{importData.PartNumber}\", \"{vehicle.Vehicle.Year}\", \"{vehicle.Vehicle.MakeName}\", \"{vehicle.Vehicle.ModelName}\", \"{vehicle.Fitment.Position}\"");
}
catch
{
// This is still a POC at this point. Oh well...
}
}
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku} - {ex.Message}", ex);
}
}
try
{
Console.WriteLine($"Did an iteration of {type}");
products = await _shopifyClient.Products.GetNext();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
products = await _shopifyClient.Products.GetPrevious();
}
}
}
await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\Wiper Fitment.csv", csvData);
;
}
}
}

240
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -19,163 +19,163 @@ using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class ProcessWhiFitment : IAutomationJob
{
private readonly ILogger<ProcessWhiFitment> _logger;
private readonly WhiSeoService _whiSeoService;
private readonly FtpConfiguration _ftpConfiguration;
private readonly SeoDataType _seoDataType;
public class ProcessWhiFitment : IAutomationJob
{
private readonly ILogger<ProcessWhiFitment> _logger;
private readonly WhiSeoService _whiSeoService;
private readonly FtpConfiguration _ftpConfiguration;
private readonly SeoDataType _seoDataType;
private readonly IDictionary<string, string> _noteDictionary;
private readonly IDictionary<string, string> _noteDictionary;
public ProcessWhiFitment(IConfiguration configuration, ILogger<ProcessWhiFitment> logger, WhiSeoService whiSeoService)
{
_logger = logger;
_whiSeoService = whiSeoService;
public ProcessWhiFitment(IConfiguration configuration, ILogger<ProcessWhiFitment> logger, WhiSeoService whiSeoService)
{
_logger = logger;
_whiSeoService = whiSeoService;
_seoDataType = SeoDataType.Fitment;
_seoDataType = SeoDataType.Fitment;
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
_noteDictionary = new ConcurrentDictionary<string, string>();
}
_noteDictionary = new ConcurrentDictionary<string, string>();
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "Not library code")]
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateFitmentTables();
_whiSeoService.GetFiles(_seoDataType);
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "<Pending>")]
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateFitmentTables();
// _whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
ConcurrentQueue<IGrouping<string, FileInfo>> fileGroups = new ConcurrentQueue<IGrouping<string, FileInfo>>();
ConcurrentQueue<IGrouping<string, FileInfo>> fileGroups = new ConcurrentQueue<IGrouping<string, FileInfo>>();
foreach (IGrouping<string, FileInfo> fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()))
{
fileGroups.Enqueue(fileGroup);
}
foreach (IGrouping<string, FileInfo> fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()))
{
fileGroups.Enqueue(fileGroup);
}
Task[] taskArray = new Task[Environment.ProcessorCount / 2];
Task[] taskArray = new Task[8];
for (int i = 0; i < taskArray.Length; i++)
{
taskArray[i] = Task.Factory.StartNew(() =>
{
while (fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup))
{
foreach (FileInfo fileInfo in fileGroup)
{
try
{
string filename = Decompress(fileInfo);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
for (int i = 0; i < taskArray.Length; i++)
{
taskArray[i] = Task.Factory.StartNew(() =>
{
while (fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup))
{
foreach (FileInfo fileInfo in fileGroup)
{
try
{
string filename = Decompress(fileInfo);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
DataTable dataTable = GetDataTable(filename);
DataTable dataTable = GetDataTable(filename);
_whiSeoService.BulkCopyFitment(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
_whiSeoService.BulkCopyFitment(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
File.Delete(filename);
}
File.Delete(filename);
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
}
}, token);
}
}
});
}
Task.WaitAll(taskArray);
Task.WaitAll(taskArray);
_whiSeoService.CreateFitmentView();
_whiSeoService.CreateFitmentView();
_whiSeoService.SaveNotes(_noteDictionary);
}
_whiSeoService.SaveNotes(_noteDictionary);
}
public string Decompress(FileInfo fileInfo)
{
string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length);
public string Decompress(FileInfo fileInfo)
{
string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length);
using FileStream filestream = File.Create(decompressedFile);
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
using FileStream filestream = File.Create(decompressedFile);
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
decompressionStream.CopyTo(filestream);
decompressionStream.CopyTo(filestream);
return decompressedFile;
}
return decompressedFile;
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("FitmentNoteHash", typeof(string));
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("FitmentNoteHash", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string[] columns = line.Split("\",\"");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
string noteTextHash = GetMD5Hash(noteText);
string noteText = columns[4].Trim();
string noteTextHash = GetMD5Hash(noteText);
if (!_noteDictionary.ContainsKey(noteTextHash))
{
_noteDictionary.Add(noteTextHash, noteText);
}
if (!_noteDictionary.ContainsKey(noteTextHash))
{
_noteDictionary.Add(noteTextHash, noteText);
}
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash });
}
}
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash });
}
}
return dataTable;
}
return dataTable;
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")]
private string GetMD5Hash(string input)
{
using MD5 md5 = MD5.Create();
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")]
private string GetMD5Hash(string input)
{
using MD5 md5 = MD5.Create();
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
byte[] hashBytes = md5.ComputeHash(inputBytes);
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
byte[] hashBytes = md5.ComputeHash(inputBytes);
StringBuilder stringBuilder = new StringBuilder();
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
stringBuilder.Append(hashBytes[i].ToString("X2"));
}
for (int i = 0; i < hashBytes.Length; i++)
{
stringBuilder.Append(hashBytes[i].ToString("X2"));
}
return stringBuilder.ToString();
}
}
return stringBuilder.ToString();
}
}
}

185
PartSource.Automation/Jobs/ProcessWhiVehicles.cs
View File

@@ -19,117 +19,118 @@ using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class ProcessWhiVehicles : IAutomationJob
{
private readonly ILogger<ProcessWhiVehicles> _logger;
private readonly WhiSeoService _whiSeoService;
private readonly FtpConfiguration _ftpConfiguration;
private readonly SeoDataType _seoDataType;
public class ProcessWhiVehicles : IAutomationJob
{
private readonly ILogger<ProcessWhiVehicles> _logger;
private readonly WhiSeoService _whiSeoService;
private readonly FtpConfiguration _ftpConfiguration;
private readonly SeoDataType _seoDataType;
public ProcessWhiVehicles(IConfiguration configuration, ILogger<ProcessWhiVehicles> logger, WhiSeoService whiSeoService)
{
_logger = logger;
_whiSeoService = whiSeoService;
public ProcessWhiVehicles(IConfiguration configuration, ILogger<ProcessWhiVehicles> logger, WhiSeoService whiSeoService)
{
_logger = logger;
_whiSeoService = whiSeoService;
_seoDataType = SeoDataType.Vehicle;
_seoDataType = SeoDataType.Vehicle;
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
}
}
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateVehicleTables();
_whiSeoService.GetFiles(_seoDataType);
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateVehicleTable();
// _whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
IEnumerable<FileInfo> files = directoryInfo.GetFiles().Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"));
IEnumerable<FileInfo> files = directoryInfo.GetFiles().Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"));
foreach (FileInfo fileInfo in files)
{
try
{
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
foreach (FileInfo fileInfo in files)
{
try
{
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
DataTable dataTable = GetDataTable(fileInfo.FullName);
DataTable dataTable = GetDataTable(fileInfo.FullName);
_whiSeoService.BulkCopyVehicle(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
_whiSeoService.BulkCopyVehicle(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
File.Delete(fileInfo.FullName);
}
File.Delete(fileInfo.FullName);
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
_whiSeoService.CreateVehicleTable();
_whiSeoService.CreateVehicleTable();
_logger.LogInformation($"Created vehicle table.");
}
_logger.LogInformation($"Created vehicle table.");
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("Year", typeof(int));
dataTable.Columns.Add("MakeId", typeof(int));
dataTable.Columns.Add("MakeName", typeof(string));
dataTable.Columns.Add("ModelId", typeof(int));
dataTable.Columns.Add("ModelName", typeof(string));
dataTable.Columns.Add("RegionId", typeof(int));
dataTable.Columns.Add("RegionName", typeof(string));
dataTable.Columns.Add("VehicleTypeId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("EngineDescription", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("VehicleToEngineConfigId", typeof(int));
dataTable.Columns.Add("SubmodelId", typeof(int));
dataTable.Columns.Add("SubmodelName", typeof(string));
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("Year", typeof(int));
dataTable.Columns.Add("MakeId", typeof(int));
dataTable.Columns.Add("MakeName", typeof(string));
dataTable.Columns.Add("ModelId", typeof(int));
dataTable.Columns.Add("ModelName", typeof(string));
dataTable.Columns.Add("RegionId", typeof(int));
dataTable.Columns.Add("RegionName", typeof(string));
dataTable.Columns.Add("VehicleTypeId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("EngineDescription", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("VehicleToEngineConfigId", typeof(int));
dataTable.Columns.Add("SubmodelId", typeof(int));
dataTable.Columns.Add("SubmodelName", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string[] columns = line.Split("\",\"");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string makeName = columns[4].Trim();
string modelName = columns[6].Trim();
string regionName = columns[8].Trim();
string submodelName = columns[34].Trim();
string engineDescription = columns[51].Trim();
string makeName = columns[4].Trim();
string modelName = columns[6].Trim();
string regionName = columns[8].Trim();
string submodelName = columns[34].Trim();
string engineDescription = columns[51].Trim();
if (!string.IsNullOrEmpty(makeName)
&& !string.IsNullOrEmpty(modelName)
&& !string.IsNullOrEmpty(regionName)
&& !string.IsNullOrEmpty(submodelName)
&& !string.IsNullOrEmpty(engineDescription)
&& int.TryParse(columns[0], out int baseVehicleId)
&& int.TryParse(columns[2], out int year)
&& int.TryParse(columns[3], out int makeId)
&& int.TryParse(columns[5], out int modelId)
&& int.TryParse(columns[7], out int regionId)
&& int.TryParse(columns[9], out int vehicleTypeId)
&& int.TryParse(columns[33], out int submodelId)
&& int.TryParse(columns[35], out int engineConfigId)
&& int.TryParse(columns[36], out int vehicleToEngineConfigId)
&& new[] { 5, 6, 7 }.Contains(vehicleTypeId))
{
dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, regionId, regionName, vehicleTypeId, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName });
}
}
if (!string.IsNullOrEmpty(makeName)
&& !string.IsNullOrEmpty(modelName)
&& !string.IsNullOrEmpty(submodelName)
&& !string.IsNullOrEmpty(engineDescription)
&& int.TryParse(columns[0], out int baseVehicleId)
&& int.TryParse(columns[2], out int year)
&& int.TryParse(columns[3], out int makeId)
&& int.TryParse(columns[5], out int modelId)
&& int.TryParse(columns[7], out int regionId)
&& int.TryParse(columns[9], out int vehicleTypeId)
&& int.TryParse(columns[33], out int submodelId)
&& int.TryParse(columns[35], out int engineConfigId)
&& int.TryParse(columns[36], out int vehicleToEngineConfigId))
{
if (regionId == 2 && new[] { 5, 6, 7 }.Contains(vehicleTypeId))
{
dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, regionId, regionName, vehicleTypeId, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName });
}
}
}
return dataTable;
}
}
return dataTable;
}
}
}

157
PartSource.Automation/Jobs/UpdateFitment.cs
View File

@@ -1,23 +1,18 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Exceptions;
using Ratermania.Shopify.Resources;
using System;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using PartSource.Automation.Services;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs
{
@@ -27,19 +22,20 @@ namespace PartSource.Automation.Jobs
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
private readonly VehicleFitmentService _vehicleFitmentService;
public UpdateFitment(ILogger<UpdateFitment> logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
public UpdateFitment(ILogger<UpdateFitment> logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleFitmentService vehicleFitmentService)
{
_logger = logger;
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
_vehicleFitmentService = vehicleFitmentService;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = null;
try
@@ -59,61 +55,56 @@ namespace PartSource.Automation.Jobs
{
foreach (Product product in products)
{
// Wiper blades are a separate fitment process.
if (product.ProductType.Contains("CA172-SC231"))
{
continue;
}
ImportData importData = null;
try
{
IEnumerable<Metafield> metafields = await _shopifyClient.Metafields.Get(new Dictionary<string, object> { { "metafield[owner_id]", product.Id }, { "metafield[owner_resource]", "product" } });
//importData = await _partSourceContext.ImportData.FirstOrDefaultAsync(parts => parts.ShopifyId == product.Id);
//if (importData == null)
//{
// continue;
importData = new ImportData
{
LineCode = metafields.FirstOrDefault(m => m.Key == "custom_label_0").Value ?? string.Empty,
PartNumber = product.Title.Split(' ')[0],
PartNumber = metafields.FirstOrDefault(m => m.Key == "custom_label_1").Value ?? string.Empty,
VariantSku = product.Variants[0].Sku // They know we can't do fitment for variants
};
// }
bool isFitment = false;
string bodyHtml = product.BodyHtml[..(product.BodyHtml.IndexOf("</ul>") + "</ul>".Length)];
string bodyHtml = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
IList<int> vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles);
if (vehicleIdFitment.Any())
if (vehicleIdFitment.Count > 0)
{
string vehicleIdString = string.Join('-', vehicleIdFitment.Select(j => $"v{j}"));
string vehicleIdString = string.Join(',', vehicleIdFitment.Select(j => $"v{j}"));
bodyHtml += $"<div id=\"vehicleIDs\" style=\"display:none;\">{vehicleIdString}</div>";
isFitment = true;
string json = JsonConvert.SerializeObject(vehicleIdFitment);
Metafield vehicleMetafield = new Metafield
if (json.Length < 100000)
{
Namespace = "fitment",
Key = "ids",
Value = json,
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};
Metafield vehicleMetafield = new Metafield
{
Namespace = "fitment",
Key = "ids",
Value = json,
Type = "json_string",
OwnerResource = "product",
OwnerId = product.Id
};
await _shopifyClient.Metafields.Add(vehicleMetafield);
await _shopifyClient.Metafields.Add(vehicleMetafield);
}
else
{
_logger.LogWarning($"Vehicle ID fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added.");
continue;
}
}
IList<string> ymmFitment = _vehicleService.GetYmmFitment(vehicles);
IList<string> ymmFitment = _vehicleFitmentService.GetYmmFitment(vehicles);
if (ymmFitment.Count > 0)
{
isFitment = true;
@@ -141,17 +132,26 @@ namespace PartSource.Automation.Jobs
bodyHtml += $"<div id=\"seoData\">{stringBuilder.ToString()}</div>";
string json = JsonConvert.SerializeObject(ymmFitment);
Metafield ymmMetafield = new Metafield
if (json.Length < 100000)
{
Namespace = "fitment",
Key = "seo",
Value = json,
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
Metafield ymmMetafield = new Metafield
{
Namespace = "fitment",
Key = "seo",
Value = json,
Type = "json_string",
OwnerResource = "product",
OwnerId = product.Id
};
await _shopifyClient.Metafields.Add(ymmMetafield);
await _shopifyClient.Metafields.Add(ymmMetafield);
}
else
{
_logger.LogWarning($"Year/make/model fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added.");
continue;
}
}
Metafield isFitmentMetafield = new Metafield
@@ -159,34 +159,34 @@ namespace PartSource.Automation.Jobs
Namespace = "Flags",
Key = "IsFitment",
Value = isFitment.ToString(),
Type = "single_line_text_field",
Type = "string",
OwnerResource = "product",
OwnerId = product.Id
};
await _shopifyClient.Metafields.Add(isFitmentMetafield);
Metafield lineCodeMetafield = new Metafield
{
Namespace = "google",
Key = "custom_label_0",
Value = importData.LineCode,
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
//Metafield lineCodeMetafield = new Metafield
//{
// Namespace = "google",
// Key = "custom_label_0",
// Value = importData.LineCode,
// Type = "string",
// OwnerResource = "product",
// OwnerId = product.Id
//};
// await _shopifyClient.Metafields.Add(lineCodeMetafield);
//await _shopifyClient.Metafields.Add(lineCodeMetafield);
Metafield partNumberMetafield = new Metafield
{
Namespace = "google",
Key = "custom_label_1",
Value = importData.PartNumber,
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
//Metafield partNumberMetafield = new Metafield
//{
// Namespace = "google",
// Key = "custom_label_1",
// Value = importData.PartNumber,
// Type = "string",
// OwnerResource = "product",
// OwnerId = product.Id
//};
//await _shopifyClient.Metafields.Add(partNumberMetafield);
@@ -212,7 +212,6 @@ namespace PartSource.Automation.Jobs
product.Tags = string.Join(',', tags);
product.BodyHtml = bodyHtml;
await _shopifyClient.Products.Update(product);
importData.IsFitment = isFitment;
@@ -224,8 +223,8 @@ namespace PartSource.Automation.Jobs
{
_logger.LogError($"Failed to updated fitment data for SKU {importData?.VariantSku} - {ex.Message}", ex);
}
}
}
try
{
Console.WriteLine(i);
@@ -244,4 +243,4 @@ namespace PartSource.Automation.Jobs
}
}
}
}
}

86
PartSource.Automation/Jobs/UpdatePositioning.cs
View File

@@ -1,6 +1,7 @@
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Automation.Services;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
@@ -23,14 +24,14 @@ namespace PartSource.Automation.Jobs
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
private readonly VehicleFitmentService _vehicleFitmentService;
public UpdatePositioning(PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
public UpdatePositioning(PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleFitmentService vehicleFitmentService)
{
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
_vehicleFitmentService = vehicleFitmentService;
}
public async Task Run(CancellationToken token, params string[] arguments)
@@ -42,8 +43,6 @@ namespace PartSource.Automation.Jobs
IEnumerable<Product> products = await _shopifyClient.Products.Get(parameters);
int i = 1;
while (products != null && products.Any())
{
foreach (Product product in products)
@@ -65,7 +64,7 @@ namespace PartSource.Automation.Jobs
}
IList<Fitment> fitments = GetPositionOrderedFitments(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
if (fitments.Count == 0 || vehicles.Count == 0)
{
@@ -97,51 +96,45 @@ namespace PartSource.Automation.Jobs
await SavePositionMetafield(product, vehicleIds, currentPosition);
IList<string> notes = fitments.Select(f => f.FitmentNoteHash)
.Distinct()
.ToList();
//IList<string> notes = fitments.Select(f => f.NoteText)
IList<object> vehicleNotes = new List<object>();
// .Distinct()
// .ToList();
foreach (string noteHash in notes)
{
FitmentNote fitmentNote = await _fitmentContext.FitmentNotes.FirstOrDefaultAsync(f => f.Hash == noteHash);
//IList<object> vehicleNotes = new List<object>();
if (fitmentNote == null)
{
continue;
}
//foreach (string noteText in notes)
//{
// vehicleIds = fitments.Where(f => f.NoteText == noteText)
// .Select(f => new { f.EngineConfigId, f.BaseVehicleId })
// .SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId))
// .Select(v => v.VehicleToEngineConfigId)
// .ToList();
vehicleIds = fitments.Where(f => f.FitmentNoteHash == noteHash)
.Select(f => new { f.EngineConfigId, f.BaseVehicleId })
.SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId))
.Select(v => v.VehicleToEngineConfigId)
.ToList();
// vehicleNotes.Add(new { noteText, vehicleIds });
//}
vehicleNotes.Add(new { fitmentNote.NoteText, vehicleIds });
}
//string json = JsonConvert.SerializeObject(vehicleNotes);
//if (json.Length >= 100000)
//{
// continue;
//}
string json = JsonConvert.SerializeObject(vehicleNotes);
if (json.Length >= 100000)
{
continue;
}
//Metafield vehicleMetafield = new Metafield
//{
// Namespace = "fitment",
// Key = "note_text",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
//};
Metafield vehicleMetafield = new Metafield
{
Namespace = "fitment",
Key = "note_text",
Value = json,
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};
//await _shopifyClient.Metafields.Add(vehicleMetafield);
await _shopifyClient.Metafields.Add(vehicleMetafield);
//importData.UpdatedAt = DateTime.Now;
//importData.UpdateType = "Positioning";
}
//importData.UpdatedAt = DateTime.Now;
//importData.UpdateType = "Positioning";
}
catch (Exception ex)
{
@@ -150,8 +143,7 @@ namespace PartSource.Automation.Jobs
}
try
{
Console.WriteLine(i);
{
products = await _shopifyClient.Products.GetNext();
}
@@ -205,12 +197,12 @@ namespace PartSource.Automation.Jobs
Namespace = "position",
Key = key,
Value = json,
Type = "json",
Type = "json_string",
OwnerResource = "product",
OwnerId = product.Id
};
//System.Diagnostics.Debug.WriteLine(json);
System.Diagnostics.Debug.WriteLine(json);
await _shopifyClient.Metafields.Add(vehicleMetafield);
}

28
PartSource.Automation/Jobs/UpdatePricing.cs
View File

@@ -35,14 +35,13 @@ namespace PartSource.Automation.Jobs
public async Task Run(CancellationToken token, params string[] arguments)
{
List<UpdatePricingResult> pricingReport = new List<UpdatePricingResult>();
IEnumerable<Product> products = null;
IEnumerable<PartPrice> prices = null;
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
prices = await _partSourceContext.PartPrices.AsNoTracking().ToListAsync(token);
prices = await _partSourceContext.PartPrices.AsNoTracking().ToListAsync();
}
catch (Exception ex)
@@ -52,6 +51,7 @@ namespace PartSource.Automation.Jobs
throw;
}
int count = 0;
while (products != null && products.Any())
{
foreach (Product product in products)
@@ -76,12 +76,12 @@ namespace PartSource.Automation.Jobs
try
{
await _shopifyClient.Metafields.Add(new Metafield
_shopifyClient.BulkActions.Add(new Metafield
{
Namespace = "Pricing",
Key = "CorePrice",
Value = partPrice.Core_Price.HasValue ? partPrice.Core_Price.Value.ToString() : "0.00",
Type = "single_line_text_field",
Type = "string",
OwnerResource = "product",
OwnerId = product.Id
});
@@ -89,13 +89,15 @@ namespace PartSource.Automation.Jobs
catch (Exception ex)
{
_logger.LogWarning(ex, $"Failed to update core price metafield for product ID {product.Id}");
_logger.LogWarning(ex, $"Failed to update pricing for product ID {product.Id}");
}
}
try
{
await _shopifyClient.Products.Update(product);
_shopifyClient.BulkActions.Update(product);
}
catch (Exception ex)
@@ -105,12 +107,11 @@ namespace PartSource.Automation.Jobs
}
}
try
{
count += products.Count();
products = await _shopifyClient.Products.GetNext();
_logger.LogInformation($"Total updated: {pricingReport.Count}");
_logger.LogInformation($"Total updated: {count}");
}
catch (Exception ex)
@@ -119,8 +120,13 @@ namespace PartSource.Automation.Jobs
products = await _shopifyClient.Products.GetPrevious();
}
}
_emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
while (_shopifyClient.BulkActions.PendingCount() > 0)
{
await Task.Delay(15 * 1000, token);
_logger.LogInformation(_shopifyClient.BulkActions.PendingCount().ToString());
}
// _emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
}
}
}

3
PartSource.Automation/PartSource.Automation.csproj
View File

@@ -21,10 +21,11 @@
<PackageReference Include="Ratermania.Automation" Version="6.16.9" />
<PackageReference Include="Ratermania.Automation.Common" Version="6.16.9" />
<PackageReference Include="Ratermania.JwtSpot" Version="6.16.9" />
<PackageReference Include="Ratermania.Shopify" Version="6.16.8" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\ratermania\Packages\Shopify\Shopify.csproj" />
<ProjectReference Include="..\PartSource.Data\PartSource.Data.csproj" />
<ProjectReference Include="..\PartSource.Services\PartSource.Services.csproj" />
</ItemGroup>

162
PartSource.Automation/Program.cs
View File

@@ -6,7 +6,6 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Jobs;
using PartSource.Automation.Jobs.POC;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Services;
using PartSource.Data;
using PartSource.Data.AutoMapper;
@@ -15,58 +14,56 @@ using PartSource.Services;
using Ratermania.Automation.DependencyInjection;
using Ratermania.Automation.Logging;
using Ratermania.Shopify.DependencyInjection;
using Ratermania.JwtSpot.Configuration;
using System;
using System.IO;
using System.Threading.Tasks;
namespace PartSource.Automation
{
class Program
{
static async Task Main(string[] args)
{
try
{
using IHost host = CreateHostBuilder().Build();
class Program
{
static async Task Main(string[] args){
try
{
using IHost host = CreateHostBuilder().Build();
await host.StartAsync();
}
await host.StartAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
private static IHostBuilder CreateHostBuilder()
{
return Host.CreateDefaultBuilder()
.ConfigureAppConfiguration(builder =>
{
string environment = Environment.GetEnvironmentVariable("AUTOMATION_ENVIRONMENT");
private static IHostBuilder CreateHostBuilder()
{
return Host.CreateDefaultBuilder()
.ConfigureAppConfiguration(builder =>
{
string environment = Environment.GetEnvironmentVariable("AUTOMATION_ENVIRONMENT");
builder.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{environment}.json", optional: true, reloadOnChange: true);
})
.ConfigureServices((builder, services) =>
{
services.AddDbContext<PartSourceContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("PartSourceDatabase"), opts => opts.EnableRetryOnFailure())
)
builder.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{environment}.json", optional: true, reloadOnChange: true);
})
.ConfigureServices((builder, services) =>
{
services.AddDbContext<PartSourceContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("PartSourceDatabase"), opts => opts.EnableRetryOnFailure())
)
.AddDbContext<FitmentContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts =>
{
opts.EnableRetryOnFailure();
opts.CommandTimeout(600);
})
)
.AddDbContext<FitmentContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts =>
{
opts.EnableRetryOnFailure();
opts.CommandTimeout(600);
})
)
.AddShopify(options =>
{
.AddShopify(options =>
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
options.ApiVersion = "2022-10";
@@ -78,42 +75,55 @@ namespace PartSource.Automation
//options.ShopDomain = "dev-partsource.myshopify.com";
})
.AddAutomation(options =>
options.HasBaseInterval(new TimeSpan(0, 15, 0))
.HasMaxFailures(3)
.HasJob<ExecuteSsisPackages>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(26)))
.HasJob<UpdatePricing>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(27))
.HasDependency<ExecuteSsisPackages>())
.UseApiServer(opts =>
opts.HasApiKey("PartsourceAPIKey")
.UseJwtSpot(jwt =>
jwt.HasAudience(builder.Configuration["JwtSpot:Audience"])
.HasIssuer(builder.Configuration["JwtSpot:Issuer"])
.UseX509Certificate(builder.Configuration["JwtSpot:CertPath"])
.UseJwksUrl(builder.Configuration["JwtSpot:JwksUrl"])))
.UseSqlServer(builder.Configuration.GetConnectionString("AutomationDatabase")))
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 15, 0))
.HasMaxFailures(1)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>()
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>()
// .HasDependency<SyncronizeProducts>()
// .StartsAt(DateTime.Today.AddHours(8))
//) ;
//.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//)
//.HasJob<ExecuteSsisPackages>(options =>
// options.HasInterval(new TimeSpan(24, 0, 0))
// //.StartsAt(DateTime.Today.AddHours(25))
// )
.AddSingleton(builder.Configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>())
.AddSingleton<FtpService>()
.AddSingleton<EmailService>()
.AddSingleton<SsisService>()
.AddSingleton<WhiSeoService>()
.AddSingleton<VehicleService>()
.AddSingleton<NexpartService>()
.HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<ExecuteSsisPackages>()
// .StartsAt(DateTime.Today.AddHours(28))
);
//);
//.AddApiServer();
})
.AddAutoMapper(typeof(PartSourceProfile));
})
.ConfigureLogging((builder, logging) =>
{
logging.AddEventLog();
logging.AddConsole();
.AddSingleton<EmailService>()
.AddSingleton<SsisService>()
.AddSingleton<WhiSeoService>()
.AddSingleton<VehicleService>()
.AddSingleton<VehicleFitmentService>()
.AddSingleton<NexpartService>()
//logging.AddProvider(new AutomationLoggerProvider());
});
}
}
.AddAutoMapper(typeof(PartSourceProfile));
})
.ConfigureLogging((builder, logging) =>
{
logging.AddEventLog();
logging.AddConsole();
// logging.AddProvider(new AutomationLoggerProvider());
});
}
}
}

159
PartSource.Automation/Services/VehicleFitmentService.cs Normal file
View File

@@ -0,0 +1,159 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using PartSource.Data.Contexts;
using PartSource.Data.Dtos;
using PartSource.Data.Models;
namespace PartSource.Automation.Services
{
public class VehicleFitmentService
{
private readonly FitmentContext _fitmentContext;
public VehicleFitmentService(FitmentContext fitmentContext)
{
_fitmentContext = fitmentContext;
}
public IList<string> GetYmmFitment(IList<Vehicle> vehicles)
{
if (vehicles.Count == 0)
{
return new string[0];
}
IList<string> fitmentTags = new List<string>();
IList<string> makeModels = vehicles.OrderBy(v => v.MakeName).ThenBy(v => v.ModelName).Select(v => $"{v.MakeName},{v.ModelName}").Distinct().ToList();
foreach (string makeModel in makeModels)
{
string make = makeModel.Split(',')[0];
string model = makeModel.Split(',')[1];
List<string> years = vehicles
.Where(v => v.MakeName == make && v.ModelName == model)
.OrderBy(v => v.Year)
.Select(v => v.Year.ToString().Trim())
.Distinct()
.ToList();
string tag = $"{string.Join('-', years)} {make.Trim()} {model.Trim()}";
System.Diagnostics.Debug.WriteLine(tag);
fitmentTags.Add(tag);
}
return fitmentTags;
}
public IList<string> GetYmmFitmentRange(IList<Vehicle> vehicles)
{
if (vehicles.Count == 0)
{
return new string[0];
}
IList<string> fitmentTags = new List<string>();
IList<string> makeModels = vehicles.Select(v => $"{v.MakeName},{v.ModelName}").Distinct().ToList();
foreach (string makeModel in makeModels)
{
string make = makeModel.Split(',')[0];
string model = makeModel.Split(',')[1];
int minYear = vehicles
.Where(v => v.MakeName == make && v.ModelName == model)
.Min(v => v.Year);
int maxYear = vehicles
.Where(v => v.MakeName == make && v.ModelName == model)
.Max(v => v.Year);
string tag = minYear == maxYear
? $"{minYear} {make.Trim()} {model.Trim()}"
: $"{minYear}-{maxYear} {make.Trim()} {model.Trim()}";
System.Diagnostics.Debug.WriteLine(tag);
fitmentTags.Add(tag);
}
return fitmentTags;
}
public IList<int> GetVehicleIdFitment(IList<Vehicle> vehicles)
{
return vehicles.Select(v => v.VehicleToEngineConfigId).Distinct().ToArray();
}
public IList<Vehicle> GetVehiclesForPart(string partNumber, string lineCode, int maxVehicles = 0)
{
if (string.IsNullOrEmpty(partNumber) || string.IsNullOrEmpty(lineCode))
{
return null;
}
partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9\\-]", string.Empty);
IQueryable<string> whiCodes = _fitmentContext.DcfMappings
.Where(d => d.LineCode == lineCode)
.Select(d => d.WhiCode);
IQueryable<Vehicle> vehicles = _fitmentContext.Fitments
.Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode))
.Join(_fitmentContext.Vehicles,
f => new { f.BaseVehicleId, f.EngineConfigId },
v => new { v.BaseVehicleId, v.EngineConfigId },
(f, v) => v)
.Distinct()
.OrderByDescending(x => x.Year);
if (maxVehicles > 0)
{
vehicles = vehicles.Take(maxVehicles);
}
return vehicles.ToList();
}
public IList<VehicleFitmentDto> GetVehicleFitmentForPart(string partNumber, string lineCode, int maxVehicles = 0)
{
if (string.IsNullOrEmpty(partNumber) || string.IsNullOrEmpty(lineCode))
{
return null;
}
partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9\\-]", string.Empty);
IQueryable<string> whiCodes = _fitmentContext.DcfMappings
.Where(d => d.LineCode == lineCode)
.Select(d => d.WhiCode);
IQueryable<VehicleFitmentDto> vehicles = _fitmentContext.Fitments
.Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode))
.Join(_fitmentContext.Vehicles,
f => new { f.BaseVehicleId, f.EngineConfigId },
v => new { v.BaseVehicleId, v.EngineConfigId },
(f, v) => new VehicleFitmentDto
{
Fitment = f,
Vehicle = v
})
.Distinct();
if (maxVehicles > 0)
{
vehicles = vehicles.Take(maxVehicles);
}
return vehicles.ToList();
}
}
}

19
PartSource.Automation/Services/WhiSeoService.cs
View File

@@ -1,17 +1,18 @@
#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities
using System;
using System.Collections.Generic;
using System.Data;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Enums;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Text;
namespace PartSource.Automation.Services
{
public class WhiSeoService
public class WhiSeoService
{
private readonly FtpService _ftpService;
private readonly string _connectionString;
@@ -52,12 +53,12 @@ namespace PartSource.Automation.Services
}
}
public void TruncateVehicleTables()
public void TruncateVehicleTable()
{
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using SqlCommand command = new SqlCommand($"exec DropVehicleTables", connection);
using SqlCommand command = new SqlCommand($"truncate table dbo.Vehicle", connection);
command.ExecuteNonQuery();
}
@@ -150,6 +151,10 @@ namespace PartSource.Automation.Services
using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection);
command.CommandTimeout = 1800;
command.ExecuteNonQuery();
using SqlCommand command2 = new SqlCommand($"exec CreateFitmentIndexes", connection);
command.CommandTimeout = 1800;
command2.ExecuteNonQuery();
}
public void CreateVehicleTable()

13
PartSource.Automation/appsettings.json
View File

@@ -1,7 +1,6 @@
{
"ConnectionStrings": {
"AutomationDatabase": "Data Source=localhost;Initial Catalog=Automation;Integrated Security=true;Trust Server Certificate=true;",
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true",
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true;TrustServerCertificate=True",
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
},
"emailConfiguration": {
@@ -32,18 +31,12 @@
"ApiSecret": "527a3b4213c2c7ecb214728a899052df",
"ShopDomain": "partsource.myshopify.com"
},
"JwtSpot": {
"Audience": "Ratermania.Automation",
"Issuer": "https://tomraterman.com",
"JwksUrl": "http://localhost:5103/jwks",
"CertPath": "C:\\Users\\tom\\Desktop\\PartsourceAutomation.pfx"
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
// "Microsoft.EntityFrameworkCore.Database.Command": "Information"
"Microsoft.Hosting.Lifetime": "Information",
// "Microsoft.EntityFrameworkCore.Database.Command": "Information"
},
"EventLog": {
"LogLevel": {