Fitment update
This commit is contained in:
@@ -19,163 +19,163 @@ using System.Threading.Tasks;
|
||||
|
||||
namespace PartSource.Automation.Jobs
|
||||
{
|
||||
public class ProcessWhiFitment : IAutomationJob
|
||||
{
|
||||
private readonly ILogger<ProcessWhiFitment> _logger;
|
||||
private readonly WhiSeoService _whiSeoService;
|
||||
private readonly FtpConfiguration _ftpConfiguration;
|
||||
private readonly SeoDataType _seoDataType;
|
||||
public class ProcessWhiFitment : IAutomationJob
|
||||
{
|
||||
private readonly ILogger<ProcessWhiFitment> _logger;
|
||||
private readonly WhiSeoService _whiSeoService;
|
||||
private readonly FtpConfiguration _ftpConfiguration;
|
||||
private readonly SeoDataType _seoDataType;
|
||||
|
||||
private readonly IDictionary<string, string> _noteDictionary;
|
||||
private readonly IDictionary<string, string> _noteDictionary;
|
||||
|
||||
public ProcessWhiFitment(IConfiguration configuration, ILogger<ProcessWhiFitment> logger, WhiSeoService whiSeoService)
|
||||
{
|
||||
_logger = logger;
|
||||
_whiSeoService = whiSeoService;
|
||||
public ProcessWhiFitment(IConfiguration configuration, ILogger<ProcessWhiFitment> logger, WhiSeoService whiSeoService)
|
||||
{
|
||||
_logger = logger;
|
||||
_whiSeoService = whiSeoService;
|
||||
|
||||
_seoDataType = SeoDataType.Fitment;
|
||||
_seoDataType = SeoDataType.Fitment;
|
||||
|
||||
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
|
||||
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
|
||||
|
||||
_noteDictionary = new ConcurrentDictionary<string, string>();
|
||||
}
|
||||
_noteDictionary = new ConcurrentDictionary<string, string>();
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "Not library code")]
|
||||
public async Task Run(CancellationToken token, params string[] arguments)
|
||||
{
|
||||
_whiSeoService.TruncateFitmentTables();
|
||||
_whiSeoService.GetFiles(_seoDataType);
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "<Pending>")]
|
||||
public async Task Run(CancellationToken token, params string[] arguments)
|
||||
{
|
||||
_whiSeoService.TruncateFitmentTables();
|
||||
// _whiSeoService.GetFiles(_seoDataType);
|
||||
|
||||
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
|
||||
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
|
||||
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
|
||||
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
|
||||
|
||||
ConcurrentQueue<IGrouping<string, FileInfo>> fileGroups = new ConcurrentQueue<IGrouping<string, FileInfo>>();
|
||||
ConcurrentQueue<IGrouping<string, FileInfo>> fileGroups = new ConcurrentQueue<IGrouping<string, FileInfo>>();
|
||||
|
||||
foreach (IGrouping<string, FileInfo> fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()))
|
||||
{
|
||||
fileGroups.Enqueue(fileGroup);
|
||||
}
|
||||
foreach (IGrouping<string, FileInfo> fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()))
|
||||
{
|
||||
fileGroups.Enqueue(fileGroup);
|
||||
}
|
||||
|
||||
Task[] taskArray = new Task[Environment.ProcessorCount / 2];
|
||||
Task[] taskArray = new Task[8];
|
||||
|
||||
for (int i = 0; i < taskArray.Length; i++)
|
||||
{
|
||||
taskArray[i] = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup))
|
||||
{
|
||||
foreach (FileInfo fileInfo in fileGroup)
|
||||
{
|
||||
try
|
||||
{
|
||||
string filename = Decompress(fileInfo);
|
||||
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
|
||||
for (int i = 0; i < taskArray.Length; i++)
|
||||
{
|
||||
taskArray[i] = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup))
|
||||
{
|
||||
foreach (FileInfo fileInfo in fileGroup)
|
||||
{
|
||||
try
|
||||
{
|
||||
string filename = Decompress(fileInfo);
|
||||
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
|
||||
|
||||
DataTable dataTable = GetDataTable(filename);
|
||||
DataTable dataTable = GetDataTable(filename);
|
||||
|
||||
_whiSeoService.BulkCopyFitment(dataTable, tableName);
|
||||
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
|
||||
_whiSeoService.BulkCopyFitment(dataTable, tableName);
|
||||
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
|
||||
|
||||
File.Delete(filename);
|
||||
}
|
||||
File.Delete(filename);
|
||||
}
|
||||
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
|
||||
_whiSeoService.CreateFitmentTable(fitmentTable);
|
||||
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
|
||||
_whiSeoService.CreateFitmentTable(fitmentTable);
|
||||
|
||||
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
|
||||
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
|
||||
|
||||
}
|
||||
}, token);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Task.WaitAll(taskArray);
|
||||
Task.WaitAll(taskArray);
|
||||
|
||||
_whiSeoService.CreateFitmentView();
|
||||
_whiSeoService.CreateFitmentView();
|
||||
|
||||
_whiSeoService.SaveNotes(_noteDictionary);
|
||||
}
|
||||
_whiSeoService.SaveNotes(_noteDictionary);
|
||||
}
|
||||
|
||||
public string Decompress(FileInfo fileInfo)
|
||||
{
|
||||
string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length);
|
||||
public string Decompress(FileInfo fileInfo)
|
||||
{
|
||||
string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length);
|
||||
|
||||
using FileStream filestream = File.Create(decompressedFile);
|
||||
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
|
||||
using FileStream filestream = File.Create(decompressedFile);
|
||||
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
|
||||
|
||||
decompressionStream.CopyTo(filestream);
|
||||
decompressionStream.CopyTo(filestream);
|
||||
|
||||
return decompressedFile;
|
||||
}
|
||||
return decompressedFile;
|
||||
}
|
||||
|
||||
private DataTable GetDataTable(string filename)
|
||||
{
|
||||
using DataTable dataTable = new DataTable();
|
||||
dataTable.Columns.Add("LineCode", typeof(string));
|
||||
dataTable.Columns.Add("PartNumber", typeof(string));
|
||||
dataTable.Columns.Add("BaseVehicleId", typeof(int));
|
||||
dataTable.Columns.Add("EngineConfigId", typeof(int));
|
||||
dataTable.Columns.Add("Position", typeof(string));
|
||||
dataTable.Columns.Add("FitmentNoteHash", typeof(string));
|
||||
private DataTable GetDataTable(string filename)
|
||||
{
|
||||
using DataTable dataTable = new DataTable();
|
||||
dataTable.Columns.Add("LineCode", typeof(string));
|
||||
dataTable.Columns.Add("PartNumber", typeof(string));
|
||||
dataTable.Columns.Add("BaseVehicleId", typeof(int));
|
||||
dataTable.Columns.Add("EngineConfigId", typeof(int));
|
||||
dataTable.Columns.Add("Position", typeof(string));
|
||||
dataTable.Columns.Add("FitmentNoteHash", typeof(string));
|
||||
|
||||
using StreamReader reader = new StreamReader(filename);
|
||||
string line = reader.ReadLine(); // Burn the header row
|
||||
using StreamReader reader = new StreamReader(filename);
|
||||
string line = reader.ReadLine(); // Burn the header row
|
||||
|
||||
while (reader.Peek() > 0)
|
||||
{
|
||||
line = reader.ReadLine();
|
||||
while (reader.Peek() > 0)
|
||||
{
|
||||
line = reader.ReadLine();
|
||||
|
||||
string[] columns = line.Split("\",\"");
|
||||
for (int i = 0; i < columns.Length; i++)
|
||||
{
|
||||
columns[i] = columns[i].Replace("\"", string.Empty);
|
||||
}
|
||||
string[] columns = line.Split("\",\"");
|
||||
for (int i = 0; i < columns.Length; i++)
|
||||
{
|
||||
columns[i] = columns[i].Replace("\"", string.Empty);
|
||||
}
|
||||
|
||||
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
|
||||
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
|
||||
string position = columns[7].Trim();
|
||||
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
|
||||
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
|
||||
string position = columns[7].Trim();
|
||||
|
||||
string noteText = columns[4].Trim();
|
||||
string noteTextHash = GetMD5Hash(noteText);
|
||||
string noteText = columns[4].Trim();
|
||||
string noteTextHash = GetMD5Hash(noteText);
|
||||
|
||||
if (!_noteDictionary.ContainsKey(noteTextHash))
|
||||
{
|
||||
_noteDictionary.Add(noteTextHash, noteText);
|
||||
}
|
||||
if (!_noteDictionary.ContainsKey(noteTextHash))
|
||||
{
|
||||
_noteDictionary.Add(noteTextHash, noteText);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(lineCode)
|
||||
&& !string.IsNullOrEmpty(partNumber)
|
||||
&& int.TryParse(columns[5], out int baseVehicleId)
|
||||
&& int.TryParse(columns[6], out int engineConfigId))
|
||||
{
|
||||
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash });
|
||||
}
|
||||
}
|
||||
if (!string.IsNullOrEmpty(lineCode)
|
||||
&& !string.IsNullOrEmpty(partNumber)
|
||||
&& int.TryParse(columns[5], out int baseVehicleId)
|
||||
&& int.TryParse(columns[6], out int engineConfigId))
|
||||
{
|
||||
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash });
|
||||
}
|
||||
}
|
||||
|
||||
return dataTable;
|
||||
}
|
||||
return dataTable;
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")]
|
||||
private string GetMD5Hash(string input)
|
||||
{
|
||||
using MD5 md5 = MD5.Create();
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")]
|
||||
private string GetMD5Hash(string input)
|
||||
{
|
||||
using MD5 md5 = MD5.Create();
|
||||
|
||||
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
|
||||
byte[] hashBytes = md5.ComputeHash(inputBytes);
|
||||
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
|
||||
byte[] hashBytes = md5.ComputeHash(inputBytes);
|
||||
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < hashBytes.Length; i++)
|
||||
{
|
||||
stringBuilder.Append(hashBytes[i].ToString("X2"));
|
||||
}
|
||||
for (int i = 0; i < hashBytes.Length; i++)
|
||||
{
|
||||
stringBuilder.Append(hashBytes[i].ToString("X2"));
|
||||
}
|
||||
|
||||
return stringBuilder.ToString();
|
||||
}
|
||||
}
|
||||
return stringBuilder.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user