soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Current state, whatever that means

Browse Source
This commit is contained in:
Tom Raterman
2022-03-17 20:04:12 -04:00
parent fb6dbdfaa7
commit 60edbee0b8
22 changed files with 1087 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -11,6 +11,7 @@ using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -24,6 +25,8 @@ namespace PartSource.Automation.Jobs
private readonly FtpConfiguration _ftpConfiguration;
private readonly SeoDataType _seoDataType;
private readonly IDictionary<string, string> _noteDictionary;
public ProcessWhiFitment(IConfiguration configuration, ILogger<ProcessWhiFitment> logger, WhiSeoService whiSeoService)
{
_logger = logger;
@@ -32,48 +35,68 @@ namespace PartSource.Automation.Jobs
_seoDataType = SeoDataType.Fitment;
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
_noteDictionary = new ConcurrentDictionary<string, string>();
}
public async Task Run()
{
_whiSeoService.Truncate();
_whiSeoService.GetFiles(_seoDataType);
_whiSeoService.TruncateFitmentTables();
// _whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
IEnumerable<IGrouping<string, FileInfo>> fileGroups = directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last());
ConcurrentQueue<IGrouping<string, FileInfo>> fileGroups = new ConcurrentQueue<IGrouping<string, FileInfo>>();
foreach (IGrouping<string, FileInfo> fileGroup in fileGroups)
foreach (IGrouping<string, FileInfo> fileGroup in directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last()))
{
foreach (FileInfo fileInfo in fileGroup)
fileGroups.Enqueue(fileGroup);
}
while (fileGroups.Count > 0)
{
Parallel.For(0, 2, i =>
{
try
bool result = fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup);
if (!result)
{
string filename = Decompress(fileInfo);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
return;
DataTable dataTable = GetDataTable(filename);
_whiSeoService.BulkCopy(_seoDataType, dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
File.Delete(filename);
}
catch (Exception ex)
foreach (FileInfo fileInfo in fileGroup)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
try
{
string filename = Decompress(fileInfo);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
DataTable dataTable = GetDataTable(filename);
_whiSeoService.BulkCopyFitment(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
File.Delete(filename);
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
}
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
});
}
_whiSeoService.CreateFitmentView();
_whiSeoService.SaveNotes(_noteDictionary);
}
public string Decompress(FileInfo fileInfo)
@@ -96,7 +119,7 @@ namespace PartSource.Automation.Jobs
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
dataTable.Columns.Add("FitmentNoteHash", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
@@ -112,20 +135,45 @@ namespace PartSource.Automation.Jobs
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9\\-]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
string noteTextHash = GetMD5Hash(noteText);
if (!_noteDictionary.ContainsKey(noteTextHash))
{
_noteDictionary.Add(noteTextHash, noteText);
}
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteTextHash });
}
}
return dataTable;
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "Not used for security")]
private string GetMD5Hash(string input)
{
using MD5 md5 = MD5.Create();
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
byte[] hashBytes = md5.ComputeHash(inputBytes);
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
stringBuilder.Append(hashBytes[i].ToString("X2"));
}
return stringBuilder.ToString();
}
}
}