soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stuff and things

Browse Source
This commit is contained in:
Tom Raterman
2021-06-29 19:00:13 -04:00
parent 0ff42f148a
commit 962ad3383f
22 changed files with 320 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -42,96 +42,38 @@ namespace PartSource.Automation.Jobs
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
ConcurrentQueue<FileInfo> files = new ConcurrentQueue<FileInfo>(directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).OrderBy(f => f.Length));
IEnumerable<IGrouping<string, FileInfo>> fileGroups = directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).GroupBy(x => x.Name.Split('_').Last());
while (files.Count > 0)
foreach (IGrouping<string, FileInfo> fileGroup in fileGroups)
{
Parallel.For(0, 4, index =>
foreach (FileInfo fileInfo in fileGroup)
{
if (!files.TryDequeue(out FileInfo fileInfo))
{
return;
}
string filename = Decompress(fileInfo);
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
try
{
int skippedLines = 0;
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
if (columns.Length != 8)
{
skippedLines++;
continue;
}
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
}
}
string filename = Decompress(fileInfo);
DataTable dataTable = GetDataTable(filename);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
_whiSeoService.BulkCopy(_seoDataType, dataTable, tableName);
if (skippedLines == 0)
{
_logger.LogInformation($"Copied {filename} to the database.");
}
_logger.LogInformation($"Copied {fileInfo.Name} to the database.");
else
{
_logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed.");
}
File.Delete(fileInfo.FullName);
}
catch (Exception ex)
{
_logger.LogError($"Failed to copy {filename} to the database.", ex);
}
try
{
reader.Close();
File.Delete(filename);
}
catch (Exception ex) { }
});
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
}
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
}
_whiSeoService.CreateFitmentView();
}
@@ -146,5 +88,45 @@ namespace PartSource.Automation.Jobs
return decompressedFile;
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
//dataTable.Columns.Add("Position", typeof(string));
//dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId }); //, position, noteText });
}
}
return dataTable;
}
}
}