soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migration to DevOps

Browse Source
This commit is contained in:
Tom Raterman
2021-03-30 10:09:57 -04:00
parent e1be26d798
commit 3754beb035
23 changed files with 342 additions and 1700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -5,10 +5,12 @@ using PartSource.Automation.Models.Enums;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -34,92 +36,103 @@ namespace PartSource.Automation.Jobs
public async Task Run()
{
_whiSeoService.Truncate(_seoDataType);
_whiSeoService.Truncate();
_whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
foreach (FileInfo fileInfo in directoryInfo.GetFiles())
ConcurrentQueue<FileInfo> files = new ConcurrentQueue<FileInfo>(directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).OrderBy(f => f.Length));
while (files.Count > 0)
{
if (!fileInfo.Name.EndsWith("csv.gz"))
Parallel.For(0, 4, index =>
{
continue;
}
string filename = Decompress(fileInfo);
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
try
{
int skippedLines = 0;
while (reader.Peek() > 0)
if (!files.TryDequeue(out FileInfo fileInfo))
{
line = reader.ReadLine();
return;
}
string[] columns = line.Replace("\"", string.Empty).Split(',');
string filename = Decompress(fileInfo);
if (columns.Length != 8)
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
try
{
int skippedLines = 0;
while (reader.Peek() > 0)
{
skippedLines++;
continue;
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
if (columns.Length != 8)
{
skippedLines++;
continue;
}
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
}
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
_whiSeoService.BulkCopy(_seoDataType, dataTable, tableName);
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
if (skippedLines == 0)
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
_logger.LogInformation($"Copied {filename} to the database.");
}
else
{
_logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed.");
}
File.Delete(fileInfo.FullName);
}
_whiSeoService.BulkCopy(_seoDataType, dataTable);
if (skippedLines == 0)
catch (Exception ex)
{
_logger.LogInformation($"Copied {filename} to the database.");
_logger.LogError($"Failed to copy {filename} to the database.", ex);
}
else
try
{
_logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed.");
reader.Close();
File.Delete(filename);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to copy {filename} to the database.", ex);
}
try
{
reader.Close();
File.Delete(filename);
File.Delete(fileInfo.FullName);
}
catch (Exception ex)
{
_logger.LogWarning($"Failed to delete {filename}. This file will need to be deleted manually.", ex);
}
catch (Exception ex) { }
});
}
_whiSeoService.CreateFitmentView();
}
public string Decompress(FileInfo fileInfo)
@@ -128,7 +141,7 @@ namespace PartSource.Automation.Jobs
using FileStream filestream = File.Create(decompressedFile);
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
decompressionStream.CopyTo(filestream);
return decompressedFile;