using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using PartSource.Automation.Models.Configuration; using PartSource.Automation.Models.Enums; using PartSource.Automation.Services; using Ratermania.Automation.Interfaces; using System; using System.Collections.Generic; using System.Data; using System.IO; using System.IO.Compression; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace PartSource.Automation.Jobs { public class ProcessWhiFitment : IAutomationJob { private readonly ILogger _logger; private readonly WhiSeoService _whiSeoService; private readonly FtpConfiguration _ftpConfiguration; private readonly SeoDataType _seoDataType; public ProcessWhiFitment(IConfiguration configuration, ILogger logger, WhiSeoService whiSeoService) { _logger = logger; _whiSeoService = whiSeoService; _seoDataType = SeoDataType.Fitment; _ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get(); } public async Task Run() { _whiSeoService.Truncate(_seoDataType); _whiSeoService.GetFiles(_seoDataType); string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant()); DirectoryInfo directoryInfo = new DirectoryInfo(directory); foreach (FileInfo fileInfo in directoryInfo.GetFiles()) { if (!fileInfo.Name.EndsWith("csv.gz")) { continue; } string filename = Decompress(fileInfo); using DataTable dataTable = new DataTable(); dataTable.Columns.Add("LineCode", typeof(string)); dataTable.Columns.Add("PartNumber", typeof(string)); dataTable.Columns.Add("BaseVehicleId", typeof(int)); dataTable.Columns.Add("EngineConfigId", typeof(int)); dataTable.Columns.Add("Position", typeof(string)); dataTable.Columns.Add("NoteText", typeof(string)); using StreamReader reader = new StreamReader(filename); string line = reader.ReadLine(); // Burn the header row try { int skippedLines = 0; while (reader.Peek() > 0) { line = reader.ReadLine(); string[] columns = line.Replace("\"", string.Empty).Split(','); if (columns.Length != 8) { skippedLines++; continue; } string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim(); string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim(); string position = columns[7].Trim(); string noteText = columns[4].Trim(); if (!string.IsNullOrEmpty(lineCode) && !string.IsNullOrEmpty(partNumber) && int.TryParse(columns[5], out int baseVehicleId) && int.TryParse(columns[6], out int engineConfigId)) { dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText }); } } _whiSeoService.BulkCopy(_seoDataType, dataTable); if (skippedLines == 0) { _logger.LogInformation($"Copied {filename} to the database."); } else { _logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed."); } } catch (Exception ex) { _logger.LogError($"Failed to copy {filename} to the database.", ex); } try { reader.Close(); File.Delete(filename); File.Delete(fileInfo.FullName); } catch (Exception ex) { _logger.LogWarning($"Failed to delete {filename}. This file will need to be deleted manually.", ex); } } } public string Decompress(FileInfo fileInfo) { string decompressedFile = fileInfo.FullName.Remove(fileInfo.FullName.Length - fileInfo.Extension.Length); using FileStream filestream = File.Create(decompressedFile); using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress); decompressionStream.CopyTo(filestream); return decompressedFile; } } }