using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using PartSource.Automation.Extensions; using PartSource.Automation.Models.Configuration; using PartSource.Automation.Models.Enums; using PartSource.Automation.Services; using Ratermania.Automation.Interfaces; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; using System.IO; using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; namespace PartSource.Automation.Jobs { public class ProcessWhiVehicles : IAutomationJob { private readonly ILogger _logger; private readonly WhiSeoService _whiSeoService; private readonly FtpConfiguration _ftpConfiguration; private readonly SeoDataType _seoDataType; public ProcessWhiVehicles(IConfiguration configuration, ILogger logger, WhiSeoService whiSeoService) { _logger = logger; _whiSeoService = whiSeoService; _seoDataType = SeoDataType.Vehicle; _ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get(); } public async Task Run(CancellationToken token, params string[] arguments) { _whiSeoService.TruncateVehicleTable(); _whiSeoService.GetFiles(_seoDataType); string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant()); DirectoryInfo directoryInfo = new DirectoryInfo(directory); IEnumerable files = directoryInfo.GetFiles() .Where(f => f.Name.StartsWith("seo_aces_vehicle_feed")) .OrderByDescending(f => f.GetWhiTimestamp()); foreach (FileInfo fileInfo in files) { try { string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); DataTable dataTable = GetDataTable(fileInfo.FullName); _whiSeoService.BulkCopyVehicle(dataTable, tableName); _logger.LogInformation($"Copied {fileInfo.Name} to the database."); File.Delete(fileInfo.FullName); } catch (Exception ex) { _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); } } _whiSeoService.CreateVehicleTable(); _logger.LogInformation($"Created vehicle table."); } private DataTable GetDataTable(string filename) { using DataTable dataTable = new DataTable(); dataTable.Columns.Add("Year", typeof(int)); dataTable.Columns.Add("MakeId", typeof(int)); dataTable.Columns.Add("MakeName", typeof(string)); dataTable.Columns.Add("ModelId", typeof(int)); dataTable.Columns.Add("ModelName", typeof(string)); dataTable.Columns.Add("RegionId", typeof(int)); dataTable.Columns.Add("RegionName", typeof(string)); dataTable.Columns.Add("VehicleTypeId", typeof(int)); dataTable.Columns.Add("EngineConfigId", typeof(int)); dataTable.Columns.Add("EngineDescription", typeof(string)); dataTable.Columns.Add("BaseVehicleId", typeof(int)); dataTable.Columns.Add("VehicleToEngineConfigId", typeof(int)); dataTable.Columns.Add("SubmodelId", typeof(int)); dataTable.Columns.Add("SubmodelName", typeof(string)); using StreamReader reader = new StreamReader(filename); string line = reader.ReadLine(); // Burn the header row while (reader.Peek() > 0) { line = reader.ReadLine(); string[] columns = line.Split("\",\""); for (int i = 0; i < columns.Length; i++) { columns[i] = columns[i].Replace("\"", string.Empty); } string makeName = columns[4].Trim(); string modelName = columns[6].Trim(); string regionName = columns[8].Trim(); string submodelName = columns[34].Trim(); string engineDescription = columns[51].Trim(); if (!string.IsNullOrEmpty(makeName) && !string.IsNullOrEmpty(modelName) && !string.IsNullOrEmpty(submodelName) && !string.IsNullOrEmpty(engineDescription) && int.TryParse(columns[0], out int baseVehicleId) && int.TryParse(columns[2], out int year) && int.TryParse(columns[3], out int makeId) && int.TryParse(columns[5], out int modelId) && int.TryParse(columns[7], out int regionId) && int.TryParse(columns[9], out int vehicleTypeId) && int.TryParse(columns[33], out int submodelId) && int.TryParse(columns[35], out int engineConfigId) && int.TryParse(columns[36], out int vehicleToEngineConfigId)) { if (regionId == 2 && new[] { 5, 6, 7 }.Contains(vehicleTypeId)) { dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, regionId, regionName, vehicleTypeId, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName }); } } } return dataTable; } } }