Partsource/PartSource.Automation/Jobs/ProcessWhiVehicles.cs

using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Extensions;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Enums;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace PartSource.Automation.Jobs
{
	public class ProcessWhiVehicles : IAutomationJob
	{
		private readonly ILogger<ProcessWhiVehicles> _logger;
		private readonly WhiSeoService _whiSeoService;
		private readonly FtpConfiguration _ftpConfiguration;
		private readonly SeoDataType _seoDataType;

		public ProcessWhiVehicles(IConfiguration configuration, ILogger<ProcessWhiVehicles> logger, WhiSeoService whiSeoService)
		{
			_logger = logger;
			_whiSeoService = whiSeoService;

			_seoDataType = SeoDataType.Vehicle;

			_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();

		}

		public async Task Run(CancellationToken token, params string[] arguments)
		{
			_whiSeoService.TruncateVehicleTable();
			_whiSeoService.GetFiles(_seoDataType);

			string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
			DirectoryInfo directoryInfo = new DirectoryInfo(directory);

			IEnumerable<FileInfo> files = directoryInfo.GetFiles()
				.Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"))
				.OrderByDescending(f => f.GetWhiTimestamp());

			foreach (FileInfo fileInfo in files)
			{
				try
				{
					string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));

					DataTable dataTable = GetDataTable(fileInfo.FullName);

					_whiSeoService.BulkCopyVehicle(dataTable, tableName);
					_logger.LogInformation($"Copied {fileInfo.Name} to the database.");

					File.Delete(fileInfo.FullName);
				}

				catch (Exception ex)
				{
					_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
				}
			}

			_whiSeoService.CreateVehicleTable();

			_logger.LogInformation($"Created vehicle table.");
		}

		private DataTable GetDataTable(string filename)
		{
			using DataTable dataTable = new DataTable();
			dataTable.Columns.Add("Year", typeof(int));
			dataTable.Columns.Add("MakeId", typeof(int));
			dataTable.Columns.Add("MakeName", typeof(string));
			dataTable.Columns.Add("ModelId", typeof(int));
			dataTable.Columns.Add("ModelName", typeof(string));
			dataTable.Columns.Add("RegionId", typeof(int));
			dataTable.Columns.Add("RegionName", typeof(string));
			dataTable.Columns.Add("VehicleTypeId", typeof(int));
			dataTable.Columns.Add("EngineConfigId", typeof(int));
			dataTable.Columns.Add("EngineDescription", typeof(string));
			dataTable.Columns.Add("BaseVehicleId", typeof(int));
			dataTable.Columns.Add("VehicleToEngineConfigId", typeof(int));
			dataTable.Columns.Add("SubmodelId", typeof(int));
			dataTable.Columns.Add("SubmodelName", typeof(string));

			using StreamReader reader = new StreamReader(filename);
			string line = reader.ReadLine(); // Burn the header row

			while (reader.Peek() > 0)
			{
				line = reader.ReadLine();

				string[] columns = line.Split("\",\"");
				for (int i = 0; i < columns.Length; i++)
				{
					columns[i] = columns[i].Replace("\"", string.Empty);
				}

				string makeName = columns[4].Trim();
				string modelName = columns[6].Trim();
				string regionName = columns[8].Trim();
				string submodelName = columns[34].Trim();
				string engineDescription = columns[51].Trim();

				if (!string.IsNullOrEmpty(makeName)
					&& !string.IsNullOrEmpty(modelName)
					&& !string.IsNullOrEmpty(submodelName)
					&& !string.IsNullOrEmpty(engineDescription)
					&& int.TryParse(columns[0], out int baseVehicleId)
					&& int.TryParse(columns[2], out int year)
					&& int.TryParse(columns[3], out int makeId)
					&& int.TryParse(columns[5], out int modelId)
					&& int.TryParse(columns[7], out int regionId)
					&& int.TryParse(columns[9], out int vehicleTypeId)
					&& int.TryParse(columns[33], out int submodelId)
					&& int.TryParse(columns[35], out int engineConfigId)
					&& int.TryParse(columns[36], out int vehicleToEngineConfigId))
				{
                    if (regionId == 2 && new[] { 5, 6, 7 }.Contains(vehicleTypeId))
                    {
                        dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, regionId, regionName, vehicleTypeId, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName });
					}
				}
			}

			return dataTable;
		}
	}
}