From d95d947bc2318a06215c6a68a175f8ec96b903bd Mon Sep 17 00:00:00 2001 From: Tom Raterman Date: Sat, 25 Mar 2023 17:50:25 -0400 Subject: [PATCH] WIP --- .../Jobs/POC/PartialInventoryUpdate.cs | 84 ++++++++++++ PartSource.Automation/Jobs/UpdateFitment.cs | 2 +- .../Models/Ftp/FtpFileInfo.cs | 19 +++ .../Models/Ftp/FtpFileType.cs | 14 ++ PartSource.Automation/Program.cs | 2 +- PartSource.Automation/Services/FtpService.cs | 129 +++++++++++++----- 6 files changed, 215 insertions(+), 35 deletions(-) create mode 100644 PartSource.Automation/Jobs/POC/PartialInventoryUpdate.cs create mode 100644 PartSource.Automation/Models/Ftp/FtpFileInfo.cs create mode 100644 PartSource.Automation/Models/Ftp/FtpFileType.cs diff --git a/PartSource.Automation/Jobs/POC/PartialInventoryUpdate.cs b/PartSource.Automation/Jobs/POC/PartialInventoryUpdate.cs new file mode 100644 index 0000000..754cc30 --- /dev/null +++ b/PartSource.Automation/Jobs/POC/PartialInventoryUpdate.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using PartSource.Automation.Models.Configuration; +using PartSource.Automation.Models.Ftp; +using PartSource.Automation.Services; +using Ratermania.Automation.Interfaces; + +namespace PartSource.Automation.Jobs.POC +{ + public class PartialInventoryUpdate : IAutomationJob + { + private readonly FtpService _ftpService; + + public PartialInventoryUpdate(IConfiguration configuration) + { + FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get(); + _ftpService = new FtpService(ftpConfiguration); + } + + public async Task Run(CancellationToken token, params string[] arguments) + { + FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended("") + .Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf("Availability") > -1) + .OrderByDescending(f => f.Modified) + .First(); + + string file = _ftpService.Download(lastUploadedFile.Filename, Path.GetTempPath()); + + DataTable dataTable = GetDataTable(file); + + using SqlConnection connection = new SqlConnection("Server=tcp:ps-automation-stage.eastus2.cloudapp.azure.com,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=stageuser;Password=]FXepK^cFYS|[H<;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=True;Connection Timeout=30;"); + connection.Open(); + + using SqlBulkCopy bulk = new SqlBulkCopy(connection) + { + DestinationTableName = $"PartAvailability", + BulkCopyTimeout = 14400 + }; + + bulk.WriteToServer(dataTable); + + return; + } + + private DataTable GetDataTable(string filename) + { + using DataTable dataTable = new DataTable(); + dataTable.Columns.Add("Store", typeof(int)); + dataTable.Columns.Add("SKU", typeof(int)); + dataTable.Columns.Add("QTY", typeof(int)); + + using StreamReader reader = new StreamReader(filename); + string line = reader.ReadLine(); // Burn the header row + + while (reader.Peek() > 0) + { + line = reader.ReadLine(); + + string[] columns = line.Split("|"); + for (int i = 0; i < columns.Length; i++) + { + columns[i] = columns[i].Replace("\"", string.Empty); + } + + if (int.TryParse(columns[0], out int store) + && int.TryParse(columns[1], out int sku) + && int.TryParse(columns[2], out int quantity)) + { + dataTable.Rows.Add(new object[] { store, sku, quantity }); + } + } + + return dataTable; + } + } +} diff --git a/PartSource.Automation/Jobs/UpdateFitment.cs b/PartSource.Automation/Jobs/UpdateFitment.cs index 3b74881..5eb9c25 100644 --- a/PartSource.Automation/Jobs/UpdateFitment.cs +++ b/PartSource.Automation/Jobs/UpdateFitment.cs @@ -40,7 +40,7 @@ namespace PartSource.Automation.Jobs try { - products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 }, { "product_type", "CA111-SC250-FL25049_Entry Ball Joints" } }); + products = await _shopifyClient.Products.Get(new Dictionary { { "limit", 250 } }); //products = new List //{ // await _shopifyClient.Products.GetById(4388919574575) diff --git a/PartSource.Automation/Models/Ftp/FtpFileInfo.cs b/PartSource.Automation/Models/Ftp/FtpFileInfo.cs new file mode 100644 index 0000000..7c76489 --- /dev/null +++ b/PartSource.Automation/Models/Ftp/FtpFileInfo.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace PartSource.Automation.Models.Ftp +{ + public class FtpFileInfo + { + public string Filename { get; set; } + + public DateTime Modified { get; set; } + + public long Size { get; set; } + + public FtpFileType FileType { get; set; } + } +} diff --git a/PartSource.Automation/Models/Ftp/FtpFileType.cs b/PartSource.Automation/Models/Ftp/FtpFileType.cs new file mode 100644 index 0000000..2c9dcf7 --- /dev/null +++ b/PartSource.Automation/Models/Ftp/FtpFileType.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace PartSource.Automation.Models.Ftp +{ + public enum FtpFileType + { + File, + Directory + } +} diff --git a/PartSource.Automation/Program.cs b/PartSource.Automation/Program.cs index 90a8d1d..8790f7e 100644 --- a/PartSource.Automation/Program.cs +++ b/PartSource.Automation/Program.cs @@ -100,7 +100,7 @@ namespace PartSource.Automation // //.StartsAt(DateTime.Today.AddHours(25)) // ) - .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) //.HasDependency() // .StartsAt(DateTime.Today.AddHours(28)) ); diff --git a/PartSource.Automation/Services/FtpService.cs b/PartSource.Automation/Services/FtpService.cs index 25ed55a..84d5a80 100644 --- a/PartSource.Automation/Services/FtpService.cs +++ b/PartSource.Automation/Services/FtpService.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Configuration; using PartSource.Automation.Models.Configuration; +using PartSource.Automation.Models.Ftp; using System; using System.Collections.Generic; using System.Configuration; @@ -9,44 +10,106 @@ using System.Threading.Tasks; namespace PartSource.Automation.Services { - public class FtpService - { - private readonly FtpConfiguration _ftpConfiguration; + public class FtpService + { + private readonly FtpConfiguration _ftpConfiguration; - public FtpService(FtpConfiguration ftpConfiguration) - { - _ftpConfiguration = ftpConfiguration; - } + public FtpService(FtpConfiguration ftpConfiguration) + { + _ftpConfiguration = ftpConfiguration; + } - public string[] ListFiles(string directory) - { - FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}")); - request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); - request.Method = WebRequestMethods.Ftp.ListDirectory; + public IList ListFilesExtended(string directory) + { + FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}")); + request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); + request.Method = WebRequestMethods.Ftp.ListDirectoryDetails; - using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); - using StreamReader reader = new StreamReader(response.GetResponseStream()); + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); + using StreamReader reader = new StreamReader(response.GetResponseStream()); - string files = reader.ReadToEnd(); + IList files = new List(); + string[] fileStrings = reader.ReadToEnd().Split("\r\n"); - return files.Length > 0 - ? files.Split("\r\n") - : Array.Empty(); - } + foreach (string fileString in fileStrings) + { + if (string.IsNullOrEmpty(fileString)) + { + continue; + } - public void Download(string filename) - { - string file = $"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}"; + string dateString = fileString[..17]; + string[] sizeAndName = fileString[18..].TrimStart().Split(" ", 2); - FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}")); - request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); - request.Method = WebRequestMethods.Ftp.DownloadFile; - - using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); - using Stream responseStream = response.GetResponseStream(); - using FileStream fileStream = new FileStream($"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}", FileMode.Create); - - responseStream.CopyTo(fileStream); - } - } + if (sizeAndName[0].ToUpperInvariant().IndexOf("DIR") > -1) + { + files.Add(new FtpFileInfo + { + Modified = DateTime.Parse(fileString[..17]), + Size = 0, + Filename = sizeAndName[1].Trim(), + FileType = FtpFileType.Directory + }); + } + + else + { + files.Add(new FtpFileInfo + { + Modified = DateTime.Parse(fileString[..17]), + Size = long.Parse(sizeAndName[0]), + Filename = sizeAndName[1].Trim(), + FileType = FtpFileType.File + }); + } + } + + return files; + } + + public string[] ListFiles(string directory) + { + FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}")); + request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); + request.Method = WebRequestMethods.Ftp.ListDirectory; + + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); + using StreamReader reader = new StreamReader(response.GetResponseStream()); + + string files = reader.ReadToEnd(); + + return files.Length > 0 + ? files.Split("\r\n") + : Array.Empty(); + } + + public string Download(string filename, string destination = null) + { + + + FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}")); + request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password); + request.Method = WebRequestMethods.Ftp.DownloadFile; + + if (string.IsNullOrEmpty(destination)) + { + destination = _ftpConfiguration.Destination; + } + + if (Path.DirectorySeparatorChar == '\\') + { + filename = filename.Replace("/", "\\"); + } + + destination = Path.Combine(destination, filename); + + using FtpWebResponse response = (FtpWebResponse)request.GetResponse(); + using Stream responseStream = response.GetResponseStream(); + using FileStream fileStream = new FileStream(destination, FileMode.Create); + + responseStream.CopyTo(fileStream); + + return destination; + } + } }