14 Commits

Author SHA1 Message Date
41a7f57988 Add support for inventory timestamps 2025-05-29 09:55:07 -04:00
bd6682e861 Merge conflicts 2025-05-28 10:35:06 -04:00
bbeb96dbda Minor changes 2025-05-28 10:34:35 -04:00
ca45a77a0f Tweaks for stability 2025-04-09 11:51:59 -04:00
8b6892df60 Some things broke 2025-04-03 21:19:33 -04:00
f1ca48c1d0 FTP configuration 2025-04-03 20:49:56 -04:00
57f42a0e47 Finalize hourly inventory configuration 2025-04-03 20:48:03 -04:00
eb928a7c56 Merge pull request 'HourlyInventory' (#1) from HourlyInventory into master
Reviewed-on: #1
2025-04-04 00:35:03 +00:00
36b05af60e Merge from master 2025-04-03 20:34:29 -04:00
a3a08d9cff WIP 2025-04-03 20:28:46 -04:00
857f94c59a Hourly inventory updates 2025-04-04 00:18:03 +00:00
cbf7bb8de6 Namespace migration 2023-09-13 06:34:53 -04:00
68c9e01ef1 . 2023-08-23 15:04:54 -04:00
d95d947bc2 WIP 2023-03-25 17:50:25 -04:00
23 changed files with 775 additions and 180 deletions

View File

@@ -23,7 +23,7 @@ namespace PartSource.Api.Controllers
[Route("sku/{sku}/storeNumber/{storeNumber}")]
public async Task<ActionResult> GetInventory(int sku, int storeNumber)
{
PartsAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
PartAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
if (inventory == null)
{
@@ -36,7 +36,8 @@ namespace PartSource.Api.Controllers
{
StoreNumber = inventory.Store,
Sku = sku,
Quantity = inventory.QTY
Quantity = inventory.QTY,
Updated = inventory.Updated ?? System.DateTime.MinValue
}
});
}

View File

@@ -50,16 +50,14 @@ namespace PartSource.Api
});
services.AddAutoMapper(typeof(PartSourceProfile));
services.AddTransient<PartService>();
services.AddTransient<FitmentService>();
services.AddTransient<PartService>();
services.AddTransient<NexpartService>();
services.AddTransient<SecurityService>();
services.AddTransient<VehicleService>();
services.AddTransient<ShopifyChangelogService>();
services.AddTransient<FitmentContext>();
services.AddCors(o => o.AddPolicy("Default", builder =>
{
builder.AllowAnyOrigin()

View File

@@ -1,6 +1,7 @@
{
"ConnectionStrings": {
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;",
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=automation;Password=)6L)XP%m(x-UU#M;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
//"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true"
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
},

View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace PartSource.Automation.Extensions
{
public static class FileInfoExtensions
{
public static DateTime GetWhiTimestamp(this FileInfo fileInfo)
{
Match match = Regex.Match(fileInfo.Name, "[0-9]{8}");
return match.Success && DateTime.TryParseExact(match.Value, "MMddyyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime timestamp)
? timestamp
: DateTime.MinValue;
}
}
}

View File

@@ -0,0 +1,105 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
namespace PartSource.Automation.Jobs
{
public class BulkUpdateInventory : IAutomationJob
{
private readonly FtpService _ftpService;
private readonly ILogger<BulkUpdateInventory> _logger;
private readonly string _connectionString;
public BulkUpdateInventory(IConfiguration configuration, ILogger<BulkUpdateInventory> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_connectionString = configuration.GetConnectionString("PartSourceDatabase");
_logger = logger;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf("Availability Full") > -1)
.OrderByDescending(f => f.Modified)
.FirstOrDefault();
if (lastUploadedFile == null)
{
_logger.LogInformation($"No full inventory file available.");
return;
}
string file = _ftpService.Download(lastUploadedFile.Filename);
DataTable dataTable = GetDataTable(file);
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using SqlCommand command = new SqlCommand("TRUNCATE TABLE PartAvailability", connection);
await command.ExecuteNonQueryAsync(token);
using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{
DestinationTableName = "PartAvailability",
BulkCopyTimeout = 14400
};
await bulk.WriteToServerAsync(dataTable, token);
_ftpService.Delete(lastUploadedFile.Filename);
return;
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("Store", typeof(int));
dataTable.Columns.Add("SKU", typeof(string));
dataTable.Columns.Add("QTY", typeof(int));
dataTable.Columns.Add("Updated", typeof(DateTime));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("|");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string sku = columns[1].Trim();
if (int.TryParse(columns[0], out int store)
&& !string.IsNullOrEmpty(sku)
&& int.TryParse(columns[2], out int quantity)
&& DateTime.TryParse(columns[3], out DateTime updated))
{
dataTable.Rows.Add(new object[] { store, sku, quantity, updated });
}
}
return dataTable;
}
}
}

View File

@@ -1,9 +1,11 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -17,11 +19,11 @@ namespace PartSource.Automation.Jobs
private readonly ILogger<ExecuteSsisPackages> _logger;
// TODO: set from config
private readonly string[] _ssisPackages = {"Parts Availability" };
private readonly string[] _ssisPackages = {"Parts Price" };
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_emailService = emailService;
_ftpService = new FtpService(ftpConfiguration);
@@ -36,7 +38,18 @@ namespace PartSource.Automation.Jobs
{
try
{
// _ftpService.Download($"{package}.txt");
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
.OrderByDescending(f => f.Modified)
.FirstOrDefault();
if (lastUploadedFile == null)
{
_logger.LogInformation($"No {package} file available.");
return;
}
_ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
@@ -45,7 +58,6 @@ namespace PartSource.Automation.Jobs
catch (Exception ex)
{
_logger.LogError($"Execution of SSIS package {package} failed.", ex);
throw;
}
}

View File

@@ -0,0 +1,119 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly FitmentContext _fitmentContext;
private readonly PartService _partService;
private readonly ShopifyClient _shopifyClient;
public GetImageUrls(NexpartService nexpartService, PartService partService, FitmentContext fitmentContext, ShopifyClient shopifyClient)
{
_nexpartService = nexpartService;
_fitmentContext = fitmentContext;
_partService = partService;
_shopifyClient = shopifyClient;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> rows = new List<string> {
"\"Line Code\", \"Part Number\", \"Image URL(s)\""
};
using StreamReader reader = new StreamReader("C:\\Users\\Tom\\Desktop\\image parts.csv");
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split(",");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string partsourceCode = columns[0].Trim();
string partNumber = columns[1].Trim();
IList<DcfMapping> dcfMappings = await _partService.GetDcfMapping(partsourceCode);
if (dcfMappings.Count == 0)
{
Console.WriteLine($"No images for {partsourceCode} {partNumber}");
}
bool hasImage = false;
foreach (DcfMapping mapping in dcfMappings)
{
if (hasImage)
{
continue;
}
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = mapping.WhiCode,
PartNumber = partNumber
}
},
DataOption = new[] { "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
rows.Add($"\"{partsourceCode}\", \"{partNumber}\", \"{string.Join(";", urls)}\"");
hasImage = true;
}
}
}
if (!hasImage)
{
Console.WriteLine($"No images for {partsourceCode} {partNumber}");
}
}
await File.WriteAllLinesAsync($"C:\\users\\Tom\\desktop\\WHI Images {DateTime.Now:yyyyMMdd}.csv", rows);
}
}
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrlsTemp : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly FitmentContext _fitmentContext;
private readonly PartService _partService;
private readonly ShopifyClient _shopifyClient;
private readonly string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
public GetImageUrlsTemp(NexpartService nexpartService, PartService partService, FitmentContext fitmentContext, ShopifyClient shopifyClient)
{
_nexpartService = nexpartService;
_fitmentContext = fitmentContext;
_partService = partService;
_shopifyClient = shopifyClient;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<KeyValuePair<string, string>> parts = new List<KeyValuePair<string, string>>();
parts.Add(new KeyValuePair<string, string>("DAY", "89310"));
parts.Add(new KeyValuePair<string, string>("CNI", "141.40113"));
parts.Add(new KeyValuePair<string, string>("PRF", "MU19631"));
parts.Add(new KeyValuePair<string, string>("TRK", "SB8100"));
parts.Add(new KeyValuePair<string, string>("MON", "906970"));
parts.Add(new KeyValuePair<string, string>("FEL", "70804"));
parts.Add(new KeyValuePair<string, string>("FEL", "SS71198"));
parts.Add(new KeyValuePair<string, string>("CFP", "STS314"));
parts.Add(new KeyValuePair<string, string>("NGK", "21517"));
parts.Add(new KeyValuePair<string, string>("NGK", "RC-XX89"));
parts.Add(new KeyValuePair<string, string>("FRA", "CA176"));
for (int i = 0; i < chars.Length; i++)
{
for (int j = 0; j < chars.Length; j++)
{
for (int k = 0; k < chars.Length; k++)
{
string actualLineCode = $"{chars[i]}{chars[j]}{chars[k]}";
System.Diagnostics.Debug.WriteLine(actualLineCode);
foreach (KeyValuePair<string, string> part in parts)
{
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = actualLineCode,
PartNumber = part.Value
}
},
DataOption = new[] { "DIST_LINE", "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
Console.WriteLine($"Image {urls[0]} found for {part.Value}. Expected: {part.Key}, Actual: {actualLineCode}");
}
}
}
}
}
}
}
}
}

View File

@@ -18,13 +18,13 @@ using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
public class ImageList : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
public ImageList(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
{
_nexpartService = nexpartService;
_partSourceContext = partSourceContext;

View File

@@ -0,0 +1,117 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
namespace PartSource.Automation.Jobs
{
public class PartialInventoryUpdate : IAutomationJob
{
private readonly FtpService _ftpService;
private readonly ILogger<PartialInventoryUpdate> _logger;
private readonly string _connectionString;
public PartialInventoryUpdate(IConfiguration configuration, ILogger<PartialInventoryUpdate> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_connectionString = _connectionString = configuration.GetConnectionString("PartSourceDatabase");
_logger = logger;
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities")]
public async Task Run(CancellationToken token, params string[] arguments)
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf("Availability Partial") > -1)
.OrderByDescending(f => f.Modified)
.FirstOrDefault();
if (lastUploadedFile == null)
{
_logger.LogInformation($"No partial inventory file available.");
return;
}
_logger.LogInformation("Processing {filename}", lastUploadedFile.Filename);
string file = _ftpService.Download($"{lastUploadedFile.Filename}");
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using StreamReader reader = new StreamReader(file);
string line = reader.ReadLine(); // Burn the header row
IDictionary<string, object> parameters = new Dictionary<string, object>();
string command = string.Empty;
int i = 0;
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("|");
for (int j = 0; j < columns.Length; j++)
{
columns[j] = columns[j].Replace("\"", string.Empty);
}
if (int.TryParse(columns[0], out int store)
&& int.TryParse(columns[1], out int sku)
&& int.TryParse(columns[2], out int quantity)
&& DateTime.TryParse(columns[3], out DateTime updated))
{
command += $"UPDATE PartAvailability SET QTY = @qty_{i}, Updated = @updated_{i} WHERE SKU = @sku_{i} AND Store = @store_{i};";
parameters.Add($"qty_{i}", quantity);
parameters.Add($"store_{i}", store);
parameters.Add($"sku_{i}", sku);
parameters.Add($"updated_{i}", updated);
i++;
}
if (i == 250)
{
using SqlCommand nested = new SqlCommand(command, connection);
foreach (KeyValuePair<string, object> parameter in parameters)
{
nested.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
}
await nested.ExecuteNonQueryAsync(token);
parameters.Clear();
command = string.Empty;
i = 0;
}
}
using SqlCommand sqlCommand = new SqlCommand(command, connection);
foreach (KeyValuePair<string, object> parameter in parameters)
{
sqlCommand.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
}
await sqlCommand.ExecuteNonQueryAsync(token);
_ftpService.Delete(lastUploadedFile.Filename);
return;
}
}
}

View File

@@ -1,25 +1,21 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Extensions;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Enums;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class ProcessWhiVehicles : IAutomationJob
public class ProcessWhiVehicles : IAutomationJob
{
private readonly ILogger<ProcessWhiVehicles> _logger;
private readonly WhiSeoService _whiSeoService;
@@ -45,7 +41,9 @@ namespace PartSource.Automation.Jobs
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
IEnumerable<FileInfo> files = directoryInfo.GetFiles().Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"));
IEnumerable<FileInfo> files = directoryInfo.GetFiles()
.Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"))
.OrderByDescending(f => f.GetWhiTimestamp());
foreach (FileInfo fileInfo in files)
{

View File

@@ -93,7 +93,7 @@ namespace PartSource.Automation.Jobs
? string.Empty
: product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
if (!vehicleIdFitment.Any())

View File

@@ -64,7 +64,7 @@ namespace PartSource.Automation.Jobs
}
IList<Fitment> fitments = GetPositionOrderedFitments(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
if (fitments.Count == 0 || vehicles.Count == 0)
{

View File

@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PartSource.Automation.Models.Ftp
{
public class FtpFileInfo
{
public string Filename { get; set; }
public DateTime Modified { get; set; }
public long Size { get; set; }
public FtpFileType FileType { get; set; }
}
}

View File

@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PartSource.Automation.Models.Ftp
{
public enum FtpFileType
{
File,
Directory
}
}

View File

@@ -1,18 +1,14 @@
using AutoMapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Jobs;
using PartSource.Automation.Jobs.POC;
using PartSource.Automation.Services;
using PartSource.Data;
using PartSource.Data.AutoMapper;
using PartSource.Data.Contexts;
using PartSource.Services;
using Ratermania.Automation.DependencyInjection;
using Ratermania.Automation.Logging;
using Ratermania.Shopify.DependencyInjection;
using System;
using System.IO;
@@ -20,50 +16,51 @@ using System.Threading.Tasks;
namespace PartSource.Automation
{
class Program
{
static async Task Main(string[] args){
try
{
using IHost host = CreateHostBuilder().Build();
class Program
{
static async Task Main(string[] args)
{
try
{
using IHost host = CreateHostBuilder().Build();
await host.StartAsync();
}
await host.StartAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
private static IHostBuilder CreateHostBuilder()
{
return Host.CreateDefaultBuilder()
.ConfigureAppConfiguration(builder =>
{
string environment = Environment.GetEnvironmentVariable("AUTOMATION_ENVIRONMENT");
private static IHostBuilder CreateHostBuilder()
{
return Host.CreateDefaultBuilder()
.ConfigureAppConfiguration(builder =>
{
string environment = Environment.GetEnvironmentVariable("AUTOMATION_ENVIRONMENT");
builder.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{environment}.json", optional: true, reloadOnChange: true);
})
.ConfigureServices((builder, services) =>
{
services.AddDbContext<PartSourceContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("PartSourceDatabase"), opts => opts.EnableRetryOnFailure())
)
builder.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{environment}.json", optional: true, reloadOnChange: true);
})
.ConfigureServices((builder, services) =>
{
services.AddDbContext<PartSourceContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("PartSourceDatabase"), opts => opts.EnableRetryOnFailure())
)
.AddDbContext<FitmentContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts =>
{
opts.EnableRetryOnFailure();
opts.CommandTimeout(600);
})
)
.AddDbContext<FitmentContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts =>
{
opts.EnableRetryOnFailure();
opts.CommandTimeout(600);
})
)
.AddShopify(options =>
{
.AddShopify(options =>
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
options.ApiVersion = "2022-10";
@@ -75,56 +72,69 @@ namespace PartSource.Automation
//options.ShopDomain = "dev-partsource.myshopify.com";
})
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 15, 0))
.HasMaxFailures(1)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>()
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>()
// .HasDependency<SyncronizeProducts>()
// .StartsAt(DateTime.Today.AddHours(8))
//) ;
//.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//)
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 5, 0))
.HasMaxFailures(5)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>()
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>()
// .HasDependency<SyncronizeProducts>()
// .StartsAt(DateTime.Today.AddHours(8))
//) ;
//.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//)
.HasJob<ExecuteSsisPackages>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(-24))
)
.HasJob<UpdatePricing>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(-22))
.HasDependency<ExecuteSsisPackages>()
)
.HasJob<BulkUpdateInventory>(options =>
options.HasInterval(new TimeSpan(1, 0, 0))
.StartsAt(DateTime.Today.AddHours(-27))
)
.HasJob<PartialInventoryUpdate>(options =>
options.HasInterval(new TimeSpan(1, 0, 0))
.StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
);
.HasJob<PartsSync>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddMinutes(15))
// )
//.HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<ExecuteSsisPackages>()
// // .StartsAt(DateTime.Today.AddHours(28))
);
//);
//.AddApiServer();
//.HasJob<PartialInventoryUpdate>(options => options.HasInterval(new TimeSpan(1, 0, 0))
//.HasDependency<ExecuteSsisPackages>()
// .StartsAt(DateTime.Today)
//);
//);
//.AddApiServer();
})
.AddSingleton<EmailService>()
.AddSingleton<SsisService>()
.AddSingleton<WhiSeoService>()
.AddSingleton<VehicleService>()
.AddSingleton<VehicleFitmentService>()
.AddSingleton<NexpartService>()
.AddSingleton<EmailService>()
.AddSingleton<SsisService>()
.AddSingleton<WhiSeoService>()
.AddSingleton<VehicleService>()
.AddSingleton<VehicleFitmentService>()
.AddSingleton<NexpartService>()
.AddSingleton<PartService>()
.AddAutoMapper(typeof(PartSourceProfile));
})
.ConfigureLogging((builder, logging) =>
{
logging.AddEventLog();
logging.AddConsole();
.AddAutoMapper(typeof(PartSourceProfile));
})
.ConfigureLogging((builder, logging) =>
{
logging.AddEventLog();
logging.AddConsole();
// logging.AddProvider(new AutomationLoggerProvider());
});
}
}
// logging.AddProvider(new AutomationLoggerProvider());
});
}
}
}

View File

@@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using System;
using System.Collections.Generic;
using System.Configuration;
@@ -9,44 +10,119 @@ using System.Threading.Tasks;
namespace PartSource.Automation.Services
{
public class FtpService
{
private readonly FtpConfiguration _ftpConfiguration;
public class FtpService
{
private readonly FtpConfiguration _ftpConfiguration;
public FtpService(FtpConfiguration ftpConfiguration)
{
_ftpConfiguration = ftpConfiguration;
}
public FtpService(FtpConfiguration ftpConfiguration)
{
_ftpConfiguration = ftpConfiguration;
}
public string[] ListFiles(string directory)
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.ListDirectory;
public IList<FtpFileInfo> ListFilesExtended(string directory = "")
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.ListDirectoryDetails;
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
using StreamReader reader = new StreamReader(response.GetResponseStream());
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
using StreamReader reader = new StreamReader(response.GetResponseStream());
string files = reader.ReadToEnd();
IList<FtpFileInfo> files = new List<FtpFileInfo>();
string[] fileStrings = reader.ReadToEnd().Split("\r\n");
return files.Length > 0
? files.Split("\r\n")
: Array.Empty<string>();
}
foreach (string fileString in fileStrings)
{
if (string.IsNullOrEmpty(fileString))
{
continue;
}
public void Download(string filename)
{
string file = $"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}";
string dateString = fileString[..17];
string[] sizeAndName = fileString[18..].TrimStart().Split(" ", 2);
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.DownloadFile;
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
using Stream responseStream = response.GetResponseStream();
using FileStream fileStream = new FileStream($"{_ftpConfiguration.Destination}\\{filename.Replace("/", "\\")}", FileMode.Create);
responseStream.CopyTo(fileStream);
}
}
if (sizeAndName[0].ToUpperInvariant().IndexOf("DIR") > -1)
{
files.Add(new FtpFileInfo
{
Modified = DateTime.Parse(fileString[..17]),
Size = 0,
Filename = sizeAndName[1].Trim(),
FileType = FtpFileType.Directory
});
}
else
{
files.Add(new FtpFileInfo
{
Modified = DateTime.Parse(fileString[..17]),
Size = long.Parse(sizeAndName[0]),
Filename = sizeAndName[1].Trim(),
FileType = FtpFileType.File
});
}
}
return files;
}
public string[] ListFiles(string directory = "")
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{directory}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.ListDirectory;
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
using StreamReader reader = new StreamReader(response.GetResponseStream());
string files = reader.ReadToEnd();
return files.Length > 0
? files.Split("\r\n")
: Array.Empty<string>();
}
public string Download(string filename, string destination = null)
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.DownloadFile;
if (string.IsNullOrEmpty(destination))
{
destination = _ftpConfiguration.Destination;
}
if (Path.DirectorySeparatorChar == '\\')
{
filename = filename.Replace("/", "\\");
}
destination = Path.Combine(destination, filename);
string destinationDirectory = Path.GetDirectoryName(destination);
if (!Directory.Exists(destinationDirectory))
{
Directory.CreateDirectory(destinationDirectory);
}
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
using Stream responseStream = response.GetResponseStream();
using FileStream fileStream = new FileStream(destination, FileMode.Create);
responseStream.CopyTo(fileStream);
return destination;
}
public void Delete(string filename)
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(new Uri($"{_ftpConfiguration.Url}/{filename}"));
request.Credentials = new NetworkCredential(_ftpConfiguration.Username, _ftpConfiguration.Password);
request.Method = WebRequestMethods.Ftp.DeleteFile;
using FtpWebResponse response = (FtpWebResponse)request.GetResponse();
}
}
}

View File

@@ -28,7 +28,7 @@ namespace PartSource.Automation.Services
{
StartInfo = new ProcessStartInfo
{
FileName = "C:\\Program Files (x86)\\Microsoft SQL Server\\130\\DTS\\Binn\\dtexec",
FileName = "C:\\Program Files\\Microsoft SQL Server\\150\\DTS\\Binn\\dtexec.exe",
Arguments = $"/file \"{_ssisConfiguration.Directory}\\{packageName}\"",
UseShellExecute = false,
CreateNoWindow = false,

View File

@@ -4,6 +4,7 @@ using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using PartSource.Data.Contexts;
using PartSource.Data.Dtos;
using PartSource.Data.Models;
@@ -95,7 +96,7 @@ namespace PartSource.Automation.Services
: new List<int>();
}
public IList<Vehicle> GetVehiclesForPart(string partNumber, string lineCode, int maxVehicles = 0)
public async Task<IList<Vehicle>> GetVehiclesForPart(string partNumber, string lineCode, int maxVehicles = 0)
{
if (string.IsNullOrEmpty(partNumber) || string.IsNullOrEmpty(lineCode))
{
@@ -104,9 +105,10 @@ namespace PartSource.Automation.Services
partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
IQueryable<string> whiCodes = _fitmentContext.DcfMappings
.Where(d => d.LineCode == lineCode)
.Select(d => d.WhiCode);
IList<string> whiCodes = await _fitmentContext.DcfMappings
.Where(dcf => dcf.LineCode == lineCode)
.Select(dcf => dcf.WhiCode)
.ToListAsync();
IQueryable<Vehicle> vehicles = _fitmentContext.Fitments
.Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode))

View File

@@ -1,7 +1,7 @@
{
"ConnectionStrings": {
//"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true;TrustServerCertificate=true",
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300",
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true;TrustServerCertificate=true",
//"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300",
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
},
"emailConfiguration": {
@@ -11,19 +11,18 @@
"SmtpHost": "localhost"
},
"FtpServers": {
"AzureConfiguration": {
"Username": "ps-ftp\\$ps-ftp",
"Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579",
"Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot",
"Destination": "C:\\Partsource.Automation\\Downloads",
"Port": 21
},
"WhiConfiguration": {
"Username": "ctc_seo",
"Password": "YD3gtaQ5kPdtNKs",
"Url": "ftp://ftp.whisolutions.com",
"Destination": "C:\\Partsource.Automation\\Downloads\\WHI",
"Port": 3001
"Port": 3001
},
"AutomationConfiguration": {
"Username": "stageuser",
"Password": "FXepK^cFYS|[H<",
"Url": "ftp://localhost",
"Destination": "C:\\Partsource.Automation\\Downloads"
}
},
"ssisConfiguration": {

View File

@@ -32,7 +32,7 @@ namespace PartSource.Data.Contexts
public DbSet<Vehicle> Vehicles { get; set; }
public DbSet<PartsAvailability> PartAvailabilities { get; set; }
public DbSet<PartAvailability> PartAvailabilities { get; set; }
public DbSet<BaseVehicle> BaseVehicles { get; set; }
@@ -48,7 +48,7 @@ namespace PartSource.Data.Contexts
{
base.OnModelCreating(modelBuilder);
modelBuilder.Entity<PartsAvailability>().HasKey(p => new { p.Store, p.SKU });
modelBuilder.Entity<PartAvailability>().HasKey(p => new { p.Store, p.SKU });
modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
modelBuilder.Entity<ShopifyChangelog>()

View File

@@ -1,9 +1,10 @@
using System.ComponentModel.DataAnnotations;
using System;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace PartSource.Data.Models
{
public class PartsAvailability
public class PartAvailability
{
[Column(Order = 0)]
[DatabaseGenerated(DatabaseGeneratedOption.None)]
@@ -13,14 +14,8 @@ namespace PartSource.Data.Models
[DatabaseGenerated(DatabaseGeneratedOption.None)]
public int SKU { get; set; }
[Column("Line Code")]
[StringLength(50)]
public string Line_Code { get; set; }
[Column("Part Number")]
[StringLength(50)]
public string Part_Number { get; set; }
public int? QTY { get; set; }
public DateTime? Updated { get; set; }
}
}

View File

@@ -21,7 +21,7 @@ namespace PartSource.Services
_fitmentContext = fitmentContext;
}
public async Task<PartsAvailability> GetInventory(int sku, int storeNumber)
public async Task<PartAvailability> GetInventory(int sku, int storeNumber)
{
return await _partSourceContext.PartAvailabilities.FirstOrDefaultAsync(s => s.Store == storeNumber && s.SKU == sku);
}