soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
Tom Raterman
2023-08-23 15:04:54 -04:00
parent d95d947bc2
commit 68c9e01ef1
24 changed files with 655 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
PartSource.Automation/Jobs/ExecuteSsisPackages.cs
View File

@@ -17,7 +17,7 @@ namespace PartSource.Automation.Jobs
private readonly ILogger<ExecuteSsisPackages> _logger;
// TODO: set from config
private readonly string[] _ssisPackages = {"Parts Price", "Parts Availability" };
private readonly string[] _ssisPackages = { "Parts Availability" };
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{

85
PartSource.Automation/Jobs/POC/BulkUpdateInventory.cs Normal file
View File

@@ -0,0 +1,85 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Configuration;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
namespace PartSource.Automation.Jobs.POC
{
public class BulkUpdateInventory : IAutomationJob
{
private readonly FtpService _ftpService;
public BulkUpdateInventory(IConfiguration configuration)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
}
public async Task Run(CancellationToken token, params string[] arguments)
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf("Availability") > -1)
.OrderByDescending(f => f.Modified)
.First();
string file = _ftpService.Download(lastUploadedFile.Filename, Path.GetTempPath());
DataTable dataTable = GetDataTable(file);
using SqlConnection connection = new SqlConnection("Server=tcp:ps-automation-stage.eastus2.cloudapp.azure.com,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=stageuser;Password=]FXepK^cFYS|[H<;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=True;Connection Timeout=30;");
connection.Open();
using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{
DestinationTableName = $"PartAvailability",
BulkCopyTimeout = 14400
};
bulk.WriteToServer(dataTable);
return;
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("Store", typeof(int));
dataTable.Columns.Add("SKU", typeof(string));
dataTable.Columns.Add("QTY", typeof(int));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("|");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string sku = columns[1].Trim();
if (int.TryParse(columns[0], out int store)
&& !string.IsNullOrEmpty(sku)
&& int.TryParse(columns[2], out int quantity))
{
dataTable.Rows.Add(new object[] { store, sku, quantity });
}
}
return dataTable;
}
}
}

119
PartSource.Automation/Jobs/POC/GetImageUrls.cs Normal file
View File

@@ -0,0 +1,119 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly FitmentContext _fitmentContext;
private readonly PartService _partService;
private readonly ShopifyClient _shopifyClient;
public GetImageUrls(NexpartService nexpartService, PartService partService, FitmentContext fitmentContext, ShopifyClient shopifyClient)
{
_nexpartService = nexpartService;
_fitmentContext = fitmentContext;
_partService = partService;
_shopifyClient = shopifyClient;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> rows = new List<string> {
"\"Line Code\", \"Part Number\", \"Image URL(s)\""
};
using StreamReader reader = new StreamReader("C:\\Users\\Tom\\Desktop\\image parts.csv");
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split(",");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string partsourceCode = columns[0].Trim();
string partNumber = columns[1].Trim();
IList<DcfMapping> dcfMappings = await _partService.GetDcfMapping(partsourceCode);
if (dcfMappings.Count == 0)
{
Console.WriteLine($"No images for {partsourceCode} {partNumber}");
}
bool hasImage = false;
foreach (DcfMapping mapping in dcfMappings)
{
if (hasImage)
{
continue;
}
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = mapping.WhiCode,
PartNumber = partNumber
}
},
DataOption = new[] { "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
rows.Add($"\"{partsourceCode}\", \"{partNumber}\", \"{string.Join(";", urls)}\"");
hasImage = true;
}
}
}
if (!hasImage)
{
Console.WriteLine($"No images for {partsourceCode} {partNumber}");
}
}
await File.WriteAllLinesAsync($"C:\\users\\Tom\\desktop\\WHI Images {DateTime.Now:yyyyMMdd}.csv", rows);
}
}
}

106
PartSource.Automation/Jobs/POC/GetImageUrlsTemp.cs Normal file
View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrlsTemp : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly FitmentContext _fitmentContext;
private readonly PartService _partService;
private readonly ShopifyClient _shopifyClient;
private readonly string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
public GetImageUrlsTemp(NexpartService nexpartService, PartService partService, FitmentContext fitmentContext, ShopifyClient shopifyClient)
{
_nexpartService = nexpartService;
_fitmentContext = fitmentContext;
_partService = partService;
_shopifyClient = shopifyClient;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<KeyValuePair<string, string>> parts = new List<KeyValuePair<string, string>>();
parts.Add(new KeyValuePair<string, string>("DAY", "89310"));
parts.Add(new KeyValuePair<string, string>("CNI", "141.40113"));
parts.Add(new KeyValuePair<string, string>("PRF", "MU19631"));
parts.Add(new KeyValuePair<string, string>("TRK", "SB8100"));
parts.Add(new KeyValuePair<string, string>("MON", "906970"));
parts.Add(new KeyValuePair<string, string>("FEL", "70804"));
parts.Add(new KeyValuePair<string, string>("FEL", "SS71198"));
parts.Add(new KeyValuePair<string, string>("CFP", "STS314"));
parts.Add(new KeyValuePair<string, string>("NGK", "21517"));
parts.Add(new KeyValuePair<string, string>("NGK", "RC-XX89"));
parts.Add(new KeyValuePair<string, string>("FRA", "CA176"));
for (int i = 0; i < chars.Length; i++)
{
for (int j = 0; j < chars.Length; j++)
{
for (int k = 0; k < chars.Length; k++)
{
string actualLineCode = $"{chars[i]}{chars[j]}{chars[k]}";
System.Diagnostics.Debug.WriteLine(actualLineCode);
foreach (KeyValuePair<string, string> part in parts)
{
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = actualLineCode,
PartNumber = part.Value
}
},
DataOption = new[] { "DIST_LINE", "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
Console.WriteLine($"Image {urls[0]} found for {part.Value}. Expected: {part.Key}, Actual: {actualLineCode}");
}
}
}
}
}
}
}
}
}

84
PartSource.Automation/Jobs/POC/ImageList.cs
View File

@@ -1,84 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly PartSourceContext _partSourceContext;
public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext)
{
_nexpartService = nexpartService;
_partSourceContext = partSourceContext;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> rows = new List<string> {
"\"Line Code\", \"Part Number\", \"Image URL(s)\""
};
IList<ImportData> importData = await _partSourceContext.ImportData
//.Take(5000)
.ToListAsync();
foreach (ImportData item in importData)
{
SmartPageDataSearch dataSearch = new SmartPageDataSearch
{
Items = new Item[]
{
new Item
{
MfrCode = item.LineCode,
PartNumber = item.PartNumber
}
},
DataOption = new[] { "DIST_LINE", "ALL" }
};
SmartPageDataSearchResponse response = await _nexpartService.SendRequest<SmartPageDataSearch, SmartPageDataSearchResponse>(dataSearch);
if (response.ResponseBody.Item?.Length > 0)
{
List<string> urls = new List<string>();
if (!string.IsNullOrEmpty(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl))
{
urls.Add(response.ResponseBody.Item[0].PrimaryImg?.ImgUrl);
};
if (response.ResponseBody.Item[0].AddImgs?.AddImg?.Length > 0)
{
urls.AddRange(response.ResponseBody.Item[0].AddImgs.AddImg.Select(i => i.AddImgUrl));
}
if (urls.Count > 0)
{
rows.Add($"\"{item.LineCode}\", \"{item.PartNumber}\", \"{string.Join(";", urls)}\"");
}
}
}
await File.WriteAllLinesAsync("C:\\users\\Tommy\\desktop\\WHI Images.csv", rows);
}
}
}

86
PartSource.Automation/Jobs/POC/PartialInventoryUpdate.cs
View File

@@ -8,6 +8,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
@@ -18,67 +19,62 @@ namespace PartSource.Automation.Jobs.POC
public class PartialInventoryUpdate : IAutomationJob
{
private readonly FtpService _ftpService;
private readonly ILogger<PartialInventoryUpdate> _logger;
public PartialInventoryUpdate(IConfiguration configuration)
public PartialInventoryUpdate(IConfiguration configuration, ILogger<PartialInventoryUpdate> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_logger = logger;
}
public async Task Run(CancellationToken token, params string[] arguments)
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended("")
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf("Availability") > -1)
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Modified >= DateTime.Now.AddHours(-24) && f.Filename.IndexOf("Availability Partial") > -1)
.OrderByDescending(f => f.Modified)
.First();
.FirstOrDefault();
string file = _ftpService.Download(lastUploadedFile.Filename, Path.GetTempPath());
if (lastUploadedFile == null)
{
_logger.LogInformation($"No partial inventory file available for the time period {DateTime.Now.AddHours(-24)} - {DateTime.Now}");
return;
}
DataTable dataTable = GetDataTable(file);
string file = _ftpService.Download($"{lastUploadedFile.Filename}", "C:\\Users\\Tom\\Desktop");
using SqlConnection connection = new SqlConnection("Server=tcp:ps-automation-stage.eastus2.cloudapp.azure.com,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=stageuser;Password=]FXepK^cFYS|[H<;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=True;Connection Timeout=30;");
using SqlConnection connection = new SqlConnection("Server=tcp:ps-automation-stage.eastus2.cloudapp.azure.com,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=stageuser;Password=]FXepK^cFYS|[H<;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=True;Connection Timeout=30;");
connection.Open();
using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{
DestinationTableName = $"PartAvailability",
BulkCopyTimeout = 14400
};
using StreamReader reader = new StreamReader(file);
string line = reader.ReadLine(); // Burn the header row
bulk.WriteToServer(dataTable);
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split(",");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
if (int.TryParse(columns[0], out int store)
&& int.TryParse(columns[1], out int quantity)
&& int.TryParse(columns[2], out int sku))
{
using SqlCommand sqlCommand = new SqlCommand("UPDATE Inventory SET QTY = @qty WHERE SKU = @sku AND Store = @store", connection);
sqlCommand.Parameters.Add(new SqlParameter("qty", quantity));
sqlCommand.Parameters.Add(new SqlParameter("sku", sku));
sqlCommand.Parameters.Add(new SqlParameter("store", store));
await sqlCommand.ExecuteNonQueryAsync();
}
}
return;
}
private DataTable GetDataTable(string filename)
{
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("Store", typeof(int));
dataTable.Columns.Add("SKU", typeof(int));
dataTable.Columns.Add("QTY", typeof(int));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("|");
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
if (int.TryParse(columns[0], out int store)
&& int.TryParse(columns[1], out int sku)
&& int.TryParse(columns[2], out int quantity))
{
dataTable.Rows.Add(new object[] { store, sku, quantity });
}
}
return dataTable;
}
}
}

5
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -96,9 +96,8 @@ namespace PartSource.Automation.Jobs
Task.WaitAll(taskArray);
// _whiSeoService.CreateFitmentView();
//_whiSeoService.SaveNotes(_noteDictionary);
_whiSeoService.SaveNotes(_noteDictionary);
//_whiSeoService.CreateFitmentView();
}
public string Decompress(FileInfo fileInfo)

5
PartSource.Automation/Jobs/ProcessWhiVehicles.cs
View File

@@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Extensions;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Enums;
using PartSource.Automation.Services;
@@ -45,7 +46,9 @@ namespace PartSource.Automation.Jobs
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
IEnumerable<FileInfo> files = directoryInfo.GetFiles().Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"));
IEnumerable<FileInfo> files = directoryInfo.GetFiles()
.Where(f => f.Name.StartsWith("seo_aces_vehicle_feed"))
.OrderByDescending(f => f.GetWhiTimestamp());
foreach (FileInfo fileInfo in files)
{

63
PartSource.Automation/Jobs/UpdateFitment.cs
View File

@@ -40,11 +40,11 @@ namespace PartSource.Automation.Jobs
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
//products = new List<Product>
//{
// await _shopifyClient.Products.GetById(4388919574575)
//};
//products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
products = new List<Product>
{
await _shopifyClient.Products.GetById(7285013446703)
};
}
catch (Exception ex)
@@ -72,40 +72,43 @@ namespace PartSource.Automation.Jobs
};
bool isFitment = false;
string bodyHtml = product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
string bodyHtml = string.IsNullOrEmpty(product.BodyHtml)
? "<ul></ul>"
: product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
if (vehicleIdFitment.Count > 0)
if (vehicleIdFitment.Count == 0)
{
string vehicleIdString = string.Join(',', vehicleIdFitment.Select(j => $"v{j}"));
continue;
}
string vehicleIdString = string.Join(',', vehicleIdFitment.Select(j => $"v{j}"));
bodyHtml += $"<div id=\"vehicleIDs\" style=\"display:none;\">{vehicleIdString}</div>";
bodyHtml += $"<div id=\"vehicleIDs\" style=\"display:none;\">{vehicleIdString}</div>";
isFitment = true;
isFitment = true;
string json = JsonConvert.SerializeObject(vehicleIdFitment);
if (json.Length < 100000)
string json = JsonConvert.SerializeObject(vehicleIdFitment);
if (json.Length < 100000)
{
Metafield vehicleMetafield = new Metafield
{
Metafield vehicleMetafield = new Metafield
{
Namespace = "fitment",
Key = "ids",
Value = json,
Type = "json_string",
OwnerResource = "product",
OwnerId = product.Id
};
Namespace = "fitment",
Key = "ids",
Value = json,
Type = "json_string",
OwnerResource = "product",
OwnerId = product.Id
};
await _shopifyClient.Metafields.Add(vehicleMetafield);
}
await _shopifyClient.Metafields.Add(vehicleMetafield);
}
else
{
_logger.LogWarning($"Vehicle ID fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added.");
continue;
}
else
{
_logger.LogWarning($"Vehicle ID fitment data for SKU {importData.VariantSku} is too large for Shopify and cannot be added.");
continue;
}
IList<string> ymmFitment = _vehicleFitmentService.GetYmmFitment(vehicles);
@@ -135,7 +138,7 @@ namespace PartSource.Automation.Jobs
bodyHtml += $"<div id=\"seoData\">{stringBuilder.ToString()}</div>";
string json = JsonConvert.SerializeObject(ymmFitment);
json = JsonConvert.SerializeObject(ymmFitment);
if (json.Length < 100000)
{
Metafield ymmMetafield = new Metafield

2
PartSource.Automation/Jobs/UpdatePositioning.cs
View File

@@ -64,7 +64,7 @@ namespace PartSource.Automation.Jobs
}
IList<Fitment> fitments = GetPositionOrderedFitments(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
if (fitments.Count == 0 || vehicles.Count == 0)
{