soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Tom Raterman
2026-02-17 09:47:41 -05:00
parent dcd1a9ccec
commit fe97a1e51a
18 changed files with 107 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
PartSource.Automation/Jobs/ExecuteSsisPackages.cs
View File

@@ -38,18 +38,18 @@ namespace PartSource.Automation.Jobs
{
try
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
.OrderByDescending(f => f.Modified)
.FirstOrDefault();
// FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
//.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
//.OrderByDescending(f => f.Modified)
//.FirstOrDefault();
if (lastUploadedFile == null)
{
_logger.LogInformation($"No {package} file available.");
return;
}
// if (lastUploadedFile == null)
// {
// _logger.LogInformation($"No {package} file available.");
// return;
// }
_ftpService.Download($"{package}.txt");
// _ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");

2
PartSource.Automation/Jobs/PartsSync.cs
View File

@@ -63,6 +63,8 @@ namespace PartSource.Automation.Jobs
Part? existing = await _fitmentContext.Parts.FirstOrDefaultAsync(p => p.Sku == part.Sku);
if (existing == null)
{
_logger.LogInformation("Adding SKU {Sku}: {LineCode} {PartNumber}", part.Sku, part.LineCode, part.PartNumber);
await _fitmentContext.Parts.AddAsync(part);
await _fitmentContext.SaveChangesAsync();
}

10
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -32,15 +32,11 @@ namespace PartSource.Automation.Jobs
{
_logger = logger;
_whiSeoService = whiSeoService;
_seoDataType = SeoDataType.Fitment;
_ftpConfiguration = configuration.GetSection("ftpServers:WhiConfiguration").Get<FtpConfiguration>();
_noteDictionary = new ConcurrentDictionary<string, string>();
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "<Pending>")]
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateFitmentTables();
@@ -56,7 +52,7 @@ namespace PartSource.Automation.Jobs
fileGroups.Enqueue(fileGroup);
}
Task[] taskArray = new Task[18];
Task[] taskArray = new Task[12];
for (int i = 0; i < taskArray.Length; i++)
{
taskArray[i] = Task.Factory.StartNew(() =>
@@ -91,12 +87,12 @@ namespace PartSource.Automation.Jobs
_logger.LogInformation($"Created fitment table for part group {tableName}.");
}
});
}, token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
Task.WaitAll(taskArray);
_whiSeoService.SaveNotes(_noteDictionary);
_whiSeoService.SaveNotes(_noteDictionary);
_whiSeoService.CreateFitmentView();
}

3
PartSource.Automation/Jobs/ProcessWhiVehicles.cs
View File

@@ -121,7 +121,8 @@ namespace PartSource.Automation.Jobs
&& int.TryParse(columns[35], out int engineConfigId)
&& int.TryParse(columns[36], out int vehicleToEngineConfigId))
{
if (regionId == 2 && new[] { 5, 6, 7 }.Contains(vehicleTypeId))
if (new[] { 5, 6, 7 }.Contains(vehicleTypeId))
//if (regionId == 2 && new[] { 5, 6, 7 }.Contains(vehicleTypeId))
{
dataTable.Rows.Add(new object[] { year, makeId, makeName, modelId, modelName, regionId, regionName, vehicleTypeId, engineConfigId, engineDescription, baseVehicleId, vehicleToEngineConfigId, submodelId, submodelName });
}

26
PartSource.Automation/Jobs/UpdateFitment.cs
View File

@@ -15,6 +15,7 @@ using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
using System.Web;
using System.IO;
namespace PartSource.Automation.Jobs
{
@@ -37,20 +38,24 @@ namespace PartSource.Automation.Jobs
public async Task Run(CancellationToken token, params string[] arguments)
{
string logfile = Path.Combine(Path.GetTempPath(), $"missing_fitment_{DateTime.Now:yyyy-MM-dd}.csv");
using StreamWriter noFitmentWriter = File.AppendText(logfile);
noFitmentWriter.WriteLine("SKU,LineCode,PartNumber");
IList<string> productTypes = await _fitmentContext.ProductTypes
_logger.LogInformation($"Logging products without fitment to {logfile}");
IList<ProductType> productTypes = await _fitmentContext.ProductTypes
.Where(p => p.Active)
.Select(p => HttpUtility.UrlEncode(p.Name))
.ToListAsync();
foreach (string productType in productTypes)
foreach (ProductType productType in productTypes)
{
_logger.LogInformation("Processing {productType}", HttpUtility.UrlDecode(productType));
_logger.LogInformation("Processing {productType}", productType.Name);
IEnumerable<Product> products = null;
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "product_type", productType } });
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "product_type", HttpUtility.UrlEncode(productType.Name) } });
//products = new List<Product>
//{
// await _shopifyClient.Products.GetById(7458071052335)
@@ -88,6 +93,12 @@ namespace PartSource.Automation.Jobs
continue;
}
// Temporarily skip castrol oils
if (importData.LineCode == "CST")
{
continue;
}
// Extract Partsource bullet points if present.
string bodyHtml = string.IsNullOrEmpty(product.BodyHtml)
? string.Empty
@@ -99,6 +110,8 @@ namespace PartSource.Automation.Jobs
if (!vehicleIdFitment.Any())
{
Console.WriteLine($"No fitment data for {importData.LineCode} {importData.PartNumber}");
noFitmentWriter.WriteLine($"\"{importData.VariantSku}\",\"{importData.LineCode}\",\"{importData.PartNumber}\"");
continue;
}
@@ -181,6 +194,9 @@ namespace PartSource.Automation.Jobs
products = await _shopifyClient.Products.GetPrevious();
}
}
productType.Active = false;
await _fitmentContext.SaveChangesAsync();
}
}
}

24
PartSource.Automation/Jobs/UpdatePricing.cs
View File

@@ -100,20 +100,20 @@ namespace PartSource.Automation.Jobs
}
}
try
{
count += products.Count();
products = await _shopifyClient.Products.GetNext();
_logger.LogInformation($"Total updated: {count}");
}
//try
//{
// count += products.Count();
// products = await _shopifyClient.Products.GetNext();
// _logger.LogInformation($"Total updated: {count}");
//}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
products = await _shopifyClient.Products.GetPrevious();
}
//catch (Exception ex)
//{
// _logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
// products = await _shopifyClient.Products.GetPrevious();
//}
_emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
// _emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
}
}
}

7
PartSource.Automation/PartSource.Automation.csproj
View File

@@ -4,11 +4,10 @@
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<Configurations>Debug;Release;Also Debug</Configurations>
<ServerGarbageCollection>true</ServerGarbageCollection>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AutoMapper" Version="11.0.1" />
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.3.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -21,11 +20,11 @@
<PackageReference Include="Ratermania.Automation" Version="6.16.9" />
<PackageReference Include="Ratermania.Automation.Common" Version="6.16.9" />
<PackageReference Include="Ratermania.JwtSpot" Version="6.16.9" />
<PackageReference Include="Ratermania.Shopify" Version="6.16.11" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\ratermania\Packages\Shopify\Shopify.csproj" />
<ProjectReference Include="..\PartSource.Data\PartSource.Data.csproj" />
<ProjectReference Include="..\PartSource.Services\PartSource.Services.csproj" />
</ItemGroup>

37
PartSource.Automation/Program.cs
View File

@@ -1,18 +1,17 @@
using Microsoft.EntityFrameworkCore;
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Jobs;
using PartSource.Automation.Services;
using PartSource.Data.AutoMapper;
using PartSource.Data.Contexts;
using PartSource.Services;
using Ratermania.Automation.DependencyInjection;
using Ratermania.Shopify.DependencyInjection;
using System;
using System.IO;
using System.Threading.Tasks;
namespace PartSource.Automation
{
@@ -63,7 +62,7 @@ namespace PartSource.Automation
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
options.ApiVersion = "2024-10";
options.ApiVersion = "2025-10";
options.ShopDomain = builder.Configuration["Shopify:ShopDomain"];
//options.ApiKey = "9a533dad460321c6ce8f30bf5b8691ed";
@@ -74,7 +73,7 @@ namespace PartSource.Automation
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 5, 0))
options.HasBaseInterval(new TimeSpan(24, 0, 0))
.HasMaxFailures(5)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
@@ -96,19 +95,19 @@ namespace PartSource.Automation
// options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-24))
// )
.HasJob<UpdatePricing>(options =>
.HasJob<PartsSync>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(-22))
//.HasDependency<ExecuteSsisPackages>()
//.HasDependency<ExecuteSsisPackages>()
);
//.HasJob<BulkUpdateInventory>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27))
// );
//.HasJob<TestJob>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
//);
//.HasJob<BulkUpdateInventory>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27))
// );
//.HasJob<TestJob>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
//);
//.HasJob<PartialInventoryUpdate>(options => options.HasInterval(new TimeSpan(1, 0, 0))
//.HasDependency<ExecuteSsisPackages>()
@@ -124,9 +123,7 @@ namespace PartSource.Automation
.AddSingleton<VehicleService>()
.AddSingleton<VehicleFitmentService>()
.AddSingleton<NexpartService>()
.AddSingleton<PartService>()
.AddAutoMapper(typeof(PartSourceProfile));
.AddSingleton<PartService>();
})
.ConfigureLogging((builder, logging) =>
{

4
PartSource.Automation/Services/WhiSeoService.cs
View File

@@ -87,9 +87,7 @@ namespace PartSource.Automation.Services
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
string sql = string.Empty;
using SqlCommand command = new SqlCommand($"EXEC CreateFitmentTempTable @tableName = '{tableName}'", connection);
using SqlCommand command = new SqlCommand($"CREATE TABLE [FitmentTemp].[{tableName}]([LineCode] [nvarchar](10) NOT NULL, [PartNumber] [nvarchar](25) NOT NULL, [BaseVehicleId] [int] NOT NULL, [EngineConfigId] [int] NOT NULL, [Position] [nvarchar](250) null, [FitmentNoteHash] [nvarchar](40) null, PartTerminologyId [int] not null)", connection);
command.ExecuteNonQuery();
using SqlBulkCopy bulk = new SqlBulkCopy(connection)

4
PartSource.Automation/appsettings.json
View File

@@ -1,7 +1,7 @@
{
"ConnectionStrings": {
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true;TrustServerCertificate=true",
//"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300",
//"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true;TrustServerCertificate=true",
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300",
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
},
"emailConfiguration": {