11 Commits

14 changed files with 149 additions and 105 deletions

View File

@@ -1,8 +1,7 @@
using Microsoft.AspNetCore.Mvc;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using PartSource.Data.Models;
using PartSource.Services;
using System.Net;
using System.Threading.Tasks;
namespace PartSource.Api.Controllers
{
@@ -23,7 +22,7 @@ namespace PartSource.Api.Controllers
[Route("sku/{sku}/storeNumber/{storeNumber}")]
public async Task<ActionResult> GetInventory(int sku, int storeNumber)
{
PartsAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
PartAvailability inventory = await _inventoryService.GetInventory(sku, storeNumber);
if (inventory == null)
{
@@ -36,7 +35,8 @@ namespace PartSource.Api.Controllers
{
StoreNumber = inventory.Store,
Sku = sku,
Quantity = inventory.QTY
Quantity = inventory.QTY,
Updated = inventory.Updated
}
});
}

View File

@@ -1,7 +1,6 @@
{
"ConnectionStrings": {
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;",
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=automation;Password=)6L)XP%m(x-UU#M;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
//"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true"
"FitmentDatabase": "Server=tcp:ps-automation.eastus2.cloudapp.azure.com,1433;Initial Catalog=WhiFitment;User ID=sa;Password=GZ0`-ekd~[2u;Encrypt=True;TrustServerCertificate=True;Connection Timeout=300"
},

View File

@@ -14,19 +14,22 @@ using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
namespace PartSource.Automation.Jobs.POC
namespace PartSource.Automation.Jobs
{
public class BulkUpdateInventory : IAutomationJob
{
private readonly FtpService _ftpService;
private readonly ILogger<BulkUpdateInventory> _logger;
private readonly string _connectionString;
public BulkUpdateInventory(IConfiguration configuration, ILogger<BulkUpdateInventory> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_logger = logger;
_connectionString = configuration.GetConnectionString("PartSourceDatabase");
_logger = logger;
}
public async Task Run(CancellationToken token, params string[] arguments)
@@ -46,11 +49,11 @@ namespace PartSource.Automation.Jobs.POC
DataTable dataTable = GetDataTable(file);
using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using SqlCommand command = new SqlCommand("TRUNCATE TABLE PartAvailability", connection);
await command.ExecuteNonQueryAsync();
await command.ExecuteNonQueryAsync(token);
using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{
@@ -58,7 +61,7 @@ namespace PartSource.Automation.Jobs.POC
BulkCopyTimeout = 14400
};
bulk.WriteToServer(dataTable);
await bulk.WriteToServerAsync(dataTable, token);
_ftpService.Delete(lastUploadedFile.Filename);
@@ -71,6 +74,7 @@ namespace PartSource.Automation.Jobs.POC
dataTable.Columns.Add("Store", typeof(int));
dataTable.Columns.Add("SKU", typeof(string));
dataTable.Columns.Add("QTY", typeof(int));
dataTable.Columns.Add("Updated", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
@@ -86,11 +90,14 @@ namespace PartSource.Automation.Jobs.POC
}
string sku = columns[1].Trim();
string updated = columns[3].Trim();
if (int.TryParse(columns[0], out int store)
&& !string.IsNullOrEmpty(sku)
&& int.TryParse(columns[2], out int quantity))
&& int.TryParse(columns[2], out int quantity)
&& !string.IsNullOrEmpty(updated))
{
dataTable.Rows.Add(new object[] { store, sku, quantity });
dataTable.Rows.Add(new object[] { store, sku, quantity, updated });
}
}

View File

@@ -1,9 +1,11 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -17,11 +19,11 @@ namespace PartSource.Automation.Jobs
private readonly ILogger<ExecuteSsisPackages> _logger;
// TODO: set from config
private readonly string[] _ssisPackages = {"Parts Availability" };
private readonly string[] _ssisPackages = {"Parts Price" };
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_emailService = emailService;
_ftpService = new FtpService(ftpConfiguration);
@@ -36,7 +38,18 @@ namespace PartSource.Automation.Jobs
{
try
{
// _ftpService.Download($"{package}.txt");
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
.Where(f => f.FileType == FtpFileType.File && f.Filename.IndexOf(package) > -1)
.OrderByDescending(f => f.Modified)
.FirstOrDefault();
if (lastUploadedFile == null)
{
_logger.LogInformation($"No {package} file available.");
return;
}
_ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
@@ -45,7 +58,6 @@ namespace PartSource.Automation.Jobs
catch (Exception ex)
{
_logger.LogError($"Execution of SSIS package {package} failed.", ex);
throw;
}
}

View File

@@ -18,13 +18,13 @@ using Ratermania.Shopify.Resources;
namespace PartSource.Automation.Jobs.POC
{
public class GetImageUrls : IAutomationJob
public class ImageList : IAutomationJob
{
private readonly NexpartService _nexpartService;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
public GetImageUrls(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
public ImageList(NexpartService nexpartService, PartSourceContext partSourceContext, FitmentContext fitmentContext)
{
_nexpartService = nexpartService;
_partSourceContext = partSourceContext;

View File

@@ -14,21 +14,25 @@ using PartSource.Automation.Models.Ftp;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
namespace PartSource.Automation.Jobs.POC
namespace PartSource.Automation.Jobs
{
public class PartialInventoryUpdate : IAutomationJob
{
private readonly FtpService _ftpService;
private readonly ILogger<PartialInventoryUpdate> _logger;
private readonly string _connectionString;
public PartialInventoryUpdate(IConfiguration configuration, ILogger<PartialInventoryUpdate> logger)
{
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AutomationConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_connectionString = _connectionString = configuration.GetConnectionString("PartSourceDatabase");
_logger = logger;
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities")]
public async Task Run(CancellationToken token, params string[] arguments)
{
FtpFileInfo lastUploadedFile = _ftpService.ListFilesExtended()
@@ -46,38 +50,70 @@ namespace PartSource.Automation.Jobs.POC
string file = _ftpService.Download($"{lastUploadedFile.Filename}");
using SqlConnection connection = new SqlConnection("Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-test;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;");
connection.Open();
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using StreamReader reader = new StreamReader(file);
string line = reader.ReadLine(); // Burn the header row
IDictionary<string, object> parameters = new Dictionary<string, object>();
string command = string.Empty;
int i = 0;
while (reader.Peek() > 0)
{
line = reader.ReadLine();
string[] columns = line.Split("|");
for (int i = 0; i < columns.Length; i++)
for (int j = 0; j < columns.Length; j++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
columns[j] = columns[j].Replace("\"", string.Empty);
}
string sku = columns[1].Trim();
string updated = columns[3].Trim();
if (int.TryParse(columns[0], out int store)
&& int.TryParse(columns[1], out int sku)
&& int.TryParse(columns[2], out int quantity))
&& !string.IsNullOrEmpty(sku)
&& int.TryParse(columns[2], out int quantity)
&& !string.IsNullOrEmpty(updated))
{
using SqlCommand sqlCommand = new SqlCommand("UPDATE PartAvailability SET QTY = @qty WHERE SKU = @sku AND Store = @store", connection);
sqlCommand.Parameters.Add(new SqlParameter("qty", quantity));
sqlCommand.Parameters.Add(new SqlParameter("sku", sku));
sqlCommand.Parameters.Add(new SqlParameter("store", store));
command += $"UPDATE PartAvailability SET QTY = @qty_{i}, Updated = @updated_{i} WHERE SKU = @sku_{i} AND Store = @store_{i};";
await sqlCommand.ExecuteNonQueryAsync();
parameters.Add($"qty_{i}", quantity);
parameters.Add($"store_{i}", store);
parameters.Add($"sku_{i}", sku);
parameters.Add($"updated_{i}", updated);
i++;
}
if (i == 250)
{
using SqlCommand nested = new SqlCommand(command, connection);
foreach (KeyValuePair<string, object> parameter in parameters)
{
nested.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
}
await nested.ExecuteNonQueryAsync(token);
parameters.Clear();
command = string.Empty;
i = 0;
}
}
using SqlCommand sqlCommand = new SqlCommand(command, connection);
foreach (KeyValuePair<string, object> parameter in parameters)
{
sqlCommand.Parameters.Add(new SqlParameter(parameter.Key, parameter.Value));
}
await sqlCommand.ExecuteNonQueryAsync(token);
_ftpService.Delete(lastUploadedFile.Filename);
return;
}
}
}
}

View File

@@ -24,7 +24,7 @@ namespace PartSource.Automation.Jobs
#pragma warning disable CS1998, CA1303
public async Task Run(CancellationToken token, params string[] arguments)
{
// _emailService.Send("Automation Test Message", "This is a test email from the automation server. If this message was in your spam folder, whitelist the address that sent this email.");
_emailService.Send("Automation Test Message", "This is a test email from the automation server. If this message was in your spam folder, whitelist the address that sent this email.");
_logger.LogInformation("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc scelerisque congue euismod. Curabitur enim eros, sollicitudin ac purus eget, dignissim mattis augue. In quam sapien, tincidunt et elementum vitae, interdum vitae sem.");
_logger.LogWarning("Praesent feugiat sapien non suscipit faucibus. Mauris fermentum ut augue a feugiat. Integer felis sem, laoreet et augue at, finibus maximus ex. Fusce sit amet erat non tortor porta condimentum condimentum quis ipsum.");

View File

@@ -93,7 +93,7 @@ namespace PartSource.Automation.Jobs
? string.Empty
: product.BodyHtml.Substring(0, product.BodyHtml.IndexOf("</ul>") + "</ul>".Length);
IList<Vehicle> vehicles = _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<Vehicle> vehicles = await _vehicleFitmentService.GetVehiclesForPart(importData.PartNumber, importData.LineCode);
IList<int> vehicleIdFitment = _vehicleFitmentService.GetVehicleIdFitment(vehicles);
if (!vehicleIdFitment.Any())

View File

@@ -76,7 +76,8 @@ namespace PartSource.Automation.Jobs
try
{
_shopifyClient.BulkActions.Add(new Metafield
await _shopifyClient.Metafields.Add(new Metafield
{
Namespace = "Pricing",
Key = "CorePrice",
@@ -85,24 +86,16 @@ namespace PartSource.Automation.Jobs
OwnerResource = "product",
OwnerId = product.Id
});
await _shopifyClient.Products.Update(product);
_logger.LogInformation("Updated product id {productId}", product.Id);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Failed to update pricing for product ID {product.Id}");
_logger.LogWarning(ex, "Failed to update pricing for product ID {productId}", product.Id);
}
}
try
{
_shopifyClient.BulkActions.Update(product);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Failed to update pricing for product ID {product.Id}");
}
}
}
@@ -119,14 +112,9 @@ namespace PartSource.Automation.Jobs
_logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
products = await _shopifyClient.Products.GetPrevious();
}
}
while (_shopifyClient.BulkActions.PendingCount() > 0)
{
await Task.Delay(15 * 1000, token);
_logger.LogInformation(_shopifyClient.BulkActions.PendingCount().ToString());
_emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
}
// _emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
}
}
}

View File

@@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Jobs;
using PartSource.Automation.Jobs.POC;
using PartSource.Automation.Services;
using PartSource.Data.AutoMapper;
using PartSource.Data.Contexts;
@@ -64,7 +63,7 @@ namespace PartSource.Automation
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
options.ApiVersion = "2022-10";
options.ApiVersion = "2024-10";
options.ShopDomain = builder.Configuration["Shopify:ShopDomain"];
//options.ApiKey = "9a533dad460321c6ce8f30bf5b8691ed";
@@ -75,32 +74,41 @@ namespace PartSource.Automation
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 5, 0))
.HasMaxFailures(1)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>()
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>()
// .HasDependency<SyncronizeProducts>()
// .StartsAt(DateTime.Today.AddHours(8))
//) ;
//.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//)
.HasJob<BulkUpdateInventory>(options =>
options.HasInterval(new TimeSpan(1, 0, 0))
.StartsAt(DateTime.Today.AddHours(-27))
)
.HasJob<PartialInventoryUpdate>(options =>
options.HasInterval(new TimeSpan(1, 0, 0))
.StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
);
options.HasBaseInterval(new TimeSpan(0, 5, 0))
.HasMaxFailures(5)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
//
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>()
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>()
// .HasDependency<SyncronizeProducts>()
// .StartsAt(DateTime.Today.AddHours(8))
//) ;
//.HasJob<StatusCheck>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//)
//.HasJob<ExecuteSsisPackages>(options =>
// options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-24))
// )
.HasJob<UpdatePricing>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(-22))
//.HasDependency<ExecuteSsisPackages>()
);
//.HasJob<BulkUpdateInventory>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27))
// );
//.HasJob<TestJob>(options =>
// options.HasInterval(new TimeSpan(1, 0, 0))
// .StartsAt(DateTime.Today.AddHours(-27).AddMinutes(30))
//);
//.HasJob<PartialInventoryUpdate>(options => options.HasInterval(new TimeSpan(1, 0, 0))
//.HasDependency<ExecuteSsisPackages>()

View File

@@ -6,24 +6,23 @@
},
"emailConfiguration": {
"From": "alerts@ps-automation.eastus2.cloudapp.azure.com",
"To": "tom@soundpress.com,Anas.Bajwa@Partsource.ca,josh@soundpress.com,alex.au@partsource.ca,michael.massara@partsource.ca",
//"To": "tom@tomraterman.com",
"To": "tom@soundpress.com,josh@soundpress.com,alex.au@partsource.ca,michael.massara@partsource.ca",
//"To": "tom@tomraterman.com,josh@soundpress.com",
"SmtpHost": "localhost"
},
"FtpServers": {
"AzureConfiguration": {
"Username": "ps-ftp\\$ps-ftp",
"Password": "ycvXptffBxqkBXW4vuRYqn4Zi1soCvnvMMolTe5HNSeAlcl3bAyJYtNhG579",
"Url": "ftp://waws-prod-yq1-007.ftp.azurewebsites.windows.net/site/wwwroot",
"Destination": "C:\\Partsource.Automation\\Downloads",
"Port": 21
},
"WhiConfiguration": {
"Username": "ctc_seo",
"Password": "YD3gtaQ5kPdtNKs",
"Url": "ftp://ftp.whisolutions.com",
"Destination": "C:\\Partsource.Automation\\Downloads\\WHI",
"Port": 3001
"Port": 3001
},
"AutomationConfiguration": {
"Username": "stageuser",
"Password": "FXepK^cFYS|[H<",
"Url": "ftp://localhost",
"Destination": "C:\\Partsource.Automation\\Downloads"
}
},
"ssisConfiguration": {

View File

@@ -32,7 +32,7 @@ namespace PartSource.Data.Contexts
public DbSet<Vehicle> Vehicles { get; set; }
public DbSet<PartsAvailability> PartAvailabilities { get; set; }
public DbSet<PartAvailability> PartAvailabilities { get; set; }
public DbSet<BaseVehicle> BaseVehicles { get; set; }
@@ -48,7 +48,7 @@ namespace PartSource.Data.Contexts
{
base.OnModelCreating(modelBuilder);
modelBuilder.Entity<PartsAvailability>().HasKey(p => new { p.Store, p.SKU });
modelBuilder.Entity<PartAvailability>().HasKey(p => new { p.Store, p.SKU });
modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
modelBuilder.Entity<ShopifyChangelog>()

View File

@@ -1,9 +1,10 @@
using System.ComponentModel.DataAnnotations;
using System;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace PartSource.Data.Models
{
public class PartsAvailability
public class PartAvailability
{
[Column(Order = 0)]
[DatabaseGenerated(DatabaseGeneratedOption.None)]
@@ -13,14 +14,8 @@ namespace PartSource.Data.Models
[DatabaseGenerated(DatabaseGeneratedOption.None)]
public int SKU { get; set; }
[Column("Line Code")]
[StringLength(50)]
public string Line_Code { get; set; }
[Column("Part Number")]
[StringLength(50)]
public string Part_Number { get; set; }
public int? QTY { get; set; }
public string Updated { get; set; }
}
}

View File

@@ -21,7 +21,7 @@ namespace PartSource.Services
_fitmentContext = fitmentContext;
}
public async Task<PartsAvailability> GetInventory(int sku, int storeNumber)
public async Task<PartAvailability> GetInventory(int sku, int storeNumber)
{
return await _partSourceContext.PartAvailabilities.FirstOrDefaultAsync(s => s.Store == storeNumber && s.SKU == sku);
}