Automation and Shopify library updates

This commit is contained in:
2022-10-30 22:12:25 -04:00
parent 48844127d7
commit b259b77967
21 changed files with 172 additions and 200 deletions

View File

@@ -35,7 +35,7 @@
<PackageReference Include="Microsoft.AspNetCore.Razor.Design" Version="2.2.0" PrivateAssets="All" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="6.0.5" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="Ratermania.Shopify" Version="1.3.1" />
<PackageReference Include="Ratermania.Shopify" Version="6.16.8" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.3.1" />
<PackageReference Include="Swashbuckle.AspNetCore.ReDoc" Version="6.3.1" />
</ItemGroup>

View File

@@ -4,6 +4,7 @@ using PartSource.Automation.Models.Configuration;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -18,15 +19,17 @@ namespace PartSource.Automation.Jobs
// TODO: set from config
private readonly string[] _ssisPackages = { "Parts Price", "Parts Availability" };
public ExecuteSsisPackages(EmailService emailService, FtpService ftpService, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
public ExecuteSsisPackages(EmailService emailService, IConfiguration configuration, SsisService ssisService, ILogger<ExecuteSsisPackages> logger)
{
_ftpService = ftpService;
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>();
_emailService = emailService;
_ftpService = new FtpService(ftpConfiguration);
_ssisService = ssisService;
_logger = logger;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
await Task.Run(() =>
{
@@ -37,12 +40,12 @@ namespace PartSource.Automation.Jobs
_ftpService.Download($"{package}.txt");
_ssisService.Execute($"{package}.dtsx");
_logger.LogInformation("Execution of SSIS package {package} completed successfully.", package);
_logger.LogInformation($"Execution of SSIS package {package} completed successfully.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Execution of SSIS package {package} failed", package);
_logger.LogError($"Execution of SSIS package {package} failed.", ex);
throw;
}

View File

@@ -5,6 +5,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -18,7 +19,7 @@ namespace PartSource.Automation.Jobs
_nexpartService = nexpartService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> rows = new List<string>
{

View File

@@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs.POC
@@ -19,7 +20,7 @@ namespace PartSource.Automation.Jobs.POC
_shopifyClient = shopifyClient;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 }, { "product_type", "CA112-SC137-FL13750_Intake Manifolds" } });

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@@ -29,7 +30,7 @@ namespace PartSource.Automation.Jobs.POC
_shopifyClient = shopifyClient;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
await BuildDatabase();
await UpdateShopify();
@@ -177,7 +178,7 @@ namespace PartSource.Automation.Jobs.POC
Namespace = "position",
Key = key,
Value = json,
ValueType = "json_string",
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};

View File

@@ -16,6 +16,7 @@ using System.Data;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -37,7 +38,7 @@ namespace PartSource.Automation.Jobs
_vehicleService = vehicleService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = null;

View File

@@ -18,6 +18,7 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs.POC
@@ -39,7 +40,7 @@ namespace PartSource.Automation.Jobs.POC
_vehicleService = vehicleService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IList<string> productTypes = new List<string>
{

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@@ -29,9 +30,9 @@ namespace PartSource.Automation.Jobs.POC
_shopifyClient = shopifyClient;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
// await BuildDatabase();
await BuildDatabase();
await UpdateShopify();
}
@@ -143,7 +144,6 @@ namespace PartSource.Automation.Jobs.POC
}
}
//[SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "It's a Shopify metafield key")]
private async Task SavePositionMetafield(Product product, IList<int> vehicleIds, string position)
{
if (vehicleIds.Count == 0)
@@ -169,7 +169,7 @@ namespace PartSource.Automation.Jobs.POC
Namespace = "position",
Key = key,
Value = json,
ValueType = "json_string",
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};

View File

@@ -14,6 +14,7 @@ using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -39,11 +40,11 @@ namespace PartSource.Automation.Jobs
_noteDictionary = new ConcurrentDictionary<string, string>();
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "<Pending>")]
public async Task Run()
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "Not library code")]
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateFitmentTables();
// _whiSeoService.GetFiles(_seoDataType);
_whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
@@ -55,7 +56,7 @@ namespace PartSource.Automation.Jobs
fileGroups.Enqueue(fileGroup);
}
Task[] taskArray = new Task[8];
Task[] taskArray = new Task[Environment.ProcessorCount / 2];
for (int i = 0; i < taskArray.Length; i++)
{
@@ -90,7 +91,7 @@ namespace PartSource.Automation.Jobs
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
}
});
}, token);
}
Task.WaitAll(taskArray);

View File

@@ -14,6 +14,7 @@ using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -36,7 +37,7 @@ namespace PartSource.Automation.Jobs
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
_whiSeoService.TruncateVehicleTables();
_whiSeoService.GetFiles(_seoDataType);

View File

@@ -1,6 +1,7 @@
using PartSource.Automation.Models;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -15,7 +16,7 @@ namespace PartSource.Automation.Jobs
_emailService = emailService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
foreach (string phoneNumber in phoneNumbers)
{

View File

@@ -6,6 +6,7 @@ using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using PartSource.Automation.Services;
using System.Threading;
namespace PartSource.Automation.Jobs
{
@@ -21,7 +22,7 @@ namespace PartSource.Automation.Jobs
}
#pragma warning disable CS1998, CA1303
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
// _emailService.Send("Automation Test Message", "This is a test email from the automation server. If this message was in your spam folder, whitelist the address that sent this email.");

View File

@@ -16,6 +16,7 @@ using System.Data;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -37,7 +38,7 @@ namespace PartSource.Automation.Jobs
_vehicleService = vehicleService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IEnumerable<Product> products = null;
@@ -104,7 +105,7 @@ namespace PartSource.Automation.Jobs
Namespace = "fitment",
Key = "ids",
Value = json,
ValueType = "json_string",
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};
@@ -145,7 +146,7 @@ namespace PartSource.Automation.Jobs
Namespace = "fitment",
Key = "seo",
Value = json,
ValueType = "json_string",
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
@@ -158,7 +159,7 @@ namespace PartSource.Automation.Jobs
Namespace = "Flags",
Key = "IsFitment",
Value = isFitment.ToString(),
ValueType = "string",
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
@@ -170,7 +171,7 @@ namespace PartSource.Automation.Jobs
Namespace = "google",
Key = "custom_label_0",
Value = importData.LineCode,
ValueType = "string",
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};
@@ -182,7 +183,7 @@ namespace PartSource.Automation.Jobs
Namespace = "google",
Key = "custom_label_1",
Value = importData.PartNumber,
ValueType = "string",
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
};

View File

@@ -13,6 +13,7 @@ using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -32,7 +33,7 @@ namespace PartSource.Automation.Jobs
_vehicleService = vehicleService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
IDictionary<string, object> parameters = new Dictionary<string, object>
{
@@ -131,7 +132,7 @@ namespace PartSource.Automation.Jobs
Namespace = "fitment",
Key = "note_text",
Value = json,
ValueType = "json_string",
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};
@@ -204,7 +205,7 @@ namespace PartSource.Automation.Jobs
Namespace = "position",
Key = key,
Value = json,
ValueType = "json_string",
Type = "json",
OwnerResource = "product",
OwnerId = product.Id
};

View File

@@ -13,6 +13,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Mail;
using System.Threading;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
@@ -32,7 +33,7 @@ namespace PartSource.Automation.Jobs
_emailService = emailService;
}
public async Task Run()
public async Task Run(CancellationToken token, params string[] arguments)
{
List<UpdatePricingResult> pricingReport = new List<UpdatePricingResult>();
IEnumerable<Product> products = null;
@@ -41,7 +42,7 @@ namespace PartSource.Automation.Jobs
try
{
products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
prices = await _partSourceContext.PartPrices.AsNoTracking().ToListAsync();
prices = await _partSourceContext.PartPrices.AsNoTracking().ToListAsync(token);
}
catch (Exception ex)
@@ -55,12 +56,8 @@ namespace PartSource.Automation.Jobs
{
foreach (Product product in products)
{
List<UpdatePricingResult> productPricingUpdate = new List<UpdatePricingResult>();
if (product.Variants.Length > 0)
{
bool hasUpdate = false;
for (int i = 0; i < product.Variants.Length; i++)
{
Variant variant = product.Variants[i];
@@ -71,45 +68,34 @@ namespace PartSource.Automation.Jobs
continue;
}
if (product.Variants[i].Price.ToString("G29") != partPrice.Your_Price.Value.ToString("G29") || product.Variants[i].CompareAtPrice.ToString("G29") != partPrice.Compare_Price.Value.ToString("G29"))
{
productPricingUpdate.Add(new UpdatePricingResult
{
Sku = variant.Sku,
OldPrice = product.Variants[i].Price,
NewPrice = partPrice.Your_Price.Value,
OldCompareAt = product.Variants[i].CompareAtPrice,
NewCompareAt = partPrice.Compare_Price.Value
});
product.Variants[i].Price = partPrice.Your_Price.Value;
product.Variants[i].CompareAtPrice = partPrice.Compare_Price.Value;
product.PublishedAt = partPrice.Active.Trim().ToUpperInvariant() == "Y" ? DateTime.Now : null;
product.PublishedAt = partPrice.Active.Trim().ToUpperInvariant() == "Y" ? (DateTime?)DateTime.Now : null;
product.PublishedScope = PublishedScope.Global;
//Metafield metafield = new Metafield
//{
// Namespace = "Pricing",
// Key = "CorePrice",
// Value = partPrice.Core_Price.HasValue ? partPrice.Core_Price.Value.ToString() : "0.00",
// ValueType = "string",
// OwnerResource = "product",
// OwnerId = product.Id
//};
hasUpdate = true;
}
}
if (hasUpdate)
{
try
{
//await _shopifyClient.Metafields.Add(metafield);
await _shopifyClient.Products.Update(product);
await _shopifyClient.Metafields.Add(new Metafield
{
Namespace = "Pricing",
Key = "CorePrice",
Value = partPrice.Core_Price.HasValue ? partPrice.Core_Price.Value.ToString() : "0.00",
Type = "single_line_text_field",
OwnerResource = "product",
OwnerId = product.Id
});
}
pricingReport.AddRange(productPricingUpdate);
catch (Exception ex)
{
_logger.LogWarning(ex, $"Failed to update core price metafield for product ID {product.Id}");
}
}
try
{
await _shopifyClient.Products.Update(product);
}
catch (Exception ex)
@@ -118,7 +104,7 @@ namespace PartSource.Automation.Jobs
}
}
}
}
try
{
@@ -134,40 +120,7 @@ namespace PartSource.Automation.Jobs
}
}
Attachment attachment = GetPricingReportAttachment(pricingReport);
_emailService.Send("Pricing Update Completed", $"The pricing update has completed. Total updated: {pricingReport.Count}", attachment);
}
private Attachment GetPricingReportAttachment(IList<UpdatePricingResult> pricingReport)
{
string directory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "Pricing Reports");
string filename = Path.Combine(directory, $"Pricing Update {DateTime.Now.ToString("yyyy-MM-dd")}.csv");
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
if (File.Exists(filename))
{
File.Delete(filename);
}
using FileStream fileStream = File.OpenWrite(filename);
using StreamWriter streamWriter = new StreamWriter(fileStream, System.Text.Encoding.UTF8);
streamWriter.WriteLine("SKU, Old Price, New Price, Old Compare At, New Compare At");
foreach (UpdatePricingResult pricingResult in pricingReport)
{
streamWriter.WriteLine($"{pricingResult.Sku},{pricingResult.OldPrice},{pricingResult.NewPrice},{pricingResult.OldCompareAt},{pricingResult.NewCompareAt}");
}
streamWriter.Close();
fileStream.Close();
return new Attachment(filename);
_emailService.Send("Pricing Update Completed", $"The pricing update has completed.");
}
}
}

View File

@@ -18,9 +18,10 @@
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0" />
<PackageReference Include="Ratermania.Automation" Version="1.0.0" />
<PackageReference Include="Ratermania.Automation.Common" Version="1.0.0" />
<PackageReference Include="Ratermania.Shopify" Version="1.3.1" />
<PackageReference Include="Ratermania.Automation" Version="6.16.9" />
<PackageReference Include="Ratermania.Automation.Common" Version="6.16.9" />
<PackageReference Include="Ratermania.JwtSpot" Version="6.16.9" />
<PackageReference Include="Ratermania.Shopify" Version="6.16.8" />
</ItemGroup>
<ItemGroup>

View File

@@ -15,6 +15,7 @@ using PartSource.Services;
using Ratermania.Automation.DependencyInjection;
using Ratermania.Automation.Logging;
using Ratermania.Shopify.DependencyInjection;
using Ratermania.JwtSpot.Configuration;
using System;
using System.IO;
using System.Threading.Tasks;
@@ -68,36 +69,33 @@ namespace PartSource.Automation
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
options.ApiSecret = builder.Configuration["Shopify:ApiSecret"];
options.ApiVersion = "2021-01";
options.ApiVersion = "2022-10";
options.ShopDomain = builder.Configuration["Shopify:ShopDomain"];
//options.ApiKey = "9a533dad460321c6ce8f30bf5b8691ed";
//options.ApiSecret = "dc9e28365d9858e544d57ac7af43fee7";
//options.ApiVersion = "2021-01";
//options.ApiVersion = "2022-10";
//options.ShopDomain = "dev-partsource.myshopify.com";
})
.AddAutomation(options =>
{
//options.HasBaseInterval(new TimeSpan(0, 15, 0))
// .HasMaxFailures(3)
// .HasJob<ExecuteSsisPackages>(options =>
// options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddHours(26))
// )
// .HasJob<UpdatePricing>(options =>
// options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddHours(27))
// .HasDependency<ExecuteSsisPackages>()
// );
options.HasBaseInterval(new TimeSpan(0, 15, 0))
.HasMaxFailures(3)
.HasJob<UpdateWiperFitment>(options =>
.HasJob<ExecuteSsisPackages>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
);
//.AddApiServer();
})
.StartsAt(DateTime.Today.AddHours(26)))
.HasJob<UpdatePricing>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(27))
.HasDependency<ExecuteSsisPackages>())
.UseApiServer(opts =>
opts.HasApiKey("PartsourceAPIKey")
.UseJwtSpot(jwt =>
jwt.HasAudience(builder.Configuration["JwtSpot:Audience"])
.HasIssuer(builder.Configuration["JwtSpot:Issuer"])
.UseX509Certificate(builder.Configuration["JwtSpot:CertPath"])
.UseJwksUrl(builder.Configuration["JwtSpot:JwksUrl"])))
.UseSqlServer(builder.Configuration.GetConnectionString("AutomationDatabase")))
.AddSingleton(builder.Configuration.GetSection("FtpServers:AzureConfiguration").Get<FtpConfiguration>())
.AddSingleton<FtpService>()
@@ -114,7 +112,7 @@ namespace PartSource.Automation
logging.AddEventLog();
logging.AddConsole();
// logging.AddProvider(new AutomationLoggerProvider());
//logging.AddProvider(new AutomationLoggerProvider());
});
}
}

View File

@@ -1,5 +1,6 @@
{
"ConnectionStrings": {
"AutomationDatabase": "Data Source=localhost;Initial Catalog=Automation;Integrated Security=true;Trust Server Certificate=true;",
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true",
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
},
@@ -31,11 +32,17 @@
"ApiSecret": "527a3b4213c2c7ecb214728a899052df",
"ShopDomain": "partsource.myshopify.com"
},
"JwtSpot": {
"Audience": "Ratermania.Automation",
"Issuer": "https://tomraterman.com",
"JwksUrl": "http://localhost:5103/jwks",
"CertPath": "C:\\Users\\tom\\Desktop\\PartsourceAutomation.pfx"
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information",
"Microsoft.Hosting.Lifetime": "Information"
// "Microsoft.EntityFrameworkCore.Database.Command": "Information"
},
"EventLog": {

View File

@@ -14,7 +14,7 @@
<ItemGroup>
<PackageReference Include="AutoMapper" Version="11.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="Ratermania.Shopify" Version="1.3.1" />
<PackageReference Include="Ratermania.Shopify" Version="6.16.8" />
</ItemGroup>
<ItemGroup>

View File

@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29613.14
# Visual Studio Version 17
VisualStudioVersion = 17.2.32519.379
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PartSource.Api", "PartSource.Api\PartSource.Api.csproj", "{126B8961-1D86-4F73-9BB9-79ECE78E9257}"
EndProject

View File

@@ -1,9 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
<add key="Ratermania" value="https://ratermanianuget.blob.core.windows.net/nuget/index.json" />
</packageSources>