soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Threaded ProcessWhiFitment

Browse Source
This commit is contained in:
Tom Raterman
2022-03-18 16:30:57 -04:00
parent 60edbee0b8
commit 98092543ab
3 changed files with 41 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -39,6 +39,7 @@ namespace PartSource.Automation.Jobs
_noteDictionary = new ConcurrentDictionary<string, string>(); _noteDictionary = new ConcurrentDictionary<string, string>();
} }
[System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "<Pending>")]
public async Task Run() public async Task Run()
{ {
_whiSeoService.TruncateFitmentTables(); _whiSeoService.TruncateFitmentTables();
@@ -54,46 +55,46 @@ namespace PartSource.Automation.Jobs
fileGroups.Enqueue(fileGroup); fileGroups.Enqueue(fileGroup);
} }
while (fileGroups.Count > 0) Task[] taskArray = new Task[8];
for (int i = 0; i < taskArray.Length; i++)
{ {
Parallel.For(0, 2, i => taskArray[i] = Task.Factory.StartNew(() =>
{ {
bool result = fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup); while (fileGroups.TryDequeue(out IGrouping<string, FileInfo> fileGroup))
if (!result)
{ {
return; foreach (FileInfo fileInfo in fileGroup)
}
foreach (FileInfo fileInfo in fileGroup)
{
try
{ {
string filename = Decompress(fileInfo); try
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); {
string filename = Decompress(fileInfo);
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
DataTable dataTable = GetDataTable(filename); DataTable dataTable = GetDataTable(filename);
_whiSeoService.BulkCopyFitment(dataTable, tableName); _whiSeoService.BulkCopyFitment(dataTable, tableName);
_logger.LogInformation($"Copied {fileInfo.Name} to the database."); _logger.LogInformation($"Copied {fileInfo.Name} to the database.");
File.Delete(filename); File.Delete(filename);
}
catch (Exception ex)
{
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
}
} }
catch (Exception ex) string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
{ _whiSeoService.CreateFitmentTable(fitmentTable);
_logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex);
} _logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
} }
string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.'));
_whiSeoService.CreateFitmentTable(fitmentTable);
_logger.LogInformation($"Created fitment table for part group {fitmentTable}.");
}); });
} }
Task.WaitAll(taskArray);
_whiSeoService.CreateFitmentView(); _whiSeoService.CreateFitmentView();
_whiSeoService.SaveNotes(_noteDictionary); _whiSeoService.SaveNotes(_noteDictionary);
@@ -135,7 +136,7 @@ namespace PartSource.Automation.Jobs
} }
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim(); string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9\\-]", string.Empty).Trim(); string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim(); string position = columns[7].Trim();
string noteText = columns[4].Trim(); string noteText = columns[4].Trim();

18
PartSource.Automation/Program.cs
View File

@@ -79,13 +79,13 @@ namespace PartSource.Automation
{ {
options.HasBaseInterval(new TimeSpan(0, 15, 0)) options.HasBaseInterval(new TimeSpan(0, 15, 0))
.HasMaxFailures(1) .HasMaxFailures(1)
//.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0))); //.HasJob<TestJob>(options => options.HasInterval(new TimeSpan(7, 0, 0, 0)));
// //
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0))) //.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0))); .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0)) //.HasJob<ProcessWhiVehicles>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasDependency<SyncronizeProducts>() //.HasDependency<SyncronizeProducts>()
.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0))); //.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)));
//.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0)) //.HasJob<UpdatePositioning>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<UpdateFitment>() // .HasDependency<UpdateFitment>()
// .HasDependency<ProcessWhiFitment>() // .HasDependency<ProcessWhiFitment>()
@@ -96,9 +96,9 @@ namespace PartSource.Automation
// .StartsAt(DateTime.Parse("2021-04-01 08:00:00")) // .StartsAt(DateTime.Parse("2021-04-01 08:00:00"))
//) //)
//.HasJob<ExecuteSsisPackages>(options => //.HasJob<ExecuteSsisPackages>(options =>
// options.HasInterval(new TimeSpan(24, 0, 0)) //options.HasInterval(new TimeSpan(24, 0, 0))
// .StartsAt(DateTime.Today.AddHours(5)) // .StartsAt(DateTime.Today.AddHours(5))
// ); // );
//.StartsAt(DateTime.Today.AddHours(26)) //.StartsAt(DateTime.Today.AddHours(26))
//) //)
//.HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0)) //.HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0))

4
PartSource.Automation/Services/WhiSeoService.cs
View File

@@ -108,7 +108,7 @@ namespace PartSource.Automation.Services
using SqlBulkCopy bulk = new SqlBulkCopy(connection) using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{ {
DestinationTableName = $"FitmentTemp.{tableName}", DestinationTableName = $"FitmentTemp.{tableName}",
BulkCopyTimeout = 1 BulkCopyTimeout = 14400
}; };
bulk.WriteToServer(dataTable); bulk.WriteToServer(dataTable);
@@ -149,9 +149,11 @@ namespace PartSource.Automation.Services
connection.Open(); connection.Open();
using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection); using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection);
command.CommandTimeout = 1800;
command.ExecuteNonQuery(); command.ExecuteNonQuery();
using SqlCommand command2 = new SqlCommand($"exec CreateFitmentIndexes", connection); using SqlCommand command2 = new SqlCommand($"exec CreateFitmentIndexes", connection);
command.CommandTimeout = 1800;
command2.ExecuteNonQuery(); command2.ExecuteNonQuery();
} }