From 98092543abb2a90b10ad91c641c39d6c5d401a87 Mon Sep 17 00:00:00 2001 From: Tom Raterman Date: Fri, 18 Mar 2022 16:30:57 -0400 Subject: [PATCH] Threaded ProcessWhiFitment --- .../Jobs/ProcessWhiFitment.cs | 57 ++++++++++--------- PartSource.Automation/Program.cs | 18 +++--- .../Services/WhiSeoService.cs | 4 +- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/PartSource.Automation/Jobs/ProcessWhiFitment.cs b/PartSource.Automation/Jobs/ProcessWhiFitment.cs index 43bdae6..c64b226 100644 --- a/PartSource.Automation/Jobs/ProcessWhiFitment.cs +++ b/PartSource.Automation/Jobs/ProcessWhiFitment.cs @@ -39,6 +39,7 @@ namespace PartSource.Automation.Jobs _noteDictionary = new ConcurrentDictionary(); } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2008:Do not create tasks without passing a TaskScheduler", Justification = "")] public async Task Run() { _whiSeoService.TruncateFitmentTables(); @@ -54,46 +55,46 @@ namespace PartSource.Automation.Jobs fileGroups.Enqueue(fileGroup); } - while (fileGroups.Count > 0) + Task[] taskArray = new Task[8]; + + for (int i = 0; i < taskArray.Length; i++) { - Parallel.For(0, 2, i => + taskArray[i] = Task.Factory.StartNew(() => { - bool result = fileGroups.TryDequeue(out IGrouping fileGroup); - - if (!result) + while (fileGroups.TryDequeue(out IGrouping fileGroup)) { - return; - - } - - foreach (FileInfo fileInfo in fileGroup) - { - try + foreach (FileInfo fileInfo in fileGroup) { - string filename = Decompress(fileInfo); - string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); + try + { + string filename = Decompress(fileInfo); + string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.')); - DataTable dataTable = GetDataTable(filename); + DataTable dataTable = GetDataTable(filename); - _whiSeoService.BulkCopyFitment(dataTable, tableName); - _logger.LogInformation($"Copied {fileInfo.Name} to the database."); + _whiSeoService.BulkCopyFitment(dataTable, tableName); + _logger.LogInformation($"Copied {fileInfo.Name} to the database."); - File.Delete(filename); + File.Delete(filename); + } + + catch (Exception ex) + { + _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); + } } - catch (Exception ex) - { - _logger.LogError($"Failed to write {fileInfo.Name} to the database - {ex.Message}", ex); - } + string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.')); + _whiSeoService.CreateFitmentTable(fitmentTable); + + _logger.LogInformation($"Created fitment table for part group {fitmentTable}."); + } - - string fitmentTable = fileGroup.Key.Substring(0, fileGroup.Key.IndexOf('.')); - _whiSeoService.CreateFitmentTable(fitmentTable); - - _logger.LogInformation($"Created fitment table for part group {fitmentTable}."); }); } + Task.WaitAll(taskArray); + _whiSeoService.CreateFitmentView(); _whiSeoService.SaveNotes(_noteDictionary); @@ -135,7 +136,7 @@ namespace PartSource.Automation.Jobs } string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim(); - string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9\\-]", string.Empty).Trim(); + string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim(); string position = columns[7].Trim(); string noteText = columns[4].Trim(); diff --git a/PartSource.Automation/Program.cs b/PartSource.Automation/Program.cs index c54bec5..3a9e150 100644 --- a/PartSource.Automation/Program.cs +++ b/PartSource.Automation/Program.cs @@ -79,13 +79,13 @@ namespace PartSource.Automation { options.HasBaseInterval(new TimeSpan(0, 15, 0)) .HasMaxFailures(1) - //.HasJob(options => options.HasInterval(new TimeSpan(7, 0, 0, 0))); - // - //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))) - // .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); - //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) - //.HasDependency() - .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); + //.HasJob(options => options.HasInterval(new TimeSpan(7, 0, 0, 0))); + // + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))) + .HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) + //.HasDependency() + //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0))); //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) // .HasDependency() // .HasDependency() @@ -96,9 +96,9 @@ namespace PartSource.Automation // .StartsAt(DateTime.Parse("2021-04-01 08:00:00")) //) //.HasJob(options => - // options.HasInterval(new TimeSpan(24, 0, 0)) + //options.HasInterval(new TimeSpan(24, 0, 0)) // .StartsAt(DateTime.Today.AddHours(5)) - // ); + // ); //.StartsAt(DateTime.Today.AddHours(26)) //) //.HasJob(options => options.HasInterval(new TimeSpan(24, 0, 0)) diff --git a/PartSource.Automation/Services/WhiSeoService.cs b/PartSource.Automation/Services/WhiSeoService.cs index d22dc3e..e49152b 100644 --- a/PartSource.Automation/Services/WhiSeoService.cs +++ b/PartSource.Automation/Services/WhiSeoService.cs @@ -108,7 +108,7 @@ namespace PartSource.Automation.Services using SqlBulkCopy bulk = new SqlBulkCopy(connection) { DestinationTableName = $"FitmentTemp.{tableName}", - BulkCopyTimeout = 1 + BulkCopyTimeout = 14400 }; bulk.WriteToServer(dataTable); @@ -149,9 +149,11 @@ namespace PartSource.Automation.Services connection.Open(); using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection); + command.CommandTimeout = 1800; command.ExecuteNonQuery(); using SqlCommand command2 = new SqlCommand($"exec CreateFitmentIndexes", connection); + command.CommandTimeout = 1800; command2.ExecuteNonQuery(); }