soundpress/Partsource
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migration to DevOps

Browse Source
This commit is contained in:
Tom Raterman
2021-03-30 10:09:57 -04:00
parent e1be26d798
commit 3754beb035
23 changed files with 342 additions and 1700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
.github/workflows/main.yml vendored
View File

@@ -1,32 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: CI
# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
# The type of runner that the job will run on
runs-on: [self-hosted]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- name: Build
run: |
cd
dotnet restore
dotnet build --configuration "Release"
- name: Deploy
run: xcopy PartSource.Automation\bin\release\netcoreapp3.1 c:\Partsource.Automation /Y

7
PartSource.Api/Controllers/PartsController.cs
View File

@@ -1,4 +1,5 @@
using Microsoft.AspNetCore.Mvc;

using Microsoft.AspNetCore.Mvc;
using PartSource.Data.Dtos;
using PartSource.Data.Models;
using PartSource.Data.Nexpart;
@@ -52,7 +53,9 @@ namespace PartSource.Api.Controllers
BaseVehicleId = baseVehicleId
}
};
PartTypeSearchResponse response = await partsController._nexpartService.SendRequest<PartTypeSearch, PartTypeSearchResponse>(requestContent);
PartTypeSearchResponse response = await _nexpartService.SendRequest<PartTypeSearch, PartTypeSearchResponse>(requestContent);
return partsController.NexpartResponse<PartTypeSearchResponse, PartTypes>(response);
}

324
PartSource.Automation/Jobs/Archive/AddAndUpdateProducts.cs
View File

@@ -1,324 +0,0 @@
//using PartSource.Automation.Models;
//using PartSource.Data;
//using PartSource.Data.Models;
//using Ratermania.Shopify;
//using Ratermania.Shopify.Resources;
//using System;
//using System.Collections.Generic;
//using System.Linq;
//using System.Threading.Tasks;
//namespace PartSource.Automation.Jobs
//{
// public class AddAndUpdateProducts// : IAutomationJob
// {
// private readonly PartSourceContext _partSourceContext;
// private readonly ShopifyClient _shopifyClient;
// public AddAndUpdateProducts(PartSourceContext partSourceContext, ShopifyClient shopifyClient)
// {
// _partSourceContext = partSourceContext;
// _shopifyClient = shopifyClient;
// }
// public async Task<AutomationJobResult> Run()
// {
// //throw new Exception("You need to add a ProductVariant resource to the Shopify client");
// await SyncronizeIdsAndSkus();
// //await AddSkus();
// // await AddVariants();
// return new AutomationJobResult
// {
// IsSuccess = true
// };
// }
// /// <summary>
// /// Ensures syncronization between Shopify IDs and Partsource SKUs
// /// </summary>
// private async Task SyncronizeIdsAndSkus()
// {
// IEnumerable<Product> products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
// //_partSourceContext.Database.ExecuteSqlCommand("UPDATE ImportData SET ShopifyId = NULL");
// while (products != null && products.Any())
// {
// foreach (Product product in products)
// {
// foreach (Variant variant in product.Variants)
// {
// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == variant.Sku);
// if (importData != null)
// {
// importData.ShopifyId = product.Id;
// }
// }
// }
// try
// {
// await _partSourceContext.SaveChangesAsync();
// }
// catch
// {
// Console.WriteLine("Failed to save a batch of products");
// }
// finally
// {
// products = await _shopifyClient.Products.GetNext();
// }
// }
// }
// public async Task AddSkus()
// {
// IList<ImportData> items = _partSourceContext.ImportData
// .Where(i => i.IsVariant == null && i.ShopifyId == null)
// // .OrderBy(i => i.Title)
// .ToList();
// // items = items.Where(i => i.Title == items.First().Title).ToList();
// //
// foreach (ImportData importData in items)
// {
// try
// {
// // Images
// IList<ProductImage> productImages = new List<ProductImage>();
// if (!string.IsNullOrEmpty(importData.ImageSrc))
// {
// foreach (string src in importData?.ImageSrc.Split(','))
// {
// productImages.Add(new ProductImage
// {
// Src = src,
// Alt = importData.ImageAltText
// });
// }
// if (productImages.Count > 0)
// {
// productImages.Add(new ProductImage
// {
// Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg",
// Alt = "No Image Found"
// });
// }
// }
// // Product Tags
// List<string> productTags = new List<string>
// {
// items[0].LineCode,
// items[0].PartNumber,
// };
// //List<ProductVariant> productVariants = new List<ProductVariant>();
// ////foreach (ImportData itemVariant in items)
// ////{
// //productVariants.Add(new ProductVariant
// //{
// // InventoryPolicy = "Deny",
// // CompareAtPrice = importData.CompareAt,
// // Price = importData.Price,
// // Sku = importData.VariantSku,
// // Title = importData.VariantTitle ?? importData.Title,
// // Option1 = importData.VariantTitle,
// // RequiresShipping = false,
// //});
// //}
// Product requestData = new Product
// {
// BodyHtml = importData.BodyHtml,
// Title = importData.Title,
// Vendor = importData.Vendor,
// Tags = string.Join(",", productTags),
// //ProductType = importData.FINELINE_NM,
// Images = productImages.ToArray(),
// //Variants = productVariants.ToArray(),
// CreatedAt = DateTime.Now,
// UpdatedAt = DateTime.Now,
// PublishedAt = DateTime.Now
// };
// requestData = await _shopifyClient.Products.Add(requestData);
// if (requestData.Id > 0)
// {
// //foreach (ImportData variant in items)
// //{
// // variant.ShopifyId = requestData.Id;
// //}
// importData.ShopifyId = requestData.Id;
// _partSourceContext.SaveChanges();
// Console.WriteLine($"{importData.VariantSku}");
// }
// else
// {
// Console.WriteLine($"SHOPIFY ID WAS 0 - {importData.VariantSku}");
// }
// }
// catch (Exception ex)
// {
// Console.WriteLine($"Failed to add SKU {importData.VariantSku}: {ex.StackTrace}");
// }
// //items = _partSourceContext.ImportData
// // .Where(i => i.Title == _partSourceContext.ImportData.First(d => d.ShopifyId == null).Title)
// // .ToList();
// }
// }
// public async Task AddVariants()
// {
// IList<ImportData> items = _partSourceContext.ImportData
// .Where(i => i.IsVariant.Value && i.ShopifyId == null)
// .OrderBy(i => i.Title)
// .ToList();
// items = items.Where(i => i.Title == items.First().Title).ToList();
// while (items != null && items.Count > 0)
// {
// // Images
// IList<ProductImage> productImages = new List<ProductImage>();
// if (!string.IsNullOrEmpty(items[0].ImageSrc))
// {
// productImages = items.SelectMany(v =>
// {
// IList<ProductImage> images = new List<ProductImage>();
// foreach (string src in v.ImageSrc?.Split(','))
// {
// images.Add(new ProductImage
// {
// Src = src,
// Alt = v.ImageAltText
// });
// }
// return images;
// }).ToList();
// if (productImages.Count > 0)
// {
// productImages.Add(new ProductImage
// {
// Src = "https://cdn.shopify.com/s/files/1/2239/4255/files/No_Image_Found.jpg",
// Alt = "No Image Found"
// });
// }
// }
// // Product Tags
// List<string> productTags = new List<string>
// {
// items[0].LineCode,
// items[0].PartNumber,
// };
// //List<ProductVariant> productVariants = new List<ProductVariant>();
// //foreach (ImportData itemVariant in items)
// //{
// // productVariants.Add(new ProductVariant
// // {
// // InventoryPolicy = "Deny",
// // CompareAtPrice = itemVariant.CompareAt,
// // Price = itemVariant.Price,
// // Sku = itemVariant.VariantSku,
// // Title = itemVariant.VariantTitle,
// // Option1 = itemVariant.VariantTitle,
// // RequiresShipping = false,
// // });
// //}
// Product requestData = new Product
// {
// BodyHtml = items[0].BodyHtml,
// Title = items[0].Title,
// Vendor = items[0].Vendor,
// Tags = string.Join(",", productTags),
// //ProductType = importData.FINELINE_NM,
// Images = productImages.ToArray(),
// //Variants = productVariants.ToArray(),
// CreatedAt = DateTime.Now,
// UpdatedAt = DateTime.Now,
// PublishedAt = DateTime.Now
// };
// requestData = await _shopifyClient.Products.Add(requestData);
// if (requestData.Id > 0)
// {
// foreach (ImportData variant in items)
// {
// variant.ShopifyId = requestData.Id;
// }
// _partSourceContext.SaveChanges();
// Console.WriteLine($"{items[0].VariantSku}");
// }
// else
// {
// Console.WriteLine($"SHOPIFY ID WAS 0 - {items[0].VariantSku}");
// }
// ImportData next = _partSourceContext.ImportData.FirstOrDefault(i => i.IsVariant != null && i.ShopifyId == null);
// if (next != null)
// {
// items = _partSourceContext.ImportData
// .Where(i => i.Title == next.Title)
// .ToList();
// }
// else
// {
// items = null;
// }
// }
// }
// }
// //private void Log(string message)
// //{
// // try
// // {
// // using (FileStream fileStream = File.OpenWrite(@"C:\users\tommy\desktop\log.txt"))
// // {
// // fileStream.Write(Encoding.UTF8.GetBytes(message + "\n"));
// // }
// // }
// // catch
// // {
// // // LOL Fix this
// // Log(message);
// // }
// //}
//}

320
PartSource.Automation/Jobs/Archive/BuildVehicleCache.cs
View File

@@ -1,320 +0,0 @@
//using Microsoft.EntityFrameworkCore;
//using PartSource.Automation.Jobs.Interfaces;
//using PartSource.Automation.Services;
//using PartSource.Data;
//using PartSource.Data.Models;
//using PartSource.Data.Nexpart;
//using PartSource.Services;
//using System;
//using System.Collections.Generic;
//using System.Linq;
//using System.Text;
//using BaseVehicle = PartSource.Data.Models.BaseVehicle;
//namespace PartSource.Automation.Jobs
//{
// public class BuildVehicleCache : IAutomationJob
// {
// private readonly NexpartService _nexpartService;
// private readonly PartSourceContext _partSourceContext;
// private readonly IServiceProvider _serviceProvider;
// public BuildVehicleCache(IServiceProvider serviceProvider, PartSourceContext partSourceContext, NexpartService nexpartService)
// {
// _nexpartService = nexpartService;
// _partSourceContext = partSourceContext;
// _serviceProvider = serviceProvider;
// }
// public void Run()
// {
// try
// {
// //AddMakes();
// //AddModels();
// // AddEngines();
// //AddYearMakeModels();
// AddSubmodels();
// }
// catch (Exception ex)
// {
// ;
// }
// }
// private void AddEngines()
// {
// IList<BaseVehicle> baseVehicles = _partSourceContext.BaseVehicles.ToList();
// foreach (BaseVehicle baseVehicle in baseVehicles)
// {
// try
// {
// WHIEngineSearch whiEngineSearch = new WHIEngineSearch
// {
// VehicleIdentifier = new VehicleIdentifier
// {
// BaseVehicleId = baseVehicle.Id
// }
// };
// WHIEngineSearchResponse response = _nexpartService.SendRequest<WHIEngineSearch, WHIEngineSearchResponse>(whiEngineSearch).Result;
// foreach (WHIEngine engine in response.ResponseBody.WHIEngine)
// {
// try
// {
// if (!_partSourceContext.Engines.Any(e => e.Id == engine.Id))
// {
// _partSourceContext.Engines.Add(new Data.Models.Engine
// {
// Id = engine.Id,
// Description = engine.Description
// });
// }
// _partSourceContext.SaveChanges();
// }
// catch (Exception ex)
// {
// Console.WriteLine($"Failed to add engine { engine.Id } - { ex.Message }");
// }
// }
// }
// catch (Exception ex)
// {
// Console.WriteLine($"Failed to add engines for base vehicle { baseVehicle.Id } - { ex.Message }");
// }
// }
// }
// private void AddSubmodels()
// {
// int maxBaseVehicleId = _partSourceContext.Submodels.Any()
// ? _partSourceContext.Submodels.Max(s => s.BaseVehicleId)
// : 0;
// IList<BaseVehicle> baseVehicles = _partSourceContext.BaseVehicles.AsNoTracking().Where(b => b.Id > maxBaseVehicleId).ToList();
// foreach (BaseVehicle baseVehicle in baseVehicles)
// {
// try
// {
// SubModelSearch smSearch = new SubModelSearch()
// {
// MakeId = baseVehicle.VehicleMakeId,
// ModelId = baseVehicle.VehicleModelId,
// Year = baseVehicle.Year,
// RegionId = 2
// };
// SubModelSearchResponse smResponse = _nexpartService.SendRequest<SubModelSearch, SubModelSearchResponse>(smSearch).Result;
// SubModel[] subModels = smResponse.ResponseBody?.SubModel;
// if (subModels == null)
// {
// continue;
// }
// foreach (SubModel submodel in subModels)
// {
// EngineSearch requestContent = new EngineSearch()
// {
// VehicleIdentifier = new VehicleIdentifier()
// {
// BaseVehicleId = baseVehicle.Id
// },
// SubModelId = int.Parse(submodel.Id)
// };
// EngineSearchResponse response = _nexpartService.SendRequest<EngineSearch, EngineSearchResponse>(requestContent).Result;
// if (response.ResponseBody == null)
// {
// continue;
// }
// foreach (Data.Nexpart.Engine engine in response.ResponseBody.Engine)
// {
// VehicleIdSearch vidSearch = new VehicleIdSearch
// {
// VehicleIdentifier = new VehicleIdentifier()
// {
// BaseVehicleId = baseVehicle.Id,
// EngineConfigId = engine.Id
// },
// Criterion = new Criterion[]
// {
// new Criterion
// {
// Attribute = "SUB_MODEL",
// Id = int.Parse(submodel.Id)
// }
// },
// RegionId = new RegionId
// {
// Value = 2
// },
// ResultOption = new ResultOption[]
// {
// new ResultOption
// {
// Value = "WHI_ENGINE"
// }
// }
// };
// VehicleIdSearchResponse vidResponse = _nexpartService.SendRequest<VehicleIdSearch, VehicleIdSearchResponse>(vidSearch).Result;
// if (vidResponse != null && vidResponse.ResponseBody.VehicleToEngineConfigId > 0)
// {
// _partSourceContext.Submodels.Add(new Submodel
// {
// VehicleToEngineConfigId = vidResponse.ResponseBody.VehicleToEngineConfigId,
// SubmodelId = int.Parse(submodel.Id),
// BaseVehicleId = baseVehicle.Id,
// EngineId = vidResponse.ResponseBody.WHIEngineId
// });
// }
// }
// }
// _partSourceContext.SaveChanges();
// }
// catch (Exception ex)
// {
// Console.WriteLine($"Failed to add BaseVehicleId {baseVehicle.Id}: {ex.Message}");
// }
// }
// }
// private void AddYearMakeModels()
// {
// BaseVehicleSearch request = new BaseVehicleSearch
// {
// Years = new Years
// {
// From = 1950,
// To = 2020
// },
// Region = new[]
// {
// new Region
// {
// Id = 2
// }
// },
// VehicleType = new[]
// {
// new VehicleType
// {
// Id = 5
// },
// new VehicleType
// {
// Id = 6
// },
// new VehicleType
// {
// Id = 7
// }
// }
// };
// BaseVehicleSearchResponse response = _nexpartService.SendRequest<BaseVehicleSearch, BaseVehicleSearchResponse>(request).Result;
// foreach (Data.Nexpart.BaseVehicle vehicle in response.ResponseBody.BaseVehicle)
// {
// _partSourceContext.BaseVehicles.Add(new Data.Models.BaseVehicle
// {
// Id = (int)vehicle.BaseVehicleId,
// VehicleMakeId = (int)vehicle.MakeId,
// VehicleModelId = (int)vehicle.ModelId,
// Year = (int)vehicle.Year
// });
// }
// _partSourceContext.SaveChanges();
// }
// private void AddMakes()
// {
// MakeSearch requestContent = new MakeSearch()
// {
// VehicleTypeId = new int[] { 5, 6, 7 },
// RegionId = new int[] { 2 }
// };
// MakeSearchResponse response = _nexpartService.SendRequest<MakeSearch, MakeSearchResponse>(requestContent).Result;
// foreach (Make make in response.ResponseBody.Make)
// {
// _partSourceContext.VehicleMakes.Add(new VehicleMake
// {
// Id = make.Id,
// Name = make.Value
// });
// }
// _partSourceContext.SaveChanges();
// }
// private void AddModels()
// {
// IList<VehicleMake> vehicleMakes = _partSourceContext.VehicleMakes.ToList();
// IDictionary<int, VehicleModel> vehicleModels = new Dictionary<int, VehicleModel>();
// foreach (VehicleMake vehicleMake in vehicleMakes)
// {
// for (int year = 1950; year <= 2020; year++)
// {
// ModelSearch requestContent = new ModelSearch()
// {
// MakeId = vehicleMake.Id,
// Year = year,
// VehicleTypeId = new int[] { 5, 6, 7 },
// RegionId = new int[] { 2 }
// };
// ModelSearchResponse response = _nexpartService.SendRequest<ModelSearch, ModelSearchResponse>(requestContent).Result;
// if (response.ResponseBody == null || response.ResponseBody.Length == 0)
// {
// continue;
// }
// foreach (Model model in response.ResponseBody[0].Model)
// {
// bool result = vehicleModels.TryGetValue(model.Id, out VehicleModel throwaway);
// if (!result)
// {
// VehicleModel vehicleModel = new VehicleModel
// {
// Id = model.Id,
// Name = model.Value,
// VehicleMakeId = vehicleMake.Id
// };
// vehicleModels.Add(model.Id, vehicleModel);
// }
// }
// }
// }
// _partSourceContext.VehicleModels.AddRange(vehicleModels.Values);
// _partSourceContext.SaveChanges();
// }
// }
//}

72
PartSource.Automation/Jobs/Archive/DeleteProducts.cs
View File

@@ -1,72 +0,0 @@
//using PartSource.Automation.Models;
//using PartSource.Data;
//using Ratermania.Shopify;
//using Ratermania.Shopify.Resources;
//using System;
//using System.Collections.Generic;
//using System.Linq;
//using System.Threading.Tasks;
//namespace PartSource.Automation.Jobs
//{
// public class DeleteProducts //: IAutomationJob
// {
// private readonly ShopifyClient _shopifyClient;
// private readonly PartSourceContext _partSourceContext;
// public DeleteProducts(ShopifyClient shopifyClient, PartSourceContext partSourceContext)
// {
// _shopifyClient = shopifyClient;
// _partSourceContext = partSourceContext;
// }
// // If this job fails, oh well. Run it again and again until it works, or use the Shopify UI (LOL)
// public async Task<AutomationJobResult> Run()
// {
// Console.WriteLine("This job will delete ALL PRODUCTS from Shopify. If you really want to delete EVERY SINGLE PRODUCT, type 'mechanical keyboard' below.");
// string input = Console.ReadLine();
// if (input != "mechanical keyboard")
// {
// return new AutomationJobResult
// {
// IsSuccess = true
// };
// }
// IList<long?> shopifyIds = _partSourceContext.ImportData
// .Select(i => i.ShopifyId)
// .Distinct()
// .ToList();
// foreach (long? id in shopifyIds)
// {
// Product product = await _shopifyClient.Products.GetById((long)id);
// await _shopifyClient.Products.Delete(product);
// Console.WriteLine(id);
// }
// //IEnumerable<Product> products = await _shopifyClient.Products.Get();
// //while (products != null)
// //{
// // foreach (Product product in products)
// // {
// // bool result = await _shopifyClient.Products.Delete(product);
// // }
// // products = await _shopifyClient.Products.GetNext();
// // Console.Write('.');
// //}
// return new AutomationJobResult
// {
// Message = "All products deleted. Don't forget to truncate the ImportData table",
// IsSuccess = true
// };
// }
// }
//}

33
PartSource.Automation/Jobs/Archive/StatusCheck.cs
View File

@@ -1,33 +0,0 @@
//using PartSource.Automation.Models;
//using PartSource.Automation.Services;
//using System.Threading.Tasks;
//namespace PartSource.Automation.Jobs
//{
// public class StatusCheck //: IAutomationJob
// {
// private readonly string[] phoneNumbers = { "8593609107", "5134008303" };
// private readonly EmailService _emailService;
// public StatusCheck(EmailService emailService)
// {
// _emailService = emailService;
// }
// public async Task<AutomationJobResult> Run()
// {
// foreach (string phoneNumber in phoneNumbers)
// {
// // TODO: One day it won't just be AT&T numbers
// string to = $"{phoneNumber}@txt.att.net";
// _emailService.Send(to, string.Empty, "The Partsource automation server is running. Check the server for more details.");
// }
// return new AutomationJobResult
// {
// IsSuccess = true
// };
// }
// }
//}

246
PartSource.Automation/Jobs/Archive/UpdateFitment - Copy.cs
View File

@@ -1,246 +0,0 @@
//using Microsoft.EntityFrameworkCore;
//using Newtonsoft.Json;
//using PartSource.Automation.Models;
//using PartSource.Data;
//using PartSource.Data.Models;
//using PartSource.Services;
//using Ratermania.Shopify;
//using Ratermania.Shopify.Exceptions;
//using Ratermania.Shopify.Resources;
//using System;
//using System.Collections.Generic;
//using System.Data;
//using System.Linq;
//using System.Text.RegularExpressions;
//using System.Threading.Tasks;
//namespace PartSource.Automation.Jobs
//{
// public class UpdateFitment //: IAutomationJob
// {
// private readonly IServiceProvider _serviceProvider;
// private readonly ShopifyClient _shopifyClient;
// private readonly PartSourceContext _partSourceContext;
// private readonly NexpartService _nexpartService;
// private readonly VehicleService _vehicleService;
// public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService)
// {
// _partSourceContext = partSourceContext;
// _shopifyClient = shopifyClient;
// _vehicleService = vehicleService;
// }
// public async Task<AutomationJobResult> Run()
// {
// IEnumerable<Product> products = null;
// try
// {
// products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
// }
// catch (Exception ex)
// {
// // TODO: Logging
// return new AutomationJobResult
// {
// Message = "Failed to get products from Shopify",
// IsSuccess = false
// };
// }
// while (products != null && products.Any())
// {
// foreach (Product product in products)
// {
// try
// {
// ImportData importData = await _partSourceContext.ImportData
// .Where(i => i.ShopifyId == product.Id && i.UpdatedAt <= DateTime.Now.AddDays(-7))
// .FirstOrDefaultAsync();
// if (importData == null)
// {
// continue;
// }
// bool isFitment = false;
// IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode, 255);
// if (vehicles.Count > 250)
// {
// continue;
// }
// IList<int> vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles);
// if (vehicleIdFitment.Count > 0 && vehicleIdFitment.Count <= 250)
// {
// isFitment = true;
// string json = JsonConvert.SerializeObject(vehicleIdFitment);
// if (json.Length >= 100000)
// {
// continue;
// }
// Metafield vehicleMetafield = new Metafield
// {
// Namespace = "fitment",
// Key = "ids",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(vehicleMetafield);
// }
// IList<string> ymmFitment = _vehicleService.GetYmmFitment(vehicles);
// if (ymmFitment.Count > 0)
// {
// isFitment = true;
// string json = JsonConvert.SerializeObject(ymmFitment);
// if (json.Length >= 100000)
// {
// continue;
// }
// Metafield ymmMetafield = new Metafield
// {
// Namespace = "fitment",
// Key = "seo",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(ymmMetafield);
// }
// Metafield isFitmentMetafield = new Metafield
// {
// Namespace = "Flags",
// Key = "IsFitment",
// Value = isFitment.ToString(),
// ValueType = "string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(isFitmentMetafield);
// //Metafield noteTextMetafield = new Metafield
// //{
// // Namespace = "Flags",
// // Key = "IsFitment",
// // Value = isFitment.ToString(),
// // ValueType = "string",
// // OwnerResource = "product",
// // OwnerId = product.Id
// //};
// //await _shopifyClient.Metafields.Add(noteTextMetafield);
// List<string> tags = new List<string>
// {
// importData.LineCode,
// importData.PartNumber
// };
// for (int j = 0; j < vehicleIdFitment.Count; j += 25)
// {
// tags.Add(string.Join('-', vehicleIdFitment.Skip(j).Take(25).Select(j => $"v{j}")));
// }
// tags.AddRange(ymmFitment);
// if (tags.Count > 250)
// {
// tags = tags.Take(250).ToList();
// }
// product.Tags = string.Join(',', tags);
// await _shopifyClient.Products.Update(product);
// importData.IsFitment = isFitment;
// importData.UpdatedAt = DateTime.Now;
// importData.UpdateType = "Fitment";
// }
// catch (ShopifyClientException ex)
// {
// // TODO: Log
// }
// catch (Exception ex)
// {
// // TODO: Log
// }
// }
// try
// {
// _partSourceContext.SaveChanges();
// products = await _shopifyClient.Products.GetNext();
// }
// catch (Exception ex)
// {
// products = await _shopifyClient.Products.GetNext();
// }
// }
// return new AutomationJobResult
// {
// IsSuccess = true
// };
// }
// private async Task DeleteFitmentMetafields(long shopifyId)
// {
// IDictionary<string, object> parameters = new Dictionary<string, object>
// {
// { "metafield[owner_id]", shopifyId},
// { "metafield[owner_resource]", "product" },
// { "namespace", "fitment" },
// };
// IEnumerable<Metafield> metafields = await _shopifyClient.Metafields.Get(parameters);
// foreach (Metafield metafield in metafields)
// {
// await _shopifyClient.Metafields.Delete(metafield);
// }
// }
// public IList<Vehicle> GetVehicles(string partNumber, string lineCode)
// {
// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
// //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')";
// string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;";
//#pragma warning disable EF1000 // Possible SQL injection vulnerability.
// IList<Vehicle> vehicles = _partSourceContext.Vehicles.FromSql(sql).ToList();
//#pragma warning restore EF1000 // Possible SQL injection vulnerability.
// return vehicles;
// }
// private void Update()
// {
// }
// }
//}

200
PartSource.Automation/Jobs/Archive/UpdateFitment.cs
View File

@@ -1,200 +0,0 @@
//using Microsoft.EntityFrameworkCore;
//using Newtonsoft.Json;
//using PartSource.Automation.Jobs.Interfaces;
//using PartSource.Automation.Models;
//using PartSource.Automation.Services;
//using PartSource.Data;
//using PartSource.Data.Models;
//using PartSource.Data.Nexpart;
//using PartSource.Services;
//using PartSource.Services.Integrations;
//using System;
//using System.Collections.Concurrent;
//using System.Collections.Generic;
//using System.Data;
//using System.Data.SqlClient;
//using System.IO;
//using System.Linq;
//using System.Text;
//using System.Text.RegularExpressions;
//using System.Threading.Tasks;
//using Ratermania.Shopify.Entities;
//namespace PartSource.Automation.Jobs
//{
// public class UpdateFitment : IAutomationJob
// {
// private readonly IServiceProvider _serviceProvider;
// private readonly ShopifyClient _shopifyClient;
// private readonly PartSourceContext _partSourceContext;
// private readonly NexpartService _nexpartService;
// private readonly VehicleService _vehicleService;
// public UpdateFitment(IServiceProvider serviceProvider, PartSourceContext partSourceContext, ShopifyClient shopifyClient, NexpartService nexpartService, VehicleService vehicleService)
// {
// _partSourceContext = partSourceContext;
// _shopifyClient = shopifyClient;
// _vehicleService = vehicleService;
// }
// public async Task<AutomationJobResult> Run()
// {
// IEnumerable<Product> products = await _shopifyClient.Products.Get();
// int i = 0;
// while (products != null && products.Any())
// {
// foreach (Product product in products)
// {
// try
// {
// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku && i.UpdatedAt == null);
// IList<VehicleData> vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
// if (vehicles == null || vehicles.Count == 0)
// {
// continue;
// }
// await DeleteFitmentMetafields(product.Id);
// bool isFitment = false;
// IList<int> vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles);
// if (vehicleIdFitment.Count > 0)
// {
// isFitment = true;
// string json = JsonConvert.SerializeObject(vehicleIdFitment);
// if (json.Length >= 100000)
// {
// continue;
// }
// Metafield vehicleMetafield = new Metafield
// {
// Namespace = "fitment",
// Key = "ids",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(vehicleMetafield);
// }
// IList<string> ymmFitment = _vehicleService.GetYmmFitment(vehicles);
// if (ymmFitment.Count > 0)
// {
// isFitment = true;
// string json = JsonConvert.SerializeObject(ymmFitment);
// if (json.Length >= 100000)
// {
// continue;
// }
// Metafield ymmMetafield = new Metafield
// {
// Namespace = "fitment",
// Key = "seo",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(ymmMetafield);
// }
// Metafield isFitmentMetafield = new Metafield
// {
// Namespace = "Flags",
// Key = "IsFitment",
// Value = isFitment.ToString(),
// ValueType = "string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(isFitmentMetafield);
// importData.UpdatedAt = DateTime.Now;
// importData.UpdateType = "Fitment";
// }
// catch (Exception ex)
// {
// Console.WriteLine($"{product.Id}: {ex.Message}");
// }
// }
// try
// {
// i++;
// Console.WriteLine(i);
// await _partSourceContext.SaveChangesAsync();
// products = await _shopifyClient.Products.GetNext();
// }
// catch (Exception ex)
// {
// i++;
// Console.WriteLine(i);
// Console.WriteLine($"Retrying request: {ex.Message}");
// await _partSourceContext.SaveChangesAsync();
// products = await _shopifyClient.Products.GetNext();
// }
// }
// return new AutomationJobResult
// {
// IsSuccess = true
// };
// }
// private async Task DeleteFitmentMetafields(long shopifyId)
// {
// IDictionary<string, object> parameters = new Dictionary<string, object>
// {
// { "metafield[owner_id]", shopifyId},
// { "metafield[owner_resource]", "product" },
// { "namespace", "fitment" },
// };
// IEnumerable<Metafield> metafields = await _shopifyClient.Metafields.Get(parameters);
// foreach (Metafield metafield in metafields)
// {
// await _shopifyClient.Metafields.Delete(metafield);
// }
// }
// public IList<VehicleData> GetVehicles(string partNumber, string lineCode)
// {
// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
// //string sql = $"select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where ManufacturerCode in (select WhiCode from DcfMapping where PartSourceCode='{lineCode}') and (partNumber = '{partNumber}' or partNumber = '{partNumber.Replace("-", string.Empty)}')";
// string sql = $"with FitmentIds (BaseVehicleId, EngineConfigId) as (select distinct BaseVehicleId, EngineConfigId from dbo.Fitment where LineCode in (select WhiCode from DcfMapping where LineCode='{lineCode}') and PartNumber = '{partNumber}') select v.* from VehicleData v join FitmentIds f on v.BaseVehicleId = f.BaseVehicleId and v.EngineConfigId = f.EngineConfigId;";
//#pragma warning disable EF1000 // Possible SQL injection vulnerability.
// IList<VehicleData> vehicles = _partSourceContext.VehicleData.FromSql(sql).ToList();
//#pragma warning restore EF1000 // Possible SQL injection vulnerability.
// return vehicles;
// }
// private void Update()
// {
// }
// }
//}

209
PartSource.Automation/Jobs/Archive/UpdatePositioning.cs
View File

@@ -1,209 +0,0 @@
//using Microsoft.EntityFrameworkCore;
//using Newtonsoft.Json;
//using PartSource.Automation.Models;
//using PartSource.Data;
//using PartSource.Data.Models;
//using PartSource.Services;
//using Ratermania.Shopify;
//using Ratermania.Shopify.Resources;
//using System;
//using System.Collections.Generic;
//using System.Diagnostics.CodeAnalysis;
//using System.Linq;
//using System.Text.RegularExpressions;
//using System.Threading.Tasks;
//namespace PartSource.Automation.Jobs
//{
// public class UpdatePositioning// : IAutomationJob
// {
// private readonly ShopifyClient _shopifyClient;
// private readonly PartSourceContext _partSourceContext;
// private readonly VehicleService _vehicleService;
// public UpdatePositioning(PartSourceContext partSourceContext, ShopifyClient shopifyClient, VehicleService vehicleService)
// {
// _partSourceContext = partSourceContext;
// _shopifyClient = shopifyClient;
// _vehicleService = vehicleService;
// }
// public async Task<AutomationJobResult> Run()
// {
// IDictionary<string, object> parameters = new Dictionary<string, object>
// {
// { "limit", 250 }
// };
// IEnumerable<Product> products = await _shopifyClient.Products.Get(parameters);
// while (products != null && products.Any())
// {
// foreach (Product product in products)
// {
// try
// {
// ImportData importData = _partSourceContext.ImportData.FirstOrDefault(i => i.VariantSku == product.Variants[0].Sku);
// if (importData == null || importData?.LineCode == "SVG") // Headlights go in front, DUH
// {
// continue;
// }
// IList<Fitment> fitments = GetPositionOrderedFitments(importData?.PartNumber, importData?.LineCode);
// IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData?.PartNumber, importData?.LineCode);
// if (fitments.Count == 0 || vehicles.Count == 0)
// {
// continue;
// }
// //await DeletePositionMetafields(product.Id);
// //string currentPosition = fitments[0].Position;
// //List<int> vehicleIds = new List<int>();
// //foreach (Fitment fitment in fitments)
// //{
// // if (fitment.Position != currentPosition)
// // {
// // await SavePositionMetafield(product, vehicleIds, currentPosition);
// // currentPosition = fitment.Position;
// // vehicleIds = new List<int>();
// // }
// // // We don't need to DCF map because these are both sourced from WHI
// // IList<int> fitmentVehicleIds = vehicles.Where(v => v.BaseVehicleId == fitment.BaseVehicleId && v.EngineConfigId == fitment.EngineConfigId)
// // .Select(v => v.VehicleToEngineConfigId)
// // .Distinct()
// // .ToList();
// // vehicleIds.AddRange(fitmentVehicleIds);
// //}
// //await SavePositionMetafield(product, vehicleIds, currentPosition);
// IList<string> notes = fitments.Select(f => f.NoteText)
// .Distinct()
// .ToList();
// IList<object> vehicleNotes = new List<object>();
// foreach (string noteText in notes)
// {
// IList<int> vehicleIds = fitments.Where(f => f.NoteText == noteText)
// .Select(f => new { f.EngineConfigId, f.BaseVehicleId })
// .SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId))
// .Select(v => v.VehicleToEngineConfigId)
// .ToList();
// vehicleNotes.Add(new { noteText, vehicleIds });
// }
// string json = JsonConvert.SerializeObject(vehicleNotes);
// if (json.Length >= 100000)
// {
// continue;
// }
// Metafield vehicleMetafield = new Metafield
// {
// Namespace = "fitment",
// Key = "note_text",
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// await _shopifyClient.Metafields.Add(vehicleMetafield);
// //importData.UpdatedAt = DateTime.Now;
// //importData.UpdateType = "Positioning";
// ;
// }
// catch (Exception ex)
// {
// Console.WriteLine($"{product.Id}: {ex.Message}");
// }
// }
// try
// {
// Console.Write('.');
// // await _partSourceContext.SaveChangesAsync();
// products = await _shopifyClient.Products.GetNext();
// }
// catch (Exception ex)
// {
// Console.WriteLine($"Retrying: {ex.Message}");
// products = await _shopifyClient.Products.GetPrevious();
// }
// }
// return new AutomationJobResult
// {
// Message = "Positioning updated successfully",
// IsSuccess = true
// };
// }
// private IList<Fitment> GetPositionOrderedFitments(string partNumber, string lineCode)
// {
// partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
// IQueryable<string> whiCodes = _partSourceContext.DcfMappings
// .Where(d => d.LineCode == lineCode)
// .Select(d => d.WhiCode);
// return _partSourceContext.Fitments
// .Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode) && !string.IsNullOrEmpty(f.Position))
// .OrderBy(f => f.Position)
// .ToList();
// }
// [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "It's a Shopify metafield key")]
// private async Task SavePositionMetafield(Product product, IList<int> vehicleIds, string position)
// {
// if (vehicleIds.Count == 0)
// {
// return;
// }
// string json = JsonConvert.SerializeObject(vehicleIds);
// if (json.Length >= 100000)
// {
// // TODO: Logging
// return;
// }
// string key = position.ToLowerInvariant().Replace(" ", "_");
// if (key.Length > 20)
// {
// key = key.Substring(0, 20);
// }
// Metafield vehicleMetafield = new Metafield
// {
// Namespace = "position",
// Key = key,
// Value = json,
// ValueType = "json_string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// System.Diagnostics.Debug.WriteLine(json);
// await _shopifyClient.Metafields.Add(vehicleMetafield);
// }
// }
//}

113
PartSource.Automation/Jobs/Archive/UpdatePricing.cs
View File

@@ -1,113 +0,0 @@
//using Microsoft.EntityFrameworkCore;
//using Microsoft.Extensions.Logging.Abstractions;
//using Ratermania.Automation.Interfaces;
//using PartSource.Automation.Models;
//using PartSource.Data;
//using PartSource.Data.Models;
//using Ratermania.Shopify;
//using Ratermania.Shopify.Resources;
//using Ratermania.Shopify.Resources.Enums;
//using System;
//using System.Collections.Generic;
//using System.Linq;
//using System.Threading.Tasks;
//using Microsoft.Extensions.Logging;
//namespace PartSource.Automation.Jobs
//{
// public class UpdatePricing : IAutomationJob
// {
// private readonly ILogger<UpdatePricing> _logger;
// private readonly PartSourceContext _partSourceContext;
// private readonly ShopifyClient _shopifyClient;
// public UpdatePricing(ILogger<UpdatePricing> logger, PartSourceContext partSourceContext, ShopifyClient shopifyClient)
// {
// _logger = logger;
// _partSourceContext = partSourceContext;
// _shopifyClient = shopifyClient;
// }
// public async Task Run()
// {
// IEnumerable<Product> products = null;
// IEnumerable<PartPrice> prices = null;
// int updateCount = 0;
// try
// {
// products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
// prices = await _partSourceContext.PartPrices.ToListAsync();
// }
// catch (Exception ex)
// {
// _logger.LogError(ex, "Failed to get the initial set of products from Shopify.");
// throw;
// }
// while (products != null && products.Any())
// {
// foreach (Product product in products)
// {
// if (product.Variants.Length > 0)
// {
// Variant variant = product.Variants[0];
// PartPrice partPrice = prices.Where(p => p.SKU == variant.Sku).FirstOrDefault();
// if (partPrice == null || !partPrice.Your_Price.HasValue || !partPrice.Compare_Price.HasValue)
// {
// continue;
// }
// if (product.Variants[0].Price != partPrice.Your_Price.Value || product.Variants[0].CompareAtPrice != partPrice.Compare_Price.Value)
// {
// product.Variants[0].Price = partPrice.Your_Price.Value;
// product.Variants[0].CompareAtPrice = partPrice.Compare_Price.Value;
// product.PublishedAt = partPrice.Active.Trim().ToUpperInvariant() == "Y" ? (DateTime?)DateTime.Now : null;
// product.PublishedScope = PublishedScope.Global;
// Metafield metafield = new Metafield
// {
// Namespace = "Pricing",
// Key = "CorePrice",
// Value = partPrice.Core_Price.HasValue ? partPrice.Core_Price.Value.ToString() : "0.00",
// ValueType = "string",
// OwnerResource = "product",
// OwnerId = product.Id
// };
// try
// {
// await _shopifyClient.Metafields.Add(metafield);
// await _shopifyClient.Products.Update(product);
// updateCount++;
// }
// catch (Exception ex)
// {
// _logger.LogWarning(ex, $"Failed to update pricing for SKU {variant.Sku}");
// }
// }
// }
// }
// try
// {
// products = await _shopifyClient.Products.GetNext();
// _logger.LogInformation($"Total updated: {updateCount}");
// }
// catch (Exception ex)
// {
// _logger.LogWarning(ex, "Failed to get the next set of products. Retrying");
// products = await _shopifyClient.Products.GetPrevious();
// }
// }
// }
// }
//}

137
PartSource.Automation/Jobs/ProcessWhiFitment.cs
View File

@@ -5,10 +5,12 @@ using PartSource.Automation.Models.Enums;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -34,92 +36,103 @@ namespace PartSource.Automation.Jobs
public async Task Run()
{
_whiSeoService.Truncate(_seoDataType);
_whiSeoService.Truncate();
_whiSeoService.GetFiles(_seoDataType);
string directory = Path.Combine(_ftpConfiguration.Destination, _seoDataType.ToString().ToLowerInvariant());
DirectoryInfo directoryInfo = new DirectoryInfo(directory);
foreach (FileInfo fileInfo in directoryInfo.GetFiles())
ConcurrentQueue<FileInfo> files = new ConcurrentQueue<FileInfo>(directoryInfo.GetFiles().Where(f => f.Name.EndsWith("csv.gz")).OrderBy(f => f.Length));
while (files.Count > 0)
{
if (!fileInfo.Name.EndsWith("csv.gz"))
Parallel.For(0, 4, index =>
{
continue;
}
string filename = Decompress(fileInfo);
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
try
{
int skippedLines = 0;
while (reader.Peek() > 0)
if (!files.TryDequeue(out FileInfo fileInfo))
{
line = reader.ReadLine();
return;
}
string[] columns = line.Replace("\"", string.Empty).Split(',');
string filename = Decompress(fileInfo);
if (columns.Length != 8)
using DataTable dataTable = new DataTable();
dataTable.Columns.Add("LineCode", typeof(string));
dataTable.Columns.Add("PartNumber", typeof(string));
dataTable.Columns.Add("BaseVehicleId", typeof(int));
dataTable.Columns.Add("EngineConfigId", typeof(int));
dataTable.Columns.Add("Position", typeof(string));
dataTable.Columns.Add("NoteText", typeof(string));
using StreamReader reader = new StreamReader(filename);
string line = reader.ReadLine(); // Burn the header row
try
{
int skippedLines = 0;
while (reader.Peek() > 0)
{
skippedLines++;
continue;
line = reader.ReadLine();
string[] columns = line.Split("\",\"");
if (columns.Length != 8)
{
skippedLines++;
continue;
}
for (int i = 0; i < columns.Length; i++)
{
columns[i] = columns[i].Replace("\"", string.Empty);
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
}
}
string lineCode = Regex.Replace(columns[0], "[^a-zA-Z0-9]", string.Empty).Trim();
string partNumber = Regex.Replace(columns[1], "[^a-zA-Z0-9]", string.Empty).Trim();
string position = columns[7].Trim();
string noteText = columns[4].Trim();
string tableName = fileInfo.Name.Substring(0, fileInfo.Name.IndexOf('.'));
_whiSeoService.BulkCopy(_seoDataType, dataTable, tableName);
if (!string.IsNullOrEmpty(lineCode)
&& !string.IsNullOrEmpty(partNumber)
&& int.TryParse(columns[5], out int baseVehicleId)
&& int.TryParse(columns[6], out int engineConfigId))
if (skippedLines == 0)
{
dataTable.Rows.Add(new object[] { lineCode, partNumber, baseVehicleId, engineConfigId, position, noteText });
_logger.LogInformation($"Copied {filename} to the database.");
}
else
{
_logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed.");
}
File.Delete(fileInfo.FullName);
}
_whiSeoService.BulkCopy(_seoDataType, dataTable);
if (skippedLines == 0)
catch (Exception ex)
{
_logger.LogInformation($"Copied {filename} to the database.");
_logger.LogError($"Failed to copy {filename} to the database.", ex);
}
else
try
{
_logger.LogWarning($"Copied {filename} to the database with warnings. {skippedLines} lines contained errors and could not be processed.");
reader.Close();
File.Delete(filename);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to copy {filename} to the database.", ex);
}
try
{
reader.Close();
File.Delete(filename);
File.Delete(fileInfo.FullName);
}
catch (Exception ex)
{
_logger.LogWarning($"Failed to delete {filename}. This file will need to be deleted manually.", ex);
}
catch (Exception ex) { }
});
}
_whiSeoService.CreateFitmentView();
}
public string Decompress(FileInfo fileInfo)
@@ -128,7 +141,7 @@ namespace PartSource.Automation.Jobs
using FileStream filestream = File.Create(decompressedFile);
using GZipStream decompressionStream = new GZipStream(fileInfo.OpenRead(), CompressionMode.Decompress);
decompressionStream.CopyTo(filestream);
return decompressedFile;

29
PartSource.Automation/Jobs/StatusCheck.cs Normal file
View File

@@ -0,0 +1,29 @@
using PartSource.Automation.Models;
using PartSource.Automation.Services;
using Ratermania.Automation.Interfaces;
using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class StatusCheck : IAutomationJob
{
private readonly string[] phoneNumbers = { "8593609107", "5134008303" };
private readonly EmailService _emailService;
public StatusCheck(EmailService emailService)
{
_emailService = emailService;
}
public async Task Run()
{
foreach (string phoneNumber in phoneNumbers)
{
// TODO: One day it won't just be AT&T numbers
string to = $"{phoneNumber}@txt.att.net";
_emailService.Send(to, string.Empty, "The Partsource automation server is running. Check the server for more details.");
}
}
}
}

75
PartSource.Automation/Jobs/SyncronizeProducts.cs Normal file
View File

@@ -0,0 +1,75 @@
using Ratermania.Automation.Interfaces;
using PartSource.Automation.Models;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using PartSource.Data;
using PartSource.Services;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
using PartSource.Data.Models;
using System.Linq;
using Microsoft.EntityFrameworkCore;
namespace PartSource.Automation.Jobs
{
/// <summary>
/// Ensures syncronization between Shopify IDs and Partsource SKUs
/// </summary>
public class SyncronizeProducts : IAutomationJob
{
private readonly PartSourceContext _partSourceContext;
private readonly ShopifyClient _shopifyClient;
private readonly ILogger<TestJob> _logger;
public SyncronizeProducts(ILogger<TestJob> logger, PartSourceContext partSourceContext, ShopifyClient shopifyClient)
{
_partSourceContext = partSourceContext;
_shopifyClient = shopifyClient;
_logger = logger;
}
public async Task Run()
{
IList<ImportData> importData = _partSourceContext.ImportData.FromSql<ImportData>($"SELECT * FROM ImportDataFilters").ToList();
IEnumerable<Product> products = await _shopifyClient.Products.Get(new Dictionary<string, object> { { "limit", 250 } });
while (products?.Any() == true)
{
foreach (Product product in products)
{
foreach (Variant variant in product.Variants)
{
ImportData item = importData.FirstOrDefault(i => i.VariantSku == variant.Sku);
if (item != null)
{
_partSourceContext.Database.ExecuteSqlCommand($"UPDATE ImportDataFilters SET ShopifyId = {product.Id} WHERE VariantSku = {variant.Sku}");
}
}
}
try
{
_logger.LogInformation("Did 250");
//await _partSourceContext.SaveChangesAsync();
}
catch
{
Console.WriteLine("Failed to save a batch of products");
}
finally
{
products = await _shopifyClient.Products.GetNext();
}
}
}
}
}

37
PartSource.Automation/Jobs/UpdateFitment.cs
View File

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
@@ -23,12 +24,14 @@ namespace PartSource.Automation.Jobs
private readonly ILogger<UpdateFitment> _logger;
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
public UpdateFitment(ILogger<UpdateFitment> logger, PartSourceContext partSourceContext, ShopifyClient shopifyClient, VehicleService vehicleService)
public UpdateFitment(ILogger<UpdateFitment> logger, PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
{
_logger = logger;
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
}
@@ -48,6 +51,12 @@ namespace PartSource.Automation.Jobs
throw;
}
IList<ImportData> parts = await _partSourceContext.ImportData
.Where(i => i.UpdatedAt <= DateTime.Now.AddDays(-7))
.ToListAsync();
int i = 1;
while (products != null && products.Any())
{
foreach (Product product in products)
@@ -56,9 +65,7 @@ namespace PartSource.Automation.Jobs
try
{
importData = await _partSourceContext.ImportData
.Where(i => i.ShopifyId == product.Id && i.UpdatedAt <= DateTime.Now.AddDays(-7))
.FirstOrDefaultAsync();
importData = parts.FirstOrDefault(parts => parts.ShopifyId == product.Id);
if (importData == null)
{
@@ -69,13 +76,13 @@ namespace PartSource.Automation.Jobs
IList<Vehicle> vehicles = _vehicleService.GetVehiclesForPart(importData.PartNumber, importData.LineCode, 255);
if (vehicles.Count > 250)
{
vehicles = vehicles.Take(250)
.ToList();
//if (vehicles.Count > 250)
//{
// vehicles = vehicles.Take(250)
// .ToList();
_logger.LogInformation($"SKU {importData.VariantSku} fits more than 250 vehicles. Only the first 250 will be used.");
}
// _logger.LogInformation($"SKU {importData.VariantSku} fits more than 250 vehicles. Only the first 250 will be used.");
//}
IList<int> vehicleIdFitment = _vehicleService.GetVehicleIdFitment(vehicles);
@@ -164,6 +171,12 @@ namespace PartSource.Automation.Jobs
tags = tags.Take(250).ToList();
}
string zzzIsFitment = isFitment
? "zzzIsFitment=true"
: "zzzIsFitment=false";
tags.Add(zzzIsFitment);
product.Tags = string.Join(',', tags);
await _shopifyClient.Products.Update(product);
@@ -181,8 +194,12 @@ namespace PartSource.Automation.Jobs
try
{
Console.WriteLine(i);
_partSourceContext.SaveChanges();
products = await _shopifyClient.Products.GetNext();
i++;
}
catch (Exception ex)

67
PartSource.Automation/Jobs/UpdatePositioning.cs
View File

@@ -2,8 +2,10 @@
using Newtonsoft.Json;
using PartSource.Automation.Models;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Models;
using PartSource.Services;
using Ratermania.Automation.Interfaces;
using Ratermania.Shopify;
using Ratermania.Shopify.Resources;
using System;
@@ -15,20 +17,22 @@ using System.Threading.Tasks;
namespace PartSource.Automation.Jobs
{
public class UpdatePositioning// : IAutomationJob
public class UpdatePositioning : IAutomationJob
{
private readonly ShopifyClient _shopifyClient;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
private readonly VehicleService _vehicleService;
public UpdatePositioning(PartSourceContext partSourceContext, ShopifyClient shopifyClient, VehicleService vehicleService)
public UpdatePositioning(PartSourceContext partSourceContext, FitmentContext fitmentContext, ShopifyClient shopifyClient, VehicleService vehicleService)
{
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
_shopifyClient = shopifyClient;
_vehicleService = vehicleService;
}
public async Task<AutomationJobResult> Run()
public async Task Run()
{
IDictionary<string, object> parameters = new Dictionary<string, object>
{
@@ -58,31 +62,29 @@ namespace PartSource.Automation.Jobs
continue;
}
//await DeletePositionMetafields(product.Id);
string currentPosition = fitments[0].Position;
List<int> vehicleIds = new List<int>();
//string currentPosition = fitments[0].Position;
//List<int> vehicleIds = new List<int>();
foreach (Fitment fitment in fitments)
{
if (fitment.Position != currentPosition)
{
await SavePositionMetafield(product, vehicleIds, currentPosition);
//foreach (Fitment fitment in fitments)
//{
// if (fitment.Position != currentPosition)
// {
// await SavePositionMetafield(product, vehicleIds, currentPosition);
currentPosition = fitment.Position;
vehicleIds = new List<int>();
}
// currentPosition = fitment.Position;
// vehicleIds = new List<int>();
// }
// We don't need to DCF map because these are both sourced from WHI
IList<int> fitmentVehicleIds = vehicles.Where(v => v.BaseVehicleId == fitment.BaseVehicleId && v.EngineConfigId == fitment.EngineConfigId)
.Select(v => v.VehicleToEngineConfigId)
.Distinct()
.ToList();
// // We don't need to DCF map because these are both sourced from WHI
// IList<int> fitmentVehicleIds = vehicles.Where(v => v.BaseVehicleId == fitment.BaseVehicleId && v.EngineConfigId == fitment.EngineConfigId)
// .Select(v => v.VehicleToEngineConfigId)
// .Distinct()
// .ToList();
vehicleIds.AddRange(fitmentVehicleIds);
}
// vehicleIds.AddRange(fitmentVehicleIds);
//}
//await SavePositionMetafield(product, vehicleIds, currentPosition);
await SavePositionMetafield(product, vehicleIds, currentPosition);
IList<string> notes = fitments.Select(f => f.NoteText)
@@ -93,13 +95,12 @@ namespace PartSource.Automation.Jobs
foreach (string noteText in notes)
{
IList<int> vehicleIds = fitments.Where(f => f.NoteText == noteText)
vehicleIds = fitments.Where(f => f.NoteText == noteText)
.Select(f => new { f.EngineConfigId, f.BaseVehicleId })
.SelectMany(f => vehicles.Where(v => v.BaseVehicleId == f.BaseVehicleId && v.EngineConfigId == f.EngineConfigId))
.Select(v => v.VehicleToEngineConfigId)
.ToList();
vehicleNotes.Add(new { noteText, vehicleIds });
}
@@ -123,8 +124,6 @@ namespace PartSource.Automation.Jobs
//importData.UpdatedAt = DateTime.Now;
//importData.UpdateType = "Positioning";
;
}
catch (Exception ex)
@@ -134,9 +133,7 @@ namespace PartSource.Automation.Jobs
}
try
{
Console.Write('.');
// await _partSourceContext.SaveChangesAsync();
{
products = await _shopifyClient.Products.GetNext();
}
@@ -147,23 +144,17 @@ namespace PartSource.Automation.Jobs
products = await _shopifyClient.Products.GetPrevious();
}
}
return new AutomationJobResult
{
Message = "Positioning updated successfully",
IsSuccess = true
};
}
private IList<Fitment> GetPositionOrderedFitments(string partNumber, string lineCode)
{
partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
IQueryable<string> whiCodes = _partSourceContext.DcfMappings
IQueryable<string> whiCodes = _fitmentContext.DcfMappings
.Where(d => d.LineCode == lineCode)
.Select(d => d.WhiCode);
return _partSourceContext.Fitments
return _fitmentContext.Fitments
.Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode) && !string.IsNullOrEmpty(f.Position))
.OrderBy(f => f.Position)
.ToList();

42
PartSource.Automation/Program.cs
View File

@@ -8,6 +8,8 @@ using PartSource.Automation.Jobs;
using PartSource.Automation.Services;
using PartSource.Data;
using PartSource.Data.AutoMapper;
using PartSource.Data.Contexts;
using PartSource.Services;
using Ratermania.Automation.DependencyInjection;
using Ratermania.Automation.Logging;
using Ratermania.Shopify.DependencyInjection;
@@ -21,9 +23,18 @@ namespace PartSource.Automation
{
static async Task Main(string[] args)
{
using IHost host = CreateHostBuilder().Build();
try
{
using IHost host = CreateHostBuilder().Build();
await host.StartAsync();
await host.StartAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw;
}
}
private static IHostBuilder CreateHostBuilder()
@@ -43,6 +54,10 @@ namespace PartSource.Automation
options.UseSqlServer(builder.Configuration.GetConnectionString("PartSourceDatabase"), opts => opts.EnableRetryOnFailure())
)
.AddDbContext<FitmentContext>(options =>
options.UseSqlServer(builder.Configuration.GetConnectionString("FitmentDatabase"), opts => opts.EnableRetryOnFailure())
)
.AddShopify(options =>
{
options.ApiKey = builder.Configuration["Shopify:ApiKey"];
@@ -53,21 +68,19 @@ namespace PartSource.Automation
.AddAutomation(options =>
{
options.HasBaseInterval(new TimeSpan(0, 15, 0))
options.HasBaseInterval(new TimeSpan(0, 1, 0))
.HasMaxFailures(5)
//.HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
//.HasJob<SyncronizeProducts>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasJob<ProcessWhiFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0))
//.HasJob<UpdateFitment>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasDependency<ProcessWhiFitment>()
// .StartsAt(DateTime.Today.AddHours(8))
//)
.HasJob<ExecuteSsisPackages>(options =>
options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(26))
)
.HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0))
.StartsAt(DateTime.Today.AddHours(27))
.HasDependency<ExecuteSsisPackages>()
.HasJob<ExecuteSsisPackages>(options => options.HasInterval(new TimeSpan(24, 0, 0))
// .HasJob<UpdatePricing>(options => options.HasInterval(new TimeSpan(24, 0, 0)))
//.HasDependency<ExecuteSsisPackages>()
//.StartsAt(DateTime.Now.AddMinutes(15))
)
.AddApiServer(options => options.HasApiKey(Environment.GetEnvironmentVariable("AUTOMATION_API_KEY")));
})
@@ -75,13 +88,16 @@ namespace PartSource.Automation
.AddSingleton<EmailService>()
.AddSingleton<SsisService>()
.AddSingleton<WhiSeoService>()
.AddSingleton<VehicleService>()
.AddAutoMapper(typeof(PartSourceProfile));
})
.ConfigureLogging((builder, logging) =>
{
logging.ClearProviders()
.AddProvider(new AutomationLoggerProvider());
logging.AddConsole();
logging.AddProvider(new AutomationLoggerProvider());
});
}
}

34
PartSource.Automation/Services/WhiSeoService.cs
View File

@@ -21,7 +21,7 @@ namespace PartSource.Automation.Services
FtpConfiguration ftpConfiguration = configuration.GetSection("FtpServers:WhiConfiguration").Get<FtpConfiguration>();
_ftpService = new FtpService(ftpConfiguration);
_connectionString = configuration.GetConnectionString("PartSourceDatabase");
_connectionString = configuration.GetConnectionString("FitmentDatabase");
_logger = logger;
}
@@ -51,29 +51,41 @@ namespace PartSource.Automation.Services
}
}
public void Truncate(SeoDataType seoDataType)
public void Truncate()
{
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using SqlCommand command = new SqlCommand($"exec DropFitmentTables", connection);
command.ExecuteNonQuery();
}
public void BulkCopy(SeoDataType seoDataType, DataTable dataTable, string tableName)
{
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities
using SqlCommand command = new SqlCommand($"TRUNCATE TABLE [{seoDataType}]", connection);
using SqlCommand command = new SqlCommand($"EXEC AddFitmentTable @tableName = '{tableName}'", connection);
command.ExecuteNonQuery();
#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities
}
public void BulkCopy(SeoDataType seoDataType, DataTable dataTable)
{
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities
using SqlBulkCopy bulk = new SqlBulkCopy(connection)
{
DestinationTableName = seoDataType.ToString(),
DestinationTableName = $"{seoDataType}.{tableName}",
BulkCopyTimeout = 14400
};
bulk.WriteToServer(dataTable);
}
public void CreateFitmentView()
{
using SqlConnection connection = new SqlConnection(_connectionString);
connection.Open();
using SqlCommand command = new SqlCommand($"exec CreateFitmentView", connection);
command.ExecuteNonQuery();
}
}
}

10
PartSource.Automation/appsettings.json
View File

@@ -1,6 +1,6 @@
{
"ConnectionStrings": {
//"PartSourceDatabase": "Server=localhost;Database=ps-whi-stage;Integrated Security=True;"
"FitmentDatabase": "Data Source=localhost;Initial Catalog=WhiFitment;Integrated Security=true",
"PartSourceDatabase": "Server=tcp:ps-whi.database.windows.net,1433;Initial Catalog=ps-whi-stage;Persist Security Info=False;User ID=ps-whi;Password=9-^*N5dw!6:|.5Q;MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
},
"emailConfiguration": {
@@ -32,7 +32,11 @@
"ApiSecret": "527a3b4213c2c7ecb214728a899052df",
"ShopDomain": "partsource.myshopify.com"
},
"LogLevel": {
"Default": "Information"
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
}

33
PartSource.Data/Contexts/FitmentContext.cs Normal file
View File

@@ -0,0 +1,33 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata;
using PartSource.Data.Models;
using System;
using System.Collections.Generic;
using System.Text;
namespace PartSource.Data.Contexts
{
public class FitmentContext : DbContext
{
public FitmentContext(DbContextOptions<FitmentContext> contextOptions) : base(contextOptions) { }
public DbSet<DcfMapping> DcfMappings { get; set; }
public DbSet<Fitment> Fitments { get; set; }
public DbSet<Vehicle> Vehicles { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
modelBuilder.Entity<Fitment>().HasKey(f => new { f.BaseVehicleId, f.EngineConfigId, f.LineCode, f.PartNumber });
foreach (IMutableEntityType entityType in modelBuilder.Model.GetEntityTypes())
{
entityType.Relational().TableName = entityType.ClrType.Name;
}
}
}
}

2
PartSource.Data/Models/Fitment.cs
View File

@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace PartSource.Data.Models
{
[Table("P8F")]
public partial class Fitment
{
public string PartNumber { get; set; }

2
PartSource.Data/Models/ImportData.cs
View File

@@ -6,7 +6,7 @@ using System.Text;
namespace PartSource.Data.Models
{
//[Table("ImportDataOld")]
[Table("ImportDataFitment")]
public class ImportData
{
public string Title { get; set; }

7
PartSource.Data/PartSourceContext.cs
View File

@@ -15,10 +15,6 @@ namespace PartSource.Data
public DbSet<ProductBackup> ProductBackups { get; set; }
public DbSet<DcfMapping> DcfMappings { get; set; }
public DbSet<Fitment> Fitments { get; set; }
public DbSet<Manufacturer> Manufacturers { get; set; }
public DbSet<ImportData> ImportData { get; set; }
@@ -58,9 +54,6 @@ namespace PartSource.Data
modelBuilder.Query<VehicleModel>().ToView(nameof(VehicleModel));
modelBuilder.Entity<PartsAvailability>().HasKey(p => new { p.Store, p.SKU });
modelBuilder.Entity<DcfMapping>().HasKey(d => new { d.LineCode, d.WhiCode });
modelBuilder.Entity<Fitment>().HasKey(f => new { f.BaseVehicleId, f.EngineConfigId, f.LineCode, f.PartNumber });
foreach (IMutableEntityType entityType in modelBuilder.Model.GetEntityTypes())
{

11
PartSource.Services/VehicleService.cs
View File

@@ -1,6 +1,7 @@
using AutoMapper;
using Microsoft.EntityFrameworkCore;
using PartSource.Data;
using PartSource.Data.Contexts;
using PartSource.Data.Dtos;
using PartSource.Data.Models;
using PartSource.Services.Extensions;
@@ -17,11 +18,13 @@ namespace PartSource.Services
{
private readonly IMapper _mapper;
private readonly PartSourceContext _partSourceContext;
private readonly FitmentContext _fitmentContext;
public VehicleService(IMapper mapper, PartSourceContext partSourceContext)
public VehicleService(IMapper mapper, PartSourceContext partSourceContext, FitmentContext fitmentContext)
{
_mapper = mapper;
_partSourceContext = partSourceContext;
_fitmentContext = fitmentContext;
}
public async Task<IList<Vehicle>> GetVehicles(VehicleDto vehicleQuery)
@@ -284,13 +287,13 @@ namespace PartSource.Services
partNumber = Regex.Replace(partNumber, "[^a-zA-Z0-9]", string.Empty);
IQueryable<string> whiCodes = _partSourceContext.DcfMappings
IQueryable<string> whiCodes = _fitmentContext.DcfMappings
.Where(d => d.LineCode == lineCode)
.Select(d => d.WhiCode);
IQueryable<Vehicle> vehicles = _partSourceContext.Fitments
IQueryable<Vehicle> vehicles = _fitmentContext.Fitments
.Where(f => f.PartNumber == partNumber && whiCodes.Contains(f.LineCode))
.Join(_partSourceContext.Vehicles,
.Join(_fitmentContext.Vehicles,
f => new { f.BaseVehicleId, f.EngineConfigId },
v => new { v.BaseVehicleId, v.EngineConfigId },
(f, v) => v);