From 9dbb234d6530bd1fbacaa1d90161d547b9027b86 Mon Sep 17 00:00:00 2001 From: KonsN Date: Fri, 6 Nov 2020 16:48:14 -0600 Subject: [PATCH] Implemented SqlReplay.Batch utility. --- SqlReplay.Batch/BatchConfig.cs | 31 +++ SqlReplay.Batch/Program.cs | 239 ++++++++++++++++++ .../Properties/launchSettings.json | 7 + SqlReplay.Batch/SampleBatchConfig.json | 22 ++ SqlReplay.Batch/SqlReplay.Batch.csproj | 14 + SqlReplay.Utilities.sln | 25 ++ 6 files changed, 338 insertions(+) create mode 100644 SqlReplay.Batch/BatchConfig.cs create mode 100644 SqlReplay.Batch/Program.cs create mode 100644 SqlReplay.Batch/Properties/launchSettings.json create mode 100644 SqlReplay.Batch/SampleBatchConfig.json create mode 100644 SqlReplay.Batch/SqlReplay.Batch.csproj create mode 100644 SqlReplay.Utilities.sln diff --git a/SqlReplay.Batch/BatchConfig.cs b/SqlReplay.Batch/BatchConfig.cs new file mode 100644 index 0000000..f3cb686 --- /dev/null +++ b/SqlReplay.Batch/BatchConfig.cs @@ -0,0 +1,31 @@ +namespace SqlReplay.Batch +{ + class BatchConfig + { + public string ReplayConsolePath { get; set; } + + public string ConnectionString { get; set; } + + public string OutputPath { get; set; } + + public Batch[] Batches { get; set; } + + public int StatusCheckIntervalInSec { get; set; } + } + + class Batch + { + public int ProcessingOrder { get; set; } + + public string BatchId { get; set; } + + public int NumOfInstances { get; set; } + + public string StartAt { get; set; } + + public int DurationInMinutes { get; set; } + + public string[] InputFiles { get; set; } + } +} + diff --git a/SqlReplay.Batch/Program.cs b/SqlReplay.Batch/Program.cs new file mode 100644 index 0000000..2407e24 --- /dev/null +++ b/SqlReplay.Batch/Program.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using Newtonsoft.Json; + +namespace SqlReplay.Batch +{ + class Program + { + const int DefaultStatusCheckIntervalInSec = 60; + + const int DefaultReplayDurationInMinutes = 0; // replay all + + static void Main(string[] args) + { + if (args == null || string.IsNullOrWhiteSpace(args[0])) + { + ShowUsage(); + Environment.Exit(-1); + } + + string logPath = string.Empty; + if (args.Length > 1 && !string.IsNullOrWhiteSpace(args[1]) && Directory.Exists(args[1])) + logPath = Path.Combine(args[1], $"SqlReplay.Batch_{GetTimestamp()}.log"); + + WriteMessage(logPath, $"{GetTimestamp()} - Validating configuration..."); + BatchConfig config = ValidateAndGetConfig(args[0], out List validationResults); + if (config == null) + { + WriteMessage(logPath, $"{GetTimestamp()} - Validation failed:"); + validationResults.ForEach(msg=>WriteMessage(logPath, msg)); + Environment.Exit(-1); + } + WriteMessage(logPath, $"{GetTimestamp()} - Validation completed"); + + string appFileName = Path.GetFileName(config.ReplayConsolePath); + string appPath = Path.GetDirectoryName(config.ReplayConsolePath); + bool isDll = Path.GetExtension(config.ReplayConsolePath).ToUpper() == ".DLL"; + + WriteMessage(logPath, $"{GetTimestamp()} - Processing started"); + + foreach (Batch batch in config.Batches) + { + var processes = new List(); + WriteMessage(logPath, $"{Environment.NewLine}{GetTimestamp()} - Running batch {batch.ProcessingOrder} of {config.Batches.Length}. Batch ID: {batch.BatchId}"); + for (int i = 0; i < batch.NumOfInstances; i++) + { + string inputFile = batch.InputFiles[i]; + string arguments = + $"run {inputFile} \"{batch.StartAt}\" {batch.DurationInMinutes} \"\" \"{config.ConnectionString}\""; + + if (isDll) + arguments = $"{appFileName} {arguments}"; + + WriteMessage(logPath, $"{arguments}"); + + processes.Add(new Process + { + StartInfo = new ProcessStartInfo + { + FileName = isDll ? "dotnet" : appFileName, + Arguments = arguments, + UseShellExecute = true, + RedirectStandardOutput = false, + RedirectStandardError = false, + CreateNoWindow = false, + WorkingDirectory = appPath + } + }); + } + + processes.ForEach(p => p.Start()); + int running = processes.Count; + WriteMessage(logPath, $"{GetTimestamp()} - Started {running} replay instances"); + + do + { + Thread.Sleep(config.StatusCheckIntervalInSec * 1000); + processes.ForEach(p => + { + if (p.HasExited) + running--; + }); + } while (running > 0); + + WriteMessage(logPath, $"{GetTimestamp()} - Finished batch {batch.ProcessingOrder} of {config.Batches.Length}. Batch ID: {batch.BatchId}"); + } + + WriteMessage(logPath, $"{Environment.NewLine}{GetTimestamp()} - Processing completed"); + Environment.Exit(0); + } + + static BatchConfig ValidateAndGetConfig(string batchConfigPath, out List validationResults) + { + if(string.IsNullOrWhiteSpace(batchConfigPath)) + throw new ArgumentNullException(nameof(batchConfigPath)); + + validationResults = new List(); + + if (!File.Exists(batchConfigPath)) + { + validationResults.Add($"Batch configuration file {batchConfigPath} not found"); + return null; + } + + string configJson = File.ReadAllText(batchConfigPath); + BatchConfig config; + try + { + config = JsonConvert.DeserializeObject(configJson); + } + catch (Exception e) + { + validationResults.Add($"Error deserializing {batchConfigPath}"); + validationResults.Add(e.ToString()); + return null; + } + + if (config.Batches == null || config.Batches.Length < 1) + validationResults.Add("No batch definitions found"); + + if (string.IsNullOrWhiteSpace(config.ConnectionString)) + validationResults.Add("connectionString is required"); + + if (string.IsNullOrWhiteSpace(config.OutputPath)) + validationResults.Add("outputPath is required"); + + if (string.IsNullOrWhiteSpace(config.ReplayConsolePath)) + validationResults.Add("replayConsolePath is required"); + + if (!File.Exists(config.ReplayConsolePath)) + validationResults.Add($"Replay console {config.ReplayConsolePath} not found"); + + if (!Directory.Exists(config.OutputPath)) + validationResults.Add($"Replay output path {config.OutputPath} not found"); + + if (validationResults.Count > 0) + return null; + + if (config.StatusCheckIntervalInSec <= 0) + config.StatusCheckIntervalInSec = DefaultStatusCheckIntervalInSec; + + // ReSharper disable once AssignNullToNotNullAttribute + config.Batches = config.Batches.OrderBy(b => b.ProcessingOrder).ToArray(); + + var orderDuplicates = config.Batches.GroupBy(x => x.ProcessingOrder) + .Where(y => y.Count() > 1) + .Select(z => z.Key) + .ToArray(); + + if (orderDuplicates.Any()) + validationResults.Add("processingOrder duplicates found in batch definitions"); + + var batchIdDuplicates = config.Batches.GroupBy(x => x.BatchId) + .Where(y => y.Count() > 1) + .Select(z => z.Key) + .ToArray(); + + if (batchIdDuplicates.Any()) + validationResults.Add("batchId duplicates found in batch definitions"); + + if (validationResults.Count > 0) + return null; + + int i = 0; + Regex regex = new Regex(@"replay[\d]+.txt"); + + // ReSharper disable once PossibleNullReferenceException + foreach (Batch batch in config.Batches) + { + int numOfInstances = batch.NumOfInstances; + string batchId = batch.BatchId; + string startAt = batch.StartAt; + int processingOrder = batch.ProcessingOrder; + i++; + + if (numOfInstances < 1 || processingOrder < 1 || string.IsNullOrWhiteSpace(batchId) || string.IsNullOrWhiteSpace(startAt)) + { + validationResults.Add($"Batch #{i}: incomplete or invalid batch definition"); + continue; + } + + string batchPath = Path.Combine(config.OutputPath, batchId); + if (!Directory.Exists(batchPath)) + { + validationResults.Add($"Batch #{i}: directory {batchPath} not found"); + continue; + } + + string[] files = Directory.EnumerateFiles(batchPath, "replay*.txt").ToArray(); + if (!files.Any()) + { + validationResults.Add($"Batch #{i}: Directory {batchPath} is empty"); + continue; + } + + var inputFiles = files.Where(file => regex.IsMatch(file)).ToArray(); + if (inputFiles.Length != batch.NumOfInstances) + { + validationResults.Add($"Batch #{i}: Number of replay*.txt input files should match numOfInstances defined for the batch"); + continue; + } + + batch.InputFiles = inputFiles; + if (batch.DurationInMinutes < 0) + batch.DurationInMinutes = DefaultReplayDurationInMinutes; + } + return validationResults.Count > 0 ? null : config; + } + + static string GetTimestamp() + { + return string.Format($"{DateTime.UtcNow:MM-dd-yyyy--HH-mm-ss}"); + } + + static void WriteMessage(string logFilePath, string msg) + { + if (string.IsNullOrWhiteSpace(msg)) + throw new ArgumentNullException(nameof(msg)); + + Console.WriteLine(msg); + + if (!string.IsNullOrWhiteSpace(logFilePath)) + File.AppendAllText(logFilePath, $"{msg}{Environment.NewLine}"); + } + + static void ShowUsage() + { + Console.WriteLine("Batch configuration file is required."); + Console.WriteLine("Usage: SqlReplay.Batch.exe [log directory path]"); + Console.WriteLine(@"Example: SqlReplay.Batch.exe C:\replay\config\SampleBatchConfig.json"); + Console.WriteLine(@"Example: SqlReplay.Batch.exe C:\replay\config\SampleBatchConfig.json C:\replay\output"); + } + } +} diff --git a/SqlReplay.Batch/Properties/launchSettings.json b/SqlReplay.Batch/Properties/launchSettings.json new file mode 100644 index 0000000..c73b7ce --- /dev/null +++ b/SqlReplay.Batch/Properties/launchSettings.json @@ -0,0 +1,7 @@ +{ + "profiles": { + "SqlReplay.Batch": { + "commandName": "Project" + } + } +} \ No newline at end of file diff --git a/SqlReplay.Batch/SampleBatchConfig.json b/SqlReplay.Batch/SampleBatchConfig.json new file mode 100644 index 0000000..a34fd99 --- /dev/null +++ b/SqlReplay.Batch/SampleBatchConfig.json @@ -0,0 +1,22 @@ +{ + "replayConsolePath": "C:\\tmp\\replay\\SqlReplay.Console.dll", + "connectionString": "DB connection string", + "outputPath": "D:\\tmp\\replay\\output", + "statusCheckIntervalInSec": 15, + "batches": [ + { + "processingOrder": 1, + "batchId": "XECapture_1", + "numOfInstances": 10, + "startAt": "10/07/2020 02:53:07 PM -04:00", + "durationInMinutes": 0 + }, + { + "processingOrder": 2, + "batchId": "XECapture_2", + "numOfInstances": 10, + "startAt": "10/07/2020 10:31:33 PM -04:00", + "durationInMinutes": 0 + } + ] +} \ No newline at end of file diff --git a/SqlReplay.Batch/SqlReplay.Batch.csproj b/SqlReplay.Batch/SqlReplay.Batch.csproj new file mode 100644 index 0000000..06e8825 --- /dev/null +++ b/SqlReplay.Batch/SqlReplay.Batch.csproj @@ -0,0 +1,14 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + diff --git a/SqlReplay.Utilities.sln b/SqlReplay.Utilities.sln new file mode 100644 index 0000000..b99099e --- /dev/null +++ b/SqlReplay.Utilities.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30204.135 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlReplay.Batch", "SqlReplay.Batch\SqlReplay.Batch.csproj", "{18936929-E0ED-4258-93F1-70E7A8E72C7F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {18936929-E0ED-4258-93F1-70E7A8E72C7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {18936929-E0ED-4258-93F1-70E7A8E72C7F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {18936929-E0ED-4258-93F1-70E7A8E72C7F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {18936929-E0ED-4258-93F1-70E7A8E72C7F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {25DED7B2-DA93-4221-A2F4-135D92F74788} + EndGlobalSection +EndGlobal