feat(export): improve xlsx export speed

- reduce NPOI API calls, it's time consuming
- limit the number of rows/columns in matrix sheets
- retrieve demos data from JSON files in commands if they are available

CLI/DownloadCommand.cs

@@ -183,7 +183,7 @@ private async Task HandleBoilerExitCode(int exitCode)
                     Console.WriteLine(@"Invalid arguments");
                     break;
                 case (int)BoilerExitCode.CommunicationFailure:
-                    Console.WriteLine(@"CSGO Game coordinator communication failure"); ;
+                    Console.WriteLine(@"CSGO Game coordinator communication failure");
                     break;
                 case (int)BoilerExitCode.AlreadyConnectedToGC:
                     Console.WriteLine(@"Already connected to CSGO GC, make sure to close CSGO and retry");

CLI/JsonCommand.cs

@@ -55,20 +55,46 @@ public override async Task Run(string[] args)
 
             try
             {
+                CacheService cacheService = new CacheService();
+                int currentDemoNumber = 0;
                 foreach (string demoPath in _demoPaths)
                 {
-                    Console.WriteLine($@"Analyzing demo {demoPath}");
+                    Console.WriteLine($@"Retrieving demo {++currentDemoNumber}/{_demoPaths.Count} {demoPath}");
                     Demo demo = DemoAnalyzer.ParseDemoHeader(demoPath);
+                    if (demo == null)
+                    {
+                        Console.WriteLine($@"Invalid demo {demoPath}");
+                        continue;
+                    }
+
                     if (_source != null)
                     {
                         demo.Source = _source;
                     }
 
-                    DemoAnalyzer analyzer = DemoAnalyzer.Factory(demo);
-                    demo = await analyzer.AnalyzeDemoAsync(new CancellationTokenSource().Token);
-                    Console.WriteLine(@"Generating JSON file");
+                    if (cacheService.HasDemoInCache(demo.Id))
+                    {
+                        demo = await cacheService.GetDemoDataFromCache(demo.Id);
+                        demo.WeaponFired = await cacheService.GetDemoWeaponFiredAsync(demo);
+                        demo.PlayerBlinded = await cacheService.GetDemoPlayerBlindedAsync(demo);
+                    }
+                    else
+                    {
+                        try
+                        {
+                            Console.WriteLine($@"Analyzing demo {demoPath}");
+                            DemoAnalyzer analyzer = DemoAnalyzer.Factory(demo);
+                            demo = await analyzer.AnalyzeDemoAsync(new CancellationTokenSource().Token);
+                            await cacheService.WriteDemoDataCache(demo);
+                        }
+                        catch (Exception)
+                        {
+                            Console.WriteLine($@"Error while analyzing demo {demoPath}");
+                            continue;
+                        }
+                    }
+
                     string outputFolderPath = BuildOutputFolderPathFromDemoPath(demoPath);
-                    CacheService cacheService = new CacheService();
                     string jsonFilePath = await cacheService.GenerateJsonAsync(demo, outputFolderPath);
                     Console.WriteLine($@"JSON file generated at {jsonFilePath}");
                 }

CLI/XlsxCommand.cs

@@ -4,6 +4,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Core.Models;
+using Services.Concrete;
 using Services.Concrete.Analyzer;
 using Services.Concrete.Excel;
 
@@ -67,19 +68,46 @@ public override async Task Run(string[] args)
             try
             {
                 ExcelService excelService = new ExcelService();
+                CacheService cacheService = new CacheService();
                 List<Demo> demos = new List<Demo>();
 
+                int currentDemoNumber = 0;
                 foreach (string demoPath in _demoPaths)
                 {
-                    Console.WriteLine($@"Analyzing demo {demoPath}");
+                    Console.WriteLine($@"Retrieving demo {++currentDemoNumber}/{_demoPaths.Count} {demoPath}");
                     Demo demo = DemoAnalyzer.ParseDemoHeader(demoPath);
+                    if (demo == null)
+                    {
+                        Console.WriteLine($@"Invalid demo {demoPath}");
+                        continue;
+                    }
+
                     if (_source != null)
                     {
                         demo.Source = _source;
                     }
 
-                    DemoAnalyzer analyzer = DemoAnalyzer.Factory(demo);
-                    demo = await analyzer.AnalyzeDemoAsync(new CancellationTokenSource().Token);
+                    if (cacheService.HasDemoInCache(demo.Id))
+                    {
+                        demo = await cacheService.GetDemoDataFromCache(demo.Id);
+                        demo.WeaponFired = await cacheService.GetDemoWeaponFiredAsync(demo);
+                        demo.PlayerBlinded = await cacheService.GetDemoPlayerBlindedAsync(demo);
+                    }
+                    else
+                    {
+                        try
+                        {
+                            Console.WriteLine($@"Analyzing demo {demoPath}");
+                            DemoAnalyzer analyzer = DemoAnalyzer.Factory(demo);
+                            demo = await analyzer.AnalyzeDemoAsync(new CancellationTokenSource().Token);
+                            await cacheService.WriteDemoDataCache(demo);
+                        }
+                        catch (Exception)
+                        {
+                            Console.WriteLine($@"Error while analyzing demo {demoPath}");
+                            continue;
+                        }
+                    }
 
                     if (_exportIntoSingleFile)
                     {

Core/Models/Events/KillEvent.cs

@@ -74,6 +74,12 @@ public class KillEvent : BaseEvent
 
         [JsonIgnore] public override string Message => KillerName + " killed " + KilledName + " with " + Weapon.Name;
 
+        [JsonIgnore]
+        public bool IsKillerBot => KillerSteamId == 0;
+
+        [JsonIgnore]
+        public bool IsVictimBot => KilledSteamId == 0;
+
         public KillEvent(int tick, float seconds) : base(tick, seconds)
         {
         }

Core/Models/Events/PlayerBlindedEvent.cs

@@ -25,6 +25,12 @@ public class PlayerBlindedEvent : BaseEvent
 
         [JsonProperty("duration")] public float Duration { get; set; }
 
+        [JsonIgnore]
+        public bool IsThrowerBot => ThrowerSteamId == 0;
+
+        [JsonIgnore]
+        public bool IsVictimBot => VictimSteamId == 0;
+
         public PlayerBlindedEvent(int tick, float seconds) : base(tick, seconds)
         {
         }

Core/Models/Player.cs

@@ -1041,7 +1041,11 @@ public bool HasEntryHoldKill
             set { Set(() => HasEntryHoldKill, ref _hasEntryHoldKill, value); }
         }
 
-        [JsonIgnore] public int MatchCount { get; set; } = 1;
+        [JsonIgnore]
+        public int MatchCount { get; set; } = 1;
+
+        [JsonIgnore]
+        public bool IsBot => SteamId == 0;
 
         #endregion
 
@@ -1383,7 +1387,7 @@ private float ComputeKast(Demo demo)
             }
 
             int eventCount = 0;
-            foreach(Round round in demo.Rounds)
+            foreach (Round round in demo.Rounds)
             {
                 bool hasKill = Kills.Any(kill => kill.RoundNumber == round.Number);
                 bool hasAssist = Assists.Any(assist => assist.RoundNumber == round.Number);

Manager/ViewModel/Demos/DemoListViewModel.cs

@@ -570,6 +570,11 @@ public RelayCommand<ObservableCollection<Demo>> ExportExcelCommand
                                                        {
                                                            demo.WeaponFired = await _cacheService.GetDemoWeaponFiredAsync(demo);
                                                        }
+
+                                                       if (demo.PlayerBlinded.Count == 0)
+                                                       {
+                                                           demo.PlayerBlinded = await _cacheService.GetDemoPlayerBlindedAsync(demo);
+                                                       }
                                                    }
 
                                                    await _excelService.GenerateXls(demos.ToList(), saveExportFileDialog.FileName,
@@ -622,6 +627,11 @@ await _dialogService.ShowErrorAsync(Properties.Resources.DialogErrorWhileExporti
                                                                demo.WeaponFired = await _cacheService.GetDemoWeaponFiredAsync(demo);
                                                            }
 
+                                                           if (demo.PlayerBlinded.Count == 0)
+                                                           {
+                                                               demo.PlayerBlinded = await _cacheService.GetDemoPlayerBlindedAsync(demo);
+                                                           }
+
                                                            string exportFilePath =
                                                                $"{directoryPath}{Path.DirectorySeparatorChar}{demo.Name.Substring(0, demo.Name.Length - 4)}-export.xlsx";
                                                            await _excelService.GenerateXls(demo, exportFilePath);

Services/Concrete/Excel/MultipleExport.cs

@@ -40,12 +40,6 @@ public MultipleExport(List<Demo> demos, long selectedStatsAccountSteamId = 0)
 
         public override async Task<IWorkbook> Generate()
         {
-            CacheService cacheService = new CacheService();
-            foreach (Demo demo in _demos)
-            {
-                demo.WeaponFired = await cacheService.GetDemoWeaponFiredAsync(demo);
-            }
-
             _generalSheet = new GeneralSheet(Workbook, _demos);
             await _generalSheet.Generate();
             _playersSheet = new PlayersSheet(Workbook, _demos);

Services/Concrete/Excel/Sheets/Multiple/FlashMatrixPlayersSheet.cs

@@ -3,84 +3,140 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Core.Models;
+using Core.Models.Events;
 using NPOI.SS.UserModel;
 
 namespace Services.Concrete.Excel.Sheets.Multiple
 {
+    internal class PlayerFlashEntry
+    {
+        public long SteamId { get; set; }
+        public string Name { get; set; }
+
+        public Dictionary<long, float> Durations { get; set; }
+    }
+
     public class FlashMatrixPlayersSheet : AbstractMultipleSheet
     {
+        private readonly List<PlayerFlashEntry> _playerEntries = new List<PlayerFlashEntry>();
+        private Dictionary<long, string> _playerNamePerSteamId = new Dictionary<long, string>();
+
         public FlashMatrixPlayersSheet(IWorkbook workbook, List<Demo> demos)
         {
             Headers = new Dictionary<string, CellType> { { string.Empty, CellType.String } };
             Demos = demos;
             Sheet = workbook.CreateSheet("Flash matrix players");
         }
 
-        public override async Task GenerateContent()
+        public override Task GenerateContent()
         {
-            // store players row and columns index
-            Dictionary<long, int> playersRow = new Dictionary<long, int>();
-            Dictionary<long, int> playersColumn = new Dictionary<long, int>();
-
-            // first row containing victims name
-            IRow firstRow = Sheet.CreateRow(0);
-            SetCellValue(firstRow, 0, CellType.String, "Flasher\\Flashed");
+            ComputePlayerNamePerSteamId();
+            InitializePlayerEntries();
+            PopulatePlayerEntries();
+            GenerateSheet();
 
-            int columnCount = 1;
-            int rowCount = 1;
+            return Task.CompletedTask;
+        }
 
-            // current row
-            IRow row;
+        private void ComputePlayerNamePerSteamId()
+        {
             foreach (Demo demo in Demos)
             {
-                CacheService cacheService = new CacheService();
-                demo.PlayerBlinded = await cacheService.GetDemoPlayerBlindedAsync(demo);
-
-                // create rows and columns with only players name
                 foreach (Player player in demo.Players)
                 {
-                    if (!playersRow.ContainsKey(player.SteamId))
+                    if (player.IsBot || _playerNamePerSteamId.ContainsKey(player.SteamId))
+                    {
+                        continue;
+                    }
+
+                    _playerNamePerSteamId.Add(player.SteamId, player.Name);
+
+                    if (IsMaxPlayerLimitReached())
                     {
-                        // add a column for this player in the first row
-                        SetCellValue(firstRow, columnCount, CellType.String, player.Name);
-                        playersRow.Add(player.SteamId, rowCount);
-                        // create a row for this player
-                        row = Sheet.CreateRow(rowCount++);
-                        SetCellValue(row, 0, CellType.String, player.Name);
-                        playersColumn.Add(player.SteamId, columnCount++);
+                        break;
                     }
                 }
 
-                foreach (Player player in demo.Players)
+                if (IsMaxPlayerLimitReached())
+                {
+                    break;
+                }
+            }
+
+            _playerNamePerSteamId = _playerNamePerSteamId.OrderBy(k => k.Value).ToDictionary(x => x.Key, x => x.Value);
+        }
+
+        private void InitializePlayerEntries()
+        {
+            foreach (var playerName in _playerNamePerSteamId)
+            {
+                var durations = new Dictionary<long, float>();
+                foreach (var sortedPlayerName in _playerNamePerSteamId)
                 {
-                    if (playersRow.ContainsKey(player.SteamId))
+                    durations.Add(sortedPlayerName.Key, 0);
+                }
+
+                _playerEntries.Add(new PlayerFlashEntry()
+                {
+                    SteamId = playerName.Key,
+                    Name = playerName.Value,
+                    Durations = durations,
+                });
+            }
+        }
+
+        private void PopulatePlayerEntries()
+        {
+            foreach (Demo demo in Demos)
+            {
+                foreach (PlayerBlindedEvent blindEvent in demo.PlayerBlinded)
+                {
+                    if (blindEvent.IsThrowerBot || blindEvent.IsVictimBot)
+                    {
+                        continue;
+                    }
+
+                    var throwerEntry = _playerEntries.Find(entry => entry.SteamId == blindEvent.ThrowerSteamId);
+                    if (throwerEntry == null)
                     {
-                        int rowIndex = playersRow[player.SteamId];
-                        foreach (Player pl in demo.Players)
-                        {
-                            if (playersColumn.ContainsKey(pl.SteamId))
-                            {
-                                int columnIndex = playersColumn[pl.SteamId];
-                                float duration = demo.PlayerBlinded.Where(e => e.ThrowerSteamId == player.SteamId && e.VictimSteamId == pl.SteamId)
-                                    .Sum(e => e.Duration);
-                                row = Sheet.GetRow(rowIndex);
-                                ICell cell = row.GetCell(columnIndex);
-                                if (cell == null)
-                                {
-                                    SetCellValue(row, columnIndex, CellType.Numeric, Math.Round(duration, 2));
-                                }
-                                else
-                                {
-                                    cell.SetCellValue(cell.NumericCellValue + Math.Round(duration, 2));
-                                }
-                            }
-                        }
+                        continue;
                     }
+
+                    var victimEntry = _playerEntries.Find(entry => entry.SteamId == blindEvent.VictimSteamId);
+                    if (victimEntry == null)
+                    {
+                        continue;
+                    }
+
+                    throwerEntry.Durations[victimEntry.SteamId] += blindEvent.Duration;
                 }
+            }
+        }
+
+        private void GenerateSheet()
+        {
+            IRow firstRow = Sheet.CreateRow(0);
+            SetCellValue(firstRow, 0, CellType.String, "Flasher\\Flashed");
+
+            for (int playerIndex = 0; playerIndex < _playerEntries.Count; playerIndex++)
+            {
+                var playerEntry = _playerEntries[playerIndex];
+                SetCellValue(firstRow, playerIndex + 1, CellType.String, playerEntry.Name);
 
-                FillEmptyCells(rowCount, columnCount);
-                demo.PlayerBlinded.Clear();
+                IRow row = Sheet.CreateRow(playerIndex + 1);
+                SetCellValue(row, 0, CellType.String, playerEntry.Name);
+
+                for (int durationIndex = 0; durationIndex < playerEntry.Durations.Count; durationIndex++)
+                {
+                    var duration = playerEntry.Durations.ElementAt(durationIndex);
+                    SetCellValue(row, durationIndex + 1, CellType.Numeric, Math.Round(duration.Value, 2));
+                }
             }
         }
+
+        private bool IsMaxPlayerLimitReached()
+        {
+            return _playerNamePerSteamId.Count >= 256;
+        }
     }
 }