refactor: decrease ram usage during xlsx exports

ref akiver#199
ConnySjoblom · Jul 15, 2022 · 46f01b7 · 46f01b7
1 parent 753a12f
commit 46f01b7
Show file tree

Hide file tree

Showing 66 changed files with 1,253 additions and 1,336 deletions.
diff --git a/CLI/XlsxCommand.cs b/CLI/XlsxCommand.cs
@@ -1,22 +1,20 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Threading;
 using System.Threading.Tasks;
-using Core.Models;
-using Services.Concrete;
-using Services.Concrete.Analyzer;
 using Services.Concrete.Excel;
+using Services.Exceptions.Export;
 
 namespace CLI
 {
     internal class XlsxCommand : ExportCommand
     {
         public const string COMMAND_NAME = "xlsx";
         private bool _exportIntoSingleFile = false;
+        private readonly ExcelService _excelService;
 
         public XlsxCommand() : base(COMMAND_NAME, @"Export demos into XLSX files")
         {
+            _excelService = new ExcelService();
         }
 
         public override void PrintHelp()
@@ -68,99 +66,95 @@ public override async Task Run(string[] args)
 
             try
             {
-                ExcelService excelService = new ExcelService();
-                CacheService cacheService = new CacheService();
-                List<Demo> demos = new List<Demo>();
-
-                int currentDemoNumber = 0;
-                foreach (string demoPath in _demoPaths)
+                if (_exportIntoSingleFile)
                 {
-                    Console.WriteLine($@"Retrieving demo {++currentDemoNumber}/{_demoPaths.Count} {demoPath}");
-                    Demo demo = DemoAnalyzer.ParseDemoHeader(demoPath);
-                    if (demo == null)
-                    {
-                        Console.WriteLine($@"Invalid demo {demoPath}");
-                        continue;
-                    }
+                    await ProcessSingleFileExport();
+                }
+                else
+                {
+                    await ProcessMultipleFilesExport();
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($@"Error while exporting demo: {ex.Message}");
+                Environment.Exit(1);
+            }
+        }
 
-                    if (_source != null)
-                    {
-                        demo.Source = _source;
-                    }
+        private async Task ProcessSingleFileExport()
+        {
+            if (string.IsNullOrEmpty(_outputFolderPath))
+            {
+                if (IsCurrentDirectoryWritable())
+                {
+                    _outputFolderPath = Directory.GetCurrentDirectory();
+                }
+                else
+                {
+                    _outputFolderPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
+                }
+            }
+            string fileName = _outputFolderPath + Path.DirectorySeparatorChar + "export-" + DateTime.Now.ToString("yy-MM-dd-hh-mm-ss") + ".xlsx";
+            MultiExportConfiguration configuration = new MultiExportConfiguration
+            {
+                FileName = fileName,
+                DemoPaths = _demoPaths,
+                FocusSteamId = 0, // TODO --steamid parameter
+                Source = _source,
+                OnProcessingDemo = (demoPath, demoNumber, totalDemoCount) => Console.WriteLine($@"Retrieving demo {demoNumber}/{totalDemoCount} {demoPath}"),
+                OnDemoNotFound = demoPath => Console.WriteLine($@"The demo doesn't exists: {demoPath}"),
+                OnInvalidDemo = demoPath => Console.WriteLine($@"Invalid demo {demoPath}"),
+                OnAnalyzeStart = demoPath => Console.WriteLine($@"Analyzing demo {demoPath}"),
+                OnAnalyzeError = demoPath => Console.WriteLine($@"Error while analyzing demo {demoPath}"),
+                OnGeneratingXlsxFile = () => Console.WriteLine(@"Generating XLSX file..."),
 
-                    if (!_forceAnalyze && cacheService.HasDemoInCache(demo.Id))
-                    {
-                        demo = await cacheService.GetDemoDataFromCache(demo.Id);
-                        demo.WeaponFired = await cacheService.GetDemoWeaponFiredAsync(demo);
-                        demo.PlayerBlinded = await cacheService.GetDemoPlayerBlindedAsync(demo);
-                    }
-                    else
-                    {
-                        try
-                        {
-                            Console.WriteLine($@"Analyzing demo {demoPath}");
-                            DemoAnalyzer analyzer = DemoAnalyzer.Factory(demo);
-                            demo = await analyzer.AnalyzeDemoAsync(new CancellationTokenSource().Token);
-                            await cacheService.WriteDemoDataCache(demo);
-                        }
-                        catch (Exception)
-                        {
-                            Console.WriteLine($@"Error while analyzing demo {demoPath}");
-                            continue;
-                        }
-                    }
+            };
+            await _excelService.GenerateXls(configuration);
+            Console.WriteLine($@"XLSX file generated at {fileName}");
+        }
 
-                    if (_exportIntoSingleFile)
-                    {
-                        demos.Add(demo);
-                    }
-                    else
+        private async Task ProcessMultipleFilesExport()
+        {
+            int currentDemoNumber = 0;
+            foreach (string demoPath in _demoPaths)
+            {
+                try
+                {
+                    string outputFolderPath = BuildOutputFolderPathFromDemoPath(demoPath);
+                    string demoName = Path.GetFileName(demoPath);
+                    string fileName = outputFolderPath + Path.DirectorySeparatorChar + demoName + ".xlsx";
+                    SingleExportConfiguration configuration = new SingleExportConfiguration
                     {
-                        Console.WriteLine($@"Generating XLSX file");
-                        string outputFolderPath = BuildOutputFolderPathFromDemoPath(demoPath);
-                        string fileName = outputFolderPath + Path.DirectorySeparatorChar + demo.Name + ".xlsx";
-                        await excelService.GenerateXls(demo, fileName);
-                        Console.WriteLine($@"XLSX file generated at {fileName}");
-                    }
+                        DemoPath = demoPath,
+                        FileName = fileName,
+                        Source = _source,
+                        ForceAnalyze = _forceAnalyze,
+                        OnProcessingDemo = () => Console.WriteLine($@"Retrieving demo {++currentDemoNumber}/{_demoPaths.Count} {demoPath}"),
+                        OnAnalyzeStart = () => Console.WriteLine($@"Analyzing demo {demoPath}"),
+                    };
+                    await _excelService.GenerateXls(configuration);
+                    Console.WriteLine($@"XLSX file generated at {fileName}");
                 }
-
-                if (_exportIntoSingleFile)
+                catch (Exception ex)
                 {
-                    if (demos.Count == 0)
+                    switch (ex)
                     {
-                        Console.WriteLine(@"No demos to export");
-
-                        return;
-                    }
-
-                    if (string.IsNullOrEmpty(_outputFolderPath))
-                    {
-                        if (IsCurrentDirectoryWritable())
-                        {
-                            _outputFolderPath = Directory.GetCurrentDirectory();
-                        }
-                        else
-                        {
-                            _outputFolderPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
-                        }
+                        case FileNotFoundException _:
+                            Console.WriteLine($@"The demo doesn't exists: {demoPath}");
+                            break;
+                        case InvalidDemoException _:
+                            Console.WriteLine($@"Invalid demo {demoPath}");
+                            break;
+                        case AnalyzeException _:
+                            Console.WriteLine($@"Error while analyzing demo {demoPath}");
+                            break;
+                        default:
+                            Console.WriteLine($@"An error occurred while exporting demo {demoPath}");
+                            break;
                     }
-
-                    Console.WriteLine(@"Generating XLSX file");
-                    string fileName = _outputFolderPath + Path.DirectorySeparatorChar + "export-" + DateTime.Now.ToString("yy-MM-dd-hh-mm-ss") + ".xlsx";
-                    await excelService.GenerateXls(demos, fileName);
-                    Console.WriteLine($@"XLSX file generated at {fileName}");
                 }
             }
-            catch (FileNotFoundException ex)
-            {
-                Console.WriteLine($@"The demo doesn't exists: {ex.FileName}");
-                Environment.Exit(1);
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine($@"Error while exporting demo: {ex.Message}");
-                Environment.Exit(1);
-            }
         }
     }
 }
diff --git a/Manager/MultilingualResources/Manager.ar.xlf b/Manager/MultilingualResources/Manager.ar.xlf
@@ -2333,10 +2333,6 @@ Unable to start the game.</target>
           <source>Adding suspects...</source>
           <target state="new">Adding suspects...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="new">Analyzing demos for export...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="new">Analyzing demo(s) for json export...</target>
@@ -2583,10 +2579,6 @@ You can find more information on {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="new">Analyzing demo {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="new">Analyzing demo {0} for export...</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="new">POV</target>

diff --git a/Manager/MultilingualResources/Manager.da.xlf b/Manager/MultilingualResources/Manager.da.xlf
@@ -2334,10 +2334,6 @@ Er ikke istand til at starte spillet</target>
           <source>Adding suspects...</source>
           <target state="final">Tilføjer mistænkte</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="final">Analysere Demoer for eksportering.</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="final">Analysere demo(er) til JSON eksport</target>
@@ -2584,10 +2580,6 @@ du kan finde mere information på {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="final">Analysere demo {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="final">Analyser demo {0} til eksport...</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="final">POV</target>

diff --git a/Manager/MultilingualResources/Manager.de.xlf b/Manager/MultilingualResources/Manager.de.xlf
@@ -2333,10 +2333,6 @@ Das Spiel kann nicht gestartet werden.</target>
           <source>Adding suspects...</source>
           <target state="final">Füge Verdächtigen hinzu...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="final">Analysiere Demos für Exportvorgang...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="final">Analysiere Demos(s) für json Exportvorgang...</target>
@@ -2583,10 +2579,6 @@ Mehr Informationen finden Sie unter {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="final">Analysiere Demo {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="final">Analyisiere Demo {0} für den Exportvorgang.</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="final">POV</target>

diff --git a/Manager/MultilingualResources/Manager.es.xlf b/Manager/MultilingualResources/Manager.es.xlf
@@ -2347,10 +2347,6 @@ No se puede iniciar el juego.</target>
           <source>Adding suspects...</source>
           <target state="translated">Añadiendo sospechosos...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="translated">Analizando repeticiones para exportar...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="translated">Analizando repetición(es) para exportar json...</target>
@@ -2597,10 +2593,6 @@ Puedes encontrar más información en {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="translated">Analizando repetición {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="translated">Analizando repetición {0} para exportar...</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="translated">POV</target>

diff --git a/Manager/MultilingualResources/Manager.fr.xlf b/Manager/MultilingualResources/Manager.fr.xlf
@@ -2287,10 +2287,6 @@ Impossible de lancer le jeu.</target>
           <source>Analyzing...</source>
           <target state="final">Analyse en cours...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="final">Analyse des démos en cours pour l'export...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="final">Analyse des démos en cours pour l'export JSON...</target>
@@ -2553,10 +2549,6 @@ Si ce n'est pas le cas, merci de l'envoyer par email à l'adresse indiquée sur
           <source>Analyzing demo {0}...</source>
           <target state="final">Analyse de la démo {0} en cours...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="final">Analyse de la démo {0} pour l'export en cours...</target>
-        </trans-unit>
         <trans-unit id="DialogExportPlayer" translate="yes" xml:space="preserve">
           <source>You are going to export data for the player "{0}".
 Do you want to continue?</source>

diff --git a/Manager/MultilingualResources/Manager.hr.xlf b/Manager/MultilingualResources/Manager.hr.xlf
@@ -2333,10 +2333,6 @@ Unable to start the game.</target>
           <source>Adding suspects...</source>
           <target state="new">Adding suspects...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="new">Analyzing demos for export...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="new">Analyzing demo(s) for json export...</target>
@@ -2583,10 +2579,6 @@ You can find more information on {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="new">Analyzing demo {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="new">Analyzing demo {0} for export...</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="new">POV</target>

diff --git a/Manager/MultilingualResources/Manager.hu.xlf b/Manager/MultilingualResources/Manager.hu.xlf
@@ -2333,10 +2333,6 @@ A játékot nem lehet elindítani.</target>
           <source>Adding suspects...</source>
           <target state="final">Gyanúsított hozzáadása</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="final">Demók elemzése exportáláshoz...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="final">Demó(k) elemzése JSON exportáláshoz...</target>
@@ -2583,10 +2579,6 @@ Több információt találhat a {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="final">Demó elemzése {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="final">Demó elemzése {0} exportáláshoz.</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="final">POV</target>

diff --git a/Manager/MultilingualResources/Manager.it.xlf b/Manager/MultilingualResources/Manager.it.xlf
@@ -2333,10 +2333,6 @@ Unable to start the game.</target>
           <source>Adding suspects...</source>
           <target state="new">Adding suspects...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemosForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demos for export...</source>
-          <target state="new">Analyzing demos for export...</target>
-        </trans-unit>
         <trans-unit id="NotificationAnalyzingForJsonExport" translate="yes" xml:space="preserve">
           <source>Analyzing demo(s) for json export...</source>
           <target state="new">Analyzing demo(s) for json export...</target>
@@ -2583,10 +2579,6 @@ You can find more information on {1}.</target>
           <source>Analyzing demo {0}...</source>
           <target state="new">Analyzing demo {0}...</target>
         </trans-unit>
-        <trans-unit id="NotificationAnalyzingDemoForExport" translate="yes" xml:space="preserve">
-          <source>Analyzing demo {0} for export...</source>
-          <target state="new">Analyzing demo {0} for export...</target>
-        </trans-unit>
         <trans-unit id="Pov" translate="yes" xml:space="preserve">
           <source>POV</source>
           <target state="new">POV</target>