From 57c9d2d55ff7d78ede64a11e1e760055deba2f35 Mon Sep 17 00:00:00 2001 From: Mark Crossley Date: Thu, 1 Jun 2023 00:45:53 +0100 Subject: [PATCH 1/2] Log the SQL string built so far on each error --- Program.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Program.cs b/Program.cs index 5321bea..8032737 100644 --- a/Program.cs +++ b/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data.SqlTypes; using System.Globalization; using System.IO; using System.Text; @@ -203,6 +204,7 @@ private static void DoSingleMonthlyExport(string filename) catch (Exception ex) { Console.WriteLine(ex.Message); + Console.WriteLine("SQL = " + sb.ToString() + "\n"); } } while (!(sr.EndOfStream)); } @@ -247,9 +249,11 @@ private static void DoDayfileExport() do { - // now process each record in the file - try - { + // now process each record in the file + StringBuilder sb = new StringBuilder(StartOfDayfileInsertSQL + " Values("); + + try + { var line = sr.ReadLine(); linenum++; var st = new List(Regex.Split(line, CultureInfo.CurrentCulture.TextInfo.ListSeparator)); @@ -262,7 +266,7 @@ private static void DoDayfileExport() Console.Write(sqldate + "\r"); - StringBuilder sb = new StringBuilder(StartOfDayfileInsertSQL + " Values('" + sqldate + "',"); + sb.Append($"'{sqldate}',"); for (int i = 1; i < 55; i++) { @@ -295,6 +299,7 @@ private static void DoDayfileExport() catch (Exception ex) { Console.WriteLine(ex.Message); + Console.WriteLine("SQL = " + sb.ToString() + "\n"); } } while (!(sr.EndOfStream)); } From 3825f7be514bf05a0f171273567f9ef5793345c4 Mon Sep 17 00:00:00 2001 From: Mark Crossley Date: Mon, 7 Aug 2023 14:37:46 +0100 Subject: [PATCH 2/2] Additional error logging/checking --- ExportToMySQL.csproj | 5 +- Program.cs | 96 ++++++++++++++++++++++---------------- Properties/AssemblyInfo.cs | 4 +- 3 files changed, 62 insertions(+), 43 deletions(-) diff --git a/ExportToMySQL.csproj b/ExportToMySQL.csproj index c4491d3..33f3544 100644 --- a/ExportToMySQL.csproj +++ b/ExportToMySQL.csproj @@ -48,6 +48,9 @@ 4 false + + + @@ -76,7 +79,7 @@ - 2.2.5 + 2.2.7 7.0.2 diff --git a/Program.cs b/Program.cs index 8032737..1ae8b25 100644 --- a/Program.cs +++ b/Program.cs @@ -27,7 +27,7 @@ private static void Main(string[] args) if (args.Length == 0) { Console.WriteLine("Specify 'dayfile', 'monthly', or the path to a monthly log file"); - Environment.Exit(0); + Environment.Exit(1); } else { @@ -38,7 +38,7 @@ private static void Main(string[] args) if (!File.Exists("Cumulus.ini")) { Console.WriteLine("Cannot find Cumulus.ini"); - Environment.Exit(0); + Environment.Exit(1); } IniFile ini = new IniFile("Cumulus.ini"); @@ -55,27 +55,6 @@ private static void Main(string[] args) Database = ini.GetValue("MySQL", "Database", "database") }; - if (File.Exists("strings.ini")) - { - IniFile iniStrs = new IniFile("strings.ini"); - compassp[0] = iniStrs.GetValue("Compass", "N", "N"); - compassp[1] = iniStrs.GetValue("Compass", "NNE", "NNE"); - compassp[2] = iniStrs.GetValue("Compass", "NE", "NE"); - compassp[3] = iniStrs.GetValue("Compass", "ENE", "ENE"); - compassp[4] = iniStrs.GetValue("Compass", "E", "E"); - compassp[5] = iniStrs.GetValue("Compass", "ESE", "ESE"); - compassp[6] = iniStrs.GetValue("Compass", "SE", "SE"); - compassp[7] = iniStrs.GetValue("Compass", "SSE", "SSE"); - compassp[8] = iniStrs.GetValue("Compass", "S", "S"); - compassp[9] = iniStrs.GetValue("Compass", "SSW", "SSW"); - compassp[10] = iniStrs.GetValue("Compass", "SW", "SW"); - compassp[11] = iniStrs.GetValue("Compass", "WSW", "WSW"); - compassp[12] = iniStrs.GetValue("Compass", "W", "W"); - compassp[13] = iniStrs.GetValue("Compass", "WNW", "WNW"); - compassp[14] = iniStrs.GetValue("Compass", "NW", "NW"); - compassp[15] = iniStrs.GetValue("Compass", "NNW", "NNW"); - } - try { mySqlConn = new MySqlConnection(ConnString.ToString()); @@ -84,14 +63,9 @@ private static void Main(string[] args) { Console.WriteLine("Error encountered opening MySQL connection"); Console.WriteLine(ex.Message); - Environment.Exit(0); + Environment.Exit(1); } - cmd = new MySqlCommand - { - Connection = mySqlConn - }; - try { mySqlConn.Open(); @@ -100,9 +74,34 @@ private static void Main(string[] args) { Console.WriteLine("Error encountered opening MySQL connection"); Console.WriteLine(ex.Message); - Environment.Exit(0); + Environment.Exit(1); } + cmd = new MySqlCommand + { + Connection = mySqlConn + }; + + if (File.Exists("strings.ini")) + { + IniFile iniStrs = new IniFile("strings.ini"); + compassp[0] = iniStrs.GetValue("Compass", "N", "N"); + compassp[1] = iniStrs.GetValue("Compass", "NNE", "NNE"); + compassp[2] = iniStrs.GetValue("Compass", "NE", "NE"); + compassp[3] = iniStrs.GetValue("Compass", "ENE", "ENE"); + compassp[4] = iniStrs.GetValue("Compass", "E", "E"); + compassp[5] = iniStrs.GetValue("Compass", "ESE", "ESE"); + compassp[6] = iniStrs.GetValue("Compass", "SE", "SE"); + compassp[7] = iniStrs.GetValue("Compass", "SSE", "SSE"); + compassp[8] = iniStrs.GetValue("Compass", "S", "S"); + compassp[9] = iniStrs.GetValue("Compass", "SSW", "SSW"); + compassp[10] = iniStrs.GetValue("Compass", "SW", "SW"); + compassp[11] = iniStrs.GetValue("Compass", "WSW", "WSW"); + compassp[12] = iniStrs.GetValue("Compass", "W", "W"); + compassp[13] = iniStrs.GetValue("Compass", "WNW", "WNW"); + compassp[14] = iniStrs.GetValue("Compass", "NW", "NW"); + compassp[15] = iniStrs.GetValue("Compass", "NNW", "NNW"); + } if (param.ToLower().Equals("dayfile")) { @@ -131,6 +130,8 @@ private static void Main(string[] args) private static void DoSingleMonthlyExport(string filename) { + Console.WriteLine("Processing file:" + filename); + var StartOfMonthlyInsertSQL = "INSERT IGNORE INTO " + MySqlMonthlyTable + " (LogDateTime,Temp,Humidity,Dewpoint,Windspeed,Windgust,Windbearing,RainRate,TodayRainSoFar,Pressure,Raincounter,InsideTemp,InsideHumidity,LatestWindGust,WindChill,HeatIndex,UVindex,SolarRad,Evapotrans,AnnualEvapTran,ApparentTemp,MaxSolarRad,HrsSunShine,CurrWindBearing,RG11rain,RainSinceMidnight,FeelsLike,Humidex,WindbearingSym,CurrWindBearingSym)"; using (var sr = new StreamReader(filename)) @@ -138,21 +139,29 @@ private static void DoSingleMonthlyExport(string filename) const int MaxBatchSize = 1000; StringBuilder sb = new StringBuilder("", MaxBatchSize * 2100); - int linenum = 0; + var linenum = 0; + var line = string.Empty; + do { sb.Clear(); - sb.Append(StartOfMonthlyInsertSQL + " Values "); + sb.Append(StartOfMonthlyInsertSQL + " VALUES "); // now process each record in the file try { for (int a = 0; a < MaxBatchSize && !(sr.EndOfStream); a++) { - var line = sr.ReadLine(); + line = sr.ReadLine(); linenum++; var st = new List(Regex.Split(line, CultureInfo.CurrentCulture.TextInfo.ListSeparator)); + if (st.Count < 16) + { + Console.WriteLine($"Error: Line {linenum} is too short. Detected {st.Count} fields present, but 16 is the minimum"); + continue; + } + var logfiledate = st[0]; // 01234567 // dd/mm/yy @@ -203,10 +212,12 @@ private static void DoSingleMonthlyExport(string filename) } catch (Exception ex) { + Console.WriteLine("Error processing line " + linenum); Console.WriteLine(ex.Message); - Console.WriteLine("SQL = " + sb.ToString() + "\n"); - } - } while (!(sr.EndOfStream)); + Console.WriteLine("SQL = " + sb.ToString()); + Console.WriteLine("Src = " + line + "\n"); + } + } while (!(sr.EndOfStream)); } } @@ -241,7 +252,8 @@ private static void DoDayfileExport() Console.WriteLine("Dayfile exists, beginning export"); string StartOfDayfileInsertSQL = "INSERT IGNORE INTO " + MySqlDayfileTable + " (LogDate,HighWindGust,HWindGBear,THWindG,MinTemp,TMinTemp,MaxTemp,TMaxTemp,MinPress,TMinPress,MaxPress,TMaxPress,MaxRainRate,TMaxRR,TotRainFall,AvgTemp,TotWindRun,HighAvgWSpeed,THAvgWSpeed,LowHum,TLowHum,HighHum,THighHum,TotalEvap,HoursSun,HighHeatInd,THighHeatInd,HighAppTemp,THighAppTemp,LowAppTemp,TLowAppTemp,HighHourRain,THighHourRain,LowWindChill,TLowWindChill,HighDewPoint,THighDewPoint,LowDewPoint,TLowDewPoint,DomWindDir,HeatDegDays,CoolDegDays,HighSolarRad,THighSolarRad,HighUV,THighUV,MaxFeelsLike,TMaxFeelsLike,MinFeelsLike,TMinFeelsLike,MaxHumidex,TMaxHumidex,ChillHours,HighRain24h,THighRain24h,HWindGBearSym,DomWindDirSym)"; - int linenum = 0; + var linenum = 0; + var line = string.Empty; using (var sr = new StreamReader(filename)) { @@ -251,10 +263,9 @@ private static void DoDayfileExport() { // now process each record in the file StringBuilder sb = new StringBuilder(StartOfDayfileInsertSQL + " Values("); - try { - var line = sr.ReadLine(); + line = sr.ReadLine(); linenum++; var st = new List(Regex.Split(line, CultureInfo.CurrentCulture.TextInfo.ListSeparator)); @@ -299,7 +310,8 @@ private static void DoDayfileExport() catch (Exception ex) { Console.WriteLine(ex.Message); - Console.WriteLine("SQL = " + sb.ToString() + "\n"); + Console.WriteLine("SQL = " + sb.ToString()); + Console.WriteLine("Src = " + line + "\n"); } } while (!(sr.EndOfStream)); } @@ -307,6 +319,10 @@ private static void DoDayfileExport() Console.WriteLine(); Console.WriteLine(linenum+" entries processed"); } + else + { + Console.WriteLine("Dafile not found - " + filename); + } } private static string CompassPoint(int bearing) diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 13ee113..4ea4835 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.8.1.0")] -[assembly: AssemblyFileVersion("1.8.1.0")] +[assembly: AssemblyVersion("1.9.1.0")] +[assembly: AssemblyFileVersion("1.9.1.0")]