Merge branch 'task/RDMP-32-regex-redaction' of https://github.com/Hic…

…Services/RDMP into task/RDMP-230-pure-integration

CHANGELOG.md

@@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add Ordering to Filters
 - [MSSQL ONLY] Add ability to perform Regex redactions on data loads and existing catalogues
 
-## [8.3.1] - 2024-10-16
+## [8.3.1] - 2024-10-22
 
 - Improve Performance of regenerating problems with child providers
 - Update UI Tab opening Logic

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandPerformRegexRedactionOnCatalogue.cs

@@ -60,11 +60,14 @@ public override void Execute()
         {
             redactionsToSaveTable = RegexRedactionHelper.GenerateRedactionsDataTable();
             pksToSave = RegexRedactionHelper.GeneratePKDataTable();
+
             var columnName = columnInfo.Name;
             var table = columnInfo.TableInfo.Name;
+
             _discoveredTable = columnInfo.TableInfo.Discover(DataAccessContext.InternalDataProcessing);
             DiscoveredColumn[] discoveredColumns = _discoveredTable.DiscoverColumns();
             _discoveredPKColumns = discoveredColumns.Where(c => c.IsPrimaryKey).ToArray();
+
             if (_discoveredPKColumns.Length != 0)
             {
                 _cataloguePKs = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.IsPrimaryKey).ToList();
@@ -81,6 +84,7 @@ public override void Execute()
                     qb.TopX = (int)_readLimit;
                 }
                 var sql = qb.SQL;
+
                 var dt = new DataTable();
                 dt.BeginLoadData();
                 var conn = _server.GetConnection();
@@ -92,13 +96,19 @@ public override void Execute()
                     da.Fill(dt);
                 }
                 conn.Close();
+
                 redactionUpates = dt.Clone();
                 redactionUpates.BeginLoadData();
                 foreach (DataRow row in dt.Rows)
                 {
                     RegexRedactionHelper.Redact(columnInfo, row, _cataloguePKs, _redactionConfiguration, redactionsToSaveTable, pksToSave, redactionUpates);
                 }
                 redactionUpates.EndLoadData();
+                if(pksToSave.Rows.Count == 0)
+                {
+                    _activator.Show("Unable to find any matching Redactions");
+                    return;
+                }
                 for (int i = 0; i < pksToSave.Rows.Count; i++)
                 {
                     pksToSave.Rows[i][nameof(RegexRedactionHelper.Constants.ID)] = i + 1;
@@ -129,7 +139,6 @@ public override void Execute()
                 throw new Exception($"Unable to identify any primary keys in table '{table}'. Redactions cannot be performed on tables without primary keys");
             }
             resultCount += redactionUpates.Rows.Count;
-
         }
     }
 }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRestoreRegexRedactedValueInCatalogue.cs

@@ -69,7 +69,7 @@ public override void Execute()
                 var updateHelper = server.GetQuerySyntaxHelper().UpdateHelper;
                 var sqlLines = new List<CustomLine>
                 {
-                    new CustomLine($"t1.{columnInfo.GetRuntimeName()} = '{newValue}'", QueryComponent.SET)
+                    new($"t1.{columnInfo.GetRuntimeName()} = '{newValue}'", QueryComponent.SET)
                 };
                 foreach (var rk in _redaction.RedactionKeys)
                 {

Rdmp.Core/Curation/DataHelper/RegexRedaction/RegexRedactionHelper.cs

@@ -113,8 +113,6 @@ public static void Redact(ColumnInfo column, DataRow match, List<CatalogueItem>
 
         public static void SaveRedactions(ICatalogueRepository catalogueRepo, DiscoveredTable pksToSave, DiscoveredTable redactionsToSaveTable, DiscoveredServer _server, int timeout = 30000)
         {
-            //the update isn't working? and do we need the +1?
-
             var sql = $@"
                 DECLARE @output TABLE (id1 int, inc int IDENTITY(1,1))
                 INSERT INTO RegexRedaction(RedactionConfiguration_ID,ColumnInfo_ID,startingIndex,ReplacementValue,RedactedValue) OUTPUT inserted.id as id1 INTO @output

Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs

@@ -73,7 +73,7 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di
         {
             if (ColumnMatches(column))
             {
-                var pkSeparator = pkColumnInfos.Count() > 0 ? "," : "";
+                var pkSeparator = pkColumnInfos.Any() ? "," : "";
                 var sql = @$"
                     SELECT {column.GetRuntimeName()} {pkSeparator} {string.Join(", ", pkColumnInfos.Select(c => c.GetRuntimeName()))}
                     FROM {table.GetRuntimeName()}

Rdmp.Core/Logging/LogManager.cs

@@ -63,7 +63,8 @@ public string[] ListDataTasks(bool hideTests = false)
 
         using var con = Server.GetConnection();
         con.Open();
-        using var cmd = Server.GetCommand("SELECT * FROM DataLoadTask", con);
+        var sh = Server.GetQuerySyntaxHelper();
+        using var cmd = Server.GetCommand($"SELECT * FROM {sh.EnsureWrapped("DataLoadTask")}", con);
         using var r = cmd.ExecuteReader();
         while (r.Read())
             if (!hideTests || !(bool)r["isTest"])
@@ -113,7 +114,8 @@ public string[] ListDataSets()
         using var con = Server.GetConnection();
         con.Open();
 
-        using var cmd = Server.GetCommand("SELECT * FROM DataSet", con);
+        var sh = Server.GetQuerySyntaxHelper();
+        using var cmd = Server.GetCommand($"SELECT * FROM {sh.EnsureWrapped("DataSet")}", con);
         using var r = cmd.ExecuteReader();
         while (r.Read())
             tasks.Add(r["dataSetID"].ToString());
@@ -133,7 +135,8 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         int? specificDataLoadRunIDOnly = null, int? topX = null)
     {
         var db = Server.GetCurrentDatabase();
-        var run = db.ExpectTable("DataLoadRun");
+        var sh = Server.GetQuerySyntaxHelper();
+        var run = db.ExpectTable(sh.EnsureWrapped("DataLoadRun"));
 
         using var con = Server.GetConnection();
         con.Open();
@@ -144,11 +147,11 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         string where;
         if (specificDataLoadRunIDOnly != null)
         {
-            where = $"WHERE ID={specificDataLoadRunIDOnly.Value}";
+            where = $"WHERE {sh.EnsureWrapped("ID")}={specificDataLoadRunIDOnly.Value}";
         }
         else
         {
-            where = "WHERE dataLoadTaskID = @dataTaskId";
+            where = $"WHERE {sh.EnsureWrapped("dataLoadTaskID")} = @dataTaskId";
             var p = cmd.CreateParameter();
             p.ParameterName = "@dataTaskId";
             p.Value = dataTaskId;
@@ -158,13 +161,13 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         TopXResponse top = null;
 
         if (topX.HasValue)
-            top = Server.GetQuerySyntaxHelper().HowDoWeAchieveTopX(topX.Value);
+            top = sh.HowDoWeAchieveTopX(topX.Value);
 
         var sb = new StringBuilder("SELECT ");
 
         if (top?.Location == QueryComponent.SELECT) sb.AppendLine(top.SQL);
 
-        sb.AppendLine($" * FROM {run.GetFullyQualifiedName()}  {where} ORDER BY ID desc");
+        sb.AppendLine($" * FROM {run.GetFullyQualifiedName()}  {where} ORDER BY {sh.EnsureWrapped("ID")} desc");
 
         if (top?.Location == QueryComponent.Postfix) sb.AppendLine(top.SQL);
 
@@ -204,7 +207,8 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
 
     private static int GetDataTaskId(string dataTask, DiscoveredServer server, DbConnection con)
     {
-        using var cmd = server.GetCommand("SELECT ID FROM DataLoadTask WHERE name = @name", con);
+        var sh = server.GetQuerySyntaxHelper();
+        using var cmd = server.GetCommand($"SELECT {sh.EnsureWrapped("ID")} FROM {sh.EnsureWrapped("DataLoadTask")} WHERE name = @name", con);
         var p = cmd.CreateParameter();
         p.ParameterName = "@name";
         p.Value = dataTask;
@@ -236,9 +240,10 @@ public IDataLoadInfo CreateDataLoadInfo(string dataLoadTaskName, string packageN
     public void CreateNewLoggingTask(int id, string dataSetID)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         var sql =
-            $"INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES ({id}, @dataSetID, @dataSetID, @date, @username, 1, 0, @dataSetID)";
+            $"INSERT INTO {sh.EnsureWrapped("DataLoadTask")} ({sh.EnsureWrapped("ID")}, description, name, {sh.EnsureWrapped("createTime")}, {sh.EnsureWrapped("userAccount")}, {sh.EnsureWrapped("statusID")}, {sh.EnsureWrapped("isTest")}, {sh.EnsureWrapped("dataSetID")}) VALUES ({id}, @dataSetID, @dataSetID, @date, @username, 1, {sh.False}, @dataSetID)";
 
         using var cmd = Server.GetCommand(sql, conn);
         Server.AddParameterWithValueToCommand("@date", cmd, DateTime.Now);
@@ -251,9 +256,10 @@ public void CreateNewLoggingTask(int id, string dataSetID)
     private void CreateNewDataSet(string datasetName)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         {
-            const string sql = "INSERT INTO DataSet (dataSetID,name) VALUES (@datasetName,@datasetName)";
+            var sql = $"INSERT INTO {sh.EnsureWrapped("DataSet")} ({sh.EnsureWrapped("dataSetID")},name) VALUES (@datasetName,@datasetName)";
 
             using var cmd = Server.GetCommand(sql, conn);
             Server.AddParameterWithValueToCommand("@datasetName", cmd, datasetName.Substring(Math.Max(0, datasetName.Length - 150)));
@@ -274,8 +280,9 @@ public void CreateNewLoggingTaskIfNotExists(string toCreate)
     private int GetMaxTaskID()
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
-        const string sql = "SELECT MAX(ID) FROM DataLoadTask";
+        var sql = $"SELECT MAX({sh.EnsureWrapped("ID")}) FROM {sh.EnsureWrapped("DataLoadTask")}";
 
         using var cmd = Server.GetCommand(sql, conn);
         var result = cmd.ExecuteScalar();
@@ -285,9 +292,10 @@ private int GetMaxTaskID()
     public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState, string newExplanation)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         var sql =
-            $"UPDATE FatalError SET explanation =@explanation, statusID=@statusID where ID in ({string.Join(",", ids)})";
+            $"UPDATE {sh.EnsureWrapped("FatalError")} SET explanation =@explanation, {sh.EnsureWrapped("statusID")}=@statusID where {sh.EnsureWrapped("ID")} in ({string.Join(",", ids)})";
 
         using var cmd = Server.GetCommand(sql, conn);
         Server.AddParameterWithValueToCommand("@explanation", cmd, newExplanation);
@@ -298,4 +306,4 @@ public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState
             throw new Exception(
                 $"Query {sql} resulted in {affectedRows}, we were expecting there to be {ids.Length} updates because that is how many FatalError IDs that were passed to this method");
     }
-}
+}

Rdmp.UI/Collections/ConfigurationsCollectionUI.cs

@@ -4,9 +4,9 @@
 using Rdmp.UI.ItemActivation;
 using Rdmp.UI.Refreshing;
 using System.Linq;
-using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Providers.Nodes;
 using Rdmp.Core.Curation.Data.Datasets;
+using Rdmp.Core.Curation.DataHelper.RegexRedaction;
 
 namespace Rdmp.UI.Collections;
 
@@ -63,6 +63,9 @@ public void RefreshBus_RefreshObject(object sender, RefreshObjectEventArgs e)
             case Dataset:
                 tlvConfigurations.RefreshObject(tlvConfigurations.Objects.OfType<AllDatasetsNode>());
                 break;
+            case RegexRedactionConfiguration:
+                tlvConfigurations.RefreshObject(tlvConfigurations.Objects.OfType<AllRegexRedactionConfigurationsNode>());
+                break;
         }
     }
 

Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandAddNewRegexRedactionConfigurationUI.cs

@@ -1,14 +1,12 @@
-using Rdmp.Core.CommandExecution;
-using Rdmp.Core.CommandExecution.AtomicCommands;
+// Copyright (c) The University of Dundee 2024-2024
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using Rdmp.Core.CommandExecution;
 using Rdmp.UI.ItemActivation;
-using Rdmp.UI.SimpleDialogs.Datasets;
 using Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm;
-using Rdmp.UI.TestsAndSetup.ServicePropogation;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Rdmp.UI.CommandExecution.AtomicCommands
 {

Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandRegexRedaction.cs

@@ -1,4 +1,10 @@
-using Rdmp.Core.CommandExecution.AtomicCommands;
+// Copyright (c) The University of Dundee 2024-2024
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using Rdmp.Core.CommandExecution.AtomicCommands;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Icons.IconProvision;
 using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;

Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsRegexRedactionConfiguration.cs

@@ -1,4 +1,10 @@
-using Rdmp.Core.CommandExecution;
+// Copyright (c) The University of Dundee 2024-2024
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation.DataHelper.RegexRedaction;
 using Rdmp.Core.ReusableLibraryCode.Annotations;
 using Rdmp.UI.ItemActivation;

Rdmp.UI/SimpleDialogs/RedactCatalogueUI.cs

@@ -41,6 +41,11 @@ public RedactCatalogueUI()
 
     }
 
+    public override string GetTabName()
+    {
+        return $"{base.GetTabName()} - Regex Redaction";   
+    }
+
     public void HandleNewRegex(object sender, EventArgs e)
     {
         //_activator

Rdmp.UI/SimpleDialogs/RegexRedactionConfigurationForm/CreateNewRegexRedactionConfigurationUI.cs

@@ -1,18 +1,10 @@
-using Rdmp.Core.CommandExecution.AtomicCommands;
-using Rdmp.Core.Curation.DataHelper.RegexRedaction;
-using Rdmp.UI.CommandExecution.AtomicCommands;
+using Rdmp.Core.Curation.DataHelper.RegexRedaction;
 using Rdmp.UI.ItemActivation;
 using Rdmp.UI.TestsAndSetup.ServicePropogation;
 using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Data;
 using System.Drawing;
-using System.Linq;
-using System.Text;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-using System.Windows.Forms;
+
 
 namespace Rdmp.UI.SimpleDialogs.RegexRedactionConfigurationForm
 {

SharedAssemblyInfo.cs

@@ -12,4 +12,4 @@
 
 [assembly: AssemblyVersion("8.4.0")]
 [assembly: AssemblyFileVersion("8.4.0")]
-[assembly: AssemblyInformationalVersion("8.4.0")]
+[assembly: AssemblyInformationalVersion("8.4.0")]