Skip to content

Commit

Permalink
Merge branch 'release/8.4.0' into task/RDMP-32-regex-redaction
Browse files Browse the repository at this point in the history
  • Loading branch information
@JFriel
JFriel authored Nov 18, 2024
2 parents 8f5a2fd + 0779ce3 commit 81faa2f
Show file tree
Hide file tree
Showing 7 changed files with 38 additions and 32 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

- Add Ordering to Filters
- [MSSQL ONLY] Add ability to perform Regex redactions on data loads and existing catalogues
- Add RAW Table Date Column Override for Delta Loads
- Fix Delta Load off by one issue
- Update Migration strategy to account for all Primary Keys when moving from staging -> live
- Fix UI issue with viewing cross-database SQL results
Expand Down
1 change: 1 addition & 0 deletions Documentation/DataLoadEngine/RemoteAttachers.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ The full configuration options are
| Culture | Optionally specify a custom date format |
| Explicit Date Time Format | Optionally specify a specific datetime format
| Selected Columns | Optionally select which columns you wish to pull from the remote server (defaults to "*" )
| RawTableDateColumn | Optionally give the date column in RAW a different name than in the select statement for delta loads. This may be due to awkward joins in a custom select query

## Configuring the Remote Database Attacher
The Remote Database Attacher has a number of configuration options the required fields are:
Expand Down
22 changes: 11 additions & 11 deletions Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void TestRemoteAttacherParameter(AttacherHistoricalDurations duration, st
RemoteTableDateColumn = "date"
};
var lmd = new LoadMetadata();
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > DATEADD({convertTime}, -1, GETDATE())"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) > DATEADD({convertTime}, -1, GETDATE())"));
}
[Test]
public void TestRemoteAttacherParameterSinceLastUse()
Expand All @@ -42,7 +42,7 @@ public void TestRemoteAttacherParameterSinceLastUse()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) > convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) > convert(Date,'{lmd.LastLoadTime.GetValueOrDefault():yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterSinceLastUse_NULL()
Expand All @@ -53,7 +53,7 @@ public void TestRemoteAttacherParameterSinceLastUse_NULL()
RemoteTableDateColumn = "date"
};
var lmd = new LoadMetadata();
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo(""));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo(""));
}
[Test]
public void TestRemoteAttacherParameterCustomRange()
Expand All @@ -69,7 +69,7 @@ public void TestRemoteAttacherParameterCustomRange()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterCustomRangeNoStart()
Expand All @@ -84,7 +84,7 @@ public void TestRemoteAttacherParameterCustomRangeNoStart()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate:yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterCustomRangeNoEnd()
Expand All @@ -99,7 +99,7 @@ public void TestRemoteAttacherParameterCustomRangeNoEnd()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate:yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterCustomRangeNoDates()
Expand All @@ -113,7 +113,7 @@ public void TestRemoteAttacherParameterCustomRangeNoDates()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo(""));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo(""));
}
[Test]
public void TestRemoteAttacherParameterDeltaReading()
Expand All @@ -130,7 +130,7 @@ public void TestRemoteAttacherParameterDeltaReading()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterDeltaReading_NoLookBack()
Expand All @@ -146,7 +146,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookBack()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays):yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterDeltaReading_NoLookForward()
Expand All @@ -162,7 +162,7 @@ public void TestRemoteAttacherParameterDeltaReading_NoLookForward()
{
LastLoadTime = DateTime.Now
};
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')"));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer,attacher.RemoteTableDateColumn), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays):yyyy-MM-dd HH:mm:ss.fff}') AND CAST(date as Date) < convert(Date,'{attacher.DeltaReadingStartDate:yyyy-MM-dd HH:mm:ss.fff}')"));
}
[Test]
public void TestRemoteAttacherParameterDeltaReadingNoDates()
Expand All @@ -173,6 +173,6 @@ public void TestRemoteAttacherParameterDeltaReadingNoDates()
RemoteTableDateColumn = "date"
};
var lmd = new LoadMetadata();
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo(""));
Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer, attacher.RemoteTableDateColumn), Is.EqualTo(""));
}
}
4 changes: 0 additions & 4 deletions Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/TableInfoCloneOperation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,6 @@ public void CloneTable(DiscoveredDatabase srcDatabaseInfo, DiscoveredDatabase de
drop = true;
}

//drop the data load run ID field and validFrom fields, we don't need them in STAGING or RAW, it will be hard coded in the MERGE migration with a fixed value anyway.
if (colName.Equals(SpecialFieldNames.DataLoadRunID) || colName.Equals(SpecialFieldNames.ValidFrom))
drop = true;

var dilution = dilutionColumns.SingleOrDefault(c => c.GetRuntimeName().Equals(colName));

if (dilution != null)
Expand Down
32 changes: 20 additions & 12 deletions Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ public RemoteAttacher() : base(true) { }
[DemandsInitialization("Which column in the remote table can be used to perform time-based data selection")]
public string RemoteTableDateColumn { get; set; }

[DemandsInitialization("Option name for the Date colunn table within RDMP if it differs from the remote table e.g Due to joins")]
public string RawTableDateColumn { get; set; }

private readonly string RemoteTableDateFormat = "yyyy-MM-dd HH:mm:ss.fff";

[DemandsInitialization("Earliest date when using a custom fetch duration")]
Expand Down Expand Up @@ -93,33 +96,33 @@ private string ConvertDateString(DatabaseType dbType, string dateString)

}

public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType dbType)
public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType dbType, string column)
{
const string dateConvert = "Date";

switch (HistoricalFetchDuration)
{
case AttacherHistoricalDurations.Past24Hours:
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}";
return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}";
case AttacherHistoricalDurations.Past7Days:
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}";
return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}";
case AttacherHistoricalDurations.PastMonth:
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}";
return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}";
case AttacherHistoricalDurations.PastYear:
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}";
return $" WHERE CAST({column} as {dateConvert}) > {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}";
case AttacherHistoricalDurations.SinceLastUse:
return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) > {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : "";
return loadMetadata.LastLoadTime is not null ? $" WHERE CAST({column} as {dateConvert}) > {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}" : "";
case AttacherHistoricalDurations.Custom:
if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate != DateTime.MinValue)
{
//end only
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}";
return $" WHERE CAST({column} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}";
}

if (CustomFetchDurationStartDate != DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue)
{
//start only
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))}";
return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))}";
}

if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue)
Expand All @@ -128,12 +131,12 @@ public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType d
return "";
}

return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}";
return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))} AND CAST({column} as {dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}";
case AttacherHistoricalDurations.DeltaReading:
if (DeltaReadingStartDate == DateTime.MinValue) return "";
var startDate = DeltaReadingStartDate.AddDays(-DeltaReadingLookBackDays);
var endDate = DeltaReadingStartDate.AddDays(DeltaReadingLookForwardDays);
return $" WHERE CAST({RemoteTableDateColumn} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {dateConvert}) < {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}";
return $" WHERE CAST({column} as {dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({column} as {dateConvert}) < {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}";
default:
return "";
}
Expand Down Expand Up @@ -173,9 +176,14 @@ private bool IsThisRemoteAttacher(IProcessTask task)
return true;
}

public void FindMostRecentDateInLoadedData(IQuerySyntaxHelper syntaxFrom, DatabaseType dbType, string table, IDataLoadJob job)
public void FindMostRecentDateInLoadedData(IQuerySyntaxHelper syntaxFrom, DatabaseType dbType, string table, IDataLoadJob job, bool userOverrideColumn = false)
{
var maxDateSql = $"SELECT MAX({RemoteTableDateColumn}) FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbType)}";
var column = RemoteTableDateColumn;
if (userOverrideColumn && !string.IsNullOrWhiteSpace(RawTableDateColumn))
{
column = RawTableDateColumn;
}
var maxDateSql = $"SELECT MAX({column}) FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbType, column)}";

using var con = _dbInfo.Server.GetConnection();
using var dt = new DataTable();
Expand Down
4 changes: 2 additions & 2 deletions Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,11 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken
? tableInfo.GetColumnsAtStage(LoadStage.AdjustRaw)
: tableInfo.ColumnInfos;
sql =
$"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbFrom.Server.DatabaseType)}";
$"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbFrom.Server.DatabaseType,RemoteTableDateColumn)}";
}
else
{
sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata,RemoteSource.DatabaseType)}";
sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata,RemoteSource.DatabaseType, RemoteTableDateColumn)}";
}
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
$"About to execute SQL:{Environment.NewLine}{sql}"));
Expand Down
6 changes: 3 additions & 3 deletions Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,12 +364,12 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken
string sql;
if (!string.IsNullOrWhiteSpace(RemoteSelectSQL))
{
var injectedWhereClause = SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType).Replace(" WHERE", "");
var injectedWhereClause = SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType, RemoteTableDateColumn).Replace(" WHERE", "");
sql = Regex.Replace(RemoteSelectSQL, "\\$RDMPDefinedWhereClause", injectedWhereClause);
}
else
{
sql = $"Select {SelectedColumns} from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType)}";
sql = $"Select {SelectedColumns} from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata, DatabaseType, RemoteTableDateColumn)}";
}

var scheduleMismatch = false;
Expand Down Expand Up @@ -466,7 +466,7 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken

if (SetDeltaReadingToLatestSeenDatePostLoad)
{
FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType, rawTableName, job);
FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType, rawTableName, job,true);
}


Expand Down

0 comments on commit 81faa2f

Please sign in to comment.