From 46cc583851e4011fe9800ea86eea7de00fdfa1f2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 2 Apr 2024 14:47:39 +0100 Subject: [PATCH] add real test --- ...atabaseMSSqlDestinationReExtractionTest.cs | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs new file mode 100644 index 0000000000..16f9a5d8ae --- /dev/null +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs @@ -0,0 +1,361 @@ +// Copyright (c) The University of Dundee 2018-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using Amazon.Runtime.Internal.Transform; +using BadMedicine.Datasets; +using FAnsi.Discovery; +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandLine.Options; +using Rdmp.Core.CommandLine.Runners; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Pipelines; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; +using Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using Tests.Common; +using Tests.Common.Scenarios; + +namespace Rdmp.Core.Tests.DataExport.DataExtraction; + +public class ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest : TestsRequiringAnExtractionConfiguration +{ + private ExternalDatabaseServer _extractionServer; + + private const string _expectedTableName = "ExecuteFullExtractionToDatabaseMSSqlDestinationTest_TestTable"; + private ColumnInfo _columnToTransform; + private Pipeline _pipeline; + private DiscoveredTable tbl; + private SupportingSQLTable sql; + + [Test] + public void SQLServerDestinationReExtraction() + { + DiscoveredDatabase dbToExtractTo = null; + + var ci = new CatalogueItem(CatalogueRepository, _catalogue, "YearOfBirth"); + _columnToTransform = _columnInfos.Single(c => + c.GetRuntimeName().Equals("DateOfBirth", StringComparison.CurrentCultureIgnoreCase)); + + var transform = $"YEAR({_columnToTransform.Name})"; + + + if (_catalogue.GetAllExtractionInformation(ExtractionCategory.Any) + .All(ei => ei.GetRuntimeName() != "YearOfBirth")) + { + var ei = new ExtractionInformation(CatalogueRepository, ci, _columnToTransform, transform) + { + Alias = "YearOfBirth", + ExtractionCategory = ExtractionCategory.Core + }; + ei.SaveToDatabase(); + + //make it part of the ExtractionConfiguration + var newColumn = new ExtractableColumn(DataExportRepository, _selectedDataSet.ExtractableDataSet, + (ExtractionConfiguration)_selectedDataSet.ExtractionConfiguration, ei, 0, ei.SelectSQL) + { + Alias = ei.Alias + }; + newColumn.SaveToDatabase(); + + _extractableColumns.Add(newColumn); + } + + CreateLookupsEtc(); + + try + { + _configuration.Name = "ExecuteFullExtractionToDatabaseMSSqlDestinationTest"; + _configuration.SaveToDatabase(); + + var dbname = TestDatabaseNames.GetConsistentName($"{_project.Name}_{_project.ProjectNumber}"); + dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + + //ExecuteRunner(); + var pipeline = SetupPipeline(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = _configuration.ID.ToString(), + ExtractGlobals = true, + Pipeline = pipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + var destinationTable = dbToExtractTo.ExpectTable(_expectedTableName); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.Multiple(() => + { + Assert.That(dt.Rows[0]["ReleaseID"], Is.EqualTo(_cohortKeysGenerated[_cohortKeysGenerated.Keys.First()].Trim())); + Assert.That(dt.Rows[0]["DateOfBirth"], Is.EqualTo(new DateTime(2001, 1, 1))); + Assert.That(dt.Rows[0]["YearOfBirth"], Is.EqualTo(2001)); + + Assert.That(destinationTable.DiscoverColumn("DateOfBirth").DataType.SQLType, Is.EqualTo(_columnToTransform.Data_type)); + Assert.That(destinationTable.DiscoverColumn("YearOfBirth").DataType.SQLType, Is.EqualTo("int")); + }); + + AssertLookupsEtcExist(dbToExtractTo); + } + finally + { + if (dbToExtractTo?.Exists() == true) + dbToExtractTo.Drop(); + + _pipeline?.DeleteInDatabase(); + } + } + + [Test] + public void SQLServerDestinationReExtractionwithNewData() + { + DiscoveredDatabase dbToExtractTo = null; + + var ci = new CatalogueItem(CatalogueRepository, _catalogue, "YearOfBirth"); + _columnToTransform = _columnInfos.Single(c => + c.GetRuntimeName().Equals("DateOfBirth", StringComparison.CurrentCultureIgnoreCase)); + + var transform = $"YEAR({_columnToTransform.Name})"; + + + if (_catalogue.GetAllExtractionInformation(ExtractionCategory.Any) + .All(ei => ei.GetRuntimeName() != "YearOfBirth")) + { + var ei = new ExtractionInformation(CatalogueRepository, ci, _columnToTransform, transform) + { + Alias = "YearOfBirth", + ExtractionCategory = ExtractionCategory.Core + }; + ei.SaveToDatabase(); + + //make it part of the ExtractionConfiguration + var newColumn = new ExtractableColumn(DataExportRepository, _selectedDataSet.ExtractableDataSet, + (ExtractionConfiguration)_selectedDataSet.ExtractionConfiguration, ei, 0, ei.SelectSQL) + { + Alias = ei.Alias + }; + newColumn.SaveToDatabase(); + + _extractableColumns.Add(newColumn); + } + + CreateLookupsEtc(); + + try + { + _configuration.Name = "ExecuteFullExtractionToDatabaseMSSqlDestinationTest"; + _configuration.SaveToDatabase(); + + var dbname = TestDatabaseNames.GetConsistentName($"{_project.Name}_{_project.ProjectNumber}"); + dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + + //ExecuteRunner(); + var pipeline = SetupPipeline(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = _configuration.ID.ToString(), + ExtractGlobals = true, + Pipeline = pipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + tbl.Insert(new Dictionary + { + { "chi","1111111111"}, + {"Healthboard","T"}, + {"SampleDate", new DateTime(2001, 1, 2) }, + {"SampleType","Blood"}, + {"TestCode","1234"}, + {"Result",1}, + {"Labnumber","1"}, + {"QuantityUnit","ml"}, + {"ReadCodeValue","1"}, + {"ArithmeticComparator","="}, + {"Interpretation","!"}, + {"RangeHighValue",1}, + {"RangeLowValue",1} + }); + sql.SQL = $"SELECT * FROM {tbl.GetFullyQualifiedName()}"; + sql.SaveToDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = _configuration.ID.ToString(), + ExtractGlobals = true, + Pipeline = pipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + var destinationTable = dbToExtractTo.ExpectTable(_expectedTableName); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(2)); + Assert.Multiple(() => + { + Assert.That(dt.Rows[0]["ReleaseID"], Is.EqualTo(_cohortKeysGenerated[_cohortKeysGenerated.Keys.First()].Trim())); + Assert.That(dt.Rows[0]["DateOfBirth"], Is.EqualTo(new DateTime(2001, 1, 1))); + Assert.That(dt.Rows[0]["YearOfBirth"], Is.EqualTo(2001)); + + Assert.That(destinationTable.DiscoverColumn("DateOfBirth").DataType.SQLType, Is.EqualTo(_columnToTransform.Data_type)); + Assert.That(destinationTable.DiscoverColumn("YearOfBirth").DataType.SQLType, Is.EqualTo("int")); + }); + + AssertLookupsEtcExist(dbToExtractTo); + } + finally + { + if (dbToExtractTo?.Exists() == true) + dbToExtractTo.Drop(); + + _pipeline?.DeleteInDatabase(); + } + } + + private static void AssertLookupsEtcExist(DiscoveredDatabase dbToExtractTo) + { + Assert.That(dbToExtractTo.ExpectTable("ExecuteFullExtractionToDatabaseMSSqlDestinationTest_TestTable_Biochem") + .Exists()); + } + + private void CreateLookupsEtc() + { + //an extractable file + var filename = Path.Combine(TestContext.CurrentContext.WorkDirectory, "bob.txt"); + + File.WriteAllText(filename, "fishfishfish"); + var doc = new SupportingDocument(CatalogueRepository, _catalogue, "bob") + { + URL = new Uri($"file://{filename}"), + Extractable = true + }; + doc.SaveToDatabase(); + + //an extractable global file (comes out regardless of datasets) + var filename2 = Path.Combine(TestContext.CurrentContext.WorkDirectory, "bob2.txt"); + + File.WriteAllText(filename2, "fishfishfish2"); + var doc2 = new SupportingDocument(CatalogueRepository, _catalogue, "bob2") + { + URL = new Uri($"file://{filename2}"), + Extractable = true, + IsGlobal = true + }; + doc2.SaveToDatabase(); + + //an supplemental table in the database (not linked against cohort) + tbl = CreateDataset(Database, 1, 1, new Random(50)); + + sql = new SupportingSQLTable(CatalogueRepository, _catalogue, "Biochem"); + var server = new ExternalDatabaseServer(CatalogueRepository, "myserver", null); + server.SetProperties(tbl.Database); + sql.ExternalDatabaseServer_ID = server.ID; + sql.SQL = $"SELECT * FROM {tbl.GetFullyQualifiedName()}"; + sql.Extractable = true; + sql.SaveToDatabase(); + + } + + + protected override Pipeline SetupPipeline() + { + //create a target server pointer + _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + Random rnd = new Random(); + + //create a pipeline + _pipeline = new Pipeline(CatalogueRepository, $"Empty extraction pipeline{rnd.Next()}"); + + //set the destination pipeline + var component = new PipelineComponent(CatalogueRepository, _pipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentAppendDataIfTableExists = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentAppendDataIfTableExists.SetValue(true); + argumentAppendDataIfTableExists.SaveToDatabase(); + AdjustPipelineComponentDelegate?.Invoke(component); + + var component2 = new PipelineComponent(CatalogueRepository, _pipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + AdjustPipelineComponentDelegate?.Invoke(component2); + + //configure the component as the destination + _pipeline.DestinationPipelineComponent_ID = component.ID; + _pipeline.SourcePipelineComponent_ID = component2.ID; + _pipeline.SaveToDatabase(); + + return _pipeline; + } +} \ No newline at end of file