From 4f0d1656b90b93ca4418d93f673a8a1f4e210a02 Mon Sep 17 00:00:00 2001 From: zwdOr20 Date: Mon, 5 Apr 2021 23:48:50 +0100 Subject: [PATCH 1/5] feat: Add Config command basic infra code - Add ConfigOption.cs and BaseConfigOption.cs files for config cmd options. - Add RunConfigOption method to handle config cmd use case. closes https://github.com/rdagumampan/yuniql/issues/172 --- yuniql-cli/BaseConfigOption.cs | 13 + yuniql-cli/BasePlatformOption.cs | 4 +- yuniql-cli/CommandLineService.cs | 12 + yuniql-cli/ConfigOption.cs | 9 + yuniql-cli/Program.cs | 9 +- .../postgresql/PostgreSqlBulkImportService.cs | 512 +++++++++--------- 6 files changed, 298 insertions(+), 261 deletions(-) create mode 100644 yuniql-cli/BaseConfigOption.cs create mode 100644 yuniql-cli/ConfigOption.cs diff --git a/yuniql-cli/BaseConfigOption.cs b/yuniql-cli/BaseConfigOption.cs new file mode 100644 index 00000000..7a3efb34 --- /dev/null +++ b/yuniql-cli/BaseConfigOption.cs @@ -0,0 +1,13 @@ +using CommandLine; + +namespace Yuniql.CLI +{ + public class BaseConfigOption : BasePlatformOption + { + //yuniql -o "" + [Option('c', "data-type", Required = false, HelpText = "The choosen Output format")] + public string DataType { get; set; } + } +} + + diff --git a/yuniql-cli/BasePlatformOption.cs b/yuniql-cli/BasePlatformOption.cs index 3d6d7c5d..d1b3ee67 100644 --- a/yuniql-cli/BasePlatformOption.cs +++ b/yuniql-cli/BasePlatformOption.cs @@ -3,8 +3,8 @@ namespace Yuniql.CLI { public class BasePlatformOption : BaseOption - { - //yuniql -d | --debug + { + //yuniql -p | --platform [Option(longName: "platform", Required = false, HelpText = "Target database platform. Default is sqlserver.")] public string Platform { get; set; } diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index b5a8ae6c..cb1ddc16 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -368,6 +368,18 @@ public int RunArchiveOption(ArchiveOption opts) return OnException(ex, "Failed to execute archive function", opts.IsDebug); } } + public int RunConfigOption(ConfigOption opts) + { + try + { + throw new NotImplementedException("Not yet implemented"); + } + catch (Exception ex) + { + return OnException(ex, "Failed to execute config function", opts.IsDebug); + } + } + private int OnException(Exception exception, string headerMessage, bool debug) { diff --git a/yuniql-cli/ConfigOption.cs b/yuniql-cli/ConfigOption.cs new file mode 100644 index 00000000..a33ab4e6 --- /dev/null +++ b/yuniql-cli/ConfigOption.cs @@ -0,0 +1,9 @@ +using CommandLine; + +namespace Yuniql.CLI +{ + [Verb("config", HelpText = "Print the environment setup in the command line")] + public class ConfigOption : BaseConfigOption + { + } +} diff --git a/yuniql-cli/Program.cs b/yuniql-cli/Program.cs index 22dc8264..58ffd4f9 100644 --- a/yuniql-cli/Program.cs +++ b/yuniql-cli/Program.cs @@ -1,5 +1,6 @@ using CommandLine; using System; +using System.Collections; using System.Reflection; using Yuniql.Core; using Yuniql.Extensibility; @@ -30,7 +31,7 @@ public static int Main(string[] args) environmentService, traceService, configurationService); - + IEnumerable errors; var resultCode = Parser.Default .ParseArguments< CheckOption, @@ -44,7 +45,8 @@ public static int Main(string[] args) BaselineOption, RebaseOption, //ArchiveOption, - PlatformsOption + PlatformsOption, + ConfigOption >(args).MapResult( (CheckOption opts) => Dispatch(commandLineService.RunCheckOption, opts, traceService), (InitOption opts) => Dispatch(commandLineService.RunInitOption, opts, traceService), @@ -58,8 +60,9 @@ public static int Main(string[] args) (RebaseOption opts) => Dispatch(commandLineService.RunRebaseOption, opts, traceService), //(ArchiveOption opts) => Dispatch(commandLineService.RunArchiveOption, opts, traceService), (PlatformsOption opts) => Dispatch(commandLineService.RunPlatformsOption, opts, traceService), + (ConfigOption opts) => Dispatch(commandLineService.RunConfigOption, opts, traceService), - errs => 1); + errs => { errors = errs; return 1; }); return resultCode; } diff --git a/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs b/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs index 31be421c..95bbbaf8 100644 --- a/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs +++ b/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs @@ -1,277 +1,277 @@ -using System.Data; -using System.IO; -using Yuniql.Extensibility; -using NpgsqlTypes; -using System.Collections.Generic; -using Npgsql; -using System; -using System.Linq; -using Yuniql.Extensibility.BulkCsvParser; +using System.Data; +using System.IO; +using Yuniql.Extensibility; +using NpgsqlTypes; +using System.Collections.Generic; +using Npgsql; +using System; +using System.Linq; +using Yuniql.Extensibility.BulkCsvParser; using System.Diagnostics; //https://github.com/22222/CsvTextFieldParser -namespace Yuniql.PostgreSql -{ - /// - public class PostgreSqlBulkImportService : IBulkImportService - { - private string _connectionString; +namespace Yuniql.PostgreSql +{ + /// + public class PostgreSqlBulkImportService : IBulkImportService + { + private string _connectionString; private readonly ITraceService _traceService; - /// - public PostgreSqlBulkImportService(ITraceService traceService) - { - this._traceService = traceService; + /// + public PostgreSqlBulkImportService(ITraceService traceService) + { + this._traceService = traceService; } - /// - public void Initialize( - string connectionString) - { - this._connectionString = connectionString; + /// + public void Initialize( + string connectionString) + { + this._connectionString = connectionString; } - /// - public void Run( - IDbConnection connection, - IDbTransaction transaction, - string fileFullPath, - string bulkSeparator = null, - int? bulkBatchSize = null, - int? commandTimeout = null) + /// + public void Run( + IDbConnection connection, + IDbTransaction transaction, + string fileFullPath, + string bulkSeparator = null, + int? bulkBatchSize = null, + int? commandTimeout = null) { //get file name segments from potentially sequenceno.schemaname.tablename filename pattern //assumes all objects are not double quoted because pgsql auto-lower case all undouble quoted objects - var fileName = Path.GetFileNameWithoutExtension(fileFullPath); - var fileNameSegments = fileName.SplitBulkFileName(defaultSchema: "public"); + var fileName = Path.GetFileNameWithoutExtension(fileFullPath); + var fileNameSegments = fileName.SplitBulkFileName(defaultSchema: "public"); var schemaName = fileNameSegments.Item2.IsDoubleQuoted() ? fileNameSegments.Item2 : fileNameSegments.Item2.ToLower(); var tableName = fileNameSegments.Item3.IsDoubleQuoted() ? fileNameSegments.Item3 : fileNameSegments.Item3.ToLower(); var stopwatch = new Stopwatch(); stopwatch.Start(); - _traceService.Info($"PostgreSqlBulkImportService: Started copying data into destination table {schemaName}.{tableName}"); + _traceService.Info($"PostgreSqlBulkImportService: Started copying data into destination table {schemaName}.{tableName}"); //read csv file and load into data table var dataTable = ParseCsvFile(fileFullPath, bulkSeparator); - - //save the csv data into staging sql table - BulkCopyWithDataTable(connection, transaction, schemaName, tableName, dataTable); - + + //save the csv data into staging sql table + BulkCopyWithDataTable(connection, transaction, schemaName, tableName, dataTable); + stopwatch.Stop(); - _traceService.Info($"PostgreSqlBulkImportService: Finished copying data into destination table {schemaName}.{tableName} in {stopwatch.ElapsedMilliseconds} ms"); - } - - private DataTable ParseCsvFile(string csvFileFullPath, string bulkSeparator) - { - if (string.IsNullOrEmpty(bulkSeparator)) - bulkSeparator = ","; - - var csvDatatable = new DataTable(); - using (var csvReader = new CsvTextFieldParser(csvFileFullPath)) - { - csvReader.Separators = (new string[] { bulkSeparator }); - csvReader.HasFieldsEnclosedInQuotes = true; - - string[] csvColumns = csvReader.ReadFields(); - foreach (string csvColumn in csvColumns) - { - var dataColumn = new DataColumn(csvColumn.ToLower()); - dataColumn.AllowDBNull = true; - csvDatatable.Columns.Add(dataColumn); - } - - while (!csvReader.EndOfData) - { - string[] fieldData = csvReader.ReadFields(); - for (int i = 0; i < fieldData.Length; i++) - { - if (fieldData[i] == "" || fieldData[i] == "NULL") - { - fieldData[i] = null; - } - } - csvDatatable.Rows.Add(fieldData); - } - } - return csvDatatable; - } - - //NOTE: This is not the most typesafe and performant way to do this and this is just to demonstrate - //possibility to bulk import data in custom means during migration execution - //https://www.npgsql.org/doc/copy.html - private void BulkCopyWithDataTable( - IDbConnection connection, - IDbTransaction transaction, - string schemaName, - string tableName, - DataTable dataTable) - { - //get destination table schema and filter out columns not in csv file - var destinationSchema = GetDestinationSchema(schemaName, tableName); - var destinationColumns = destinationSchema.ToList().Where(f => dataTable.Columns.Contains(f.Key)).Select(k => k.Key).ToArray(); - - //prepare statement for binary import - var sqlStatement = $"COPY {schemaName}.{tableName} ({string.Join(',', destinationColumns)}) FROM STDIN (FORMAT BINARY)"; - _traceService.Info("PostgreSqlBulkImportService: " + sqlStatement); - - var pgsqlConnection = connection as NpgsqlConnection; - using (var writer = pgsqlConnection.BeginBinaryImport(sqlStatement)) - { - //writes each data row as datastream into pgsql database - foreach (DataRow dataRow in dataTable.Rows) - { - writer.StartRow(); - foreach (DataColumn dataColumn in dataTable.Columns) - { - if (dataRow.IsNull(dataColumn.ColumnName)) - { - writer.Write(DBNull.Value); - continue; - } - - if (!destinationSchema.ContainsKey(dataColumn.ColumnName)) - continue; - - var dataType = destinationSchema[dataColumn.ColumnName.ToLower()].DataType; - - if (dataType == "boolean" || dataType == "bit" || dataType == "bit varying") - { - writer.Write(bool.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Boolean); - continue; - } - else if (dataType == "smallint" || dataType == "int2") - { - writer.Write(short.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Smallint); - continue; - } - else if (dataType == "integer" || dataType == "int4") - { - writer.Write(int.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Integer); - continue; - } - else if (dataType == "bigint" || dataType == "int8") - { - writer.Write(long.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Bigint); - continue; - } - else if (dataType == "real") - { - writer.Write(float.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Real); - continue; - } - else if (dataType == "double precision") - { - writer.Write(double.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Double); - continue; - } - else if (dataType == "numeric" || dataType == "money") - { - writer.Write(decimal.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Numeric); - continue; - } - else if (dataType == "uuid") - { - writer.Write(Guid.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Uuid); - continue; - } - else if (dataType == "date") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Date); - continue; - } - else if (dataType == "interval") - { - writer.Write(TimeSpan.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Interval); - continue; - } - else if (dataType == "timestamp" || dataType == "timestamp without time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Timestamp); - continue; - } - else if (dataType == "timestamp with time zone") - { - writer.Write(DateTimeOffset.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimestampTz); - continue; - } - else if (dataType == "time" || dataType == "time without time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Time); - continue; - } - else if (dataType == "time with time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimeTz); - continue; - } - else if (dataType == "name") - { - writer.Write(dataRow[dataColumn.ColumnName].ToString(), NpgsqlDbType.Name); - continue; - } - else if (dataType == "(internal) char") - { - writer.Write(byte.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.InternalChar); - continue; - } - else if (dataType == "text" - || dataType == "character varying" - || dataType == "character" - || dataType == "citext" - || dataType == "json" - || dataType == "jsonb" - || dataType == "xml") - { - writer.Write(dataRow[dataColumn.ColumnName].ToString()); - continue; - } - else - { - //not supported types: lseg,path,polygon,line,circle,box,hstore,cidr,inet,macaddr,tsquery,tsvector,bytea,oid,xid,cid,oidvector,composite types,range types,enum types,array types - throw new NotSupportedException($"PostgreSqlBulkImportService: Data type '{dataType}' on destination table {schemaName}.{tableName} is not support for bulk import operations."); - } - } - } - - //wraps up everything, closes the stream - writer.Complete(); - } - } - - //https://www.npgsql.org/doc/types/basic.html - private IDictionary GetDestinationSchema(string schemaName, string tableName) - { - var result = new Dictionary(); - using (var connection = new NpgsqlConnection(_connectionString)) - { - connection.Open(); - - var command = connection.CreateCommand(); - command.CommandType = CommandType.Text; - command.CommandText = $"SELECT column_name, data_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '{schemaName}' AND TABLE_NAME = '{tableName}'"; - command.CommandTimeout = 0; - - using (var reader = command.ExecuteReader()) - { - while (reader.Read()) - { - result.Add(reader.GetString(0), new ColumnDefinition - { - ColumnName = reader.GetString(0), - DataType = reader.GetString(1) - }); - } - } - } - - return result; - } - } - - public class ColumnDefinition - { - public string ColumnName { get; set; } - public string DataType { get; set; } - } -} - + _traceService.Info($"PostgreSqlBulkImportService: Finished copying data into destination table {schemaName}.{tableName} in {stopwatch.ElapsedMilliseconds} ms"); + } + + private DataTable ParseCsvFile(string csvFileFullPath, string bulkSeparator) + { + if (string.IsNullOrEmpty(bulkSeparator)) + bulkSeparator = ","; + + var csvDatatable = new DataTable(); + using (var csvReader = new CsvTextFieldParser(csvFileFullPath)) + { + csvReader.Separators = (new string[] { bulkSeparator }); + csvReader.HasFieldsEnclosedInQuotes = true; + + string[] csvColumns = csvReader.ReadFields(); + foreach (string csvColumn in csvColumns) + { + var dataColumn = new DataColumn(csvColumn.ToLower()); + dataColumn.AllowDBNull = true; + csvDatatable.Columns.Add(dataColumn); + } + + while (!csvReader.EndOfData) + { + string[] fieldData = csvReader.ReadFields(); + for (int i = 0; i < fieldData.Length; i++) + { + if (fieldData[i] == "" || fieldData[i] == "NULL") + { + fieldData[i] = null; + } + } + csvDatatable.Rows.Add(fieldData); + } + } + return csvDatatable; + } + + //NOTE: This is not the most typesafe and performant way to do this and this is just to demonstrate + //possibility to bulk import data in custom means during migration execution + //https://www.npgsql.org/doc/copy.html + private void BulkCopyWithDataTable( + IDbConnection connection, + IDbTransaction transaction, + string schemaName, + string tableName, + DataTable dataTable) + { + //get destination table schema and filter out columns not in csv file + var destinationSchema = GetDestinationSchema(schemaName, tableName); + var destinationColumns = destinationSchema.ToList().Where(f => dataTable.Columns.Contains(f.Key)).Select(k => k.Key).ToArray(); + + //prepare statement for binary import + var sqlStatement = $"COPY {schemaName}.{tableName} ({string.Join(',', destinationColumns)}) FROM STDIN (FORMAT BINARY)"; + _traceService.Info("PostgreSqlBulkImportService: " + sqlStatement); + + var pgsqlConnection = connection as NpgsqlConnection; + using (var writer = pgsqlConnection.BeginBinaryImport(sqlStatement)) + { + //writes each data row as datastream into pgsql database + foreach (DataRow dataRow in dataTable.Rows) + { + writer.StartRow(); + foreach (DataColumn dataColumn in dataTable.Columns) + { + if (dataRow.IsNull(dataColumn.ColumnName)) + { + writer.Write(DBNull.Value); + continue; + } + + if (!destinationSchema.ContainsKey(dataColumn.ColumnName)) + continue; + + var dataType = destinationSchema[dataColumn.ColumnName.ToLower()].DataType; + + if (dataType == "boolean" || dataType == "bit" || dataType == "bit varying") + { + writer.Write(bool.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Boolean); + continue; + } + else if (dataType == "smallint" || dataType == "int2") + { + writer.Write(short.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Smallint); + continue; + } + else if (dataType == "integer" || dataType == "int4") + { + writer.Write(int.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Integer); + continue; + } + else if (dataType == "bigint" || dataType == "int8") + { + writer.Write(long.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Bigint); + continue; + } + else if (dataType == "real") + { + writer.Write(float.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Real); + continue; + } + else if (dataType == "double precision") + { + writer.Write(double.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Double); + continue; + } + else if (dataType == "numeric" || dataType == "money") + { + writer.Write(decimal.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Numeric); + continue; + } + else if (dataType == "uuid") + { + writer.Write(Guid.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Uuid); + continue; + } + else if (dataType == "date") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Date); + continue; + } + else if (dataType == "interval") + { + writer.Write(TimeSpan.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Interval); + continue; + } + else if (dataType == "timestamp" || dataType == "timestamp without time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Timestamp); + continue; + } + else if (dataType == "timestamp with time zone") + { + writer.Write(DateTimeOffset.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimestampTz); + continue; + } + else if (dataType == "time" || dataType == "time without time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Time); + continue; + } + else if (dataType == "time with time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimeTz); + continue; + } + else if (dataType == "name") + { + writer.Write(dataRow[dataColumn.ColumnName].ToString(), NpgsqlDbType.Name); + continue; + } + else if (dataType == "(internal) char") + { + writer.Write(byte.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.InternalChar); + continue; + } + else if (dataType == "text" + || dataType == "character varying" + || dataType == "character" + || dataType == "citext" + || dataType == "json" + || dataType == "jsonb" + || dataType == "xml") + { + writer.Write(dataRow[dataColumn.ColumnName].ToString()); + continue; + } + else + { + //not supported types: lseg,path,polygon,line,circle,box,hstore,cidr,inet,macaddr,tsquery,tsvector,bytea,oid,xid,cid,oidvector,composite types,range types,enum types,array types + throw new NotSupportedException($"PostgreSqlBulkImportService: Data type '{dataType}' on destination table {schemaName}.{tableName} is not support for bulk import operations."); + } + } + } + + //wraps up everything, closes the stream + writer.Complete(); + } + } + + //https://www.npgsql.org/doc/types/basic.html + private IDictionary GetDestinationSchema(string schemaName, string tableName) + { + var result = new Dictionary(); + using (var connection = new NpgsqlConnection(_connectionString)) + { + connection.Open(); + + var command = connection.CreateCommand(); + command.CommandType = CommandType.Text; + command.CommandText = $"SELECT column_name, data_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '{schemaName}' AND TABLE_NAME = '{tableName}'"; + command.CommandTimeout = 0; + + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + result.Add(reader.GetString(0), new ColumnDefinition + { + ColumnName = reader.GetString(0), + DataType = reader.GetString(1) + }); + } + } + } + + return result; + } + } + + public class ColumnDefinition + { + public string ColumnName { get; set; } + public string DataType { get; set; } + } +} + From 620bd40bf48cd9213f2df9d1d3a0a5bfb3a9c950 Mon Sep 17 00:00:00 2001 From: zwdOr20 Date: Tue, 6 Apr 2021 12:46:02 +0100 Subject: [PATCH 2/5] Complete feature : Configuration variables print Implment RunConfigurationOption method to handle config command use case. closes https://github.com/rdagumampan/yuniql/issues/172 --- yuniql-cli/BaseConfigOption.cs | 10 ++++-- yuniql-cli/BaseOption.cs | 2 +- yuniql-cli/CommandLineService.cs | 52 +++++++++++++++++++++++++++----- yuniql-cli/Program.cs | 5 ++- yuniql-core/Source.cs | 25 +++++++++++++++ 5 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 yuniql-core/Source.cs diff --git a/yuniql-cli/BaseConfigOption.cs b/yuniql-cli/BaseConfigOption.cs index 7a3efb34..afed1fe7 100644 --- a/yuniql-cli/BaseConfigOption.cs +++ b/yuniql-cli/BaseConfigOption.cs @@ -4,9 +4,13 @@ namespace Yuniql.CLI { public class BaseConfigOption : BasePlatformOption { - //yuniql -o "" - [Option('c', "data-type", Required = false, HelpText = "The choosen Output format")] - public string DataType { get; set; } + //yuniql -o "" + [Option('c', "data-type", Required = false, HelpText = "The choosen Output format")] + public string DataType { get; set; } + + //yuniql -a true | --auto-create-db true + [Option('a', "auto-create-db", Required = false, HelpText = "Create database automatically.")] + public bool? IsAutoCreateDatabase { get; set; } } } diff --git a/yuniql-cli/BaseOption.cs b/yuniql-cli/BaseOption.cs index 950ce958..fa4661cd 100644 --- a/yuniql-cli/BaseOption.cs +++ b/yuniql-cli/BaseOption.cs @@ -10,7 +10,7 @@ public class BaseOption //yuniql -d | --debug [Option('d', "debug", Required = false, HelpText = "Print debug information including all raw scripts.")] - public bool IsDebug { get; set; } + public bool? IsDebug { get; set; } //yuniql --trace-sensitive-data [Option("trace-sensitive-data", Required = false, HelpText = "Include sensitive data like connection string in the log messages.", Default = false)] diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index cb1ddc16..a2bd36e7 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -40,7 +40,7 @@ private Configuration SetupRunConfiguration(BaseRunPlatformOption opts, bool isV var tokens = opts.Tokens.Select(t => new KeyValuePair(t.Split("=")[0], t.Split("=")[1])).ToList(); configuration.Workspace = opts.Workspace; - configuration.IsDebug = opts.IsDebug; + configuration.IsDebug = opts.IsDebug?? false; configuration.Platform = platform; configuration.ConnectionString = opts.ConnectionString; @@ -82,7 +82,7 @@ public int RunCheckOption(CheckOption opts) } catch (Exception ex) { - return OnException(ex, "Failed to execute ping function", opts.IsDebug); + return OnException(ex, "Failed to execute ping function", opts.IsDebug??false); } return 0; @@ -171,7 +171,7 @@ public int RunListOption(ListOption opts) var configuration = Configuration.Instance; configuration.Workspace = opts.Workspace; - configuration.IsDebug = opts.IsDebug; + configuration.IsDebug = opts.IsDebug??false; configuration.Platform = platform; configuration.ConnectionString = opts.ConnectionString; @@ -222,7 +222,7 @@ public int RunEraseOption(EraseOption opts) var configuration = Configuration.Instance; configuration.Workspace = opts.Workspace; - configuration.IsDebug = opts.IsDebug; + configuration.IsDebug = opts.IsDebug??false; configuration.Platform = platform; configuration.ConnectionString = opts.ConnectionString; @@ -255,7 +255,7 @@ public int RunDestroyOption(DestroyOption opts) var configuration = Configuration.Instance; configuration.Workspace = opts.Workspace; - configuration.IsDebug = opts.IsDebug; + configuration.IsDebug = opts.IsDebug??false; configuration.Platform = platform; configuration.ConnectionString = connectionString; @@ -372,7 +372,42 @@ public int RunConfigOption(ConfigOption opts) { try { - throw new NotImplementedException("Not yet implemented"); + var versionPrettyPrint = new TablePrinter("Property", "Value", "Source"); + // platform + var platformValue = _configurationService.GetValueOrDefault(opts.Platform, ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM, defaultValue: SUPPORTED_DATABASES.SQLSERVER); + var platformSource = opts.Platform != null ? Source.CmdLine_Options : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM) != null ? Source.Environment_variable + : Source.Default; + versionPrettyPrint.AddRow("Platform", platformValue, platformSource); + //workspace + var workspaceValue = _configurationService.GetValueOrDefault(opts.Workspace, ENVIRONMENT_VARIABLE.YUNIQL_WORKSPACE,"undefined"); + var workspaceSource = opts.Workspace != null ? Source.CmdLine_Options : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_WORKSPACE) != null ? Source.Environment_variable + : Source.Default; + versionPrettyPrint.AddRow("Workspace", workspaceValue, workspaceSource); + + //connection string + var connectionStringValue = _configurationService.GetValueOrDefault(opts.ConnectionString, ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING,"undefined"); + var connectionStringSource = opts.ConnectionString != null ? Source.CmdLine_Options : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING) != null ? Source.Environment_variable + : Source.Default; + versionPrettyPrint.AddRow("ConnectionString", connectionStringValue, connectionStringSource); + + //connection string + var IsDebugValue = opts.IsDebug?? false; + var IsDebugSource = opts.IsDebug != null ? Source.CmdLine_Options : Source.Default; + versionPrettyPrint.AddRow("IsDebug", IsDebugValue, IsDebugSource); + + // Auto Create Database + var IsAutoCreateDatabaseValue = opts.IsAutoCreateDatabase ?? false; + var IsAutoCreateDatabaseSource = opts.IsAutoCreateDatabase != null ? Source.CmdLine_Options : Source.Default; + versionPrettyPrint.AddRow("IsAutoCreateDatabase", IsAutoCreateDatabaseValue, IsAutoCreateDatabaseSource); + + //print table + versionPrettyPrint.Print(); + _traceService.Success($"Listed all configuration variables successfully"); + + return 0; } catch (Exception ex) { @@ -381,9 +416,10 @@ public int RunConfigOption(ConfigOption opts) } - private int OnException(Exception exception, string headerMessage, bool debug) + private int OnException(Exception exception, string headerMessage, bool? debug) { - var stackTraceMessage = debug ? exception.ToString().Replace(exception.Message, string.Empty) + bool debugOption = debug ?? false; + var stackTraceMessage = debugOption? exception.ToString().Replace(exception.Message, string.Empty) : $"{exception.Message} {exception.InnerException?.Message}"; _traceService.Error($"{headerMessage}. {exception.Message}{Environment.NewLine}" + diff --git a/yuniql-cli/Program.cs b/yuniql-cli/Program.cs index 58ffd4f9..6b0326ce 100644 --- a/yuniql-cli/Program.cs +++ b/yuniql-cli/Program.cs @@ -61,8 +61,7 @@ public static int Main(string[] args) //(ArchiveOption opts) => Dispatch(commandLineService.RunArchiveOption, opts, traceService), (PlatformsOption opts) => Dispatch(commandLineService.RunPlatformsOption, opts, traceService), (ConfigOption opts) => Dispatch(commandLineService.RunConfigOption, opts, traceService), - - errs => { errors = errs; return 1; }); + errs => 1 ); return resultCode; } @@ -81,7 +80,7 @@ private static int Dispatch(Func command, T opts, ITraceService trace Console.WriteLine($"Visit https://yuniql.io for documentation and working samples{Environment.NewLine}"); Console.ResetColor(); - traceService.IsDebugEnabled = opts.IsDebug; + traceService.IsDebugEnabled = opts.IsDebug?? false; traceService.IsTraceSensitiveData = opts.IsTraceSensitiveData; traceService.IsTraceToFile = opts.IsTraceToFile; traceService.TraceToDirectory = opts.TraceToDirectory; diff --git a/yuniql-core/Source.cs b/yuniql-core/Source.cs new file mode 100644 index 00000000..dbfc4321 --- /dev/null +++ b/yuniql-core/Source.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Yuniql.Core +{ + /// + /// Information source for yuniql configuration values + /// + public enum Source + { + /// + /// Default value assigned to configuration parameter + /// + Default, + /// + /// Environment variable is the source of the configuration parameter + /// + Environment_variable, + /// + /// command line is the source of the configuration parameter + /// + CmdLine_Options + } +} From 3b20357ab420bac4a5fcffc075ac1b21e6890129 Mon Sep 17 00:00:00 2001 From: zwdOr20 Date: Wed, 7 Apr 2021 14:04:58 +0100 Subject: [PATCH 3/5] feat: Add Json Printer option (infra) - add IPrinter interface - add JsonPrinter class closes https://github.com/rdagumampan/yuniql/issues/172 --- yuniql-cli/BaseConfigOption.cs | 2 +- yuniql-cli/CommandLineService.cs | 8 +++++++- yuniql-cli/JsonPrinter.cs | 22 +++++++++++++++++++++ yuniql-core/IPrinter.cs | 12 +++++++++++ {yuniql-cli => yuniql-core}/TablePrinter.cs | 8 +++----- 5 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 yuniql-cli/JsonPrinter.cs create mode 100644 yuniql-core/IPrinter.cs rename {yuniql-cli => yuniql-core}/TablePrinter.cs (90%) diff --git a/yuniql-cli/BaseConfigOption.cs b/yuniql-cli/BaseConfigOption.cs index afed1fe7..f4979743 100644 --- a/yuniql-cli/BaseConfigOption.cs +++ b/yuniql-cli/BaseConfigOption.cs @@ -4,7 +4,7 @@ namespace Yuniql.CLI { public class BaseConfigOption : BasePlatformOption { - //yuniql -o "" + //yuniql -o json [Option('c', "data-type", Required = false, HelpText = "The choosen Output format")] public string DataType { get; set; } diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index a2bd36e7..1e06bb61 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -372,7 +372,13 @@ public int RunConfigOption(ConfigOption opts) { try { - var versionPrettyPrint = new TablePrinter("Property", "Value", "Source"); + IPrinter versionPrettyPrint; + if (opts.DataType != null && opts.DataType.Equals("json", + StringComparison.OrdinalIgnoreCase)) + versionPrettyPrint = new JsonPrinter(); + else + versionPrettyPrint = new TablePrinter("Property", "Value", "Source"); + // platform var platformValue = _configurationService.GetValueOrDefault(opts.Platform, ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM, defaultValue: SUPPORTED_DATABASES.SQLSERVER); var platformSource = opts.Platform != null ? Source.CmdLine_Options : diff --git a/yuniql-cli/JsonPrinter.cs b/yuniql-cli/JsonPrinter.cs new file mode 100644 index 00000000..2251f128 --- /dev/null +++ b/yuniql-cli/JsonPrinter.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Yuniql.Core; + +namespace Yuniql.CLI +{ + public class JsonPrinter : IPrinter + { + private (string property, string value, Source source)[] ParametersList = new (string, string, Source)[5]; + + public void AddRow(params object[] row) + { + throw new NotImplementedException(); + } + + public void Print() + { + throw new NotImplementedException(); + } + } +} diff --git a/yuniql-core/IPrinter.cs b/yuniql-core/IPrinter.cs new file mode 100644 index 00000000..d7cbf8c7 --- /dev/null +++ b/yuniql-core/IPrinter.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Yuniql.Core +{ + public interface IPrinter + { + void Print(); + void AddRow(params object[] row); + } +} diff --git a/yuniql-cli/TablePrinter.cs b/yuniql-core/TablePrinter.cs similarity index 90% rename from yuniql-cli/TablePrinter.cs rename to yuniql-core/TablePrinter.cs index 25936104..d86f33ba 100644 --- a/yuniql-cli/TablePrinter.cs +++ b/yuniql-core/TablePrinter.cs @@ -2,12 +2,9 @@ using System.Collections.Generic; using System.Linq; -namespace Yuniql.CLI +namespace Yuniql.Core { - //TODO: Move this into Yuniql.Core as TablePrintService - //thanks https://stackoverflow.com/users/1547699/sumudu - //https://stackoverflow.com/a/54943087/3449591 - public class TablePrinter + public class TablePrinter : IPrinter { private readonly string[] titles; private readonly List lengths; @@ -71,5 +68,6 @@ public void Print() Console.WriteLine("+"); Console.WriteLine(); } + } } From 57da2b7e7daf8ac801d20bd3f8260fc188bfc71f Mon Sep 17 00:00:00 2001 From: zwdOr20 Date: Thu, 8 Apr 2021 12:40:00 +0100 Subject: [PATCH 4/5] feat: Complete feature : Add json option - Choose text format for printing configuration variable information. - Add json option (by default the format is table/rows) closes closes https://github.com/rdagumampan/yuniql/issues/172 --- yuniql-cli/BaseConfigOption.cs | 2 +- yuniql-cli/JsonPrinter.cs | 33 ++++++++++++++++++----- yuniql-cli/Program.cs | 1 - yuniql-cli/Properties/launchSettings.json | 8 ------ yuniql-core/IPrinter.cs | 1 + 5 files changed, 29 insertions(+), 16 deletions(-) delete mode 100644 yuniql-cli/Properties/launchSettings.json diff --git a/yuniql-cli/BaseConfigOption.cs b/yuniql-cli/BaseConfigOption.cs index f4979743..b5c1d109 100644 --- a/yuniql-cli/BaseConfigOption.cs +++ b/yuniql-cli/BaseConfigOption.cs @@ -5,7 +5,7 @@ namespace Yuniql.CLI public class BaseConfigOption : BasePlatformOption { //yuniql -o json - [Option('c', "data-type", Required = false, HelpText = "The choosen Output format")] + [Option('o', "data-type", Required = false, HelpText = "The choosen Output format")] public string DataType { get; set; } //yuniql -a true | --auto-create-db true diff --git a/yuniql-cli/JsonPrinter.cs b/yuniql-cli/JsonPrinter.cs index 2251f128..d7104606 100644 --- a/yuniql-cli/JsonPrinter.cs +++ b/yuniql-cli/JsonPrinter.cs @@ -1,22 +1,43 @@ -using System; +using Newtonsoft.Json.Linq; +using System; using System.Collections.Generic; -using System.Text; +using System.Linq; using Yuniql.Core; namespace Yuniql.CLI { + /// + /// This class is used to print configuration variable as json text + /// public class JsonPrinter : IPrinter { - private (string property, string value, Source source)[] ParametersList = new (string, string, Source)[5]; - + private readonly string[] titles = new string[] { "property","value","source"}; + private readonly List rows = new List(); + public void AddRow(params object[] row) { - throw new NotImplementedException(); + if (row.Length != titles.Length) + { + throw new Exception($"Added row length [{row.Length}] is not equal to title row length [{titles.Length}]"); + } + rows.Add(row.Select(o => o.ToString()).ToArray()); } public void Print() { - throw new NotImplementedException(); + var jarray = new JArray("Properties"); + var jproperty = new JProperty("Properties", jarray); + var jobject = new JObject(jproperty); + foreach (var row in rows) + { + JObject obj = new JObject(); + for (int i = 0; i < row.Length; i++) + { + obj.Add(new JProperty(titles[i], row[i])); + } + jarray.Add(obj); + } + Console.WriteLine(jobject.ToString()); } } } diff --git a/yuniql-cli/Program.cs b/yuniql-cli/Program.cs index 6b0326ce..06cc3595 100644 --- a/yuniql-cli/Program.cs +++ b/yuniql-cli/Program.cs @@ -31,7 +31,6 @@ public static int Main(string[] args) environmentService, traceService, configurationService); - IEnumerable errors; var resultCode = Parser.Default .ParseArguments< CheckOption, diff --git a/yuniql-cli/Properties/launchSettings.json b/yuniql-cli/Properties/launchSettings.json deleted file mode 100644 index 8afc538c..00000000 --- a/yuniql-cli/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Yuniql.CLI": { - "commandName": "Project", - "commandLineArgs": "destroy --force --debug -c \"Server=localhost;Port=3306;Database=helloyuniql;Uid=root;Pwd=P@ssw0rd!;\"" - } - } -} \ No newline at end of file diff --git a/yuniql-core/IPrinter.cs b/yuniql-core/IPrinter.cs index d7cbf8c7..9a07319f 100644 --- a/yuniql-core/IPrinter.cs +++ b/yuniql-core/IPrinter.cs @@ -8,5 +8,6 @@ public interface IPrinter { void Print(); void AddRow(params object[] row); + } } From a88b3853951621f7f1608e523d97f942ca8d2c27 Mon Sep 17 00:00:00 2001 From: zwdOr20 Date: Sat, 10 Apr 2021 12:26:13 +0100 Subject: [PATCH 5/5] refactor: Enhance code readability - change data-type option name to output - change Source enum values fomat to ALLCAPS - chekout yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs to master. --- yuniql-cli/BaseConfigOption.cs | 4 +- yuniql-cli/CommandLineService.cs | 25 +- yuniql-core/Source.cs | 6 +- .../postgresql/PostgreSqlBulkImportService.cs | 512 +++++++++--------- 4 files changed, 274 insertions(+), 273 deletions(-) diff --git a/yuniql-cli/BaseConfigOption.cs b/yuniql-cli/BaseConfigOption.cs index b5c1d109..a45c3dd9 100644 --- a/yuniql-cli/BaseConfigOption.cs +++ b/yuniql-cli/BaseConfigOption.cs @@ -5,8 +5,8 @@ namespace Yuniql.CLI public class BaseConfigOption : BasePlatformOption { //yuniql -o json - [Option('o', "data-type", Required = false, HelpText = "The choosen Output format")] - public string DataType { get; set; } + [Option('o', "output", Required = false, HelpText = "The choosen Output format")] + public string Output { get; set; } //yuniql -a true | --auto-create-db true [Option('a', "auto-create-db", Required = false, HelpText = "Create database automatically.")] diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index 1e06bb61..45473a4a 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -371,9 +371,10 @@ public int RunArchiveOption(ArchiveOption opts) public int RunConfigOption(ConfigOption opts) { try + { IPrinter versionPrettyPrint; - if (opts.DataType != null && opts.DataType.Equals("json", + if (opts.Output != null && opts.Output.Equals("json", StringComparison.OrdinalIgnoreCase)) versionPrettyPrint = new JsonPrinter(); else @@ -381,32 +382,32 @@ public int RunConfigOption(ConfigOption opts) // platform var platformValue = _configurationService.GetValueOrDefault(opts.Platform, ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM, defaultValue: SUPPORTED_DATABASES.SQLSERVER); - var platformSource = opts.Platform != null ? Source.CmdLine_Options : - _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM) != null ? Source.Environment_variable - : Source.Default; + var platformSource = opts.Platform != null ? Source.CMD_LINE_OPTIONS : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_PLATFORM) != null ? Source.ENVIRONMENT_VARIABLE + : Source.DEFAULT; versionPrettyPrint.AddRow("Platform", platformValue, platformSource); //workspace var workspaceValue = _configurationService.GetValueOrDefault(opts.Workspace, ENVIRONMENT_VARIABLE.YUNIQL_WORKSPACE,"undefined"); - var workspaceSource = opts.Workspace != null ? Source.CmdLine_Options : - _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_WORKSPACE) != null ? Source.Environment_variable - : Source.Default; + var workspaceSource = opts.Workspace != null ? Source.CMD_LINE_OPTIONS : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_WORKSPACE) != null ? Source.ENVIRONMENT_VARIABLE + : Source.DEFAULT; versionPrettyPrint.AddRow("Workspace", workspaceValue, workspaceSource); //connection string var connectionStringValue = _configurationService.GetValueOrDefault(opts.ConnectionString, ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING,"undefined"); - var connectionStringSource = opts.ConnectionString != null ? Source.CmdLine_Options : - _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING) != null ? Source.Environment_variable - : Source.Default; + var connectionStringSource = opts.ConnectionString != null ? Source.CMD_LINE_OPTIONS : + _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING) != null ? Source.ENVIRONMENT_VARIABLE + : Source.DEFAULT; versionPrettyPrint.AddRow("ConnectionString", connectionStringValue, connectionStringSource); //connection string var IsDebugValue = opts.IsDebug?? false; - var IsDebugSource = opts.IsDebug != null ? Source.CmdLine_Options : Source.Default; + var IsDebugSource = opts.IsDebug != null ? Source.CMD_LINE_OPTIONS : Source.DEFAULT; versionPrettyPrint.AddRow("IsDebug", IsDebugValue, IsDebugSource); // Auto Create Database var IsAutoCreateDatabaseValue = opts.IsAutoCreateDatabase ?? false; - var IsAutoCreateDatabaseSource = opts.IsAutoCreateDatabase != null ? Source.CmdLine_Options : Source.Default; + var IsAutoCreateDatabaseSource = opts.IsAutoCreateDatabase != null ? Source.CMD_LINE_OPTIONS : Source.DEFAULT; versionPrettyPrint.AddRow("IsAutoCreateDatabase", IsAutoCreateDatabaseValue, IsAutoCreateDatabaseSource); //print table diff --git a/yuniql-core/Source.cs b/yuniql-core/Source.cs index dbfc4321..78c00c14 100644 --- a/yuniql-core/Source.cs +++ b/yuniql-core/Source.cs @@ -12,14 +12,14 @@ public enum Source /// /// Default value assigned to configuration parameter /// - Default, + DEFAULT, /// /// Environment variable is the source of the configuration parameter /// - Environment_variable, + ENVIRONMENT_VARIABLE, /// /// command line is the source of the configuration parameter /// - CmdLine_Options + CMD_LINE_OPTIONS } } diff --git a/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs b/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs index 95bbbaf8..31be421c 100644 --- a/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs +++ b/yuniql-platforms/postgresql/PostgreSqlBulkImportService.cs @@ -1,277 +1,277 @@ -using System.Data; -using System.IO; -using Yuniql.Extensibility; -using NpgsqlTypes; -using System.Collections.Generic; -using Npgsql; -using System; -using System.Linq; -using Yuniql.Extensibility.BulkCsvParser; +using System.Data; +using System.IO; +using Yuniql.Extensibility; +using NpgsqlTypes; +using System.Collections.Generic; +using Npgsql; +using System; +using System.Linq; +using Yuniql.Extensibility.BulkCsvParser; using System.Diagnostics; //https://github.com/22222/CsvTextFieldParser -namespace Yuniql.PostgreSql -{ - /// - public class PostgreSqlBulkImportService : IBulkImportService - { - private string _connectionString; +namespace Yuniql.PostgreSql +{ + /// + public class PostgreSqlBulkImportService : IBulkImportService + { + private string _connectionString; private readonly ITraceService _traceService; - /// - public PostgreSqlBulkImportService(ITraceService traceService) - { - this._traceService = traceService; + /// + public PostgreSqlBulkImportService(ITraceService traceService) + { + this._traceService = traceService; } - /// - public void Initialize( - string connectionString) - { - this._connectionString = connectionString; + /// + public void Initialize( + string connectionString) + { + this._connectionString = connectionString; } - /// - public void Run( - IDbConnection connection, - IDbTransaction transaction, - string fileFullPath, - string bulkSeparator = null, - int? bulkBatchSize = null, - int? commandTimeout = null) + /// + public void Run( + IDbConnection connection, + IDbTransaction transaction, + string fileFullPath, + string bulkSeparator = null, + int? bulkBatchSize = null, + int? commandTimeout = null) { //get file name segments from potentially sequenceno.schemaname.tablename filename pattern //assumes all objects are not double quoted because pgsql auto-lower case all undouble quoted objects - var fileName = Path.GetFileNameWithoutExtension(fileFullPath); - var fileNameSegments = fileName.SplitBulkFileName(defaultSchema: "public"); + var fileName = Path.GetFileNameWithoutExtension(fileFullPath); + var fileNameSegments = fileName.SplitBulkFileName(defaultSchema: "public"); var schemaName = fileNameSegments.Item2.IsDoubleQuoted() ? fileNameSegments.Item2 : fileNameSegments.Item2.ToLower(); var tableName = fileNameSegments.Item3.IsDoubleQuoted() ? fileNameSegments.Item3 : fileNameSegments.Item3.ToLower(); var stopwatch = new Stopwatch(); stopwatch.Start(); - _traceService.Info($"PostgreSqlBulkImportService: Started copying data into destination table {schemaName}.{tableName}"); + _traceService.Info($"PostgreSqlBulkImportService: Started copying data into destination table {schemaName}.{tableName}"); //read csv file and load into data table var dataTable = ParseCsvFile(fileFullPath, bulkSeparator); - - //save the csv data into staging sql table - BulkCopyWithDataTable(connection, transaction, schemaName, tableName, dataTable); - + + //save the csv data into staging sql table + BulkCopyWithDataTable(connection, transaction, schemaName, tableName, dataTable); + stopwatch.Stop(); - _traceService.Info($"PostgreSqlBulkImportService: Finished copying data into destination table {schemaName}.{tableName} in {stopwatch.ElapsedMilliseconds} ms"); - } - - private DataTable ParseCsvFile(string csvFileFullPath, string bulkSeparator) - { - if (string.IsNullOrEmpty(bulkSeparator)) - bulkSeparator = ","; - - var csvDatatable = new DataTable(); - using (var csvReader = new CsvTextFieldParser(csvFileFullPath)) - { - csvReader.Separators = (new string[] { bulkSeparator }); - csvReader.HasFieldsEnclosedInQuotes = true; - - string[] csvColumns = csvReader.ReadFields(); - foreach (string csvColumn in csvColumns) - { - var dataColumn = new DataColumn(csvColumn.ToLower()); - dataColumn.AllowDBNull = true; - csvDatatable.Columns.Add(dataColumn); - } - - while (!csvReader.EndOfData) - { - string[] fieldData = csvReader.ReadFields(); - for (int i = 0; i < fieldData.Length; i++) - { - if (fieldData[i] == "" || fieldData[i] == "NULL") - { - fieldData[i] = null; - } - } - csvDatatable.Rows.Add(fieldData); - } - } - return csvDatatable; - } - - //NOTE: This is not the most typesafe and performant way to do this and this is just to demonstrate - //possibility to bulk import data in custom means during migration execution - //https://www.npgsql.org/doc/copy.html - private void BulkCopyWithDataTable( - IDbConnection connection, - IDbTransaction transaction, - string schemaName, - string tableName, - DataTable dataTable) - { - //get destination table schema and filter out columns not in csv file - var destinationSchema = GetDestinationSchema(schemaName, tableName); - var destinationColumns = destinationSchema.ToList().Where(f => dataTable.Columns.Contains(f.Key)).Select(k => k.Key).ToArray(); - - //prepare statement for binary import - var sqlStatement = $"COPY {schemaName}.{tableName} ({string.Join(',', destinationColumns)}) FROM STDIN (FORMAT BINARY)"; - _traceService.Info("PostgreSqlBulkImportService: " + sqlStatement); - - var pgsqlConnection = connection as NpgsqlConnection; - using (var writer = pgsqlConnection.BeginBinaryImport(sqlStatement)) - { - //writes each data row as datastream into pgsql database - foreach (DataRow dataRow in dataTable.Rows) - { - writer.StartRow(); - foreach (DataColumn dataColumn in dataTable.Columns) - { - if (dataRow.IsNull(dataColumn.ColumnName)) - { - writer.Write(DBNull.Value); - continue; - } - - if (!destinationSchema.ContainsKey(dataColumn.ColumnName)) - continue; - - var dataType = destinationSchema[dataColumn.ColumnName.ToLower()].DataType; - - if (dataType == "boolean" || dataType == "bit" || dataType == "bit varying") - { - writer.Write(bool.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Boolean); - continue; - } - else if (dataType == "smallint" || dataType == "int2") - { - writer.Write(short.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Smallint); - continue; - } - else if (dataType == "integer" || dataType == "int4") - { - writer.Write(int.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Integer); - continue; - } - else if (dataType == "bigint" || dataType == "int8") - { - writer.Write(long.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Bigint); - continue; - } - else if (dataType == "real") - { - writer.Write(float.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Real); - continue; - } - else if (dataType == "double precision") - { - writer.Write(double.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Double); - continue; - } - else if (dataType == "numeric" || dataType == "money") - { - writer.Write(decimal.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Numeric); - continue; - } - else if (dataType == "uuid") - { - writer.Write(Guid.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Uuid); - continue; - } - else if (dataType == "date") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Date); - continue; - } - else if (dataType == "interval") - { - writer.Write(TimeSpan.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Interval); - continue; - } - else if (dataType == "timestamp" || dataType == "timestamp without time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Timestamp); - continue; - } - else if (dataType == "timestamp with time zone") - { - writer.Write(DateTimeOffset.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimestampTz); - continue; - } - else if (dataType == "time" || dataType == "time without time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Time); - continue; - } - else if (dataType == "time with time zone") - { - writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimeTz); - continue; - } - else if (dataType == "name") - { - writer.Write(dataRow[dataColumn.ColumnName].ToString(), NpgsqlDbType.Name); - continue; - } - else if (dataType == "(internal) char") - { - writer.Write(byte.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.InternalChar); - continue; - } - else if (dataType == "text" - || dataType == "character varying" - || dataType == "character" - || dataType == "citext" - || dataType == "json" - || dataType == "jsonb" - || dataType == "xml") - { - writer.Write(dataRow[dataColumn.ColumnName].ToString()); - continue; - } - else - { - //not supported types: lseg,path,polygon,line,circle,box,hstore,cidr,inet,macaddr,tsquery,tsvector,bytea,oid,xid,cid,oidvector,composite types,range types,enum types,array types - throw new NotSupportedException($"PostgreSqlBulkImportService: Data type '{dataType}' on destination table {schemaName}.{tableName} is not support for bulk import operations."); - } - } - } - - //wraps up everything, closes the stream - writer.Complete(); - } - } - - //https://www.npgsql.org/doc/types/basic.html - private IDictionary GetDestinationSchema(string schemaName, string tableName) - { - var result = new Dictionary(); - using (var connection = new NpgsqlConnection(_connectionString)) - { - connection.Open(); - - var command = connection.CreateCommand(); - command.CommandType = CommandType.Text; - command.CommandText = $"SELECT column_name, data_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '{schemaName}' AND TABLE_NAME = '{tableName}'"; - command.CommandTimeout = 0; - - using (var reader = command.ExecuteReader()) - { - while (reader.Read()) - { - result.Add(reader.GetString(0), new ColumnDefinition - { - ColumnName = reader.GetString(0), - DataType = reader.GetString(1) - }); - } - } - } - - return result; - } - } - - public class ColumnDefinition - { - public string ColumnName { get; set; } - public string DataType { get; set; } - } -} - + _traceService.Info($"PostgreSqlBulkImportService: Finished copying data into destination table {schemaName}.{tableName} in {stopwatch.ElapsedMilliseconds} ms"); + } + + private DataTable ParseCsvFile(string csvFileFullPath, string bulkSeparator) + { + if (string.IsNullOrEmpty(bulkSeparator)) + bulkSeparator = ","; + + var csvDatatable = new DataTable(); + using (var csvReader = new CsvTextFieldParser(csvFileFullPath)) + { + csvReader.Separators = (new string[] { bulkSeparator }); + csvReader.HasFieldsEnclosedInQuotes = true; + + string[] csvColumns = csvReader.ReadFields(); + foreach (string csvColumn in csvColumns) + { + var dataColumn = new DataColumn(csvColumn.ToLower()); + dataColumn.AllowDBNull = true; + csvDatatable.Columns.Add(dataColumn); + } + + while (!csvReader.EndOfData) + { + string[] fieldData = csvReader.ReadFields(); + for (int i = 0; i < fieldData.Length; i++) + { + if (fieldData[i] == "" || fieldData[i] == "NULL") + { + fieldData[i] = null; + } + } + csvDatatable.Rows.Add(fieldData); + } + } + return csvDatatable; + } + + //NOTE: This is not the most typesafe and performant way to do this and this is just to demonstrate + //possibility to bulk import data in custom means during migration execution + //https://www.npgsql.org/doc/copy.html + private void BulkCopyWithDataTable( + IDbConnection connection, + IDbTransaction transaction, + string schemaName, + string tableName, + DataTable dataTable) + { + //get destination table schema and filter out columns not in csv file + var destinationSchema = GetDestinationSchema(schemaName, tableName); + var destinationColumns = destinationSchema.ToList().Where(f => dataTable.Columns.Contains(f.Key)).Select(k => k.Key).ToArray(); + + //prepare statement for binary import + var sqlStatement = $"COPY {schemaName}.{tableName} ({string.Join(',', destinationColumns)}) FROM STDIN (FORMAT BINARY)"; + _traceService.Info("PostgreSqlBulkImportService: " + sqlStatement); + + var pgsqlConnection = connection as NpgsqlConnection; + using (var writer = pgsqlConnection.BeginBinaryImport(sqlStatement)) + { + //writes each data row as datastream into pgsql database + foreach (DataRow dataRow in dataTable.Rows) + { + writer.StartRow(); + foreach (DataColumn dataColumn in dataTable.Columns) + { + if (dataRow.IsNull(dataColumn.ColumnName)) + { + writer.Write(DBNull.Value); + continue; + } + + if (!destinationSchema.ContainsKey(dataColumn.ColumnName)) + continue; + + var dataType = destinationSchema[dataColumn.ColumnName.ToLower()].DataType; + + if (dataType == "boolean" || dataType == "bit" || dataType == "bit varying") + { + writer.Write(bool.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Boolean); + continue; + } + else if (dataType == "smallint" || dataType == "int2") + { + writer.Write(short.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Smallint); + continue; + } + else if (dataType == "integer" || dataType == "int4") + { + writer.Write(int.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Integer); + continue; + } + else if (dataType == "bigint" || dataType == "int8") + { + writer.Write(long.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Bigint); + continue; + } + else if (dataType == "real") + { + writer.Write(float.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Real); + continue; + } + else if (dataType == "double precision") + { + writer.Write(double.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Double); + continue; + } + else if (dataType == "numeric" || dataType == "money") + { + writer.Write(decimal.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Numeric); + continue; + } + else if (dataType == "uuid") + { + writer.Write(Guid.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Uuid); + continue; + } + else if (dataType == "date") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Date); + continue; + } + else if (dataType == "interval") + { + writer.Write(TimeSpan.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Interval); + continue; + } + else if (dataType == "timestamp" || dataType == "timestamp without time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Timestamp); + continue; + } + else if (dataType == "timestamp with time zone") + { + writer.Write(DateTimeOffset.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimestampTz); + continue; + } + else if (dataType == "time" || dataType == "time without time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.Time); + continue; + } + else if (dataType == "time with time zone") + { + writer.Write(DateTime.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.TimeTz); + continue; + } + else if (dataType == "name") + { + writer.Write(dataRow[dataColumn.ColumnName].ToString(), NpgsqlDbType.Name); + continue; + } + else if (dataType == "(internal) char") + { + writer.Write(byte.Parse(dataRow[dataColumn.ColumnName].ToString()), NpgsqlDbType.InternalChar); + continue; + } + else if (dataType == "text" + || dataType == "character varying" + || dataType == "character" + || dataType == "citext" + || dataType == "json" + || dataType == "jsonb" + || dataType == "xml") + { + writer.Write(dataRow[dataColumn.ColumnName].ToString()); + continue; + } + else + { + //not supported types: lseg,path,polygon,line,circle,box,hstore,cidr,inet,macaddr,tsquery,tsvector,bytea,oid,xid,cid,oidvector,composite types,range types,enum types,array types + throw new NotSupportedException($"PostgreSqlBulkImportService: Data type '{dataType}' on destination table {schemaName}.{tableName} is not support for bulk import operations."); + } + } + } + + //wraps up everything, closes the stream + writer.Complete(); + } + } + + //https://www.npgsql.org/doc/types/basic.html + private IDictionary GetDestinationSchema(string schemaName, string tableName) + { + var result = new Dictionary(); + using (var connection = new NpgsqlConnection(_connectionString)) + { + connection.Open(); + + var command = connection.CreateCommand(); + command.CommandType = CommandType.Text; + command.CommandText = $"SELECT column_name, data_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '{schemaName}' AND TABLE_NAME = '{tableName}'"; + command.CommandTimeout = 0; + + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + result.Add(reader.GetString(0), new ColumnDefinition + { + ColumnName = reader.GetString(0), + DataType = reader.GetString(1) + }); + } + } + } + + return result; + } + } + + public class ColumnDefinition + { + public string ColumnName { get; set; } + public string DataType { get; set; } + } +} +