From b4cd668a439d792e4c95160ad22c1f8d8f7cc4b7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 4 Apr 2022 16:08:46 +0000 Subject: [PATCH] Backport #35733 to 22.3: Added settings for insert of invalid IPv6, IPv4 values --- src/Core/Settings.h | 2 + .../Serializations/SerializationIP.cpp | 14 +++- src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 3 + .../02244_ip_address_invalid_insert.reference | 10 +++ .../02244_ip_address_invalid_insert.sql | 81 +++++++++++++++++++ 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02244_ip_address_invalid_insert.reference create mode 100644 tests/queries/0_stateless/02244_ip_address_invalid_insert.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 058536ebf250..0c69b864d528 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -632,6 +632,8 @@ class IColumn; M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ \ + M(Bool, input_format_ipv4_default_on_conversion_error, false, "Deserialization of IPv4 will use default values instead of throwing exception on conversion error.", 0) \ + M(Bool, input_format_ipv6_default_on_conversion_error, false, "Deserialization of IPV6 will use default values instead of throwing exception on conversion error.", 0) \ M(String, bool_true_representation, "true", "Text to represent bool value in TSV/CSV formats.", 0) \ M(String, bool_false_representation, "false", "Text to represent bool value in TSV/CSV formats.", 0) \ \ diff --git a/src/DataTypes/Serializations/SerializationIP.cpp b/src/DataTypes/Serializations/SerializationIP.cpp index 916edced0a31..ed0e9d54415c 100644 --- a/src/DataTypes/Serializations/SerializationIP.cpp +++ b/src/DataTypes/Serializations/SerializationIP.cpp @@ -6,6 +6,8 @@ #include #include #include +#include + namespace DB { @@ -47,9 +49,11 @@ void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, con char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'}; istr.read(buffer, sizeof(buffer) - 1); UInt32 ipv4_value = 0; - if (!parseIPv4(buffer, reinterpret_cast(&ipv4_value))) + + bool parse_result = parseIPv4(buffer, reinterpret_cast(&ipv4_value)); + if (!parse_result && !settings.input_format_ipv4_default_on_conversion_error) { - throw Exception("Invalid IPv4 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); + throw Exception("Invalid IPv4 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); } col->insert(ipv4_value); @@ -89,9 +93,11 @@ void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, con istr.read(buffer, sizeof(buffer) - 1); std::string ipv6_value(IPV6_BINARY_LENGTH, '\0'); - if (!parseIPv6(buffer, reinterpret_cast(ipv6_value.data()))) + + bool parse_result = parseIPv6(buffer, reinterpret_cast(ipv6_value.data())); + if (!parse_result && !settings.input_format_ipv6_default_on_conversion_error) { - throw Exception("Invalid IPv6 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); + throw Exception("Invalid IPv6 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); } col->insertString(ipv6_value); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 08554cf7e073..6bd8d2c460d2 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -77,6 +77,8 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; format_settings.date_time_input_format = settings.date_time_input_format; format_settings.date_time_output_format = settings.date_time_output_format; + format_settings.input_format_ipv4_default_on_conversion_error = settings.input_format_ipv4_default_on_conversion_error; + format_settings.input_format_ipv6_default_on_conversion_error = settings.input_format_ipv6_default_on_conversion_error; format_settings.bool_true_representation = settings.bool_true_representation; format_settings.bool_false_representation = settings.bool_false_representation; format_settings.enable_streaming = settings.output_format_enable_streaming; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 4881c1a43c8e..a99d7dc46616 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -66,6 +66,9 @@ struct FormatSettings DateTimeOutputFormat date_time_output_format = DateTimeOutputFormat::Simple; + bool input_format_ipv4_default_on_conversion_error = false; + bool input_format_ipv6_default_on_conversion_error = false; + UInt64 input_allow_errors_num = 0; Float32 input_allow_errors_ratio = 0; diff --git a/tests/queries/0_stateless/02244_ip_address_invalid_insert.reference b/tests/queries/0_stateless/02244_ip_address_invalid_insert.reference new file mode 100644 index 000000000000..60e6a5da083b --- /dev/null +++ b/tests/queries/0_stateless/02244_ip_address_invalid_insert.reference @@ -0,0 +1,10 @@ +1.1.1.1 1.1.1.1 + 0.0.0.0 +1.1.1.1 1.1.1.1 + 0.0.0.0 +fe80::9801:43ff:fe1f:7690 fe80::9801:43ff:fe1f:7690 +1.1.1.1 :: + :: +fe80::9801:43ff:fe1f:7690 fe80::9801:43ff:fe1f:7690 +1.1.1.1 ::ffff:1.1.1.1 + :: diff --git a/tests/queries/0_stateless/02244_ip_address_invalid_insert.sql b/tests/queries/0_stateless/02244_ip_address_invalid_insert.sql new file mode 100644 index 000000000000..4057b9b2d98d --- /dev/null +++ b/tests/queries/0_stateless/02244_ip_address_invalid_insert.sql @@ -0,0 +1,81 @@ +DROP TABLE IF EXISTS test_table_ipv4; +CREATE TABLE test_table_ipv4 +( + ip String, + ipv4 IPv4 +) ENGINE = TinyLog; + +INSERT INTO test_table_ipv4 VALUES ('1.1.1.1', '1.1.1.1'), ('', ''); --{clientError 441} + +SET input_format_ipv4_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv4 VALUES ('1.1.1.1', '1.1.1.1'), ('', ''); +SELECT ip, ipv4 FROM test_table_ipv4; + +SET input_format_ipv4_default_on_conversion_error = 0; + +DROP TABLE test_table_ipv4; + +DROP TABLE IF EXISTS test_table_ipv4_materialized; +CREATE TABLE test_table_ipv4_materialized +( + ip String, + ipv6 IPv4 MATERIALIZED toIPv4(ip) +) ENGINE = TinyLog; + +INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), (''); --{serverError 441} + +SET input_format_ipv4_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), (''); --{serverError 441} + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), (''); +SELECT ip, ipv6 FROM test_table_ipv4_materialized; + +SET input_format_ipv4_default_on_conversion_error = 0; +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +DROP TABLE test_table_ipv4_materialized; + +DROP TABLE IF EXISTS test_table_ipv6; +CREATE TABLE test_table_ipv6 +( + ip String, + ipv6 IPv6 +) ENGINE = TinyLog; + +INSERT INTO test_table_ipv6 VALUES ('fe80::9801:43ff:fe1f:7690', 'fe80::9801:43ff:fe1f:7690'), ('1.1.1.1', '1.1.1.1'), ('', ''); --{clientError 441} + +SET input_format_ipv6_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv6 VALUES ('fe80::9801:43ff:fe1f:7690', 'fe80::9801:43ff:fe1f:7690'), ('1.1.1.1', '1.1.1.1'), ('', ''); +SELECT ip, ipv6 FROM test_table_ipv6; + +SET input_format_ipv6_default_on_conversion_error = 0; + +DROP TABLE test_table_ipv6; + +DROP TABLE IF EXISTS test_table_ipv6_materialized; +CREATE TABLE test_table_ipv6_materialized +( + ip String, + ipv6 IPv6 MATERIALIZED toIPv6(ip) +) ENGINE = TinyLog; + +INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), (''); --{serverError 441} + +SET input_format_ipv6_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), (''); --{serverError 441} + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + +INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), (''); +SELECT ip, ipv6 FROM test_table_ipv6_materialized; + +SET input_format_ipv6_default_on_conversion_error = 0; +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +DROP TABLE test_table_ipv6_materialized;