Merge pull request ClickHouse#35928 from ClickHouse/backport/22.3/35733

Backport ClickHouse#35733 to 22.3: Added settings for insert of invalid IPv6, IPv4 values
Kyligence · Apr 5, 2022 · 82735cb · 82735cb
2 parents ad0a62d + b4cd668
commit 82735cb
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 4 deletions.
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
@@ -632,6 +632,8 @@ class IColumn;
     M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \
     M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \
     \
+    M(Bool, input_format_ipv4_default_on_conversion_error, false, "Deserialization of IPv4 will use default values instead of throwing exception on conversion error.", 0) \
+    M(Bool, input_format_ipv6_default_on_conversion_error, false, "Deserialization of IPV6 will use default values instead of throwing exception on conversion error.", 0) \
     M(String, bool_true_representation, "true", "Text to represent bool value in TSV/CSV formats.", 0) \
     M(String, bool_false_representation, "false", "Text to represent bool value in TSV/CSV formats.", 0) \
     \

diff --git a/src/DataTypes/Serializations/SerializationIP.cpp b/src/DataTypes/Serializations/SerializationIP.cpp
@@ -6,6 +6,8 @@
 #include <Common/formatIPv6.h>
 #include <IO/WriteBuffer.h>
 #include <IO/ReadBuffer.h>
+#include <Formats/FormatSettings.h>
+
 
 namespace DB
 {
@@ -47,9 +49,11 @@ void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, con
     char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'};
     istr.read(buffer, sizeof(buffer) - 1);
     UInt32 ipv4_value = 0;
-    if (!parseIPv4(buffer, reinterpret_cast<unsigned char *>(&ipv4_value)))
+
+    bool parse_result = parseIPv4(buffer, reinterpret_cast<unsigned char *>(&ipv4_value));
+    if (!parse_result && !settings.input_format_ipv4_default_on_conversion_error)
     {
-        throw Exception("Invalid IPv4 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
+        throw Exception("Invalid IPv4 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
     }
 
     col->insert(ipv4_value);
@@ -89,9 +93,11 @@ void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, con
     istr.read(buffer, sizeof(buffer) - 1);
 
     std::string ipv6_value(IPV6_BINARY_LENGTH, '\0');
-    if (!parseIPv6(buffer, reinterpret_cast<unsigned char *>(ipv6_value.data())))
+
+    bool parse_result = parseIPv6(buffer, reinterpret_cast<unsigned char *>(ipv6_value.data()));
+    if (!parse_result && !settings.input_format_ipv6_default_on_conversion_error)
     {
-        throw Exception("Invalid IPv6 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
+        throw Exception("Invalid IPv6 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
     }
 
     col->insertString(ipv6_value);

diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
@@ -77,6 +77,8 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
     format_settings.date_time_input_format = settings.date_time_input_format;
     format_settings.date_time_output_format = settings.date_time_output_format;
+    format_settings.input_format_ipv4_default_on_conversion_error = settings.input_format_ipv4_default_on_conversion_error;
+    format_settings.input_format_ipv6_default_on_conversion_error = settings.input_format_ipv6_default_on_conversion_error;
     format_settings.bool_true_representation = settings.bool_true_representation;
     format_settings.bool_false_representation = settings.bool_false_representation;
     format_settings.enable_streaming = settings.output_format_enable_streaming;

diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
@@ -66,6 +66,9 @@ struct FormatSettings
 
     DateTimeOutputFormat date_time_output_format = DateTimeOutputFormat::Simple;
 
+    bool input_format_ipv4_default_on_conversion_error = false;
+    bool input_format_ipv6_default_on_conversion_error = false;
+
     UInt64 input_allow_errors_num = 0;
     Float32 input_allow_errors_ratio = 0;
 

diff --git a/tests/queries/0_stateless/02244_ip_address_invalid_insert.reference b/tests/queries/0_stateless/02244_ip_address_invalid_insert.reference
@@ -0,0 +1,10 @@
+1.1.1.1	1.1.1.1
+	0.0.0.0
+1.1.1.1	1.1.1.1
+	0.0.0.0
+fe80::9801:43ff:fe1f:7690	fe80::9801:43ff:fe1f:7690
+1.1.1.1	::
+	::
+fe80::9801:43ff:fe1f:7690	fe80::9801:43ff:fe1f:7690
+1.1.1.1	::ffff:1.1.1.1
+	::
diff --git a/tests/queries/0_stateless/02244_ip_address_invalid_insert.sql b/tests/queries/0_stateless/02244_ip_address_invalid_insert.sql
@@ -0,0 +1,81 @@
+DROP TABLE IF EXISTS test_table_ipv4;
+CREATE TABLE test_table_ipv4
+(
+    ip String,
+    ipv4 IPv4
+) ENGINE = TinyLog;
+
+INSERT INTO test_table_ipv4 VALUES ('1.1.1.1', '1.1.1.1'), ('', ''); --{clientError 441}
+
+SET input_format_ipv4_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv4 VALUES ('1.1.1.1', '1.1.1.1'), ('', '');
+SELECT ip, ipv4 FROM test_table_ipv4;
+
+SET input_format_ipv4_default_on_conversion_error = 0;
+
+DROP TABLE test_table_ipv4;
+
+DROP TABLE IF EXISTS test_table_ipv4_materialized;
+CREATE TABLE test_table_ipv4_materialized
+(
+    ip String,
+    ipv6 IPv4 MATERIALIZED toIPv4(ip)
+) ENGINE = TinyLog;
+
+INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), (''); --{serverError 441}
+
+SET input_format_ipv4_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), (''); --{serverError 441}
+
+SET cast_ipv4_ipv6_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv4_materialized(ip) VALUES ('1.1.1.1'), ('');
+SELECT ip, ipv6 FROM test_table_ipv4_materialized;
+
+SET input_format_ipv4_default_on_conversion_error = 0;
+SET cast_ipv4_ipv6_default_on_conversion_error = 0;
+
+DROP TABLE test_table_ipv4_materialized;
+
+DROP TABLE IF EXISTS test_table_ipv6;
+CREATE TABLE test_table_ipv6
+(
+    ip String,
+    ipv6 IPv6
+) ENGINE = TinyLog;
+
+INSERT INTO test_table_ipv6 VALUES ('fe80::9801:43ff:fe1f:7690', 'fe80::9801:43ff:fe1f:7690'), ('1.1.1.1', '1.1.1.1'), ('', ''); --{clientError 441}
+
+SET input_format_ipv6_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv6 VALUES ('fe80::9801:43ff:fe1f:7690', 'fe80::9801:43ff:fe1f:7690'), ('1.1.1.1', '1.1.1.1'), ('', '');
+SELECT ip, ipv6 FROM test_table_ipv6;
+
+SET input_format_ipv6_default_on_conversion_error = 0;
+
+DROP TABLE test_table_ipv6;
+
+DROP TABLE IF EXISTS test_table_ipv6_materialized;
+CREATE TABLE test_table_ipv6_materialized
+(
+    ip String,
+    ipv6 IPv6 MATERIALIZED toIPv6(ip)
+) ENGINE = TinyLog;
+
+INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), (''); --{serverError 441}
+
+SET input_format_ipv6_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), (''); --{serverError 441}
+
+SET cast_ipv4_ipv6_default_on_conversion_error = 1;
+
+INSERT INTO test_table_ipv6_materialized(ip) VALUES ('fe80::9801:43ff:fe1f:7690'), ('1.1.1.1'), ('');
+SELECT ip, ipv6 FROM test_table_ipv6_materialized;
+
+SET input_format_ipv6_default_on_conversion_error = 0;
+SET cast_ipv4_ipv6_default_on_conversion_error = 0;
+
+DROP TABLE test_table_ipv6_materialized;