Merge remote-tracking branch 'origin/master' into changelog-22.3

docs/tools/requirements.txt

@@ -1,4 +1,4 @@
-Babel==2.8.0
+Babel==2.9.1
 backports-abc==0.5
 backports.functools-lru-cache==1.6.1
 beautifulsoup4==4.9.1
@@ -10,22 +10,22 @@ cssmin==0.2.0
 future==0.18.2
 htmlmin==0.1.12
 idna==2.10
-Jinja2>=2.11.3
+Jinja2>=3.0.3
 jinja2-highlight==0.6.1
 jsmin==3.0.0
-livereload==2.6.2
+livereload==2.6.3
 Markdown==3.3.2
-MarkupSafe==1.1.1
+MarkupSafe==2.1.0
 mkdocs==1.1.2
 mkdocs-htmlproofer-plugin==0.0.3
 mkdocs-macros-plugin==0.4.20
-nltk==3.5
+nltk==3.7
 nose==1.3.7
 protobuf==3.14.0
 numpy==1.21.2
 pymdown-extensions==8.0
 python-slugify==4.0.1
-PyYAML==5.4.1
+PyYAML==6.0
 repackage==0.7.3
 requests==2.25.1
 singledispatch==3.4.0.3
@@ -34,5 +34,6 @@ soupsieve==2.0.1
 termcolor==1.1.0
 tornado==6.1
 Unidecode==1.1.1
-urllib3>=1.26.5
-Pygments>=2.7.4
+urllib3>=1.26.8
+Pygments>=2.11.2
+

programs/local/LocalServer.cpp

@@ -304,8 +304,8 @@ void LocalServer::setupUsers()
 
     ConfigurationPtr users_config;
     auto & access_control = global_context->getAccessControl();
-    access_control.setPlaintextPasswordSetting(config().getBool("allow_plaintext_password", true));
-    access_control.setNoPasswordSetting(config().getBool("allow_no_password", true));
+    access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true));
+    access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true));
     if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml"))
     {
         const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));

programs/server/Server.cpp

@@ -1069,9 +1069,10 @@ if (ThreadFuzzer::instance().isEffective())
     auto & access_control = global_context->getAccessControl();
     if (config().has("custom_settings_prefixes"))
         access_control.setCustomSettingsPrefixes(config().getString("custom_settings_prefixes"));
-    ///set the allow_plaintext_and_no_password setting in context.
-    access_control.setPlaintextPasswordSetting(config().getBool("allow_plaintext_password", true));
-    access_control.setNoPasswordSetting(config().getBool("allow_no_password", true));
+
+    access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true));
+    access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true));
+
     /// Initialize access storages.
     try
     {

programs/server/config.xml

@@ -243,7 +243,7 @@
                   openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
                  Only file format with BEGIN DH PARAMETERS is supported.
               -->
-	    <!-- <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>-->
+            <!-- <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>-->
             <verificationMode>none</verificationMode>
             <loadDefaultCAFile>true</loadDefaultCAFile>
             <cacheSessions>true</cacheSessions>
@@ -368,7 +368,7 @@
     <!-- Path to temporary data for processing hard queries. -->
     <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
 
-    <!-- Disable AuthType Plaintext_password and No_password for ACL. --> 
+    <!-- Disable AuthType plaintext_password and no_password for ACL. -->
     <!-- <allow_plaintext_password>0</allow_plaintext_password> -->
     <!-- <allow_no_password>0</allow_no_password> -->`
 

src/Access/AccessControl.cpp

@@ -173,7 +173,8 @@ void AccessControl::addUsersConfigStorage(const String & storage_name_, const Po
     auto check_setting_name_function = [this](const std::string_view & setting_name) { checkSettingNameIsAllowed(setting_name); };
     auto is_no_password_allowed_function = [this]() -> bool { return isNoPasswordAllowed(); };
     auto is_plaintext_password_allowed_function = [this]() -> bool { return isPlaintextPasswordAllowed(); };
-    auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function,is_no_password_allowed_function,is_plaintext_password_allowed_function);
+    auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function,
+                                                                  is_no_password_allowed_function, is_plaintext_password_allowed_function);
     new_storage->setConfig(users_config_);
     addStorage(new_storage);
     LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}",
@@ -209,7 +210,8 @@ void AccessControl::addUsersConfigStorage(
     auto check_setting_name_function = [this](const std::string_view & setting_name) { checkSettingNameIsAllowed(setting_name); };
     auto is_no_password_allowed_function = [this]() -> bool { return isNoPasswordAllowed(); };
     auto is_plaintext_password_allowed_function = [this]() -> bool { return isPlaintextPasswordAllowed(); };
-    auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function,is_no_password_allowed_function,is_plaintext_password_allowed_function);
+    auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function,
+                                                                  is_no_password_allowed_function, is_plaintext_password_allowed_function);
     new_storage->load(users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
     addStorage(new_storage);
     LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
@@ -411,7 +413,8 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne
 {
     try
     {
-        return MultipleAccessStorage::authenticate(credentials, address, *external_authenticators,allow_no_password, allow_plaintext_password);
+        return MultipleAccessStorage::authenticate(credentials, address, *external_authenticators, allow_no_password,
+                                                   allow_plaintext_password);
     }
     catch (...)
     {
@@ -447,23 +450,35 @@ void AccessControl::setCustomSettingsPrefixes(const String & comma_separated_pre
     setCustomSettingsPrefixes(prefixes);
 }
 
-void AccessControl::setPlaintextPasswordSetting(bool allow_plaintext_password_)
+bool AccessControl::isSettingNameAllowed(const std::string_view setting_name) const
 {
-    allow_plaintext_password = allow_plaintext_password_;
+    return custom_settings_prefixes->isSettingNameAllowed(setting_name);
 }
-void AccessControl::setNoPasswordSetting(bool allow_no_password_)
+
+void AccessControl::checkSettingNameIsAllowed(const std::string_view setting_name) const
+{
+    custom_settings_prefixes->checkSettingNameIsAllowed(setting_name);
+}
+
+
+void AccessControl::setNoPasswordAllowed(bool allow_no_password_)
 {
     allow_no_password = allow_no_password_;
 }
 
-bool AccessControl::isSettingNameAllowed(const std::string_view & setting_name) const
+bool AccessControl::isNoPasswordAllowed() const
 {
-    return custom_settings_prefixes->isSettingNameAllowed(setting_name);
+    return allow_no_password;
 }
 
-void AccessControl::checkSettingNameIsAllowed(const std::string_view & setting_name) const
+void AccessControl::setPlaintextPasswordAllowed(bool allow_plaintext_password_)
 {
-    custom_settings_prefixes->checkSettingNameIsAllowed(setting_name);
+    allow_plaintext_password = allow_plaintext_password_;
+}
+
+bool AccessControl::isPlaintextPasswordAllowed() const
+{
+    return allow_plaintext_password;
 }
 
 
@@ -550,15 +565,6 @@ std::vector<QuotaUsage> AccessControl::getAllQuotasUsage() const
     return quota_cache->getAllQuotasUsage();
 }
 
-bool AccessControl::isPlaintextPasswordAllowed() const
-{
-    return allow_plaintext_password;
-}
-
-bool AccessControl::isNoPasswordAllowed() const
-{
-    return allow_no_password;
-}
 
 std::shared_ptr<const EnabledSettings> AccessControl::getEnabledSettings(
     const UUID & user_id,

src/Access/AccessControl.h

@@ -49,8 +49,6 @@ class AccessControl : public MultipleAccessStorage
 public:
     AccessControl();
     ~AccessControl() override;
-    std::atomic_bool allow_plaintext_password;
-    std::atomic_bool allow_no_password;
 
     /// Parses access entities from a configuration loaded from users.xml.
     /// This function add UsersConfigAccessStorage if it wasn't added before.
@@ -113,12 +111,16 @@ class AccessControl : public MultipleAccessStorage
     /// This function also enables custom prefixes to be used.
     void setCustomSettingsPrefixes(const Strings & prefixes);
     void setCustomSettingsPrefixes(const String & comma_separated_prefixes);
-    bool isSettingNameAllowed(const std::string_view & name) const;
-    void checkSettingNameIsAllowed(const std::string_view & name) const;
+    bool isSettingNameAllowed(const std::string_view name) const;
+    void checkSettingNameIsAllowed(const std::string_view name) const;
 
-    //sets allow_plaintext_password and allow_no_password setting
-    void setPlaintextPasswordSetting(const bool allow_plaintext_password_);
-    void setNoPasswordSetting(const bool allow_no_password_);
+    /// Allows users without password (by default it's allowed).
+    void setNoPasswordAllowed(const bool allow_no_password_);
+    bool isNoPasswordAllowed() const;
+
+    /// Allows users with plaintext password (by default it's allowed).
+    void setPlaintextPasswordAllowed(const bool allow_plaintext_password_);
+    bool isPlaintextPasswordAllowed() const;
 
     UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const;
     void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
@@ -153,9 +155,6 @@ class AccessControl : public MultipleAccessStorage
 
     std::vector<QuotaUsage> getAllQuotasUsage() const;
 
-    bool isPlaintextPasswordAllowed() const;
-    bool isNoPasswordAllowed() const;
-
     std::shared_ptr<const EnabledSettings> getEnabledSettings(
         const UUID & user_id,
         const SettingsProfileElements & settings_from_user,
@@ -177,6 +176,8 @@ class AccessControl : public MultipleAccessStorage
     std::unique_ptr<SettingsProfilesCache> settings_profiles_cache;
     std::unique_ptr<ExternalAuthenticators> external_authenticators;
     std::unique_ptr<CustomSettingsPrefixes> custom_settings_prefixes;
+    std::atomic_bool allow_plaintext_password = true;
+    std::atomic_bool allow_no_password = true;
 };
 
 }

src/Access/AccessEntityIO.cpp

@@ -120,7 +120,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition)
             if (res)
                 throw Exception("Two access entities attached in the same file", ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION);
             res = user = std::make_unique<User>();
-            InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query);
+            InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true);
         }
         else if (auto * create_role_query = query->as<ASTCreateRoleQuery>())
         {

src/Access/IAccessStorage.cpp

@@ -441,7 +441,9 @@ void IAccessStorage::notify(const Notifications & notifications)
 UUID IAccessStorage::authenticate(
     const Credentials & credentials,
     const Poco::Net::IPAddress & address,
-    const ExternalAuthenticators & external_authenticators, bool allow_no_password, bool allow_plaintext_password) const
+    const ExternalAuthenticators & external_authenticators,
+    bool allow_no_password,
+    bool allow_plaintext_password) const
 {
     return *authenticateImpl(credentials, address, external_authenticators, /* throw_if_user_not_exists = */ true, allow_no_password, allow_plaintext_password);
 }
@@ -451,7 +453,9 @@ std::optional<UUID> IAccessStorage::authenticate(
     const Credentials & credentials,
     const Poco::Net::IPAddress & address,
     const ExternalAuthenticators & external_authenticators,
-    bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const
+    bool throw_if_user_not_exists,
+    bool allow_no_password,
+    bool allow_plaintext_password) const
 {
     return authenticateImpl(credentials, address, external_authenticators, throw_if_user_not_exists, allow_no_password, allow_plaintext_password);
 }
@@ -461,16 +465,21 @@ std::optional<UUID> IAccessStorage::authenticateImpl(
     const Credentials & credentials,
     const Poco::Net::IPAddress & address,
     const ExternalAuthenticators & external_authenticators,
-    bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const
+    bool throw_if_user_not_exists,
+    bool allow_no_password,
+    bool allow_plaintext_password) const
 {
     if (auto id = find<User>(credentials.getUserName()))
     {
         if (auto user = tryRead<User>(*id))
         {
             if (!isAddressAllowed(*user, address))
                 throwAddressNotAllowed(address);
-            if (isNoPasswordAllowed(*user, allow_no_password) || isPlaintextPasswordAllowed(*user, allow_plaintext_password))
-                throwPasswordTypeNotAllowed();
+
+            auto auth_type = user->auth_data.getType();
+            if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) ||
+                ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password))
+                throwAuthenticationTypeNotAllowed(auth_type);
 
             if (!areCredentialsValid(*user, credentials, external_authenticators))
                 throwInvalidCredentials();
@@ -506,15 +515,6 @@ bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddr
     return user.allowed_client_hosts.contains(address);
 }
 
-bool IAccessStorage::isPlaintextPasswordAllowed(const User & user, bool allow_plaintext_password)
-{
-    return !allow_plaintext_password && user.auth_data.getType() == AuthenticationType::PLAINTEXT_PASSWORD;
-}
-
-bool IAccessStorage::isNoPasswordAllowed(const User & user, bool allow_no_password)
-{
-    return !allow_no_password && user.auth_data.getType() == AuthenticationType::NO_PASSWORD;
-}
 
 UUID IAccessStorage::generateRandomID()
 {
@@ -610,11 +610,12 @@ void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address
     throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
 }
 
-void IAccessStorage::throwPasswordTypeNotAllowed()
+void IAccessStorage::throwAuthenticationTypeNotAllowed(AuthenticationType auth_type)
 {
     throw Exception(
-        "Authentication denied for users configured with AuthType PLAINTEXT_PASSWORD and NO_PASSWORD. Please check with Clickhouse admin to allow allow  PLAINTEXT_PASSWORD and NO_PASSWORD through server configuration ",
-        ErrorCodes::AUTHENTICATION_FAILED);
+        ErrorCodes::AUTHENTICATION_FAILED,
+        "Authentication type {} is not allowed, check the setting allow_{} in the server configuration",
+        toString(auth_type), AuthenticationTypeInfo::get(auth_type).name);
 }
 void IAccessStorage::throwInvalidCredentials()
 {

src/Access/IAccessStorage.h

@@ -18,6 +18,7 @@ namespace DB
 struct User;
 class Credentials;
 class ExternalAuthenticators;
+enum class AuthenticationType;
 
 /// Contains entities, i.e. instances of classes derived from IAccessEntity.
 /// The implementations of this class MUST be thread-safe.
@@ -148,7 +149,7 @@ class IAccessStorage
 
     /// Finds a user, check the provided credentials and returns the ID of the user if they are valid.
     /// Throws an exception if no such user or credentials are invalid.
-    UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool allow_no_password=true, bool allow_plaintext_password=true) const;
+    UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool allow_no_password, bool allow_plaintext_password) const;
     std::optional<UUID> authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
 
 protected:
@@ -164,8 +165,6 @@ class IAccessStorage
     virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
     virtual bool areCredentialsValid(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const;
     virtual bool isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const;
-    static bool isPlaintextPasswordAllowed(const User & user, bool allow_plaintext_password) ;
-    static bool isNoPasswordAllowed(const User & user, bool allow_no_password);
     static UUID generateRandomID();
     Poco::Logger * getLogger() const;
     static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); }
@@ -181,7 +180,7 @@ class IAccessStorage
     [[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const;
     [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
     [[noreturn]] static void throwInvalidCredentials();
-    [[noreturn]] static void throwPasswordTypeNotAllowed();
+    [[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type);
     using Notification = std::tuple<OnChangedHandler, UUID, AccessEntityPtr>;
     using Notifications = std::vector<Notification>;
     static void notify(const Notifications & notifications);

src/Access/LDAPAccessStorage.cpp

@@ -481,7 +481,9 @@ std::optional<UUID> LDAPAccessStorage::authenticateImpl(
     const Credentials & credentials,
     const Poco::Net::IPAddress & address,
     const ExternalAuthenticators & external_authenticators,
-    bool throw_if_user_not_exists,bool allow_no_password __attribute__((unused)), bool allow_plaintext_password __attribute__((unused))) const
+    bool throw_if_user_not_exists,
+    bool /* allow_no_password */,
+    bool /* allow_plaintext_password */) const
 {
     std::scoped_lock lock(mutex);
     auto id = memory_storage.find<User>(credentials.getUserName());

src/Access/MultipleAccessStorage.cpp

@@ -449,14 +449,20 @@ void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock
 }
 
 
-std::optional<UUID> MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists,bool allow_no_password, bool allow_plaintext_password) const
+std::optional<UUID>
+MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address,
+                                        const ExternalAuthenticators & external_authenticators,
+                                        bool throw_if_user_not_exists,
+                                        bool allow_no_password, bool allow_plaintext_password) const
 {
     auto storages = getStoragesInternal();
     for (size_t i = 0; i != storages->size(); ++i)
     {
         const auto & storage = (*storages)[i];
         bool is_last_storage = (i == storages->size() - 1);
-        auto id = storage->authenticate(credentials, address, external_authenticators, (throw_if_user_not_exists && is_last_storage), allow_no_password, allow_plaintext_password);
+        auto id = storage->authenticate(credentials, address, external_authenticators,
+                                        (throw_if_user_not_exists && is_last_storage),
+                                        allow_no_password, allow_plaintext_password);
         if (id)
         {
             std::lock_guard lock{mutex};