[CELEBORN-1521] Introduce celeborn-spi module for authentication exte…

…nsions ### What changes were proposed in this pull request? Introduce celeborn-spi module for authentication extensions. ### Why are the changes needed? Address comments: apache#2632 (comment) ### Does this PR introduce _any_ user-facing change? No, this interface has not been released. ### How was this patch tested? UT. Closes apache#2644 from turboFei/celeborn_spi. Authored-by: Wang, Fei <[email protected]> Signed-off-by: Wang, Fei <[email protected]>
ChenYi015 · Jul 25, 2024 · ea6617c · ea6617c
1 parent c0ca952
commit ea6617c
Show file tree

Hide file tree

Showing 20 changed files with 172 additions and 56 deletions.
diff --git a/build/release/release.sh b/build/release/release.sh
@@ -128,8 +128,8 @@ upload_nexus_staging() {
   echo "Deploying celeborn-openapi-client_2.12"
   ${PROJECT_DIR}/build/sbt "clean;celeborn-openapi-client/publishSigned"
 
-  echo "Deploying celeborn-common_2.12"
-  ${PROJECT_DIR}/build/sbt "clean;celeborn-common/publishSigned"
+  echo "Deploying celeborn-spi"
+  ${PROJECT_DIR}/build/sbt "clean;celeborn-spi/publishSigned"
 }
 
 finalize_svn() {

diff --git a/common/pom.xml b/common/pom.xml
@@ -30,6 +30,11 @@
   <name>Celeborn Common</name>
 
   <dependencies>
+    <dependency>
+      <groupId>org.apache.celeborn</groupId>
+      <artifactId>celeborn-spi</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-core</artifactId>

diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -2298,7 +2298,7 @@ object CelebornConf extends Logging {
       .categories("master")
       .version("0.6.0")
       .doc("User-defined password authentication implementation of " +
-        "org.apache.celeborn.common.authentication.PasswdAuthenticationProvider")
+        "org.apache.celeborn.spi.authentication.PasswdAuthenticationProvider")
       .stringConf
       .createWithDefault(classOf[AnonymousAuthenticationProviderImpl].getName)
 
@@ -2307,7 +2307,7 @@ object CelebornConf extends Logging {
       .categories("master")
       .version("0.6.0")
       .doc("User-defined token authentication implementation of " +
-        "org.apache.celeborn.common.authentication.TokenAuthenticationProvider")
+        "org.apache.celeborn.spi.authentication.TokenAuthenticationProvider")
       .stringConf
       .createWithDefault(classOf[AnonymousAuthenticationProviderImpl].getName)
 

diff --git a/...scala/org/apache/celeborn/common/authentication/AnonymousAuthenticationProviderImpl.scala b/...scala/org/apache/celeborn/common/authentication/AnonymousAuthenticationProviderImpl.scala
@@ -19,6 +19,8 @@ package org.apache.celeborn.common.authentication
 
 import java.security.Principal
 
+import org.apache.celeborn.spi.authentication.{PasswdAuthenticationProvider, PasswordCredential, TokenAuthenticationProvider, TokenCredential}
+
 class AnonymousAuthenticationProviderImpl extends PasswdAuthenticationProvider
   with TokenAuthenticationProvider {
   override def authenticate(credential: PasswordCredential): Principal = {

diff --git a/common/src/main/scala/org/apache/celeborn/common/authentication/Credential.scala b/common/src/main/scala/org/apache/celeborn/common/authentication/Credential.scala
@@ -17,26 +17,17 @@
 
 package org.apache.celeborn.common.authentication
 
-trait PasswordCredential {
-  def username: String
-  def password: String
-  def extraInfo: Map[String, String] = Map.empty
-}
+import java.util.{Collections, Map => JMap}
+
+import org.apache.celeborn.spi.authentication.{PasswordCredential, TokenCredential}
 
 case class DefaultPasswordCredential(
     username: String,
     password: String,
-    override val extraInfo: Map[String, String] = Map.empty) extends PasswordCredential
-
-trait TokenCredential {
-  def token: String
-  def extraInfo: Map[String, String] = Map.empty
-}
+    override val extraInfo: JMap[String, String] = Collections.emptyMap())
+  extends PasswordCredential
 
 case class DefaultTokenCredential(
     token: String,
-    override val extraInfo: Map[String, String] = Map.empty) extends TokenCredential
-
-object Credential {
-  val CLIENT_IP_KEY = "clientIp"
-}
+    override val extraInfo: JMap[String, String] = Collections.emptyMap())
+  extends TokenCredential
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
@@ -45,8 +45,8 @@ license: |
 | celeborn.master.heartbeat.worker.timeout | 120s | false | Worker heartbeat timeout. | 0.3.0 | celeborn.worker.heartbeat.timeout | 
 | celeborn.master.host | &lt;localhost&gt; | false | Hostname for master to bind. | 0.2.0 |  | 
 | celeborn.master.http.auth.administers |  | false | A comma-separated list of users who have admin privileges, Note, when celeborn.master.http.auth.supportedSchemes is not set, everyone is treated as administrator. | 0.6.0 |  | 
-| celeborn.master.http.auth.basic.provider | org.apache.celeborn.common.authentication.AnonymousAuthenticationProviderImpl | false | User-defined password authentication implementation of org.apache.celeborn.common.authentication.PasswdAuthenticationProvider | 0.6.0 |  | 
-| celeborn.master.http.auth.bearer.provider | org.apache.celeborn.common.authentication.AnonymousAuthenticationProviderImpl | false | User-defined token authentication implementation of org.apache.celeborn.common.authentication.TokenAuthenticationProvider | 0.6.0 |  | 
+| celeborn.master.http.auth.basic.provider | org.apache.celeborn.common.authentication.AnonymousAuthenticationProviderImpl | false | User-defined password authentication implementation of org.apache.celeborn.spi.authentication.PasswdAuthenticationProvider | 0.6.0 |  | 
+| celeborn.master.http.auth.bearer.provider | org.apache.celeborn.common.authentication.AnonymousAuthenticationProviderImpl | false | User-defined token authentication implementation of org.apache.celeborn.spi.authentication.TokenAuthenticationProvider | 0.6.0 |  | 
 | celeborn.master.http.auth.supportedSchemes |  | false | A comma-separated list of master http auth supported schemes.<ul> <li>SPNEGO: Kerberos/GSSAPI authentication.</li> <li>BASIC: User-defined password authentication, the concreted implementation is configurable via `celeborn.master.http.auth.basic.provider`.</li> <li>BEARER: User-defined bearer token authentication, the concreted implementation is configurable via `celeborn.master.http.auth.bearer.provider`.</li></ul> | 0.6.0 |  | 
 | celeborn.master.http.host | &lt;localhost&gt; | false | Master's http host. | 0.4.0 | celeborn.metrics.master.prometheus.host,celeborn.master.metrics.prometheus.host | 
 | celeborn.master.http.idleTimeout | 30s | false | Master http server idle timeout. | 0.5.0 |  | 

diff --git a/pom.xml b/pom.xml
@@ -33,6 +33,7 @@
   <modules>
     <module>openapi/openapi-client</module>
     <module>openapi/openapi-model</module>
+    <module>spi</module>
     <module>common</module>
     <module>client</module>
     <module>service</module>

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -350,6 +350,7 @@ object CelebornBuild extends sbt.internal.BuildDef {
       CelebornOpenApi.openapiInternalMasterModel,
       CelebornOpenApi.openapiInternalWorkerModel,
       CelebornOpenApi.openapiModel,
+      CelebornSpi.spi,
       CelebornCommon.common,
       CelebornClient.client,
       CelebornService.service,
@@ -444,6 +445,16 @@ object Utils {
   }
 }
 
+object CelebornSpi {
+  lazy val spi = Project("celeborn-spi", file("spi"))
+    .settings(
+      commonSettings,
+      releaseSettings,
+      crossPaths := false,
+      Compile / doc / javacOptions := Seq("-encoding", UTF_8.name(), "-source", "1.8")
+    )
+}
+
 object CelebornCommon {
 
   lazy val hadoopAwsDependencies = if(profiles.exists(_.startsWith("hadoop-aws"))){
@@ -453,10 +464,10 @@ object CelebornCommon {
   }
 
   lazy val common = Project("celeborn-common", file("common"))
+    .dependsOn(CelebornSpi.spi)
     .settings (
       commonSettings,
       protoSettings,
-      releaseSettings,
       libraryDependencies ++= Seq(
         Dependencies.protobufJava,
         Dependencies.findbugsJsr305,

diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpAuthUtils.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpAuthUtils.scala
@@ -17,19 +17,23 @@
 
 package org.apache.celeborn.server.common.http
 
-import org.apache.celeborn.common.authentication.Credential
+import java.util.{Map => JMap}
+
+import scala.collection.JavaConverters._
+
 import org.apache.celeborn.server.common.http.authentication.AuthenticationFilter
+import org.apache.celeborn.spi.authentication.Credential
 
 object HttpAuthUtils {
   // HTTP header used by the server endpoint during an authentication sequence.
   val WWW_AUTHENTICATE_HEADER = "WWW-Authenticate"
   // HTTP header used by the client endpoint during an authentication sequence.
   val AUTHORIZATION_HEADER = "Authorization"
 
-  def getCredentialExtraInfo: Map[String, String] = {
+  def getCredentialExtraInfo: JMap[String, String] = {
     Map(Credential.CLIENT_IP_KEY ->
       Option(
         AuthenticationFilter.HTTP_PROXY_HEADER_CLIENT_IP_ADDRESS.get()).getOrElse(
-        AuthenticationFilter.HTTP_CLIENT_IP_ADDRESS.get()))
+        AuthenticationFilter.HTTP_CLIENT_IP_ADDRESS.get())).asJava
   }
 }
diff --git a/...la/org/apache/celeborn/server/common/http/authentication/BasicAuthenticationHandler.scala b/...la/org/apache/celeborn/server/common/http/authentication/BasicAuthenticationHandler.scala
@@ -22,11 +22,12 @@ import java.util.Base64
 import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
 
 import org.apache.celeborn.common.CelebornConf
-import org.apache.celeborn.common.authentication.{AnonymousAuthenticationProviderImpl, DefaultPasswordCredential, PasswdAuthenticationProvider}
+import org.apache.celeborn.common.authentication.{AnonymousAuthenticationProviderImpl, DefaultPasswordCredential}
 import org.apache.celeborn.common.authentication.HttpAuthSchemes._
 import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.server.common.http.HttpAuthUtils
 import org.apache.celeborn.server.common.http.HttpAuthUtils.{AUTHORIZATION_HEADER, WWW_AUTHENTICATE_HEADER}
+import org.apache.celeborn.spi.authentication.PasswdAuthenticationProvider
 
 class BasicAuthenticationHandler(providerClass: String) extends AuthenticationHandler with Logging {
 

diff --git a/...a/org/apache/celeborn/server/common/http/authentication/BearerAuthenticationHandler.scala b/...a/org/apache/celeborn/server/common/http/authentication/BearerAuthenticationHandler.scala
@@ -22,11 +22,12 @@ import java.util.Base64
 import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
 
 import org.apache.celeborn.common.CelebornConf
-import org.apache.celeborn.common.authentication.{AnonymousAuthenticationProviderImpl, Credential, DefaultTokenCredential, TokenAuthenticationProvider}
+import org.apache.celeborn.common.authentication.{AnonymousAuthenticationProviderImpl, DefaultTokenCredential}
 import org.apache.celeborn.common.authentication.HttpAuthSchemes._
 import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.server.common.http.HttpAuthUtils
 import org.apache.celeborn.server.common.http.HttpAuthUtils.{AUTHORIZATION_HEADER, WWW_AUTHENTICATE_HEADER}
+import org.apache.celeborn.spi.authentication.TokenAuthenticationProvider
 
 class BearerAuthenticationHandler(providerClass: String)
   extends AuthenticationHandler with Logging {

diff --git a/...ala/org/apache/celeborn/server/common/http/authentication/HttpAuthenticationFactory.scala b/...ala/org/apache/celeborn/server/common/http/authentication/HttpAuthenticationFactory.scala
@@ -23,9 +23,9 @@ import org.eclipse.jetty.server.{Handler, Request}
 import org.eclipse.jetty.server.handler.HandlerWrapper
 
 import org.apache.celeborn.common.CelebornConf
-import org.apache.celeborn.common.authentication.{PasswdAuthenticationProvider, TokenAuthenticationProvider}
 import org.apache.celeborn.common.exception.CelebornException
 import org.apache.celeborn.reflect.DynConstructors
+import org.apache.celeborn.spi.authentication.{PasswdAuthenticationProvider, TokenAuthenticationProvider}
 
 object HttpAuthenticationFactory {
   def wrapHandler(handler: Handler): HandlerWrapper = {

diff --git a/...born/server/common/http/authentication/UserDefinePasswordAuthenticationProviderImpl.scala b/...born/server/common/http/authentication/UserDefinePasswordAuthenticationProviderImpl.scala
@@ -20,15 +20,15 @@ package org.apache.celeborn.server.common.http.authentication
 import java.security.Principal
 import javax.security.sasl.AuthenticationException
 
-import org.apache.celeborn.common.authentication.{BasicPrincipal, Credential, PasswdAuthenticationProvider, PasswordCredential}
+import org.apache.celeborn.common.authentication.BasicPrincipal
 import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.server.common.http.authentication.UserDefinePasswordAuthenticationProviderImpl.VALID_PASSWORD
+import org.apache.celeborn.spi.authentication.{Credential, PasswdAuthenticationProvider, PasswordCredential}
 
 class UserDefinePasswordAuthenticationProviderImpl
   extends PasswdAuthenticationProvider with Logging {
   override def authenticate(credential: PasswordCredential): Principal = {
-    val clientIp =
-      credential.extraInfo.getOrElse(Credential.CLIENT_IP_KEY, null)
+    val clientIp = credential.extraInfo.get(Credential.CLIENT_IP_KEY)
     if (credential.password == VALID_PASSWORD) {
       logInfo(s"Success log in of user: ${credential.username} with clientIp: $clientIp")
       new BasicPrincipal(credential.username)

diff --git a/...eleborn/server/common/http/authentication/UserDefineTokenAuthenticationProviderImpl.scala b/...eleborn/server/common/http/authentication/UserDefineTokenAuthenticationProviderImpl.scala
@@ -20,14 +20,14 @@ package org.apache.celeborn.server.common.http.authentication
 import java.security.Principal
 import javax.security.sasl.AuthenticationException
 
-import org.apache.celeborn.common.authentication.{BasicPrincipal, Credential, TokenAuthenticationProvider, TokenCredential}
+import org.apache.celeborn.common.authentication.BasicPrincipal
 import org.apache.celeborn.common.internal.Logging
 import org.apache.celeborn.server.common.http.authentication.UserDefineTokenAuthenticationProviderImpl.VALID_TOKEN
+import org.apache.celeborn.spi.authentication.{Credential, TokenAuthenticationProvider, TokenCredential}
 
 class UserDefineTokenAuthenticationProviderImpl extends TokenAuthenticationProvider with Logging {
   override def authenticate(credential: TokenCredential): Principal = {
-    val clientIp =
-      credential.extraInfo.getOrElse(Credential.CLIENT_IP_KEY, null)
+    val clientIp = credential.extraInfo.get(Credential.CLIENT_IP_KEY)
     if (credential.token == VALID_TOKEN) {
       logInfo(s"Success log in of token: ${credential.token} with clientIp: $clientIp")
       new BasicPrincipal("user")

diff --git a/spi/pom.xml b/spi/pom.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.celeborn</groupId>
+    <artifactId>celeborn-parent_${scala.binary.version}</artifactId>
+    <version>${project.version}</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>celeborn-spi</artifactId>
+  <packaging>jar</packaging>
+  <name>Celeborn SPI</name>
+</project>
diff --git a/spi/src/main/java/org/apache/celeborn/spi/authentication/Credential.java b/spi/src/main/java/org/apache/celeborn/spi/authentication/Credential.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.spi.authentication;
+
+public class Credential {
+  public static String CLIENT_IP_KEY = "clientIp";
+}
diff --git a/...cation/PasswdAuthenticationProvider.scala → ...ication/PasswdAuthenticationProvider.java b/...cation/PasswdAuthenticationProvider.scala → ...ication/PasswdAuthenticationProvider.java
@@ -15,22 +15,19 @@
  * limitations under the License.
  */
 
-package org.apache.celeborn.common.authentication
+package org.apache.celeborn.spi.authentication;
 
-import java.security.Principal
-
-trait PasswdAuthenticationProvider {
+import java.security.Principal;
 
+public interface PasswdAuthenticationProvider {
   /**
-   * The authenticate method is called by the celeborn authentication layer
-   * to authenticate password credential for their requests.
-   * If a credential is to be granted, return nothing/throw nothing.
+   * The authenticate method is called by the celeborn authentication layer to authenticate password
+   * credential for their requests. If a credential is to be granted, return nothing/throw nothing.
    * When a credential is to be disallowed, throw an appropriate [[SecurityException]].
    *
    * @param credential The credential received over the connection request
-   *
+   * @return The identifier associated with the credential
    * @throws SecurityException When a user is found to be invalid by the implementation
    */
-  @throws[SecurityException]
-  def authenticate(credential: PasswordCredential): Principal
+  Principal authenticate(PasswordCredential credential);
 }
diff --git a/spi/src/main/java/org/apache/celeborn/spi/authentication/PasswordCredential.java b/spi/src/main/java/org/apache/celeborn/spi/authentication/PasswordCredential.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.spi.authentication;
+
+import java.util.Map;
+
+public interface PasswordCredential {
+  String username();
+
+  String password();
+
+  Map<String, String> extraInfo();
+}
diff --git a/...ication/TokenAuthenticationProvider.scala → ...tication/TokenAuthenticationProvider.java b/...ication/TokenAuthenticationProvider.scala → ...tication/TokenAuthenticationProvider.java
@@ -15,23 +15,19 @@
  * limitations under the License.
  */
 
-package org.apache.celeborn.common.authentication
+package org.apache.celeborn.spi.authentication;
 
-import java.security.Principal
-
-trait TokenAuthenticationProvider {
+import java.security.Principal;
 
+public interface TokenAuthenticationProvider {
   /**
-   * The authenticate method is called by the celeborn authentication layer
-   * to authenticate credential for their requests.
-   * If the credential is to be granted, return nothing/throw nothing.
-   * When the credential is to be disallowed, throw an appropriate [[SecurityException]].
+   * The authenticate method is called by the celeborn authentication layer to authenticate
+   * credential for their requests. If the credential is to be granted, return nothing/throw
+   * nothing. When the credential is to be disallowed, throw an appropriate [[SecurityException]].
    *
    * @param credential The credential received over the connection request
    * @return The identifier associated with the token
-   *
    * @throws SecurityException When the credential is found to be invalid by the implementation
    */
-  @throws[SecurityException]
-  def authenticate(credential: TokenCredential): Principal
+  Principal authenticate(TokenCredential credential);
 }