Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[KYUUBI #6402]: engine.share.level=GROUP enable for a list of hadoop … #6779

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[KYUUBI #6402]: engine.share.level=GROUP enable for a list of hadoop … #6779

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/configuration/settings.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.session.idle.timeout | PT6H | session idle timeout, it will be closed when it's not accessed for this duration | duration | 1.2.0 |
| kyuubi.session.local.dir.allow.list || The local dir list that are allowed to access by the kyuubi session application. End-users might set some parameters such as `spark.files` and it will upload some local files when launching the kyuubi engine, if the local dir allow list is defined, kyuubi will check whether the path to upload is in the allow list. Note that, if it is empty, there is no limitation for that. And please use absolute paths. | set | 1.6.0 |
| kyuubi.session.name | <undefined> | A human readable name of the session and we use empty string by default. This name will be recorded in the event. Note that, we only apply this value from session conf. | string | 1.4.0 |
| kyuubi.session.preferGroups | <undefined> | Comma separated list of hadoop group name for the group engine launch. The first preferred and valid group from this list will be used for GROUP SHARE LEVEL execution. If this is not configured, the engine will use the first group name from the users list of groups as the primary group. | seq | 1.10.0 |
| kyuubi.session.proxy.user | <undefined> | An alternative to hive.server2.proxy.user. The current behavior is consistent with hive.server2.proxy.user and now only takes effect in RESTFul API. When both parameters are set, kyuubi.session.proxy.user takes precedence. | string | 1.9.0 |
| kyuubi.session.timeout | PT6H | (deprecated)session timeout, it will be closed when it's not accessed for this duration | duration | 1.0.0 |
| kyuubi.session.user.sign.enabled | false | Whether to verify the integrity of session user name on the engine side, e.g. Authz plugin in Spark. | boolean | 1.7.0 |
Expand Down
11 changes: 11 additions & 0 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2779,6 +2779,17 @@ object KyuubiConf {
}
.createWithDefault("hadoop")

val PREFERRED_GROUPS: OptionalConfigEntry[Seq[String]] =
buildConf("kyuubi.session.preferGroups")
.doc("Comma separated list of hadoop group name for the group engine launch. " +
"The first preferred and valid group from this list will be used for GROUP " +
"SHARE LEVEL execution. If this is not configured, the engine will use " +
"the first group name from the users list of groups as the primary group.")
.version("1.10.0")
.stringConf
.toSequence()
.createOptional

val SERVER_NAME: OptionalConfigEntry[String] =
buildConf("kyuubi.server.name")
.doc("The name of Kyuubi Server.")
Expand Down
5 changes: 4 additions & 1 deletion kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ private[kyuubi] class EngineRef(
}

// user for launching engine
private[kyuubi] val appUser: String = if (doAsEnabled) routingUser else Utils.currentUser
private[kyuubi] val appUser: String = shareLevel match {
case GROUP => if (doAsEnabled) sessionUser else Utils.currentUser
case _ => if (doAsEnabled) routingUser else Utils.currentUser
}

@VisibleForTesting
private[kyuubi] val subdomain: String = conf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN) match {
Expand Down
26 changes: 24 additions & 2 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/session/HadoopGroupProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,37 @@ import java.util.{Map => JMap}
import org.apache.hadoop.security.UserGroupInformation

import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.plugin.GroupProvider

/**
* Hadoop based group provider, see more information at
* https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/GroupsMapping.html
*/
class HadoopGroupProvider extends GroupProvider with Logging {
override def primaryGroup(user: String, sessionConf: JMap[String, String]): String =
groups(user, sessionConf).head
override def primaryGroup(user: String, sessionConf: JMap[String, String]): String = {
val preferredGroups: Option[Seq[String]] = if (sessionConf != null) {
Option(sessionConf.get(KyuubiConf.PREFERRED_GROUPS.key)).map(_.split(",").toSeq)
} else {
None
}

val userGroups: Array[String] = groups(user, sessionConf)

val primaryGroup = preferredGroups match {
case Some(groups) =>
groups.find(userGroups.contains) match {
case Some(group) => group
case None => userGroups.headOption.getOrElse {
throw new NoSuchElementException("No groups available for the user")
}
}
case None => userGroups.headOption.getOrElse {
throw new NoSuchElementException("No groups available for the user")
}
}
primaryGroup
}

override def groups(user: String, sessionConf: JMap[String, String]): Array[String] =
UserGroupInformation.createRemoteUser(user).getGroupNames match {
Expand Down
28 changes: 25 additions & 3 deletions kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerGroupSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,34 @@ class KyuubiOperationPerGroupSuite extends WithKyuubiServer with SparkQueryTests
}

test("kyuubi defined function - system_user/session_user") {
withSessionConf(Map("hive.server2.proxy.user" -> "user1"))(Map.empty)(Map.empty) {
withSessionConf(Map("hive.server2.proxy.user" -> "user2"))(Map.empty)(Map.empty) {
withJdbcStatement() { statement =>
val res = statement.executeQuery("select system_user() as c1, session_user() as c2")
assert(res.next())
assert(res.getString("c1") === "testGG")
assert(res.getString("c2") === "user1")
assert(res.getString("c1") === "user1")
assert(res.getString("c2") === "user2")
}
}
}

test("ensure preferred group is chosen from list of groups") {
withSessionConf(Map("hive.server2.proxy.user" -> "user1"))(Map(
KyuubiConf.PREFERRED_GROUPS.key -> "test,group_tt,testGG"))(Map.empty) {
withJdbcStatement() { statement =>
val res = statement.executeQuery("set spark.app.name")
assert(res.next())
val engineName = res.getString("value")
assert(engineName.startsWith(s"kyuubi_GROUP_${conf.get(KyuubiConf.ENGINE_TYPE)}_group_tt"))
}
}

withSessionConf(Map("hive.server2.proxy.user" -> "user1"))(Map(
KyuubiConf.PREFERRED_GROUPS.key -> "test"))(Map.empty) {
withJdbcStatement() { statement =>
val res = statement.executeQuery("set spark.app.name")
assert(res.next())
val engineName = res.getString("value")
assert(engineName.startsWith(s"kyuubi_GROUP_${conf.get(KyuubiConf.ENGINE_TYPE)}_testGG"))
}
}
}
Expand Down
Loading