tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[BugFix] Fix broker kerberos ticket timeout bug when there are broker load continuously (#16131) 

The current broker has two types of kerberos token, one is the global, one unique to each HDFS client, access to HDFS will use each client unique token, and access to other components (such as kms) use the global. HDFS client unique token is not automatically refreshed, but HDFS client in the broker has an expiration time, if more than 5min is not used, the HDFS client will be destroyed. If there is a request the next time, a new client will be created, the kerberos login will be executed again, this is actually an indirect refresh token. But if there are access continuously, HDFS client will never be rebuilt, then the token will not be refreshed, it will time out after 1 day.
To fix this bug, destroy the client before token is expired.
This commit is contained in:
gengjun-git 2023-01-03 20:56:09 +08:00 committed by GitHub
parent db1490b340
commit 51094916a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
fs_brokers/apache_hdfs_broker/src/main/java/com/starrocks/broker/hdfs/BrokerConfig.java
View File

@ -33,6 +33,15 @@ public class BrokerConfig extends ConfigBase {
@ConfField
public static int broker_ipc_port = 8000;
/**
* If the kerberos HDFS client is alive beyond this time,
* client checker will destroy it to avoid kerberos token expire.
* Set this value a little smaller than the actual token expire seconds,
* to make sure the client is destroyed before the timeout reaches.
*/
@ConfField
public static int kerberos_token_expire_seconds = 86000;
@ConfField
public static String sys_log_dir = System.getenv("BROKER_HOME") + "/log";

25
fs_brokers/apache_hdfs_broker/src/main/java/com/starrocks/broker/hdfs/BrokerFileSystem.java
View File

@ -31,20 +31,27 @@ public class BrokerFileSystem {
private ReentrantLock lock;
private FileSystemIdentity identity;
private FileSystem dfsFileSystem;
private long lastAccessTimestamp;
private long lastAccessTimestamp = -1L;
private long createTimestamp = -1L;
private UUID fileSystemId;
private boolean isKerberosAuth = false;
public BrokerFileSystem(FileSystemIdentity identity) {
this.identity = identity;
this.lock = new ReentrantLock();
this.dfsFileSystem = null;
this.lastAccessTimestamp = System.currentTimeMillis();
this.fileSystemId = UUID.randomUUID();
}
public synchronized void setFileSystem(FileSystem fileSystem) {
public synchronized void setFileSystem(FileSystem fileSystem, boolean isKerberosAuth) {
this.dfsFileSystem = fileSystem;
this.lastAccessTimestamp = System.currentTimeMillis();
this.createTimestamp = System.currentTimeMillis();
this.isKerberosAuth = isKerberosAuth;
}
public synchronized void setFileSystem(FileSystem fileSystem) {
this.setFileSystem(fileSystem, false);
}
public void closeFileSystem() {
@ -81,8 +88,14 @@ public class BrokerFileSystem {
return lock;
}
public boolean isExpired(long expirationIntervalSecs) {
if (System.currentTimeMillis() - lastAccessTimestamp > expirationIntervalSecs * 1000) {
public boolean isExpired() {
if (lastAccessTimestamp > 0
&& System.currentTimeMillis() - lastAccessTimestamp > BrokerConfig.client_expire_seconds * 1000) {
return true;
}
if (createTimestamp > 0
&& isKerberosAuth
&& System.currentTimeMillis() - createTimestamp > BrokerConfig.kerberos_token_expire_seconds * 1000) {
return true;
}
return false;

4
fs_brokers/apache_hdfs_broker/src/main/java/com/starrocks/broker/hdfs/FileSystemManager.java
View File

@ -437,7 +437,7 @@ public class FileSystemManager {
} else {
dfsFileSystem = FileSystem.get(pathUri.getUri(), conf);
}
fileSystem.setFileSystem(dfsFileSystem);
fileSystem.setFileSystem(dfsFileSystem, authentication.equals(AUTHENTICATION_KERBEROS));
}
return fileSystem;
} catch (Exception e) {
@ -994,7 +994,7 @@ public class FileSystemManager {
public void run() {
try {
for (BrokerFileSystem fileSystem : cachedFileSystem.values()) {
if (fileSystem.isExpired(BrokerConfig.client_expire_seconds)) {
if (fileSystem.isExpired()) {
logger.info("file system " + fileSystem + " is expired, close and remove it");
fileSystem.getLock().lock();
try {