tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Enhancement] mask crendential info in submit task (backport #62311) (#62554) 

Signed-off-by: starrocks-xupeng <xupeng@starrocks.com>
This commit is contained in:
starrocks-xupeng 2025-09-01 12:30:57 +08:00 committed by GitHub
parent 148829d642
commit 67f90d7979
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 68 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TaskRunsSystemTable.java
View File

@ -26,6 +26,7 @@ import com.starrocks.catalog.system.SystemId;
import com.starrocks.catalog.system.SystemTable;
import com.starrocks.cluster.ClusterNamespace;
import com.starrocks.common.Config;
import com.starrocks.common.util.SqlCredentialRedactor;
import com.starrocks.qe.ConnectContext;
import com.starrocks.scheduler.Task;
import com.starrocks.scheduler.TaskManager;
@ -198,13 +199,21 @@ public class TaskRunsSystemTable extends SystemTable {
info.setCatalog(status.getCatalogName());
info.setDatabase(ClusterNamespace.getNameFromFullName(status.getDbName()));
if (!Strings.isEmpty(status.getDefinition())) {
info.setDefinition(status.getDefinition());
if (Config.enable_task_info_mask_credential) {
info.setDefinition(SqlCredentialRedactor.redact(status.getDefinition()));
} else {
info.setDefinition(status.getDefinition());
}
} else {
try {
// NOTE: use task's definition to display task-run's definition here
Task task = taskManager.getTaskWithoutLock(taskName);
if (task != null) {
info.setDefinition(task.getDefinition());
if (Config.enable_task_info_mask_credential) {
info.setDefinition(SqlCredentialRedactor.redact(task.getDefinition()));
} else {
info.setDefinition(task.getDefinition());
}
}
} catch (Exception e) {
LOG.warn("Get taskName {} definition failed: {}", taskName, e);

10
fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TasksSystemTable.java
View File

@ -22,6 +22,8 @@ import com.starrocks.catalog.Table;
import com.starrocks.catalog.system.SystemId;
import com.starrocks.catalog.system.SystemTable;
import com.starrocks.cluster.ClusterNamespace;
import com.starrocks.common.Config;
import com.starrocks.common.util.SqlCredentialRedactor;
import com.starrocks.qe.ConnectContext;
import com.starrocks.scheduler.Constants;
import com.starrocks.scheduler.Task;
@ -96,12 +98,16 @@ public class TasksSystemTable {
scheduleStr = task.getType().name();
}
if (task.getType() == Constants.TaskType.PERIODICAL) {
scheduleStr += task.getSchedule();
scheduleStr += " " + task.getSchedule();
}
info.setSchedule(scheduleStr);
info.setCatalog(task.getCatalogName());
info.setDatabase(ClusterNamespace.getNameFromFullName(task.getDbName()));
info.setDefinition(task.getDefinition());
if (Config.enable_task_info_mask_credential) {
info.setDefinition(SqlCredentialRedactor.redact(task.getDefinition()));
} else {
info.setDefinition(task.getDefinition());
}
info.setExpire_time(task.getExpireTime() / 1000);
info.setProperties(task.getPropertiesString());
if (task.getUserIdentity() != null) {

5
fe/fe-core/src/main/java/com/starrocks/common/Config.java
View File

@ -418,6 +418,11 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean enable_task_run_fe_evaluation = true;
// whether mask credential info in `information_schema.tasks` and `information_schema.task_runs`
// if task count is big, mask process can be time consuming
@ConfField(mutable = true)
public static boolean enable_task_info_mask_credential = true;
/**
* The max keep time of some kind of jobs.
* like schema change job and rollup job.

2
fe/fe-core/src/main/java/com/starrocks/scheduler/persist/TaskSchedule.java
View File

@ -67,7 +67,7 @@ public class TaskSchedule {
public String toString() {
if (startTime > 0) {
return " START(" + Utils.getDatetimeFromLong(startTime)
return "START(" + Utils.getDatetimeFromLong(startTime)
+ ") EVERY(" + period + " " + timeUnit + ")";
}
return "EVERY(" + period + " " + timeUnit + ")";

3
fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AstToSQLBuilder.java
View File

@ -22,6 +22,7 @@ import com.starrocks.analysis.TableName;
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.common.util.ParseUtil;
import com.starrocks.common.util.SqlCredentialRedactor;
import com.starrocks.sql.ast.ArrayExpr;
import com.starrocks.sql.ast.CTERelation;
import com.starrocks.sql.ast.FieldReference;
@ -82,7 +83,7 @@ public class AstToSQLBuilder {
return toSQL(statement);
} catch (Exception e) {
LOG.info("Ast to sql failed.", e);
return defaultSql;
return SqlCredentialRedactor.redact(defaultSql);
}
}

20
fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AstToStringBuilder.java
View File

@ -85,6 +85,7 @@ import com.starrocks.catalog.View;
import com.starrocks.common.Pair;
import com.starrocks.common.util.ParseUtil;
import com.starrocks.common.util.PrintableMap;
import com.starrocks.common.util.SqlCredentialRedactor;
import com.starrocks.credential.CredentialUtil;
import com.starrocks.sql.ast.AlterStorageVolumeStmt;
import com.starrocks.sql.ast.AlterUserStmt;
@ -134,6 +135,7 @@ import com.starrocks.sql.ast.SetQualifier;
import com.starrocks.sql.ast.SetStmt;
import com.starrocks.sql.ast.SetType;
import com.starrocks.sql.ast.SetUserPropertyStmt;
import com.starrocks.sql.ast.SubmitTaskStmt;
import com.starrocks.sql.ast.SubqueryRelation;
import com.starrocks.sql.ast.SystemVariable;
import com.starrocks.sql.ast.TableFunctionRelation;
@ -1602,6 +1604,24 @@ public class AstToStringBuilder {
return hintBuilder.toString();
}
@Override
public String visitSubmitTaskStatement(SubmitTaskStmt stmt, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("SUBMIT TASK ");
sb.append(stmt.getTaskName());
if (stmt.getSchedule() != null) {
sb.append(" ");
sb.append(stmt.getSchedule().toString());
}
if (!stmt.getProperties().isEmpty()) {
sb.append(" PROPERTIES (")
.append(new PrintableMap<>(stmt.getProperties(), "=", true, false, false)).append(")");
}
sb.append(" AS ");
sb.append(SqlCredentialRedactor.redact(stmt.getSqlText()));
return sb.toString();
}
}
public static void getDdlStmt(Table table, List<String> createTableStmt, List<String> addPartitionStmt,

6
fe/fe-core/src/main/java/com/starrocks/sql/common/AuditEncryptionChecker.java
View File

@ -40,6 +40,7 @@ import com.starrocks.sql.ast.SetOperationRelation;
import com.starrocks.sql.ast.SetPassVar;
import com.starrocks.sql.ast.SetStmt;
import com.starrocks.sql.ast.StatementBase;
import com.starrocks.sql.ast.SubmitTaskStmt;
import com.starrocks.sql.ast.SubqueryRelation;
import com.starrocks.sql.ast.integration.ShowCreateSecurityIntegrationStatement;
import com.starrocks.sql.ast.pipe.CreatePipeStmt;
@ -84,6 +85,11 @@ public class AuditEncryptionChecker implements AstVisitor<Boolean, Void> {
return visit(queryRelation);
}
@Override
public Boolean visitSubmitTaskStatement(SubmitTaskStmt statement, Void context) {
return true;
}
@Override
public Boolean visitInsertStatement(InsertStmt statement, Void context) {
boolean tableFunctionAsTargetTable = statement.useTableFunctionAsTargetTable();

18
fe/fe-core/src/test/java/com/starrocks/analysis/SubmitTaskStmtTest.java
View File

@ -30,9 +30,11 @@ import com.starrocks.scheduler.TaskRun;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.server.WarehouseManager;
import com.starrocks.sql.analyzer.TaskAnalyzer;
import com.starrocks.sql.analyzer.AstToStringBuilder;
import com.starrocks.sql.ast.StatementBase;
import com.starrocks.sql.ast.SubmitTaskStmt;
import com.starrocks.sql.ast.UserIdentity;
import com.starrocks.sql.common.AuditEncryptionChecker;
import com.starrocks.sql.optimizer.rule.transformation.materialization.MVTestBase;
import com.starrocks.sql.parser.ParsingException;
import com.starrocks.utframe.UtFrameUtils;
@ -112,6 +114,16 @@ public class SubmitTaskStmtTest extends MVTestBase {
Assertions.assertEquals(submitTaskStmt4.getDbName(), "test");
Assertions.assertEquals(submitTaskStmt4.getTaskName(), "task_name");
Assertions.assertEquals(submitTaskStmt4.getProperties().size(), 0);
String submitSQL5 = "submit /*+ SET_VAR(query_timeout = 1) */ task as " +
"insert into files(\"path\"=\"data\", \"format\"=\"csv\", \"aws.s3.access_key\"=\"aaa\", " +
"\"aws.s3.secret_key\"=\"bbb\") select * from tbl1";
SubmitTaskStmt submitTaskStmt5 = (SubmitTaskStmt) UtFrameUtils.parseStmtWithNewParser(submitSQL5, ctx);
AuditEncryptionChecker.getInstance().visitSubmitTaskStatement(submitTaskStmt5, null);
AstToStringBuilder.AST2StringBuilderVisitor builder = new AstToStringBuilder.AST2StringBuilderVisitor();
String str = builder.visitSubmitTaskStatement(submitTaskStmt5, null);
Assertions.assertFalse(str.contains("aaa"));
Assertions.assertFalse(str.contains("bbb"));
}
@Test
@ -259,7 +271,7 @@ public class SubmitTaskStmtTest extends MVTestBase {
"as insert overwrite tbl1 select * from tbl1");
Task task = tm.getTask("t2");
Assertions.assertNotNull(task);
Assertions.assertEquals(" START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals("START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals(Constants.TaskSource.INSERT, task.getSource());
Assertions.assertEquals(Constants.TaskType.PERIODICAL, task.getType());
connectContext.executeSql("drop task t2");
@ -282,7 +294,7 @@ public class SubmitTaskStmtTest extends MVTestBase {
"as insert overwrite tbl1 select * from tbl1");
Task task = tm.getTask("t4");
Assertions.assertNotNull(task);
Assertions.assertEquals(" START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals("START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals(Constants.TaskSource.INSERT, task.getSource());
Assertions.assertEquals(Constants.TaskType.PERIODICAL, task.getType());
connectContext.executeSql("drop task t4");
@ -295,7 +307,7 @@ public class SubmitTaskStmtTest extends MVTestBase {
"as insert overwrite tbl1 select * from tbl1");
Task task = tm.getTask("t4");
Assertions.assertNotNull(task);
Assertions.assertEquals(" START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals("START(1997-01-01T00:00) EVERY(1 MINUTES)", task.getSchedule().toString());
Assertions.assertEquals(Constants.TaskSource.INSERT, task.getSource());
Assertions.assertEquals(Constants.TaskType.PERIODICAL, task.getType());
connectContext.executeSql("drop task t4");