tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Enhancement] support s3 path style in shared-data cluster (backport #62591) (#63082) 

Signed-off-by: starrocks-xupeng <xupeng@starrocks.com>
Co-authored-by: starrocks-xupeng <xupeng@starrocks.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
mergify[bot] 2025-09-14 16:50:15 +08:00 committed by GitHub
parent 4a3309228a
commit 8861a1027e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 50 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/sql-reference/sql-statements/cluster-management/storage_volume/CREATE_STORAGE_VOLUME.md
View File

@ -50,6 +50,7 @@ import Beta from '../../../../_assets/commonMarkdown/_beta.mdx'
| aws.s3.secret_key | The Secret Access Key used to access your S3 bucket. |
| aws.s3.iam_role_arn | The ARN of the IAM role that has privileges on your S3 bucket in which your data files are stored. |
| aws.s3.external_id | The external ID of the AWS account that is used for cross-account access to your S3 bucket. |
| aws.s3.enable_path_style_access | Whether to use the Path Style to access S3.<br />Valid values: `true` and `false`. Default: `false`. |
| azure.blob.endpoint | The endpoint of your Azure Blob Storage Account, for example, `https://test.blob.core.windows.net`. |
| azure.blob.shared_key | The Shared Key used to authorize requests for your Azure Blob Storage. |
| azure.blob.sas_token | The shared access signatures (SAS) used to authorize requests for your Azure Blob Storage. |

1
docs/ja/sql-reference/sql-statements/cluster-management/storage_volume/CREATE_STORAGE_VOLUME.md
View File

@ -52,6 +52,7 @@ import Beta from '../../../../_assets/commonMarkdown/_beta.mdx'
| aws.s3.secret_key | S3 バケットにアクセスするためのシークレットアクセスキーです。 |
| aws.s3.iam_role_arn | データファイルが保存されている S3 バケットに対して権限を持つ IAM ロールの ARN です。 |
| aws.s3.external_id | S3 バケットへのクロスアカウントアクセスに使用される AWS アカウントの外部 ID です。 |
| aws.s3.enable_path_style_access | Path Style を使用して S3 にアクセスするかどうか。<br />有効な値: `true` および `false`。デフォルト: `false`。 |
| azure.blob.endpoint | Azure Blob Storage アカウントのエンドポイントです。例：`https://test.blob.core.windows.net`。 |
| azure.blob.shared_key | Azure Blob Storage へのリクエストを承認するために使用される共有キーです。 |
| azure.blob.sas_token | Azure Blob Storage へのリクエストを承認するために使用される共有アクセス署名 (SAS) です。 |

1
docs/zh/sql-reference/sql-statements/cluster-management/storage_volume/CREATE_STORAGE_VOLUME.md
View File

@ -53,6 +53,7 @@ import Beta from '../../../../_assets/commonMarkdown/_beta.mdx'
| aws.s3.secret_key | 访问 S3 存储空间的 Secret Key。 |
| aws.s3.iam_role_arn | 有访问 S3 存储空间权限 IAM Role 的 ARN。 |
| aws.s3.external_id | 用于跨 AWS 账户访问 S3 存储空间的外部 ID。 |
| aws.s3.enable_path_style_access | 指定是否使用 Path Style 方式访问 S3。<br />有效值：`true` 和 `false`。默认值：`false`。 |
| azure.blob.endpoint | Azure Blob Storage 的链接地址，如 `https://test.blob.core.windows.net`。 |
| azure.blob.shared_key | 访问 Azure Blob Storage 的共享密钥（Shared Key）。 |
| azure.blob.sas_token | 访问 Azure Blob Storage 的共享访问签名（SAS）。 |

24
fe/fe-core/src/main/java/com/starrocks/credential/aws/AwsCloudConfiguration.java
View File

@ -32,7 +32,10 @@ public class AwsCloudConfiguration extends CloudConfiguration {
private final AwsCloudCredential awsCloudCredential;
private boolean enablePathStyleAccess = false;
// shared-data cluster uses 2 methods to determine whether to enable path style access:
// if it is null, BE/CN will auto-determine whether to use path style access; otherwise, it will
// be determined by the corresponding value.
private Boolean enablePathStyleAccess = null;
private boolean enableSSL = true;
@ -49,7 +52,7 @@ public class AwsCloudConfiguration extends CloudConfiguration {
}
public boolean getEnablePathStyleAccess() {
return this.enablePathStyleAccess;
return this.enablePathStyleAccess == null ? false : this.enablePathStyleAccess;
}
public void setEnableSSL(boolean enableSSL) {
@ -85,7 +88,7 @@ public class AwsCloudConfiguration extends CloudConfiguration {
// Default value is 20
configuration.set(Constants.MAX_ERROR_RETRIES, "5");
configuration.set(Constants.PATH_STYLE_ACCESS, String.valueOf(enablePathStyleAccess));
configuration.set(Constants.PATH_STYLE_ACCESS, String.valueOf(getEnablePathStyleAccess()));
configuration.set(Constants.SECURE_CONNECTIONS, String.valueOf(enableSSL));
awsCloudCredential.applyToConfiguration(configuration);
}
@ -117,6 +120,10 @@ public class AwsCloudConfiguration extends CloudConfiguration {
}
}
}
if (properties.containsKey(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS)) {
enablePathStyleAccess = Boolean.parseBoolean(
properties.get(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS));
}
}
@Override
@ -125,7 +132,7 @@ public class AwsCloudConfiguration extends CloudConfiguration {
tCloudConfiguration.setCloud_type(TCloudType.AWS);
Map<String, String> properties = tCloudConfiguration.getCloud_properties();
properties.put(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS,
String.valueOf(enablePathStyleAccess));
String.valueOf(getEnablePathStyleAccess()));
properties.put(CloudConfigurationConstants.AWS_S3_ENABLE_SSL, String.valueOf(enableSSL));
awsCloudCredential.toThrift(properties);
}
@ -142,6 +149,13 @@ public class AwsCloudConfiguration extends CloudConfiguration {
builder.getS3FsInfoBuilder()
.setPartitionedPrefixEnabled(enablePartitionedPrefix)
.setNumPartitionedPrefix(numOfPartitionedPrefix);
if (enablePathStyleAccess == null) {
builder.getS3FsInfoBuilder().setPathStyleAccess(0 /* auto */);
} else if (enablePathStyleAccess) {
builder.getS3FsInfoBuilder().setPathStyleAccess(1 /* path style */);
} else {
builder.getS3FsInfoBuilder().setPathStyleAccess(2 /* virtual host */);
}
return builder.build();
}
@ -150,7 +164,7 @@ public class AwsCloudConfiguration extends CloudConfiguration {
// TODO: add enable_partitioned_prefix, num_partitioned_prefix output
return "AWSCloudConfiguration{" + getCommonFieldsString() +
", cred=" + awsCloudCredential.toCredString() +
", enablePathStyleAccess=" + enablePathStyleAccess +
", enablePathStyleAccess=" + getEnablePathStyleAccess() +
", enableSSL=" + enableSSL +
'}';
}

7
fe/fe-core/src/main/java/com/starrocks/storagevolume/StorageVolume.java
View File

@ -292,6 +292,13 @@ public class StorageVolume implements Writable, GsonPostProcessable {
params.put(CloudConfigurationConstants.AWS_S3_NUM_PARTITIONED_PREFIX,
Integer.toString(s3FileStoreInfo.getNumPartitionedPrefix()));
}
if (s3FileStoreInfo.getPathStyleAccess() == 1) {
params.put(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS,
Boolean.toString(true));
} else if (s3FileStoreInfo.getPathStyleAccess() == 2) {
params.put(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS,
Boolean.toString(false));
}
AwsCredentialInfo credentialInfo = s3FileStoreInfo.getCredential();
if (credentialInfo.hasSimpleCredential()) {
params.put(CloudConfigurationConstants.AWS_S3_USE_INSTANCE_PROFILE, "false");

21
fe/fe-core/src/test/java/com/starrocks/storagevolume/StorageVolumeTest.java
View File

@ -55,6 +55,7 @@ import java.util.HashMap;
import java.util.Map;
import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_ACCESS_KEY;
import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS;
import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_ENDPOINT;
import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_EXTERNAL_ID;
import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_IAM_ROLE_ARN;
@ -120,6 +121,7 @@ public class StorageVolumeTest {
storageParams.put(AWS_S3_ACCESS_KEY, "access_key");
storageParams.put(AWS_S3_SECRET_KEY, "secret_key");
storageParams.put(AWS_S3_USE_AWS_SDK_DEFAULT_BEHAVIOR, "false");
storageParams.put(AWS_S3_ENABLE_PATH_STYLE_ACCESS, "true");
StorageVolume sv = new StorageVolume("1", "test", "s3", Arrays.asList("s3://abc"),
storageParams, true, "");
@ -130,6 +132,7 @@ public class StorageVolumeTest {
Assertions.assertTrue(fileStore.hasS3FsInfo());
S3FileStoreInfo s3FileStoreInfo = fileStore.getS3FsInfo();
Assertions.assertTrue(s3FileStoreInfo.getCredential().hasSimpleCredential());
Assertions.assertEquals(s3FileStoreInfo.getPathStyleAccess(), 1);
AwsSimpleCredentialInfo simpleCredentialInfo = s3FileStoreInfo.getCredential().getSimpleCredential();
Assertions.assertEquals("access_key", simpleCredentialInfo.getAccessKey());
Assertions.assertEquals("secret_key", simpleCredentialInfo.getAccessKeySecret());
@ -612,6 +615,24 @@ public class StorageVolumeTest {
Assertions.assertEquals("32", params.get(CloudConfigurationConstants.AWS_S3_NUM_PARTITIONED_PREFIX));
}
fsInfoBuilder.getS3FsInfoBuilder()
.setPathStyleAccess(1);
{
FileStoreInfo fs = fsInfoBuilder.build();
Map<String, String> params = StorageVolume.getParamsFromFileStoreInfo(fs);
Assertions.assertEquals("true", params.get(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS));
}
fsInfoBuilder.getS3FsInfoBuilder()
.setPathStyleAccess(2);
{
FileStoreInfo fs = fsInfoBuilder.build();
Map<String, String> params = StorageVolume.getParamsFromFileStoreInfo(fs);
Assertions.assertEquals("false", params.get(CloudConfigurationConstants.AWS_S3_ENABLE_PATH_STYLE_ACCESS));
}
// It's OK to have trailing '/' after bucket name
fsInfoBuilder.addLocations("s3://bucket/");
{