Support s3 cross region bucket access

trinodb · Oct 2, 2024 · e2c5666 · e2c5666
1 parent b43e2a3
commit e2c5666
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 2 deletions.
diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java
@@ -115,6 +115,7 @@ public static RetryStrategy getRetryStrategy(RetryMode retryMode)
     private RetryMode retryMode = RetryMode.LEGACY;
     private int maxErrorRetries = 10;
     private boolean supportsExclusiveCreate = true;
+    private boolean crossRegionAccessEnabled;
 
     public String getAwsAccessKey()
     {
@@ -511,4 +512,17 @@ public S3FileSystemConfig setSupportsExclusiveCreate(boolean supportsExclusiveCr
         this.supportsExclusiveCreate = supportsExclusiveCreate;
         return this;
     }
+
+    public boolean isCrossRegionAccessEnabled()
+    {
+        return crossRegionAccessEnabled;
+    }
+
+    @Config("s3.cross-region-access-enabled")
+    @ConfigDescription("Whether to allow cross-region bucket access")
+    public S3FileSystemConfig setCrossRegionAccessEnabled(boolean crossRegionAccessEnabled)
+    {
+        this.crossRegionAccessEnabled = crossRegionAccessEnabled;
+        return this;
+    }
 }
diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java
@@ -134,6 +134,7 @@ private static S3ClientFactory s3ClientFactory(SdkHttpClient httpClient, OpenTel
         Optional<String> staticRegion = Optional.ofNullable(config.getRegion());
         Optional<String> staticEndpoint = Optional.ofNullable(config.getEndpoint());
         boolean pathStyleAccess = config.isPathStyleAccess();
+        boolean isCrossRegionAccessEnabled = config.isCrossRegionAccessEnabled();
         boolean useWebIdentityTokenCredentialsProvider = config.isUseWebIdentityTokenCredentialsProvider();
         Optional<String> staticIamRole = Optional.ofNullable(config.getIamRole());
         String staticRoleSessionName = config.getRoleSessionName();
@@ -154,6 +155,17 @@ private static S3ClientFactory s3ClientFactory(SdkHttpClient httpClient, OpenTel
             s3.overrideConfiguration(overrideConfiguration);
             s3.httpClient(httpClient);
 
+            Optional<Region> awsRegion = region.map(Region::of);
+            Optional<URI> awsEndpoint = endpoint.map(URI::create);
+            awsRegion.ifPresent(s3::region);
+            awsEndpoint.ifPresent(s3::endpointOverride);
+            if (isCrossRegionAccessEnabled && awsEndpoint.isEmpty()) {
+                if (awsRegion.isEmpty()) {
+                    s3.region(Region.US_EAST_1);
+                }
+                s3.crossRegionAccessEnabled(true);
+            }
+
             region.map(Region::of).ifPresent(s3::region);
             endpoint.map(URI::create).ifPresent(s3::endpointOverride);
             s3.forcePathStyle(pathStyleAccess);

diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java
@@ -67,7 +67,8 @@ public void testDefaults()
                 .setHttpProxyUsername(null)
                 .setHttpProxyPassword(null)
                 .setHttpProxyPreemptiveBasicProxyAuth(false)
-                .setSupportsExclusiveCreate(true));
+                .setSupportsExclusiveCreate(true)
+                .setCrossRegionAccessEnabled(false));
     }
 
     @Test
@@ -105,6 +106,7 @@ public void testExplicitPropertyMappings()
                 .put("s3.http-proxy.password", "test")
                 .put("s3.http-proxy.preemptive-basic-auth", "true")
                 .put("s3.exclusive-create", "false")
+                .put("s3.cross-region-access-enabled", "true")
                 .buildOrThrow();
 
         S3FileSystemConfig expected = new S3FileSystemConfig()
@@ -138,7 +140,8 @@ public void testExplicitPropertyMappings()
                 .setHttpProxyUsername("test")
                 .setHttpProxyPassword("test")
                 .setHttpProxyPreemptiveBasicProxyAuth(true)
-                .setSupportsExclusiveCreate(false);
+                .setSupportsExclusiveCreate(false)
+                .setCrossRegionAccessEnabled(true);
 
         assertFullMapping(properties, expected);
     }