Update software.amazon.awssdk:* to v2.42.37 (branch_9x) - abandoned

changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml

@@ -0,0 +1,7 @@
+title: Update software.amazon.awssdk:* to v2.41.0 (branch_9x)
+type: dependency_update
+authors:
+- name: solrbot
+links:
+- name: PR#3593
+  url: https://github.com/apache/solr/pull/3593

solr/licenses/annotations-2.41.0.jar.sha1

@@ -0,0 +1 @@
+38fedba1c0c3f3c65afe4896c3f936ef77129620

solr/licenses/apache-client-2.41.0.jar.sha1

@@ -0,0 +1 @@
+908533ce5db358d9b965ec87bdc5de4dd97fc0c9

solr/licenses/arns-2.41.0.jar.sha1

@@ -0,0 +1 @@
+fc2479d009ff23a6e8b29bfdbc65b097dcdf7451

solr/licenses/auth-2.41.0.jar.sha1

@@ -0,0 +1 @@
+3303638f0e18cb52b63b36435dc099dd23bcff38

solr/licenses/aws-core-2.41.0.jar.sha1

@@ -0,0 +1 @@
+8c43ee4145f1566651ddd484a3bece07ccb9af13

solr/licenses/aws-query-protocol-2.41.0.jar.sha1

@@ -0,0 +1 @@
+5eaa9a7ad5037963118d81825328f958bc78a28d

solr/licenses/aws-xml-protocol-2.41.0.jar.sha1

@@ -0,0 +1 @@
+6d6437cd2619553e7fce5d0499584d3afb399dde

solr/licenses/checksums-2.41.0.jar.sha1

@@ -0,0 +1 @@
+65efd61cbbf6e368cc1aa43c11d01d325987d794

solr/licenses/checksums-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+7996b92051d6348ef0429f97ac52e3b1c1ede292

solr/licenses/crt-core-2.41.0.jar.sha1

@@ -0,0 +1 @@
+ea1ace621f14f62f5677d32b343bb8e95f6ce064

solr/licenses/endpoints-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+8b183fc09f4996069d572b92a8ebabc8fd5c5747

solr/licenses/http-auth-2.41.0.jar.sha1

@@ -0,0 +1 @@
+fad29613fa347c9377c7f05e5bd425ac3f1c7b12

solr/licenses/http-auth-aws-2.41.0.jar.sha1

@@ -0,0 +1 @@
+31196b2da7f78ad5dd8c00229a9d2666104d96b6

solr/licenses/http-auth-aws-eventstream-2.41.0.jar.sha1

@@ -0,0 +1 @@
+3ad44b3298ae201982f6c78fd317689116242fab

solr/licenses/http-auth-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+dd75ccc1d5fb92d4b63847e86b14d56ee01fd470

solr/licenses/http-client-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+39f0d9581cb02d152917af7af1930a52526e0a36

solr/licenses/identity-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+e780d4fd05f77159671a124d1a354b0f1f156fbc

solr/licenses/json-utils-2.41.0.jar.sha1

@@ -0,0 +1 @@
+28669255117df36fadb30aa20bcfe7d1d7d8554f

solr/licenses/metrics-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+22db7213d930e6f80b03d8c8df0d5e9cb431008e

solr/licenses/profiles-2.41.0.jar.sha1

@@ -0,0 +1 @@
+67f147fbfd6ff92d7ff1c5143228643fcd795ce9

solr/licenses/protocol-core-2.41.0.jar.sha1

@@ -0,0 +1 @@
+92e71779441a3045cf6e71aefbc88e96a81c1a5b

solr/licenses/regions-2.41.0.jar.sha1

@@ -0,0 +1 @@
+32927c5905c139b3ebede179084f3a0184f445c7

solr/licenses/retries-2.41.0.jar.sha1

@@ -0,0 +1 @@
+3d08ea778e7c49741f02563020efd3f0fa14189d

solr/licenses/retries-spi-2.41.0.jar.sha1

@@ -0,0 +1 @@
+95dd9d0eed2c678bfd51f3f6d5caee2f1e873c29

solr/licenses/s3-2.41.0.jar.sha1

@@ -0,0 +1 @@
+9e8e9a162f502f31ef2020c05d9a379bce583015

solr/licenses/sdk-core-2.41.0.jar.sha1

@@ -0,0 +1 @@
+f83030af2f845f036dc1b529b917bdcd3e200234

solr/licenses/sts-2.41.0.jar.sha1

@@ -0,0 +1 @@
+cb69d34e47304181cae83809d07f3dc89fd4fc75

solr/licenses/third-party-jackson-core-2.41.0.jar.sha1

@@ -0,0 +1 @@
+eb418464f58798f68f800aa3e28257e47670e306

solr/licenses/url-connection-client-2.41.0.jar.sha1

@@ -0,0 +1 @@
+7b0bf0cc123ccc6256796ab298b26b0fb51a3873

solr/licenses/utils-2.41.0.jar.sha1

@@ -0,0 +1 @@
+ac0fbab3deab0720e168ac08eb9bb82c39972dc5

solr/licenses/utils-lite-2.41.0.jar.sha1

@@ -0,0 +1 @@
+a08dd775d9257f8680bf426541d9630d29fcd8f8

solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java

@@ -44,6 +44,7 @@
 import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.http.apache.ProxyConfiguration;
 import software.amazon.awssdk.regions.Region;
@@ -92,6 +93,9 @@ public class S3StorageClient {
   /** The S3 bucket where we read/write all data. */
   private final String bucketName;
 
+  /** The HTTP client used by the S3 client. Needs to be closed separately. */
+  private final SdkHttpClient httpClient;
+
   S3StorageClient(
       String bucketName,
       String profile,
@@ -100,30 +104,19 @@ public class S3StorageClient {
       boolean proxyUseSystemSettings,
       String endpoint,
       boolean disableRetries) {
-    this(
-        createInternalClient(
-            profile, region, proxyUrl, proxyUseSystemSettings, endpoint, disableRetries),
-        bucketName);
+    this.bucketName = bucketName;
+    this.httpClient = createHttpClient(proxyUrl, proxyUseSystemSettings);
+    this.s3Client = createInternalClient(profile, region, endpoint, disableRetries, httpClient);
   }
 
   @VisibleForTesting
   S3StorageClient(S3Client s3Client, String bucketName) {
     this.s3Client = s3Client;
     this.bucketName = bucketName;
+    this.httpClient = null;
   }
 
-  private static S3Client createInternalClient(
-      String profile,
-      String region,
-      String proxyUrl,
-      boolean proxyUseSystemSettings,
-      String endpoint,
-      boolean disableRetries) {
-    S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true);
-    if (StrUtils.isNotNullOrEmpty(profile)) {
-      configBuilder.profileName(profile);
-    }
-
+  private static SdkHttpClient createHttpClient(String proxyUrl, boolean proxyUseSystemSettings) {
     ApacheHttpClient.Builder sdkHttpClientBuilder = ApacheHttpClient.builder();
     // If configured, add proxy
     ProxyConfiguration.Builder proxyConfigurationBuilder = ProxyConfiguration.builder();
@@ -135,6 +128,20 @@ private static S3Client createInternalClient(
     sdkHttpClientBuilder.proxyConfiguration(proxyConfigurationBuilder.build());
     sdkHttpClientBuilder.useIdleConnectionReaper(false);
 
+    return sdkHttpClientBuilder.build();
+  }
+
+  private static S3Client createInternalClient(
+      String profile,
+      String region,
+      String endpoint,
+      boolean disableRetries,
+      SdkHttpClient httpClient) {
+    S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true);
+    if (StrUtils.isNotNullOrEmpty(profile)) {
+      configBuilder.profileName(profile);
+    }
+
     /*
      * Retry logic
      */
@@ -167,7 +174,7 @@ private static S3Client createInternalClient(
             .credentialsProvider(credentialsProviderBuilder.build())
             .overrideConfiguration(builder -> builder.retryStrategy(retryStrategy))
             .serviceConfiguration(configBuilder.build())
-            .httpClient(sdkHttpClientBuilder.build());
+            .httpClient(httpClient);
 
     if (StrUtils.isNotNullOrEmpty(endpoint)) {
       clientBuilder.endpointOverride(URI.create(endpoint));
@@ -374,23 +381,27 @@ InputStream pullStream(String path) throws S3Exception {
       GetObjectRequest.Builder getBuilder =
           GetObjectRequest.builder().bucket(bucketName).key(s3Path);
       // This InputStream instance needs to be closed by the caller
-      return s3Client.getObject(
-          getBuilder.build(),
-          ResponseTransformer.unmanaged(
-              (response, inputStream) -> {
-                final long contentLength = response.contentLength();
-                return new ResumableInputStream(
-                    inputStream,
-                    bytesRead -> {
-                      if (contentLength > 0 && bytesRead >= contentLength) {
-                        // No more bytes to read
-                        return null;
-                      } else if (bytesRead > 0) {
-                        getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead));
-                      }
-                      return s3Client.getObject(getBuilder.build());
-                    });
-              }));
+      // Use Duration.ZERO to disable timeout and prevent response-input-stream-timeout-scheduler
+      // thread leak (see https://github.com/aws/aws-sdk-java-v2/issues/6567)
+      software.amazon.awssdk.core.ResponseInputStream<
+              software.amazon.awssdk.services.s3.model.GetObjectResponse>
+          responseStream =
+              s3Client.getObject(
+                  getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO));
+      final long contentLength = responseStream.response().contentLength();
+      return new ResumableInputStream(
+          responseStream,
+          bytesRead -> {
+            if (contentLength > 0 && bytesRead >= contentLength) {
+              // No more bytes to read
+              return null;
+            } else if (bytesRead > 0) {
+              getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead));
+            }
+            // Use Duration.ZERO to disable timeout on resumed streams as well
+            return s3Client.getObject(
+                getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO));
+          });
     } catch (SdkException sdke) {
       throw handleAmazonException(sdke);
     }
@@ -419,6 +430,9 @@ OutputStream pushStream(String path) throws S3Exception {
   /** Override {@link Closeable} since we throw no exception. */
   void close() {
     s3Client.close();
+    if (httpClient != null) {
+      httpClient.close();
+    }
   }
 
   /** Any file path that specifies a non-existent file will not be treated as an error. */
@@ -544,7 +558,7 @@ private String getParentDirectory(String path) {
   }
 
   /** Ensures path adheres to some rules: -Doesn't start with a leading slash */
-  String sanitizedPath(String path) throws S3Exception {
+  String sanitizedPath(String path) {
     // Trim space from start and end
     String sanitizedPath = path.trim();
 
@@ -580,7 +594,7 @@ String sanitizedFilePath(String path) throws S3Exception {
    * Ensures directory path adheres to some rules: -Overall Path rules from `sanitizedPath` -Add a
    * trailing slash if one does not exist
    */
-  String sanitizedDirPath(String path) throws S3Exception {
+  String sanitizedDirPath(String path) {
     // Trim space from start and end
     String sanitizedPath = sanitizedPath(path);
 

solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java

@@ -338,7 +338,11 @@ private void pushObject(String path, String content) {
   private File pullObject(String path) throws IOException {
     try (S3Client s3 = S3_MOCK_RULE.createS3ClientV2()) {
       File file = temporaryFolder.newFile();
-      InputStream input = s3.getObject(b -> b.bucket(BUCKET_NAME).key(path));
+      InputStream input =
+          s3.getObject(
+              b -> b.bucket(BUCKET_NAME).key(path),
+              software.amazon.awssdk.core.sync.ResponseTransformer.toInputStream(
+                  java.time.Duration.ZERO));
       FileUtils.copyInputStreamToFile(input, file);
       return file;
     }

solr/modules/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java

@@ -21,11 +21,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import org.apache.solr.SolrTestCaseJ4;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
 import software.amazon.awssdk.services.s3.S3Client;
 
 public class S3OutputStreamTest extends SolrTestCaseJ4 {
@@ -63,7 +65,10 @@ public void testWriteByteByByte() throws IOException {
     }
 
     // Check we can re-read same content
-    try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"))) {
+    try (InputStream input =
+        s3.getObject(
+            b -> b.bucket(BUCKET).key("byte-by-byte"),
+            ResponseTransformer.toInputStream(Duration.ZERO))) {
       String read = new String(input.readAllBytes(), StandardCharsets.UTF_8);
       assertEquals("Contents saved to S3 file did not match expected", "hello", read);
     }
@@ -82,7 +87,10 @@ public void testWriteSmallBuffer() throws IOException {
     }
 
     // Check we can re-read same content
-    try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"))) {
+    try (InputStream input =
+        s3.getObject(
+            b -> b.bucket(BUCKET).key("small-buffer"),
+            ResponseTransformer.toInputStream(Duration.ZERO))) {
       String read = new String(input.readAllBytes(), StandardCharsets.UTF_8);
       assertEquals("hello", read);
     }
@@ -103,7 +111,10 @@ public void testWriteLargeBuffer() throws IOException {
     }
 
     // Check we can re-read same content
-    try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"))) {
+    try (InputStream input =
+        s3.getObject(
+            b -> b.bucket(BUCKET).key("large-buffer"),
+            ResponseTransformer.toInputStream(Duration.ZERO))) {
       String read = new String(input.readAllBytes(), StandardCharsets.UTF_8);
       assertEquals(new String(buffer, StandardCharsets.UTF_8), read);
     }
@@ -125,7 +136,10 @@ public void testFlushSmallBuffer() throws IOException {
     }
 
     // Check we can re-read same content
-    try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"))) {
+    try (InputStream input =
+        s3.getObject(
+            b -> b.bucket(BUCKET).key("flush-small"),
+            ResponseTransformer.toInputStream(Duration.ZERO))) {
       String read = new String(input.readAllBytes(), StandardCharsets.UTF_8);
       assertEquals(
           "Flushing a small frame of an S3OutputStream should not impact data written",
@@ -152,7 +166,10 @@ public void testFlushLargeBuffer() throws IOException {
     }
 
     // Check we can re-read same content
-    try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"))) {
+    try (InputStream input =
+        s3.getObject(
+            b -> b.bucket(BUCKET).key("flush-large"),
+            ResponseTransformer.toInputStream(Duration.ZERO))) {
       String read = new String(input.readAllBytes(), StandardCharsets.UTF_8);
       assertEquals(
           "Flushing a large frame of an S3OutputStream should not impact data written",