Add support for request compression

aws/client/aws-client-awsjson/src/it/java/software/amazon/smithy/java/client/aws/jsonprotocols/AwsJson1ProtocolTests.java

@@ -20,10 +20,6 @@ public class AwsJson1ProtocolTests {
     @HttpClientRequestTests
     @ProtocolTestFilter(
             skipTests = {
-                    // TODO: implement content-encoding
-                    "SDKAppliedContentEncoding_awsJson1_0",
-                    "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_0",
-
                     // Skipping top-level input defaults isn't necessary in Smithy-Java given it uses builders and
                     // the defaults don't impact nullability. This applies to the following tests.
                     "AwsJson10ClientSkipsTopLevelDefaultValuesInInput",
@@ -42,8 +38,9 @@ public void requestTest(DataStream expected, DataStream actual) {
                     Node.parse(new String(ByteBufferUtils.getBytes(expected.asByteBuffer()),
                             StandardCharsets.UTF_8)));
         }
-        assertEquals(expectedJson, new StringBuildingSubscriber(actual).getResult());
-
+        if (expected.contentType() != null) { // Skip request compression tests since they do not have expected body
+            assertEquals(expectedJson, new StringBuildingSubscriber(actual).getResult());
+        }
     }
 
     @HttpClientResponseTests

aws/client/aws-client-restjson/src/it/java/software/amazon/smithy/java/client/aws/restjson/RestJson1ProtocolTests.java

@@ -26,8 +26,6 @@
         skipOperations = {
                 // We dont ignore defaults on input shapes
                 "aws.protocoltests.restjson#OperationWithDefaults",
-                // TODO: support content-encoding
-                "aws.protocoltests.restjson#PutWithContentEncoding"
         })
 public class RestJson1ProtocolTests {
     private static final String EMPTY_BODY = "";
@@ -50,7 +48,7 @@ public void requestTest(DataStream expected, DataStream actual) {
             } else {
                 assertEquals(expectedStr, actualStr);
             }
-        } else {
+        } else if (expected.contentType() != null) { // Skip request compression tests since they do not have expected body
             assertEquals(EMPTY_BODY, actualStr);
         }
     }

aws/client/aws-client-restxml/src/it/java/software/amazon/smithy/java/aws/client/restxml/RestXmlProtocolTests.java

@@ -25,7 +25,6 @@
 import software.amazon.smithy.java.protocoltests.harness.HttpClientRequestTests;
 import software.amazon.smithy.java.protocoltests.harness.HttpClientResponseTests;
 import software.amazon.smithy.java.protocoltests.harness.ProtocolTest;
-import software.amazon.smithy.java.protocoltests.harness.ProtocolTestFilter;
 import software.amazon.smithy.java.protocoltests.harness.StringBuildingSubscriber;
 import software.amazon.smithy.java.protocoltests.harness.TestType;
 
@@ -34,11 +33,6 @@
         testType = TestType.CLIENT)
 public class RestXmlProtocolTests {
     @HttpClientRequestTests
-    @ProtocolTestFilter(
-            skipTests = {
-                    "SDKAppliedContentEncoding_restXml",
-                    "SDKAppendedGzipAfterProvidedEncoding_restXml",
-            })
     public void requestTest(DataStream expected, DataStream actual) {
         if (expected.contentLength() != 0) {
             var a = new String(ByteBufferUtils.getBytes(actual.asByteBuffer()), StandardCharsets.UTF_8);
@@ -51,7 +45,7 @@ public void requestTest(DataStream expected, DataStream actual) {
             } else {
                 assertEquals(a, b);
             }
-        } else {
+        } else if (expected.contentType() != null) { // Skip request compression tests since they do not have expected body
             assertEquals("", new StringBuildingSubscriber(actual).getResult());
         }
     }

client/client-core/src/test/java/software/amazon/smithy/java/client/core/ClientTest.java

@@ -35,6 +35,7 @@
 import software.amazon.smithy.java.client.http.mock.MockQueue;
 import software.amazon.smithy.java.client.http.plugins.ApplyHttpRetryInfoPlugin;
 import software.amazon.smithy.java.client.http.plugins.HttpChecksumPlugin;
+import software.amazon.smithy.java.client.http.plugins.RequestCompressionPlugin;
 import software.amazon.smithy.java.client.http.plugins.UserAgentPlugin;
 import software.amazon.smithy.java.core.serde.document.Document;
 import software.amazon.smithy.java.dynamicclient.DynamicClient;
@@ -83,6 +84,7 @@ public class ClientTest {
             SimpleAuthDetectionPlugin.class,
             UserAgentPlugin.class,
             ApplyHttpRetryInfoPlugin.class,
+            RequestCompressionPlugin.class,
             HttpChecksumPlugin.class,
             FooPlugin.class);
 

client/client-http/src/main/java/software/amazon/smithy/java/client/http/HttpContext.java

@@ -27,5 +27,17 @@ public final class HttpContext {
     public static final Context.Key<HttpHeaders> ENDPOINT_RESOLVER_HTTP_HEADERS = Context.key(
             "HTTP headers to use with the request returned from an endpoint resolver");
 
+    /**
+     * The minimum length of bytes threshold for a request body to be compressed. Defaults to 10240 bytes if not set.
+     */
+    public static final Context.Key<Integer> REQUEST_MIN_COMPRESSION_SIZE_BYTES =
+            Context.key("Minimum bytes size for request compression");
+
+    /**
+     * If request compression is disabled.
+     */
+    public static final Context.Key<Boolean> DISABLE_REQUEST_COMPRESSION =
+            Context.key("If request compression is disabled");
+
     private HttpContext() {}
 }

client/client-http/src/main/java/software/amazon/smithy/java/client/http/compression/CompressionAlgorithm.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.client.http.compression;
+
+import java.util.List;
+import software.amazon.smithy.java.io.datastream.DataStream;
+import software.amazon.smithy.utils.ListUtils;
+
+/**
+ * Represents a compression algorithm that can be used to compress request
+ * bodies.
+ */
+public interface CompressionAlgorithm {
+    /**
+     * The ID of the compression algorithm. This is matched against the algorithm
+     * names used in the trait e.g. "gzip"
+     */
+    String algorithmId();
+
+    /**
+     * Compresses content of fixed length
+     */
+    DataStream compress(DataStream data);
+
+    static List<CompressionAlgorithm> supportedAlgorithms() {
+        return ListUtils.of(new Gzip());
+    }
+}

client/client-http/src/main/java/software/amazon/smithy/java/client/http/compression/Gzip.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.client.http.compression;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.zip.GZIPOutputStream;
+import software.amazon.smithy.java.io.ByteBufferOutputStream;
+import software.amazon.smithy.java.io.datastream.DataStream;
+
+public final class Gzip implements CompressionAlgorithm {
+
+    @Override
+    public String algorithmId() {
+        return "gzip";
+    }
+
+    @Override
+    public DataStream compress(DataStream data) {
+        if (!data.hasKnownLength()) { // Using streaming
+            return DataStream.ofInputStream(
+                    new GzipCompressingInputStream(data.asInputStream()),
+                    data.contentType(),
+                    -1);
+        }
+
+        try (var bos = new ByteBufferOutputStream();
+                var in = data.asInputStream()) {
+            var gzip = new GZIPOutputStream(bos);
+            in.transferTo(gzip);
+            gzip.close();
+            return DataStream.ofBytes(bos.toByteBuffer().array());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}

client/client-http/src/main/java/software/amazon/smithy/java/client/http/compression/GzipCompressingInputStream.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.client.http.compression;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPOutputStream;
+
+final class GzipCompressingInputStream extends InputStream {
+
+    private final InputStream source;
+    private byte[] compressedData;
+    private int position = 0;
+
+    public GzipCompressingInputStream(InputStream source) {
+        this.source = source;
+    }
+
+    @Override
+    public int read() throws IOException {
+        ensureCompressed();
+        return position < compressedData.length ? compressedData[position++] & 0xFF : -1;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        ensureCompressed();
+        var available = compressedData.length - position;
+        if (available <= 0)
+            return -1;
+
+        var toRead = Math.min(len, available);
+        System.arraycopy(compressedData, position, b, off, toRead);
+        position += toRead;
+        return toRead;
+    }
+
+    private void ensureCompressed() throws IOException {
+        if (compressedData == null) {
+            var buffer = new ByteArrayOutputStream();
+            try (GZIPOutputStream gzip = new GZIPOutputStream(buffer)) {
+                source.transferTo(gzip);
+            }
+            compressedData = buffer.toByteArray();
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        source.close();
+    }
+}

client/client-http/src/main/java/software/amazon/smithy/java/client/http/plugins/RequestCompressionPlugin.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.client.http.plugins;
+
+import java.util.List;
+import software.amazon.smithy.java.client.core.AutoClientPlugin;
+import software.amazon.smithy.java.client.core.ClientConfig;
+import software.amazon.smithy.java.client.core.interceptors.ClientInterceptor;
+import software.amazon.smithy.java.client.core.interceptors.RequestHook;
+import software.amazon.smithy.java.client.http.HttpContext;
+import software.amazon.smithy.java.client.http.HttpMessageExchange;
+import software.amazon.smithy.java.client.http.compression.CompressionAlgorithm;
+import software.amazon.smithy.java.context.Context;
+import software.amazon.smithy.java.core.schema.TraitKey;
+import software.amazon.smithy.java.http.api.HttpRequest;
+import software.amazon.smithy.java.io.datastream.DataStream;
+import software.amazon.smithy.model.traits.RequestCompressionTrait;
+import software.amazon.smithy.utils.SmithyInternalApi;
+
+/**
+ * Compress the request body using provided compression algorithm if @requestCompression trait is applied.
+ */
+@SmithyInternalApi
+public final class RequestCompressionPlugin implements AutoClientPlugin {
+
+    @Override
+    public void configureClient(ClientConfig.Builder config) {
+        if (config.isUsingMessageExchange(HttpMessageExchange.INSTANCE)) {
+            config.addInterceptor(RequestCompressionInterceptor.INSTANCE);
+        }
+    }
+
+    static final class RequestCompressionInterceptor implements ClientInterceptor {
+
+        private static final int DEFAULT_MIN_COMPRESSION_SIZE_BYTES = 10240;
+        // This cap matches ApiGateway's spec: https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-openapi-minimum-compression-size.html
+        private static final int MIN_COMPRESSION_SIZE_CAP = 10485760;
+        private static final String CONTENT_ENCODING_HEADER = "Content-Encoding";
+        private static final ClientInterceptor INSTANCE = new RequestCompressionInterceptor();
+        private static final TraitKey<RequestCompressionTrait> REQUEST_COMPRESSION_TRAIT_KEY =
+                TraitKey.get(RequestCompressionTrait.class);
+        // Currently only Gzip is supported in Smithy model: https://smithy.io/2.0/spec/behavior-traits.html#requestcompression-trait
+        private static final List<CompressionAlgorithm> supportedAlgorithms =
+                CompressionAlgorithm.supportedAlgorithms();
+
+        @Override
+        public <RequestT> RequestT modifyBeforeTransmit(RequestHook<?, ?, RequestT> hook) {
+            return hook.mapRequest(HttpRequest.class, RequestCompressionInterceptor::processRequest);
+        }
+
+        private static HttpRequest processRequest(RequestHook<?, ?, HttpRequest> hook) {
+            if (shouldCompress(hook)) {
+                var compressionTrait =
+                        hook.operation().schema().getTrait(REQUEST_COMPRESSION_TRAIT_KEY);
+                var request = hook.request();
+                // Will pick the first supported algorithm to compress the body.
+                for (var algorithmId : compressionTrait.getEncodings()) {
+                    for (var algorithm : supportedAlgorithms) {
+                        if (algorithmId.equals(algorithm.algorithmId())) {
+                            var compressed = algorithm.compress(request.body());
+                            return request.toBuilder()
+                                    .body(compressed)
+                                    .withAddedHeader(CONTENT_ENCODING_HEADER, algorithmId)
+                                    .build();
+                        }
+                    }
+                }
+            }
+            return hook.request();
+        }
+
+        private static boolean shouldCompress(RequestHook<?, ?, HttpRequest> hook) {
+            var context = hook.context();
+            var operation = hook.operation();
+            if (!operation.schema().hasTrait(REQUEST_COMPRESSION_TRAIT_KEY)
+                    || context.getOrDefault(HttpContext.DISABLE_REQUEST_COMPRESSION, false)) {
+                return false;
+            }
+            var requestBody = hook.request().body();
+            // Streaming should not have known length
+            if (operation.inputStreamMember() != null && !requestBody.hasKnownLength()) {
+                return true;
+            }
+            return isBodySizeValid(requestBody, context);
+        }
+
+        private static boolean isBodySizeValid(DataStream requestBody, Context context) {
+            var minCompressionSize = context.getOrDefault(HttpContext.REQUEST_MIN_COMPRESSION_SIZE_BYTES,
+                    DEFAULT_MIN_COMPRESSION_SIZE_BYTES);
+            validateCompressionSize(minCompressionSize);
+            return requestBody.contentLength() >= minCompressionSize;
+        }
+
+        private static void validateCompressionSize(int minCompressionSize) {
+            if (minCompressionSize < 0 || minCompressionSize > MIN_COMPRESSION_SIZE_CAP) {
+                throw new IllegalArgumentException("Min compression size must be between 0 and 10485760");
+            }
+        }
+    }
+}

client/client-http/src/main/resources/META-INF/services/software.amazon.smithy.java.client.core.AutoClientPlugin

@@ -1,3 +1,4 @@
 software.amazon.smithy.java.client.http.plugins.UserAgentPlugin
 software.amazon.smithy.java.client.http.plugins.ApplyHttpRetryInfoPlugin
+software.amazon.smithy.java.client.http.plugins.RequestCompressionPlugin
 software.amazon.smithy.java.client.http.plugins.HttpChecksumPlugin