diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java index c6eff8066cd..a67bd28317a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java @@ -31,6 +31,15 @@ /** Utilities for unit-testing S3 endpoints. */ public final class EndpointTestUtils { + /** Get key tags. */ + public static Response getTagging( + ObjectEndpoint subject, + String bucket, + String key + ) throws IOException, OS3Exception { + return subject.get(bucket, key, 0, null, 0, null, ""); + } + /** Put without content. */ public static Response putDir( ObjectEndpoint subject, @@ -50,6 +59,23 @@ public static Response put( return put(subject, bucket, key, 0, null, content); } + /** Add tagging on key. */ + public static Response putTagging( + ObjectEndpoint subject, + String bucket, + String key, + String content + ) throws IOException, OS3Exception { + if (content == null) { + return subject.put(bucket, key, 0, 0, null, "", null, null); + } else { + final long length = content.length(); + try (ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes(UTF_8))) { + return subject.put(bucket, key, length, 0, null, "", null, body); + } + } + } + /** Put with content, part number, upload ID. */ public static Response put( ObjectEndpoint subject, @@ -69,6 +95,15 @@ public static Response put( } } + /** Delete key tags. */ + public static Response deleteTagging( + ObjectEndpoint subject, + String bucket, + String key + ) throws IOException, OS3Exception { + return subject.delete(bucket, key, null, ""); + } + /** Verify response is success for {@code request}. */ public static void assertSucceeds(CheckedSupplier request) throws E { try (Response response = request.get()) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java index 488474e3039..6cf7eea1336 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingDelete.java @@ -17,10 +17,10 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; import static java.net.HttpURLConnection.HTTP_NO_CONTENT; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.deleteTagging; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; @@ -28,16 +28,13 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -65,29 +62,25 @@ public class TestObjectTaggingDelete { @BeforeEach public void init() throws OS3Exception, IOException { //GIVEN - OzoneConfiguration config = new OzoneConfiguration(); client = new OzoneClientStub(); client.getObjectStore().createS3Bucket(BUCKET_NAME); HttpHeaders headers = Mockito.mock(HttpHeaders.class); rest = EndpointBuilder.newObjectEndpointBuilder() .setClient(client) - .setConfig(config) .setHeaders(headers) .build(); - ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); // Create a key with object tags Mockito.when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) .thenReturn("mockSignature"); - rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(), - 1, null, null, null, body); + put(rest, BUCKET_NAME, KEY_WITH_TAG, CONTENT); } @Test public void testDeleteTagging() throws IOException, OS3Exception { - Response response = rest.delete(BUCKET_NAME, KEY_WITH_TAG, null, ""); + Response response = deleteTagging(rest, BUCKET_NAME, KEY_WITH_TAG); assertEquals(HTTP_NO_CONTENT, response.getStatus()); assertTrue(client.getObjectStore().getS3Bucket(BUCKET_NAME) @@ -95,25 +88,13 @@ public void testDeleteTagging() throws IOException, OS3Exception { } @Test - public void testDeleteTaggingNoKeyFound() throws Exception { - try { - rest.delete(BUCKET_NAME, "nonexistent", null, ""); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_KEY.getCode(), ex.getCode()); - } + public void testDeleteTaggingNoKeyFound() { + assertErrorResponse(NO_SUCH_KEY, () -> deleteTagging(rest, BUCKET_NAME, "nonexistent")); } @Test - public void testDeleteTaggingNoBucketFound() throws Exception { - try { - rest.delete("nonexistent", "nonexistent", null, ""); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_BUCKET.getCode(), ex.getCode()); - } + public void testDeleteTaggingNoBucketFound() { + assertErrorResponse(NO_SUCH_BUCKET, () -> deleteTagging(rest, "nonexistent", "any")); } @Test @@ -134,12 +115,6 @@ public void testDeleteObjectTaggingNotImplemented() throws Exception { doThrow(new OMException("DeleteObjectTagging is not currently supported for FSO directory", ResultCodes.NOT_SUPPORTED_OPERATION)).when(mockBucket).deleteObjectTagging("dir/"); - try { - endpoint.delete("fsoBucket", "dir/", null, ""); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_IMPLEMENTED, ex.getHttpCode()); - assertEquals(NOT_IMPLEMENTED.getCode(), ex.getCode()); - } + assertErrorResponse(NOT_IMPLEMENTED, () -> deleteTagging(endpoint, "fsoBucket", "dir/")); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java index 1885e7d0cf6..94942479cff 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingGet.java @@ -17,9 +17,11 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getTagging; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; @@ -28,11 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; -import java.io.ByteArrayInputStream; import java.io.IOException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.s3.endpoint.S3Tagging.Tag; @@ -52,9 +52,8 @@ public class TestObjectTaggingGet { private ObjectEndpoint rest; @BeforeEach - public void init() throws OS3Exception, IOException { + public void init() throws Exception { //GIVEN - OzoneConfiguration config = new OzoneConfiguration(); OzoneClient client = new OzoneClientStub(); client.getObjectStore().createS3Bucket(BUCKET_NAME); @@ -64,21 +63,18 @@ public void init() throws OS3Exception, IOException { rest = EndpointBuilder.newObjectEndpointBuilder() .setClient(client) - .setConfig(config) .setHeaders(headers) .build(); - ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); // Create a key with object tags Mockito.when(headers.getHeaderString(TAG_HEADER)).thenReturn("tag1=value1&tag2=value2"); - rest.put(BUCKET_NAME, KEY_WITH_TAG, CONTENT.length(), - 1, null, null, null, body); + assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_WITH_TAG, CONTENT)); } @Test public void testGetTagging() throws IOException, OS3Exception { //WHEN - Response response = rest.get(BUCKET_NAME, KEY_WITH_TAG, 0, null, 0, null, ""); + Response response = getTagging(rest, BUCKET_NAME, KEY_WITH_TAG); assertEquals(HTTP_OK, response.getStatus()); S3Tagging s3Tagging = (S3Tagging) response.getEntity(); @@ -97,24 +93,12 @@ public void testGetTagging() throws IOException, OS3Exception { } @Test - public void testGetTaggingNoKeyFound() throws Exception { - try { - rest.get(BUCKET_NAME, "nonexistent", 0, null, 0, null, ""); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_KEY.getCode(), ex.getCode()); - } + public void testGetTaggingNoKeyFound() { + assertErrorResponse(NO_SUCH_KEY, () -> getTagging(rest, BUCKET_NAME, "nonexistent")); } @Test - public void testGetTaggingNoBucketFound() throws Exception { - try { - rest.get("nonexistent", "nonexistent", 0, null, 0, null, ""); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_BUCKET.getCode(), ex.getCode()); - } + public void testGetTaggingNoBucketFound() { + assertErrorResponse(NO_SUCH_BUCKET, () -> getTagging(rest, "nonexistent", "any")); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java index d1651d6b59c..f6f26515ea9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectTaggingPut.java @@ -17,30 +17,23 @@ package org.apache.hadoop.ozone.s3.endpoint; -import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putTagging; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; +import com.google.common.collect.ImmutableMap; import java.util.Map; -import java.util.function.Supplier; import javax.ws.rs.core.HttpHeaders; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -50,7 +43,6 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; -import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -65,15 +57,11 @@ public class TestObjectTaggingPut { private static final String BUCKET_NAME = "b1"; private static final String KEY_NAME = "key=value/1"; + private static final Map TAGS = ImmutableMap.of("tag1", "val1", "tag2", "val2"); @BeforeEach - void setup() throws IOException, OS3Exception { - OzoneConfiguration config = new OzoneConfiguration(); - - //Create client stub and object store stub. + void setup() throws Exception { clientStub = new OzoneClientStub(); - - // Create bucket clientStub.getObjectStore().createS3Bucket(BUCKET_NAME); HttpHeaders headers = mock(HttpHeaders.class); @@ -82,84 +70,44 @@ void setup() throws IOException, OS3Exception { // Create PutObject and setClient to OzoneClientStub objectEndpoint = EndpointBuilder.newObjectEndpointBuilder() .setClient(clientStub) - .setConfig(config) .setHeaders(headers) .build(); - - ByteArrayInputStream body = - new ByteArrayInputStream("".getBytes(UTF_8)); - - objectEndpoint.put(BUCKET_NAME, KEY_NAME, 0, 1, null, null, null, body); + assertSucceeds(() -> put(objectEndpoint, BUCKET_NAME, KEY_NAME, "")); } @Test - public void testPutObjectTaggingWithEmptyBody() throws Exception { - try { - objectEndpoint.put(BUCKET_NAME, KEY_NAME, 0, 1, null, "", null, - null); - fail(); - } catch (OS3Exception ex) { - assertEquals(HTTP_BAD_REQUEST, ex.getHttpCode()); - assertEquals(MALFORMED_XML.getCode(), ex.getCode()); - } + public void testPutObjectTaggingWithEmptyBody() { + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, "")); } @Test public void testPutValidObjectTagging() throws Exception { - assertEquals(HTTP_OK, objectEndpoint.put(BUCKET_NAME, KEY_NAME, 0, 1, null, - "", null, twoTags()).getStatus()); + assertSucceeds(() -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, twoTags())); OzoneKeyDetails keyDetails = clientStub.getObjectStore().getS3Bucket(BUCKET_NAME).getKey(KEY_NAME); - assertEquals(2, keyDetails.getTags().size()); - assertEquals("val1", keyDetails.getTags().get("tag1")); - assertEquals("val2", keyDetails.getTags().get("tag2")); + assertThat(keyDetails.getTags()) + .containsExactlyEntriesOf(TAGS); } @Test - public void testPutInvalidObjectTagging() throws Exception { - testInvalidObjectTagging(this::emptyBody, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - testInvalidObjectTagging(this::invalidXmlStructure, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - testInvalidObjectTagging(this::noTagSet, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - testInvalidObjectTagging(this::emptyTags, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - testInvalidObjectTagging(this::tagKeyNotSpecified, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - testInvalidObjectTagging(this::tagValueNotSpecified, HTTP_BAD_REQUEST, MALFORMED_XML.getCode()); - } - - private void testInvalidObjectTagging(Supplier inputStream, - int expectedHttpCode, String expectedErrorCode) throws Exception { - try { - objectEndpoint.put(BUCKET_NAME, KEY_NAME, 0, 1, null, "", null, - inputStream.get()); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(expectedHttpCode, ex.getHttpCode()); - assertEquals(expectedErrorCode, ex.getCode()); - } + public void testPutInvalidObjectTagging() { + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, emptyBody())); + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, invalidXmlStructure())); + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, noTagSet())); + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, emptyTags())); + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, tagKeyNotSpecified())); + assertErrorResponse(MALFORMED_XML, () -> putTagging(objectEndpoint, BUCKET_NAME, KEY_NAME, tagValueNotSpecified())); } @Test - public void testPutObjectTaggingNoKeyFound() throws Exception { - try { - objectEndpoint.put(BUCKET_NAME, "nonexistent", 0, 1, - null, "", null, twoTags()); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_KEY.getCode(), ex.getCode()); - } + public void testPutObjectTaggingNoKeyFound() { + assertErrorResponse(NO_SUCH_KEY, () -> putTagging(objectEndpoint, BUCKET_NAME, "nonexistent", twoTags())); } @Test - public void testPutObjectTaggingNoBucketFound() throws Exception { - try { - objectEndpoint.put("nonexistent", "nonexistent", 0, 1, - null, "", null, twoTags()); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); - assertEquals(NO_SUCH_BUCKET.getCode(), ex.getCode()); - } + public void testPutObjectTaggingNoBucketFound() { + assertErrorResponse(NO_SUCH_BUCKET, () -> putTagging(objectEndpoint, "nonexistent", "any", twoTags())); } @Test @@ -177,40 +125,27 @@ public void testPutObjectTaggingNotImplemented() throws Exception { ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() .setClient(mockClient) .build(); - Map twoTagsMap = new HashMap<>(); - twoTagsMap.put("tag1", "val1"); - twoTagsMap.put("tag2", "val2"); - doThrow(new OMException("PutObjectTagging is not currently supported for FSO directory", - ResultCodes.NOT_SUPPORTED_OPERATION)).when(mockBucket).putObjectTagging("dir/", twoTagsMap); + ResultCodes.NOT_SUPPORTED_OPERATION)).when(mockBucket).putObjectTagging("dir/", TAGS); - try { - endpoint.put("fsoBucket", "dir/", 0, 1, null, "", - null, twoTags()); - fail("Expected an OS3Exception to be thrown"); - } catch (OS3Exception ex) { - assertEquals(HTTP_NOT_IMPLEMENTED, ex.getHttpCode()); - assertEquals(NOT_IMPLEMENTED.getCode(), ex.getCode()); - } + assertErrorResponse(NOT_IMPLEMENTED, () -> putTagging(endpoint, "fsoBucket", "dir/", twoTags())); } - private InputStream emptyBody() { + private String emptyBody() { return null; } - private InputStream invalidXmlStructure() { - String xml = + private String invalidXmlStructure() { + return "" + " " + " "; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } - private InputStream twoTags() { - String xml = + private String twoTags() { + return "" + " " + " " + @@ -223,29 +158,24 @@ private InputStream twoTags() { " " + " " + ""; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } - private InputStream noTagSet() { - String xml = + private String noTagSet() { + return "" + ""; - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } - private InputStream emptyTags() { - String xml = + private String emptyTags() { + return "" + " " + " " + ""; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } - public InputStream tagKeyNotSpecified() { - String xml = + public String tagKeyNotSpecified() { + return "" + " " + " " + @@ -253,12 +183,10 @@ public InputStream tagKeyNotSpecified() { " " + " " + ""; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } - public InputStream tagValueNotSpecified() { - String xml = + public String tagValueNotSpecified() { + return "" + " " + " " + @@ -266,8 +194,6 @@ public InputStream tagValueNotSpecified() { " " + " " + ""; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 8e9eef2d974..a51f4c60d61 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -18,7 +18,9 @@ package org.apache.hadoop.ozone.s3.endpoint; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putTagging; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -34,9 +36,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; @@ -276,10 +277,7 @@ public void testPutKey() throws IOException { .setConfig(conf) .build(); - OS3Exception e = assertThrows(OS3Exception.class, () -> objectEndpoint.put( - "bucketName", "keyPath", 1024, 0, null, null, null, - new ByteArrayInputStream(new byte[]{}))); - assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); + assertErrorResponse(S3ErrorTable.ACCESS_DENIED, () -> put(objectEndpoint, "bucketName", "keyPath", "")); } @Test @@ -337,20 +335,11 @@ public void testObjectTagging() throws Exception { " " + ""; - InputStream tagInput = new ByteArrayInputStream(xml.getBytes(UTF_8)); - - OS3Exception e = assertThrows(OS3Exception.class, () -> - objectEndpoint.put("bucketName", "keyPath", 0, 1, - null, "", null, tagInput)); - assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); - - e = assertThrows(OS3Exception.class, () -> - objectEndpoint.delete("bucketName", "keyPath", "", "")); - assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); - - e = assertThrows(OS3Exception.class, () -> - objectEndpoint.get("bucketName", "keyPath", 0, null, - 0, null, "")); - assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); + assertErrorResponse(S3ErrorTable.ACCESS_DENIED, + () -> putTagging(objectEndpoint, "bucketName", "keyPath", xml)); + assertErrorResponse(S3ErrorTable.ACCESS_DENIED, + () -> objectEndpoint.delete("bucketName", "keyPath", null, "")); + assertErrorResponse(S3ErrorTable.ACCESS_DENIED, + () -> objectEndpoint.get("bucketName", "keyPath", 0, null, 0, null, "")); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index bfc471e22d5..1849bd81e40 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -19,7 +19,12 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static java.net.HttpURLConnection.HTTP_OK; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.deleteTagging; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getTagging; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putTagging; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS; import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; @@ -32,7 +37,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -288,28 +292,18 @@ public void testHeadKeyFailure() throws Exception { @Test public void testCreateKeySuccess() throws Exception { - long oriMetric = metrics.getCreateKeySuccess(); - // Create an input stream - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - // Create the file - keyEndpoint.put(bucketName, keyName, CONTENT - .length(), 1, null, null, null, body); - body.close(); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); long curMetric = metrics.getCreateKeySuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testCreateKeyFailure() throws Exception { + public void testCreateKeyFailure() { long oriMetric = metrics.getCreateKeyFailure(); // Create the file in a bucket that does not exist - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.put( - "unknownBucket", keyName, CONTENT.length(), 1, null, null, - null, null)); - assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_BUCKET, () -> put(keyEndpoint, "unknownBucket", keyName, CONTENT)); long curMetric = metrics.getCreateKeyFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -325,11 +319,9 @@ public void testDeleteKeySuccess() throws Exception { } @Test - public void testDeleteKeyFailure() throws Exception { + public void testDeleteKeyFailure() { long oriMetric = metrics.getDeleteKeyFailure(); - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.delete( - "unknownBucket", keyName, null, null)); - assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_BUCKET, () -> keyEndpoint.delete("unknownBucket", keyName, null, null)); long curMetric = metrics.getDeleteKeyFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -338,12 +330,8 @@ public void testDeleteKeyFailure() throws Exception { public void testGetKeySuccess() throws Exception { long oriMetric = metrics.getGetKeySuccess(); - // Create an input stream - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); // Create the file - keyEndpoint.put(bucketName, keyName, CONTENT - .length(), 1, null, null, null, body); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); // GET the key from the bucket Response response = keyEndpoint.get(bucketName, keyName, 0, null, 0, null, null); StreamingOutput stream = (StreamingOutput) response.getEntity(); @@ -353,13 +341,12 @@ public void testGetKeySuccess() throws Exception { } @Test - public void testGetKeyFailure() throws Exception { + public void testGetKeyFailure() { long oriMetric = metrics.getGetKeyFailure(); // Fetching a non-existent key - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.get( - bucketName, "unknownKey", 0, null, 0, null, null)); - assertEquals(S3ErrorTable.NO_SUCH_KEY.getCode(), e.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_KEY, + () -> keyEndpoint.get(bucketName, "unknownKey", 0, null, 0, null, null)); long curMetric = metrics.getGetKeyFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -374,11 +361,10 @@ public void testInitMultiPartUploadSuccess() throws Exception { } @Test - public void testInitMultiPartUploadFailure() throws Exception { + public void testInitMultiPartUploadFailure() { long oriMetric = metrics.getInitMultiPartUploadFailure(); - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint - .initializeMultipartUpload("unknownBucket", keyName)); - assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_BUCKET, + () -> keyEndpoint.initializeMultipartUpload("unknownBucket", keyName)); long curMetric = metrics.getInitMultiPartUploadFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -399,94 +385,76 @@ public void testAbortMultiPartUploadSuccess() throws Exception { } @Test - public void testAbortMultiPartUploadFailure() throws Exception { + public void testAbortMultiPartUploadFailure() { long oriMetric = metrics.getAbortMultiPartUploadFailure(); // Fail the Abort Method by providing wrong uploadID - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.delete( - bucketName, keyName, "wrongId", null)); - assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getCode(), e.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD, () -> keyEndpoint.delete(bucketName, keyName, "wrongId", null)); long curMetric = metrics.getAbortMultiPartUploadFailure(); assertEquals(1L, curMetric - oriMetric); } @Test public void testCompleteMultiPartUploadSuccess() throws Exception { - - // Initiate the Upload and fetch the upload ID + long oriMetric = metrics.getCompleteMultiPartUploadSuccess(); String uploadID = initiateMultipartUpload(bucketName, keyName); + CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(); + + assertSucceeds(() -> keyEndpoint.completeMultipartUpload(bucketName, keyName, uploadID, request)); - long oriMetric = metrics.getCompleteMultiPartUploadSuccess(); - // complete multipart upload - CompleteMultipartUploadRequest completeMultipartUploadRequest = new - CompleteMultipartUploadRequest(); - Response response = keyEndpoint.completeMultipartUpload(bucketName, keyName, - uploadID, completeMultipartUploadRequest); long curMetric = metrics.getCompleteMultiPartUploadSuccess(); - assertEquals(200, response.getStatus()); assertEquals(1L, curMetric - oriMetric); } @Test - public void testCompleteMultiPartUploadFailure() throws Exception { + public void testCompleteMultiPartUploadFailure() { long oriMetric = metrics.getCompleteMultiPartUploadFailure(); - CompleteMultipartUploadRequest completeMultipartUploadRequestNew = new - CompleteMultipartUploadRequest(); - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint - .completeMultipartUpload(bucketName, "key2", "random", - completeMultipartUploadRequestNew)); - assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getCode(), e.getCode()); + CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(); + + assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD, + () -> keyEndpoint.completeMultipartUpload(bucketName, "key2", "random", request)); + long curMetric = metrics.getCompleteMultiPartUploadFailure(); assertEquals(1L, curMetric - oriMetric); } @Test public void testCreateMultipartKeySuccess() throws Exception { - - // Initiate the Upload and fetch the upload ID String uploadID = initiateMultipartUpload(bucketName, keyName); - long oriMetric = metrics.getCreateMultipartKeySuccess(); - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - keyEndpoint.put(bucketName, keyName, CONTENT.length(), - 1, uploadID, null, null, body); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, 1, uploadID, CONTENT)); long curMetric = metrics.getCreateMultipartKeySuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testCreateMultipartKeyFailure() throws Exception { + public void testCreateMultipartKeyFailure() { long oriMetric = metrics.getCreateMultipartKeyFailure(); - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.put( - bucketName, keyName, CONTENT.length(), 1, "randomId", null, null, null)); - assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getCode(), e.getCode()); + + assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD, () -> put(keyEndpoint, bucketName, keyName, 1, "random", CONTENT)); + long curMetric = metrics.getCreateMultipartKeyFailure(); assertEquals(1L, curMetric - oriMetric); } @Test public void testListPartsSuccess() throws Exception { - long oriMetric = metrics.getListPartsSuccess(); - // Initiate the Upload and fetch the upload ID String uploadID = initiateMultipartUpload(bucketName, keyName); - // Listing out the parts by providing the uploadID - keyEndpoint.get(bucketName, keyName, 0, - uploadID, 3, null, null); + keyEndpoint.get(bucketName, keyName, 0, uploadID, 3, null, null); + long curMetric = metrics.getListPartsSuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testListPartsFailure() throws Exception { - + public void testListPartsFailure() { long oriMetric = metrics.getListPartsFailure(); - // Listing out the parts by providing the uploadID after aborting - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.get( - bucketName, keyName, 0, "wrong_id", 3, null, null)); - assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getCode(), e.getCode()); + + assertErrorResponse(S3ErrorTable.NO_SUCH_UPLOAD, + () -> keyEndpoint.get(bucketName, keyName, 0, "wrong_id", 3, null, null)); + long curMetric = metrics.getListPartsFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -502,28 +470,24 @@ public void testCopyObject() throws Exception { // Test for Success of CopyObjectSuccess Metric long oriMetric = metrics.getCopyObjectSuccess(); - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - keyEndpoint.put(bucketName, keyName, - CONTENT.length(), 1, null, null, null, body); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); // Add copy header, and then call put when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( bucketName + "/" + urlEncode(keyName)); - keyEndpoint.put(destBucket, destKey, CONTENT.length(), 1, - null, null, null, body); + assertSucceeds(() -> put(keyEndpoint, destBucket, destKey, CONTENT)); + long curMetric = metrics.getCopyObjectSuccess(); assertEquals(1L, curMetric - oriMetric); // Test for Failure of CopyObjectFailure Metric oriMetric = metrics.getCopyObjectFailure(); - // source and dest same + when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(""); - OS3Exception e = assertThrows(OS3Exception.class, () -> keyEndpoint.put( - bucketName, keyName, CONTENT.length(), 1, null, null, null, body), - "Test for CopyObjectMetric failed"); + OS3Exception e = assertErrorResponse(S3ErrorTable.INVALID_REQUEST, + () -> put(keyEndpoint, bucketName, keyName, CONTENT)); assertThat(e.getErrorMessage()).contains("This copy request is illegal"); curMetric = metrics.getCopyObjectFailure(); assertEquals(1L, curMetric - oriMetric); @@ -532,31 +496,21 @@ public void testCopyObject() throws Exception { @Test public void testPutObjectTaggingSuccess() throws Exception { long oriMetric = metrics.getPutObjectTaggingSuccess(); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - // Create the file - keyEndpoint.put(bucketName, keyName, CONTENT - .length(), 1, null, null, null, body); - body.close(); - - // Put object tagging - keyEndpoint.put(bucketName, keyName, 0, 1, null, "", null, getPutTaggingBody()); + assertSucceeds(() -> putTagging(keyEndpoint, bucketName, keyName, getPutTaggingBody())); long curMetric = metrics.getPutObjectTaggingSuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testPutObjectTaggingFailure() throws Exception { + public void testPutObjectTaggingFailure() { long oriMetric = metrics.getPutObjectTaggingFailure(); // Put object tagging for nonexistent key - OS3Exception ex = assertThrows(OS3Exception.class, () -> - keyEndpoint.put(bucketName, "nonexistent", 0, 1, null, "", - null, getPutTaggingBody()) - ); - assertEquals(S3ErrorTable.NO_SUCH_KEY.getCode(), ex.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_KEY, + () -> putTagging(keyEndpoint, bucketName, "nonexistent", getPutTaggingBody())); long curMetric = metrics.getPutObjectTaggingFailure(); assertEquals(1L, curMetric - oriMetric); @@ -565,34 +519,20 @@ null, getPutTaggingBody()) @Test public void testGetObjectTaggingSuccess() throws Exception { long oriMetric = metrics.getGetObjectTaggingSuccess(); - - // Create the file - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - keyEndpoint.put(bucketName, keyName, CONTENT - .length(), 1, null, null, null, body); - body.close(); - - // Put object tagging - keyEndpoint.put(bucketName, keyName, 0, 1, null, "", null, getPutTaggingBody()); - - // Get object tagging - keyEndpoint.get(bucketName, keyName, 0, - null, 0, null, ""); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); + assertSucceeds(() -> putTagging(keyEndpoint, bucketName, keyName, getPutTaggingBody())); + assertSucceeds(() -> getTagging(keyEndpoint, bucketName, keyName)); long curMetric = metrics.getGetObjectTaggingSuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testGetObjectTaggingFailure() throws Exception { + public void testGetObjectTaggingFailure() { long oriMetric = metrics.getGetObjectTaggingFailure(); - // Get object tagging for nonexistent key - OS3Exception ex = assertThrows(OS3Exception.class, () -> - keyEndpoint.get(bucketName, "nonexistent", 0, null, - 0, null, "")); - assertEquals(S3ErrorTable.NO_SUCH_KEY.getCode(), ex.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_KEY, () -> getTagging(keyEndpoint, bucketName, "nonexistent")); + long curMetric = metrics.getGetObjectTaggingFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -600,32 +540,20 @@ public void testGetObjectTaggingFailure() throws Exception { @Test public void testDeleteObjectTaggingSuccess() throws Exception { long oriMetric = metrics.getDeleteObjectTaggingSuccess(); - - // Create the file - ByteArrayInputStream body = - new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); - keyEndpoint.put(bucketName, keyName, CONTENT - .length(), 1, null, null, null, body); - body.close(); - - // Put object tagging - keyEndpoint.put(bucketName, keyName, 0, 1, null, "", null, getPutTaggingBody()); - - // Delete object tagging - keyEndpoint.delete(bucketName, keyName, null, ""); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, CONTENT)); + assertSucceeds(() -> putTagging(keyEndpoint, bucketName, keyName, getPutTaggingBody())); + deleteTagging(keyEndpoint, bucketName, keyName); long curMetric = metrics.getDeleteObjectTaggingSuccess(); assertEquals(1L, curMetric - oriMetric); } @Test - public void testDeleteObjectTaggingFailure() throws Exception { + public void testDeleteObjectTaggingFailure() { long oriMetric = metrics.getDeleteObjectTaggingFailure(); - // Delete object tagging for nonexistent key - OS3Exception ex = assertThrows(OS3Exception.class, () -> - keyEndpoint.delete(bucketName, "nonexistent", null, "")); - assertEquals(S3ErrorTable.NO_SUCH_KEY.getCode(), ex.getCode()); + assertErrorResponse(S3ErrorTable.NO_SUCH_KEY, () -> deleteTagging(keyEndpoint, bucketName, "nonexistent")); + long curMetric = metrics.getDeleteObjectTaggingFailure(); assertEquals(1L, curMetric - oriMetric); } @@ -646,8 +574,8 @@ private String initiateMultipartUpload(String bktName, String key) return "Invalid-Id"; } - private static InputStream getPutTaggingBody() { - String xml = + private static String getPutTaggingBody() { + return "" + " " + " " + @@ -656,7 +584,6 @@ private static InputStream getPutTaggingBody() { " " + " " + ""; - - return new ByteArrayInputStream(xml.getBytes(UTF_8)); } + }