diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java index d6ac13334..95599230e 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java @@ -21,19 +21,10 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.sync.RequestBody; @@ -75,6 +66,18 @@ public class BackblazeDataTransferClient { private static final String DATA_TRANSFER_BUCKET_PREFIX_FORMAT_STRING = "%s-data-transfer"; private static final int MAX_BUCKET_CREATION_ATTEMPTS = 10; + // Backblaze B2 Native API only supports IPv4; until it supports IPv6 we cannot use + // the b2_authorize_account endpoint to look up what region an account is in. For now + // a hard-coded list will be maintained and updated if the cluster list changes. + private static final List REGIONS = List.of( + "us-west-000", + "us-west-001", + "us-west-002", + "us-west-004", + "us-east-005", + "eu-central-003", + "ca-east-006" + ); private final long sizeThresholdForMultipartUpload; private final long partSizeForMultiPartUpload; @@ -97,21 +100,28 @@ public BackblazeDataTransferClient( this.partSizeForMultiPartUpload = partSizeForMultiPartUpload; } - public void init( - String keyId, String applicationKey, String exportService, CloseableHttpClient httpClient) + public void init(String keyId, String applicationKey, String exportService) throws BackblazeCredentialsException, IOException { // Fetch all the available buckets and use that to find which region the user is in ListBucketsResponse listBucketsResponse = null; + String successfulRegion = null; Throwable s3Exception = null; - String userRegion = getAccountRegion(httpClient, keyId, applicationKey); - s3Client = backblazeS3ClientFactory.createS3Client(keyId, applicationKey, userRegion); - try { - listBucketsResponse = s3Client.listBuckets(); - } catch (S3Exception e) { - s3Exception = e; - if (s3Client != null) { - s3Client.close(); + for (String region : REGIONS) { + S3Client potentialS3Client = + backblazeS3ClientFactory.createS3Client(keyId, applicationKey, region); + try { + listBucketsResponse = potentialS3Client.listBuckets(); + s3Client = potentialS3Client; + successfulRegion = region; + monitor.info(() -> "Successfully connected to Backblaze region: " + region); + break; + } catch (S3Exception e) { + monitor.info(() -> "Failed to connect to Backblaze region: " + region); + s3Exception = e; + if (potentialS3Client != null) { + potentialS3Client.close(); + } } } @@ -120,7 +130,7 @@ public void init( "User's credentials or permissions are not valid for any regions available", s3Exception); } - bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion, exportService); + bucketName = getOrCreateBucket(s3Client, listBucketsResponse, successfulRegion, exportService); } public String uploadFile(String fileKey, File file) throws IOException { @@ -154,42 +164,6 @@ public String uploadFile(String fileKey, File file) throws IOException { } } - private String getAccountRegion( - CloseableHttpClient httpClient, String keyId, String applicationKey) - throws BackblazeCredentialsException { - - String auth = keyId + ":" + applicationKey; - byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes()); - String authHeaderValue = "Basic " + new String(encodedAuth); - - HttpGet request = new HttpGet("https://api.backblazeb2.com/b2api/v2/b2_authorize_account"); - request.addHeader("Authorization", authHeaderValue); - - try { - CloseableHttpResponse response = httpClient.execute(request); - try (response) { - int statusCode = response.getStatusLine().getStatusCode(); - - if (statusCode == 200) { - String responseBody = EntityUtils.toString(response.getEntity()); - JSONParser parser = new JSONParser(); - JSONObject jsonResponse = (JSONObject) parser.parse(responseBody); - String s3ApiUrl = (String) jsonResponse.get("s3ApiUrl"); - String region = s3ApiUrl.split("s3.")[1].split("\\.")[0]; - monitor.info(() -> "Region extracted from s3ApiUrl: " + region); - return region; - } else if (statusCode >= 400 && statusCode < 500) { - // Don't retry on client errors (4xx) - throw new BackblazeCredentialsException( - "Failed to retrieve account's region. Status code: " + statusCode, null); - } else { - throw new IOException("Server returned status code: " + statusCode); - } - } - } catch (IOException | ParseException e) { - throw new BackblazeCredentialsException("Failed to retrieve account's region", e); - } - } private String uploadFileUsingMultipartUpload(String fileKey, File file, long contentLength) throws IOException, AwsServiceException, SdkClientException { diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java index b6a6cb8da..62ac48708 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import org.apache.http.impl.client.HttpClientBuilder; import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; import org.datatransferproject.transfer.JobMetadata; @@ -49,12 +48,10 @@ public BackblazeDataTransferClient getOrCreateB2Client( UUID jobId, SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD, PART_SIZE_FOR_MULTIPART_UPLOAD); String exportService = JobMetadata.getExportService(); - CustomIPv4DnsResolver customDnsResolver = new CustomIPv4DnsResolver(); backblazeDataTransferClient.init( authData.getToken(), authData.getSecret(), - exportService, - HttpClientBuilder.create().setDnsResolver(customDnsResolver).build() + exportService ); backblazeDataTransferClientMap.put(jobId, backblazeDataTransferClient); } diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeIntegrationTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeIntegrationTest.java index 8da01e6ad..293fea1e5 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeIntegrationTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeIntegrationTest.java @@ -1,6 +1,4 @@ package org.datatransferproject.datatransfer.backblaze.common; - -import org.apache.http.impl.client.HttpClientBuilder; import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.launcher.monitor.ConsoleMonitor; @@ -34,7 +32,7 @@ public static void main(String[] args) throws Exception { // Initialize client with your credentials // The "test-service" string is used as a prefix for bucket naming - client.init(keyId, appKey, "test-service", HttpClientBuilder.create().build()); + client.init(keyId, appKey, "test-service"); System.out.println("Client initialized successfully!"); diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/CustomIPv4DnsResolver.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/CustomIPv4DnsResolver.java deleted file mode 100644 index e5b69a2b3..000000000 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/CustomIPv4DnsResolver.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.datatransferproject.datatransfer.backblaze.common; - -import org.apache.http.impl.conn.SystemDefaultDnsResolver; -import org.apache.http.conn.DnsResolver; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; - -// This class is only required because Backblaze only supports IPV4 on the B2 Native APIs, the S3 -// compatible APIs support IPV6. This can be deleted when all B2 API endpoints support IPV6. -public class CustomIPv4DnsResolver implements DnsResolver { - private final SystemDefaultDnsResolver systemDefaultDnsResolver = new SystemDefaultDnsResolver(); - - @Override - public InetAddress[] resolve(String host) throws UnknownHostException { - // Filter to return only IPv4 addresses - return Arrays.stream(systemDefaultDnsResolver.resolve(host)) - .filter(inetAddress -> inetAddress.getAddress().length == 4) - .toArray(InetAddress[]::new); - } -} - diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java index ad3e45a61..f3cc8ed19 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java @@ -23,18 +23,9 @@ import java.io.File; import java.io.IOException; -import java.io.ByteArrayInputStream; -import java.io.InputStream; - -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.protocol.HttpContext; + import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,7 +48,6 @@ import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; -import org.apache.http.StatusLine; @ExtendWith(MockitoExtension.class) public class BackblazeDataTransferClientTest { @@ -65,18 +55,11 @@ public class BackblazeDataTransferClientTest { @Mock private Monitor monitor; @Mock private BackblazeS3ClientFactory backblazeS3ClientFactory; @Mock private S3Client s3Client; - @Mock private CloseableHttpClient httpClient; - @Mock private CloseableHttpResponse authorizeAccountHttpResponse; - @Mock private HttpEntity httpEntity; private static final String KEY_ID = "test-key-id"; private static final String APP_KEY = "test-app-key"; private static final String VALID_RESPONSE = "{\"s3ApiUrl\": \"https://s3.us-west-002.backblazeb2.com\"}"; - @Mock private CloseableHttpResponse httpResponse; - - @Mock private StatusLine statusLine; - @Mock private BackblazeS3ClientFactory s3ClientFactory; private BackblazeDataTransferClient client; @@ -93,105 +76,50 @@ public static void setUpClass() { @BeforeEach public void setUp() throws IOException, InterruptedException { - StatusLine statusLine = mock(StatusLine.class); s3Client = mock(S3Client.class); - lenient().when(httpResponse.getStatusLine()).thenReturn(statusLine); - lenient().when(statusLine.getStatusCode()).thenReturn(200); - lenient().when(httpResponse.getEntity()).thenReturn(httpEntity); - lenient() - .when(httpEntity.getContent()) - .thenReturn(new ByteArrayInputStream(VALID_RESPONSE.getBytes())); - lenient().when(httpClient.execute(any(HttpGet.class))).thenReturn(httpResponse); - InputStream mockInputStream = - new ByteArrayInputStream( - "{\"s3ApiUrl\":\"https://s3.us-west-910.backblazeb2.pet\"}".getBytes()); - lenient().when(httpEntity.getContent()).thenReturn(mockInputStream); - lenient() - .doReturn(authorizeAccountHttpResponse) - .when(httpClient) - .execute((HttpUriRequest) any(), (HttpContext) any()); lenient().doReturn(s3Client).when(backblazeS3ClientFactory).createS3Client(any(), any(), any()); - client = new BackblazeDataTransferClient(monitor, s3ClientFactory, 1000L, 100L); + client = new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, 1000L, 100L); } @Test - public void testGetAccountRegionSuccess() throws Exception { - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(200); - when(httpResponse.getEntity()).thenReturn(httpEntity); - when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(VALID_RESPONSE.getBytes())); - when(httpClient.execute(any(HttpGet.class))).thenReturn(httpResponse); - - // Mock the S3 client creation and bucket listing - doReturn(s3Client) - .when(s3ClientFactory) - .createS3Client(eq(KEY_ID), eq(APP_KEY), eq("us-west-002")); - + public void testInitSuccess() throws Exception { + // Mock the S3 client creation and bucket listing for the first region when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().build()); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); - verify(monitor) - .info( - argThat( - message -> message.get().contains("Region extracted from s3ApiUrl: us-west-002"))); - verify(s3ClientFactory, times(1)).createS3Client(eq(KEY_ID), eq(APP_KEY), eq("us-west-002")); + verify(monitor).info(any()); + verify(backblazeS3ClientFactory).createS3Client(eq(KEY_ID), eq(APP_KEY), eq("us-west-000")); } @Test - public void testGetAccountRegionWithClientError() throws Exception { - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(403); - when(httpClient.execute(any(HttpGet.class))).thenReturn(httpResponse); - - BackblazeCredentialsException exception = - assertThrows( - BackblazeCredentialsException.class, - () -> client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient)); - verify(httpClient, times(1)).execute(any(HttpGet.class)); - } + public void testInitSuccessOnSecondRegion() throws Exception { + S3Client s3Client1 = mock(S3Client.class); + S3Client s3Client2 = mock(S3Client.class); - @Test - public void testGetAccountRegionWithServerError() throws Exception { - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(500); - when(httpClient.execute(any(HttpGet.class))).thenReturn(httpResponse); - - BackblazeCredentialsException exception = - assertThrows( - BackblazeCredentialsException.class, - () -> client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient)); - } + when(backblazeS3ClientFactory.createS3Client(eq(KEY_ID), eq(APP_KEY), eq("us-west-000"))).thenReturn(s3Client1); + when(backblazeS3ClientFactory.createS3Client(eq(KEY_ID), eq(APP_KEY), eq("us-west-001"))).thenReturn(s3Client2); - @Test - public void testGetAccountRegionWithIOException() throws Exception { - // Given - when(httpClient.execute(any(HttpGet.class))).thenThrow(new IOException("Network error")); + when(s3Client1.listBuckets()).thenThrow(S3Exception.builder().build()); + when(s3Client2.listBuckets()).thenReturn(ListBucketsResponse.builder().build()); - // When/Then - BackblazeCredentialsException exception = - assertThrows( - BackblazeCredentialsException.class, - () -> client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient)); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); - assertEquals("Network error", exception.getCause().getMessage()); + verify(monitor, atLeastOnce()).info(any()); + verify(s3Client1).close(); } @Test - public void testGetAccountRegionWithMalformedJson() throws Exception { - // Given - String malformedJson = "{ invalid json }"; - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(200); - when(httpResponse.getEntity()).thenReturn(httpEntity); - when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(malformedJson.getBytes())); - when(httpClient.execute(any(HttpGet.class))).thenReturn(httpResponse); - - // When/Then + public void testInitFailureAllRegions() throws Exception { + when(backblazeS3ClientFactory.createS3Client(any(), any(), any())).thenReturn(s3Client); + when(s3Client.listBuckets()).thenThrow(S3Exception.builder().build()); + assertThrows( BackblazeCredentialsException.class, - () -> client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient)); + () -> client.init(KEY_ID, APP_KEY, EXPORT_SERVICE)); + + verify(s3Client, times(7)).listBuckets(); // Number of hardcoded regions } private void createValidBucketList() { @@ -220,7 +148,7 @@ public void testWrongPartSize() { public void testInitBucketNameMatches() throws BackblazeCredentialsException, IOException { createValidBucketList(); BackblazeDataTransferClient client = createDefaultClient(); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); verify(s3Client, times(0)).createBucket(any(CreateBucketRequest.class)); } @@ -229,7 +157,7 @@ public void testInitBucketCreated() throws BackblazeCredentialsException, IOExce Bucket bucket = Bucket.builder().name("invalid-name").build(); when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(bucket).build()); BackblazeDataTransferClient client = createDefaultClient(); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); verify(s3Client, times(1)).createBucket(any(CreateBucketRequest.class)); } @@ -243,7 +171,7 @@ public void testInitBucketNameExists() throws BackblazeCredentialsException, IOE assertThrows( IOException.class, () -> { - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); }); verify(monitor, atLeast(1)).info(any()); } @@ -257,7 +185,7 @@ public void testInitErrorCreatingBucket() throws BackblazeCredentialsException, assertThrows( IOException.class, () -> { - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); }); } @@ -268,7 +196,7 @@ public void testInitListBucketException() throws BackblazeCredentialsException, assertThrows( BackblazeCredentialsException.class, () -> { - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); }); verify(s3Client, atLeast(1)).close(); // verify(monitor, atLeast(1)).debug(any()); @@ -291,7 +219,7 @@ public void testUploadFileSingle() throws BackblazeCredentialsException, IOExcep when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class))) .thenReturn(PutObjectResponse.builder().versionId(expectedVersionId).build()); BackblazeDataTransferClient client = createDefaultClient(); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); String actualVersionId = client.uploadFile(FILE_KEY, testFile); verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); assertEquals(expectedVersionId, actualVersionId); @@ -303,7 +231,7 @@ public void testUploadFileSingleException() throws BackblazeCredentialsException when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class))) .thenThrow(AwsServiceException.builder().build()); BackblazeDataTransferClient client = createDefaultClient(); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); assertThrows( IOException.class, () -> { @@ -326,7 +254,7 @@ public void testUploadFileMultipart() throws BackblazeCredentialsException, IOEx final long expectedParts = fileSize / partSize + (fileSize % partSize == 0 ? 0 : 1); BackblazeDataTransferClient client = new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, fileSize / 2, partSize); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); String actualVersionId = client.uploadFile(FILE_KEY, testFile); verify(s3Client, times((int) expectedParts)) .uploadPart(any(UploadPartRequest.class), any(RequestBody.class)); @@ -344,7 +272,7 @@ public void testUploadFileMultipartException() throws BackblazeCredentialsExcept BackblazeDataTransferClient client = new BackblazeDataTransferClient( monitor, backblazeS3ClientFactory, fileSize / 2, fileSize / 8); - client.init(KEY_ID, APP_KEY, EXPORT_SERVICE, httpClient); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); assertThrows( IOException.class, () -> {