From 37bbc0b656d3f5a51c5ed7da509331dfa12a692a Mon Sep 17 00:00:00 2001 From: Alessandro Liparoti Date: Thu, 23 Feb 2023 16:21:12 +0100 Subject: [PATCH 1/2] support all media (photo and video) transfers --- .../backblaze/BackblazeTransferExtension.java | 25 +++++++++++---- .../backblaze/common/BackblazeConstants.java | 7 +++++ .../photos/BackblazePhotosImporter.java | 10 +++--- .../videos/BackblazeVideosImporter.java | 8 +++-- .../photos/BackblazePhotosImporterTest.java | 31 +++++++++---------- .../videos/BackblazeVideosImporterTest.java | 28 +++++++++-------- 6 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeConstants.java diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java index 898d202f6..dd12ad994 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java @@ -16,25 +16,26 @@ package org.datatransferproject.datatransfer.backblaze; -import static org.datatransferproject.types.common.models.DataVertical.PHOTOS; -import static org.datatransferproject.types.common.models.DataVertical.VIDEOS; - import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; import org.datatransferproject.api.launcher.ExtensionContext; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter; import org.datatransferproject.datatransfer.backblaze.videos.BackblazeVideosImporter; import org.datatransferproject.spi.cloud.connection.ConnectionProvider; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; +import org.datatransferproject.spi.transfer.provider.converter.MediaImporterDecorator; import org.datatransferproject.types.common.models.DataVertical; import org.datatransferproject.spi.transfer.extension.TransferExtension; import org.datatransferproject.spi.transfer.provider.Exporter; import org.datatransferproject.spi.transfer.provider.Importer; +import static org.datatransferproject.types.common.models.DataVertical.*; + public class BackblazeTransferExtension implements TransferExtension { public static final String SERVICE_ID = "Backblaze"; private static final List SUPPORTED_TYPES = ImmutableList.of(PHOTOS, VIDEOS); @@ -56,7 +57,7 @@ public String getServiceId() { @Override public Importer getImporter(DataVertical transferDataType) { Preconditions.checkArgument( - initialized, "Trying to call getImporter before initalizing BackblazeTransferExtension"); + initialized, "Trying to call getImporter before initializing BackblazeTransferExtension"); Preconditions.checkArgument( SUPPORTED_TYPES.contains(transferDataType), "Import of " + transferDataType + " not supported by Backblaze"); @@ -82,11 +83,23 @@ public void initialize(ExtensionContext context) { importerBuilder.put( PHOTOS, new BackblazePhotosImporter( - monitor, jobStore, isProvider, backblazeDataTransferClientFactory)); + monitor, jobStore, isProvider, backblazeDataTransferClientFactory, + BackblazeConstants.PHOTOS_BASE_FOLDER_NAME)); importerBuilder.put( VIDEOS, new BackblazeVideosImporter( - monitor, jobStore, isProvider, backblazeDataTransferClientFactory)); + monitor, jobStore, isProvider, backblazeDataTransferClientFactory, + BackblazeConstants.VIDEOS_BASE_FOLDER_NAME)); + importerBuilder.put(MEDIA, + new MediaImporterDecorator<>( + new BackblazePhotosImporter( + monitor, jobStore, isProvider, backblazeDataTransferClientFactory, + BackblazeConstants.MEDIA_BASE_FOLDER_NAME), + new BackblazeVideosImporter( + monitor, jobStore, isProvider, backblazeDataTransferClientFactory, + BackblazeConstants.MEDIA_BASE_FOLDER_NAME)) + ); + importerMap = importerBuilder.build(); initialized = true; } diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeConstants.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeConstants.java new file mode 100644 index 000000000..7acf16520 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeConstants.java @@ -0,0 +1,7 @@ +package org.datatransferproject.datatransfer.backblaze.common; + +public interface BackblazeConstants { + String VIDEOS_BASE_FOLDER_NAME = "Video Transfer"; + String PHOTOS_BASE_FOLDER_NAME = "Photo Transfer"; + String MEDIA_BASE_FOLDER_NAME = "Media Transfer"; +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java index 38f8de402..dedae71a2 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java @@ -41,22 +41,24 @@ public class BackblazePhotosImporter implements Importer { - private static final String PHOTO_TRANSFER_MAIN_FOLDER = "Photo Transfer"; - private final TemporaryPerJobDataStore jobStore; private final ConnectionProvider connectionProvider; private final Monitor monitor; private final BackblazeDataTransferClientFactory b2ClientFactory; + private final String baseFolderName; + public BackblazePhotosImporter( Monitor monitor, TemporaryPerJobDataStore jobStore, ConnectionProvider connectionProvider, - BackblazeDataTransferClientFactory b2ClientFactory) { + BackblazeDataTransferClientFactory b2ClientFactory, + String baseFolderName) { this.monitor = monitor; this.jobStore = jobStore; this.connectionProvider = connectionProvider; this.b2ClientFactory = b2ClientFactory; + this.baseFolderName = baseFolderName; } @Override @@ -115,7 +117,7 @@ private ItemImportResult importSinglePhoto( } String response = b2Client.uploadFile( - String.format("%s/%s/%s.jpg", PHOTO_TRANSFER_MAIN_FOLDER, albumName, photo.getDataId()), + String.format("%s/%s/%s.jpg", baseFolderName, albumName, photo.getDataId()), file); long size = file.length(); diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java index 3c196ed8d..bb71b8eae 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java @@ -46,15 +46,19 @@ public class BackblazeVideosImporter private final Monitor monitor; private final BackblazeDataTransferClientFactory b2ClientFactory; + private final String baseFolderName; + public BackblazeVideosImporter( Monitor monitor, TemporaryPerJobDataStore jobStore, ConnectionProvider connectionProvider, - BackblazeDataTransferClientFactory b2ClientFactory) { + BackblazeDataTransferClientFactory b2ClientFactory, + String baseFolderName) { this.monitor = monitor; this.jobStore = jobStore; this.connectionProvider = connectionProvider; this.b2ClientFactory = b2ClientFactory; + this.baseFolderName = baseFolderName; } @Override @@ -96,7 +100,7 @@ private ItemImportResult importSingleVideo( File file = jobStore.getTempFileFromInputStream(videoFileStream, video.getDataId(), ".mp4"); String res = b2Client.uploadFile( - String.format("%s/%s.mp4", VIDEO_TRANSFER_MAIN_FOLDER, video.getDataId()), file); + String.format("%s/%s.mp4", baseFolderName, video.getDataId()), file); return ItemImportResult.success(res, file.length()); } catch (FileNotFoundException e) { monitor.info( diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java index 5c04bb9b0..78c2c3978 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java @@ -31,6 +31,7 @@ import java.util.UUID; import org.apache.commons.io.IOUtils; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; import org.datatransferproject.spi.cloud.connection.ConnectionProvider; @@ -43,11 +44,13 @@ import org.datatransferproject.types.common.models.photos.PhotoModel; import org.datatransferproject.types.common.models.photos.PhotosContainerResource; import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class BackblazePhotosImporterTest { Monitor monitor; TemporaryPerJobDataStore dataStore; @@ -56,8 +59,9 @@ public class BackblazePhotosImporterTest { IdempotentImportExecutor executor; TokenSecretAuthData authData; BackblazeDataTransferClient client; + BackblazePhotosImporter photosImporter; - @BeforeEach + @BeforeAll public void setUp() { monitor = mock(Monitor.class); dataStore = mock(TemporaryPerJobDataStore.class); @@ -66,27 +70,26 @@ public void setUp() { executor = mock(IdempotentImportExecutor.class); authData = mock(TokenSecretAuthData.class); client = mock(BackblazeDataTransferClient.class); + photosImporter = new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory, + BackblazeConstants.PHOTOS_BASE_FOLDER_NAME); } @TempDir public Path folder; @Test public void testNullData() throws Exception { - BackblazePhotosImporter sut = - new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, null); + ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, null); assertEquals(ImportResult.OK, result); } @Test public void testNullPhotosAndAlbums() throws Exception { + System.out.println(photosImporter); PhotosContainerResource data = mock(PhotosContainerResource.class); when(data.getAlbums()).thenReturn(null); when(data.getPhotos()).thenReturn(null); - BackblazePhotosImporter sut = - new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, data); assertEquals(ImportResult.ResultType.OK, result.getType()); } @@ -96,9 +99,7 @@ public void testEmptyPhotosAndAlbums() throws Exception { when(data.getAlbums()).thenReturn(new ArrayList<>()); when(data.getPhotos()).thenReturn(new ArrayList<>()); - BackblazePhotosImporter sut = - new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, data); assertEquals(ImportResult.ResultType.OK, result.getType()); } @@ -126,9 +127,7 @@ public void testImportPhoto() throws Exception { File file = folder.toFile(); when(dataStore.getTempFileFromInputStream(any(), any(), any())).thenReturn(file); - BackblazePhotosImporter sut = - new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); - sut.importItem(jobId, executor, authData, data); + photosImporter.importItem(jobId, executor, authData, data); ArgumentCaptor> importCapture = ArgumentCaptor.forClass(ImportFunction.class); @@ -148,9 +147,7 @@ public void testImportAlbum() throws Exception { PhotosContainerResource data = mock(PhotosContainerResource.class); when(data.getAlbums()).thenReturn(albums); - BackblazePhotosImporter sut = - new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); - sut.importItem(UUID.randomUUID(), executor, authData, data); + photosImporter.importItem(UUID.randomUUID(), executor, authData, data); verify(executor, times(1)) .executeAndSwallowIOExceptions( diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java index def53b674..abee3a1ae 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java @@ -29,8 +29,10 @@ import java.util.UUID; import org.apache.commons.io.IOUtils; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; +import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter; import org.datatransferproject.spi.cloud.connection.ConnectionProvider; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper; @@ -40,11 +42,15 @@ import org.datatransferproject.types.common.models.videos.VideoModel; import org.datatransferproject.types.common.models.videos.VideosContainerResource; import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; +import org.junit.Before; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class BackblazeVideosImporterTest { Monitor monitor; @@ -54,9 +60,11 @@ public class BackblazeVideosImporterTest { IdempotentImportExecutor executor; TokenSecretAuthData authData; BackblazeDataTransferClient client; + BackblazeDataTransferClientFactory b2ClientFactory; + BackblazeVideosImporter videosImporter; @TempDir public Path folder; - @BeforeEach + @BeforeAll public void setUp() { monitor = mock(Monitor.class); dataStore = mock(TemporaryPerJobDataStore.class); @@ -65,13 +73,13 @@ public void setUp() { executor = mock(IdempotentImportExecutor.class); authData = mock(TokenSecretAuthData.class); client = mock(BackblazeDataTransferClient.class); + videosImporter = new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory, + BackblazeConstants.VIDEOS_BASE_FOLDER_NAME); } @Test public void testNullData() throws Exception { - BackblazeVideosImporter sut = - new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, null); + ImportResult result = videosImporter.importItem(UUID.randomUUID(), executor, authData, null); assertEquals(ImportResult.OK, result); } @@ -80,9 +88,7 @@ public void testNullVideos() throws Exception { VideosContainerResource data = mock(VideosContainerResource.class); when(data.getVideos()).thenReturn(null); - BackblazeVideosImporter sut = - new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + ImportResult result = videosImporter.importItem(UUID.randomUUID(), executor, authData, data); assertEquals(ImportResult.ResultType.OK, result.getType()); } @@ -91,9 +97,7 @@ public void testEmptyVideos() throws Exception { VideosContainerResource data = mock(VideosContainerResource.class); when(data.getVideos()).thenReturn(new ArrayList<>()); - BackblazeVideosImporter sut = - new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); - ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + ImportResult result = videosImporter.importItem(UUID.randomUUID(), executor, authData, data); assertEquals(ImportResult.ResultType.OK, result.getType()); } @@ -125,9 +129,7 @@ public void testImportVideo() throws Exception { when(client.uploadFile(eq("Video Transfer/dataId.mp4"), any())).thenReturn(response); when(clientFactory.getOrCreateB2Client(jobId, authData)).thenReturn(client); - BackblazeVideosImporter sut = - new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); - sut.importItem(jobId, executor, authData, data); + videosImporter.importItem(jobId, executor, authData, data); ArgumentCaptor> importCapture = ArgumentCaptor.forClass(ImportFunction.class); From 4e6f9aa9d2459392a2c853624722c655765b081d Mon Sep 17 00:00:00 2001 From: Alessandro Liparoti Date: Thu, 23 Feb 2023 17:02:24 +0100 Subject: [PATCH 2/2] fix naming of photo or video to actual name rather than id set created_time metadata to S3 object --- .../common/BackblazeDataTransferClient.java | 26 +++++++++++-------- .../photos/BackblazePhotosImporter.java | 6 +++-- .../videos/BackblazeVideosImporter.java | 2 +- .../BackblazeDataTransferClientTest.java | 16 ++++++++---- .../photos/BackblazePhotosImporterTest.java | 2 +- .../videos/BackblazeVideosImporterTest.java | 5 +--- 6 files changed, 33 insertions(+), 24 deletions(-) diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java index a764c4594..b1a170207 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java @@ -20,21 +20,17 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.RandomStringUtils; import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; -import org.datatransferproject.transfer.JobMetadata; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; @@ -64,6 +60,7 @@ public class BackblazeDataTransferClient { private final long partSizeForMultiPartUpload; private final BackblazeS3ClientFactory backblazeS3ClientFactory; private final Monitor monitor; + private final DateFormat dateFormatter; private S3Client s3Client; private String bucketName; @@ -79,6 +76,9 @@ public BackblazeDataTransferClient( throw new IllegalArgumentException("Part size for multipart upload must be positive."); this.sizeThresholdForMultipartUpload = sizeThresholdForMultipartUpload; this.partSizeForMultiPartUpload = partSizeForMultiPartUpload; + + dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); // Quoted "Z" to indicate UTC, no timezone offset + dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); } public void init(String keyId, String applicationKey, String exportService) @@ -125,7 +125,7 @@ public void init(String keyId, String applicationKey, String exportService) bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion, exportService); } - public String uploadFile(String fileKey, File file) throws IOException { + public String uploadFile(String fileKey, File file, Date createdTime) throws IOException { if (s3Client == null || bucketName == null) { throw new IllegalStateException("BackblazeDataTransferClient has not been initialised"); } @@ -144,8 +144,12 @@ public String uploadFile(String fileKey, File file) throws IOException { return uploadFileUsingMultipartUpload(fileKey, file, contentLength); } + String createdTimeIso = dateFormatter.format(createdTime); + PutObjectRequest putObjectRequest = - PutObjectRequest.builder().bucket(bucketName).key(fileKey).build(); + PutObjectRequest.builder().bucket(bucketName).key(fileKey) + .metadata(ImmutableMap.of("created_time", createdTimeIso)) + .build(); PutObjectResponse putObjectResponse = s3Client.putObject(putObjectRequest, RequestBody.fromFile(file)); diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java index dedae71a2..9ca3c0fa1 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java @@ -117,8 +117,10 @@ private ItemImportResult importSinglePhoto( } String response = b2Client.uploadFile( - String.format("%s/%s/%s.jpg", baseFolderName, albumName, photo.getDataId()), - file); + String.format("%s/%s/%s", baseFolderName, albumName, photo.getTitle()), + file, + photo.getUploadedTime() + ); long size = file.length(); try { diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java index bb71b8eae..9becb4dba 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java @@ -100,7 +100,7 @@ private ItemImportResult importSingleVideo( File file = jobStore.getTempFileFromInputStream(videoFileStream, video.getDataId(), ".mp4"); String res = b2Client.uploadFile( - String.format("%s/%s.mp4", baseFolderName, video.getDataId()), file); + String.format("%s/%s", baseFolderName, video.getName()), file, video.getUploadedTime()); return ItemImportResult.success(res, file.length()); } catch (FileNotFoundException e) { monitor.info( diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java index 8c42343ef..598933e75 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java @@ -28,6 +28,10 @@ import java.io.File; import java.io.IOException; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; + import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; import org.junit.jupiter.api.BeforeAll; @@ -63,6 +67,7 @@ public class BackblazeDataTransferClientTest { @Mock private S3Client s3Client; private static File testFile; + private static Date testUploadedDate = new GregorianCalendar(2023, Calendar.FEBRUARY,23, 15, 0).getTime(); private static final String KEY_ID = "keyId"; private static final String APP_KEY = "appKey"; private static final String EXPORT_SERVICE = "exp-serv"; @@ -156,7 +161,7 @@ public void testInitListBucketException() throws BackblazeCredentialsException, public void testUploadFileNonInitialized() throws IOException { BackblazeDataTransferClient client = createDefaultClient(); assertThrows(IllegalStateException.class, () -> { - client.uploadFile(FILE_KEY, testFile); + client.uploadFile(FILE_KEY, testFile, testUploadedDate); }); } @@ -168,7 +173,8 @@ public void testUploadFileSingle() throws BackblazeCredentialsException, IOExcep .thenReturn(PutObjectResponse.builder().versionId(expectedVersionId).build()); BackblazeDataTransferClient client = createDefaultClient(); client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); - String actualVersionId = client.uploadFile(FILE_KEY, testFile); + String actualVersionId = client.uploadFile(FILE_KEY, testFile, + new GregorianCalendar(2023, Calendar.FEBRUARY,23, 15, 0).getTime()); verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); assertEquals(expectedVersionId, actualVersionId); } @@ -181,7 +187,7 @@ public void testUploadFileSingleException() throws BackblazeCredentialsException BackblazeDataTransferClient client = createDefaultClient(); client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); assertThrows(IOException.class, () -> { - client.uploadFile(FILE_KEY, testFile); + client.uploadFile(FILE_KEY, testFile, testUploadedDate); }); } @@ -201,7 +207,7 @@ public void testUploadFileMultipart() throws BackblazeCredentialsException, IOEx BackblazeDataTransferClient client = new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, fileSize / 2, partSize); client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); - String actualVersionId = client.uploadFile(FILE_KEY, testFile); + String actualVersionId = client.uploadFile(FILE_KEY, testFile, testUploadedDate); verify(s3Client, times((int) expectedParts)) .uploadPart(any(UploadPartRequest.class), any(RequestBody.class)); assertEquals(expectedVersionId, actualVersionId); @@ -220,7 +226,7 @@ public void testUploadFileMultipartException() throws BackblazeCredentialsExcept fileSize / 8); client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); assertThrows(IOException.class, () -> { - client.uploadFile(FILE_KEY, testFile); + client.uploadFile(FILE_KEY, testFile, testUploadedDate); }); } } diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java index 78c2c3978..e110391cd 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java @@ -121,7 +121,7 @@ public void testImportPhoto() throws Exception { when(streamProvider.getInputStreamForItem(jobId, photoModel)) .thenReturn(new InputStreamWrapper(IOUtils.toInputStream("photo content", "UTF-8"))); - when(client.uploadFile(eq("Photo Transfer/albumName/dataId.jpg"), any())).thenReturn(response); + when(client.uploadFile(eq("Photo Transfer/albumName/dataId.jpg"), any(), any())).thenReturn(response); when(clientFactory.getOrCreateB2Client(jobId, authData)).thenReturn(client); File file = folder.toFile(); diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java index abee3a1ae..60b162fce 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java @@ -32,7 +32,6 @@ import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient; import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; -import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter; import org.datatransferproject.spi.cloud.connection.ConnectionProvider; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper; @@ -42,9 +41,7 @@ import org.datatransferproject.types.common.models.videos.VideoModel; import org.datatransferproject.types.common.models.videos.VideosContainerResource; import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; -import org.junit.Before; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; @@ -126,7 +123,7 @@ public void testImportVideo() throws Exception { .thenReturn(new InputStreamWrapper(IOUtils.toInputStream("video content", "UTF-8"))); when(dataStore.getTempFileFromInputStream(any(), any(), any())).thenReturn(folder.toFile()); - when(client.uploadFile(eq("Video Transfer/dataId.mp4"), any())).thenReturn(response); + when(client.uploadFile(eq("Video Transfer/dataId.mp4"), any(), any())).thenReturn(response); when(clientFactory.getOrCreateB2Client(jobId, authData)).thenReturn(client); videosImporter.importItem(jobId, executor, authData, data);