Improvements to backblaze data transfer

extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java

@@ -16,25 +16,26 @@
 
 package org.datatransferproject.datatransfer.backblaze;
 
-import static org.datatransferproject.types.common.models.DataVertical.PHOTOS;
-import static org.datatransferproject.types.common.models.DataVertical.VIDEOS;
-
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import java.util.List;
 import org.datatransferproject.api.launcher.ExtensionContext;
 import org.datatransferproject.api.launcher.Monitor;
+import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants;
 import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory;
 import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter;
 import org.datatransferproject.datatransfer.backblaze.videos.BackblazeVideosImporter;
 import org.datatransferproject.spi.cloud.connection.ConnectionProvider;
 import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
+import org.datatransferproject.spi.transfer.provider.converter.MediaImporterDecorator;
 import org.datatransferproject.types.common.models.DataVertical;
 import org.datatransferproject.spi.transfer.extension.TransferExtension;
 import org.datatransferproject.spi.transfer.provider.Exporter;
 import org.datatransferproject.spi.transfer.provider.Importer;
 
+import static org.datatransferproject.types.common.models.DataVertical.*;
+
 public class BackblazeTransferExtension implements TransferExtension {
   public static final String SERVICE_ID = "Backblaze";
   private static final List<DataVertical> SUPPORTED_TYPES = ImmutableList.of(PHOTOS, VIDEOS);
@@ -56,7 +57,7 @@ public String getServiceId() {
   @Override
   public Importer<?, ?> getImporter(DataVertical transferDataType) {
     Preconditions.checkArgument(
-        initialized, "Trying to call getImporter before initalizing BackblazeTransferExtension");
+        initialized, "Trying to call getImporter before initializing BackblazeTransferExtension");
     Preconditions.checkArgument(
         SUPPORTED_TYPES.contains(transferDataType),
         "Import of " + transferDataType + " not supported by Backblaze");
@@ -82,11 +83,23 @@ public void initialize(ExtensionContext context) {
     importerBuilder.put(
             PHOTOS,
             new BackblazePhotosImporter(
-                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory));
+                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory,
+                    BackblazeConstants.PHOTOS_BASE_FOLDER_NAME));
     importerBuilder.put(
             VIDEOS,
             new BackblazeVideosImporter(
-                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory));
+                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory,
+                    BackblazeConstants.VIDEOS_BASE_FOLDER_NAME));
+    importerBuilder.put(MEDIA,
+            new MediaImporterDecorator<>(
+                    new BackblazePhotosImporter(
+                            monitor, jobStore, isProvider, backblazeDataTransferClientFactory,
+                            BackblazeConstants.MEDIA_BASE_FOLDER_NAME),
+                    new BackblazeVideosImporter(
+                            monitor, jobStore, isProvider, backblazeDataTransferClientFactory,
+                            BackblazeConstants.MEDIA_BASE_FOLDER_NAME))
+            );
+
     importerMap = importerBuilder.build();
     initialized = true;
   }

extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeConstants.java

@@ -0,0 +1,7 @@
+package org.datatransferproject.datatransfer.backblaze.common;
+
+public interface BackblazeConstants {
+    String VIDEOS_BASE_FOLDER_NAME = "Video Transfer";
+    String PHOTOS_BASE_FOLDER_NAME = "Photo Transfer";
+    String MEDIA_BASE_FOLDER_NAME = "Media Transfer";
+}

extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java

@@ -20,21 +20,17 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException;
-import org.datatransferproject.transfer.JobMetadata;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.awscore.exception.AwsServiceException;
-import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.Bucket;
 import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
@@ -64,6 +60,7 @@ public class BackblazeDataTransferClient {
   private final long partSizeForMultiPartUpload;
   private final BackblazeS3ClientFactory backblazeS3ClientFactory;
   private final Monitor monitor;
+  private final DateFormat dateFormatter;
   private S3Client s3Client;
   private String bucketName;
 
@@ -79,6 +76,9 @@ public BackblazeDataTransferClient(
       throw new IllegalArgumentException("Part size for multipart upload must be positive.");
     this.sizeThresholdForMultipartUpload = sizeThresholdForMultipartUpload;
     this.partSizeForMultiPartUpload = partSizeForMultiPartUpload;
+
+    dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); // Quoted "Z" to indicate UTC, no timezone offset
+    dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
   }
 
   public void init(String keyId, String applicationKey, String exportService)
@@ -125,7 +125,7 @@ public void init(String keyId, String applicationKey, String exportService)
     bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion, exportService);
   }
 
-  public String uploadFile(String fileKey, File file) throws IOException {
+  public String uploadFile(String fileKey, File file, Date createdTime) throws IOException {
     if (s3Client == null || bucketName == null) {
       throw new IllegalStateException("BackblazeDataTransferClient has not been initialised");
     }
@@ -144,8 +144,12 @@ public String uploadFile(String fileKey, File file) throws IOException {
         return uploadFileUsingMultipartUpload(fileKey, file, contentLength);
       }
 
+      String createdTimeIso = dateFormatter.format(createdTime);
+
       PutObjectRequest putObjectRequest =
-          PutObjectRequest.builder().bucket(bucketName).key(fileKey).build();
+          PutObjectRequest.builder().bucket(bucketName).key(fileKey)
+                  .metadata(ImmutableMap.of("created_time", createdTimeIso))
+                  .build();
 
       PutObjectResponse putObjectResponse =
           s3Client.putObject(putObjectRequest, RequestBody.fromFile(file));

extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java

@@ -41,22 +41,24 @@
 public class BackblazePhotosImporter
     implements Importer<TokenSecretAuthData, PhotosContainerResource> {
 
-  private static final String PHOTO_TRANSFER_MAIN_FOLDER = "Photo Transfer";
-
   private final TemporaryPerJobDataStore jobStore;
   private final ConnectionProvider connectionProvider;
   private final Monitor monitor;
   private final BackblazeDataTransferClientFactory b2ClientFactory;
 
+  private final String baseFolderName;
+
   public BackblazePhotosImporter(
       Monitor monitor,
       TemporaryPerJobDataStore jobStore,
       ConnectionProvider connectionProvider,
-      BackblazeDataTransferClientFactory b2ClientFactory) {
+      BackblazeDataTransferClientFactory b2ClientFactory,
+      String baseFolderName) {
     this.monitor = monitor;
     this.jobStore = jobStore;
     this.connectionProvider = connectionProvider;
     this.b2ClientFactory = b2ClientFactory;
+    this.baseFolderName = baseFolderName;
   }
 
   @Override
@@ -115,8 +117,10 @@ private ItemImportResult<String> importSinglePhoto(
     }
     String response =
         b2Client.uploadFile(
-            String.format("%s/%s/%s.jpg", PHOTO_TRANSFER_MAIN_FOLDER, albumName, photo.getDataId()),
-            file);
+            String.format("%s/%s/%s", baseFolderName, albumName, photo.getTitle()),
+            file,
+            photo.getUploadedTime()
+        );
     long size = file.length();
 
     try {

extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java

@@ -46,15 +46,19 @@ public class BackblazeVideosImporter
   private final Monitor monitor;
   private final BackblazeDataTransferClientFactory b2ClientFactory;
 
+  private final String baseFolderName;
+
   public BackblazeVideosImporter(
       Monitor monitor,
       TemporaryPerJobDataStore jobStore,
       ConnectionProvider connectionProvider,
-      BackblazeDataTransferClientFactory b2ClientFactory) {
+      BackblazeDataTransferClientFactory b2ClientFactory,
+      String baseFolderName) {
     this.monitor = monitor;
     this.jobStore = jobStore;
     this.connectionProvider = connectionProvider;
     this.b2ClientFactory = b2ClientFactory;
+    this.baseFolderName = baseFolderName;
   }
 
   @Override
@@ -96,7 +100,7 @@ private ItemImportResult<String> importSingleVideo(
       File file = jobStore.getTempFileFromInputStream(videoFileStream, video.getDataId(), ".mp4");
       String res =
           b2Client.uploadFile(
-              String.format("%s/%s.mp4", VIDEO_TRANSFER_MAIN_FOLDER, video.getDataId()), file);
+              String.format("%s/%s", baseFolderName, video.getName()), file, video.getUploadedTime());
       return ItemImportResult.success(res, file.length());
     } catch (FileNotFoundException e) {
       monitor.info(

extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java

@@ -28,6 +28,10 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException;
 import org.junit.jupiter.api.BeforeAll;
@@ -63,6 +67,7 @@ public class BackblazeDataTransferClientTest {
   @Mock
   private S3Client s3Client;
   private static File testFile;
+  private static Date testUploadedDate = new GregorianCalendar(2023, Calendar.FEBRUARY,23, 15, 0).getTime();
   private static final String KEY_ID = "keyId";
   private static final String APP_KEY = "appKey";
   private static final String EXPORT_SERVICE = "exp-serv";
@@ -156,7 +161,7 @@ public void testInitListBucketException() throws BackblazeCredentialsException,
   public void testUploadFileNonInitialized() throws IOException {
     BackblazeDataTransferClient client = createDefaultClient();
     assertThrows(IllegalStateException.class, () -> {
-      client.uploadFile(FILE_KEY, testFile);
+      client.uploadFile(FILE_KEY, testFile, testUploadedDate);
     });
   }
 
@@ -168,7 +173,8 @@ public void testUploadFileSingle() throws BackblazeCredentialsException, IOExcep
         .thenReturn(PutObjectResponse.builder().versionId(expectedVersionId).build());
     BackblazeDataTransferClient client = createDefaultClient();
     client.init(KEY_ID, APP_KEY, EXPORT_SERVICE);
-    String actualVersionId = client.uploadFile(FILE_KEY, testFile);
+    String actualVersionId = client.uploadFile(FILE_KEY, testFile,
+            new GregorianCalendar(2023, Calendar.FEBRUARY,23, 15, 0).getTime());
     verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class));
     assertEquals(expectedVersionId, actualVersionId);
   }
@@ -181,7 +187,7 @@ public void testUploadFileSingleException() throws BackblazeCredentialsException
     BackblazeDataTransferClient client = createDefaultClient();
     client.init(KEY_ID, APP_KEY, EXPORT_SERVICE);
     assertThrows(IOException.class, () -> {
-      client.uploadFile(FILE_KEY, testFile);
+      client.uploadFile(FILE_KEY, testFile, testUploadedDate);
     });
   }
 
@@ -201,7 +207,7 @@ public void testUploadFileMultipart() throws BackblazeCredentialsException, IOEx
     BackblazeDataTransferClient client =
         new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, fileSize / 2, partSize);
     client.init(KEY_ID, APP_KEY, EXPORT_SERVICE);
-    String actualVersionId = client.uploadFile(FILE_KEY, testFile);
+    String actualVersionId = client.uploadFile(FILE_KEY, testFile, testUploadedDate);
     verify(s3Client, times((int) expectedParts))
         .uploadPart(any(UploadPartRequest.class), any(RequestBody.class));
     assertEquals(expectedVersionId, actualVersionId);
@@ -220,7 +226,7 @@ public void testUploadFileMultipartException() throws BackblazeCredentialsExcept
             fileSize / 8);
     client.init(KEY_ID, APP_KEY, EXPORT_SERVICE);
     assertThrows(IOException.class, () -> {
-      client.uploadFile(FILE_KEY, testFile);
+      client.uploadFile(FILE_KEY, testFile, testUploadedDate);
     });
   }
 }

extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java

@@ -31,6 +31,7 @@
 import java.util.UUID;
 import org.apache.commons.io.IOUtils;
 import org.datatransferproject.api.launcher.Monitor;
+import org.datatransferproject.datatransfer.backblaze.common.BackblazeConstants;
 import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient;
 import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory;
 import org.datatransferproject.spi.cloud.connection.ConnectionProvider;
@@ -43,11 +44,13 @@
 import org.datatransferproject.types.common.models.photos.PhotoModel;
 import org.datatransferproject.types.common.models.photos.PhotosContainerResource;
 import org.datatransferproject.types.transfer.auth.TokenSecretAuthData;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.io.TempDir;
 import org.mockito.ArgumentCaptor;
 
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
 public class BackblazePhotosImporterTest {
   Monitor monitor;
   TemporaryPerJobDataStore dataStore;
@@ -56,8 +59,9 @@ public class BackblazePhotosImporterTest {
   IdempotentImportExecutor executor;
   TokenSecretAuthData authData;
   BackblazeDataTransferClient client;
+  BackblazePhotosImporter photosImporter;
 
-  @BeforeEach
+  @BeforeAll
   public void setUp() {
     monitor = mock(Monitor.class);
     dataStore = mock(TemporaryPerJobDataStore.class);
@@ -66,27 +70,26 @@ public void setUp() {
     executor = mock(IdempotentImportExecutor.class);
     authData = mock(TokenSecretAuthData.class);
     client = mock(BackblazeDataTransferClient.class);
+    photosImporter = new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory,
+            BackblazeConstants.PHOTOS_BASE_FOLDER_NAME);
   }
 
   @TempDir public Path folder;
 
   @Test
   public void testNullData() throws Exception {
-    BackblazePhotosImporter sut =
-        new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory);
-    ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, null);
+    ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, null);
     assertEquals(ImportResult.OK, result);
   }
 
   @Test
   public void testNullPhotosAndAlbums() throws Exception {
+    System.out.println(photosImporter);
     PhotosContainerResource data = mock(PhotosContainerResource.class);
     when(data.getAlbums()).thenReturn(null);
     when(data.getPhotos()).thenReturn(null);
 
-    BackblazePhotosImporter sut =
-        new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory);
-    ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data);
+    ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, data);
     assertEquals(ImportResult.ResultType.OK, result.getType());
   }
 
@@ -96,9 +99,7 @@ public void testEmptyPhotosAndAlbums() throws Exception {
     when(data.getAlbums()).thenReturn(new ArrayList<>());
     when(data.getPhotos()).thenReturn(new ArrayList<>());
 
-    BackblazePhotosImporter sut =
-        new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory);
-    ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data);
+    ImportResult result = photosImporter.importItem(UUID.randomUUID(), executor, authData, data);
     assertEquals(ImportResult.ResultType.OK, result.getType());
   }
 
@@ -120,15 +121,13 @@ public void testImportPhoto() throws Exception {
     when(streamProvider.getInputStreamForItem(jobId, photoModel))
         .thenReturn(new InputStreamWrapper(IOUtils.toInputStream("photo content", "UTF-8")));
 
-    when(client.uploadFile(eq("Photo Transfer/albumName/dataId.jpg"), any())).thenReturn(response);
+    when(client.uploadFile(eq("Photo Transfer/albumName/dataId.jpg"), any(), any())).thenReturn(response);
     when(clientFactory.getOrCreateB2Client(jobId, authData)).thenReturn(client);
 
     File file = folder.toFile();
     when(dataStore.getTempFileFromInputStream(any(), any(), any())).thenReturn(file);
 
-    BackblazePhotosImporter sut =
-        new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory);
-    sut.importItem(jobId, executor, authData, data);
+    photosImporter.importItem(jobId, executor, authData, data);
 
     ArgumentCaptor<ImportFunction<PhotoModel, String>> importCapture =
         ArgumentCaptor.forClass(ImportFunction.class);
@@ -148,9 +147,7 @@ public void testImportAlbum() throws Exception {
     PhotosContainerResource data = mock(PhotosContainerResource.class);
     when(data.getAlbums()).thenReturn(albums);
 
-    BackblazePhotosImporter sut =
-        new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory);
-    sut.importItem(UUID.randomUUID(), executor, authData, data);
+    photosImporter.importItem(UUID.randomUUID(), executor, authData, data);
 
     verify(executor, times(1))
         .executeAndSwallowIOExceptions(