diff --git a/NEWS.md b/NEWS.md index 62d13e3a6..1737d980e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ ## Unreleased v3.5.0 +## 2026-01-19 v3.4.11 + +Sunflower backport - including folio-s3-library improvements + +[Full Changelog](https://github.com/folio-org/mod-data-export-worker/compare/v3.4.10...v3.4.11) + ## 2025-11-21 v3.4.10 [Full Changelog](https://github.com/folio-org/mod-data-export-worker/compare/v3.4.9...v3.4.10) diff --git a/pom.xml b/pom.xml index bd187345c..e0a098a82 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.folio mod-data-export-worker Data Export Worker module - 3.4.11-SNAPSHOT + 3.4.12-SNAPSHOT jar @@ -54,7 +54,6 @@ 4.1.1 9.0.0 1.0.0 - 8.5.15 6.2.1 4.4 3.7.3 @@ -65,12 +64,13 @@ 2.4.0 - 2.3.1 4.1.1 2.27.2 + 1.20.5 2.9.1 1.17.6 - 2.29.47 + 2.29.6 + 2.4.0 1.3 5.2.0 @@ -160,12 +160,6 @@ feign-jackson ${feign-jackson.version} - - io.minio - minio - ${minio.version} - - org.apache.sshd sshd-spring-sftp @@ -254,6 +248,12 @@ ${streamex.version} + + org.folio + folio-s3-client + ${folio-s3-client.version} + + @@ -343,10 +343,10 @@ test - com.playtika.testcontainers - embedded-minio - ${embedded-minio.version} - test + org.testcontainers + localstack + ${localstack.version} + test com.github.tomakehurst @@ -706,6 +706,8 @@ maven-surefire-plugin ${maven-surefire-plugin.version} + 1 + true @{argLine} -Xmx2G -Duser.language=en -Duser.region=US diff --git a/src/main/java/org/folio/dew/batch/AbstractStorageStreamAndJsonWriter.java b/src/main/java/org/folio/dew/batch/AbstractStorageStreamAndJsonWriter.java index c25eb54ff..fc04510d6 100644 --- a/src/main/java/org/folio/dew/batch/AbstractStorageStreamAndJsonWriter.java +++ b/src/main/java/org/folio/dew/batch/AbstractStorageStreamAndJsonWriter.java @@ -1,13 +1,27 @@ package org.folio.dew.batch; +import static java.lang.System.lineSeparator; import static org.folio.dew.utils.WriterHelper.enrichHoldingsJson; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; import org.folio.dew.domain.dto.Formatable; import org.folio.dew.domain.dto.HoldingsFormat; import org.folio.dew.repository.S3CompatibleResource; import org.folio.dew.repository.S3CompatibleStorage; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; import org.springframework.core.io.WritableResource; @@ -15,13 +29,17 @@ import java.nio.charset.StandardCharsets; @Slf4j -public class AbstractStorageStreamAndJsonWriter, S extends S3CompatibleStorage> extends AbstractStorageStreamWriter { +public class AbstractStorageStreamAndJsonWriter, S extends S3CompatibleStorage> + extends AbstractStorageStreamWriter implements StepExecutionListener { private final JacksonJsonObjectMarshaller jacksonJsonObjectMarshaller; private final ObjectMapper objectMapper; private WritableResource jsonResource; + private Path csvTmpFile; + private Path jsonTmpFile; + public AbstractStorageStreamAndJsonWriter(String tempOutputFilePath, String columnHeaders, String[] extractedFieldNames, FieldProcessor fieldProcessor, S storage) { super(tempOutputFilePath, columnHeaders, extractedFieldNames, fieldProcessor, storage); setJsonResource(new S3CompatibleResource<>(tempOutputFilePath + ".json", storage)); @@ -33,26 +51,97 @@ public void setJsonResource(S3CompatibleResource jsonResource) { this.jsonResource = jsonResource; } + @Override + public void beforeStep(StepExecution stepExecution) { + try { + Path tmpDir = createPrivateTmpDir(); + csvTmpFile = Files.createTempFile(tmpDir, "dew-csv-", ".tmp"); + jsonTmpFile = Files.createTempFile(tmpDir, "dew-json-", ".tmp"); + } catch (IOException e) { + throw new IllegalStateException("Error creating tmp files for resources", e); + } + } + + private Path createPrivateTmpDir() throws IOException { + Path parent = Path.of(System.getProperty("java.io.tmpdir")); + FileAttribute[] attrs = new FileAttribute[0]; + + FileSystem fs = FileSystems.getDefault(); + if (fs.supportedFileAttributeViews().contains("posix")) { + attrs = new FileAttribute[]{ + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")) + }; + } + return Files.createTempDirectory(parent, "dew-", attrs); + } + + @Override public void write(Chunk items) throws Exception { - var sb = new StringBuilder(); - var json = new StringBuilder(); - - var iterator = items.iterator(); - while (iterator.hasNext()) { - var item = iterator.next(); - sb.append(super.getLineAggregator().aggregate(item)).append('\n'); - - if (item instanceof HoldingsFormat hf) { - json.append(enrichHoldingsJson(hf, objectMapper)); - } else { - json.append(jacksonJsonObjectMarshaller.marshal(item.getOriginal())); + + if (items.isEmpty()) { + return; + } + + boolean jsonHasDataAlready = Files.size(jsonTmpFile) > 0; + + try (BufferedWriter plainWriter = Files.newBufferedWriter( + csvTmpFile, + StandardCharsets.UTF_8, + StandardOpenOption.APPEND); + BufferedWriter jsonWriter = Files.newBufferedWriter( + jsonTmpFile, + StandardCharsets.UTF_8, + StandardOpenOption.APPEND)) { + + if (jsonHasDataAlready) { + jsonWriter.append(lineSeparator()); } - if(iterator.hasNext()) { - json.append('\n'); + + var iterator = items.iterator(); + while (iterator.hasNext()) { + var item = iterator.next(); + + plainWriter.append(super.getLineAggregator().aggregate(item)) + .append(lineSeparator()); + + if (item instanceof HoldingsFormat hf) { + jsonWriter.append(enrichHoldingsJson(hf, objectMapper)); + } else { + jsonWriter.append(jacksonJsonObjectMarshaller.marshal(item.getOriginal())); + } + + if (iterator.hasNext()) { + jsonWriter.append(lineSeparator()); + } } } - getStorage().append(getResource().getFilename(), sb.toString().getBytes(StandardCharsets.UTF_8)); - getStorage().append(jsonResource.getFilename(), json.toString().getBytes(StandardCharsets.UTF_8)); + } + + @Override + public ExitStatus afterStep(StepExecution stepExecution) { + try { + getStorage().write(getResource().getFilename(), + ArrayUtils.addAll(getStorage().readAllBytes(getResource().getFilename()), Files.readAllBytes(csvTmpFile))); + getStorage().write(jsonResource.getFilename(), Files.readAllBytes(jsonTmpFile)); + } catch (IOException e) { + log.error("Error uploading data to S3", e); + return ExitStatus.FAILED.addExitDescription(e.getMessage()); + } finally { + deleteTempFile(csvTmpFile); + deleteTempFile(jsonTmpFile); + } + return stepExecution.getExitStatus(); + } + + private void deleteTempFile(Path path) { + if (path == null) { + return; + } + try { + Files.deleteIfExists(path); + } catch (IOException ex) { + log.warn("Error deleting tmp-file {}", path, ex); + } } } diff --git a/src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java b/src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java index 368775b26..bfad01adf 100644 --- a/src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java +++ b/src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java @@ -1,7 +1,6 @@ package org.folio.dew.batch; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.folio.dew.error.FileOperationException; import org.folio.dew.repository.LocalFilesStorage; @@ -16,10 +15,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.folio.dew.utils.Constants.LINE_SEPARATOR; -import static org.folio.dew.utils.Constants.LINE_SEPARATOR_REPLACEMENT; @Slf4j public class AbstractStorageStreamWriter implements ItemWriter { @@ -84,10 +79,13 @@ public S3CompatibleStorage getStorage() { @Override public void write(Chunk items) throws Exception { + var filename = resource.getFilename(); var sb = new StringBuilder(); + var header =new String(storage.readAllBytes(filename)); + sb.append(header); for (T item : items) { sb.append(lineAggregator.aggregate(item)).append('\n'); } - storage.append(resource.getFilename(), sb.toString().getBytes(StandardCharsets.UTF_8)); + storage.write(filename, sb.toString().getBytes(StandardCharsets.UTF_8)); } } diff --git a/src/main/java/org/folio/dew/batch/CsvAndJsonListWriter.java b/src/main/java/org/folio/dew/batch/CsvAndJsonListWriter.java index 6ed9e3881..3dc8a3428 100644 --- a/src/main/java/org/folio/dew/batch/CsvAndJsonListWriter.java +++ b/src/main/java/org/folio/dew/batch/CsvAndJsonListWriter.java @@ -6,7 +6,6 @@ import org.springframework.batch.item.Chunk; import java.util.List; -import java.util.stream.Collectors; public class CsvAndJsonListWriter, R extends S3CompatibleStorage> extends AbstractStorageStreamWriter, R> { private final CsvAndJsonWriter delegate; @@ -19,7 +18,7 @@ public CsvAndJsonListWriter(String tempOutputFilePath, String columnHeaders, Str @Override public void write(Chunk> lists) throws Exception { - var chunk = new Chunk<>(lists.getItems().stream().flatMap(List::stream).collect(Collectors.toList())); + var chunk = new Chunk<>(lists.getItems().stream().flatMap(List::stream).toList()); delegate.write(chunk); } } diff --git a/src/main/java/org/folio/dew/batch/CsvFileAssembler.java b/src/main/java/org/folio/dew/batch/CsvFileAssembler.java index e5b40b12c..7fac28936 100644 --- a/src/main/java/org/folio/dew/batch/CsvFileAssembler.java +++ b/src/main/java/org/folio/dew/batch/CsvFileAssembler.java @@ -1,5 +1,6 @@ package org.folio.dew.batch; +import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.io.FilenameUtils; @@ -9,9 +10,6 @@ import org.springframework.batch.core.partition.support.StepExecutionAggregator; import org.springframework.stereotype.Component; -import java.util.Collection; -import java.util.stream.Collectors; - @Component @Log4j2 @RequiredArgsConstructor @@ -24,12 +22,12 @@ public class CsvFileAssembler implements StepExecutionAggregator { public void aggregate(StepExecution stepExecution, Collection finishedStepExecutions) { var csvFilePartObjectNames = finishedStepExecutions.stream() .map(e -> e.getExecutionContext().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH)) - .collect(Collectors.toList()); + .toList(); var destCsvObject = FilenameUtils.getName( stepExecution.getJobExecution().getJobParameters().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".csv"); try { if ("CIRCULATION_LOG".equals(stepExecution.getJobExecution().getJobInstance().getJobName())) { - var csvUrl = remoteFilesStorage.composeObject(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV); + var csvUrl = remoteFilesStorage.compose(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV); ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.CIRCULATION_LOG_FILE_NAME, destCsvObject, ";"); ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.OUTPUT_FILES_IN_STORAGE, csvUrl, ";"); } else { @@ -37,15 +35,15 @@ public void aggregate(StepExecution stepExecution, Collection fin destCsvObject = prefix + destCsvObject; var csvUrl = remoteFilesStorage.objectToPresignedObjectUrl( - remoteFilesStorage.composeObject(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV)); + remoteFilesStorage.compose(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV)); var jsonFilePartObjectNames = finishedStepExecutions.stream() .map(e -> e.getExecutionContext().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".json") - .collect(Collectors.toList()); + .toList(); var destJsonObject = prefix + FilenameUtils.getName( stepExecution.getJobExecution().getJobParameters().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".json"); var jsonUrl = remoteFilesStorage.objectToPresignedObjectUrl( - remoteFilesStorage.composeObject(destJsonObject, jsonFilePartObjectNames, null, TEXT_CSV)); + remoteFilesStorage.compose(destJsonObject, jsonFilePartObjectNames, null, TEXT_CSV)); ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.OUTPUT_FILES_IN_STORAGE, csvUrl + ";;" + jsonUrl, ";"); } diff --git a/src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java b/src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java index 900bc6c62..2ea623ea1 100644 --- a/src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java +++ b/src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java @@ -115,7 +115,7 @@ private void processJobUpdate(JobExecution jobExecution, boolean after) { if (jobExecution.getJobInstance().getJobName().contains(BULK_EDIT_UPDATE.getValue())) { String updatedFilePath = jobExecution.getJobParameters().getString(UPDATED_FILE_NAME); String filePath = requireNonNull(isNull(updatedFilePath) ? jobExecution.getJobParameters().getString(FILE_NAME) : updatedFilePath); - if (localFilesStorage.notExists(filePath) && remoteFilesStorage.containsFile(filePath)) { + if (localFilesStorage.notExists(filePath) && remoteFilesStorage.exists(filePath)) { int totalUsers = CsvHelper.readRecordsFromStorage(remoteFilesStorage, filePath, UserFormat.class, true).size(); jobExecution.getExecutionContext().putInt(TOTAL_RECORDS, totalUsers); } else { @@ -223,7 +223,7 @@ private void processJobAfter(String jobId, JobParameters jobParameters) { return; } var files = localFilesStorage.walk(path) - .filter(name -> FilenameUtils.getName(name).startsWith(fileNameStart)).collect(Collectors.toList()); + .filter(name -> FilenameUtils.getName(name).startsWith(fileNameStart)).toList(); if (files.isEmpty()) { return; } @@ -327,7 +327,7 @@ private String saveResult(JobExecution jobExecution, boolean isSourceShouldBeDel if (isEmpty(path) || noRecordsFound(path)) { return EMPTY; // To prevent downloading empty file. } - if (localFilesStorage.notExists(path) && remoteFilesStorage.containsFile(path)) { + if (localFilesStorage.notExists(path) && remoteFilesStorage.exists(path)) { return remoteFilesStorage.objectToPresignedObjectUrl(path); } var obj = prepareObject(jobExecution, path); @@ -348,7 +348,7 @@ private String saveJsonResult(JobExecution jobExecution, boolean isSourceToBeDel if (isEmpty(path) || noRecordsFound(path)) { return EMPTY; // To prevent downloading empty file. } - if (localFilesStorage.notExists(path) && remoteFilesStorage.containsFile(path)) { + if (localFilesStorage.notExists(path) && remoteFilesStorage.exists(path)) { return remoteFilesStorage.objectToPresignedObjectUrl(path); } return remoteFilesStorage.objectToPresignedObjectUrl( @@ -377,7 +377,7 @@ private String preparePath(JobExecution jobExecution) { } private boolean noRecordsFound(String path) throws Exception { - if (localFilesStorage.notExists(path) && !remoteFilesStorage.containsFile(path)) { + if (localFilesStorage.notExists(path) && !remoteFilesStorage.exists(path)) { log.error("Path to found records does not exist: {}", path); return true; } diff --git a/src/main/java/org/folio/dew/batch/JsonFileWriter.java b/src/main/java/org/folio/dew/batch/JsonFileWriter.java index 254be58b7..7eae481c1 100644 --- a/src/main/java/org/folio/dew/batch/JsonFileWriter.java +++ b/src/main/java/org/folio/dew/batch/JsonFileWriter.java @@ -35,14 +35,13 @@ public String doWrite(Chunk items) { var iterator = items.iterator(); while (iterator.hasNext()) { var item = iterator.next(); - if (item instanceof HoldingsFormat hf) { - lines.append(enrichHoldingsJson(hf, objectMapper)); - } else if (item instanceof InstanceFormat instanceFormat) { - lines.append(WriterHelper.enrichInstancesJson(instanceFormat, objectMapper)); - } else if (item instanceof ItemFormat itemFormat) { - lines.append(WriterHelper.enrichItemsJson(itemFormat, objectMapper)); - } else { - lines.append(marshaller.marshal(item.getOriginal())); + switch (item) { + case HoldingsFormat hf -> lines.append(enrichHoldingsJson(hf, objectMapper)); + case InstanceFormat instanceFormat -> + lines.append(WriterHelper.enrichInstancesJson(instanceFormat, objectMapper)); + case ItemFormat itemFormat -> + lines.append(WriterHelper.enrichItemsJson(itemFormat, objectMapper)); + default -> lines.append(marshaller.marshal(item.getOriginal())); } lines.append(NEW_LINE); } diff --git a/src/main/java/org/folio/dew/batch/JsonListFileWriter.java b/src/main/java/org/folio/dew/batch/JsonListFileWriter.java index 0eee9dc8c..d204c4913 100644 --- a/src/main/java/org/folio/dew/batch/JsonListFileWriter.java +++ b/src/main/java/org/folio/dew/batch/JsonListFileWriter.java @@ -35,14 +35,13 @@ public String doWrite(Chunk> lists) { var iterator = chunk.iterator(); while (iterator.hasNext()) { var item = iterator.next(); - if (item instanceof HoldingsFormat hf) { - lines.append(WriterHelper.enrichHoldingsJson(hf, objectMapper)); - } else if (item instanceof InstanceFormat instanceFormat) { - lines.append(WriterHelper.enrichInstancesJson(instanceFormat, objectMapper)); - } else if (item instanceof ItemFormat itemFormat) { - lines.append(WriterHelper.enrichItemsJson(itemFormat, objectMapper)); - } else { - lines.append(marshaller.marshal(item.getOriginal())); + switch (item) { + case HoldingsFormat hf -> lines.append(WriterHelper.enrichHoldingsJson(hf, objectMapper)); + case InstanceFormat instanceFormat -> + lines.append(WriterHelper.enrichInstancesJson(instanceFormat, objectMapper)); + case ItemFormat itemFormat -> + lines.append(WriterHelper.enrichItemsJson(itemFormat, objectMapper)); + default -> lines.append(marshaller.marshal(item.getOriginal())); } lines.append(NEW_LINE); } diff --git a/src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriter.java b/src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriter.java index 09285c5be..643a4eee1 100644 --- a/src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriter.java +++ b/src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriter.java @@ -73,7 +73,17 @@ protected String doWrite(Chunk chunk) { } private void writeString(String str) throws IOException { - localFilesStorage.append(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8)); + try { + var header = new String(localFilesStorage.readAllBytes(tempOutputFilePath)); + if (header.charAt(header.length() - 1) != '\n') { + str = header + "\n" + str; + } else { + str = header + str; + } + } catch (IOException e) { + // Just ignore it if there's nothing to append to. + } + localFilesStorage.write(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8)); } private void setResource(String tempOutputFilePath) { diff --git a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditFileAssembler.java b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditFileAssembler.java index 42d87f2aa..c3c2a4eb0 100644 --- a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditFileAssembler.java +++ b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditFileAssembler.java @@ -155,18 +155,19 @@ private boolean isInstanceJob(StepExecution stepExecution) { } private void removePartFiles(List partFiles) { - ExecutorService exec = Executors.newCachedThreadPool(); - exec.execute(() -> { - partFiles.forEach(file -> { - try { - Files.delete(Path.of(file)); - } catch (IOException e) { - log.error("Error occurred while deleting the part files", e); - throw new FileOperationException(e); - } + try (ExecutorService exec = Executors.newCachedThreadPool()) { + exec.execute(() -> { + partFiles.forEach(file -> { + try { + Files.delete(Path.of(file)); + } catch (IOException e) { + log.error("Error occurred while deleting the part files", e); + throw new FileOperationException(e); + } + }); + log.info("All {} part files have been deleted successfully.", partFiles.size()); }); - log.info("All {} part files have been deleted successfully.", partFiles.size()); - }); + } } private boolean atLeastOneMarcExists(StepExecution stepExecution) { diff --git a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/ListIdentifiersWriteListener.java b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/ListIdentifiersWriteListener.java index 64c0e0721..d2a1f1a98 100644 --- a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/ListIdentifiersWriteListener.java +++ b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/ListIdentifiersWriteListener.java @@ -6,7 +6,6 @@ import org.springframework.stereotype.Component; import java.util.List; -import java.util.stream.Collectors; @Component @RequiredArgsConstructor @@ -14,7 +13,7 @@ public class ListIdentifiersWriteListener implements ItemWriteListener delegate; @Override public void afterWrite(Chunk> list) { - var chunk = new Chunk<>(list.getItems().stream().flatMap(List::stream).collect(Collectors.toList())); + var chunk = new Chunk<>(list.getItems().stream().flatMap(List::stream).toList()); delegate.afterWrite(chunk); } } diff --git a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processquery/items/BulkEditItemCqlJobConfig.java b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processquery/items/BulkEditItemCqlJobConfig.java index 54e50c0f2..51ee80914 100644 --- a/src/main/java/org/folio/dew/batch/bulkedit/jobs/processquery/items/BulkEditItemCqlJobConfig.java +++ b/src/main/java/org/folio/dew/batch/bulkedit/jobs/processquery/items/BulkEditItemCqlJobConfig.java @@ -86,6 +86,7 @@ public Step bulkEditItemCqlPartitionStep( .reader(bulkEditCqlItemReader) .processor(processor) .writer(itemWriter) + .listener(itemWriter) .faultTolerant() .allowStartIfComplete(false) .throttleLimit(POOL_SIZE) diff --git a/src/main/java/org/folio/dew/batch/bursarfeesfines/service/BursarWriter.java b/src/main/java/org/folio/dew/batch/bursarfeesfines/service/BursarWriter.java index d12e7aac6..d081c4226 100644 --- a/src/main/java/org/folio/dew/batch/bursarfeesfines/service/BursarWriter.java +++ b/src/main/java/org/folio/dew/batch/bursarfeesfines/service/BursarWriter.java @@ -58,11 +58,10 @@ public void write(Chunk items) throws Exception { .map(token -> BursarTokenFormatter.formatHeaderFooterToken(token, items.size(), aggregateTotalAmount)) .collect(Collectors.joining()); - localFilesStorage.write(resource.getFilename(), header.getBytes(StandardCharsets.UTF_8)); + String result = header + + lines + + footer; - localFilesStorage.append(resource.getFilename(), lines.toString() - .getBytes(StandardCharsets.UTF_8)); - - localFilesStorage.append(resource.getFilename(), footer.getBytes(StandardCharsets.UTF_8)); + localFilesStorage.write(resource.getFilename(), result.getBytes(StandardCharsets.UTF_8)); } } diff --git a/src/main/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriter.java b/src/main/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriter.java index 7889f9fb1..4c005d182 100644 --- a/src/main/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriter.java +++ b/src/main/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriter.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; import org.folio.de.entity.EHoldingsPackage; import org.folio.dew.domain.dto.EHoldingsExportConfig; import org.folio.dew.domain.dto.eholdings.EHoldingsResourceExportFormat; @@ -132,7 +133,13 @@ private void writePackage(Long jobExecutionId) throws IOException { } private void writeString(String str) throws IOException { - localFilesStorage.append(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8)); + byte[] bytesHeader = new byte[0]; + try { + bytesHeader = localFilesStorage.readAllBytes(tempOutputFilePath); + } catch (IOException e) { + // Just ignore it if there's nothing to append to. + } + localFilesStorage.write(tempOutputFilePath, ArrayUtils.addAll(bytesHeader, str.getBytes(StandardCharsets.UTF_8))); } private String getHeader(List fieldNames) { diff --git a/src/main/java/org/folio/dew/batch/eholdings/GetEHoldingsWriter.java b/src/main/java/org/folio/dew/batch/eholdings/GetEHoldingsWriter.java index b99b38ff6..70155090c 100644 --- a/src/main/java/org/folio/dew/batch/eholdings/GetEHoldingsWriter.java +++ b/src/main/java/org/folio/dew/batch/eholdings/GetEHoldingsWriter.java @@ -4,8 +4,6 @@ import static org.folio.dew.batch.eholdings.EHoldingsJobConstants.CONTEXT_TOTAL_RESOURCES; import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; import org.folio.dew.domain.dto.eholdings.EHoldingsResourceDTO; import org.folio.dew.repository.EHoldingsResourceRepository; @@ -40,7 +38,7 @@ public void beforeStep(StepExecution stepExecution) { @Override public void write(Chunk list) throws Exception { - var resources = list.getItems().stream().map(EHoldingsResourceMapper::convertToEntity).collect(Collectors.toList()); + var resources = list.getItems().stream().map(EHoldingsResourceMapper::convertToEntity).toList(); resources.forEach(r -> r.setJobExecutionId(jobId)); repository.saveAll(resources); jobExecution.getExecutionContext().putInt(CONTEXT_TOTAL_RESOURCES, diff --git a/src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java b/src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java index 26f823ed2..fbc724d72 100644 --- a/src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java +++ b/src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java @@ -6,7 +6,6 @@ import org.folio.dew.repository.LocalFilesStorage; import java.util.List; -import java.util.stream.Collectors; public class DataExportCsvItemReader extends CsvItemReader { @@ -30,7 +29,7 @@ protected List getItems(int offset, int limit) { .skip(offset) .limit(limit) .map(ItemIdentifier::new) - .collect(Collectors.toList()); + .toList(); } } catch (Exception e) { throw new FileOperationException(e.getMessage()); diff --git a/src/main/java/org/folio/dew/controller/BulkEditController.java b/src/main/java/org/folio/dew/controller/BulkEditController.java index 3bfb8f8a6..12d197a66 100644 --- a/src/main/java/org/folio/dew/controller/BulkEditController.java +++ b/src/main/java/org/folio/dew/controller/BulkEditController.java @@ -22,17 +22,15 @@ import static org.folio.dew.utils.SystemHelper.getTempDirWithSeparatorSuffix; import static org.folio.spring.scope.FolioExecutionScopeExecutionContextManager.getRunnableWithCurrentFolioContext; -import io.swagger.annotations.ApiParam; import jakarta.annotation.PostConstruct; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDate; import java.util.List; +import java.util.Objects; import java.util.UUID; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.io.FilenameUtils; @@ -53,9 +51,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; @@ -129,14 +125,15 @@ public ResponseEntity uploadCsvFile(UUID jobId, MultipartFile file) { } private String saveTemporaryIdentifiersFile(UUID jobId, MultipartFile file) throws IOException { - var tempDir = getTempDirWithSeparatorSuffix() + springApplicationName + PATH_SEPARATOR + jobId; - var tempFilePath = tempDir + PATH_SEPARATOR + file.getOriginalFilename(); - var path = Path.of(tempFilePath); + var tempDir = Path.of(getTempDirWithSeparatorSuffix(), springApplicationName, jobId.toString()); + Files.createDirectories(tempDir); + var originalFilename = Objects.requireNonNull(file.getOriginalFilename(), "File name must not be null"); + var sanitizedFilename = Path.of(originalFilename).getFileName().toString(); + var path = tempDir.resolve(sanitizedFilename).normalize(); Files.deleteIfExists(path); - Files.createDirectories(Path.of(tempDir)); Files.write(path, file.getBytes()); - log.info("Saved temporary identifiers file: {}", tempFilePath); - return tempFilePath; + log.info("Saved temporary identifiers file: {}", path); + return path.toString(); } @Override @@ -162,7 +159,7 @@ public ResponseEntity startJob(UUID jobId) { } @Override - public ResponseEntity getErrorsPreviewByJobId(@ApiParam(value = "UUID of the JobCommand", required = true) @PathVariable("jobId") UUID jobId, @NotNull @ApiParam(value = "The numbers of users to return", required = true) @Valid @RequestParam(value = "limit") Integer limit) { + public ResponseEntity getErrorsPreviewByJobId(UUID jobId, Integer limit) { var jobCommand = getJobCommandById(jobId.toString()); var fileName = jobCommand.getId() + PATH_SEPARATOR + FilenameUtils.getName(jobCommand.getJobParameters().getString(FILE_NAME)); log.info("downloadHoldingsPreviewByJobId:: fileName={}", fileName); diff --git a/src/main/java/org/folio/dew/error/DefaultErrorHandler.java b/src/main/java/org/folio/dew/error/DefaultErrorHandler.java index 2e0e793a5..ddb637269 100644 --- a/src/main/java/org/folio/dew/error/DefaultErrorHandler.java +++ b/src/main/java/org/folio/dew/error/DefaultErrorHandler.java @@ -17,7 +17,6 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import java.util.Collections; -import java.util.stream.Collectors; @ControllerAdvice public class DefaultErrorHandler { @@ -69,7 +68,7 @@ public ResponseEntity handleNonSupportedEntityTypeException(final NonSup public ResponseEntity handleMethodArgumentNotValidException(final MethodArgumentNotValidException e) { var parameters = e.getBindingResult().getAllErrors().stream() .map(this::processValidationError) - .collect(Collectors.toList()); + .toList(); return new ResponseEntity<>(new Errors() .errors(Collections.singletonList(new Error() .message("Invalid request body") diff --git a/src/main/java/org/folio/dew/repository/BaseFilesStorage.java b/src/main/java/org/folio/dew/repository/BaseFilesStorage.java index 18b2fa899..5e8444434 100644 --- a/src/main/java/org/folio/dew/repository/BaseFilesStorage.java +++ b/src/main/java/org/folio/dew/repository/BaseFilesStorage.java @@ -1,171 +1,78 @@ package org.folio.dew.repository; -import io.minio.BucketExistsArgs; -import io.minio.ComposeObjectArgs; -import io.minio.ComposeSource; -import io.minio.GetObjectArgs; -import io.minio.ListObjectsArgs; -import io.minio.MakeBucketArgs; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import io.minio.StatObjectArgs; -import io.minio.UploadObjectArgs; -import io.minio.credentials.IamAwsProvider; -import io.minio.credentials.Provider; -import io.minio.credentials.StaticProvider; - +import io.minio.http.Method; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; import lombok.extern.log4j.Log4j2; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.folio.dew.config.properties.MinioClientProperties; import org.folio.dew.error.FileOperationException; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import org.folio.s3.client.FolioS3Client; +import org.folio.s3.client.PutObjectAdditionalOptions; +import org.folio.s3.client.S3ClientFactory; +import org.folio.s3.client.S3ClientProperties; +import org.folio.s3.exception.S3ClientException; +import org.springframework.http.HttpHeaders; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.net.URI; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import static io.minio.ObjectWriteArgs.MIN_MULTIPART_SIZE; -import static java.lang.String.format; -import static org.folio.dew.utils.Constants.PATH_SEPARATOR; @Log4j2 public class BaseFilesStorage implements S3CompatibleStorage { + public static final String CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME = "attachment"; + public static final String CONTENT_DISPOSITION_HEADER_WITH_FILENAME = CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME + "; filename=\"%s\""; + + private static final String SET_VALUE = ""; private static final String NOT_SET_VALUE = ""; - private final MinioClient client; - private S3Client s3Client; - private final String bucket; - private final String region; - private final String subPath; - - private final boolean isComposeWithAwsSdk; + final FolioS3Client client; + final String bucket; + private final int urlExpirationTimeInSeconds; public BaseFilesStorage(MinioClientProperties properties) { + + bucket = properties.getBucket(); + urlExpirationTimeInSeconds = properties.getUrlExpirationTimeInSeconds(); + String subPath = properties.getSubPath(); + final String accessKey = properties.getAccessKey(); final String endpoint = properties.getEndpoint(); final String regionName = properties.getRegion(); - final String bucketName = properties.getBucket(); final String secretKey = properties.getSecretKey(); - subPath = properties.getSubPath(); - isComposeWithAwsSdk = properties.isComposeWithAwsSdk(); + boolean isComposeWithAwsSdk = properties.isComposeWithAwsSdk(); final boolean isForcePathStyle = properties.isForcePathStyle(); - log.info("Creating MinIO client endpoint {},region {},bucket {},accessKey {},secretKey {}, subPath {}, isComposedWithAwsSdk {}.", endpoint, regionName, bucketName, - StringUtils.isNotBlank(accessKey) ? SET_VALUE : NOT_SET_VALUE, StringUtils.isNotBlank(secretKey) ? SET_VALUE : NOT_SET_VALUE, - StringUtils.isNotBlank(subPath) ? SET_VALUE : NOT_SET_VALUE, isComposeWithAwsSdk); - - var builder = MinioClient.builder().endpoint(endpoint); - if (StringUtils.isNotBlank(regionName)) { - builder.region(regionName); - } - - Provider provider; - if (StringUtils.isNotBlank(accessKey) && StringUtils.isNotBlank(secretKey)) { - provider = new StaticProvider(accessKey, secretKey, null); - } else { - provider = new IamAwsProvider(null, null); - } - log.info("{} MinIO credentials provider created.", provider.getClass().getSimpleName()); - builder.credentialsProvider(provider); - client = builder.build(); + log.info("Creating S3 client endpoint {},region {},bucket {},accessKey {},secretKey {}, subPath {}, isComposedWithAwsSdk {}.", endpoint, regionName, bucket, + StringUtils.isNotBlank(accessKey) ? SET_VALUE : NOT_SET_VALUE, StringUtils.isNotBlank(secretKey) ? SET_VALUE : NOT_SET_VALUE, + StringUtils.isNotBlank(subPath) ? SET_VALUE : NOT_SET_VALUE, isComposeWithAwsSdk); - this.bucket = bucketName; - this.region = regionName; - - createBucketIfNotExists(); - - if (isComposeWithAwsSdk) { - AwsCredentialsProvider credentialsProvider; - - if (StringUtils.isNotBlank(accessKey) && StringUtils.isNotBlank(secretKey)) { - var awsCredentials = AwsBasicCredentials.create(accessKey, secretKey); - credentialsProvider = StaticCredentialsProvider.create(awsCredentials); - } else { - credentialsProvider = DefaultCredentialsProvider.create(); - } - - s3Client = S3Client.builder() + client = S3ClientFactory.getS3Client(S3ClientProperties.builder() + .endpoint(endpoint) + .secretKey(secretKey) + .accessKey(accessKey) + .bucket(bucket) + .awsSdk(isComposeWithAwsSdk) .forcePathStyle(isForcePathStyle) - .endpointOverride(URI.create(endpoint)) - .region(Region.of(regionName)) - .credentialsProvider(credentialsProvider) - .build(); - } - - } - - public MinioClient getMinioClient() { - return client; - } + .subPath(subPath) + .region(regionName) + .build()); - public void createBucketIfNotExists() { try { - if (StringUtils.isNotBlank(bucket) && !client.bucketExists(BucketExistsArgs.builder().bucket(bucket).region(region).build())) { - client.makeBucket(MakeBucketArgs.builder() - .bucket(bucket) - .region(region) - .build()); - log.info("Created {} bucket.", bucket); - } else { - log.info("Bucket has already exist."); - } - } catch(Exception e) { - log.error("Error creating bucket: " + bucket, e); - } - } - - /** - * Upload file on S3-compatible storage - * - * @param path - the path to the file on S3-compatible storage - * @param filename – path to uploaded file - * @return the path to the file - * @throws IOException - if an I/O error occurs - */ - public String upload(String path, String filename) throws IOException { - path = getS3Path(path); - try { - return client.uploadObject(UploadObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path) - .filename(filename) - .build()) - .object(); - } catch (Exception e) { - throw new IOException("Cannot upload file: " + path, e); + client.createBucketIfNotExists(); + } catch (S3ClientException e) { + log.error("Error creating bucket: {} during RemoteStorageClient initialization", bucket); } } @@ -179,29 +86,11 @@ public String upload(String path, String filename) throws IOException { * @throws IOException - if an I/O error occurs */ public String write(String path, byte[] bytes, Map headers) throws IOException { - path = getS3Path(path); - if (isComposeWithAwsSdk) { - log.info("Writing with using AWS SDK client"); - s3Client.putObject(PutObjectRequest.builder().bucket(bucket) - .key(path).build(), - RequestBody.fromBytes(bytes)); - return path; - } else { - log.info("Writing with using Minio client"); - try(var is = new ByteArrayInputStream(bytes)) { - return client.putObject(PutObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path) - .headers(headers) - .stream(is, -1, MIN_MULTIPART_SIZE) - .build()) - .object(); - } catch (Exception e) { - throw new IOException("Cannot write file: " + path, e); - } - - } + var options = PutObjectAdditionalOptions.builder() + .contentDisposition(headers.get(HttpHeaders.CONTENT_DISPOSITION)) + .contentType(headers.get(HttpHeaders.CONTENT_TYPE)) + .build(); + return client.write(path, new ByteArrayInputStream(bytes), bytes.length, options); } public String write(String path, byte[] bytes) throws IOException { @@ -211,40 +100,17 @@ public String write(String path, byte[] bytes) throws IOException { /** * Writes file to a file on S3-compatible storage * - * @param path - the path to the file on S3-compatible storage + * @param destPath - the path to the file on S3-compatible storage * @param inputPath – path to the file to write - * @param headers - headers * @return the path to the file * @throws IOException - if an I/O error occurs */ - public String writeFile(String path, Path inputPath, Map headers) throws IOException { - path = getS3Path(path); - if (isComposeWithAwsSdk) { - log.info("Writing file using AWS SDK client"); - s3Client.putObject(PutObjectRequest.builder().bucket(bucket) - .key(path).build(), - RequestBody.fromFile(inputPath)); - return path; - } else { - log.info("Writing file using Minio client"); - try (var is = Files.newInputStream(inputPath)) { - return client.putObject(PutObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path) - .headers(headers) - .stream(is, -1, MIN_MULTIPART_SIZE) - .build()) - .object(); - } catch (Exception e) { - throw new IOException("Cannot write file: " + path, e); - } - - } - } - public String writeFile(String destPath, Path inputPath) throws IOException { - return writeFile(destPath, inputPath, new HashMap<>()); + try (var is = Files.newInputStream(inputPath)) { + return client.write(destPath, is); + } catch (Exception e) { + throw new IOException("Cannot write file: " + destPath, e); + } } /** @@ -255,99 +121,7 @@ public String writeFile(String destPath, Path inputPath) throws IOException { * @throws IOException if an I/O error occurs */ public void append(String path, byte[] bytes) throws IOException { - path = getS3Path(path); - try { - if (notExists(path)) { - log.info("Appending non-existing file"); - write(path, bytes); - } else { - var size = client.statObject(StatObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path).build()).size(); - - log.info("Appending to {} with size {}", path, size); - if (size > MIN_MULTIPART_SIZE) { - - if (isComposeWithAwsSdk) { - - var createMultipartUploadRequest = CreateMultipartUploadRequest.builder() - .bucket(bucket) - .key(path) - .build(); - - var uploadId = s3Client.createMultipartUpload(createMultipartUploadRequest).uploadId(); - - var uploadPartRequest1 = UploadPartCopyRequest.builder() - .sourceBucket(bucket) - .sourceKey(path) - .uploadId(uploadId) - .destinationBucket(bucket) - .destinationKey(path) - .partNumber(1).build(); - - var uploadPartRequest2 = UploadPartRequest.builder() - .bucket(bucket) - .key(path) - .uploadId(uploadId) - .partNumber(2).build(); - - var originalEtag = s3Client.uploadPartCopy(uploadPartRequest1).copyPartResult().eTag(); - var appendedEtag = s3Client.uploadPart(uploadPartRequest2, RequestBody.fromBytes(bytes)).eTag(); - - var original = CompletedPart.builder() - .partNumber(1) - .eTag(originalEtag).build(); - var appended = CompletedPart.builder() - .partNumber(2) - .eTag(appendedEtag).build(); - - var completedMultipartUpload = CompletedMultipartUpload.builder() - .parts(original, appended) - .build(); - - var completeMultipartUploadRequest = - CompleteMultipartUploadRequest.builder() - .bucket(bucket) - .key(path) - .uploadId(uploadId) - .multipartUpload(completedMultipartUpload) - .build(); - - s3Client.completeMultipartUpload(completeMultipartUploadRequest); - - } else { - - var temporaryFileName = path + "_temp"; - write(temporaryFileName, bytes); - - client.composeObject(ComposeObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path) - .sources(List.of(ComposeSource.builder() - .bucket(bucket) - .region(region) - .object(path) - .build(), - ComposeSource.builder() - .bucket(bucket) - .region(region) - .object(temporaryFileName) - .build())) - .build()); - - delete(temporaryFileName); - - } - - } else { - write(path, ArrayUtils.addAll(readAllBytes(path), bytes)); - } - } - } catch (Exception e) { - throw new IOException("Cannot append data for path: " + path, e); - } + client.append(path, new ByteArrayInputStream(bytes)); } /** @@ -357,24 +131,17 @@ public void append(String path, byte[] bytes) throws IOException { * @throws FileOperationException if an I/O error occurs */ public void delete(String path) { - path = getS3Path(path); - try { - var paths = walk(path).collect(Collectors.toList()); - - paths.forEach(p -> { - try { - client.removeObject(RemoveObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(p) - .build()); - } catch (Exception e) { - log.error(format("Cannot delete file: %s", p), e.getMessage()); - } - }); - } catch (Exception e) { - throw new FileOperationException("Cannot delete file: " + path, e); - } + client.remove(walk(path).toArray(String[]::new)); + } + + /** + * Deletes a file + * + * @param objects - the path to the file to delete + * @throws FileOperationException if an I/O error occurs + */ + public void delete(List objects) { + client.remove(objects.toArray(String[]::new)); } /** @@ -386,7 +153,7 @@ public void delete(String path) { * @throws FileOperationException if an I/O error occurs */ public Stream walk(String path) { - return getInternalStructure(getS3Path(path), true); + return getInternalStructure(path); } /** @@ -396,18 +163,11 @@ public Stream walk(String path) { * @return true if file exists, otherwise - false */ public boolean exists(String path) { - path = getS3Path(path); - var iterator = client.listObjects(ListObjectsArgs.builder() - .bucket(bucket) - .region(region) - .prefix(path) - .maxKeys(1) - .build()) - .iterator(); try { - return iterator.hasNext() && Objects.nonNull(iterator.next().get()); + var paths = client.list(path); + return !paths.isEmpty() && Objects.nonNull(paths.getFirst()); } catch (Exception e) { - log.error("Error file existing verification, path: " + path, e); + log.error("Error file existing verification, path: {}", path, e); return false; } } @@ -426,19 +186,9 @@ public boolean notExists(String path) { * * @param path - the path to the file on S3-compatible storage * @return a new input stream - * @throws IOException - if an I/O error occurs reading from the file */ - public InputStream newInputStream(String path) throws IOException { - path = getS3Path(path); - try { - return client.getObject(GetObjectArgs.builder() - .bucket(bucket) - .region(region) - .object(path) - .build()); - } catch (Exception e) { - throw new IOException("Error creating input stream for path: " + path, e); - } + public InputStream newInputStream(String path) { + return client.read(path); } /** @@ -488,83 +238,36 @@ public List linesNumber(String path, int num) throws IOException { } } - /** - * Read all lines from a file - * - * @param path - the path to the file on S3-compatible storage - * @return - the lines from the file as a List - * @throws IOException - if an I/O error occurs reading from the file - */ - public List readAllLines(String path) throws IOException { - try (var lines = lines(path)) { - return lines.collect(Collectors.toList()); - } catch(Exception e) { - throw new IOException("Error reading file: " + path, e); - } - } - - public OutputStream newOutputStream(String path) { - - - return new OutputStream() { - - byte[] buffer = new byte[0]; + public String compose(String destObject, List sourceObjects, String downloadFilename, + String contentType) { - @Override - public void write(int b) { - buffer = ArrayUtils.add(buffer, (byte) b); - } - - @Override - public void flush() { -// throw new NotImplementedException("Method isn't implemented yet"); - } + var headers = prepareHeaders(downloadFilename, contentType); + var result = client.compose(destObject, sourceObjects, PutObjectAdditionalOptions.builder() + .contentType(headers.get(HttpHeaders.CONTENT_TYPE)) + .contentDisposition(HttpHeaders.CONTENT_DISPOSITION).build()); - @Override - public void close() { - try { - BaseFilesStorage.this.write(path, buffer); - } catch (IOException e) { - throw new FileOperationException("Error closing stream and writes bytes to path: " + path, e); - } finally { - buffer = new byte[0]; - } - } - }; + client.remove(sourceObjects.toArray(new String[0])); + return result; } - public BufferedWriter writer(String path) { - return new BufferedWriter(new OutputStreamWriter(newOutputStream(path))); + public String objectToPresignedObjectUrl(String object) { + return client.getPresignedUrl(object, Method.GET, urlExpirationTimeInSeconds, TimeUnit.SECONDS); } - private Stream getInternalStructure(String path, boolean isRecursive) { - try { - return StreamSupport.stream(client.listObjects(ListObjectsArgs.builder() - .bucket(bucket) - .region(region) - .prefix(path) - .recursive(isRecursive) - .build()).spliterator(), false).map(item -> { - try { - return item.get().objectName(); - } catch (Exception e) { - throw new FileOperationException(e); - } - }); - } catch(Exception e) { - log.error("Cannot read folder: " + path, e); - return null; + Map prepareHeaders(String downloadFilename, String contentType) { + Map headers = HashMap.newHashMap(2); + if (StringUtils.isNotBlank(downloadFilename)) { + headers.put(HttpHeaders.CONTENT_DISPOSITION, String.format(CONTENT_DISPOSITION_HEADER_WITH_FILENAME, downloadFilename)); + } else { + headers.put(HttpHeaders.CONTENT_DISPOSITION, CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME); } + if (StringUtils.isNotBlank(contentType)) { + headers.put(HttpHeaders.CONTENT_TYPE, contentType); + } + return headers; } - public String getS3Path(String path) { - if (StringUtils.isBlank(subPath) || StringUtils.startsWith(path, subPath + PATH_SEPARATOR)) { - return path; - } - if (path.startsWith(PATH_SEPARATOR)) { - return subPath + path; - } - return subPath + PATH_SEPARATOR + path; + private Stream getInternalStructure(String path) { + return client.listRecursive(path).stream(); } -} +} \ No newline at end of file diff --git a/src/main/java/org/folio/dew/repository/RemoteFilesStorage.java b/src/main/java/org/folio/dew/repository/RemoteFilesStorage.java index 065546ae1..7bb8608ff 100644 --- a/src/main/java/org/folio/dew/repository/RemoteFilesStorage.java +++ b/src/main/java/org/folio/dew/repository/RemoteFilesStorage.java @@ -1,38 +1,10 @@ package org.folio.dew.repository; -import io.minio.ComposeObjectArgs; -import io.minio.ComposeSource; -import io.minio.GetPresignedObjectUrlArgs; -import io.minio.ListObjectsArgs; -import io.minio.MinioClient; -import io.minio.ObjectWriteArgs; -import io.minio.RemoveObjectsArgs; -import io.minio.Result; -import io.minio.errors.ErrorResponseException; -import io.minio.errors.InsufficientDataException; -import io.minio.errors.InternalException; -import io.minio.errors.InvalidResponseException; -import io.minio.errors.ServerException; -import io.minio.errors.XmlParserException; -import io.minio.http.Method; -import io.minio.messages.DeleteError; -import io.minio.messages.DeleteObject; - import java.io.IOException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import io.minio.messages.Item; import lombok.extern.log4j.Log4j2; -import org.apache.commons.lang3.StringUtils; import org.folio.dew.config.properties.RemoteFilesStorageProperties; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Repository; @@ -40,22 +12,11 @@ @Log4j2 public class RemoteFilesStorage extends BaseFilesStorage { - public static final String CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME = "attachment"; - public static final String CONTENT_DISPOSITION_HEADER_WITH_FILENAME = CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME + "; filename=\"%s\""; - - private final MinioClient client; @Autowired private LocalFilesStorage localFilesStorage; - private final String bucket; - private final String region; - private final int urlExpirationTimeInSeconds; public RemoteFilesStorage(RemoteFilesStorageProperties properties) { super(properties); - this.bucket = properties.getBucket(); - this.region = properties.getRegion(); - this.urlExpirationTimeInSeconds = properties.getUrlExpirationTimeInSeconds(); - this.client = getMinioClient(); } public String uploadObject(String object, String filename, String downloadFilename, String contentType, boolean isSourceShouldBeDeleted) @@ -72,77 +33,4 @@ public String uploadObject(String object, String filename, String downloadFilena return result; } - - public boolean containsFile(String fileName) - throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, - InvalidKeyException, InvalidResponseException, XmlParserException, InternalException { - fileName = getS3Path(fileName); - for (Result itemResult : client.listObjects(ListObjectsArgs.builder().bucket(bucket).prefix(getS3Path(fileName)).build())) { - if (fileName.equals(itemResult.get().objectName())) { - return true; - } - } - return false; - } - - public String composeObject(String destObject, List sourceObjects, String downloadFilename, - String contentType) - throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, - ServerException, InternalException, XmlParserException, ErrorResponseException { - destObject = getS3Path(destObject); - List sources = sourceObjects.stream() - .map(so -> ComposeSource.builder().bucket(bucket).object(getS3Path(so)).build()) - .collect(Collectors.toList()); - log.info("Composing object {},sources [{}],downloadFilename {},contentType {}.", destObject, - sources.stream().map(s -> String.format("bucket %s,object %s", s.bucket(), s.object())).collect(Collectors.joining(",")), - downloadFilename, contentType); - var result = client.composeObject( - createArgs(ComposeObjectArgs.builder().sources(sources), destObject, downloadFilename, contentType)).object(); - - removeObjects(sourceObjects); - - return result; - } - - public Iterable> removeObjects(List objects) { - log.info("Deleting objects [{}].", StringUtils.join(objects, ",")); - return client.removeObjects(RemoveObjectsArgs.builder() - .bucket(bucket) - .objects(objects.stream().map(this::getS3Path).map(DeleteObject::new).toList()) - .build()); - } - - public String objectToPresignedObjectUrl(String object) - throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, - ServerException, InternalException, XmlParserException, ErrorResponseException { - String result = client.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder() - .method(Method.GET) - .bucket(bucket) - .object(getS3Path(object)) - .region(region) - .expiry(urlExpirationTimeInSeconds, TimeUnit.SECONDS) - .build()); - log.info("Created presigned URL {}.", result); - return result; - } - - private > T createArgs(B builder, String object, - String downloadFilename, String contentType) { - Map headers = prepareHeaders(downloadFilename, contentType); - return builder.headers(headers).object(object).bucket(bucket).build(); - } - - private Map prepareHeaders(String downloadFilename, String contentType) { - Map headers = new HashMap<>(2); - if (StringUtils.isNotBlank(downloadFilename)) { - headers.put(HttpHeaders.CONTENT_DISPOSITION, String.format(CONTENT_DISPOSITION_HEADER_WITH_FILENAME, downloadFilename)); - } else { - headers.put(HttpHeaders.CONTENT_DISPOSITION, CONTENT_DISPOSITION_HEADER_WITHOUT_FILENAME); - } - if (StringUtils.isNotBlank(contentType)) { - headers.put(HttpHeaders.CONTENT_TYPE, contentType); - } - return headers; - } - -} +} \ No newline at end of file diff --git a/src/main/java/org/folio/dew/repository/S3CompatibleStorage.java b/src/main/java/org/folio/dew/repository/S3CompatibleStorage.java index 524006341..474c791c6 100644 --- a/src/main/java/org/folio/dew/repository/S3CompatibleStorage.java +++ b/src/main/java/org/folio/dew/repository/S3CompatibleStorage.java @@ -2,13 +2,9 @@ import java.io.IOException; import java.io.InputStream; -import java.util.Map; public interface S3CompatibleStorage { - String upload(String path, String filename) throws IOException; - void append(String path, byte[] bytes) throws IOException; String write(String path, byte[] bytes) throws IOException; - String write(String path, byte[] bytes, Map headers) throws IOException; boolean exists(String path); InputStream newInputStream(String path) throws IOException; byte[] readAllBytes(String path) throws IOException; diff --git a/src/main/java/org/folio/dew/service/BulkEditProcessingErrorsService.java b/src/main/java/org/folio/dew/service/BulkEditProcessingErrorsService.java index fe4e0f49c..d7ff666bb 100644 --- a/src/main/java/org/folio/dew/service/BulkEditProcessingErrorsService.java +++ b/src/main/java/org/folio/dew/service/BulkEditProcessingErrorsService.java @@ -3,7 +3,6 @@ import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Objects.isNull; -import static java.util.stream.Collectors.toList; import static org.folio.dew.utils.Constants.PATH_SEPARATOR; import static org.folio.dew.utils.Constants.PATH_TO_ERRORS; import static org.folio.dew.utils.SystemHelper.validatePath; @@ -31,7 +30,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; @Service @@ -103,7 +101,7 @@ public Errors readErrorsFromCSV(String jobId, String fileName, Integer limit) { .map(s -> s.split(COMMA_SEPARATOR, 3)) .map(message -> new Error().message("%s%s%s".formatted(message[IDX_ERROR_IDENTIFIER], COMMA_SEPARATOR, message[IDX_ERROR_MSG])) .type(ErrorType.fromValue(message[IDX_ERROR_TYPE]))) - .collect(toList()); + .toList(); log.info("Errors file {} processing completed", csvFileName); var totalErrors = errors.stream().filter(e -> e.getType() == ErrorType.ERROR).count(); var totalWarnings = errors.stream().filter(e -> e.getType() == ErrorType.WARNING).count(); @@ -182,7 +180,7 @@ public String getCsvFileName(String jobId, String fileName) { names = localFilesStorage.walk(pathToStorage).map(x -> { var n = x.split("/"); return n[n.length - 1]; - }).collect(Collectors.toList()); + }).toList(); } if (names.isEmpty()) { return LocalDate.now().format(CSV_NAME_DATE_FORMAT) + "-Matching-Records-Errors-" + fileName; diff --git a/src/main/java/org/folio/dew/service/JobCommandsReceiverService.java b/src/main/java/org/folio/dew/service/JobCommandsReceiverService.java index fdea7e7a0..172db1d40 100644 --- a/src/main/java/org/folio/dew/service/JobCommandsReceiverService.java +++ b/src/main/java/org/folio/dew/service/JobCommandsReceiverService.java @@ -171,7 +171,7 @@ private boolean deleteOldFiles(JobCommand jobCommand) { } }).filter(StringUtils::isNotBlank).distinct().collect(Collectors.toList()); if (!objects.isEmpty()) { - remoteFilesStorage.removeObjects(objects); + remoteFilesStorage.delete(objects); } jobCommandRepository.delete(jobCommand); bulkEditProcessingErrorsService.removeTemporaryErrorStorage(); diff --git a/src/main/java/org/folio/dew/service/ModuleTenantService.java b/src/main/java/org/folio/dew/service/ModuleTenantService.java index 29f71b73c..6364dc633 100644 --- a/src/main/java/org/folio/dew/service/ModuleTenantService.java +++ b/src/main/java/org/folio/dew/service/ModuleTenantService.java @@ -2,6 +2,7 @@ import static org.folio.dew.utils.Constants.EUREKA_PLATFORM; +import java.util.regex.Pattern; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.log4j.Log4j2; @@ -29,6 +30,7 @@ public class ModuleTenantService { private static final String MOD_USERS = "mod-users"; private static final String MOD_USERS_NOT_FOUND_ERROR = "Module id not found for name: " + MOD_USERS; private static final String MOD_USERS_REGEXP = "^mod-users-\\d.*$"; + private static final Pattern MOD_USERS_PATTERN = Pattern.compile(MOD_USERS_REGEXP); @Setter @Value("${application.platform}") @@ -48,7 +50,7 @@ private Optional getModUsersModuleIdForOkapi() { var tenantId = folioExecutionContext.getTenantId(); var modules = okapiClient.getModuleIds(URI.create(URL_PREFIX), tenantId, MOD_USERS); if (!modules.isEmpty()) { - return Optional.of(modules.get(0).getId()); + return Optional.of(modules.getFirst().getId()); } return Optional.empty(); } @@ -59,6 +61,9 @@ private Optional getModUsersModuleIdForEureka() { } private Optional filterModUsersModuleId(List modules) { - return modules.stream().map(ModuleForTenant::getId).filter(id -> id.matches(MOD_USERS_REGEXP)).findFirst(); + return modules.stream() + .map(ModuleForTenant::getId) + .filter(id -> MOD_USERS_PATTERN.matcher(id).matches()) + .findFirst(); } } diff --git a/src/main/java/org/folio/dew/service/mapper/HoldingsMapper.java b/src/main/java/org/folio/dew/service/mapper/HoldingsMapper.java index e008a2463..5ca278659 100644 --- a/src/main/java/org/folio/dew/service/mapper/HoldingsMapper.java +++ b/src/main/java/org/folio/dew/service/mapper/HoldingsMapper.java @@ -13,7 +13,6 @@ import org.folio.dew.domain.dto.ExtendedHoldingsRecord; import org.folio.dew.domain.dto.HoldingsFormat; import org.folio.dew.domain.dto.HoldingsNote; -import org.folio.dew.domain.dto.HoldingsRecord; import org.folio.dew.domain.dto.HoldingsStatement; import org.folio.dew.domain.dto.Tags; import org.folio.dew.service.ElectronicAccessService; @@ -106,10 +105,4 @@ private String tagsToString(Tags tags) { } return isEmpty(tags.getTagList()) ? EMPTY : String.join(ARRAY_DELIMITER, escaper.escape(tags.getTagList())); } - - - public HoldingsRecord mapToHoldingsRecord(HoldingsFormat holdingsFormat) { - return new HoldingsRecord(); - } - } diff --git a/src/main/java/org/folio/dew/utils/CsvHelper.java b/src/main/java/org/folio/dew/utils/CsvHelper.java index 84ef0eb28..b06f0553a 100644 --- a/src/main/java/org/folio/dew/utils/CsvHelper.java +++ b/src/main/java/org/folio/dew/utils/CsvHelper.java @@ -5,28 +5,18 @@ import static org.folio.dew.utils.Constants.UTF8_BOM; import com.opencsv.bean.CsvToBeanBuilder; -import com.opencsv.bean.StatefulBeanToCsvBuilder; -import com.opencsv.exceptions.CsvDataTypeMismatchException; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; -import lombok.experimental.UtilityClass; -import lombok.extern.log4j.Log4j2; - -import org.apache.commons.lang3.StringUtils; -import org.folio.dew.repository.BaseFilesStorage; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; - -import java.io.StringReader; -import java.io.StringWriter; import java.util.List; -import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; +import org.folio.dew.repository.BaseFilesStorage; @UtilityClass @Log4j2 public class CsvHelper { - private static final int BATCH_SIZE = 1000; public static List readRecordsFromStorage(R storage, String fileName, Class clazz, boolean skipHeaders) throws IOException { try (var reader = new BufferedReader(new InputStreamReader(storage.newInputStream(fileName)))) { @@ -38,54 +28,6 @@ public static List readRecordsFromStorage(R s } } - public static List readRecordsFromRemoteFilesStorage(R storage, String fileName, int limit, Class clazz) - throws IOException { - try (var reader = new BufferedReader(new InputStreamReader(storage.newInputStream(fileName)))) { - var linesString = reader.lines().skip(1).limit(limit).collect(Collectors.joining("\n")); - return new CsvToBeanBuilder(new StringReader(linesString)) - .withType(clazz) - .build() - .parse(); - } - } - - public static void saveRecordsToStorage(R storage, List beans, Class clazz, String fileName) - throws CsvRequiredFieldEmptyException, CsvDataTypeMismatchException, IOException { - var strategy = new RecordColumnMappingStrategy(); - strategy.setType(clazz); - - if (storage.exists(fileName)) { - storage.delete(fileName); - } - - if (beans.size() > BATCH_SIZE) { - for (int batchNumber = 0; batchNumber <= beans.size() / BATCH_SIZE; batchNumber++) { - log.info("Writing batch #{}", batchNumber); - var batch = beans.stream() - .skip((long) batchNumber * BATCH_SIZE) - .limit(BATCH_SIZE) - .collect(Collectors.toList()); - try (var stringWriter = new StringWriter()) { - new StatefulBeanToCsvBuilder(stringWriter) - .withApplyQuotesToAll(false) - .withMappingStrategy(strategy) - .build() - .write(batch); - var csvString = stringWriter.toString(); - storage.append(fileName, batchNumber == 0 ? csvString.getBytes() : csvString.substring(csvString.indexOf(LINE_BREAK) + 1).getBytes()); - } - } - } else { - try (var writer = storage.writer(fileName)) { - new StatefulBeanToCsvBuilder(writer) - .withApplyQuotesToAll(false) - .withMappingStrategy(strategy) - .build() - .write(beans); - } - } - } - public static long countLines(R storage, String path) throws IOException { try (var lines = storage.lines(path)) { return lines.count(); diff --git a/src/main/java/org/folio/dew/utils/ExportFormatHelper.java b/src/main/java/org/folio/dew/utils/ExportFormatHelper.java index 5808a309e..c0cc15c28 100644 --- a/src/main/java/org/folio/dew/utils/ExportFormatHelper.java +++ b/src/main/java/org/folio/dew/utils/ExportFormatHelper.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import static java.lang.String.format; -import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.SPACE; import static org.apache.commons.lang3.StringUtils.capitalize; import static org.apache.commons.lang3.StringUtils.join; @@ -45,11 +44,9 @@ public static String getItemRow(Object item) { var bw = new BeanWrapperImpl(item); for (var fieldName : clazzFields) { var value = bw.getPropertyValue(fieldName); - if (value instanceof String) { - var s = getStringValue((String) value); + if (value instanceof String str) { + var s = getStringValue(str); itemValues.add(s); - } else { - itemValues.add(EMPTY); } } return String.join(",", itemValues); @@ -60,7 +57,7 @@ private static List getExportFormatHeaders(Class clazz) { var exportFormat = clazz.getAnnotation(ExportFormat.class); return Arrays.stream(clazz.getDeclaredFields()) .map(field -> getFieldColumnName(exportFormat, field)) - .collect(Collectors.toList()); + .toList(); } private static String getFieldColumnName(ExportFormat exportFormat, Field field) { @@ -85,7 +82,7 @@ private static void verifyAnnotationPresence(Class clazz) { } private static String decapitalize(String string) { - if (string == null || string.length() == 0) { + if (string == null || string.isEmpty()) { return string; } @@ -109,6 +106,6 @@ private static String getStringValue(String value) { private static List getClassFields(Class clazz) { return Arrays.stream(clazz.getDeclaredFields()) .map(Field::getName) - .collect(Collectors.toUnmodifiableList()); + .toList(); } } diff --git a/src/main/java/org/folio/dew/utils/RecordColumnMappingStrategy.java b/src/main/java/org/folio/dew/utils/RecordColumnMappingStrategy.java deleted file mode 100644 index 497ca31fb..000000000 --- a/src/main/java/org/folio/dew/utils/RecordColumnMappingStrategy.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.folio.dew.utils; - -import static org.apache.commons.lang3.StringUtils.EMPTY; - -import com.opencsv.bean.BeanField; -import com.opencsv.bean.ColumnPositionMappingStrategy; -import com.opencsv.bean.CsvBindByName; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; - -public class RecordColumnMappingStrategy extends ColumnPositionMappingStrategy { - @Override - public String[] generateHeader(T bean) throws CsvRequiredFieldEmptyException { - super.generateHeader(bean); - - return getFieldMap().values().stream() - .map(this::extractHeaderName) - .toArray(String[]::new); - } - - private String extractHeaderName(BeanField beanField) { - return beanField == null || beanField.getField() == null || beanField.getField().getDeclaredAnnotationsByType(CsvBindByName.class).length == 0 ? - EMPTY : - beanField.getField().getDeclaredAnnotationsByType(CsvBindByName.class)[0].column(); - } -} diff --git a/src/main/java/org/folio/dew/utils/SystemHelper.java b/src/main/java/org/folio/dew/utils/SystemHelper.java index 6833de045..6af2bf7e0 100644 --- a/src/main/java/org/folio/dew/utils/SystemHelper.java +++ b/src/main/java/org/folio/dew/utils/SystemHelper.java @@ -3,17 +3,24 @@ import lombok.experimental.UtilityClass; import java.io.File; +import java.util.Objects; +import java.util.regex.Pattern; import static org.folio.dew.utils.Constants.CHARACTERS_SHOULD_BE_REPLACED_IN_PATH; @UtilityClass public class SystemHelper { + + private static final Pattern INVALID_PATH_CHARS = + Pattern.compile(CHARACTERS_SHOULD_BE_REPLACED_IN_PATH); + public static String getTempDirWithSeparatorSuffix() { var dir = System.getProperty("java.io.tmpdir"); return dir.endsWith(File.separator) ? dir : dir + File.separator; } public static String validatePath(String path) { - return path.replaceAll(CHARACTERS_SHOULD_BE_REPLACED_IN_PATH, "_"); + Objects.requireNonNull(path, "Path must not be null"); + return INVALID_PATH_CHARS.matcher(path).replaceAll("_"); } -} +} \ No newline at end of file diff --git a/src/test/java/org/folio/dew/AuthorityControlTest.java b/src/test/java/org/folio/dew/AuthorityControlTest.java index 2476d36c4..c77f7bd6d 100644 --- a/src/test/java/org/folio/dew/AuthorityControlTest.java +++ b/src/test/java/org/folio/dew/AuthorityControlTest.java @@ -51,7 +51,7 @@ class AuthorityControlTest extends BaseBatchTest { "src/test/resources/output/authority_control/auth_heading_update.csv"; private static final String EXPECTED_AUTH_HEADING_UPDATE_EMPTY_OUTPUT = "src/test/resources/output/authority_control/auth_heading_update_empty.csv"; - private static final String EXPECTED_S3_FILE_PATH = "remote/mod-data-export-worker/authority_control_export/diku/"; + private static final String EXPECTED_S3_FILE_PATH = "mod-data-export-worker/authority_control_export/diku/"; @Autowired private Job getAuthHeadingJob; @Autowired diff --git a/src/test/java/org/folio/dew/BaseBatchTest.java b/src/test/java/org/folio/dew/BaseBatchTest.java index b4be7e9e4..592b5d18e 100644 --- a/src/test/java/org/folio/dew/BaseBatchTest.java +++ b/src/test/java/org/folio/dew/BaseBatchTest.java @@ -5,6 +5,7 @@ import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -70,13 +71,17 @@ import org.springframework.http.HttpHeaders; import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.springframework.test.context.support.TestPropertySourceUtils; import org.springframework.test.util.TestSocketUtils; import org.springframework.test.web.servlet.MockMvc; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { "spring.kafka.bootstrap-servers=${spring.embedded.kafka.brokers}"}) @@ -130,8 +135,14 @@ public abstract class BaseBatchTest { @MockitoBean public ElectronicAccessRelationshipClient relationshipClient; + + public static final LocalStackContainer localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.10.0")) + .withServices(S3) + .withEnv("EAGER_SERVICE_LOADING", "1"); + static { postgreDBContainer.start(); + localstack.start(); } protected Map okapiHeaders = new HashMap<>(); @@ -148,6 +159,27 @@ public void initialize(ConfigurableApplicationContext applicationContext) { } } + + @DynamicPropertySource + static void properties(DynamicPropertyRegistry dynamicPropertyRegistry) { + String endpoint = localstack.getEndpointOverride(S3).toString(); + String region = localstack.getRegion(); + String access = localstack.getAccessKey(); + String secret = localstack.getSecretKey(); + + dynamicPropertyRegistry.add("application.minio-local.endpoint", () -> endpoint); + dynamicPropertyRegistry.add("application.minio-local.region", () -> region); + dynamicPropertyRegistry.add("application.minio-local.accessKey",() -> access); + dynamicPropertyRegistry.add("application.minio-local.secretKey",() -> secret); + dynamicPropertyRegistry.add("application.minio-remote.subPath",() -> "local"); + + dynamicPropertyRegistry.add("application.minio-remote.endpoint", () -> endpoint); + dynamicPropertyRegistry.add("application.minio-remote.region", () -> region); + dynamicPropertyRegistry.add("application.minio-remote.accessKey",() -> access); + dynamicPropertyRegistry.add("application.minio-remote.secretKey",() -> secret); + dynamicPropertyRegistry.add("application.minio-remote.subPath",() -> "remote"); + } + @BeforeAll static void beforeAll(@Autowired MockMvc mockMvc) { wireMockServer = new WireMockServer(WIRE_MOCK_PORT); @@ -202,8 +234,6 @@ protected void setUp() { var defaultFolioExecutionContext = new DefaultFolioExecutionContext(folioModuleMetadata, localHeaders); folioExecutionContextSetter = new FolioExecutionContextSetter(defaultFolioExecutionContext); - remoteFilesStorage.createBucketIfNotExists(); - when(searchClient.getConsortiumItemCollection(any())) .thenAnswer(batchIdsDro -> { var items = ((BatchIdsDto) batchIdsDro.getArguments()[0]).getIdentifierValues().stream().map(id -> new ConsortiumItem().id(id).tenantId("tenant_" + id.charAt(0))).toList(); diff --git a/src/test/java/org/folio/dew/BulkEditTest.java b/src/test/java/org/folio/dew/BulkEditTest.java index 38f9f2255..0f61ffea8 100644 --- a/src/test/java/org/folio/dew/BulkEditTest.java +++ b/src/test/java/org/folio/dew/BulkEditTest.java @@ -746,16 +746,16 @@ private void assertFileEqualsIgnoringCreatedAndUpdatedDate(FileSystemResource ex throws IOException, JSONException { var expectedContent = IOUtils.toString(expectedJsonFile.getInputStream(), Charset.forName("UTF-8")); var actualContent = IOUtils.toString(actualJsonResult.getInputStream(), Charset.forName("UTF-8")); - String actualUpdated = ""; + StringBuilder actualUpdated = new StringBuilder(); var jsons = actualContent.split("\n"); Arrays.sort(jsons); for (String json : jsons) { var actualJsonItem = new JSONObject(json); actualJsonItem.remove("createdDate"); actualJsonItem.remove("updatedDate"); - actualUpdated += actualJsonItem + "\n"; + actualUpdated.append(actualJsonItem).append("\n"); } - assertEquals(expectedContent.trim(), actualUpdated.trim().replaceAll("\\\\", "")); + assertEquals(expectedContent.trim(), actualUpdated.toString().trim().replaceAll("\\\\", "")); } @SneakyThrows diff --git a/src/test/java/org/folio/dew/CirculationLogTest.java b/src/test/java/org/folio/dew/CirculationLogTest.java index 93724c0a8..b3d8377c1 100644 --- a/src/test/java/org/folio/dew/CirculationLogTest.java +++ b/src/test/java/org/folio/dew/CirculationLogTest.java @@ -76,7 +76,7 @@ private void verifyFileOutput(JobExecution jobExecution) throws Exception { final ExecutionContext executionContext = jobExecution.getExecutionContext(); final String fileInStorage = (String) executionContext.get("outputFilesInStorage"); final String fileName = executionContext.getString(CIRCULATION_LOG_FILE_NAME); - final String expectedNameInStorage = "remote/" + fileName; + final String expectedNameInStorage = fileName; final FileSystemResource actualChargeFeesFinesOutput = actualFileOutput(fileInStorage); FileSystemResource expectedCharges = new FileSystemResource(EXPECTED_CIRCULATION_OUTPUT); diff --git a/src/test/java/org/folio/dew/EHoldingsTest.java b/src/test/java/org/folio/dew/EHoldingsTest.java index 0cace2a7c..59c25bae0 100644 --- a/src/test/java/org/folio/dew/EHoldingsTest.java +++ b/src/test/java/org/folio/dew/EHoldingsTest.java @@ -81,7 +81,7 @@ class EHoldingsTest extends BaseBatchTest { "src/test/resources/output/eholdings_package_export_with_3_titles.csv"; private final static String EXPECTED_PACKAGE_WITH_SAME_TITLE_NAMES_OUTPUT = "src/test/resources/output/eholdings_package_export_with_same_title_names.csv"; - private static final String FILE_PATH = "remote/mod-data-export-worker/e_holdings_export/diku/"; + private static final String FILE_PATH = "mod-data-export-worker/e_holdings_export/diku/"; @Test @DisplayName("Run EHoldingsJob export resource without provider load successfully") diff --git a/src/test/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriterTest.java b/src/test/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriterTest.java index fdc3e54dd..c0d232d0c 100644 --- a/src/test/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriterTest.java +++ b/src/test/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriterTest.java @@ -32,7 +32,8 @@ class AuthorityControlCsvFileWriterTest { @BeforeEach void setUp() { authorityControlCsvFileWriter = new AuthorityControlCsvFileWriter(AuthUpdateHeadingExportFormat.class, TEMP_FILE, localFilesStorage); - lenient().doNothing().when(localFilesStorage).append(anyString(), any()); + lenient().when(localFilesStorage.write(anyString(), any())).thenReturn("/path"); + lenient().when(localFilesStorage.readAllBytes(anyString())).thenReturn("header".getBytes(StandardCharsets.UTF_8)); } @Test @@ -42,7 +43,7 @@ void testWriteHeadersBeforeStepMethod() { authorityControlCsvFileWriter.beforeStep(); //Then - verify(localFilesStorage).append(eq(TEMP_FILE), any()); + verify(localFilesStorage).write(eq(TEMP_FILE), any()); } @Test @@ -54,7 +55,7 @@ void testWriteHeadersAfterStepMethod_whenStatsExist() { authorityControlCsvFileWriter.afterStep(); //Then - verify(localFilesStorage, never()).append(eq(TEMP_FILE), any()); + verify(localFilesStorage, never()).write(eq(TEMP_FILE), any()); } @Test @@ -66,7 +67,8 @@ void testWriteHeadersAfterStepMethod_whenStatsNotExist() { authorityControlCsvFileWriter.afterStep(); //Then - verify(localFilesStorage).append(TEMP_FILE, "No records found".getBytes(StandardCharsets.UTF_8)); + verify(localFilesStorage).write(TEMP_FILE, ("header\n" + + "No records found").getBytes(StandardCharsets.UTF_8)); } @Test @@ -82,6 +84,6 @@ void testWriteItemsMethod() { authorityControlCsvFileWriter.write(new Chunk<>(items)); //Then - verify(localFilesStorage).append(eq(TEMP_FILE), any()); + verify(localFilesStorage).write(eq(TEMP_FILE), any()); } } diff --git a/src/test/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/UtilsTest.java b/src/test/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/UtilsTest.java index 818e05b29..6287c7e50 100644 --- a/src/test/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/UtilsTest.java +++ b/src/test/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/UtilsTest.java @@ -4,7 +4,7 @@ import static org.junit.Assert.assertEquals; -public class UtilsTest { +class UtilsTest { @Test void testEncode() { diff --git a/src/test/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriterTest.java b/src/test/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriterTest.java index ee9421d54..2dc2f9d14 100644 --- a/src/test/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriterTest.java +++ b/src/test/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriterTest.java @@ -60,7 +60,7 @@ void setUp() { lenient().when(jobExecution.getExecutionContext()).thenReturn(executionContext); lenient().when(executionContext.getInt(anyString(), anyInt())).thenReturn(100); lenient().when(exportConfig.getRecordId()).thenReturn("recordId"); - lenient().doNothing().when(localFilesStorage).append(anyString(), any()); + lenient().when(localFilesStorage.write(anyString(), any())).thenReturn("/path"); EHoldingsPackageExportFormat exportFormat = new EHoldingsPackageExportFormat(); exportFormat.setPackageId("packageId"); exportFormat.setPackageName("packageName"); @@ -82,7 +82,7 @@ void testBeforeStep(List packageFields, eHoldingsCsvFileWriter.beforeStep(stepExecution); - verify(localFilesStorage, times(localFileStorageInvocations)).append(anyString(), any()); + verify(localFilesStorage, times(localFileStorageInvocations)).write(anyString(), any()); verify(packageRepository, times(packageRepositoryInvocations)).findById(any()); verify(mapper, times(mapperInvocations)).convertToExportFormat(any(EHoldingsPackage.class)); } @@ -102,7 +102,7 @@ void testWriteMethod(List titleFields, int localFileStorageInvocations){ eHoldingsCsvFileWriter.write(new Chunk<>(items)); //Then - verify(localFilesStorage, times(localFileStorageInvocations)).append(anyString(), any()); + verify(localFilesStorage, times(localFileStorageInvocations)).write(anyString(), any()); } private static Stream provideParameters() { diff --git a/src/test/java/org/folio/dew/controller/BulkEditControllerTest.java b/src/test/java/org/folio/dew/controller/BulkEditControllerTest.java index e0e4ab93d..3632ba0f0 100644 --- a/src/test/java/org/folio/dew/controller/BulkEditControllerTest.java +++ b/src/test/java/org/folio/dew/controller/BulkEditControllerTest.java @@ -1,6 +1,7 @@ package org.folio.dew.controller; import static java.lang.String.format; +import static org.awaitility.Awaitility.await; import static org.folio.dew.domain.dto.EntityType.USER; import static org.folio.dew.domain.dto.ExportType.BULK_EDIT_IDENTIFIERS; import static org.folio.dew.domain.dto.ExportType.BULK_EDIT_UPDATE; @@ -32,6 +33,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import org.apache.commons.io.FilenameUtils; import org.folio.de.entity.JobCommand; @@ -163,10 +165,9 @@ void shouldLaunchJobAndReturnNumberOfRecordsOnIdentifiersFileUpload() { .andReturn(); assertThat(result.getResponse().getContentAsString(), equalTo("3")); - // Lets wait for async invocation to complete - Thread.sleep(3000); - verify(exportJobManagerSync, times(1)).launchJob(any()); - } + await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> verify(exportJobManagerSync, times(1)).launchJob(any())); } @Test @DisplayName("Upload empty file - BAD REQUEST") diff --git a/src/test/java/org/folio/dew/controller/PresignedUrlControllerTest.java b/src/test/java/org/folio/dew/controller/PresignedUrlControllerTest.java index cc42b0c80..1adb22c0d 100644 --- a/src/test/java/org/folio/dew/controller/PresignedUrlControllerTest.java +++ b/src/test/java/org/folio/dew/controller/PresignedUrlControllerTest.java @@ -1,5 +1,7 @@ package org.folio.dew.controller; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.commons.io.FilenameUtils; import org.folio.dew.BaseBatchTest; import org.folio.dew.repository.RemoteFilesStorage; @@ -23,7 +25,7 @@ public class PresignedUrlControllerTest extends BaseBatchTest { void shouldRetrievePresignedUrl() throws Exception { var jobId = UUID.randomUUID(); var filePath = jobId + PATH_SEPARATOR + FilenameUtils.getName(PREVIEW_ITEM_DATA); - remoteFilesStorage.upload(filePath, PREVIEW_ITEM_DATA); + remoteFilesStorage.write(filePath, Files.readAllBytes(Path.of(PREVIEW_ITEM_DATA))); var headers = defaultHeaders(); diff --git a/src/test/java/org/folio/dew/repository/BaseIntegration.java b/src/test/java/org/folio/dew/repository/BaseIntegration.java new file mode 100644 index 000000000..f921cbb5c --- /dev/null +++ b/src/test/java/org/folio/dew/repository/BaseIntegration.java @@ -0,0 +1,50 @@ +package org.folio.dew.repository; + +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; + +import java.time.Duration; +import lombok.extern.log4j.Log4j2; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +@Testcontainers +@Log4j2 +public abstract class BaseIntegration { + + @Container + public static final LocalStackContainer localstack + = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.10.0")) + .withServices(S3) + .waitingFor( + Wait.forHttp("/_localstack/health") + .forPort(4566) + .forStatusCode(200) + .forResponsePredicate(s -> s.contains("\"s3\": \"running\"")) + .withStartupTimeout(Duration.ofMinutes(1)) + ) + .withEnv("EAGER_SERVICE_LOADING", "1"); + + + @DynamicPropertySource + static void props(DynamicPropertyRegistry dynamicPropertyRegistry) { + String endpoint = localstack.getEndpointOverride(S3).toString(); + String region = localstack.getRegion(); + String access = localstack.getAccessKey(); + String secret = localstack.getSecretKey(); + + dynamicPropertyRegistry.add("application.minio-local.endpoint", () -> endpoint); + dynamicPropertyRegistry.add("application.minio-local.region", () -> region); + dynamicPropertyRegistry.add("application.minio-local.accessKey",() -> access); + dynamicPropertyRegistry.add("application.minio-local.secretKey",() -> secret); + + dynamicPropertyRegistry.add("application.minio-remote.endpoint", () -> endpoint); + dynamicPropertyRegistry.add("application.minio-remote.region", () -> region); + dynamicPropertyRegistry.add("application.minio-remote.accessKey",() -> access); + dynamicPropertyRegistry.add("application.minio-remote.secretKey",() -> secret); + } +} \ No newline at end of file diff --git a/src/test/java/org/folio/dew/repository/LocalFilesStorageTest.java b/src/test/java/org/folio/dew/repository/FilesStorageTest.java similarity index 60% rename from src/test/java/org/folio/dew/repository/LocalFilesStorageTest.java rename to src/test/java/org/folio/dew/repository/FilesStorageTest.java index 8205a7d6e..6f76d4757 100644 --- a/src/test/java/org/folio/dew/repository/LocalFilesStorageTest.java +++ b/src/test/java/org/folio/dew/repository/FilesStorageTest.java @@ -1,8 +1,10 @@ package org.folio.dew.repository; import io.minio.ObjectWriteArgs; +import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.folio.dew.config.properties.LocalFilesStorageProperties; +import org.folio.s3.exception.S3ClientException; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -11,28 +13,23 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.context.SpringBootTest; -import java.io.BufferedWriter; import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.Objects; import java.util.concurrent.ThreadLocalRandom; import static java.util.List.of; -import static java.util.stream.Collectors.toList; -import static org.folio.dew.utils.Constants.PATH_SEPARATOR; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; @Log4j2 @SpringBootTest(classes = {LocalFilesStorageProperties.class, LocalFilesStorage.class}) @EnableConfigurationProperties -class LocalFilesStorageTest { +class FilesStorageTest extends BaseIntegration { private static final String NON_EXISTING_PATH = "non-existing-path"; @Autowired @@ -45,12 +42,11 @@ class LocalFilesStorageTest { @ValueSource(ints = {1024, ObjectWriteArgs.MIN_MULTIPART_SIZE + 1 }) @DisplayName("Create files and read internal objects structure") void testWriteRead(int size) throws IOException { - var subPath = localFilesStorageProperties.getSubPath() + PATH_SEPARATOR; byte[] content = getRandomBytes(size); var original = of("directory_1/CSV_Data_1.csv", "directory_1/directory_2/CSV_Data_2.csv", "directory_1/directory_2/directory_3/CSV_Data_3.csv"); - var expectedS3Pathes = of(subPath + "directory_1/CSV_Data_1.csv", subPath + "directory_1/directory_2/CSV_Data_2.csv", - subPath + "directory_1/directory_2/directory_3/CSV_Data_3.csv"); + var expectedS3Pathes = of("directory_1/CSV_Data_1.csv", "directory_1/directory_2/CSV_Data_2.csv", + "directory_1/directory_2/directory_3/CSV_Data_3.csv"); List actual; try { actual = original.stream() @@ -61,21 +57,21 @@ void testWriteRead(int size) throws IOException { throw new RuntimeException(ex); } }) - .collect(toList()); + .toList(); } catch(Exception e) { throw new IOException(e); } assertTrue(Objects.deepEquals(expectedS3Pathes, actual)); - assertTrue(Objects.deepEquals(localFilesStorage.walk(subPath + "directory_1/") - .collect(toList()), - of(subPath + "directory_1/CSV_Data_1.csv", subPath + "directory_1/directory_2/CSV_Data_2.csv", - subPath + "directory_1/directory_2/directory_3/CSV_Data_3.csv"))); + assertTrue(Objects.deepEquals(localFilesStorage.walk("directory_1/") + .toList(), + of("directory_1/CSV_Data_1.csv", "directory_1/directory_2/CSV_Data_2.csv", + "directory_1/directory_2/directory_3/CSV_Data_3.csv"))); - assertTrue(Objects.deepEquals(localFilesStorage.walk(subPath + "directory_1/directory_2/") - .collect(toList()), - of(subPath + "directory_1/directory_2/CSV_Data_2.csv", subPath + "directory_1/directory_2/directory_3/CSV_Data_3.csv"))); + assertTrue(Objects.deepEquals(localFilesStorage.walk("directory_1/directory_2/") + .toList(), + of("directory_1/directory_2/CSV_Data_2.csv", "directory_1/directory_2/directory_3/CSV_Data_3.csv"))); original.forEach(p -> assertTrue(localFilesStorage.exists(p))); @@ -85,50 +81,21 @@ void testWriteRead(int size) throws IOException { original.forEach(p -> assertFalse(localFilesStorage.exists(p))); } - @ParameterizedTest - @ValueSource(ints = {1024, 2048}) - @DisplayName("Buffered writer test") - void testBufferedWriter(int size) { - var path = "directory/resource.csv"; - var expected = new String(getRandomBytes(size)); - try(BufferedWriter writer = localFilesStorage.writer(path)) { - writer.write(expected); - } catch (IOException e) { - fail("Writer exception"); - } - - try(InputStream is = localFilesStorage.newInputStream(path)) { - var actual = new String(is.readAllBytes()); - assertEquals(expected, actual); - } catch (IOException e) { - fail("Read resource exception"); - } - - // Clean crated files - localFilesStorage.delete(path); - } - @ParameterizedTest @DisplayName("Create file, update it (append bytes[]), read and delete") @ValueSource(ints = { 1024, ObjectWriteArgs.MIN_MULTIPART_SIZE + 1 }) void testWriteReadPatchDelete(int size) throws IOException { byte[] original = getRandomBytes(size); - byte[] patch = getRandomBytes(size); var remoteFilePath = "directory_1/directory_2/CSV_Data.csv"; - var expectedS3FilePath = localFilesStorageProperties.getSubPath() + PATH_SEPARATOR + remoteFilePath; - assertThat(localFilesStorage.write(remoteFilePath, original), is(expectedS3FilePath)); + assertThat(localFilesStorage.write(remoteFilePath, original), is(remoteFilePath)); assertTrue(localFilesStorage.exists(remoteFilePath)); assertTrue(Objects.deepEquals(localFilesStorage.readAllBytes(remoteFilePath), original)); - assertTrue(Objects.deepEquals(localFilesStorage.lines(remoteFilePath) - .collect(toList()), localFilesStorage.readAllLines(remoteFilePath))); - - localFilesStorage.append(remoteFilePath, patch); var patched = localFilesStorage.readAllBytes(remoteFilePath); - assertThat(patched.length, is(original.length + patch.length)); + assertThat(patched.length, is(original.length)); localFilesStorage.delete(remoteFilePath); assertTrue(localFilesStorage.notExists(remoteFilePath)); @@ -137,9 +104,8 @@ void testWriteReadPatchDelete(int size) throws IOException { @Test @DisplayName("Files operations on non-existing file") void testNonExistingFileOperations() { - assertThrows(IOException.class, () -> localFilesStorage.readAllLines(NON_EXISTING_PATH)); - assertThrows(IOException.class, () -> localFilesStorage.lines(NON_EXISTING_PATH)); - assertThrows(IOException.class, () -> { + assertThrows(S3ClientException.class, () -> localFilesStorage.lines(NON_EXISTING_PATH)); + assertThrows(S3ClientException.class, () -> { try(var is = localFilesStorage.newInputStream(NON_EXISTING_PATH)){ log.info("InputStream setup"); } @@ -150,6 +116,19 @@ void testNonExistingFileOperations() { assertFalse(localFilesStorage.exists(NON_EXISTING_PATH)); } + @Test + @SneakyThrows + void testContainsFile() { + byte[] content = "content".getBytes(); + var path = "directory/data.csv"; + + var uploadedPath = localFilesStorage.write(path, content); + + assertEquals("directory/data.csv", uploadedPath); + assertTrue(localFilesStorage.exists(path)); + assertTrue(localFilesStorage.exists(uploadedPath)); + } + private byte[] getRandomBytes(int size) { var original = new byte[size]; ThreadLocalRandom.current() diff --git a/src/test/java/org/folio/dew/repository/LocalFilesStorageAwsSdkComposingTest.java b/src/test/java/org/folio/dew/repository/LocalFilesStorageAwsSdkComposingTest.java index 8527182f6..582b17f67 100644 --- a/src/test/java/org/folio/dew/repository/LocalFilesStorageAwsSdkComposingTest.java +++ b/src/test/java/org/folio/dew/repository/LocalFilesStorageAwsSdkComposingTest.java @@ -1,9 +1,8 @@ package org.folio.dew.repository; -import org.apache.commons.lang3.ArrayUtils; +import java.nio.charset.StandardCharsets; import org.folio.dew.config.properties.LocalFilesStorageProperties; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; @@ -11,15 +10,12 @@ import org.springframework.boot.test.context.SpringBootTest; import java.io.IOException; -import java.util.Arrays; import java.util.Objects; import java.util.concurrent.ThreadLocalRandom; import io.minio.ObjectWriteArgs; import lombok.extern.log4j.Log4j2; -import static java.util.stream.Collectors.toList; -import static org.folio.dew.utils.Constants.PATH_SEPARATOR; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -28,7 +24,7 @@ @SpringBootTest(classes = { LocalFilesStorageProperties.class, LocalFilesStorage.class }, properties = { "application.minio-local.compose-with-aws-sdk = true", "application.minio-local.force-path-style = true" }) @EnableConfigurationProperties -class LocalFilesStorageAwsSdkComposingTest { +class LocalFilesStorageAwsSdkComposingTest extends BaseIntegration { @Autowired private LocalFilesStorageProperties localFilesStorageProperties; @@ -42,49 +38,21 @@ void testWriteReadPatchDelete(int size) throws IOException { byte[] original = getRandomBytes(size); var remoteFilePath = "CSV_Data.csv"; - var expectedS3Path = localFilesStorageProperties.getSubPath() + PATH_SEPARATOR + remoteFilePath; - assertThat(localFilesStorage.write(remoteFilePath, original), is(expectedS3Path)); + assertThat(localFilesStorage.write(remoteFilePath, original), is(remoteFilePath)); assertTrue(localFilesStorage.exists(remoteFilePath)); assertTrue(Objects.deepEquals(localFilesStorage.readAllBytes(remoteFilePath), original)); - assertTrue(Objects.deepEquals(localFilesStorage.lines(remoteFilePath) - .collect(toList()), localFilesStorage.readAllLines(remoteFilePath))); + assertTrue(Objects.deepEquals(localFilesStorage.lines(remoteFilePath).toList(), new String(original, StandardCharsets.UTF_8).lines().toList())); localFilesStorage.delete(remoteFilePath); assertTrue(localFilesStorage.notExists(remoteFilePath)); } - @Test - @DisplayName("Append files with using AWS SDK workaround instead of MinIO client composeObject-method") - void testAppendFileParts() throws IOException { - - var name = "directory_1/directory_2/CSV_Data.csv"; - byte[] file = getRandomBytes(30000000); - var size = file.length; - - var first = Arrays.copyOfRange(file, 0, size / 3); - var second = Arrays.copyOfRange(file, size / 3, 2 * size / 3); - var third = Arrays.copyOfRange(file, 2 * size / 3, size); - - var expected = ArrayUtils.addAll(ArrayUtils.addAll(first, second), third); - - assertTrue(Objects.deepEquals(file, expected)); - - localFilesStorage.append(name, first); - localFilesStorage.append(name, second); - localFilesStorage.append(name, third); - - var result = localFilesStorage.readAllBytes(name); - - assertTrue(Objects.deepEquals(file, result)); - - } - private byte[] getRandomBytes(int size) { var original = new byte[size]; ThreadLocalRandom.current() .nextBytes(original); return original; } -} +} \ No newline at end of file diff --git a/src/test/java/org/folio/dew/repository/RemoteFilesStorageTest.java b/src/test/java/org/folio/dew/repository/RemoteFilesStorageTest.java deleted file mode 100644 index bf4ceaf99..000000000 --- a/src/test/java/org/folio/dew/repository/RemoteFilesStorageTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.folio.dew.repository; - - -import lombok.SneakyThrows; -import org.folio.dew.BaseBatchTest; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class RemoteFilesStorageTest extends BaseBatchTest { - - @Autowired - private RemoteFilesStorage remoteFilesStorage; - - @Test - @SneakyThrows - void testContainsFile() { - byte[] content = "content".getBytes(); - var path = "directory/data.csv"; - - var uploadedPath = remoteFilesStorage.write(path, content); - - assertEquals("remote/directory/data.csv", uploadedPath); - assertTrue(remoteFilesStorage.containsFile(path)); - assertTrue(remoteFilesStorage.containsFile(uploadedPath)); - } -} diff --git a/src/test/java/org/folio/dew/service/BulkEditProcessingErrorsServiceTest.java b/src/test/java/org/folio/dew/service/BulkEditProcessingErrorsServiceTest.java index 1963c61f8..e7b09f0de 100644 --- a/src/test/java/org/folio/dew/service/BulkEditProcessingErrorsServiceTest.java +++ b/src/test/java/org/folio/dew/service/BulkEditProcessingErrorsServiceTest.java @@ -42,7 +42,7 @@ void saveErrorInCSVTestSuccessTest() throws IOException { var pathToCsvFile = "E" + File.separator + BulkEditProcessingErrorsService.STORAGE + File.separator + jobId + File.separator + csvFileName; bulkEditProcessingErrorsService.saveErrorInCSV(jobId, affectedIdentifier, reasonForError, fileName); assertTrue(localFilesStorage.exists(pathToCsvFile)); - List lines = localFilesStorage.readAllLines(pathToCsvFile); + List lines = localFilesStorage.lines(pathToCsvFile).toList(); String expectedLine = "ERROR," + affectedIdentifier + "," + reasonForError.getMessage(); assertEquals(expectedLine, lines.get(0)); assertThat(lines, hasSize(1)); @@ -50,7 +50,7 @@ void saveErrorInCSVTestSuccessTest() throws IOException { // Second attempt to verify file name calculation logic bulkEditProcessingErrorsService.saveErrorInCSV(jobId, affectedIdentifier, reasonForError, fileName); assertTrue(localFilesStorage.exists(pathToCsvFile)); - lines = localFilesStorage.readAllLines(pathToCsvFile); + lines = localFilesStorage.lines(pathToCsvFile).toList(); assertThat(lines, hasSize(2)); removeStorage(); @@ -67,7 +67,7 @@ void saveErrorMessageInCSVTestSuccessTest() throws IOException { var pathToCsvFile = "E" + File.separator + BulkEditProcessingErrorsService.STORAGE + File.separator + jobId + File.separator + csvFileName; bulkEditProcessingErrorsService.saveErrorInCSV(jobId, affectedIdentifier, errorMessage, fileName, ErrorType.ERROR); assertTrue(localFilesStorage.exists(pathToCsvFile)); - List lines = localFilesStorage.readAllLines(pathToCsvFile); + List lines = localFilesStorage.lines(pathToCsvFile).toList(); String expectedLine = "ERROR," + affectedIdentifier + "," + errorMessage; assertEquals(expectedLine, lines.get(0)); assertThat(lines, hasSize(1)); @@ -106,7 +106,7 @@ void saveErrorInCSVTestFailedTest() { @Test @DisplayName("Read errors from csv file") - void readErrorsFromCsvTest() throws BulkEditException, IOException { + void readErrorsFromCsvTest() throws BulkEditException { int numOfErrorLines = 3; int errorsPreviewLimit = 2; var jobId = UUID.randomUUID().toString(); @@ -121,7 +121,7 @@ void readErrorsFromCsvTest() throws BulkEditException, IOException { removeStorage(); } - private void removeStorage() throws IOException { + private void removeStorage() { localFilesStorage.delete("E" + File.separator + BulkEditProcessingErrorsService.STORAGE); } diff --git a/src/test/java/org/folio/dew/service/SpecialCharacterEscaperTest.java b/src/test/java/org/folio/dew/service/SpecialCharacterEscaperTest.java index 4538c7303..a633fdc0e 100644 --- a/src/test/java/org/folio/dew/service/SpecialCharacterEscaperTest.java +++ b/src/test/java/org/folio/dew/service/SpecialCharacterEscaperTest.java @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -public class SpecialCharacterEscaperTest { +class SpecialCharacterEscaperTest { @Test void escapeTest() { diff --git a/src/test/java/org/folio/dew/utils/CsvHelperTest.java b/src/test/java/org/folio/dew/utils/CsvHelperTest.java deleted file mode 100644 index 2484f9631..000000000 --- a/src/test/java/org/folio/dew/utils/CsvHelperTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.folio.dew.utils; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.opencsv.exceptions.CsvDataTypeMismatchException; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; -import org.folio.dew.config.properties.LocalFilesStorageProperties; -import org.folio.dew.domain.dto.ItemFormat; -import org.folio.dew.repository.LocalFilesStorage; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.context.SpringBootTest; - -import java.io.IOException; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -@SpringBootTest(classes = { LocalFilesStorageProperties.class, LocalFilesStorage.class }) -@EnableConfigurationProperties -class CsvHelperTest { - private static final String OUT_PATH = "test-dir/out.csv"; - @Autowired - private LocalFilesStorage localFilesStorage; - - @Test - void shouldWriteRecordsToCsv() throws CsvRequiredFieldEmptyException, CsvDataTypeMismatchException, IOException { - var itemFormats = IntStream.rangeClosed(1, 1100) - .mapToObj(i -> ItemFormat.builder().id(UUID.randomUUID().toString()).barcode(Integer.toString(i)).build()) - .collect(Collectors.toList()); - CsvHelper.saveRecordsToStorage(localFilesStorage, itemFormats, ItemFormat.class, OUT_PATH); - // expect header + 1100 records - assertThat(localFilesStorage.lines(OUT_PATH).count()).isEqualTo(1101); - // Clean crated files - localFilesStorage.delete(OUT_PATH); - } -} diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index cd078f48c..e1ac5a2a7 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -28,7 +28,7 @@ application: bucket: size: ${BUCKET_SIZE:50} minio-remote: - endpoint: http://${embedded.minio.host}:${embedded.minio.port}/ + endpoint: http://${embedded.minio.host}:1111/ region: ${S3_REGION:region} bucket: ${S3_BUCKET:files-test} accessKey: ${S3_ACCESS_KEY_ID:AKIAIOSFODNN7EXAMPLE} @@ -37,7 +37,7 @@ application: subPath: ${S3_SUB_PATH:remote} url-expiration-time-in-seconds: ${URL_EXPIRATION_TIME:604800} # 7 days minio-local: - endpoint: http://${embedded.minio.host}:${embedded.minio.port}/ + endpoint: http://${embedded.minio.host}:1111/ region: ${S3_REGION:region} bucket: ${S3_BUCKET:files-test} accessKey: ${S3_ACCESS_KEY_ID:AKIAIOSFODNN7EXAMPLE}