Feature/mig 400 documents and attachments

migration-app/web/src/dialogs/settings/AdvancedSettingsForm.tsx

@@ -70,6 +70,7 @@ export function AdvancedSettingsForm({ settings, setSettings }: SettingsFormProp
                                     projectConfig: {
                                         ...prev.projectConfig,
                                         paths: {
+                                            ...prev.projectConfig.paths,
                                             images: e.target.value || undefined,
                                         },
                                     },
@@ -87,13 +88,50 @@ export function AdvancedSettingsForm({ settings, setSettings }: SettingsFormProp
                                     projectConfig: {
                                         ...prev.projectConfig,
                                         paths: {
+                                            ...prev.projectConfig.paths,
                                             fonts: e.target.value || undefined,
                                         },
                                     },
                                 }))
                             }
                         />
                     </div>
+                    <div className="grid gap-3">
+                        <Label>Documents</Label>
+                        <Input
+                            value={settings.projectConfig.paths.documents ?? ""}
+                            onChange={(e) =>
+                                setSettings((prev) => ({
+                                    ...prev,
+                                    projectConfig: {
+                                        ...prev.projectConfig,
+                                        paths: {
+                                            ...prev.projectConfig.paths,
+                                            documents: e.target.value || undefined,
+                                        },
+                                    },
+                                }))
+                            }
+                        />
+                    </div>
+                    <div className="grid gap-3">
+                        <Label>Attachments</Label>
+                        <Input
+                            value={settings.projectConfig.paths.attachments ?? ""}
+                            onChange={(e) =>
+                                setSettings((prev) => ({
+                                    ...prev,
+                                    projectConfig: {
+                                        ...prev.projectConfig,
+                                        paths: {
+                                            ...prev.projectConfig.paths,
+                                            attachments: e.target.value || undefined,
+                                        },
+                                    },
+                                }))
+                            }
+                        />
+                    </div>
                 </CardContent>
             </Card>
             <Card>

migration-app/web/src/dialogs/settings/settingsTypes.ts

@@ -51,6 +51,8 @@ export type ProjectConfig = {
 export type PathsConfig = {
     images?: string | undefined;
     fonts?: string | undefined;
+    documents?: string | undefined;
+    attachments?: string | undefined;
 };
 
 export const inspireOutputOptions = ["Designer", "Interactive", "Evolve"] as const;

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/FilesExport.groovy

@@ -0,0 +1,53 @@
+//! ---
+//! displayName: Export Files
+//! category: Mapping
+//! description: Creates CSV files with file details from the migration project. The generated CSV columns can be updated and later imported back into the database using a dedicated import task.
+//! target: gradle
+//! ---
+package com.quadient.migration.example.common.mapping
+
+import com.quadient.migration.api.Migration
+import com.quadient.migration.example.common.util.Csv
+import com.quadient.migration.example.common.util.Mapping
+import com.quadient.migration.service.deploy.ResourceType
+
+import java.nio.file.Path
+
+import static com.quadient.migration.example.common.util.InitMigration.initMigration
+
+def migration = initMigration(this.binding)
+
+def filesPath = Mapping.csvPath(binding, migration.projectConfig.name, "files")
+
+run(migration, filesPath)
+
+static void run(Migration migration, Path filesDstPath) {
+    def files = migration.fileRepository.listAll()
+
+    filesDstPath.toFile().createParentDirectories()
+
+    filesDstPath.toFile().withWriter { writer ->
+        def headers = ["id", "name", "sourcePath", "fileType", "targetFolder", "status", "skip", "skipPlaceholder", "skipReason", Mapping.displayHeader("originalName", true), Mapping.displayHeader("originLocations", true)]
+        writer.writeLine(headers.join(","))
+        files.each { obj ->
+            def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(obj.id,
+                    ResourceType.File,
+                    migration.projectConfig.inspireOutput)
+
+            def builder = new StringBuilder()
+            builder.append(Csv.serialize(obj.id))
+            builder.append("," + Csv.serialize(obj.name))
+            builder.append("," + Csv.serialize(obj.sourcePath))
+            builder.append("," + Csv.serialize(obj.fileType))
+            builder.append("," + Csv.serialize(obj.targetFolder))
+            builder.append("," + Csv.serialize(status.class.simpleName))
+            builder.append("," + Csv.serialize(obj.skip.skipped))
+            builder.append("," + Csv.serialize(obj.skip.placeholder))
+            builder.append("," + Csv.serialize(obj.skip.reason))
+            builder.append("," + Csv.serialize(obj.customFields["originalName"]))
+            builder.append("," + Csv.serialize(obj.originLocations))
+
+            writer.writeLine(builder.toString())
+        }
+    }
+}

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/FilesImport.groovy

@@ -0,0 +1,75 @@
+//! ---
+//! displayName: Import Files
+//! category: Mapping
+//! description: Imports file details from CSV files into the migration project, applying any updates made to the columns during editing.
+//! target: gradle
+//! ---
+package com.quadient.migration.example.common.mapping
+
+import com.quadient.migration.api.Migration
+import com.quadient.migration.example.common.util.Csv
+import com.quadient.migration.example.common.util.Mapping
+import com.quadient.migration.service.deploy.ResourceType
+import com.quadient.migration.shared.FileType
+import com.quadient.migration.shared.SkipOptions
+
+import java.nio.file.Path
+
+import static com.quadient.migration.example.common.util.InitMigration.initMigration
+
+def migration = initMigration(this.binding)
+
+def path = Mapping.csvPath(binding, migration.projectConfig.name, "files")
+
+run(migration, path)
+
+static void run(Migration migration, Path filesFilePath) {
+    def deploymentId = UUID.randomUUID().toString()
+    def now = new Date().getTime()
+    def output = migration.projectConfig.inspireOutput
+    def fileLines = filesFilePath.toFile().readLines()
+    def fileColumnNames = Csv.parseColumnNames(fileLines.removeFirst()).collect { Mapping.normalizeHeader(it) }
+
+    for (line in fileLines) {
+        def values = Csv.getCells(line, fileColumnNames)
+        def id = values.get("id")
+        def existingFile = migration.fileRepository.find(id)
+        def existingMapping = migration.mappingRepository.getFileMapping(id)
+
+        if (existingFile == null) {
+            throw new Exception("File with id ${id} not found")
+        }
+        def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(existingFile.id,
+                ResourceType.File,
+                migration.projectConfig.inspireOutput)
+
+        def newName = Csv.deserialize(values.get("name"), String.class)
+        Mapping.mapProp(existingMapping, existingFile, "name", newName)
+
+        def newSourcePath = Csv.deserialize(values.get("sourcePath"), String.class)
+        Mapping.mapProp(existingMapping, existingFile, "sourcePath", newSourcePath)
+
+        def newFileType = Csv.deserialize(values.get("fileType"), FileType.class)
+        Mapping.mapProp(existingMapping, existingFile, "fileType", newFileType)
+
+        def newTargetFolder = Csv.deserialize(values.get("targetFolder"), String.class)
+        Mapping.mapProp(existingMapping, existingFile, "targetFolder", newTargetFolder)
+
+        def csvStatus = values.get("status")
+        if (status != null && csvStatus == "Active" && status.class.simpleName != "Active") {
+            migration.statusTrackingRepository.active(existingFile.id, ResourceType.File, [reason: "Manual"])
+        }
+        if (status != null && csvStatus == "Deployed" && status.class.simpleName != "Deployed") {
+            migration.statusTrackingRepository.deployed(existingFile.id, deploymentId, now, ResourceType.File, null, output, [reason: "Manual"])
+        }
+
+        boolean newSkip = Csv.deserialize(values.get("skip"), boolean)
+        def newSkipReason = Csv.deserialize(values.get("skipReason"), String.class)
+        def newSkipPlaceholder = Csv.deserialize(values.get("skipPlaceholder"), String.class)
+        def skipObj = new SkipOptions(newSkip, newSkipPlaceholder, newSkipReason)
+        Mapping.mapProp(existingMapping, existingFile, "skip", skipObj)
+
+        migration.mappingRepository.upsert(id, existingMapping)
+        migration.mappingRepository.applyFileMapping(id)
+    }
+}

migration-examples/src/main/groovy/com/quadient/migration/example/common/report/ComplexityReport.groovy

@@ -35,6 +35,7 @@ def header = ["Id",
               "Display rules",
               "Translated display rules",
               "Images",
+              "Files",
               "Hyperlinks",
               "Paragraph styles",
               "Text styles",
@@ -87,6 +88,7 @@ file.withWriter { writer ->
         writer.write("$stats.displayRulesCount,") // Display rules count
         writer.write("$stats.translatedDisplayRulesCount,") // Translated display rules
         writer.write("$stats.imagesCount,") // Images count
+        writer.write("$stats.filesCount,") // Files count
         writer.write("$stats.usedHyperlinksCount,") // Used Hyperlinks Count
         writer.write("$stats.usedParagraphStylesCount,") // Used Paragraph Styles Count
         writer.write("$stats.usedTextStylesCount,") // Used Text Styles Count
@@ -107,6 +109,7 @@ class Stats {
     Set<String> usedDisplayRules = new HashSet()
     Set<String> usedTranslatedDisplayRules = new HashSet()
     Set<String> usedImages = new HashSet()
+    Set<String> usedFiles = new HashSet()
     Set<String> usedHyperlinks = new HashSet()
     Set<String> usedParagraphStyles = new HashSet()
     Set<String> usedTextStyles = new HashSet()
@@ -134,6 +137,7 @@ class Stats {
             switch (content) {
                 case DocumentObjectRef -> this.collectDocumentObjectRef(content)
                 case ImageRef -> this.usedImages.add(content.id)
+                case FileRef -> this.usedFiles.add(content.id)
                 case Table -> this.collectTable(content)
                 case Paragraph -> this.collectParagraph(content)
                 case Area -> this.collectContent(content.content)
@@ -163,6 +167,7 @@ class Stats {
             switch (content) {
                 case DocumentObjectRef -> this.collectDocumentObjectRef(content)
                 case ImageRef -> this.usedImages.add(content.id)
+                case FileRef -> this.usedFiles.add(content.id)
                 case StringValue -> {
                     this.characterCount += content.value.chars.length
                     this.wordCount += content.value.split("\\s+").length
@@ -251,6 +256,10 @@ class Stats {
         return this.usedImages.size()
     }
 
+    Number getFilesCount() {
+        return this.usedFiles.size()
+    }
+
     Number getUsedHyperlinksCount() {
         return this.usedHyperlinks.size()
     }

migration-examples/src/main/groovy/com/quadient/migration/example/common/util/InitMigration.groovy

@@ -48,19 +48,23 @@ static Migration initMigration(Binding binding) {
 
     def imagesPathArg = getValueOfArg("--images-path", argsList).orElse(fileProjectConfig.paths.images?.toString())
     def fontsPathArg = getValueOfArg("--fonts-path", argsList).orElse(fileProjectConfig.paths.fonts?.toString())
+    def documentsPathArg = getValueOfArg("--documents-path", argsList).orElse(fileProjectConfig.paths.documents?.toString())
+    def attachmentsPathArg = getValueOfArg("--attachments-path", argsList).orElse(fileProjectConfig.paths.attachments?.toString())
 
     def styleDefinitionPath = (styleDefinitionPathArg == null || styleDefinitionPathArg.isEmpty()) ? null : IcmPath.from(styleDefinitionPathArg)
     def defFolder = (defaultTargetFolder == null || defaultTargetFolder.isEmpty()) ? null : IcmPath.from(defaultTargetFolder)
     def imagesPath = (imagesPathArg == null || imagesPathArg.isEmpty()) ? null : IcmPath.from(imagesPathArg)
     def fontsPath = (fontsPathArg == null || fontsPathArg.isEmpty()) ? null : IcmPath.from(fontsPathArg)
+    def documentsPath = (documentsPathArg == null || documentsPathArg.isEmpty()) ? null : IcmPath.from(documentsPathArg)
+    def attachmentsPath = (attachmentsPathArg == null || attachmentsPathArg.isEmpty()) ? null : IcmPath.from(attachmentsPathArg)
 
     def projectConfig = new ProjectConfig(projectName,
             baseTemplatePath,
             styleDefinitionPath,
             inputDataPath,
             interactiveTenant,
             defFolder,
-            new PathsConfig(imagesPath, fontsPath),
+            new PathsConfig(imagesPath, fontsPath, documentsPath, attachmentsPath),
             InspireOutput.valueOf(inspireOutput),
             sourceBaseTemplate,
             defaultVariableStructure,

migration-examples/src/main/groovy/com/quadient/migration/example/example/Import.groovy

@@ -11,6 +11,7 @@ import com.quadient.migration.api.dto.migrationmodel.DisplayRuleRef
 import com.quadient.migration.api.dto.migrationmodel.ParagraphStyleRef
 import com.quadient.migration.api.dto.migrationmodel.builder.DisplayRuleBuilder
 import com.quadient.migration.api.dto.migrationmodel.builder.DocumentObjectBuilder
+import com.quadient.migration.api.dto.migrationmodel.builder.FileBuilder
 import com.quadient.migration.api.dto.migrationmodel.builder.ImageBuilder
 import com.quadient.migration.api.dto.migrationmodel.builder.ParagraphBuilder
 import com.quadient.migration.api.dto.migrationmodel.builder.ParagraphStyleBuilder
@@ -19,6 +20,7 @@ import com.quadient.migration.api.dto.migrationmodel.builder.VariableBuilder
 import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder
 import com.quadient.migration.shared.DataType
 import com.quadient.migration.shared.DocumentObjectType
+import com.quadient.migration.shared.FileType
 import com.quadient.migration.shared.GroupOp
 import com.quadient.migration.shared.ImageOptions
 import com.quadient.migration.shared.ImageType
@@ -149,6 +151,14 @@ def logo = new ImageBuilder("logo")
     .alternateText("Example logo image")
     .build()
 
+def logoPdfFile = this.class.getClassLoader().getResource('exampleResources/migrationModelExample/logo.pdf')
+migration.storage.write("logo.pdf", logoPdfFile.bytes)
+def logoDocument = new FileBuilder("logoDocument").fileType(FileType.Document)
+        .sourcePath("logo.pdf")
+        .build()
+
+migration.fileRepository.upsert(logoDocument)
+
 // Table containing some data with the first address row being optionally hidden
 // by using displayRuleRef to the display displayHeaderRule defined above.
 // The table also contains some merged cells and custom column widths.
@@ -310,13 +320,13 @@ def firstMatchBlock = new DocumentObjectBuilder("firstMatch", DocumentObjectType
     .firstMatch { fb ->
         fb.case { cb ->
             cb.name("Czech Variant").appendContent(new ParagraphBuilder().styleRef(paragraphStyle.id).text {
-                it.appendContent("Nashledanou.")
+                it.string("Nashledanou.")
             }.build()).displayRule(displayRuleStateCzechia.id)
         }.case { cb ->
             cb.name("French Variant").appendContent(new ParagraphBuilder().styleRef(paragraphStyle.id).text {
-                it.appendContent("Au revoir.")
+                it.string("Au revoir.")
             }.build()).displayRule(displayRuleStateFrance.id)
-        }.default(new ParagraphBuilder().styleRef(paragraphStyle.id).text { it.appendContent("Goodbye.") }.build())
+        }.default(new ParagraphBuilder().styleRef(paragraphStyle.id).text { it.string("Goodbye.") }.build())
     }.build()
 
 // SelectByLanguage demonstrates language-based content selection.
@@ -326,17 +336,17 @@ def selectByLanguageBlock = new DocumentObjectBuilder("selectByLanguage", Docume
         sb.case { cb ->
             cb.language("en_us")
             cb.appendContent(new ParagraphBuilder().styleRef(paragraphStyle.id).text {
-                it.appendContent("This document was created in English.")
+                it.string("This document was created in English.")
             }.build())
         }.case { cb ->
             cb.language("de")
             cb.appendContent(new ParagraphBuilder().styleRef(paragraphStyle.id).text {
-                it.appendContent("Dieses Dokument wurde auf Deutsch erstellt.")
+                it.string("Dieses Dokument wurde auf Deutsch erstellt.")
             }.build())
         }.case { cb ->
             cb.language("es")
             cb.appendContent(new ParagraphBuilder().styleRef(paragraphStyle.id).text {
-                it.appendContent("Este documento fue creado en español.")
+                it.string("Este documento fue creado en español.")
             }.build())
         }
     }.build()
@@ -395,6 +405,7 @@ def page = new DocumentObjectBuilder("page1", DocumentObjectType.Page)
         }
             .documentObjectRef(signature.id)
     }
+    .fileRef(logoDocument.id)
     .variableStructureRef(variableStructure.id)
     .build()
 

migration-examples/src/main/resources/azureai-project-config-template.toml

@@ -12,6 +12,8 @@ inspireOutput = "Designer" # "Evolve", "Interactive", "Designer"
 [paths]
 #images = ""
 #fonts = ""
+#documents = ""
+#attachments = ""
 
 [context]
 endpoint = "https://<<ENDPOINT>>/documentintelligence/documentModels/prebuilt-layout:analyze?_overload=analyzeDocument&api-version=2024-11-30&features=styleFont,barcodes"

migration-examples/src/main/resources/project-config-template.toml

@@ -12,5 +12,7 @@ inspireOutput = "Designer" # "Evolve", "Interactive", "Designer"
 [paths]
 #images = ""
 #fonts = ""
+#documents = ""
+#attachments = ""
 
 [context]

migration-examples/src/test/groovy/AreasExportTest.groovy

@@ -19,7 +19,7 @@ import static org.mockito.Mockito.when
 
 class AreasExportTest {
     @TempDir
-    File dir
+    java.io.File dir
 
     Migration migration
 

migration-examples/src/test/groovy/AreasImportTest.groovy

@@ -19,7 +19,7 @@ import static org.mockito.Mockito.times
 
 class AreasImportTest {
     @TempDir
-    File dir
+    java.io.File dir
 
     Migration migration