USFM New Experimental Importer

src/exportHandler/exportHandler.ts

@@ -853,6 +853,206 @@ async function exportCodexContentAsObsRoundtrip(
     );
 }
 
+/**
+ * USFM (Unified Standard Format Marker) Round-trip export
+ * Rebuilds original USFM file with translated content
+ */
+async function exportCodexContentAsUsfmRoundtrip(
+    userSelectedPath: string,
+    filesToExport: string[],
+    _options?: ExportOptions
+) {
+    const workspaceFolders = vscode.workspace.workspaceFolders;
+    if (!workspaceFolders) {
+        vscode.window.showErrorMessage("No workspace folder found.");
+        return;
+    }
+
+    const exportFolder = vscode.Uri.file(userSelectedPath);
+
+    return vscode.window.withProgress(
+        {
+            location: vscode.ProgressLocation.Notification,
+            title: "Exporting USFM Round-trip",
+            cancellable: false,
+        },
+        async (progress) => {
+            const increment = filesToExport.length > 0 ? 100 / filesToExport.length : 100;
+
+            // Import USFM exporter from experimental (now standalone implementation)
+            const experimentalExporter = await import("../../webviews/codex-webviews/src/NewSourceUploader/importers/usfm/experimental/usfmExporter");
+            const exportUsfmRoundtrip = experimentalExporter.exportUsfmRoundtrip;
+
+            // For each selected codex file, reconstruct the USFM with translations
+            for (const [index, filePath] of filesToExport.entries()) {
+                progress.report({ message: `Processing ${index + 1}/${filesToExport.length}`, increment });
+                try {
+                    const file = vscode.Uri.file(filePath);
+                    const fileName = basename(file.fsPath);
+                    const bookCode = fileName.split(".")[0] || "";
+
+                    console.log(`[USFM Export] Processing ${fileName} using USFM round-trip exporter`);
+
+                    // Read codex notebook
+                    const codexNotebook = await readCodexNotebookFromUri(file);
+
+                    // Check if this is a USFM file (experimental or standalone)
+                    const importerType = (codexNotebook.metadata as any)?.importerType;
+                    const corpusMarker = (codexNotebook.metadata as any)?.corpusMarker;
+
+                    if (importerType !== 'usfm-experimental' && corpusMarker !== 'usfm') {
+                        console.warn(`[USFM Export] Skipping ${fileName} - not imported with USFM importer (importerType: ${importerType}, corpusMarker: ${corpusMarker})`);
+                        vscode.window.showWarningMessage(`Skipping ${fileName} - not imported with USFM importer`);
+                        continue;
+                    }
+
+                    // Get original file name from metadata with fallback
+                    // Try multiple sources: codex metadata, source notebook metadata, or construct from bookCode
+                    let metadataOriginalFileName = (codexNotebook.metadata as any)?.originalFileName;
+                    const metadataBookCode = (codexNotebook.metadata as any)?.bookCode;
+                    const finalBookCode = metadataBookCode || bookCode;
+
+                    // If not found in codex metadata, try source notebook metadata
+                    if (!metadataOriginalFileName) {
+                        try {
+                            const sourceFileName = fileName.replace('.codex', '.source');
+                            const sourceFileUri = vscode.Uri.joinPath(
+                                workspaceFolders[0].uri,
+                                ".project",
+                                "sourceTexts",
+                                sourceFileName
+                            );
+                            const sourceNotebook = await readCodexNotebookFromUri(sourceFileUri);
+                            metadataOriginalFileName = (sourceNotebook.metadata as any)?.originalFileName;
+                            if (metadataOriginalFileName) {
+                                console.log(`[USFM Export] Found originalFileName in source notebook: ${metadataOriginalFileName}`);
+                            }
+                        } catch (error) {
+                            // Source notebook not found or error reading it, continue with fallbacks
+                            console.log(`[USFM Export] Could not read source notebook for originalFileName`);
+                        }
+                    }
+
+                    // Try common USFM file extensions
+                    const possibleExtensions = ['.usfm', '.sfm', '.USFM', '.SFM'];
+                    let originalFileName = metadataOriginalFileName;
+
+                    // If no originalFileName, try to find it in originals folder
+                    if (!originalFileName && finalBookCode) {
+                        const originalsDir = vscode.Uri.joinPath(
+                            workspaceFolders[0].uri,
+                            ".project",
+                            "attachments",
+                            "originals"
+                        );
+
+                        // Try each extension
+                        for (const ext of possibleExtensions) {
+                            const testFileName = `${finalBookCode}${ext}`;
+                            const testUri = vscode.Uri.joinPath(originalsDir, testFileName);
+                            try {
+                                await vscode.workspace.fs.stat(testUri);
+                                originalFileName = testFileName;
+                                console.log(`[USFM Export] Found original file: ${testFileName}`);
+                                break;
+                            } catch {
+                                // File doesn't exist, try next extension
+                            }
+                        }
+                    }
+
+                    // Final fallback: construct filename from bookCode
+                    if (!originalFileName) {
+                        originalFileName = `${finalBookCode}.usfm`;
+                        console.log(`[USFM Export] Using fallback filename: ${originalFileName}`);
+                    }
+
+                    // Load original USFM file from attachments/originals
+                    const originalsDir = vscode.Uri.joinPath(
+                        workspaceFolders[0].uri,
+                        ".project",
+                        "attachments",
+                        "originals"
+                    );
+                    const originalFileUri = vscode.Uri.joinPath(originalsDir, originalFileName);
+
+                    let originalUsfmContent: string;
+                    try {
+                        const originalFileData = await vscode.workspace.fs.readFile(originalFileUri);
+                        originalUsfmContent = new TextDecoder('utf-8').decode(originalFileData);
+                        console.log(`[USFM Export] Loaded original USFM file: ${originalFileName}`);
+                    } catch (error) {
+                        // Fallback: try to get from structureMetadata if available
+                        const structureMetadata = (codexNotebook.metadata as any)?.structureMetadata;
+                        if (structureMetadata?.originalUsfmContent) {
+                            originalUsfmContent = structureMetadata.originalUsfmContent;
+                            console.log(`[USFM Export] Using original USFM content from metadata (file not found at ${originalFileUri.fsPath})`);
+                        } else {
+                            throw new Error(`Original USFM file not found at ${originalFileUri.fsPath} and no original content in metadata`);
+                        }
+                    }
+
+                    // Build codex cells array
+                    // Include id property if it exists (some cells have id at top level, others in metadata.id)
+                    const codexCells = codexNotebook.cells.map(cell => {
+                        const cellData: any = {
+                            kind: cell.kind,
+                            value: cell.value,
+                            metadata: cell.metadata,
+                        };
+                        // Include id if it exists at top level (for ProcessedCell structure)
+                        if ((cell as any).id) {
+                            cellData.id = (cell as any).id;
+                        }
+                        return cellData;
+                    });
+
+                    // Get lineMappings from structureMetadata if available (for standalone exporter)
+                    const structureMetadata = (codexNotebook.metadata as any)?.structureMetadata;
+                    const lineMappings = structureMetadata?.lineMappings;
+
+                    // Debug: Log structureMetadata and lineMappings
+                    if (lineMappings) {
+                        console.log(`[USFM Export] Found lineMappings: ${lineMappings.length} entries`);
+                        const sampleMapping = lineMappings.find((m: any) => m.cellId);
+                        console.log(`[USFM Export] Sample mapping with cellId:`, sampleMapping);
+                        console.log(`[USFM Export] Mappings with cellId count:`, lineMappings.filter((m: any) => m.cellId && m.cellId !== '').length);
+                    } else {
+                        console.warn(`[USFM Export] No lineMappings found in structureMetadata`);
+                        console.log(`[USFM Export] structureMetadata keys:`, structureMetadata ? Object.keys(structureMetadata) : 'null');
+                    }
+
+                    // Export USFM with translations
+                    // If we have lineMappings, use them for precise round-trip export
+                    let updatedUsfmContent: string;
+                    if (lineMappings) {
+                        updatedUsfmContent = await exportUsfmRoundtrip(originalUsfmContent, lineMappings, codexCells);
+                    } else {
+                        // Use backward-compatible signature (no lineMappings - fallback mode)
+                        updatedUsfmContent = await exportUsfmRoundtrip(originalUsfmContent, codexCells);
+                    }
+
+                    // Save to export folder
+                    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+                    const exportedName = originalFileName.replace(/\.(usfm|sfm|USFM|SFM)$/i, `_${timestamp}_translated.$1`);
+                    const exportedUri = vscode.Uri.joinPath(exportFolder, exportedName);
+
+                    const encoder = new TextEncoder();
+                    await vscode.workspace.fs.writeFile(exportedUri, encoder.encode(updatedUsfmContent));
+
+                    console.log(`[USFM Export] ✓ Exported ${exportedName}`);
+
+                } catch (error) {
+                    console.error(`[USFM Export] Error exporting ${filePath}:`, error);
+                    vscode.window.showErrorMessage(`Failed to export ${basename(filePath)}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+                }
+            }
+
+            vscode.window.showInformationMessage(`USFM round-trip export completed to ${userSelectedPath}`);
+        }
+    );
+}
+
 /**
  * TMS (Translation Memory System) Round-trip export
  * Supports both TMX and XLIFF formats
@@ -1074,6 +1274,19 @@ async function exportCodexContentAsRebuild(
                         // TMS (Translation Memory System) files use the TMS exporter
                         filesByType['tms'] = filesByType['tms'] || [];
                         filesByType['tms'].push(filePath);
+                    } else if (
+                        corpusMarker === 'usfm' ||
+                        importerType === 'usfm-experimental' ||
+                        importerType === 'usfm' ||
+                        // Also check for NT/OT corpus markers with USFM file extensions (Bible books imported as USFM)
+                        ((corpusMarker === 'NT' || corpusMarker === 'OT') &&
+                            originalFileName &&
+                            (originalFileName.endsWith('.usfm') || originalFileName.endsWith('.sfm') || originalFileName.endsWith('.USFM') || originalFileName.endsWith('.SFM'))) ||
+                        (originalFileName && (originalFileName.endsWith('.usfm') || originalFileName.endsWith('.sfm') || originalFileName.endsWith('.USFM') || originalFileName.endsWith('.SFM')))
+                    ) {
+                        // USFM files use the USFM round-trip exporter
+                        filesByType['usfm'] = filesByType['usfm'] || [];
+                        filesByType['usfm'].push(filePath);
                     } else {
                         unsupportedFiles.push({ file: basename(filePath), marker: corpusMarker || importerType || 'unknown' });
                     }
@@ -1193,6 +1406,22 @@ async function exportCodexContentAsRebuild(
                 }
             }
 
+            // Export USFM files
+            if (filesByType['usfm']?.length > 0) {
+                console.log(`[Rebuild Export] Exporting ${filesByType['usfm'].length} USFM file(s)...`);
+                progress.report({
+                    message: `Exporting ${filesByType['usfm'].length} USFM file(s)...`,
+                    increment: 20
+                });
+                try {
+                    await exportCodexContentAsUsfmRoundtrip(userSelectedPath, filesByType['usfm'], options);
+                    processedCount += filesByType['usfm'].length;
+                } catch (error) {
+                    console.error('[Rebuild Export] USFM export failed:', error);
+                    vscode.window.showErrorMessage(`USFM export failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+                }
+            }
+
             progress.report({ message: "Complete", increment: 30 });
 
             // Show summary

src/projectManager/projectExportView.ts

@@ -334,13 +334,14 @@ function getWebviewContent(
                                 <i class="codicon codicon-refresh"></i>
                                 <div>
                                     <strong>Rebuild Export</strong>
-                                    <p>Intelligently detects file type and exports back to original format (DOCX, IDML, Biblica, OBS, TMS)</p>
+                                    <p>Intelligently detects file type and exports back to original format (DOCX, IDML, Biblica, OBS, TMS, USFM)</p>
                                     <div style="display: flex; gap: 0.5rem; margin-top: 0.25rem; flex-wrap: wrap;">
                                         <span class="format-tag" style="background-color: var(--vscode-charts-blue);">DOCX</span>
                                         <span class="format-tag" style="background-color: var(--vscode-charts-blue);">IDML</span>
                                         <span class="format-tag" style="background-color: var(--vscode-charts-blue);">Biblica</span>
                                         <span class="format-tag" style="background-color: var(--vscode-charts-blue);">OBS</span>
                                         <span class="format-tag" style="background-color: var(--vscode-charts-blue);">TMS</span>
+                                        <span class="format-tag" style="background-color: var(--vscode-charts-blue);">USFM</span>
                                     </div>
                                 </div>
                             </div>

src/providers/NewSourceUploader/NewSourceUploaderProvider.ts

@@ -5,7 +5,7 @@ import { promisify } from "util";
 import { exec } from "child_process";
 import { getWebviewHtml } from "../../utils/webviewTemplate";
 import { createNoteBookPair } from "./codexFIleCreateUtils";
-import { WriteNotebooksMessage, WriteTranslationMessage, OverwriteResponseMessage, WriteNotebooksWithAttachmentsMessage, SelectAudioFileMessage, ReprocessAudioFileMessage, RequestAudioSegmentMessage, FinalizeAudioImportMessage, UpdateAudioSegmentsMessage } from "../../../webviews/codex-webviews/src/NewSourceUploader/types/plugin";
+import { WriteNotebooksMessage, WriteTranslationMessage, OverwriteResponseMessage, WriteNotebooksWithAttachmentsMessage, SelectAudioFileMessage, ReprocessAudioFileMessage, RequestAudioSegmentMessage, FinalizeAudioImportMessage, UpdateAudioSegmentsMessage, SaveFileMessage } from "../../../webviews/codex-webviews/src/NewSourceUploader/types/plugin";
 import {
     handleSelectAudioFile,
     handleReprocessAudioFile,
@@ -317,8 +317,12 @@ export class NewSourceUploaderProvider implements vscode.CustomTextEditorProvide
                         message as FinalizeAudioImportMessage,
                         token,
                         webviewPanel,
-                        (msg, tok, pan) => this.handleWriteNotebooks(msg as WriteNotebooksMessage, tok, pan)
+                        async (msg, tok, pan) => {
+                            await this.handleWriteNotebooks(msg as WriteNotebooksMessage, tok, pan);
+                        }
                     );
+                } else if (message.command === "saveFile") {
+                    await this.handleSaveFile(message as SaveFileMessage, webviewPanel);
                 }
             } catch (error) {
                 console.error("Error handling message:", error);
@@ -1328,6 +1332,76 @@ export class NewSourceUploaderProvider implements vscode.CustomTextEditorProvide
         }
     }
 
+    /**
+     * Handle saveFile command from webview - saves a file using VS Code's save dialog
+     */
+    private async handleSaveFile(message: SaveFileMessage, webviewPanel: vscode.WebviewPanel): Promise<void> {
+        try {
+            const { fileName, dataBase64, mime } = message;
+
+            // Extract base64 data (handle data: URL format)
+            let base64Data = dataBase64;
+            if (base64Data.includes(',')) {
+                // Remove data: URL prefix if present
+                base64Data = base64Data.split(',')[1];
+            }
+
+            // Convert base64 to Buffer
+            const buffer = Buffer.from(base64Data, 'base64');
+
+            if (buffer.length === 0) {
+                throw new Error('File data is empty');
+            }
+
+            // Show save dialog
+            const workspaceFolder = vscode.workspace.workspaceFolders?.[0];
+            const defaultUri = workspaceFolder
+                ? vscode.Uri.joinPath(workspaceFolder.uri, fileName)
+                : undefined;
+
+            const saveUri = await vscode.window.showSaveDialog({
+                defaultUri,
+                saveLabel: 'Save',
+                filters: mime
+                    ? {
+                        'All Files': ['*'],
+                        [mime]: [fileName.split('.').pop() || '*']
+                    }
+                    : undefined
+            });
+
+            if (!saveUri) {
+                // User cancelled
+                webviewPanel.webview.postMessage({
+                    command: "notification",
+                    type: "info",
+                    message: "File save cancelled"
+                });
+                return;
+            }
+
+            // Write file
+            await vscode.workspace.fs.writeFile(saveUri, buffer);
+
+            // Send success notification
+            webviewPanel.webview.postMessage({
+                command: "notification",
+                type: "success",
+                message: `File saved successfully: ${path.basename(saveUri.fsPath)}`
+            });
+
+            console.log(`[NEW SOURCE UPLOADER] File saved: ${saveUri.fsPath} (${buffer.length} bytes)`);
+
+        } catch (error) {
+            console.error("[NEW SOURCE UPLOADER] Error saving file:", error);
+            webviewPanel.webview.postMessage({
+                command: "notification",
+                type: "error",
+                message: error instanceof Error ? error.message : "Failed to save file"
+            });
+        }
+    }
+
     private checkIfBibleContent(document: vscode.NotebookDocument): boolean {
         // Check first few cells to see if they contain Bible verse references
         const cellsToCheck = Math.min(5, document.cellCount);