From a89b81acf887996ebcd14212a5df1eac0888fbdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 26 Oct 2025 17:22:07 +0100 Subject: [PATCH 01/57] iterating on insurance domain and puting generic import in collectionsplugin --- Directory.Packages.props | 1 + .../InsuranceAIExtensions.cs | 2 + .../MeshWeaver.Insurance.AI/InsuranceAgent.cs | 41 +- .../MeshWeaver.Insurance.AI.csproj | 6 + .../RiskImportAgent.cs | 356 +++++++++++++ .../SlipImportAgent.cs | 470 ++++++++++++++++++ .../InsuranceApplicationExtensions.cs | 10 +- .../MeshWeaver.Insurance.Domain/Structure.cs | 110 ++++ src/MeshWeaver.AI/MeshWeaver.AI.csproj | 1 + src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 72 +++ .../Configuration/ImportConfiguration.cs | 44 +- .../Implementation/ImportManager.cs | 10 +- .../ImportRegistryExtensions.cs | 8 + src/MeshWeaver.Import/ImportRequest.cs | 4 +- .../MeshWeaver.Import.csproj | 1 + .../MessageHubExtensions.cs | 51 ++ .../CollectionPluginImportTest.cs | 189 +++++++ .../CollectionSourceImportTest.cs | 148 ++++++ .../MeshWeaver.Import.Test.csproj | 1 + 19 files changed, 1502 insertions(+), 23 deletions(-) create mode 100644 modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs create mode 100644 test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs create mode 100644 test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f0b889ee9..c4e7ecd92 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,6 +36,7 @@ + diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAIExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAIExtensions.cs index 7a562ce36..9a42b4f16 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAIExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAIExtensions.cs @@ -14,6 +14,8 @@ public static class InsuranceAIExtensions public static IServiceCollection AddInsuranceAI(this IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs index e6b406b3f..c9f2587fd 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs @@ -12,8 +12,8 @@ namespace MeshWeaver.Insurance.AI; /// /// Main Insurance agent that provides access to insurance pricing data and collections. /// -[DefaultAgent] -public class InsuranceAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPlugins, IAgentWithContext +[ExposedInNavigator] +public class InsuranceAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPlugins, IAgentWithContext, IAgentWithDelegations { private Dictionary? typeDefinitionMap; private Dictionary? layoutAreaMap; @@ -23,12 +23,42 @@ public class InsuranceAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPl public string Description => "Handles all questions and actions related to insurance pricings, property risks, and dimensions. " + "Provides access to pricing data, allows creation and management of pricings and property risks. " + - "Also manages submission documents and files for each pricing."; + "Also manages submission documents and files for each pricing. " + + "Can delegate to specialized import agents for processing risk data files and slip documents."; + + public IEnumerable Delegations + { + get + { + yield return new DelegationDescription( + nameof(RiskImportAgent), + "Delegate to RiskImportAgent when the user wants to import property risks from Excel files, " + + "or when working with risk data files (.xlsx, .xls) that contain property information like " + + "location, TSI (Total Sum Insured), address, country, currency, building values, etc. " + + "Common file names include: risks.xlsx, exposure.xlsx, property schedule, location schedule, etc." + ); + + yield return new DelegationDescription( + nameof(SlipImportAgent), + "Delegate to SlipImportAgent when the user wants to import insurance slips from PDF documents, " + + "or when working with slip files (.pdf) that contain insurance submission information like " + + "insured details, coverage terms, premium information, reinsurance structure layers, limits, rates, etc. " + + "Common file names include: slip.pdf, submission.pdf, placement.pdf, quote.pdf, etc." + ); + } + } public string Instructions => $$$""" The agent is the InsuranceAgent, specialized in managing insurance pricings: + ## Content Collection Context + + IMPORTANT: The current context is set to pricing/{pricingId} where pricingId follows the format {company}-{uwy}. + - The submission files collection is named "Submissions-{pricingId}" + - All file paths are relative to the root (/) of this collection + - Example: For pricing "AXA-2024", the collection is "Submissions-AXA-2024" and files are at paths like "/slip.pdf", "/risks.xlsx" + ## Working with Submission Documents and Files CRITICAL: When users ask about submission files, documents, or content: @@ -36,12 +66,13 @@ public class InsuranceAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPl - DO NOT try to verify the pricing exists before accessing files - The SubmissionPlugin is already configured for the current pricing context - Simply call the SubmissionPlugin functions directly + - All file paths should start with "/" (e.g., "/slip.pdf", "/risks.xlsx") Available SubmissionPlugin functions (all collectionName parameters are optional): - {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - List all files in the current pricing's submissions - {{{nameof(ContentCollectionPlugin.ListFolders)}}}() - List all folders - {{{nameof(ContentCollectionPlugin.ListCollectionItems)}}}() - List both files and folders - - {{{nameof(ContentCollectionPlugin.GetDocument)}}}(documentPath) - Get document content + - {{{nameof(ContentCollectionPlugin.GetDocument)}}}(documentPath) - Get document content (use path like "/Slip.md") - {{{nameof(ContentCollectionPlugin.SaveDocument)}}}(documentPath, content) - Save a document - {{{nameof(ContentCollectionPlugin.DeleteFile)}}}(filePath) - Delete a file - {{{nameof(ContentCollectionPlugin.CreateFolder)}}}(folderPath) - Create a folder @@ -50,7 +81,7 @@ Available SubmissionPlugin functions (all collectionName parameters are optional Examples: - User: "Show me the submission files" → You: Call {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - User: "What files are in the submissions?" → You: Call {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - - User: "Read the slip document" → You: Call {{{nameof(ContentCollectionPlugin.GetDocument)}}}("Slip.md") + - User: "Read the slip document" → You: Call {{{nameof(ContentCollectionPlugin.GetDocument)}}}("/Slip.md") ## Working with Pricing Data diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/MeshWeaver.Insurance.AI.csproj b/modules/Insurance/MeshWeaver.Insurance.AI/MeshWeaver.Insurance.AI.csproj index f5e01a264..292eb0d9d 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/MeshWeaver.Insurance.AI.csproj +++ b/modules/Insurance/MeshWeaver.Insurance.AI/MeshWeaver.Insurance.AI.csproj @@ -14,4 +14,10 @@ + + + + + + diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs new file mode 100644 index 000000000..00e1fc21d --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -0,0 +1,356 @@ +using System.ComponentModel; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ClosedXML.Excel; +using MeshWeaver.AI; +using MeshWeaver.AI.Plugins; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; +using MeshWeaver.Insurance.Domain; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +namespace MeshWeaver.Insurance.AI; + +public class RiskImportAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPlugins, IAgentWithContext +{ + private Dictionary? typeDefinitionMap; + private string? propertyRiskSchema; + private string? excelImportConfigSchema; + + public string Name => nameof(RiskImportAgent); + + public string Description => "Runs risk imports for a pricing. Creates mappings and imports property risk data from Excel files."; + + public string Instructions + { + get + { + var baseText = + $$$""" + You control risk imports for a specific pricing. Use the provided tool: + + ## Content Collection Context + + IMPORTANT: The current context is set to pricing/{pricingId} where pricingId follows the format {company}-{uwy}. + - The submission files collection is named "Submissions-{pricingId}" + - All file paths are relative to the root (/) of this collection + - When listing files, you'll see paths like "/risks.xlsx", "/exposure.xlsx" + - When accessing files, use paths starting with "/" (e.g., "/risks.xlsx") + + # Importing Risks + When the user asks you to import risks, you should: + 1) Get the existing risk mapping configuration for the specified file using the function {{{nameof(RiskImportPlugin.GetRiskImportConfiguration)}}} with the filename. + 2) If no import configuration was returned in 1, get a sample of the worksheet using {{{nameof(RiskImportPlugin.GetWorksheetSample)}}} with the filename and extract the table start row as well as the mapping as in the schema provided below. + Consider any input from the user to modify the configuration. Use the {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} function to update. + 3) call Import with the filename. + + # Updating Risk Import Configuration + When the user asks you to update the risk import configuration, you should: + 1) Get the existing risk mapping configuration for the specified file using the function {{{nameof(RiskImportPlugin.GetRiskImportConfiguration)}}} with the filename. + 2) Modify it according to the user's input, ensuring it follows the schema provided below. + 3) Upload the new configuration using the function {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} with the filename and the updated mapping. + + # Automatic Risk Import Configuration + - Read the column header from the row which you determine to be the first of the Data Table and map to column numbers. + - Map to the properties of the PropertyRisk type (see schema below). Only these names are allowed for mappings. Read the descriptions contained in the schema to get guidance on which field to map where + - Columns you cannot map ==> ignore. + - Watch out for empty columns at the beginning of the table. In this case, see that you get the column index right. + + Notes: + - The agent defaults to ignoring rows where Id or Address is missing (adds "Id == null" and "Address == null" to ignoreRowExpressions). + - Provide only the file name (e.g., "risks.xlsx"); it is resolved relative to the pricing's content collection. + + IMPORTANT OUTPUT RULES: + - do not output JSON to the user. + - When the user asks you to import, your job is not finished by creating the risk import configuration. You will actually have to call import. + """; + + if (excelImportConfigSchema is not null) + baseText += $"\n\n# Schema for ExcelImportConfiguration\n{excelImportConfigSchema}"; + if (propertyRiskSchema is not null) + baseText += $"\n\n# Schema for PropertyRisk (Target for Mapping)\n{propertyRiskSchema}"; + + return baseText; + } + } + + public bool Matches(AgentContext? context) + { + return context?.Address?.Type == PricingAddress.TypeName; + } + + IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) + { + yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); + + // Add ContentCollectionPlugin for submissions + var submissionPluginConfig = CreateSubmissionPluginConfig(chat); + yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); + + // Add CollectionPlugin for import functionality + var collectionPlugin = new CollectionPlugin(hub); + yield return KernelPluginFactory.CreateFromObject(collectionPlugin); + + // Add risk import specific plugin + var plugin = new RiskImportPlugin(hub, chat); + yield return KernelPluginFactory.CreateFromObject(plugin); + } + + private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + { + return new ContentCollectionPluginConfig + { + Collections = [], + ContextToConfigMap = context => + { + // Only handle pricing contexts + if (context?.Address?.Type != PricingAddress.TypeName) + return null!; + + var pricingId = context.Address.Id; + + // Parse pricingId in format {company}-{uwy} + var parts = pricingId.Split('-'); + if (parts.Length != 2) + return null!; + + var company = parts[0]; + var uwy = parts[1]; + var subPath = $"{company}/{uwy}"; + + // Create Hub-based collection config pointing to the pricing address + return new ContentCollectionConfig + { + SourceType = HubStreamProviderFactory.SourceType, + Name = $"Submissions-{pricingId}", + Address = context.Address, + BasePath = subPath + }; + } + }; + } + + async Task IInitializableAgent.InitializeAsync() + { + try + { + var typesResponse = await hub.AwaitResponse( + new GetDomainTypesRequest(), + o => o.WithTarget(new PricingAddress("default"))); + typeDefinitionMap = typesResponse.Message.Types.Select(t => t with { Address = null }).ToDictionary(x => x.Name); + } + catch + { + typeDefinitionMap = null; + } + + try + { + var resp = await hub.AwaitResponse( + new GetSchemaRequest("ExcelImportConfiguration"), + o => o.WithTarget(new PricingAddress("default"))); + excelImportConfigSchema = resp.Message.Schema; + } + catch + { + excelImportConfigSchema = null; + } + + try + { + var resp = await hub.AwaitResponse( + new GetSchemaRequest(nameof(PropertyRisk)), + o => o.WithTarget(new PricingAddress("default"))); + propertyRiskSchema = resp.Message.Schema; + } + catch + { + propertyRiskSchema = null; + } + } +} + +public class RiskImportPlugin(IMessageHub hub, IAgentChat chat) +{ + private JsonSerializerOptions GetJsonOptions() + { + return hub.JsonSerializerOptions; + } + + //[KernelFunction] + //[Description("Imports a file with filename")] + //public async Task Import(string filename) + //{ + // if (chat.Context?.Address?.Type != PricingAddress.TypeName) + // return "Please navigate to the pricing for which you want to import risks."; + + // var pricingId = chat.Context.Address.Id; + // var collectionName = $"Submissions-{pricingId}"; + // var address = $"{PricingAddress.TypeName}/{pricingId}"; + + // // Delegate to CollectionPlugin's Import method + // var collectionPlugin = new CollectionPlugin(hub); + // return await collectionPlugin.Import( + // path: filename, + // collection: collectionName, + // address: address, + // format: "PropertyRiskImport" + // ); + //} + + [KernelFunction] + [Description("Gets the risk configuration for a particular file")] + public async Task GetRiskImportConfiguration(string filename) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing for which you want to create a risk import mapping."; + + try + { + var response = await hub.AwaitResponse( + new GetDataRequest(new EntityReference("ExcelImportConfiguration", filename)), + o => o.WithTarget(new PricingAddress(chat.Context.Address.Id)) + ); + return JsonSerializer.Serialize(response.Message.Data, hub.JsonSerializerOptions); + } + catch (Exception e) + { + return $"Error processing file '{filename}': {e.Message}"; + } + } + + [KernelFunction] + [Description("Updates the mapping configuration for risks import")] + public async Task UpdateRiskImportConfiguration( + string filename, + [Description("Needs to follow the schema provided in the system prompt")] string mappingJson) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing for which you want to update the risk import configuration."; + + var pa = new PricingAddress(chat.Context.Address.Id); + if (string.IsNullOrWhiteSpace(mappingJson)) + return "Json mapping is empty. Please provide valid JSON."; + + try + { + var parsed = EnsureTypeFirst((JsonObject)JsonNode.Parse(ExtractJson(mappingJson))!, "ExcelImportConfiguration"); + parsed["entityId"] = pa.Id; + parsed["name"] = filename; + var response = await hub.AwaitResponse(new DataChangeRequest() { Updates = [parsed] }, o => o.WithTarget(pa)); + return JsonSerializer.Serialize(response.Message, hub.JsonSerializerOptions); + } + catch (Exception e) + { + return $"Mapping JSON is invalid. Please provide valid JSON. Exception: {e.Message}"; + } + } + + private static JsonObject EnsureTypeFirst(JsonObject source, string typeName) + { + var ordered = new JsonObject + { + ["$type"] = typeName + }; + foreach (var kv in source) + { + if (string.Equals(kv.Key, "$type", StringComparison.Ordinal)) continue; + ordered[kv.Key] = kv.Value?.DeepClone(); + } + return ordered; + } + + [KernelFunction] + [Description("Gets the first 20 rows for each worksheet in the workbook to help determine the mapping")] + public async Task GetWorksheetSample(string filename) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing first."; + + try + { + var pricingId = chat.Context.Address.Id; + var contentService = hub.ServiceProvider.GetRequiredService(); + var stream = await OpenContentReadStreamAsync(contentService, pricingId, filename); + + if (stream is null) + return $"Content not found: {filename}"; + + await using (stream) + { + using var wb = new XLWorkbook(stream); + var sb = new StringBuilder(); + + foreach (var ws in wb.Worksheets) + { + var used = ws.RangeUsed(); + sb.AppendLine($"Sheet: {ws.Name}"); + if (used is null) + { + sb.AppendLine("(No data)"); + sb.AppendLine(); + continue; + } + + var firstRow = used.FirstRow().RowNumber(); + var lastRow = Math.Min(used.FirstRow().RowNumber() + 19, used.LastRow().RowNumber()); + var firstCol = 1; + var lastCol = used.LastColumn().ColumnNumber(); + + for (var r = firstRow; r <= lastRow; r++) + { + var rowVals = new List(); + for (var c = firstCol; c <= lastCol; c++) + { + var raw = ws.Cell(r, c).GetValue(); + var val = raw?.Replace('\n', ' ').Replace('\r', ' ').Trim(); + rowVals.Add(string.IsNullOrEmpty(val) ? "null" : val); + } + + sb.AppendLine(string.Join('\t', rowVals)); + } + + sb.AppendLine(); + } + + return sb.ToString(); + } + } + catch (Exception e) + { + return $"Error reading sample: {e.Message}"; + } + } + + + private static async Task OpenContentReadStreamAsync( + IContentService contentService, + string pricingId, + string filename) + { + try + { + var collectionName = $"Submissions-{pricingId}"; + + var collection = await contentService.GetCollectionAsync(collectionName, CancellationToken.None); + if (collection is null) + return null; + + return await collection.GetContentAsync(filename); + } + catch + { + return null; + } + } + + private string ExtractJson(string json) + { + return json.Replace("```json", "") + .Replace("```", "") + .Trim(); + } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs new file mode 100644 index 000000000..93db5db39 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -0,0 +1,470 @@ +using iText.Kernel.Pdf; +using iText.Kernel.Pdf.Canvas.Parser; +using iText.Kernel.Pdf.Canvas.Parser.Listener; +using MeshWeaver.AI; +using MeshWeaver.AI.Plugins; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; +using MeshWeaver.Insurance.Domain; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using System.ComponentModel; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace MeshWeaver.Insurance.AI; + +public class SlipImportAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPlugins, IAgentWithContext +{ + private Dictionary? typeDefinitionMap; + private string? pricingSchema; + private string? structureSchema; + + public string Name => nameof(SlipImportAgent); + + public string Description => "Imports insurance slip documents from PDF files and structures them into Pricing and Structure data models using LLM-based extraction."; + + public string Instructions + { + get + { + var baseText = + $$$""" + You are a slip import agent that processes PDF documents containing insurance submission slips. + Your task is to extract structured data and map it to the insurance domain models using the provided schemas. + + ## Content Collection Context + + IMPORTANT: The current context is set to pricing/{pricingId} where pricingId follows the format {company}-{uwy}. + - The submission files collection is named "Submissions-{pricingId}" + - All file paths are relative to the root (/) of this collection + - When listing files, you'll see paths like "/slip.pdf", "/submission.pdf" + - When accessing files, use paths starting with "/" (e.g., "/slip.pdf") + + # Importing Slips + When the user asks you to import a slip: + 1) First, use {{{nameof(ContentCollectionPlugin.ListFiles)}}}() to see available files in the submissions collection + 2) Use {{{nameof(SlipImportPlugin.ExtractCompleteText)}}} to extract the PDF content (e.g., "slip.pdf" without the leading /) + 3) Review the extracted text and identify data that matches the domain schemas + 4) Use {{{nameof(SlipImportPlugin.ImportSlipData)}}} to save the structured data as JSON + 5) Provide feedback on what data was successfully imported or if any issues were encountered + + # Data Mapping Guidelines + Based on the extracted PDF text, create JSON objects that match the schemas provided below: + - **Pricing**: Basic pricing information (insured name, broker, dates, premium, country, legal entity) + - **Structure**: Reinsurance layer structure and financial terms (cession, limits, rates, commissions) + + # Important Rules + - Only extract data that is explicitly present in the PDF text + - Use null or default values for missing data points + - Ensure all monetary values are properly formatted as numbers + - Convert percentages to decimal format (e.g., 25% → 0.25) + - Provide clear feedback on what data was successfully extracted + - If data is ambiguous or unclear, note it in your response + - For Structure records, generate appropriate LayerId values (e.g., "Layer1", "Layer2") + - Multiple layers can be imported if the slip contains multiple layer structures + + # PDF Section Processing + Look for common sections in insurance slips: + - Insured information (name, location, industry) + - Coverage details (inception/expiration dates, policy terms) + - Premium and financial information + - Layer structures (limits, attachments, rates) + - Reinsurance terms (commission, brokerage, taxes) + + Notes: + - When listing files, paths will include "/" prefix (e.g., "/slip.pdf") + - When calling import functions, provide only the filename without "/" (e.g., "slip.pdf") + """; + + if (pricingSchema is not null) + baseText += $"\n\n# Pricing Schema\n```json\n{pricingSchema}\n```"; + if (structureSchema is not null) + baseText += $"\n\n# Structure Schema\n```json\n{structureSchema}\n```"; + + return baseText; + } + } + + IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) + { + yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); + + // Add ContentCollectionPlugin for submissions + var submissionPluginConfig = CreateSubmissionPluginConfig(chat); + yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); + + // Add slip import specific plugin + var plugin = new SlipImportPlugin(hub, chat); + yield return KernelPluginFactory.CreateFromObject(plugin); + } + + private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + { + return new ContentCollectionPluginConfig + { + Collections = [], + ContextToConfigMap = context => + { + // Only handle pricing contexts + if (context?.Address?.Type != PricingAddress.TypeName) + return null!; + + var pricingId = context.Address.Id; + + // Parse pricingId in format {company}-{uwy} + var parts = pricingId.Split('-'); + if (parts.Length != 2) + return null!; + + var company = parts[0]; + var uwy = parts[1]; + var subPath = $"{company}/{uwy}"; + + // Create Hub-based collection config pointing to the pricing address + return new ContentCollectionConfig + { + SourceType = HubStreamProviderFactory.SourceType, + Name = $"Submissions-{pricingId}", + Address = context.Address, + BasePath = subPath + }; + } + }; + } + + async Task IInitializableAgent.InitializeAsync() + { + var pricingAddress = new PricingAddress("default"); + + try + { + var typesResponse = await hub.AwaitResponse( + new GetDomainTypesRequest(), + o => o.WithTarget(pricingAddress)); + typeDefinitionMap = typesResponse.Message.Types.Select(t => t with { Address = null }).ToDictionary(x => x.Name); + } + catch + { + typeDefinitionMap = null; + } + + try + { + var resp = await hub.AwaitResponse( + new GetSchemaRequest(nameof(Pricing)), + o => o.WithTarget(pricingAddress)); + pricingSchema = resp.Message.Schema; + } + catch + { + pricingSchema = null; + } + + try + { + var resp = await hub.AwaitResponse( + new GetSchemaRequest(nameof(Structure)), + o => o.WithTarget(pricingAddress)); + structureSchema = resp.Message.Schema; + } + catch + { + structureSchema = null; + } + } + + public bool Matches(AgentContext? context) + { + return context?.Address?.Type == PricingAddress.TypeName; + } +} + +public class SlipImportPlugin(IMessageHub hub, IAgentChat chat) +{ + private JsonSerializerOptions GetJsonOptions() + { + return hub.JsonSerializerOptions; + } + + [KernelFunction] + [Description("Extracts the complete text from a PDF slip document and returns it for LLM processing")] + public async Task ExtractCompleteText(string filename) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing first."; + + try + { + var pricingId = chat.Context.Address.Id; + var contentService = hub.ServiceProvider.GetRequiredService(); + var stream = await OpenContentReadStreamAsync(contentService, pricingId, filename); + + if (stream is null) + return $"Content not found: {filename}"; + + await using (stream) + { + var completeText = await ExtractCompletePdfText(stream); + + var sb = new StringBuilder(); + sb.AppendLine("=== INSURANCE SLIP DOCUMENT TEXT ==="); + sb.AppendLine(); + sb.AppendLine(completeText); + + return sb.ToString(); + } + } + catch (Exception e) + { + return $"Error extracting PDF text: {e.Message}"; + } + } + + [KernelFunction] + [Description("Imports the structured slip data as JSON into the pricing")] + public async Task ImportSlipData( + [Description("Pricing data as JSON (optional if updating existing)")] string? pricingJson, + [Description("Array of Structure layer data as JSON (can contain multiple layers)")] string? structuresJson) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing first."; + + var pricingId = chat.Context.Address.Id; + var pricingAddress = new PricingAddress(pricingId); + + try + { + // Step 1: Retrieve existing Pricing data + var existingPricing = await GetExistingPricingAsync(pricingAddress, pricingId); + + var updates = new List(); + + // Step 2: Update Pricing if provided + if (!string.IsNullOrWhiteSpace(pricingJson)) + { + var newPricingData = JsonNode.Parse(ExtractJson(pricingJson)); + if (newPricingData is JsonObject newPricingObj) + { + var mergedPricing = MergeWithExistingPricing(existingPricing, newPricingObj, pricingId); + RemoveNullProperties(mergedPricing); + updates.Add(mergedPricing); + } + } + + // Step 3: Process Structure layers (can be multiple) + if (!string.IsNullOrWhiteSpace(structuresJson)) + { + var structuresData = JsonNode.Parse(ExtractJson(structuresJson)); + + // Handle both array and single object + var structureArray = structuresData is JsonArray arr ? arr : new JsonArray { structuresData }; + + foreach (var structureData in structureArray) + { + if (structureData is JsonObject structureObj) + { + var processedStructure = EnsureTypeFirst(structureObj, nameof(Structure)); + processedStructure["pricingId"] = pricingId; + RemoveNullProperties(processedStructure); + updates.Add(processedStructure); + } + } + } + + if (updates.Count == 0) + return "No valid data provided for import."; + + // Step 4: Post DataChangeRequest + var updateRequest = new DataChangeRequest { Updates = updates }; + var response = await hub.AwaitResponse(updateRequest, o => o.WithTarget(pricingAddress)); + + return response.Message.Status switch + { + DataChangeStatus.Committed => $"Slip data imported successfully. Updated {updates.Count} entities.", + _ => $"Data update failed:\n{string.Join('\n', response.Message.Log.Messages.Select(l => l.LogLevel + ": " + l.Message))}" + }; + } + catch (Exception e) + { + return $"Import failed: {e.Message}"; + } + } + + private async Task GetExistingPricingAsync(Address pricingAddress, string pricingId) + { + try + { + var response = await hub.AwaitResponse( + new GetDataRequest(new EntityReference(nameof(Pricing), pricingId)), + o => o.WithTarget(pricingAddress)); + + return response.Message.Data as Pricing; + } + catch + { + return null; + } + } + + private JsonObject MergeWithExistingPricing(Pricing? existing, JsonObject newData, string pricingId) + { + JsonObject baseData; + if (existing != null) + { + var existingJson = JsonSerializer.Serialize(existing, GetJsonOptions()); + baseData = JsonNode.Parse(existingJson) as JsonObject ?? new JsonObject(); + } + else + { + baseData = new JsonObject(); + } + + var merged = MergeJsonObjects(baseData, newData); + merged = EnsureTypeFirst(merged, nameof(Pricing)); + merged["id"] = pricingId; + return merged; + } + + private static JsonObject MergeJsonObjects(JsonObject? existing, JsonObject? newData) + { + if (existing == null) + return newData?.DeepClone() as JsonObject ?? new JsonObject(); + + if (newData == null) + return existing.DeepClone() as JsonObject ?? new JsonObject(); + + var merged = existing.DeepClone() as JsonObject ?? new JsonObject(); + + foreach (var kvp in newData) + { + var isNullValue = kvp.Value == null || + (kvp.Value?.GetValueKind() == System.Text.Json.JsonValueKind.String && + kvp.Value.GetValue() == "null"); + + if (!isNullValue) + { + if (merged.ContainsKey(kvp.Key) && + merged[kvp.Key] is JsonObject existingObj && + kvp.Value is JsonObject newObj) + { + merged[kvp.Key] = MergeJsonObjects(existingObj, newObj); + } + else + { + merged[kvp.Key] = kvp.Value?.DeepClone(); + } + } + } + + return merged; + } + + private static JsonObject EnsureTypeFirst(JsonObject source, string typeName) + { + var ordered = new JsonObject + { + ["$type"] = typeName + }; + foreach (var kv in source) + { + if (string.Equals(kv.Key, "$type", StringComparison.Ordinal)) continue; + ordered[kv.Key] = kv.Value?.DeepClone(); + } + return ordered; + } + + private static void RemoveNullProperties(JsonNode? node) + { + if (node is JsonObject obj) + { + foreach (var kvp in obj.ToList()) + { + var value = kvp.Value; + if (value is null) + { + obj.Remove(kvp.Key); + } + else + { + RemoveNullProperties(value); + } + } + } + else if (node is JsonArray arr) + { + foreach (var item in arr) + { + RemoveNullProperties(item); + } + } + } + + private async Task ExtractCompletePdfText(Stream stream) + { + var completeText = new StringBuilder(); + + try + { + using var pdfReader = new PdfReader(stream); + using var pdfDocument = new PdfDocument(pdfReader); + + for (int pageNum = 1; pageNum <= pdfDocument.GetNumberOfPages(); pageNum++) + { + var page = pdfDocument.GetPage(pageNum); + var strategy = new SimpleTextExtractionStrategy(); + var pageText = PdfTextExtractor.GetTextFromPage(page, strategy); + + if (!string.IsNullOrWhiteSpace(pageText)) + { + completeText.AppendLine($"=== PAGE {pageNum} ==="); + completeText.AppendLine(pageText.Trim()); + completeText.AppendLine(); + } + } + } + catch (Exception ex) + { + completeText.AppendLine($"Error extracting PDF: {ex.Message}"); + } + + return completeText.ToString(); + } + + private static async Task OpenContentReadStreamAsync( + IContentService contentService, + string pricingId, + string filename) + { + try + { + // Parse pricingId in format {company}-{uwy} + var parts = pricingId.Split('-'); + if (parts.Length != 2) + return null; + + var company = parts[0]; + var uwy = parts[1]; + var contentPath = $"{company}/{uwy}/{filename}"; + + var collection = await contentService.GetCollectionAsync("Submissions", CancellationToken.None); + if (collection is null) + return null; + + return await collection.GetContentAsync(contentPath); + } + catch + { + return null; + } + } + + private string ExtractJson(string json) + { + return json.Replace("```json", "") + .Replace("```", "") + .Trim(); + } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index f809ed66b..9a5987414 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -1,5 +1,6 @@ -using MeshWeaver.ContentCollections; +using MeshWeaver.ContentCollections; using MeshWeaver.Data; +using MeshWeaver.Import; using MeshWeaver.Import.Configuration; using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; @@ -20,7 +21,7 @@ public static class InsuranceApplicationExtensions /// public static MessageHubConfiguration ConfigureInsuranceApplication(this MessageHubConfiguration configuration) => configuration - .WithTypes(typeof(PricingAddress)) + .WithTypes(typeof(PricingAddress), typeof(ExcelImportConfiguration), typeof(Structure)) .AddData(data => { var svc = data.Hub.ServiceProvider.GetRequiredService(); @@ -43,7 +44,6 @@ public static MessageHubConfiguration ConfigureInsuranceApplication(this Message public static MessageHubConfiguration ConfigureSinglePricingApplication(this MessageHubConfiguration configuration) { return configuration - .WithTypes(typeof(InsuranceApplicationExtensions)) .AddContentCollection(sp => { var hub = sp.GetRequiredService(); @@ -89,6 +89,7 @@ public static MessageHubConfiguration ConfigureSinglePricingApplication(this Mes })) .WithType(t => t.WithInitialData(async ct => (IEnumerable)await svc.GetRisksAsync(pricingId, ct))) + .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) .WithType(t => t.WithInitialData(async ct => await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) ); @@ -104,6 +105,7 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) LayoutAreas.RiskMapLayoutArea.RiskMap) .WithView(nameof(LayoutAreas.ImportConfigsLayoutArea.ImportConfigs), LayoutAreas.ImportConfigsLayoutArea.ImportConfigs) - ); + ) + .AddImport(); } } diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs new file mode 100644 index 000000000..0c1ac7ffe --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs @@ -0,0 +1,110 @@ +using System.ComponentModel.DataAnnotations; + +namespace MeshWeaver.Insurance.Domain; + +/// +/// Represents the reinsurance layer structure and financial terms. +/// +public record Structure +{ + /// + /// Gets or initializes the unique layer identifier. + /// + [Key] + public required string LayerId { get; init; } + + /// + /// Gets or initializes the pricing/contract ID this structure belongs to. + /// + public string? PricingId { get; init; } + + /// + /// Gets or initializes the layer type. + /// + public string? Type { get; init; } + + /// + /// Gets or initializes the cession percentage. + /// + public decimal Cession { get; init; } + + /// + /// Gets or initializes the share percentage. + /// + public decimal Share { get; init; } + + /// + /// Gets or initializes the attachment point. + /// + public decimal Attach { get; init; } + + /// + /// Gets or initializes the layer limit. + /// + public decimal Limit { get; init; } + + /// + /// Gets or initializes the aggregate attachment point. + /// + public decimal AggAttach { get; init; } + + /// + /// Gets or initializes the aggregate limit. + /// + public decimal AggLimit { get; init; } + + /// + /// Gets or initializes the number of reinstatements. + /// + public int NumReinst { get; init; } + + /// + /// Gets or initializes the Estimated Premium Income (EPI). + /// + public decimal EPI { get; init; } + + /// + /// Gets or initializes the rate on line. + /// + public decimal Rate { get; init; } + + /// + /// Gets or initializes the commission percentage. + /// + public decimal Commission { get; init; } + + /// + /// Gets or initializes the brokerage percentage. + /// + public decimal Brokerage { get; init; } + + /// + /// Gets or initializes the tax percentage. + /// + public decimal Tax { get; init; } + + /// + /// Gets or initializes the reinstatement premium. + /// + public decimal ReinstPrem { get; init; } + + /// + /// Gets or initializes the no claims bonus percentage. + /// + public decimal NoClaimsBonus { get; init; } + + /// + /// Gets or initializes the profit commission percentage. + /// + public decimal ProfitComm { get; init; } + + /// + /// Gets or initializes the management expense percentage. + /// + public decimal MgmtExp { get; init; } + + /// + /// Gets or initializes the minimum and deposit premium. + /// + public decimal MDPrem { get; init; } +} diff --git a/src/MeshWeaver.AI/MeshWeaver.AI.csproj b/src/MeshWeaver.AI/MeshWeaver.AI.csproj index 56ee53906..208ac46b0 100644 --- a/src/MeshWeaver.AI/MeshWeaver.AI.csproj +++ b/src/MeshWeaver.AI/MeshWeaver.AI.csproj @@ -18,6 +18,7 @@ + diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index 614b343bf..85b3e26b7 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -1,5 +1,10 @@ using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Nodes; using MeshWeaver.ContentCollections; +using MeshWeaver.Domain; +using MeshWeaver.Import; +using MeshWeaver.Import.Configuration; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; @@ -140,6 +145,73 @@ public string GenerateUniqueFileName( return $"{baseName}_{timestamp}.{extension.TrimStart('.')}"; } + [KernelFunction] + [Description("Imports data from a file in a collection to a specified address.")] + public async Task Import( + [Description("The path to the file to import")] string path, + [Description("The name of the collection containing the file (optional if default collection is configured)")] string? collection = null, + [Description("The target address for the import (optional if default address is configured), can be a string like 'AddressType/id' or an Address object")] object? address = null, + [Description("The import format to use (optional, defaults to 'Default')")] string? format = null, + CancellationToken cancellationToken = default) + { + try + { + if (string.IsNullOrWhiteSpace(collection)) + return "Collection name is required."; + + if (address == null) + return "Target address is required."; + + // Parse the address - handle both string and Address types + Address targetAddress; + if (address is string addressString) + { + targetAddress = hub.GetAddress(addressString); + } + else if (address is Address addr) + { + targetAddress = addr; + } + else + { + return $"Invalid address type: {address.GetType().Name}. Expected string or Address."; + } + + // Construct the ImportRequest directly + var importRequest = new ImportRequest(new CollectionSource(collection, path)) + { + Format = format ?? ImportFormat.Default + }; + + // Post the request to the hub + var response = await hub.AwaitResponse( + importRequest, + o => o.WithTarget(targetAddress), + cancellationToken + ); + + // Format the response + var log = response.Message.Log; + var status = log.Status.ToString(); + var result = $"Import {status.ToLower()}.\n"; + + if (log.Messages.Any()) + { + result += "Log messages:\n"; + foreach (var msg in log.Messages) + { + result += $" [{msg.LogLevel}] {msg.Message}\n"; + } + } + + return result; + } + catch (Exception ex) + { + return $"Error importing file '{path}' from collection '{collection}' to address '{address}': {ex.Message}"; + } + } + /// /// Ensures that the directory structure exists for the given file path within the collection. /// diff --git a/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs b/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs index ce7e5d922..408096f9c 100644 --- a/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs +++ b/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs @@ -3,23 +3,29 @@ using System.ComponentModel.DataAnnotations; using System.Linq.Expressions; using System.Reflection; +using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.DataSetReader; using MeshWeaver.DataSetReader.Csv; using MeshWeaver.DataSetReader.Excel; using MeshWeaver.Domain; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Import.Configuration; public record ImportConfiguration { public IWorkspace Workspace { get; } + public IServiceProvider? ServiceProvider { get; init; } public ImportConfiguration( - IWorkspace workspace + IWorkspace workspace, + IServiceProvider? serviceProvider = null ) { this.Workspace = workspace; + this.ServiceProvider = serviceProvider; + StreamProviders = InitializeStreamProviders(); Validations = ImmutableList .Empty.Add(StandardValidations) .Add(CategoriesValidation); @@ -90,12 +96,15 @@ this with DataSetReaders = DataSetReaders.SetItem(fileType, dataSetReader) }; - internal ImmutableDictionary> StreamProviders { get; init; } = - ImmutableDictionary> + internal ImmutableDictionary>> StreamProviders { get; init; } + + private ImmutableDictionary>> InitializeStreamProviders() => + ImmutableDictionary>> .Empty.Add(typeof(StringStream), CreateMemoryStream) - .Add(typeof(EmbeddedResource), CreateEmbeddedResourceStream); + .Add(typeof(EmbeddedResource), CreateEmbeddedResourceStream) + .Add(typeof(CollectionSource), CreateCollectionStreamAsync); - private static Stream CreateEmbeddedResourceStream(ImportRequest request) + private static Task CreateEmbeddedResourceStream(ImportRequest request) { var embeddedResource = (EmbeddedResource)request.Source; var assembly = embeddedResource.Assembly; @@ -105,22 +114,41 @@ private static Stream CreateEmbeddedResourceStream(ImportRequest request) { throw new ArgumentException($"Resource '{resourceName}' not found."); } - return stream; + return Task.FromResult(stream); } - private static Stream CreateMemoryStream(ImportRequest request) + private static Task CreateMemoryStream(ImportRequest request) { var stream = new MemoryStream(); var writer = new StreamWriter(stream); writer.Write(((StringStream)request.Source).Content); writer.Flush(); stream.Position = 0; + return Task.FromResult(stream); + } + + private async Task CreateCollectionStreamAsync(ImportRequest request) + { + var collectionSource = (CollectionSource)request.Source; + + // Resolve from ContentCollection + if (ServiceProvider == null) + throw new ImportException("ServiceProvider is not available to resolve CollectionSource from ContentCollection"); + + var contentService = ServiceProvider.GetService(); + if (contentService == null) + throw new ImportException("IContentService is not registered. Ensure ContentCollections are configured."); + + var stream = await contentService.GetContentAsync(collectionSource.Collection, collectionSource.Path); + if (stream == null) + throw new ImportException($"Could not find content at collection '{collectionSource.Collection}' path '{collectionSource.Path}'"); + return stream; } public ImportConfiguration WithStreamReader( Type sourceType, - Func reader + Func> reader ) => this with { StreamProviders = StreamProviders.SetItem(sourceType, reader) }; internal ImmutableList Validations { get; init; } diff --git a/src/MeshWeaver.Import/Implementation/ImportManager.cs b/src/MeshWeaver.Import/Implementation/ImportManager.cs index 859c075cf..4ac3edc2b 100644 --- a/src/MeshWeaver.Import/Implementation/ImportManager.cs +++ b/src/MeshWeaver.Import/Implementation/ImportManager.cs @@ -22,7 +22,7 @@ public ImportManager(IWorkspace workspace, IMessageHub hub) Workspace = workspace; Hub = hub; - Configuration = hub.Configuration.GetListOfLambdas().Aggregate(new ImportConfiguration(workspace), (c, l) => l.Invoke(c)); + Configuration = hub.Configuration.GetListOfLambdas().Aggregate(new ImportConfiguration(workspace, hub.ServiceProvider), (c, l) => l.Invoke(c)); // Don't initialize the import hub in constructor - do it lazily to avoid timing issues logger?.LogDebug("ImportManager constructor completed for hub {HubAddress}", hub.Address); @@ -83,18 +83,17 @@ private void FinishWithException(IMessageDelivery request, Exception e, private async Task ImportImpl(IMessageDelivery request, CancellationToken cancellationToken) { var activity = new Activity(ActivityCategory.Import, Hub, autoClose: false); + var importActivity = activity.StartSubActivity(ActivityCategory.Import); try { activity.LogInformation("Starting import {ActivityId} for request {RequestId}", activity.Id, request.Id); - var importActivity = activity.StartSubActivity(ActivityCategory.Import); var imported = await ImportInstancesAsync(request.Message, importActivity, cancellationToken); importActivity.Complete(log => { if (log.HasErrors()) { - Hub.Post(new ImportResponse(Hub.Version, log), o => o.ResponseFor(request)); return; } @@ -114,6 +113,9 @@ private async Task ImportImpl(IMessageDelivery request, Cancellat } catch (Exception e) { + importActivity.LogError(e.Message); + importActivity.Complete(); + activity.LogError("Import {ImportId} for {RequestId} failed with exception: {Exception}", activity.Id, request.Id, e.Message); FinishWithException(request, e, activity); } @@ -137,7 +139,7 @@ public async Task ImportInstancesAsync( if (!Configuration.StreamProviders.TryGetValue(sourceType, out var streamProvider)) throw new ImportException($"Unknown stream type: {sourceType.FullName}"); - var stream = streamProvider.Invoke(importRequest); + var stream = await streamProvider.Invoke(importRequest); if (stream == null) throw new ImportException($"Could not open stream: {importRequest.Source}"); diff --git a/src/MeshWeaver.Import/ImportRegistryExtensions.cs b/src/MeshWeaver.Import/ImportRegistryExtensions.cs index caecc9f83..3e83aa5f8 100644 --- a/src/MeshWeaver.Import/ImportRegistryExtensions.cs +++ b/src/MeshWeaver.Import/ImportRegistryExtensions.cs @@ -30,6 +30,14 @@ Func importConfiguration .AddData() .WithServices(x => x.AddScoped()) .AddHandlers() + .WithTypes( + typeof(ImportRequest), + typeof(ImportResponse), + typeof(Source), + typeof(StringStream), + typeof(CollectionSource), + typeof(EmbeddedResource) + ) .WithInitialization(h => h.ServiceProvider.GetRequiredService()) ; diff --git a/src/MeshWeaver.Import/ImportRequest.cs b/src/MeshWeaver.Import/ImportRequest.cs index 952c718f1..0e935154f 100644 --- a/src/MeshWeaver.Import/ImportRequest.cs +++ b/src/MeshWeaver.Import/ImportRequest.cs @@ -17,7 +17,7 @@ public ImportRequest(string content) public string MimeType { get; init; } = MimeTypes.MapFileExtension( - Source is StreamSource stream ? Path.GetExtension(stream.Name) : "" + Source is CollectionSource stream ? Path.GetExtension(stream.Path) : "" ) ?? ""; public string Format { get; init; } = ImportFormat.Default; @@ -47,6 +47,6 @@ public abstract record Source { } public record StringStream(string Content) : Source; -public record StreamSource(string Name, Stream Stream) : Source; +public record CollectionSource(string Collection, string Path) : Source; //public record FileStream(string FileName) : Source; diff --git a/src/MeshWeaver.Import/MeshWeaver.Import.csproj b/src/MeshWeaver.Import/MeshWeaver.Import.csproj index e9296714c..75e65cbc7 100644 --- a/src/MeshWeaver.Import/MeshWeaver.Import.csproj +++ b/src/MeshWeaver.Import/MeshWeaver.Import.csproj @@ -10,5 +10,6 @@ + diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs b/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs index e3355f35b..669fb5bcc 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs @@ -67,4 +67,55 @@ public static Address GetAddress(this IMessageHub hub, string address) return (Address)Activator.CreateInstance(type, [string.Join('/', split.Skip(1))])!; } + + /// + /// Sends a request deserialized from JSON and awaits the response. + /// This is useful when working with JSON-based messaging without direct type references. + /// + /// The message hub + /// The request object (deserialized from JSON) + /// Post options + /// Cancellation token + /// The response message + public static async Task AwaitResponse( + this IMessageHub hub, + object request, + Func options, + CancellationToken cancellationToken = default) + { + // Find the IRequest interface to get the response type + var requestType = request.GetType(); + var requestInterface = requestType.GetInterfaces() + .FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IRequest<>)); + + if (requestInterface == null) + throw new InvalidOperationException($"Request type {requestType.Name} does not implement IRequest"); + + var responseType = requestInterface.GetGenericArguments()[0]; + + // Use reflection to call AwaitResponse + var awaitResponseMethod = typeof(IMessageHub).GetMethods() + .FirstOrDefault(m => + m.Name == nameof(IMessageHub.AwaitResponse) && + m.IsGenericMethodDefinition && + m.GetGenericArguments().Length == 2 && + m.GetParameters().Length == 3); + + if (awaitResponseMethod == null) + throw new InvalidOperationException("Could not find AwaitResponse method"); + + var genericMethod = awaitResponseMethod.MakeGenericMethod(responseType, responseType); + + // Create the result selector lambda: (IMessageDelivery d) => d.Message + var deliveryParam = System.Linq.Expressions.Expression.Parameter(typeof(IMessageDelivery<>).MakeGenericType(responseType), "d"); + var messageProperty = System.Linq.Expressions.Expression.Property(deliveryParam, "Message"); + var lambda = System.Linq.Expressions.Expression.Lambda(messageProperty, deliveryParam); + var resultSelector = lambda.Compile(); + + var task = (Task)genericMethod.Invoke(hub, new object[] { request, options, resultSelector })!; + await task.ConfigureAwait(false); + + var resultProperty = task.GetType().GetProperty("Result"); + return resultProperty!.GetValue(task)!; + } } diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs new file mode 100644 index 000000000..d1c38b03a --- /dev/null +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -0,0 +1,189 @@ +using System; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Extensions; +using MeshWeaver.AI.Plugins; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; +using MeshWeaver.Data.TestDomain; +using MeshWeaver.Fixture; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace MeshWeaver.Import.Test; + +public class CollectionPluginImportTest(ITestOutputHelper output) : HubTestBase(output) +{ + private readonly string _testFilesPath = Path.Combine(AppContext.BaseDirectory, "TestFiles", "CollectionPluginImport"); + + protected override MessageHubConfiguration ConfigureRouter(MessageHubConfiguration configuration) + { + // Ensure test directory and file exist + Directory.CreateDirectory(_testFilesPath); + var csvContent = @"@@LineOfBusiness +SystemName,DisplayName +1,LoB 1 +2,LoB 2 +3,LoB 3"; + File.WriteAllText(Path.Combine(_testFilesPath, "test-data.csv"), csvContent); + + return base.ConfigureRouter(configuration) + .WithTypes(typeof(ImportAddress)) + .AddContentCollections() + .AddFileSystemContentCollection("TestCollection", _ => _testFilesPath) + .WithRoutes(forward => + forward + .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) + .RouteAddressToHostedHub(c => c.ConfigureImportHub()) + ); + } + + [Fact] + public async Task CollectionPlugin_Import_ShouldImportSuccessfully() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: "TestCollection", + address: new ImportAddress(2024), + format: null, // Use default format + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("succeeded", "import should succeed"); + result.Should().NotContain("Error", "there should be no errors"); + + // Verify data was imported + var referenceDataHub = Router.GetHostedHub(new ReferenceDataAddress()); + var workspace = referenceDataHub.ServiceProvider.GetRequiredService(); + + var allData = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count >= 3); + + allData.Should().HaveCount(3); + var items = allData.OrderBy(x => x.SystemName).ToList(); + items[0].DisplayName.Should().Be("LoB 1"); + items[1].DisplayName.Should().Be("LoB 2"); + items[2].DisplayName.Should().Be("LoB 3"); + } + + [Fact] + public async Task CollectionPlugin_Import_WithNonExistentFile_ShouldFail() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "non-existent.csv", + collection: "TestCollection", + address: new ImportAddress(2024), + format: null, + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("failed", "import should fail for non-existent file"); + } + + [Fact] + public async Task CollectionPlugin_Import_WithNonExistentCollection_ShouldFail() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: "NonExistentCollection", + address: new ImportAddress(2024), + format: null, + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("Error", "import should fail for non-existent collection"); + } + + [Fact] + public async Task CollectionPlugin_Import_WithMissingCollection_ShouldReturnError() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: null, + address: "ImportAddress/2024", + format: null, + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("Collection name is required"); + } + + [Fact] + public async Task CollectionPlugin_Import_WithMissingAddress_ShouldReturnError() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: "TestCollection", + address: null, + format: null, + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("Target address is required"); + } + + [Fact] + public async Task CollectionPlugin_Import_WithCustomFormat_ShouldImportSuccessfully() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: "TestCollection", + address: new ImportAddress(2024), + format: "Default", // Explicit format + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + result.Should().Contain("succeeded", "import should succeed with explicit format"); + + // Verify data was imported + var referenceDataHub = Router.GetHostedHub(new ReferenceDataAddress()); + var workspace = referenceDataHub.ServiceProvider.GetRequiredService(); + + var allData = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count >= 3); + + allData.Should().HaveCount(3); + } +} diff --git a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs new file mode 100644 index 000000000..c8bfdb68e --- /dev/null +++ b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs @@ -0,0 +1,148 @@ +using System; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Extensions; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; +using MeshWeaver.Data.TestDomain; +using MeshWeaver.Fixture; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace MeshWeaver.Import.Test; + +public class CollectionSourceImportTest(ITestOutputHelper output) : HubTestBase(output) +{ + private readonly string _testFilesPath = Path.Combine(AppContext.BaseDirectory, "TestFiles", "CollectionSource"); + + protected override MessageHubConfiguration ConfigureRouter(MessageHubConfiguration configuration) + { + // Ensure test directory and file exist + Directory.CreateDirectory(_testFilesPath); + var csvContent = @"@@LineOfBusiness +SystemName,DisplayName +1,LoB 1 +2,LoB 2 +3,LoB 3"; + File.WriteAllText(Path.Combine(_testFilesPath, "test-data.csv"), csvContent); + + return base.ConfigureRouter(configuration) + .AddContentCollections() + .AddFileSystemContentCollection("TestCollection", _ => _testFilesPath) + .WithRoutes(forward => + forward + .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) + .RouteAddressToHostedHub(c => c.ConfigureImportHub()) + ); + } + + [Fact] + public async Task ImportFromCollectionSource_ShouldResolveAndImportSuccessfully() + { + // Arrange + var client = GetClient(); + + // Create ImportRequest with CollectionSource - stream will be resolved automatically + var importRequest = new ImportRequest(new CollectionSource("TestCollection", "test-data.csv")); + + // Act + var importResponse = await client.AwaitResponse( + importRequest, + o => o.WithTarget(new ImportAddress(2024)), + TestContext.Current.CancellationToken + ); + + // Assert + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded); + + // Verify data was imported + var referenceDataHub = Router.GetHostedHub(new ReferenceDataAddress()); + var workspace = referenceDataHub.ServiceProvider.GetRequiredService(); + + var allData = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count >= 3); + + allData.Should().HaveCount(3); + var items = allData.OrderBy(x => x.SystemName).ToList(); + items[0].DisplayName.Should().Be("LoB 1"); + items[1].DisplayName.Should().Be("LoB 2"); + items[2].DisplayName.Should().Be("LoB 3"); + } + + [Fact] + public async Task ImportFromCollectionSource_WithSubfolder_ShouldResolveAndImportSuccessfully() + { + // Arrange + var client = GetClient(); + + // Test with path containing subfolder + var importRequest = new ImportRequest(new CollectionSource("TestCollection", "/test-data.csv")); + + // Act + var importResponse = await client.AwaitResponse( + importRequest, + o => o.WithTarget(new ImportAddress(2024)), + TestContext.Current.CancellationToken + ); + + // Assert + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded); + + // Verify data was imported + var referenceDataHub = Router.GetHostedHub(new ReferenceDataAddress()); + var workspace = referenceDataHub.ServiceProvider.GetRequiredService(); + + var allData = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count >= 3); + + allData.Should().HaveCount(3); + } + + [Fact] + public async Task ImportFromCollectionSource_NonExistentFile_ShouldFail() + { + // Arrange + var client = GetClient(); + + var importRequest = new ImportRequest(new CollectionSource("TestCollection", "non-existent.csv")); + + // Act + var importResponse = await client.AwaitResponse( + importRequest, + o => o.WithTarget(new ImportAddress(2024)), + TestContext.Current.CancellationToken + ); + + // Assert + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Failed); + var errors = importResponse.Message.Log.Errors(); + errors.Should().Contain(m => m.Message.Contains("Could not find content")); + } + + [Fact] + public async Task ImportFromCollectionSource_NonExistentCollection_ShouldFail() + { + // Arrange + var client = GetClient(); + + var importRequest = new ImportRequest(new CollectionSource("NonExistentCollection", "test-data.csv")); + + // Act + var importResponse = await client.AwaitResponse( + importRequest, + o => o.WithTarget(new ImportAddress(2024)), + TestContext.Current.CancellationToken + ); + + // Assert + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Failed); + var errors = importResponse.Message.Log.Errors(); + errors.Should().Contain(m => m.Message.Contains("Could not find content")); + } +} diff --git a/test/MeshWeaver.Import.Test/MeshWeaver.Import.Test.csproj b/test/MeshWeaver.Import.Test/MeshWeaver.Import.Test.csproj index 5e81e56a8..1365efc1c 100644 --- a/test/MeshWeaver.Import.Test/MeshWeaver.Import.Test.csproj +++ b/test/MeshWeaver.Import.Test/MeshWeaver.Import.Test.csproj @@ -14,5 +14,6 @@ + From 12437fa8530346685f7858c0ca754548d97df89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 27 Oct 2025 14:01:34 +0100 Subject: [PATCH 02/57] enabling json in AwaitResponse and converting CollectionPlugin.Import to use json --- .../RiskImportAgent.cs | 40 ++++---- src/MeshWeaver.AI/MeshWeaver.AI.csproj | 1 - src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 51 ++++++---- src/MeshWeaver.Import/ImportRequest.cs | 28 +++++- src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 27 ++++-- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 94 +++---------------- .../MessageHubExtensions.cs | 50 +++++----- .../MessageService.cs | 22 ++--- .../CollectionPluginImportTest.cs | 2 +- .../CollectionSourceImportTest.cs | 6 +- 10 files changed, 149 insertions(+), 172 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index 00e1fc21d..6a3452232 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -180,26 +180,26 @@ private JsonSerializerOptions GetJsonOptions() return hub.JsonSerializerOptions; } - //[KernelFunction] - //[Description("Imports a file with filename")] - //public async Task Import(string filename) - //{ - // if (chat.Context?.Address?.Type != PricingAddress.TypeName) - // return "Please navigate to the pricing for which you want to import risks."; - - // var pricingId = chat.Context.Address.Id; - // var collectionName = $"Submissions-{pricingId}"; - // var address = $"{PricingAddress.TypeName}/{pricingId}"; - - // // Delegate to CollectionPlugin's Import method - // var collectionPlugin = new CollectionPlugin(hub); - // return await collectionPlugin.Import( - // path: filename, - // collection: collectionName, - // address: address, - // format: "PropertyRiskImport" - // ); - //} + [KernelFunction] + [Description("Imports a file with filename")] + public async Task Import(string filename) + { + if (chat.Context?.Address?.Type != PricingAddress.TypeName) + return "Please navigate to the pricing for which you want to import risks."; + + var pricingId = chat.Context.Address.Id; + var collectionName = $"Submissions-{pricingId}"; + var address = new PricingAddress(pricingId); + + // Delegate to CollectionPlugin's Import method + var collectionPlugin = new CollectionPlugin(hub); + return await collectionPlugin.Import( + path: filename, + collection: collectionName, + address: address, + format: "PropertyRiskImport" + ); + } [KernelFunction] [Description("Gets the risk configuration for a particular file")] diff --git a/src/MeshWeaver.AI/MeshWeaver.AI.csproj b/src/MeshWeaver.AI/MeshWeaver.AI.csproj index 208ac46b0..56ee53906 100644 --- a/src/MeshWeaver.AI/MeshWeaver.AI.csproj +++ b/src/MeshWeaver.AI/MeshWeaver.AI.csproj @@ -18,7 +18,6 @@ - diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index 85b3e26b7..9477a0291 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -2,9 +2,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using MeshWeaver.ContentCollections; -using MeshWeaver.Domain; -using MeshWeaver.Import; -using MeshWeaver.Import.Configuration; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; @@ -36,7 +33,7 @@ public async Task GetFile( return $"File '{filePath}' not found in collection '{collectionName}'."; using var reader = new StreamReader(stream); - var content = await reader.ReadToEndAsync(); + var content = await reader.ReadToEndAsync(cancellationToken); return content; } @@ -177,34 +174,56 @@ public async Task Import( return $"Invalid address type: {address.GetType().Name}. Expected string or Address."; } - // Construct the ImportRequest directly - var importRequest = new ImportRequest(new CollectionSource(collection, path)) + // Build ImportRequest JSON structure + var importRequestJson = new JsonObject { - Format = format ?? ImportFormat.Default + ["$type"] = "MeshWeaver.Import.ImportRequest", + ["source"] = new JsonObject + { + ["$type"] = "MeshWeaver.Import.CollectionSource", + ["collection"] = collection, + ["path"] = path + }, + ["format"] = format ?? "Default" }; + // Serialize and deserialize through hub's serializer to get proper type + var jsonString = importRequestJson.ToJsonString(); + var importRequestObj = JsonSerializer.Deserialize(jsonString, hub.JsonSerializerOptions)!; + // Post the request to the hub - var response = await hub.AwaitResponse( - importRequest, + var responseMessage = await hub.AwaitResponse( + importRequestObj, o => o.WithTarget(targetAddress), cancellationToken ); - // Format the response - var log = response.Message.Log; - var status = log.Status.ToString(); - var result = $"Import {status.ToLower()}.\n"; + // Serialize the response back to JSON for processing + var responseJson = JsonSerializer.Serialize(responseMessage, hub.JsonSerializerOptions); + var responseObj = JsonNode.Parse(responseJson)!; - if (log.Messages.Any()) + var log = responseObj["log"] as JsonObject; + var status = log?["status"]?.ToString() ?? "Unknown"; + var messages = log?["messages"] as JsonArray ?? new JsonArray(); + + var result = $"Import {status.ToLower()}.\n"; + if (messages.Count > 0) { result += "Log messages:\n"; - foreach (var msg in log.Messages) + foreach (var msg in messages) { - result += $" [{msg.LogLevel}] {msg.Message}\n"; + if (msg is JsonObject msgObj) + { + var level = msgObj["logLevel"]?.ToString() ?? "Info"; + var message = msgObj["message"]?.ToString() ?? ""; + result += $" [{level}] {message}\n"; + } } } return result; + + return "Import completed but response format was unexpected."; } catch (Exception ex) { diff --git a/src/MeshWeaver.Import/ImportRequest.cs b/src/MeshWeaver.Import/ImportRequest.cs index 0e935154f..1185fb653 100644 --- a/src/MeshWeaver.Import/ImportRequest.cs +++ b/src/MeshWeaver.Import/ImportRequest.cs @@ -1,4 +1,5 @@ -using MeshWeaver.Data; +using System.Text.Json.Serialization; +using MeshWeaver.Data; using MeshWeaver.DataSetReader; using MeshWeaver.Import.Configuration; using MeshWeaver.Messaging; @@ -9,16 +10,26 @@ namespace MeshWeaver.Import; /// This is a request entity triggering import when executing in a data hub /// using the Import Plugin. See also AddImport method. /// -/// Content of the source to be imported, e.g. a string (shipping the entire content) or a file name (together with StreamType = File) -public record ImportRequest(Source Source) : IRequest +public record ImportRequest : IRequest { public ImportRequest(string content) : this(new StringStream(content)) { } - public string MimeType { get; init; } = - MimeTypes.MapFileExtension( + /// + /// This is a request entity triggering import when executing in a data hub + /// using the Import Plugin. See also AddImport method. + /// + /// Content of the source to be imported, e.g. a string (shipping the entire content) or a file name (together with StreamType = File) + [JsonConstructor] + public ImportRequest(Source Source) + { + this.Source = Source; + MimeType = MimeTypes.MapFileExtension( Source is CollectionSource stream ? Path.GetExtension(stream.Path) : "" ) ?? ""; + } + + public string MimeType { get; init; } public string Format { get; init; } = ImportFormat.Default; public object? TargetDataSource { get; init; } @@ -39,6 +50,13 @@ public ImportRequest(string content) public bool SaveLog { get; init; } + /// Content of the source to be imported, e.g. a string (shipping the entire content) or a file name (together with StreamType = File) + public Source Source { get; init; } + + public void Deconstruct(out Source Source) + { + Source = this.Source; + } } public record ImportResponse(long Version, ActivityLog Log); diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index 0af3e4b4f..d33934dfe 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -16,19 +16,30 @@ public interface IMessageHub : IMessageHandlerRegistry, IDisposable Task> AwaitResponse(IRequest request) => AwaitResponse(request, new CancellationTokenSource(DefaultTimeout).Token); - Task> AwaitResponse(IMessageDelivery> request, CancellationToken cancellationToken); - - Task> AwaitResponse(IRequest request, CancellationToken cancellationToken); - - Task> AwaitResponse(IRequest request, Func options, CancellationToken cancellationToken = default); - Task AwaitResponse(IRequest request, Func, TResult> selector) + async Task> AwaitResponse(IMessageDelivery> request, CancellationToken cancellationToken) + => (IMessageDelivery)(await AwaitResponse(request, o => o, o => o, cancellationToken))!; + + Task> AwaitResponse(IRequest request, + CancellationToken cancellationToken) + => AwaitResponse(request, x => x, x => x, cancellationToken)!; + + Task?> AwaitResponse(IRequest request, + Func options, CancellationToken cancellationToken = default) + => AwaitResponse(request, options, o => o, cancellationToken); + Task AwaitResponse(IRequest request, + Func, TResult> selector) => AwaitResponse(request, x => x, selector); - Task AwaitResponse(IRequest request, Func, TResult> selector, CancellationToken cancellationToken) + Task AwaitResponse(IRequest request, + Func, TResult> selector, CancellationToken cancellationToken) => AwaitResponse(request, x => x, selector, cancellationToken); - Task AwaitResponse(IRequest request, Func options, Func, TResult> selector, CancellationToken cancellationToken = default); + async Task AwaitResponse(IRequest request, Func options, + Func, TResult> selector, CancellationToken cancellationToken = default) + => (TResult?)await AwaitResponse((object)request, options, o => selector((IMessageDelivery)o), cancellationToken); + + Task AwaitResponse(object request, Func options, Func selector, CancellationToken cancellationToken = default); Task RegisterCallback(IMessageDelivery> request, AsyncDelivery callback, CancellationToken cancellationToken = default) => RegisterCallback((IMessageDelivery)request, (r, c) => callback((IMessageDelivery)r, c), diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 67c179e25..cf68dc577 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -6,7 +6,6 @@ using MeshWeaver.Domain; using MeshWeaver.Reflection; using MeshWeaver.ServiceProvider; -using MeshWeaver.ShortGuid; using MeshWeaver.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -22,6 +21,8 @@ public void InvokeAsync(Func action, Func> callbacks = new(); private readonly HashSet pendingCallbackCancellations = new(); @@ -339,66 +340,13 @@ private IMessageDelivery FinishDelivery(IMessageDelivery delivery) - public Task> AwaitResponse( - IMessageDelivery> request, CancellationToken cancellationToken) - { - var tcs = new TaskCompletionSource>(cancellationToken); - var callbackTask = RegisterCallback( - request.Id, - d => - { - tcs.SetResult((IMessageDelivery)d); - return d.Processed(); - }, - cancellationToken - ); - return callbackTask.ContinueWith(_ => tcs.Task.Result, cancellationToken); - } - public Task> AwaitResponse( - IRequest request, - CancellationToken cancellationToken - ) => AwaitResponse(request, x => x, x => x, cancellationToken); - - public Task> AwaitResponse( - IRequest request, - Func options - ) => - AwaitResponse( - request, - options, - new CancellationTokenSource(IMessageHub.DefaultTimeout).Token - ); - public Task> AwaitResponse( - IRequest request, - Func options, - CancellationToken cancellationToken - ) => AwaitResponse(request, options, x => x, cancellationToken); - public Task AwaitResponse( - IRequest request, - Func options, - Func, TResult> selector, - CancellationToken cancellationToken - ) - { - var id = Guid.NewGuid().AsString(); - var ret = AwaitResponse( - id, - selector, - cancellationToken - ); - Post(request, o => options.Invoke(o).WithMessageId(id)); - return ret; - } - public Task AwaitResponse( - IMessageDelivery request, - Func, TResult> selector, - CancellationToken cancellationToken - ) + public Task AwaitResponse(object r, Func options, Func selector, CancellationToken cancellationToken = default) { + var request = r as IMessageDelivery ?? Post(r, options)!; var response = RegisterCallback( request.Id, d => d, @@ -406,43 +354,25 @@ CancellationToken cancellationToken ); var task = response .ContinueWith(t => - InnerCallback(request.Id, t.Result, selector), + { + var ret = t.Result; + return InnerCallback(request.Id, ret, selector); + + }, cancellationToken ); return task; } - public Task AwaitResponse( - string id, - Func, TResult> selector, - CancellationToken cancellationToken - ) - { - var response = RegisterCallback( - id, - d => d, - cancellationToken - ); - var task = response.ContinueWith(async t => - { - // Await the task to propagate the original exception - var result = await t; - return InnerCallback(id, result, selector); - }, cancellationToken).Unwrap(); - - return task; - } - private TResult InnerCallback( + private object? InnerCallback( string id, IMessageDelivery response, - Func, TResult> selector) + Func selector) { try { - if (response is IMessageDelivery tResponse) - return selector.Invoke(tResponse); - throw new DeliveryFailureException($"Response for {id} was of unexpected type: {response}"); + return selector.Invoke(response); } catch (DeliveryFailureException) { diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs b/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs index 669fb5bcc..413e33d78 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubExtensions.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Text.Json; using System.Text.Json.Nodes; using MeshWeaver.Domain; using MeshWeaver.Messaging.Serialization; @@ -77,45 +78,46 @@ public static Address GetAddress(this IMessageHub hub, string address) /// Post options /// Cancellation token /// The response message - public static async Task AwaitResponse( + public static async Task AwaitResponse( this IMessageHub hub, object request, Func options, CancellationToken cancellationToken = default) { + // If the request is a JsonElement, we need to deserialize it to the concrete type first + if (request is JsonElement jsonElement) + { + // Get the type discriminator from the JSON + if (!jsonElement.TryGetProperty("$type", out var typeElement)) + throw new InvalidOperationException("JSON request must have a '$type' property"); + + var typeName = typeElement.GetString(); + if (string.IsNullOrEmpty(typeName)) + throw new InvalidOperationException("'$type' property cannot be empty"); + + // Find the type in the type registry + var concreteType = hub.GetTypeRegistry().GetType(typeName); + if (concreteType == null) + concreteType = typeof(JsonElement); + + // Deserialize to the concrete type + request = JsonSerializer.Deserialize(jsonElement.GetRawText(), concreteType, hub.JsonSerializerOptions)!; + } + // Find the IRequest interface to get the response type var requestType = request.GetType(); var requestInterface = requestType.GetInterfaces() .FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IRequest<>)); - if (requestInterface == null) - throw new InvalidOperationException($"Request type {requestType.Name} does not implement IRequest"); - - var responseType = requestInterface.GetGenericArguments()[0]; - - // Use reflection to call AwaitResponse - var awaitResponseMethod = typeof(IMessageHub).GetMethods() - .FirstOrDefault(m => - m.Name == nameof(IMessageHub.AwaitResponse) && - m.IsGenericMethodDefinition && - m.GetGenericArguments().Length == 2 && - m.GetParameters().Length == 3); - if (awaitResponseMethod == null) - throw new InvalidOperationException("Could not find AwaitResponse method"); - - var genericMethod = awaitResponseMethod.MakeGenericMethod(responseType, responseType); // Create the result selector lambda: (IMessageDelivery d) => d.Message - var deliveryParam = System.Linq.Expressions.Expression.Parameter(typeof(IMessageDelivery<>).MakeGenericType(responseType), "d"); + var deliveryParam = System.Linq.Expressions.Expression.Parameter(typeof(IMessageDelivery), "d"); var messageProperty = System.Linq.Expressions.Expression.Property(deliveryParam, "Message"); - var lambda = System.Linq.Expressions.Expression.Lambda(messageProperty, deliveryParam); + var lambda = System.Linq.Expressions.Expression.Lambda>(messageProperty, deliveryParam); var resultSelector = lambda.Compile(); - var task = (Task)genericMethod.Invoke(hub, new object[] { request, options, resultSelector })!; - await task.ConfigureAwait(false); - - var resultProperty = task.GetType().GetProperty("Result"); - return resultProperty!.GetValue(task)!; + var resultProperty = await hub.AwaitResponse(request, options, resultSelector, cancellationToken); + return resultProperty; } } diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 0f341ff5e..dd1b76e71 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Threading.Tasks.Dataflow; -using Json.More; using Microsoft.Extensions.Logging; // ReSharper disable InconsistentlySynchronizedField @@ -271,11 +270,6 @@ private IMessageDelivery UnpackIfNecessary(IMessageDelivery delivery) return delivery.Failed($"Deserialization failed: {ex.Message}"); } - if (delivery.Message is JsonElement je) - { - return delivery.Failed($"Could not deserialize message {je.ToJsonString()}"); - } - return delivery; } private IMessageDelivery DeserializeDelivery(IMessageDelivery delivery) @@ -290,7 +284,16 @@ private IMessageDelivery DeserializeDelivery(IMessageDelivery delivery) } private IMessageDelivery PostImpl(object message, PostOptions opt) - => (IMessageDelivery)PostImplMethod.MakeGenericMethod(message.GetType()).Invoke(this, new[] { message, opt })!; + { + if (message is JsonElement je) + message = new RawJson(je.ToString()); + if (message is JsonNode jn) + message = new RawJson(jn.ToString()); + + return (IMessageDelivery)PostImplMethod.MakeGenericMethod(message.GetType()) + .Invoke(this, [message, opt])!; + + } private static readonly MethodInfo PostImplMethod = typeof(MessageService).GetMethod(nameof(PostImplGeneric), BindingFlags.Instance | BindingFlags.NonPublic)!; @@ -300,11 +303,6 @@ private IMessageDelivery PostImplGeneric(TMessage message, PostOptions if (message == null) throw new ArgumentNullException(nameof(message)); - if (typeof(TMessage) != message.GetType()) - return (IMessageDelivery)PostImplMethod - .MakeGenericMethod(message.GetType()) - .Invoke(this, [message, opt])!; - var delivery = new MessageDelivery(message, opt, hub.JsonSerializerOptions) { Id = opt.MessageId diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs index d1c38b03a..6919b1b1e 100644 --- a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -32,7 +32,7 @@ protected override MessageHubConfiguration ConfigureRouter(MessageHubConfigurati File.WriteAllText(Path.Combine(_testFilesPath, "test-data.csv"), csvContent); return base.ConfigureRouter(configuration) - .WithTypes(typeof(ImportAddress)) + .WithTypes(typeof(ImportAddress), typeof(ImportRequest), typeof(CollectionSource)) .AddContentCollections() .AddFileSystemContentCollection("TestCollection", _ => _testFilesPath) .WithRoutes(forward => diff --git a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs index c8bfdb68e..c11a6ebc2 100644 --- a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs @@ -80,8 +80,8 @@ public async Task ImportFromCollectionSource_WithSubfolder_ShouldResolveAndImpor // Arrange var client = GetClient(); - // Test with path containing subfolder - var importRequest = new ImportRequest(new CollectionSource("TestCollection", "/test-data.csv")); + // Test with path without leading slash (file is in root of collection) + var importRequest = new ImportRequest(new CollectionSource("TestCollection", "test-data.csv")); // Act var importResponse = await client.AwaitResponse( @@ -143,6 +143,6 @@ public async Task ImportFromCollectionSource_NonExistentCollection_ShouldFail() // Assert importResponse.Message.Log.Status.Should().Be(ActivityStatus.Failed); var errors = importResponse.Message.Log.Errors(); - errors.Should().Contain(m => m.Message.Contains("Could not find content")); + errors.Should().Contain(m => m.Message.Contains("Collection") && m.Message.Contains("not found")); } } From 9f81dcf34a14aee9a9472b13d14405286a6f800d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 27 Oct 2025 21:36:05 +0100 Subject: [PATCH 03/57] correcting import with Configuration in ImportRequest --- .claude/settings.local.json | 3 +- .../Files/Microsoft/2026/Microsoft.xlsx | Bin 30360 -> 29723 bytes .../RiskImportAgent.cs | 37 +- .../InsuranceApplicationExtensions.cs | 2 +- .../LayoutAreas/ImportConfigsLayoutArea.cs | 46 +-- .../MicrosoftImportTests.cs | 366 ++++++++---------- src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 13 +- .../Configuration/AutoEntityBuilder.cs | 113 ++++++ .../Configuration/ExcelImportConfiguration.cs | 12 +- .../Configuration/ImportBuilder.cs | 238 ++++++++++++ .../Configuration/ImportConfiguration.cs | 247 +----------- .../ConfiguredExcelImporter.cs | 46 +-- .../ExcelImportExtensions.cs | 4 +- .../Implementation/ImportManager.cs | 112 +++++- .../ImportUnpartitionedDataSource.cs | 6 +- .../ImportRegistryExtensions.cs | 6 +- src/MeshWeaver.Import/ImportRequest.cs | 6 + .../CollectionPluginImportTest.cs | 46 +++ 18 files changed, 762 insertions(+), 541 deletions(-) create mode 100644 src/MeshWeaver.Import/Configuration/AutoEntityBuilder.cs create mode 100644 src/MeshWeaver.Import/Configuration/ImportBuilder.cs diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 580a43ba2..dfd9f1beb 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -44,7 +44,8 @@ "Bash(python:*)", "Bash(python3:*)", "Bash(test:*)", - "Bash(Select-Object -Last 20)" + "Bash(Select-Object -Last 20)", + "Bash(git mv:*)" ], "deny": [] } diff --git a/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx b/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx index 33fd6cc1e94b41c5624fe15fff5d9608aac3780d..1513904a46c93963c530727861023994f5519884 100644 GIT binary patch delta 15735 zcmZv@1ymeO(>9Dd1b25^+%-UOcXxMpS)AaqxCM6!F2UWM;1=8o!Sy5e`^f*C^Y5P7 zQ?=E5O-)brRXyEX>F3~CN8mV0vXD@iU@%~CU|?WmVD_|O1I6HAV6k;Lln_8!#a$NE zuG#_MS3mJsM27su3OZO~$x7?~d+| z{W!s5TgAn9)`%B}7LhdV??9VYY$-G>5LM}=#ew87L8fIYr4go9JcZJ7cI%O7{Bqoq zT&8=4aGFkfHN|LA`&Ial9(6Gg#|GuvPoOJ-4<`6C#R`&cU89}3huR!kQfOPGk^ z#I7F|y!CMXKdA-GPq$dy__u!e#Q!*SF|*C{QOdYTw!C!N-_xClPQ%3k-UcL?s;a7} z`I`4fTnN7Gvy5Hy5ok#r)`F4Dh^(8EjyH6NLbwK^TAzbODENX44y;!o9VE8;9)=)3 z#Q1-X&`>aJS|^93F^>KbZ^)cj@V=YV2=zub9Qu0N1$}-+3JN=S{89PpX~lDy6a9uC z@aBjtU?2Q0q`(mm`>M>h*qnNS9cKvX9j@|w|7|5IIt6CnE)ET0eI_~y_10sXP zY=D4bhlKB!AbNL!0t4Fu3Blq5^&JZ(F#T&6-h1Yx)`U=Td;I98#&BmOWHicmYPmUScx+unzi1zo81wioII`sWAAwY^r4q-arDM==+%mQV!yJ}OL3{`Th&u}FZQnS*;#`ZmN@E2 zEl#(%bREb@V7CGWFhke-#mso(98vb

0G;Lp8M$SSz89oOYLh`I)ERhj}5x7%;A@ zi?cr)1$Fgkw*KPi`E!6Dt!M3BXYoCe?0x}Gh0Wkl=h1C-FCCOQYg65c|6q*8<4 z?;+d|Xm=RD7}r9dfzWp8+`i50&ghaJMUT0sR(D`4nTL<52l0BLNZediaQ?8&tr$f} zE#zBPUrbQc37~#k93j*`p_xGeCJlZ9yzXqHl;m?XEa-rx!AdgHhJ$ z%`QY?IAZ`NojKZWYRtxK0>a1ZQ21&o>OvGFPaT4*m5Q>l zd2Lbi?_adGfxHf1k%lmFxCI3sQsx;c*zU^?N!~~E;z<A<_krMqMc^^}fal#CN^G9_zU^+Pl3h{ZEcJ)}W%XZMe0Y-OoQ8faEWmUF$BEi^es>)L9~n zD6(4K?SCPA872tQ(6sm}5CYpW4bFM**>a~YlKg4nYx=Aj-(P0U*q&VR-)EWg-xFqT z!^drDR6El~9}Pcp$NM`Z*uQcKh{OUGL7?{63GGKFIfiQJL!<`>%G>s{AcbFsyhdz4 z|DxPzXFa+3&U)m2YNWYK>s%%rT^ev!be92XXagKF5@S{Gstv+Y{D19XQ@-ICFj z93uT4arN6A{a2SIFt(%;)6`}-pF|il(dl`{gQDNdl}3kX`@(Tez>a1-@tBfy5KKQP zXf~Rxs6S&E_nH?kplZz2U)q@Fl$EOdkg^m4ON7!v8!^}Fq zqBZNA&64jhXjGZry!g)|CQ@P1X;AJ$$Id3oxUoOJSa%%$5_h0g$3{iKD~wgyA8|IT z&*x3#Vzvfn0Mg;`DqpEq5osgHXdEcz^5&kxC@5$J(f3K|^>15`q5BMj>=h8zMW80X zLcH&Xr_|@~k8-wVe@(Mrk@YfwFd1bliGzu%62RN2MPwRb3u*ndOcUvjdi8sI&9R2{ zPj!|{Djbt{g&iq>%Jlga1&xc83whISH$sTND z!d!$dd_<*6iPh!Y7*m`KrH>`%Md;4|6ECRFm9Nc&6g>x5c7{Wu#mnuPu-zwqu(_BJzS=<-$&ztt~+lMRp-soqtEe~MHF zL%F^VDzTAZ=W$0Q27Rd`@(;CV)p05Is#%hEq)V591{R0?p5V#z|2ff;#UKTp1EKcD z=G=RQ+3Zmv-D~cLI3E$;r!$*u#}1A6I;vrV0W6#$_UR?Pf+^E}9=bQA)paDrJX>R# zZlO1Y_Rr(b>@Ao=)=R%as)to%uzcD82Z*5zT`<&JirM=uMpff{7jOyS$pgTCoF+)EG<8&zOLjDW<(@3Qkhoey}x`DZC&*cfv4Xa92z_3e^b=|(d*DUXs-bjF1KU!*m zw=djXJy}2cHxMQLT#nLO?rRMLyq}-;YgK;X%3|JoGDG8$*WdNXdgdHPrx^Z7XTCac zNYk$hj2=Yy^`s!#LiurhJ0Aa*dUwBX0la)Wo)crVcQm_gBKQq`6Wc0Vu1F;NK=63& zp<3e+_T$L{b}2sW>~_iUM;9|NueLk*bQpb*OCZv7YwOK}(U3?JmmFS~NodRJT8_Cc;_L;{C5 zkz7}QPcG{74}m0dU;S|mI+q!7dX*wD=$}#&@QIJegQBxXzPy&W0~UcOw`0z*^$B{py%jmCc+uW(o0>?y-Wm@K zrW5huro@oc^S#Z;P2aXi&Yf_U(+G*D^Ve@AZ-*Cy-$;bI+6eA|wc#6`wv$_1>l<6* zoo9x(&z7B-N*PJfvUb>`gK4TRqwM7DNqc5>!SO(qd>-HZqQvMLoPis+gv3x3%EaU# zs2lPip|D;QVB~rbJhjTzapu=CxO`?ej5uUeKy`KWQJh{^n&N;Px7UR!Q%V_1+Mu7e z?giNw_`dzRaZEHI0==}CnGc+@U0pEVUQvi~5|1tR&^O64{DW^ak`dFBt1akBf`fd} z;SQf5Bb}6$Sd!o*BciFHYpRwV`htpdP&^4Q4r_#-keN9vJ{eR4$(8Zye)4eUvyn%bhN}&i z6lXcPO`xJ2;-K6gW6YvaB&hZ+Pa;63HL0#xB?2VPozD{v(fSc6ETgfgi@Mqmw~+-x z8ceEL+!oV8?MBQooLQF)WYGX+`D$`yTonfGazm<%&nI@Frjs7B++u%}$zlb5`Gc*? zTJyOg2W2M#C4;6#r)iJ4dGDtHl^(CV1ldB~Y#^L{Vk&BHT5Ha24S$ApOb(-EG>jCM2!Q0a1{br{Nnzs91Awz0bi!wKP-maUs%PlCiO_Y9AhT{zjn`Co^7DRp3YpZjPRkM}|$2D=0L-6cK z%Iy~e2GT4Mu>@EHyeJsCf6+*WAEle;rv3>$Y)oi?u2EB@_X>$tp3F=X(X_9Zy_Dw% zd>6<^7;0LPErVt{LP9%@{=+_nn#fKOvJR0r6}u%ez7yBq*<(jeSimh@2d6LcY1TyP zt>k&S2sfrzd(XlYetNfT*~Ff0OaQk9Wvbl<=vJR+15%#Sj!QvAd8V}stPcA^KqD8O z_N9J(o4!anEInNy9}@$lzI=_qAEbS~iinfO3yO#!fIB(Qgk~7jc3r{oDGVsb)&0r85(PT5ij5|sa~iCCh!pQ zKy60ABMl+%lnswoo_-)J4wQ_3gxuKj^dxQZ2Xd&ffC33$9DV`|A(rko#9q-6;G+Gs zB93~r4jCaHHSp-l+d5(&s@sqf+y3GR8#@4ej!dbcffkzE2PN18zWeAK(P(jO)72VJDEhWy9{5X`p?Rja+d&;aT zbszd`74^0ziGkil!T*P>ZE2E9_*_x~~o`EM*Nv6oaS$l79pU7$JaMGoe+Y zBJ>k**XeHv3r0kMDVAhcvQ!BGApp?ku2a&z%h~1^T&dLMN(PiuC^T3d;#sP78VKz+ zAZLk&gjk4_N-w<_BN;LcYFN3XVLf51u5reV0>>;;PjNE11dAs$%%q(_H^DNj+r zSwBra@o1+B-+Q4ZoMyIm&j}5zdAf;w7^BFR$aNAh9awOyTlmcwyL+6-8p|>-K83w} z^Wo^ADtA! z@l8xvg4?2xG^MVzURaFpAL=d4h!lAd?nsX7?Eo6)4a5n->dy+R*v=I>Ny94L4IBZb zl0b{Sd=$M4x%!NG-5Y-OMGfMQ9jrseHvOndn~Eb^#bC;%9ikMAk>{%C zM%Vfdc+N_YAXvZLEA|a4FXV!9;oA*lDypAeROx2NRQte{u+Q5nU+(LUO713cQ{kXL ztkNn(F2iq7l!WaDU4Y>5FkGyXJ_Z-CgPd?uy+-s`0-2L`(!@ZZA!=_$Ni7NliuRp`cMOc$?YW^`<{Hw8=u?gD5QdBNk zEg30#!SduQF+hyD7|a_ltG_IOurLa@x7VAkedyZ+T)04jzLiS+5y$WmThbTueHruV z*x9dxL;5^7t(dxJA73cA&oxybqS&h21RzQ`jLDMsD?wCyW7V5Q%7I3LsdLw#lW|)W zhIfSYgU0&}joJ3I(}qUn*A%zlqKLpB0YFE>02r0;Xi*|#b!A0x{&Typ3U>VEYDljTv3gNb=6 zinA}Ug7wkuruAE#ECw0t4<*(FeK;J?UL)$AkKmVlMk_%jp8GpFV*Y;WbZ6FsIwCD* zdoj6ptK!R!xd60Lx}$dcS;#CrMG8B?i4}{O(qJNA1Fvpcb9}^tt|TTaRTwS+vq<`r z#`{XsK%c@%FFg4woR=`j9PFIbF1R{foP(1K^=gR$^0!6Tvc9t zcka4MivdQTvm}#=rvc!mswRrhy2nZj!;3z4#$cciq+sFib>AfeM0^bSWA-`EFA%)e zsuL*4vab_Oye-Fxhj*L1QxPW7kkn~>xYLJpKPyDEi(%!M^qrI-^}`XY7ZQf^@Iy`c zNjlmkjP9a-#YQy#&CZAuTuu=^@>%GsbhBdpDw3}Dg!1t*COzt9Ci*`D3(G@(Yo(f0 z<1PT{W_{&o^qo*4pYM`nq$obFbufXyD4p|aX>J}6I@ISK9Nzdc#P`8R4J~QmJ@Qdx z0i#QQ1g%*aZ<&`4T1zA)Skp08v=i>B!_<0P&(zd)S&u5@5$J|FM-lQ}u-Jhz+gHsn zG9v3rE-d$Bwzg0nhtm_A!~b>@IN`DPfm*}<$)|o^Me9VpCRXz}(UNZ46;Lm zdmZHCQfh{*{diLyK#&aq@}37%cQZ>*1$y`NGv6{CMLghAo{87Bs@?_28Bwj0gs{+8 zS3M{DYh(#eFU&N0OoFGRG|S0K*y(TBu)KyhlO+J{d)pT3^|D+uiSgWx>t$}oo;pYA z)xZBeW*2yeaFz>9&>A;#70mvb1k)u3jv>B>~mzd+GGcd+EHk^9cG)qpTnXX`OBt zl8^%|ZzO8$zOYafS2-Ph3eEgdn+KwQ-R#3pKt&KQ^^pM;c2+Uh2im}PqH}f+p=zCZ z5N`>}(1&oqfvgtGTTj&Q-axhZEGjku_Rc23DP7^8Zgi6FZsz@%g!Xu%-SE^cG_0p%WDw&4^+$c>II#4w z)2&1~EHb0G=3y3MGfE0e0d{1wnM$m*h|YkFu*phWs1j_hPu72*NN~~vm^HRb0Cs{I zHoT0S0Pdm28!2ko3Ff~jhN2wQ=~2xtBg`L)QTul55oL7rM_fk;!0eL}=5QWW?#Q=? zh1|oDmG61ZxU_!Ex0qSnD`IH*rGX7#k7}_SnlBsQyi{C+g4+tCg9*zjJ>? zppw!d+L++LIlwSLjzR{cQlXmB$|iVfrROWUL0uXHR4^4#K(03Un8TmP=}38?J}2?~ z3O5KdH7xoGH>|+89?Gr_^{ygs(60FHE0Z83Z!ywKB$A*UqdP12(BRF8mln(xK*KI_ zOU+ru%{jgnmVUK{YvJ^wSem7`;<;LOP~zd>9t#(L8}sauNc0*=~e zbKQEoI#;xVWO$Zg;~a0`Bsb^V;hGA{Lt4`)9qlNmVh|S=?7WZvnPmgP6m5(z<2oTf8Q2W9k2*BS>tk?Hco>g z$$Dkt4H)sX1J3|b{0~8wuFP>u&-mgfSh7%uTY)P3nck4SZK*Hp(*HAEky!HG>$Yvh zb?zQwA0A-FvF+d245sL4IkktlR;*w~3oIKBTSpPdV?H0Z?1#WljPlqTf;ffyrz2TC z-Fl6+Nc~&UYv5=IZpCeP;M}E+&8nTSm^%DI`XQR=9MOZyDgI`zzmFEy%oB9R{gw;o=hZg)4PwM`020r9O*Jo!aM1nJ8(yMOB@`9C_Vu?Qa(T6vbhN`?un zKFlh$j1m@xNp{~Zgw?a7K!Cz}#LFIvPoB7HJDRD(650-i#Qs?zg0_II`^Yl<^6cHc z-zW8lwE?(Gvk`H-5H&zrEZt0!8|B95cB}N+49^=QGCJ1P3#9} zP)MDlop6m7rbpi-heQ{8+Bo+!2j{PMxL(V}l;Q#qo(tX7Gv4n+Ui_6wT=yK_- zfimQ~et?pf5^@bYS$ki$4FPn%sj}kdJ8GAIqm>{Nq&v1L`4`&O*NMMqwddCfsr^I| zzCosUkblllM@DCj9cX;B*5VsnST0?kXPoOS)Oq(WOa3LvwQvFmZvPsQA6GU`aNQQ)0uY@P zkn{6oCMKX0T<@U2z`IcOn7n2d;hMid>arGN%pt!84P z$LXCW5d4eDd0v3%RB=c_S#lJI)KU5lpM3x?2bnEe@BPI$&d$pI53fK-kNaQo?|yHG zm+x;geh=3>-)a(kw||#^kKXLtW~K^&P$=lU65kxJm9gm2@FYOvp>%G1-I{$Xy< zxPZk7@@`ncj_0?#meP&N(v8hk%1c_vNb#%~`_Qpxp9Doyw4NV~XA?2W59`JnRxrBqpWS#nvA8#bF> zY2ynrqGsUP=`xJ_Y>R<(|HUE;y*1Eq>hU~Zk`)wWJ2(JlKj_krZvPc>$;s9ffJckD?;vPw$coGXP2Lt)9Ow4!_8)j!Jm~B_y5T=q9*FPv`vw$R1 zohbq=*Jh_}vV|!QpUhjR$4k!S3Q<|JP_kM7ux5#rd$W(U)DnY{*s%*Qdju#USN}kN zqBlI2IjRx3AW!lv{<^0RpQuwX#PV8~D@cm6OHtNf{`F_Y-q-UzTQ(>gzYq(RX(sNdy$dstr z=>6kS$6t>w&tU7%&)QY6%i8CJgq4qSLRV^TP@@i;H?IY(-viQwyx1ieB3&C19PAM2(sOE1 zmxBQA%}O@&9j+@XDPbxN2RY-ZGRRq;OdnSJ%{II8g;wvw`j0;OYw)ooTbue+k#BnF z^LX6yiA3nzCNSXr?)_t5{9k)nU*hCZf>}_*YUd`vzp#W$tuigdB5G#mjVa;h&m#fF zV>8kKZ5D9T_XXTB>()QA`yfmKi?As-|bw@73p$uN;E?S#N6mL;ys^uzch!E7KfbE5J>-t5Z$*xf$?RaD;HVo>FiTL);bnvX*-OHCJLdypa z5cL?qf;WFfN<1yRVwWy!yUG6r8=-}}tttXg+ zY87_VLm!0n6z^|H?RdZs|L~CMa!&}ZaG%L3q)uG{&<<3G`>Uj0fysw5QqeK-3!4ry zbln}%JwtKD^u_8H91w!9Z~aTmnYLa)gBLevV?F$#EaAqVl@&{V_vy>ease-uc`uVo z%=--nn{_{V+cg*$7_5i-(=B3^nBb_Whx%VYE(DaMXy|)n^0cZvJR~EOmi45ARD+CC zBS~l?*syfc&HIgXKcB9J#GaV(+(U2wH4po} z4D6TD9s`X%{#{WS4ERDSm>8MwlB6V(Gq6xhE_zG1RmAm-85{U!%wPZ_IRuPFJxJXz&^wi0JuKKH|Pq zn2=5v@oE8(hd_@Uw~*j@`ooi8|G`(<>H$(RLMrHV&auMAVwPzJ+4AytvI&$d%^_G# z1m`3@_`-MWODycHR`pMxS7^~=%a{ikD5FoBZgO0}blm~yivxu?`4QO1_Upx%0Put| za$@G7`pEdGIl#L^6MEIss7F$Qda6m{A!=R_dNCnAH!wL-F;8B3?W>cDc+iwI?GV|_ zZvB^)0Y&s3xUast^uPuq|5$GH8mUkBceV86rVdV$$Rts2kr~WuQ2CLPw52LTKv6+h zP8Sue>|Jb2+0E0K41N|rr=BIYh?T(L%_3>MZ6xy^@F1iNf29IFYf^>%#Cxd`q8<0B z+HvZV8=(8FlKKkb%`@0rf7jcYO3pcvet6)c<0LAo&w0Zk%9|>Zvr51ljQR%rW9w4O zwRT$NXRwa%l>+>45&HWvUmilM$ZOTdsvz{0B;{{ZrWj9ut5zcKHL`7wiceSO-1%PV zz~jH%5h%=v?nj*N!4_T=sr@v64Y3wUT~bnNc>;!XUKnZmGyJkJD+)lp)mJ%$L34(d z&KuoInUBDs{Q;6(fUr|SJz{ywkRu#7ydku2K3d7-)W%)KTydSunWpY+lVo+Yni-?O$0cFE_t5gS1k@nzLN80h_D-zC&YDZG0+ ze>%KWg$067Tkd(&ZzszM#P7~o8DT%!Q0h9_Y%%}1e!tCUeAng<{}#&N6Sf}4 z8vaO(BV(~@wQ3fGp`lE$WH&CADSj&4aS zhp+TYh_+Wf-5HMYQ#lb27uLK~;ZSYN|>14f;fS+Ey2aLQET zJ?%)%@7FqB#5exkTy}xAp#mL6(Bqe#i6Y;Ni6XxlpcXn4r*!X7Rr4CTv`(qIghu%> zkQF!|dgv6*i!!Rk)L>t`V{*C+gPF9aNSXMX`xvm~5UZfhQoe8EiDi!VN+W%cY`>Yc z3bZO;REi*WP^ne3J2P@Ko5k7(Tqkc#Y}F~jd@S955yIFEoRFD+`CL5^NSw(xIDjEHj{Yewl%#0F&0Cb zwZrJNcx~oV`%?ySY}WT;!(*t;4K>7@xNX)|K4V*jR8$Uo&%<2UU~1kIE|ru-gEOHoFe+rz3Li# z9yNIqM3s%LxGiyo;JgZ_0$%eB(UgJ*&GAkUCa)ts!X5Fsu;2TfN#J6DXQt1eEW&ps zy1ua7yce~A(+jOU9=pTd>*e2qGM?vFX29qD z+ioOCDoH?SEayRYzrg+XRiGXDNu$27JLVJgjsAD zN>qL)@|%R5lQ&4fU~rX1jsC$05{K7^>v-m!KZ_(6?@qo8U6xo#*NiF!f=olcFul7N zT>iP^`k0==*hQZORi)$oK}K*JHRM!<5}=nIR#~1ehgL8Wl1S&l`AEi5PZHlEzo@tV z;Fh%i1Q;^P>o9W>qs41gY?kD;t{A*$5x*UGis5e6%*Rpz8dJ9TuGMk~54oE7+Bv(J zwT+$Aw|wu#SY)sTD8nv(V=-KP@eY#ZLAzFViOsd^A#CxPV2ANHLic0$Hg;59v4z!e z_K!9S-A(lFi$!rAkDS`q5(dMW>AX%gwB6R!5P0eOvt78I@@YKk!q!pfalD0wwIjfj z@t1Pb)xrK0FePW|wHqb`#r~5}=iH+Td__t@bGEF_j*}TCImAyPJA zHi*f3)>yR4&{P6~VNBqRql{`0WJw?!W01p(?J(;z0059yJ~yLwP>Y&~a9n#esBU2fs+i=m-rw|(i>p#Ol20>p zGX&)kVp46^qI8ck5?83{`l>}jZUlYJ3)N8<3@nHvITmxAFp3SdR4-2c`zn22DhS!#$>s~Z zTZl|a&nVM&CU-gJ*YO|R=WJsmJk4o^SR;V3or9(?HsDS&RTnKJ9>QJk4+o3oQz@|d zfp0%YYzH|+=?g*^dVv>&>)u{Yr`1y%ia>sn&2v2;O!Bwt+5s)$Pe@dmEbY%jh&kh$ zB*G7#X6-FV?098kjCGWzF?DKstb7WwxAGid^!Kxf<;v~X_q7#0B^Gdp4|)5BAg##yy!WDuORG;}LMP2shk z|Btsp^Xl|56XdArcel$+;qIH-Z(YE*?3bspV#C^tmzP;Y;jIn{a@DUSK>57Jk`x3w zBRcT!@2`^~mk#}|_avS3ufI++hA+1Q8y=z{j>Q3;JfBdhAo%#djpCm@5#(GFyLGif z7@qq5(V>-MZp{PK7#XrsA=D?`NxR`Q_AuJ0qKk2P_WEK<3MeSk7BvxCF%SY(O-!&| zMziUluJ>*x)Y6Xt<+39(6aBBx3@F8osakqandq3RWz12V8k3dWZE|YY*Kdk++G_xNRN=oKS&QuJ!%fh{5l6);NU-yE zDI?`H7|fM*hm)0fh$0?D<~V4w@$IAfW#Q4ORQ_b%d$- zNs*&V!m}C_d1)*q$y%!gj(LD)wDS^@ZLxKpZ7S1voh=^RIr^zk&-s8ZVsh{`B5Q%@ z@a@S}2?>0Hu#ZW)5XyDE`j04h@ZMZ7xu3AvltQR9dcd8jF4#0~F{QPe{Xj7}Kem_4 zCN6gOkW8gz8H131$&BD(GUuXoup14kno$>sI|oLB2~t#IAwxOn!E<(cBZ#|*r-qgF z%BdzC$w7S_<}Ap!alhW(xN4mi;R zMe56{C7DO7PNaCvArRtbBOSnjIRQk!xTX7AG&5lrCrz-2y zD7y$KN|DzS#?=9CwZN>DJsahR|S~})yp6@3SvU6W;s6e&9w3jfC z@hI~@mg89=-XeEof+mx1(!o|>psUET9$M>Trl!s>(hT#>1X{1-FJ1{YJTKiHUXaxT zYs)B%Wf%gLM=nNl?)PS7;{%tx-Q+d0nY&`yerjHW%lQ}zA>&dt69cdP?{|LG2m0SH zzTTbE%A3)hmBQp>PECif@7k!(cdd24rRt4krF~oXp{kj*+D7sFofZYg(Gna(qREgo zPU7Yvl<>JzIB?PXl?}lSp1Xot{#Uo})ayUh4Gn}+5RmXIa#?YDA1h#=KyR`*K)Z!; znf}k1fjgbs{74Ng)(09XCHj~hTL@2JWJFn?nW!+nY;7vb4Zb-*JhBsZjsBWQyOalP z`jJ%BOq%g!$n2~Ytf`%IftaUpXi8N=$sD=8H=2Y%Btj$mzaFd%_T1~}|5RW(ixRMD zqx4@0NQ){zQGWtC9Iid=H=Snr0l(J1a>4)DJj_z^5$13&C{a!0a1PX)K%BL_wk|0u zBNvT&I?Z^w8TKa`ha3|eBN_vYA&+s6VU1CZ@x2u}KEUaHBMRquTke*e^O_D}+XM*1Z(&HRc+HbC-5zh8^X2wtBkdN{-eWKKsE%e z^w=43u@IyqPyAKGR{HFWn^|A%KJx0SyTTBj61eN_-F? zUDv;6!G~toJx~@h; z7oD=C<~^W3n`cz%hB_%;F+N^Tzq?uf4noMv_GeK#Yf@06hP7(#==8aLfOvgiAMLCb zr0tv->C)oxCx+wJM&d5t)1Jcc1aWGNRyW)0&?BB=5g`$VK1OmmMj@RogqZx=a*S*3 z1>k4dA?!wIOqskqNaG7?iTYACmx7OiO-cwVI*ys(-p|q6BJo;KTjAUUD*I3?HfQ?T z*39?Gx}$YG`Gz66f%;36?cPafw>UTV8!htqoJ$s0G7#k&GA;|tpPLGXE&akazIBs2 z%PER5N_~Q;uH^#zR&S`2zTh>25+E$Lr46hVJrj&JX$U6m^uTT5Vf#i!-i`)-`Adg{ zvn;FY$vCTZ{>%cl>u2PJaL?7Z2pqxn!G|qY6*gX8dYBY($A_c6d>7&OYj;ktonB8MLrELet(niN3h07mLunlCUsL1gDsniAWIjMkef?}PB3I0yK zpJvn?U|?Q$%&uYN+j=K9-)LeR6Wg{Yw(W^0|IF{6bI<+G`g*PIuIlQp z?s{tPr*`eGopJmn?cxijq6`=~8VDrFR}c^oVh|u&&RF{w5Rmg)Ofpb_oWmL;Tqoi( zfYBA-wg{gQP9&3H^|A%s66(BMT9#JS0O<$)^J76WJ60&GyGNhJ2M#$yHjrWXr6oVxg> zVK_(@13_04h!w5=9E=4Fpg=usqz(=O(u)8Bg813Lp0*4g4$d~l4h}Z- zo_4mC8a58Q;%M*sWgop~;;X-dOyUfd)kk1VbfVDIsf~c@!osLN;Sx6$drm}=zq=n} zdxL4$QXr9*v%{t{xm+$M4tD2hrVAoEla zxygf%ew5p!R~@Ed+~V|_gnh~kM2wW;keobB7o-tcKi=^Yb=4^XT^wS-IFpYp4LSWC z4AG^P__GTvjf~fM3_)b#&MO?CxKlW#!T7pyCyce_ctdigfkRc^ME%!6U=CPA8$~Av zps?&~XM$m4O|Ow;@z@9|x=oAO;JHU#U9;xzZ&-7&;v$AszrGQKKgzS(jLW4mFkrBf zHrt#@x0H8n>sG2#$Y57{r76lMcC7~)w~{%~XsSHJPv)w5xPgtH9{idDEXhibfO25x z1Sar!hQqD7yUE;xo4_&4YX)#(r$9gv+8?@A9%-K&X$STfK|fdIMGG1RbQ@MX4%UWU zo!LyNlo@Fm5O~)I^G*&G%3pmA#mYV~n=>uUj)`nDY6%~u1&ugeX{q0nU2^ql+SBbI zr1=fnss%^gX9czN~#xP9ln;|A!vQ1sqWwFI)Yefi~w?VDf=>J2WlJ7EyL$p{0+ zqNrP9@vS&ScF;*z#0iQoxIw=Z`^yVtyK#T|L4mH-4FoLVr zk{`uAF~rb_yDc)lEat?^t<3bj+y*=eIn0A36Bvi`8>8?TQeJk@%*t~v8w|88g0JuKl9=xr+^l-;0C-?c@{to4IGSN)>+k;^M$^A|L4?t)~{ixP- zhoZ;O%WO|K2dh1%ahD(HxdDaI{f^$%RG8Us>?w-ASYc!oJP1bWRLb0&#`3T2B(VPA zn9KTGbS3^MTo2C)&Zl=OBW2SH#3fF_nL|?htg*@SoDrErfX`%H{5)?&=3%XKI%UHA z8v~bzhw^g2kx49ZKXSACnMXn~&#o!yH-ELe-w2}+A*5e_tmc-q7UzeA7Nx~je2cX@ z9-D!?pE(T?I@Z~R!&vF|h{S7|bn3Wt2r>L&JX8WV9CvMDDvZIONxzGPObvCtL4Kwa zTZ1h)8;z6+NF7;3+iI2@gdyoa%`?Q(cGPz_9lXc9nh?Zq^vf{Z6m+LRsc$-uuxTjU zP^bTep0Rf;XM7+B%9Jp<_VMcm(mW>tj7Opw${o)iTVWXLcPC8AUHkFOR8TD;9Kw}A znk*zhzF(k?c!3b6P2bVU&aT{@z4Z+j1mxob3`9`|NW}^V6EA1;OZIa_F^8xhfu;EZ z5K`!8gbTTnd<&!TICT$#pJ~Gvlx**dZxJtB6SqmE`ThOfCqJeti*>AMwI~t)p;JLV z+ak(AMIz@|iYFt2RbDh$y;XOO8TG0nHNyuPStetARcm4ZLyGorNuyq)k~}P@bPirj zXi%Ag=I{{tyK0EI@;Oe!PNCzbBtj7bATPn1P9|6>QMSi}ZQxCmMf>o_^*HZSO_QAC zAA>dY5cQ|?g`wO77K~i>fhpfK?r>=X5(|>pHF`^NmPoL=?{}K^L^{%keOcvld2Q2th#N%Ylptgn%SHhgC+j&@0Ld0nskX2$8SQ zB&s2S>SgKWZX4Bdjy={W7O~H#4If=5l#u1J)D_@(sB6>D-U>q;VZ8P3s;beLOnpsJ zwAy68gOP60+1(7@#C0@Mk?5nILME<-=xslI|FCYl+$h^oC1W5((um_yk0y7t`i>@U9#-#IuRcL`#Nu zul}rKZs2W&Ohvt^qw|5DmJy`Y8roM9Nl2zFcy;TA&@zvI{PIdxivY?`;QKlhE zR4;lRSuzJ%r!u@QFW^iPMn;b;Pv78GF@~^mH`#ve5hp`ne6Ra)44m zr!#o$0oXw-o^}0&`#Z>MI;Ve{^*XF`&=>>$^01dRZTe5{sOH(elI&0-tp)F1Ut%t; zs;O!b~pXA~e8E$nj)5!8cARR0)r)vegHDM+*2yu-CM z7L9%%aA^sdQO>^2xPz^V=wFpt<^dHA$7R`GGnx(Gd7~Foy{{C544!5CZKTV|UXTrx zwFKD^o`hRsiKs^b3N)!0*$@{T;UeHlOo%gB76-C2r6Cb^vC5tRywb}MH~VVM^9g{l z6a!*`^W~T6L8Hj2`S(IrFx&oPsEbqK4Q^>LmjY}<_&r_ZQjK9Xq@P_dzY4#vUAAK@ z^@G@8301Uie1)6Wu#t^1BCQi@Pq|W*mktR*mRhhrT`I%eSrRDpig_}<0DhgCT@)#CV{iZvS25ap}b%3Xq|};D7h# z(e*uO9e{T?*t?~_(dGI3Vlm31TPh(2joKj_dQQUhGQi?eIJF`f;KpXKeK4u#UJFW( zA%oU@DZklys!@+Qz4`2;y_FL6Q1vQ-d1d0{Ls{S5hH4{M%Y-N^FZhLh_N@>=mWiKKN) zz^4YixD|}7bffc8-j~PZiYI}P{Ap|^xG13xdGG$2Kw#xxdFlDJ2wPgc@?&D7=VAXnYzqK*BZae5N(>uD{6N@SodhZX{NJB;;)Y*t z*7hU}NMfA3no>E=#=ka2e1CmZE1NN2(=2q<=tv1*UrS&nU$`SOTVHalf?9QBFf+G( z4v*MJpR?TRJ&|m6zrQ`bP6DUnuJkv-fA^2$a@d?} zC?HmpES{b?J=`~VYu99FYbaza&bvP}xSYZRM;oCkR>tMxtgBhUHXU;K{e54z7~;Au z=Gq>W7##W>aV=D#ba8RGCnzYRP}+LcLxgiSs8P7jg319NkRl8I_^xV6umVB5c;4aX znTcvJW(mWR)h2}m+zKd?HdP^TEZm#ZT#06*^edvyL-Z@sRkYPnc*TeWevf0*Rj&_s z_F0F&?#1B04jwQR2q!2VZKkTsjiBamNsE|?vEaq1Qo+@>XR_|aRbz_(eAkPIwCfYu zH?Jjjy;BExWfsRKXopgODv3&W=}#^ud|O_e>J#fcZlQ3VsjFJ&nf0OMV4~E$;$rSv z=eyG2KXEgO&QJ~1zs&aV!0bd*{DGsP3&SPHke7=REfETZI*7rNrrN}fHI$xZ9AlYl zJ;L-m5naZl*8_m0E{=vI5A(3X>i%93?;*?6*EX?JcV$!@*KY)s$vbU3CoM+sHF%tPiHv zAKl>L`m|n|g(zHCQ!5={MD#sRtfiC|ODDJhOfn19LfzG3XM3Il&z@^NMz0fbKb|?I zzyu0FUj`MDP!()ub`?LqthBpZH?m?337l1*g_ZR|w;4vzW20}^bv#F+#YC=aA@8hF zPAHAmw8^Yg#7H9mMy8_j~BV6x^ACgfx%G>dKq~mnGIuhWRz3;EzdN}t2PTeg6p0h zvU2>Td`NY1R-cfTi}R~SqIz{ODe0I}DHRtU+wMUcj!i@mJHJcIj^p@FY`@=ks74R~ z$}i=oZ%0>&x=(@JAF7vO3BpqtEmu#nzt{Yy2!f4Z+58U8Gpl1&9NJO41WdBZLmBam?{P(DxdM(X!Y{H0W)0>D1B_ zh(puxfPHi|g?o;K+r4g3(FWSEn`-Trz3l{ORLiwzUcI$QJ_NiS-Id z7G8mKktG|4mlsZ73|&s=%|p`xsx4o`=mcopu0ms`vO7}x5kZj$GHH{Jz2MD<-|)kl zDerdW8b^hU(%fy6|K!mFG|q-I3p&xYd-Up)e)~laMJ)51T!RLsBu?`Te-uF;1#=jR z6t$N7$K@1?l{vC~!0nvgi>z(aIT%-(S+%-%Q6a}i@+4P&%p9hwRk1|%9+d1A?jT0X z;W~|XgnCFdPU9Cflk^kI8u-;11mQWn2IYiY_?@&~PZR_-Vn&W(0IJj^jK0zMj=Z7u z^J4(s!jLoVtYa8fF*#0N?k{nF)}r7E$GvfbBo)Km7PWhn4VXckX3+AXLYB;N#|ouW z^{-qeoh_eA5p$fb{BcGWKz`QR7(W=Pl_|X2)BBH9@VeE3a-{|5J3jQ}@lMqS^m}F& z-EtDwFH<%3umKL$XJr~_To=u^6$vQmEq>E4BjPo@BMOhX)7r+BN}@?rCbyxZSU|>N z)gho_*~UtKC1h_I9<7-xQWz*6{nUyv&A2J5u{}K0&>K#f1E)R`qk?5nOd=gI1}8#& z4ZGkxUU^xIM|w^y%<~NHuUbF<=zXf1cqQ9qWIu`6F5pdxN!HnB=P;ND)*=+nTfFF? zsQo@-*}4*Oe~Y_qzx<4(0|DM@sTPPyN7*>|GIZ{8rph%M^Yaf%IQDLQhn+T3S zOVnAeZd4i~pT1Dp+*mRbo6-0ykF8fqJYz4G-T>!!9sgu`!*zDVKQ4qL#Cij`?fT-= zWMe0I^MDX1?(y|XS|_jkWl+?Js{GV#&2u!H4!Y9S!ra*j(~ZY%;UM$nlyFBQNkO%N zpO%o@mdISqDzf6Rrxn=;(&9)BuH!Nuc$Z+jY0gu9Kb@b!wPD5%@qmG~T;uC@11#E~ z(!j}Mi{2@kp;D{+>!BSG0cykB`rENES+0Ar-BDXz!yV5yalzm%(dIzlriQ07L0Ub8w}DX+hm zm?8Lsg4u?{^ zU$VZfn`*GqSd?g1!0WNSWYI6%?8ROJzha|0TP8}2<+Bf5nYNBefjULdIBL}qi%8Ox zhH5N3x1f6VdW#T_2z9e?(}3NL{qv#LPWw#-=wZy{8lJ`T8XmJ#e?K+-jx&-@z2)rk z_$NwH`=v0B*8UMdPzhrbJg#OmIDmuK_C*<~j1T`1K z@O%z?P}Ro8yizVsHw@YrNw9Ws)ilG4QJ$Iy-8Kx`jo;O-PVNS0uQ0$yO~VaKRI>}9 z1TCcyM2I@-y(cC40%DXU$-b8p!b%G9$mMlHuNT0L9e~H1#&YUAUA&r=mrFqHyo5|h zvi5Yl6zpP$GL<>hU|14Z#rrvy?Z}pJueANyF8_U-n=L|4=GGG%0Ve|8a40pBNY3m= z0-Nr6i<*rBwq(V%Ui8h;>**U@apoT2HqAOyk(-ahP%XcL^hK(=68;myh5mdodDZdODG9xFm z87OeP5u#lP=B%O}ts?&!j+;pZe`?Q;Vp1q1<8i{GG|v*>Z#A^sPf}l2Y(Z5@nYivV zFyQbxEKnx$0G-PXd3VYJtJpt5IEZ#9t{xS`C5=nbxU4fuBp#nBaI`IWBg5kxQG#tP zhE+SYWk)goTdi`sfHKAiK*(x)SG_3rt}OJk;=-K-nZ3Orz$a&Uqy^I8pe}sckgU1Z zvO@Uy;*j2W5|aLS5@LOUqMV=U|J^6}xz?RqK5Z|8Vd(uW?pT(bQ}@t7ZLG120WjY# zUv_iD+cHaC{ZCpA$*UOJpND`lQiK(eWMxo=lLnCd!u$l6VrOv?RN@u67>QucvE{&# z(osk+N)Q1@+eCq4x@0vYFIPRNj>8CL`RVot4&#z4E4`|FBwH(mYD{Z!L4Omr1{fcW zRvDIRkY>*Q_LS7_%oqwMy{sbqJpzut%IDy^eSCEWdmn(S>m?YT%~IT!;j(A{$$8eh zfSNn%rj~wHm026tpJ+CNxl^(E8x0=4LU0Rb7;|v&UT~2@vBAu*k)QfGA@7*=FH`F! z+SPzIK8Strc!qItIy1v1gP^t4CU%Mm~Z7rLr1 zI*Wnef?>805wO61fZmJf!XU!_mX5i6Bi+$Z(`S8HEx6^&x4-riX|M>P=6c1kUT)kP z==2;l&F5S?goh}BZXZ>uL1;<+8DKpyiGelk1*_7MF_v_?sMsNF(@j~_5ca?eyz(bAjI&x(4+puS5&&Z zsR1X!s$=4L+Mr89#hgILab@9&@Lsy{`-B#BCU^1r7JQd_OclCuMMW=Ug4Q;8&H#Hf zJ&ZI{8XS!hp275vPwR|qem3uUC80GaKFzco3yU5FlbKr)qNW-nji6$O7(Ptr6U%vM zGRGh(0L51WGmNM*elx1l#oTjYT?l*M9x+<80IlMDSl#SPG&_U^!HqO??iX))VU8^! z&L#srXbI+k0>zs`1{xB9pJlDuyNpZwj(=wW_+hUghz=IfyhKfD)+(PW2sai!ptd~( zWoq2Rt1=5gui$|Me+T(9&61ltJitB@R_YzFDJ{19l~^EY)PI6ZZLC@o)@wK$0R@2t z<{tySxVv{2%2^SYsi;d|KMqaBZak!xjxTW%H_e8ojh1mK*ix1q-HsB$OgAOXwr$%8 zQ%91;X`hJ-mWlpH)MP~kL@_-(q7~;SJ)oxjVO8G_)w9i#v*MC{pu-$$zL%tc0?0C< z>+RAnV&pRot2QJDIn&EbFzGq!dZ1L?5EUkf5Cl0R(Jz1Dw3#XCKXZlk*rW*VNZbs zGpOSSvsjM^E5hySCW;_X>gxdL9Fd!bl;GA|_t_dhXC%e$YTKa~<;b``OpC2>5N-r{ zaH6jCe$O^I0o}e0YOVoM^I5UtI-sZQTxPCNdlA-_4?$7*>o=vrrWUnv5u~BVQk;iw z3w^ckP4$ixQQO@Wqn`|pI=g5=6qXclH96Bipi1gu6fGO4ySn*S+6^$?Fi~u)w?)^> zZT_j!pl4S~xhVrQ6gNm{obOe`o1m>LEDa{@?^U9DVhVHQJ|n}^IsMi5!z+leW54KA zjQ$#)oYJ&I>bNSM;~_Wne9ZX}W{l|^FlF1Ic9B{6RT!!phgHJ#oD*2A03^i3`v>XW zJ|@acc4I8R?x0|FfKg~)GUWsDG>Wk})TQ?A+&?ljjb);pcB(-iZE%u+Oh+|7+aW9o z(OyvY!n01jYAv*6mpFQSR;$tgrQwvu7%oq<)7QUJ8gjqf@=l$uQeJm&0&g9l9c=;4 zq@7!&qu=&-?zoeDIN|`LEx}33%N--&Wv+x6OgY>T1Vp+|OMS;cvuglxD-b%d+}b>E z=@H^whCvAOBggYnJ)>HPE<)(5?Wn~-IQse@t2h^9DE5G%WEJ>1OA5RtyuteXNiaz^ zDf1oeVv{C7d!(QW!*wG~s;#XPBgGa~m-7%$t*D&K4xD?;o2gvs<`}(yL(QFeK#FCDpKdz)`AKiIM_l& z!r*UxJK_t%zkxqWA#O-UwRgzWLXR9AHFMbyv`8kZ_iuvz$n{SQ~x*K7K1hh`NgEHTo$EIFyd z4!PauY0z!oNPP-X7JnqYqmNOEWd+0mcY5ke%l00NeRLg-B$_ydB`K8qGQRR_SZhuxC zq1^s&gFWKG_zwQXN8xyW9#nt1{Ld=dzqn0PU$4-i<t<4q}A*jsZ>FHIiZ3jjAp?0heg;Pv@|_oI6Mwtgo3 z1=l`Z7ZPE$S?6l@0!AitYN;IzVPu^Ix8!tR7q3YQ**HR>+IKieBQ5UH(o4M^*UEvIT1Z*88Hg z!Ofxzv%`s!Yk*HIkC+bIaT`;NYp*`MYrirlX(%mtW4ycoN?gE_I;?K6%2Hb!%fFh< z*)@bf5!nM@oM3`FNW($YiLu!ZNqhL=4BOH&!^`Xcg1MNKri+jwKsm6;|sT0U~yjSERI&|s`Orw~cgCrHT z6_vZ)ciX6}{B&YFFFliSi+5J)8QCwuF3H?{5}td7lS5)LxV1`~W>$n_LbPGsFiz16 zp%8yUm<4-k($ND5U9bN9nt$viN*0QR(Pd{d zBy&(dM&kX7GGun5UOPd1rWUS>;zsAfIxw*Qm`rfU33rVzO-6(r>U8cXvH%SXDi%nCt~l4VXa}hFzif z%5Tt#Zsb9#kX$(Xrt$y%6|UrWWdOzz+@Lqh_#C*~dLe(Nks8R>?&cW}j8#obo4l+L z<%~nU(G+e?DmI(VWZSZZpI#buMauJkFG|N`cmT+CXxJS7mMme^UHI1EXnPdBiRO8r zh_}zZ>gZ%O4&T=Z*R5t(^fQTUZHH0GfUc033tEpHM)3qe?uaBvY#Kb za*O+lsD5Bn5UVRvrx-iKdH#{IeD=FX&{HrOj@;0nt&Rxx_Sh=VP z6Ff7z@wPM>np{pDIYu12wAW!`;Uv?#*k~Zaou-kF6V#F-0-C0!i&?$z_hv- zO`*xZKs(jpjdXANM3Tx--vMXhBm*IDA&YL$b{@ALRGK1YHo$~sK~bc*;|Z~J9#q-j z2^7yunoR*AC{JcWmgg+Tb4BUZRJZ*}43VUZGpsr>ewia&A^Lck!eqZsUTurO$@HJ$ zrk#RU#AgzVIFCEED1u+^zUEfD%_9~z+N+3Y&pV?swKXe5gGnpb88!=QkViov%<$`K zxTy)0QhhnNqdG|`$L8s%Ir)(!o18B(u}F#dDun^?VUj8OZVHV%l64WE^l?i8j=xq+ zd!YRz9%=vn6){JEI`;67*WLVNjC+Mu%^yW;IW4-AOQc%&h_@#V@#Tu?oLh(14x_4# zwiE_cSo_d-2q~IkUBgI3xYrY?li9fMq*)4xlA@&pJx|{$nG&Cv#w=vtzD`pg&SUsT z7s&AgY_Go<)eLrE1wDj!-r?)J=Wt2PPXH+jy@<25f49ZHO`M%tg_V4}a+6fxgr3C- zh~m+PJ&UqkTDz{Ce!nAJ3;S4>yUibHL>T zT`V;1A}k_FG$t|(j(sH3?;nf-7c6#j8vwx8>iys{f)20xA?&lA@yhdMECLam z#n*Uoh{_@}?<_qah|j+j?03Q53^r6^by*wb`KdNflI_Rqli+Wgpgo-BqGZyuZMA&I z75^(rT|NDIt2*Uvk!CfTq(jztU@sE^U`d2O);u!^<`H~tI*7+Vq+cXK69loQ7F^&8nD%D4R1Gj_L_oNVhlq)Ll z9OW{xYdhtY<+MDT-E-Xo(%r<}gC!j{US&)h+n5@-XcfOmP^qVt1#v4Yz(ErZ$X%gX zwkPfbN49k*z7?%W+$H~#gF-G^_Co?%zZT>0>l}aJhhtI;r-sF=8j!@0mt)640vV~{ zwB$GHTM5-}A@IU0|FCO0(eS+Wz^Ie~SEMUjk9DkhLrj6QDK_|!)Jk8)-qkoPI@;UL03|Hb-@{^S3)1jSsjJztX;Qeb6-u&UISheujU4$5p2R2$ z+KTjN<_t00_z@Tr7~5CnwE?3HvY59VpA6gRl2B~am5yiqOSf}F?Fo6~W+C(}_ zml7TXWC+;EMGvTU@1w{QCtXRT8|_QH7;vyFqjU3Or8^{%@B+n%9H+pg((EHA&KQh` zCysY4qssdZ^#Sd9r!FSlcOY=Slt|mYz|z;qK%?Vwc;?bJ{a};jd#Kw_MrY2Wt65S^ zGTf&Utc9+0pnkEiAnhg3!!L%Hs;y=X6P1;2&0=x-WBzlALci3o6ZB?drizN&k{v%E z--YUFrBIeh{wOW)3YvRR+tX6D9FJVaY`KcXgk9SFtfQxSn@}s87)@XOx{6Azr*Whw z24Ko$XgkBw*!}cgDlCWpB%7UgEmZlUWYo9L(|MK8&sm!viMNay5D>#EhiA#Y~)diMn5GT-%dDm$PpDSzS)WWchQM{llr)6>h;N1GGq_7%ktjy1gC28JB9oEP zk&ma6owbL~L~o-QrGE0%td-HLlJuT#5soi&Q_hv4ed=9AAuM$hVt1Ks(M0GO(E8Ae zx(Enr(n_Y*^s=67>PJ6_adzP#BevG#Y3zXmj7OMSRK%c**neNTU}7C zg|*G7WT7gzHy77rs|!aYCsgdJs@BujNxw_5WhNjo+u|q$#(R3Qi#k^h%Az19fbO{= z=By2TU-NR$I1j_ijxZrh_?qz0c0J{y>l040;O4Vf&2z^3KX zvipQV5I(NQ6_o~`@_h=Iz2;Zao`E?+fqRj>vn%P-gPLf>wc+ z8w;lxUvC(%2)bQgfykP{5xTYt28|M`R1;N8x&Y$Ex5O3%Iy!oa6Q^t8?0HB;c5kx; zQ;N*rU*rYVxsk3FM4_g^ZkLP*zfK5+og+>eg@yR4{2KXC*@TQ7n*FWdz{z<;V}~cV z&T&Iv`TM6!`)jX9c*kYPI3l4hhA%yUv^Okk$!BYwC(!U}bK^3(CJ@3WRp3(Z8BXT0 zdNZr#Uh-$V%id^v4kT$xAoJ}GE2HeH#9C)1EwUn$)J^&c@C@;JPmy)LnmNN>u}Hr> zAEI~B+J~y+@$4%aRbvH)iuDC?d+^aOCBd0K;g_Y<>bDnv3F>AjX4FLaZf{9ISc@wZ zKQb+6xn{iI?Wj-Z6lQPT>mYW16u~LssF&mus`6G|d5p*H!fvs05@IyN5_PtYTKJ`# z{_>e`!_vvQoBm4s$D97qt^QGe=hQ>zR8{9xs!!JAZ*B%5!LlroU_;bx=rp<+L&F4! zwh-;Y-%_t6sCE1n)gd#Bl84=ZtuC?XmrJ|EBXz&Gn8kqgVAisdFM%CG$-3OM;oO<= zg}Cc~vf7qCVEj-YIvmnVRbbuiJ)7ntlMo{0m#Kz}j~Q5%he|egtDI(S4T!La^lUiv zIzIMda`azL#C*GM_cjO}V#gng0)?{0q#7s(Jq?JS-V5cXU97~c9!Z4&kqa!zx131O zp)(ga(#&mAKLmKqp~H^xOvayZNm~u6MAGtWnMOqODb_d6_Uc|yP#I^wvREN$Ltd&} z%s!=`rZLhe1qFP$!dk!taBb zeNfDq&asp&q`f8e^^ZKCtA4{t2|KZ?MUL#Zi}r%A`kj2pH@P~Vd&ZA*@{gBWK|&IB zar~_d@5<2S$%##w3RXLgrg$WQm7M||m9y@a07vKqHa}eqs^?o8fMPz3v?oOX^%t!~ ztz&RuG6pYlxH0f&0FNk~KxLm6m?jg%H{r340RK)hC~^vk#$NEC?x~kj(_^om0Y5ks zcq5k?HR_D!?=0yqx!L(~c{UZ^Uj!qZa#t(^CxT!solvbxg0-AjZ-OP~6mL_jlT%Z^ zu(JHw_XOY2@L>%=0EFcuY0*4s7Nxpqk1595nE!r)RyJ)=PIaNxstz4-bcads>Xu_? z);eb9=>nA$N~9RKWD8!Ye5?{)O{!N3H+A8ENxEmgzu|gfjDJ}t@RUhSkW8x#F}2>z zDPok*xp671MCgpIh(#Ns&kJ^7nm>2Y7#ngLmHH-nh&Ff#1Zdk!Wv@aH_V{5~HEeN3 z?+Nel*ICguCNA_PTyyp%hV(H;!XrhXKg}3+VPj_L{EFnB;?2+0I0yIoRgkI57hl}Y zfsfEPOg=8mnb`iF0ljQ8SvXzV<{3Q=UFkROK%`v7YTFSbyei~IXDHn;WWZz%2|(9F3brKpG}Y8r=9YJXycYa8PU%EZm#3hrMv^7jP-p10dSAJ8j}ZlI zZFdI_Jw)27N>v~9JBn!(!SO8oPjE7rtAq)fCz+5hcm(QWig11G_I51s`=ni3{;00{ z{V`J5k;FwXP>+-QwYCY8yu3ynb0eYL&eo#Pu0dn60e~#M9wrB6DKicCnl*GO_1+HN zsz7@s_U2sjdD!nN#t%kE{#OO^oLE z-=!Xd);3@fg$Drl$yjF=-}6LKH!nN?_&)9)3_o^-PuwqF6Gmc2YCqG;bCw6A4$;uk z2{7nWC;{l03wq_=pv=G1CL7AALYMTeuU~Y82gv8YzaB_9Y(-cx(FFo;3Cm^fw5g*s zBg4=CIFI(*zxv1`-4t zqF3QQ$#u)->{10?;4x<)W0@h3V(6HMhG&`l5dexPi)|#t?|mrNBC1SOeh7~eEw$97 z^FIS00s%J0%^gJB zLu;*qBEsg8@JQ9Ke*|6r1gv1l8@YaQ6B@fiWgk zPe?gL?qwQW1h&v~Yaq99)(Z-ctJe^6UcW?S2FIoYAlU?pTl{lEu^Lz~z%t@A&zF#r zGc=bPLjngBc?Cl2QK-uf)A18%YrLCRpdh&ayk16g#+O@6A@VaMrHZ8Sn>YI#EWQg?ro(xu84x}mC_$Aqy+yeO z{aC9MH*U#?J1;t_(}pZU1i(=HCsE17$AgoJMN(#zaoioubx67-v-F-^PgNR+ro%9#Tg;TNwy9F~z zicat0q~!I1t+(z*9$|3-`_O?A($HMG7sIAJCCf>JXr5W_S!gCD_ zqqDW~aQGPbhos!h76BeE_0Kw8N_;K|k3oH<@S#=ozeFdVXRmu}bb0T5xo$(O9*MmR zX`UH+QRe?XI$;4)Fz{ZsdnkW5vn(rJkQk z`R&jkC5L{jO+Ah?6y=|rl1LEN=DdmfS{ zA3#D@^!iS5P%uCh(}G^%ryp+hb4UN&KLh<7&755v>|OqK{u}ORI@yWP#|Q`hIg|4! z?Ep5wm6pHMGcomniI%O2TA$N2DSG)tVzp;qcG-l{(GB9(ixwl$s0at_dquN)TzhXa zu%YG0iZ?SFK@p;1z1)@QJ;KkEMq=;?&4~&r>fzc9LdnX@M)KER7Y@19@yg8>B(oLZ zk`q~LpEp)oSL|T!v^1k6!n}2w^MrShTo|Ap?W@8aJ#Qy-WayIpe8>FjqXW$8m5UpG zMhg)`f`B0XPeV+MY)wQgjjZhdX~V*}b&w#I;ErG36;Zu|>|sjNfGP6vJBB5p-~=lS zH>8C0>+h`4w3*D9ZKdHWjkJ~B_A(&DQb_s3kF@9^5~nW9rQ|Ks-M#n2U1hSo>38lL zoh#;>y+F;0>1lH@fIj9pbB!;G#x*gj*_K1iJS$p}%`KyK^VIOS9J{4`Ki&&=t7Uh3 zgRr^Qt}+!ag0+SS*zo5{$u`*zxn#-ma3raB5821}hncY3z}x8C;M?%q@6K_~Va`!? z#;vPy%<&q)Co_+Jd>Op^Ldm2ZAYRT~%W$q{61DVOEu}*kK&y||D1}ibjeG*d9J)UE zZ&sc0TN?TV%xC;mpjL0C;omIV1dcg;eW+G{rO_`&n}4!k^#LnAwubDCxM}bckmi50 zeYQqTj5=v_6By>O^&x+=*?qhr*yC^xFWm|BVYBkLjB+R*HyceE1niH`44-I!08?m@ zf%V!1KnoeV`ajxEUx12E5Wom^s`^abz%TV)n%tm3DH#IbtTtVJlFT&>&_S23zE*wY z3%KHNyGT8WKJypAQo_0GBn=t;3_M>|®G*$kT6YIPkHvf}Cw|GG?0DW!Xg#5jex zI9>hDCb{3B{FXNAg=tKQ0SW3>Dm5cllX8EOO}~UF<<~;07lsK|mrv)U%sE-{z1{x` zkbozSlBIAwQQwE2b&?M94O9=-5>3tlt((}B5AO#1!+V3e>v5bN zN{KHR%(Sl&p0Qe2Pi`lJE7K+Km(~VYOCe4_!K{S=h@4~Cs^i`Hrh#q0A#>)`!RKtbLJ|K99W6^FCnX7FAow{GR$31M(1`UuJR zcC;gg=)v^IBRok~J~mRwY!;WFyS3&m3-H?2N~|_Ic9)sV)xQ1{LcrkqdGMX_m(<7h zVw$A~sDSxfA||axlpNQQS#Q)h=@6e#@{|hE!mq~y3|}s>6KD9Ld98w;iz#`lEh+ph zj|akkJm9Z}Z*b868{(a4uMBe!1_EN>Y-a1i0Nk-h$Nles;ZK?P*G379wkIL@PcR<{ z2+qHY{yCS!f`9<0U~z!0E=ail2}}b4!TSH)#tuYw5CgTN18O>uS`zz`HbfKFOypX$K?x~&Kd za1sC;Rr-4g8@TR-1l)AQ0mV@Pesv}X5<204G3)=&-441CKs6^MyuZp){K;~(pXek1 z=T)C6=}*U-Gao3W32@4p7Zl4J2vV;g%upf{$H6_ T7Ze1nPppkTd6U@jZ Import(string filename) var collectionName = $"Submissions-{pricingId}"; var address = new PricingAddress(pricingId); + // Try to get the saved configuration for this file + string? configuration = null; + try + { + var configJson = await GetRiskImportConfiguration(filename); + // Check if we got a valid configuration (not an error message) + if (!configJson.StartsWith("Error") && !configJson.StartsWith("Please navigate")) + { + configuration = configJson; + } + } + catch + { + // If we can't get the configuration, fall back to format-based import + } + // Delegate to CollectionPlugin's Import method var collectionPlugin = new CollectionPlugin(hub); return await collectionPlugin.Import( path: filename, collection: collectionName, address: address, - format: "PropertyRiskImport" + format: configuration != null ? null : "PropertyRiskImport", // Use format only if no configuration + configuration: configuration // Pass configuration if available ); } @@ -214,7 +231,19 @@ public async Task GetRiskImportConfiguration(string filename) new GetDataRequest(new EntityReference("ExcelImportConfiguration", filename)), o => o.WithTarget(new PricingAddress(chat.Context.Address.Id)) ); - return JsonSerializer.Serialize(response.Message.Data, hub.JsonSerializerOptions); + + // Serialize the data + var json = JsonSerializer.Serialize(response.Message.Data, hub.JsonSerializerOptions); + + // Parse and ensure $type is set to ExcelImportConfiguration + var jsonObject = JsonNode.Parse(json) as JsonObject; + if (jsonObject != null) + { + var withType = EnsureTypeFirst(jsonObject, "ExcelImportConfiguration"); + return JsonSerializer.Serialize(withType, hub.JsonSerializerOptions); + } + + return json; } catch (Exception e) { diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index 9a5987414..bf1a9cdc3 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -21,7 +21,7 @@ public static class InsuranceApplicationExtensions /// public static MessageHubConfiguration ConfigureInsuranceApplication(this MessageHubConfiguration configuration) => configuration - .WithTypes(typeof(PricingAddress), typeof(ExcelImportConfiguration), typeof(Structure)) + .WithTypes(typeof(PricingAddress), typeof(ImportConfiguration), typeof(ExcelImportConfiguration), typeof(Structure), typeof(ImportRequest), typeof(CollectionSource)) .AddData(data => { var svc = data.Hub.ServiceProvider.GetRequiredService(); diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ImportConfigsLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ImportConfigsLayoutArea.cs index cecf8a2a9..4f0240afa 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ImportConfigsLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ImportConfigsLayoutArea.cs @@ -1,4 +1,4 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using System.Text.Json; using MeshWeaver.Import.Configuration; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; @@ -42,51 +42,9 @@ public static IObservable ImportConfigs(LayoutAreaHost host, Renderin foreach (var cfg in list.OrderBy(x => x.Name)) { parts.Add($"\n## {cfg.Name}"); - parts.Add($"\n**Worksheet:** {cfg.WorksheetName}"); - parts.Add($"**Data Start Row:** {cfg.DataStartRow}"); - - if (cfg.Mappings.Any()) - { - parts.Add("\n### Column Mappings"); - parts.Add("\n| Target Property | Mapping Kind | Source Columns | Constant Value |"); - parts.Add("|----------------|--------------|----------------|----------------|"); - foreach (var mapping in cfg.Mappings) - { - var sourceColumns = string.Join(", ", mapping.SourceColumns); - var constantValue = mapping.ConstantValue?.ToString() ?? ""; - parts.Add($"| {mapping.TargetProperty} | {mapping.Kind} | {sourceColumns} | {constantValue} |"); - } - } - - if (cfg.Allocations.Any()) - { - parts.Add("\n### Allocations"); - parts.Add("\n| Target Property | Total Cell | Weight Columns | Currency Property |"); - parts.Add("|----------------|------------|----------------|-------------------|"); - foreach (var alloc in cfg.Allocations) - { - var weightColumns = string.Join(", ", alloc.WeightColumns); - parts.Add($"| {alloc.TargetProperty} | {alloc.TotalCell} | {weightColumns} | {alloc.CurrencyProperty ?? ""} |"); - } - } - - if (cfg.TotalRowMarkers.Any()) - { - parts.Add($"\n**Total Row Markers:** {string.Join(", ", cfg.TotalRowMarkers)}"); - } - - if (cfg.IgnoreRowExpressions.Any()) - { - parts.Add("\n**Ignore Row Expressions:**"); - foreach (var expr in cfg.IgnoreRowExpressions) - { - parts.Add($"- `{expr}`"); - } - } - // Add full JSON configuration in a collapsible section var json = JsonSerializer.Serialize(cfg, options); - parts.Add($"\n

\nView Full JSON Configuration\n\n```json\n{json}\n```\n
\n"); + parts.Add($"```json\n{json}\n```"); } var md = string.Join("\n", parts); diff --git a/modules/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs b/modules/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs index 263130574..2d75ab7e0 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs @@ -1,70 +1,124 @@ +using System.Reactive.Linq; using FluentAssertions; +using FluentAssertions.Extensions; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; using MeshWeaver.Import; using MeshWeaver.Import.Configuration; using MeshWeaver.Insurance.Domain; +using MeshWeaver.Insurance.Domain.Services; +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; using Xunit; namespace MeshWeaver.Insurance.Test; public class MicrosoftImportTests(ITestOutputHelper output) : InsuranceTestBase(output) { - [Fact] - public void Import_Microsoft_File() + private readonly string _testFilesPath = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "Files", "Microsoft", "2026"); + private const string MicrosoftPricingId = "Microsoft-2026"; + + protected override MeshBuilder ConfigureMesh(MeshBuilder builder) { - // Arrange - Create import configuration - var config = new ExcelImportConfiguration - { - Name = "Microsoft.xlsx", - EntityId = "Microsoft", - WorksheetName = "Locations", // Adjust based on actual worksheet name - DataStartRow = 2, // Assuming row 1 is headers - TotalRowMarkers = new HashSet { "Total", "Grand Total" }, - TotalRowScanAllCells = true, - TotalRowMatchExact = false, - Mappings = new List - { - // Basic identification - new() { TargetProperty = "Id", Kind = MappingKind.Direct, SourceColumns = new List { "A" } }, - new() { TargetProperty = "LocationName", Kind = MappingKind.Direct, SourceColumns = new List { "B" } }, - new() { TargetProperty = "PricingId", Kind = MappingKind.Constant, ConstantValue = "Microsoft" }, + // Ensure test directory exists + Directory.CreateDirectory(_testFilesPath); + + return base.ConfigureMesh(builder) + .ConfigureServices(services => services + .AddSingleton() + ) + .ConfigureHub(c => c + .AddContentCollections() + .AddFileSystemContentCollection($"Submissions-{MicrosoftPricingId}", _ => _testFilesPath) + .AddImport() + .AddData(data => data.AddSource(source => source.WithType())) + ); + } + private static readonly ExcelImportConfiguration Config = new() + { + Name = "Microsoft.xlsx", + EntityId = MicrosoftPricingId, + TypeName = nameof(PropertyRisk), // Auto-generate entity builder for PropertyRisk + //WorksheetName = "Locations", // Adjust based on actual worksheet name + DataStartRow = 7, // Assuming row 1 is headers + TotalRowMarkers = ["Total", "Grand Total"], + TotalRowScanAllCells = true, + TotalRowMatchExact = false, + Mappings = + [ + new () { TargetProperty = "Id", Kind = MappingKind.Direct, SourceColumns = new List { "C" } }, + new() + { + TargetProperty = "LocationName", + Kind = MappingKind.Direct, + SourceColumns = ["D"] + }, + new() { TargetProperty = "PricingId", Kind = MappingKind.Constant, ConstantValue = MicrosoftPricingId }, // Address fields - new() { TargetProperty = "Address", Kind = MappingKind.Direct, SourceColumns = new List { "C" } }, - new() { TargetProperty = "City", Kind = MappingKind.Direct, SourceColumns = new List { "D" } }, - new() { TargetProperty = "State", Kind = MappingKind.Direct, SourceColumns = new List { "E" } }, - new() { TargetProperty = "Country", Kind = MappingKind.Direct, SourceColumns = new List { "F" } }, - new() { TargetProperty = "ZipCode", Kind = MappingKind.Direct, SourceColumns = new List { "G" } }, - - // Values - adjust column letters based on actual Excel file - new() { TargetProperty = "Currency", Kind = MappingKind.Direct, SourceColumns = new List { "H" } }, - new() { TargetProperty = "TsiBuilding", Kind = MappingKind.Direct, SourceColumns = new List { "I" } }, - new() { TargetProperty = "TsiContent", Kind = MappingKind.Direct, SourceColumns = new List { "J" } }, - new() { TargetProperty = "TsiBi", Kind = MappingKind.Direct, SourceColumns = new List { "K" } }, - }, - IgnoreRowExpressions = new List - { - "Id == null", // Skip rows without an ID - "Address == null" // Skip rows without an address - } - }; - - var filePath = "../../Files/Microsoft/2026/Microsoft.xlsx"; - var fullPath = Path.Combine(Directory.GetCurrentDirectory(), filePath); + new() + { + TargetProperty = "Address", + Kind = MappingKind.Direct, + SourceColumns = ["E"] + }, + new() { TargetProperty = "Country", Kind = MappingKind.Direct, SourceColumns = new List { "B" } }, + new() + { + TargetProperty = "TsiBuilding", + Kind = MappingKind.Direct, + SourceColumns = ["H"] + }, + new() + { + TargetProperty = "TsiContent", + Kind = MappingKind.Direct, + SourceColumns = ["G", "I", "J", "K", "L", "M", "N", "O", "P"] + }, + ], + Allocations = [new() { TargetProperty = "TsiBi", WeightColumns = ["Q"] }], + IgnoreRowExpressions = + [ + "Id == null", // Skip rows without an ID + "Address == null" + ] + }; + [Fact] + public async Task Import_Microsoft_File_WithConfiguration() + { // Skip test if file doesn't exist + var fullPath = Path.Combine(_testFilesPath, "Microsoft.xlsx"); if (!File.Exists(fullPath)) + throw new FileNotFoundException(fullPath); + + // Arrange - Create import configuration with TypeName + + // Act - Import using ImportRequest with Configuration + var importRequest = new ImportRequest(new CollectionSource($"Submissions-{MicrosoftPricingId}", "Microsoft.xlsx")) { - Output.WriteLine($"Test file not found: {fullPath}"); - return; - } + Configuration = Config + }; - // Act - Import using the generic importer - var importer = new ConfiguredExcelImporter(BuildPropertyRisk); - var risks = importer.Import(fullPath, config).ToList(); + var importResponse = await Mesh.AwaitResponse( + importRequest, + o => o.WithTarget(Mesh.Address), + TestContext.Current.CancellationToken + ); // Assert + importResponse.Should().NotBeNull(); + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded, + $"import should succeed. Errors: {string.Join(", ", importResponse.Message.Log.Errors().Select(e => e.Message))}"); + + // Verify data was imported by querying the workspace + var workspace = Mesh.ServiceProvider.GetRequiredService(); + var risks = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count > 0); + risks.Should().NotBeEmpty("import should return at least one risk"); - risks.All(r => r.PricingId == "Microsoft").Should().BeTrue("all risks should have PricingId set to Microsoft"); + risks.All(r => r.PricingId == MicrosoftPricingId).Should().BeTrue("all risks should have PricingId set to Microsoft-2026"); risks.All(r => !string.IsNullOrWhiteSpace(r.Id)).Should().BeTrue("all risks should have an Id"); // Verify source tracking @@ -73,52 +127,42 @@ public void Import_Microsoft_File() // Output summary Output.WriteLine($"Successfully imported {risks.Count} property risks"); - Output.WriteLine($"Sample risk: Id={risks.First().Id}, Location={risks.First().LocationName}, Country={risks.First().Country}"); + if (risks.Any()) + { + var first = risks.First(); + Output.WriteLine($"Sample risk: Id={first.Id}, Location={first.LocationName}, Country={first.Country}"); + } } [Fact] - public void Import_Microsoft_WithAllocation() + public async Task Import_Microsoft_WithAllocation() { - // This test demonstrates allocation mapping - distributing a total value proportionally - var config = new ExcelImportConfiguration - { - Name = "Microsoft.xlsx", - EntityId = "Microsoft", - WorksheetName = "Locations", - DataStartRow = 2, - Mappings = new List - { - new() { TargetProperty = "Id", Kind = MappingKind.Direct, SourceColumns = new List { "A" } }, - new() { TargetProperty = "LocationName", Kind = MappingKind.Direct, SourceColumns = new List { "B" } }, - new() { TargetProperty = "PricingId", Kind = MappingKind.Constant, ConstantValue = "Microsoft" }, - new() { TargetProperty = "TsiBuilding", Kind = MappingKind.Direct, SourceColumns = new List { "I" } }, - new() { TargetProperty = "TsiContent", Kind = MappingKind.Direct, SourceColumns = new List { "J" } }, - }, - Allocations = new List - { - // Example: Allocate total BI from cell C3 proportionally based on TsiBuilding + TsiContent weights - new() - { - TargetProperty = "TsiBi", - TotalCell = "C3", // Adjust to actual total cell in Excel - WeightColumns = new List { "I", "J" }, // Weight by TsiBuilding + TsiContent - CurrencyProperty = "Currency" - } - } - }; - - var filePath = "../../Files/Microsoft/2026/Microsoft.xlsx"; - var fullPath = Path.Combine(Directory.GetCurrentDirectory(), filePath); - + // Skip test if file doesn't exist + var fullPath = Path.Combine(_testFilesPath, "Microsoft.xlsx"); if (!File.Exists(fullPath)) + throw new FileNotFoundException(fullPath); + + // Act - Import using ImportRequest with Configuration + var importRequest = new ImportRequest(new CollectionSource($"Submissions-{MicrosoftPricingId}", "Microsoft.xlsx")) { - Output.WriteLine($"Test file not found: {fullPath}"); - return; - } + Configuration = Config + }; - var importer = new ConfiguredExcelImporter(BuildPropertyRisk); - var risks = importer.Import(fullPath, config).ToList(); + var importResponse = await Mesh.AwaitResponse( + importRequest, + o => o.WithTarget(Mesh.Address), + TestContext.Current.CancellationToken + ); + // Assert + importResponse.Should().NotBeNull(); + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded); + + // Verify data was imported + var workspace = Mesh.ServiceProvider.GetRequiredService(); + var risks = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count > 0); risks.Should().NotBeEmpty(); // Verify allocation worked - sum of allocated TsiBi should equal proportional distribution @@ -144,143 +188,37 @@ public void Import_Microsoft_WithAllocation() } [Fact] - public void Import_Microsoft_UsingSumMapping() + public async Task Import_Microsoft_UsingSumMapping() { - // Demonstrate Sum mapping - combining multiple columns - var config = new ExcelImportConfiguration - { - Name = "Microsoft.xlsx", - EntityId = "Microsoft", - WorksheetName = "Locations", - DataStartRow = 2, - Mappings = new List - { - new() { TargetProperty = "Id", Kind = MappingKind.Direct, SourceColumns = new List { "A" } }, - new() { TargetProperty = "PricingId", Kind = MappingKind.Constant, ConstantValue = "Microsoft" }, - - // Sum example: Total TSI = Building + Content + BI - new() { TargetProperty = "TsiBuilding", Kind = MappingKind.Direct, SourceColumns = new List { "I" } }, - new() { TargetProperty = "TsiContent", Kind = MappingKind.Direct, SourceColumns = new List { "J" } }, - new() { TargetProperty = "TsiBi", Kind = MappingKind.Direct, SourceColumns = new List { "K" } }, - } - }; - - var filePath = "../../Files/Microsoft/2026/Microsoft.xlsx"; - var fullPath = Path.Combine(Directory.GetCurrentDirectory(), filePath); - + // Skip test if file doesn't exist + var fullPath = Path.Combine(_testFilesPath, "Microsoft.xlsx"); if (!File.Exists(fullPath)) - { - Output.WriteLine($"Test file not found: {fullPath}"); - return; - } - - var importer = new ConfiguredExcelImporter(BuildPropertyRisk); - var risks = importer.Import(fullPath, config).ToList(); + throw new FileNotFoundException(fullPath); - risks.Should().NotBeEmpty(); - Output.WriteLine($"Imported {risks.Count} risks using sum mapping"); - } + // Demonstrate Sum mapping - combining multiple columns - /// - /// Builder function to construct PropertyRisk from dictionary of properties. - /// This handles type conversion and provides defaults. - /// - private static PropertyRisk BuildPropertyRisk(Dictionary values) - { - return new PropertyRisk + // Act - Import using ImportRequest with Configuration + var importRequest = new ImportRequest(new CollectionSource($"Submissions-{MicrosoftPricingId}", "Microsoft.xlsx")) { - Id = Get(values, nameof(PropertyRisk.Id)) ?? Guid.NewGuid().ToString(), - PricingId = Get(values, nameof(PropertyRisk.PricingId)), - SourceRow = Get(values, nameof(PropertyRisk.SourceRow)), - SourceFile = Get(values, nameof(PropertyRisk.SourceFile)), - LocationName = Get(values, nameof(PropertyRisk.LocationName)), - Country = Get(values, nameof(PropertyRisk.Country)), - State = Get(values, nameof(PropertyRisk.State)), - County = Get(values, nameof(PropertyRisk.County)), - ZipCode = Get(values, nameof(PropertyRisk.ZipCode)), - City = Get(values, nameof(PropertyRisk.City)), - Address = Get(values, nameof(PropertyRisk.Address)), - Currency = Get(values, nameof(PropertyRisk.Currency)), - TsiBuilding = Get(values, nameof(PropertyRisk.TsiBuilding)), - TsiBuildingCurrency = Get(values, nameof(PropertyRisk.TsiBuildingCurrency)) ?? Get(values, nameof(PropertyRisk.Currency)), - TsiContent = Get(values, nameof(PropertyRisk.TsiContent)), - TsiContentCurrency = Get(values, nameof(PropertyRisk.TsiContentCurrency)) ?? Get(values, nameof(PropertyRisk.Currency)), - TsiBi = Get(values, nameof(PropertyRisk.TsiBi)), - TsiBiCurrency = Get(values, nameof(PropertyRisk.TsiBiCurrency)) ?? Get(values, nameof(PropertyRisk.Currency)), - AccountNumber = Get(values, nameof(PropertyRisk.AccountNumber)), - OccupancyScheme = Get(values, nameof(PropertyRisk.OccupancyScheme)), - OccupancyCode = Get(values, nameof(PropertyRisk.OccupancyCode)), - ConstructionScheme = Get(values, nameof(PropertyRisk.ConstructionScheme)), - ConstructionCode = Get(values, nameof(PropertyRisk.ConstructionCode)), - BuildYear = Get(values, nameof(PropertyRisk.BuildYear)), - UpgradeYear = Get(values, nameof(PropertyRisk.UpgradeYear)), - NumberOfStories = Get(values, nameof(PropertyRisk.NumberOfStories)), - Sprinklers = Get(values, nameof(PropertyRisk.Sprinklers)), - GeocodedLocation = null + Configuration = Config }; - } - - /// - /// Generic value getter with type conversion support. - /// - private static T? Get(IDictionary dict, string key) - { - if (dict.TryGetValue(key, out var val) && val is not null) - { - if (val is T t) return t; - - var targetType = typeof(T); - var underlying = Nullable.GetUnderlyingType(targetType) ?? targetType; - // Empty strings should be treated as null/default - if (val is string s && string.IsNullOrWhiteSpace(s)) return default; + var importResponse = await Mesh.AwaitResponse( + importRequest, + o => o.WithTarget(Mesh.Address), + TestContext.Current.CancellationToken + ); - try - { - // Handle common type conversions - if (underlying == typeof(string)) - return (T)(object)val.ToString()!; - - if (underlying == typeof(int)) - { - if (val is int i) return (T)(object)i; - if (val is decimal dm) return (T)(object)(int)dm; - if (val is double d) return (T)(object)(int)d; - if (int.TryParse(val.ToString(), out var parsed)) return (T)(object)parsed; - } - - if (underlying == typeof(double)) - { - if (val is double d) return (T)(object)d; - if (val is decimal dm) return (T)(object)(double)dm; - if (val is int i) return (T)(object)(double)i; - if (double.TryParse(val.ToString(), out var parsed)) return (T)(object)parsed; - } - - if (underlying == typeof(decimal)) - { - if (val is decimal dm) return (T)(object)dm; - if (val is double d) return (T)(object)(decimal)d; - if (decimal.TryParse(val.ToString(), out var parsed)) return (T)(object)parsed; - } - - if (underlying == typeof(bool)) - { - if (val is bool b) return (T)(object)b; - var str = val.ToString()?.Trim().ToLowerInvariant(); - if (str == "true" || str == "yes" || str == "1") return (T)(object)true; - if (str == "false" || str == "no" || str == "0") return (T)(object)false; - } - - // Fallback to Convert.ChangeType - if (val is IConvertible) - return (T)Convert.ChangeType(val, underlying); - } - catch - { - // Swallow conversion errors and return default - } - } - return default; + // Assert + importResponse.Should().NotBeNull(); + importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded); + + // Verify data was imported + var workspace = Mesh.ServiceProvider.GetRequiredService(); + var risks = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count > 0); + risks.Should().NotBeEmpty(); + Output.WriteLine($"Imported {risks.Count} risks using direct mapping"); } } diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index 9477a0291..5b74e47d7 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -149,6 +149,7 @@ public async Task Import( [Description("The name of the collection containing the file (optional if default collection is configured)")] string? collection = null, [Description("The target address for the import (optional if default address is configured), can be a string like 'AddressType/id' or an Address object")] object? address = null, [Description("The import format to use (optional, defaults to 'Default')")] string? format = null, + [Description("Optional import configuration as JSON string. When provided, this will be used instead of the format parameter.")] string? configuration = null, CancellationToken cancellationToken = default) { try @@ -187,6 +188,16 @@ public async Task Import( ["format"] = format ?? "Default" }; + // Add configuration if provided + if (!string.IsNullOrWhiteSpace(configuration)) + { + var configNode = JsonNode.Parse(configuration); + if (configNode != null) + { + importRequestJson["configuration"] = configNode; + } + } + // Serialize and deserialize through hub's serializer to get proper type var jsonString = importRequestJson.ToJsonString(); var importRequestObj = JsonSerializer.Deserialize(jsonString, hub.JsonSerializerOptions)!; @@ -222,8 +233,6 @@ public async Task Import( } return result; - - return "Import completed but response format was unexpected."; } catch (Exception ex) { diff --git a/src/MeshWeaver.Import/Configuration/AutoEntityBuilder.cs b/src/MeshWeaver.Import/Configuration/AutoEntityBuilder.cs new file mode 100644 index 000000000..d759c25a2 --- /dev/null +++ b/src/MeshWeaver.Import/Configuration/AutoEntityBuilder.cs @@ -0,0 +1,113 @@ +using System.Reflection; + +namespace MeshWeaver.Import.Configuration; + +/// +/// Helper class to automatically build entities from property dictionaries using reflection. +/// +public static class AutoEntityBuilder +{ + /// + /// Creates an entity builder function for a given type using reflection. + /// + /// The type to instantiate + /// A function that builds instances from property dictionaries + public static Func, object> CreateBuilder(Type type) + { + return properties => + { + var instance = Activator.CreateInstance(type); + if (instance == null) + throw new InvalidOperationException($"Failed to create instance of type {type.FullName}"); + + foreach (var (key, value) in properties) + { + var prop = type.GetProperty(key, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); + if (prop == null || !prop.CanWrite) + continue; + + try + { + var convertedValue = ConvertValue(value, prop.PropertyType); + prop.SetValue(instance, convertedValue); + } + catch + { + // Ignore conversion errors for individual properties + } + } + + return instance; + }; + } + + /// + /// Creates a typed entity builder function. + /// + public static Func, T> CreateBuilder() where T : class + { + var builder = CreateBuilder(typeof(T)); + return properties => (T)builder(properties); + } + + /// + /// Converts a value to the target type with support for common conversions. + /// + private static object? ConvertValue(object? value, Type targetType) + { + if (value == null) + return targetType.IsValueType ? Activator.CreateInstance(targetType) : null; + + // If already correct type + if (targetType.IsInstanceOfType(value)) + return value; + + var underlying = Nullable.GetUnderlyingType(targetType) ?? targetType; + + // Empty strings should be treated as null/default + if (value is string s && string.IsNullOrWhiteSpace(s)) + return targetType.IsValueType ? Activator.CreateInstance(targetType) : null; + + // Handle string conversions + if (underlying == typeof(string)) + return value.ToString(); + + // Handle numeric conversions + if (underlying == typeof(int)) + { + if (value is int i) return i; + if (value is decimal dm) return (int)dm; + if (value is double d) return (int)d; + if (int.TryParse(value.ToString(), out var parsed)) return parsed; + } + + if (underlying == typeof(double)) + { + if (value is double d) return d; + if (value is decimal dm) return (double)dm; + if (value is int i) return (double)i; + if (double.TryParse(value.ToString(), out var parsed)) return parsed; + } + + if (underlying == typeof(decimal)) + { + if (value is decimal dm) return dm; + if (value is double d) return (decimal)d; + if (decimal.TryParse(value.ToString(), out var parsed)) return parsed; + } + + if (underlying == typeof(bool)) + { + if (value is bool b) return b; + var str = value.ToString()?.Trim().ToLowerInvariant(); + if (str == "true" || str == "yes" || str == "1") return true; + if (str == "false" || str == "no" || str == "0") return false; + } + + // Fallback to Convert.ChangeType + if (value is IConvertible) + return Convert.ChangeType(value, underlying); + + return value; + } +} diff --git a/src/MeshWeaver.Import/Configuration/ExcelImportConfiguration.cs b/src/MeshWeaver.Import/Configuration/ExcelImportConfiguration.cs index e438913b8..f02db810a 100644 --- a/src/MeshWeaver.Import/Configuration/ExcelImportConfiguration.cs +++ b/src/MeshWeaver.Import/Configuration/ExcelImportConfiguration.cs @@ -6,7 +6,7 @@ namespace MeshWeaver.Import.Configuration; /// /// Configuration describing how to transform an Excel worksheet into typed entities. /// -public class ExcelImportConfiguration +public class ExcelImportConfiguration : ImportConfiguration { public ExcelImportConfiguration() { @@ -26,13 +26,11 @@ public ExcelImportConfiguration(string name, string entityId, string worksheetNa } /// - /// File name of the Excel workbook used for this mapping (key). + /// The fully qualified type name of the entity to import (e.g., "MeshWeaver.Insurance.Domain.PropertyRisk"). + /// Used for automatic entity instantiation. /// - [Key] public required string Name { get; init; } - /// - /// Entity identifier that this configuration applies to (e.g., PricingId, ProjectId, etc.). - /// - public required string EntityId { get; init; } + public string? TypeName { get; set; } + /// /// Name of the worksheet within the Excel file to process. /// diff --git a/src/MeshWeaver.Import/Configuration/ImportBuilder.cs b/src/MeshWeaver.Import/Configuration/ImportBuilder.cs new file mode 100644 index 000000000..dd866954e --- /dev/null +++ b/src/MeshWeaver.Import/Configuration/ImportBuilder.cs @@ -0,0 +1,238 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq.Expressions; +using System.Reflection; +using MeshWeaver.ContentCollections; +using MeshWeaver.Data; +using MeshWeaver.DataSetReader; +using MeshWeaver.DataSetReader.Csv; +using MeshWeaver.DataSetReader.Excel; +using MeshWeaver.Domain; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.Import.Configuration; + +public record ImportBuilder +{ + public IWorkspace Workspace { get; } + public IServiceProvider? ServiceProvider { get; init; } + + public ImportBuilder( + IWorkspace workspace, + IServiceProvider? serviceProvider = null + ) + { + this.Workspace = workspace; + this.ServiceProvider = serviceProvider; + StreamProviders = InitializeStreamProviders(); + Validations = ImmutableList + .Empty.Add(StandardValidations) + .Add(CategoriesValidation); + if (workspace.MappedTypes.Any()) + ImportFormatBuilders = ImportFormatBuilders.Add( + ImportFormat.Default, + [f => f.WithMappings(m => m.WithAutoMappingsForTypes(workspace.MappedTypes))] + ); + } + + + private readonly ConcurrentDictionary ImportFormats = new(); + + public ImportBuilder WithFormat( + string format, + Func configuration + ) => + this with + { + ImportFormatBuilders = ImportFormatBuilders.SetItem( + format, + ( + ImportFormatBuilders.GetValueOrDefault(format) + ?? ImmutableList>.Empty + ).Add(configuration) + ) + }; + + private ImmutableDictionary< + string, + ImmutableList> + > ImportFormatBuilders { get; init; } = + ImmutableDictionary>>.Empty; + + public ImportFormat? GetFormat(string format) + { + if (ImportFormats.TryGetValue(format, out var ret)) + return ret; + + var builders = ImportFormatBuilders.GetValueOrDefault(format); + if (builders == null) + return null; + + return ImportFormats.GetOrAdd( + format, + builders.Aggregate( + new ImportFormat(format, Workspace, Validations), + (a, b) => b.Invoke(a) + ) + ); + } + + internal ImmutableDictionary DataSetReaders { get; init; } = + ImmutableDictionary + .Empty.Add( + MimeTypes.csv, + (stream, options, _) => DataSetCsvSerializer.ReadAsync(stream, options) + ) + .Add( + MimeTypes.xlsx, + (stream, _, _) => Task.FromResult(new ExcelDataSetReader().Read(stream)) + ) + .Add(MimeTypes.xls, new ExcelDataSetReaderOld().ReadAsync); + + public ImportBuilder WithDataSetReader(string fileType, ReadDataSet dataSetReader) => + this with + { + DataSetReaders = DataSetReaders.SetItem(fileType, dataSetReader) + }; + + internal ImmutableDictionary>> StreamProviders { get; init; } + + private ImmutableDictionary>> InitializeStreamProviders() => + ImmutableDictionary>> + .Empty.Add(typeof(StringStream), CreateMemoryStream) + .Add(typeof(EmbeddedResource), CreateEmbeddedResourceStream) + .Add(typeof(CollectionSource), CreateCollectionStreamAsync); + + private static Task CreateEmbeddedResourceStream(ImportRequest request) + { + var embeddedResource = (EmbeddedResource)request.Source; + var assembly = embeddedResource.Assembly; + var resourceName = $"{assembly.GetName().Name}.{embeddedResource.Resource}"; + var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) + { + throw new ArgumentException($"Resource '{resourceName}' not found."); + } + return Task.FromResult(stream); + } + + private static Task CreateMemoryStream(ImportRequest request) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(((StringStream)request.Source).Content); + writer.Flush(); + stream.Position = 0; + return Task.FromResult(stream); + } + + private async Task CreateCollectionStreamAsync(ImportRequest request) + { + var collectionSource = (CollectionSource)request.Source; + + // Resolve from ContentCollection + if (ServiceProvider == null) + throw new ImportException("ServiceProvider is not available to resolve CollectionSource from ContentCollection"); + + var contentService = ServiceProvider.GetService(); + if (contentService == null) + throw new ImportException("IContentService is not registered. Ensure ContentCollections are configured."); + + var stream = await contentService.GetContentAsync(collectionSource.Collection, collectionSource.Path); + if (stream == null) + throw new ImportException($"Could not find content at collection '{collectionSource.Collection}' path '{collectionSource.Path}'"); + + return stream; + } + + public ImportBuilder WithStreamReader( + Type sourceType, + Func> reader + ) => this with { StreamProviders = StreamProviders.SetItem(sourceType, reader) }; + + internal ImmutableList Validations { get; init; } + + public ImportBuilder WithValidation(ValidationFunction validation) => + this with + { + Validations = Validations.Add(validation) + }; + + private bool StandardValidations(object instance, ValidationContext validationContext, Activity activity) + { + var ret = true; + var validationResults = new List(); + Validator.TryValidateObject(instance, validationContext, validationResults, true); + + foreach (var validation in validationResults) + { + activity.LogError(validation.ToString()); + ret = false; + } + return ret; + } + + public static string MissingCategoryErrorMessage = "Category with name {0} was not found."; + + private static readonly ConcurrentDictionary< + Type, + (Type, string, Func)[] + > TypesWithCategoryAttributes = new(); + + private bool CategoriesValidation(object instance, ValidationContext validationContext, Activity activity) + { + var type = instance.GetType(); + var dimensions = TypesWithCategoryAttributes.GetOrAdd( + type, + key => + key.GetProperties() + .Where(x => x.PropertyType == typeof(string)) + .Select(x => new + { + Attr = x.GetCustomAttribute(), + x.Name + }) + .Where(x => x.Attr != null) + .Select(x => + ( + x.Attr!.Type, + x.Name, + CreateGetter(type, x.Name) + ) + ) + .ToArray() + ); + + var ret = true; + foreach (var (dimensionType, propertyName, propGetter) in dimensions) + { + if (!Workspace.DataContext.DataSourcesByType.ContainsKey(dimensionType)) + { + activity.LogError(string.Format(MissingCategoryErrorMessage, dimensionType)); + ret = false; + continue; + } + //if (!string.IsNullOrEmpty(value)) + // TODO V10: Need to restore categories validation here (03.12.2024, Roland Bürgi) + //if (false) + //{ + // activity.LogError( + // string.Format(UnknownValueErrorMessage, propertyName, type.FullName, propGetter(instance)) + // ); + // ret = false; + //} + } + return ret; + } + + private static Func CreateGetter(Type type, string property) + { + var prm = Expression.Parameter(typeof(object)); + var typedPrm = Expression.Convert(prm, type); + var propertyExpression = Expression.Property(typedPrm, property); + return Expression.Lambda>(propertyExpression, prm).Compile(); + } + + +} diff --git a/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs b/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs index 408096f9c..ca3aa0539 100644 --- a/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs +++ b/src/MeshWeaver.Import/Configuration/ImportConfiguration.cs @@ -1,238 +1,21 @@ -using System.Collections.Concurrent; -using System.Collections.Immutable; using System.ComponentModel.DataAnnotations; -using System.Linq.Expressions; -using System.Reflection; -using MeshWeaver.ContentCollections; -using MeshWeaver.Data; -using MeshWeaver.DataSetReader; -using MeshWeaver.DataSetReader.Csv; -using MeshWeaver.DataSetReader.Excel; -using MeshWeaver.Domain; -using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Import.Configuration; -public record ImportConfiguration +/// +/// Base configuration for import operations. +/// Contains common properties shared across different import types. +/// +public class ImportConfiguration { - public IWorkspace Workspace { get; } - public IServiceProvider? ServiceProvider { get; init; } - - public ImportConfiguration( - IWorkspace workspace, - IServiceProvider? serviceProvider = null - ) - { - this.Workspace = workspace; - this.ServiceProvider = serviceProvider; - StreamProviders = InitializeStreamProviders(); - Validations = ImmutableList - .Empty.Add(StandardValidations) - .Add(CategoriesValidation); - if (workspace.MappedTypes.Any()) - ImportFormatBuilders = ImportFormatBuilders.Add( - ImportFormat.Default, - [f => f.WithMappings(m => m.WithAutoMappingsForTypes(workspace.MappedTypes))] - ); - } - - - private readonly ConcurrentDictionary ImportFormats = new(); - - public ImportConfiguration WithFormat( - string format, - Func configuration - ) => - this with - { - ImportFormatBuilders = ImportFormatBuilders.SetItem( - format, - ( - ImportFormatBuilders.GetValueOrDefault(format) - ?? ImmutableList>.Empty - ).Add(configuration) - ) - }; - - private ImmutableDictionary< - string, - ImmutableList> - > ImportFormatBuilders { get; init; } = - ImmutableDictionary>>.Empty; - - public ImportFormat? GetFormat(string format) - { - if (ImportFormats.TryGetValue(format, out var ret)) - return ret; - - var builders = ImportFormatBuilders.GetValueOrDefault(format); - if (builders == null) - return null; - - return ImportFormats.GetOrAdd( - format, - builders.Aggregate( - new ImportFormat(format, Workspace, Validations), - (a, b) => b.Invoke(a) - ) - ); - } - - internal ImmutableDictionary DataSetReaders { get; init; } = - ImmutableDictionary - .Empty.Add( - MimeTypes.csv, - (stream, options, _) => DataSetCsvSerializer.ReadAsync(stream, options) - ) - .Add( - MimeTypes.xlsx, - (stream, _, _) => Task.FromResult(new ExcelDataSetReader().Read(stream)) - ) - .Add(MimeTypes.xls, new ExcelDataSetReaderOld().ReadAsync); - - public ImportConfiguration WithDataSetReader(string fileType, ReadDataSet dataSetReader) => - this with - { - DataSetReaders = DataSetReaders.SetItem(fileType, dataSetReader) - }; - - internal ImmutableDictionary>> StreamProviders { get; init; } - - private ImmutableDictionary>> InitializeStreamProviders() => - ImmutableDictionary>> - .Empty.Add(typeof(StringStream), CreateMemoryStream) - .Add(typeof(EmbeddedResource), CreateEmbeddedResourceStream) - .Add(typeof(CollectionSource), CreateCollectionStreamAsync); - - private static Task CreateEmbeddedResourceStream(ImportRequest request) - { - var embeddedResource = (EmbeddedResource)request.Source; - var assembly = embeddedResource.Assembly; - var resourceName = $"{assembly.GetName().Name}.{embeddedResource.Resource}"; - var stream = assembly.GetManifestResourceStream(resourceName); - if (stream == null) - { - throw new ArgumentException($"Resource '{resourceName}' not found."); - } - return Task.FromResult(stream); - } - - private static Task CreateMemoryStream(ImportRequest request) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(((StringStream)request.Source).Content); - writer.Flush(); - stream.Position = 0; - return Task.FromResult(stream); - } - - private async Task CreateCollectionStreamAsync(ImportRequest request) - { - var collectionSource = (CollectionSource)request.Source; - - // Resolve from ContentCollection - if (ServiceProvider == null) - throw new ImportException("ServiceProvider is not available to resolve CollectionSource from ContentCollection"); - - var contentService = ServiceProvider.GetService(); - if (contentService == null) - throw new ImportException("IContentService is not registered. Ensure ContentCollections are configured."); - - var stream = await contentService.GetContentAsync(collectionSource.Collection, collectionSource.Path); - if (stream == null) - throw new ImportException($"Could not find content at collection '{collectionSource.Collection}' path '{collectionSource.Path}'"); - - return stream; - } - - public ImportConfiguration WithStreamReader( - Type sourceType, - Func> reader - ) => this with { StreamProviders = StreamProviders.SetItem(sourceType, reader) }; - - internal ImmutableList Validations { get; init; } - - public ImportConfiguration WithValidation(ValidationFunction validation) => - this with - { - Validations = Validations.Add(validation) - }; - - private bool StandardValidations(object instance, ValidationContext validationContext, Activity activity) - { - var ret = true; - var validationResults = new List(); - Validator.TryValidateObject(instance, validationContext, validationResults, true); - - foreach (var validation in validationResults) - { - activity.LogError(validation.ToString()); - ret = false; - } - return ret; - } - - public static string MissingCategoryErrorMessage = "Category with name {0} was not found."; - - private static readonly ConcurrentDictionary< - Type, - (Type, string, Func)[] - > TypesWithCategoryAttributes = new(); - - private bool CategoriesValidation(object instance, ValidationContext validationContext, Activity activity) - { - var type = instance.GetType(); - var dimensions = TypesWithCategoryAttributes.GetOrAdd( - type, - key => - key.GetProperties() - .Where(x => x.PropertyType == typeof(string)) - .Select(x => new - { - Attr = x.GetCustomAttribute(), - x.Name - }) - .Where(x => x.Attr != null) - .Select(x => - ( - x.Attr!.Type, - x.Name, - CreateGetter(type, x.Name) - ) - ) - .ToArray() - ); - - var ret = true; - foreach (var (dimensionType, propertyName, propGetter) in dimensions) - { - if (!Workspace.DataContext.DataSourcesByType.ContainsKey(dimensionType)) - { - activity.LogError(string.Format(MissingCategoryErrorMessage, dimensionType)); - ret = false; - continue; - } - //if (!string.IsNullOrEmpty(value)) - // TODO V10: Need to restore categories validation here (03.12.2024, Roland Bürgi) - //if (false) - //{ - // activity.LogError( - // string.Format(UnknownValueErrorMessage, propertyName, type.FullName, propGetter(instance)) - // ); - // ret = false; - //} - } - return ret; - } - - private static Func CreateGetter(Type type, string property) - { - var prm = Expression.Parameter(typeof(object)); - var typedPrm = Expression.Convert(prm, type); - var propertyExpression = Expression.Property(typedPrm, property); - return Expression.Lambda>(propertyExpression, prm).Compile(); - } - - + /// + /// Unique identifier for this configuration (e.g., file name). + /// + [Key] + public required string Name { get; init; } + + /// + /// Entity identifier that this configuration applies to (e.g., PricingId, ProjectId, etc.). + /// + public required string EntityId { get; init; } } diff --git a/src/MeshWeaver.Import/ConfiguredExcelImporter.cs b/src/MeshWeaver.Import/ConfiguredExcelImporter.cs index 5436c09cc..141192721 100644 --- a/src/MeshWeaver.Import/ConfiguredExcelImporter.cs +++ b/src/MeshWeaver.Import/ConfiguredExcelImporter.cs @@ -1,4 +1,4 @@ -using ClosedXML.Excel; +using ClosedXML.Excel; using MeshWeaver.Import.Configuration; using MeshWeaver.Utils; @@ -8,22 +8,16 @@ namespace MeshWeaver.Import; /// Imports entities from Excel files using declarative configuration. /// /// The entity type to import -public class ConfiguredExcelImporter where T : class +public class ConfiguredExcelImporter(Func, T> entityBuilder) + where T : class { - private readonly Func, T> entityBuilder; - - public ConfiguredExcelImporter(Func, T> entityBuilder) - { - this.entityBuilder = entityBuilder; - } - public IEnumerable Import(Stream stream, string sourceName, ExcelImportConfiguration config) { using var wb = new XLWorkbook(stream); var ws = string.IsNullOrWhiteSpace(config.WorksheetName) ? wb.Worksheets.First() : wb.Worksheet(config.WorksheetName); // Pre-read total cells for allocations (tolerate duplicates/invalid addresses) - var allocationTotals = new Dictionary(StringComparer.OrdinalIgnoreCase); + var allocationTotals = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var a in config.Allocations) { if (string.IsNullOrWhiteSpace(a.TotalCell)) @@ -31,7 +25,7 @@ public IEnumerable Import(Stream stream, string sourceName, ExcelImportConfig // Only accept simple A1-style addresses; skip anything suspicious if (!IsValidCellAddress(a.TotalCell)) continue; - var totalVal = GetCellDecimal(ws, a.TotalCell) ?? 0m; + var totalVal = GetCellDouble(ws, a.TotalCell) ?? 0; // Last write wins if duplicates exist; avoids ArgumentException allocationTotals[a.TotalCell] = totalVal; } @@ -49,10 +43,10 @@ public IEnumerable Import(Stream stream, string sourceName, ExcelImportConfig rows = rows.Where(r => !IsIgnoredByExpressions(r, config)); // Pre-calc denominators for each allocation (sum of weights over data rows) - var allocationDenominators = new Dictionary(); + var allocationDenominators = new Dictionary(); foreach (var alloc in config.Allocations) { - decimal denom = 0m; + var denom = 0.0; foreach (var row in rows) { denom += SumWeightColumns(row, alloc.WeightColumns); @@ -198,43 +192,43 @@ private static bool EvaluatePropertyNull(IXLRow row, ExcelImportConfiguration co return s; } - private static decimal SumColumns(IXLRow row, IEnumerable columnLetters) - => columnLetters.Select(c => GetCellDecimal(row.Worksheet, c + row.RowNumber()) ?? 0m).Sum(); + private static double SumColumns(IXLRow row, IEnumerable columnLetters) + => columnLetters.Select(c => GetCellDouble(row.Worksheet, c + row.RowNumber()) ?? 0).Sum(); - private static decimal SumWeightColumns(IXLRow row, IEnumerable columnLetters) + private static double SumWeightColumns(IXLRow row, IEnumerable columnLetters) { - decimal sum = 0m; + var sum = 0.0; foreach (var col in columnLetters) { - var val = GetCellDecimal(row.Worksheet, col + row.RowNumber()); + var val = GetCellDouble(row.Worksheet, col + row.RowNumber()); if (val.HasValue) sum += val.Value; } return sum; } - private static decimal DiffColumns(IXLRow row, IEnumerable columnLetters) + private static double DiffColumns(IXLRow row, IEnumerable columnLetters) { var cols = columnLetters.Take(2).ToArray(); - var a = GetCellDecimal(row.Worksheet, cols.ElementAtOrDefault(0) + row.RowNumber()) ?? 0m; - var b = GetCellDecimal(row.Worksheet, cols.ElementAtOrDefault(1) + row.RowNumber()) ?? 0m; + var a = GetCellDouble(row.Worksheet, cols.ElementAtOrDefault(0) + row.RowNumber()) ?? 0; + var b = GetCellDouble(row.Worksheet, cols.ElementAtOrDefault(1) + row.RowNumber()) ?? 0; return b - a; } - private static decimal? GetCellDecimal(IXLWorksheet ws, string cellAddress) + private static double? GetCellDouble(IXLWorksheet ws, string cellAddress) { var cell = ws.Cell(cellAddress); cell = ResolveMergedAnchor(cell); if (cell.DataType == XLDataType.Number) { - if (cell.TryGetValue(out decimal dec)) return dec; - if (cell.TryGetValue(out double dbl)) return (decimal)dbl; + if (cell.TryGetValue(out double dec)) return dec; + if (cell.TryGetValue(out double dbl)) return (double)dbl; if (cell.TryGetValue(out int i)) return i; } var str = GetStringSafe(cell); // Try invariant culture first, then current culture, allow currency/thousands - if (decimal.TryParse(str, System.Globalization.NumberStyles.Number | System.Globalization.NumberStyles.AllowCurrencySymbol, System.Globalization.CultureInfo.InvariantCulture, out var parsed)) + if (double.TryParse(str, System.Globalization.NumberStyles.Number | System.Globalization.NumberStyles.AllowCurrencySymbol, System.Globalization.CultureInfo.InvariantCulture, out var parsed)) return parsed; - if (decimal.TryParse(str, System.Globalization.NumberStyles.Number | System.Globalization.NumberStyles.AllowCurrencySymbol, System.Globalization.CultureInfo.CurrentCulture, out parsed)) + if (double.TryParse(str, System.Globalization.NumberStyles.Number | System.Globalization.NumberStyles.AllowCurrencySymbol, System.Globalization.CultureInfo.CurrentCulture, out parsed)) return parsed; return null; } diff --git a/src/MeshWeaver.Import/ExcelImportExtensions.cs b/src/MeshWeaver.Import/ExcelImportExtensions.cs index 97cfee11f..79b032b75 100644 --- a/src/MeshWeaver.Import/ExcelImportExtensions.cs +++ b/src/MeshWeaver.Import/ExcelImportExtensions.cs @@ -14,7 +14,7 @@ public static class ExcelImportExtensions /// Function to build an entity from property dictionary /// Enumerable of imported entities public static IEnumerable ImportExcel( - this ImportConfiguration configuration, + this ImportBuilder configuration, Stream stream, ExcelImportConfiguration excelConfig, Func, T> entityBuilder) where T : class @@ -33,7 +33,7 @@ public static IEnumerable ImportExcel( /// Function to build an entity from property dictionary /// Enumerable of imported entities public static IEnumerable ImportExcel( - this ImportConfiguration configuration, + this ImportBuilder configuration, string filePath, ExcelImportConfiguration excelConfig, Func, T> entityBuilder) where T : class diff --git a/src/MeshWeaver.Import/Implementation/ImportManager.cs b/src/MeshWeaver.Import/Implementation/ImportManager.cs index 4ac3edc2b..7589ee18a 100644 --- a/src/MeshWeaver.Import/Implementation/ImportManager.cs +++ b/src/MeshWeaver.Import/Implementation/ImportManager.cs @@ -10,7 +10,7 @@ namespace MeshWeaver.Import.Implementation; public class ImportManager { - public ImportConfiguration Configuration { get; } + public ImportBuilder Configuration { get; } public IWorkspace Workspace { get; } public IMessageHub Hub { get; } @@ -22,7 +22,7 @@ public ImportManager(IWorkspace workspace, IMessageHub hub) Workspace = workspace; Hub = hub; - Configuration = hub.Configuration.GetListOfLambdas().Aggregate(new ImportConfiguration(workspace, hub.ServiceProvider), (c, l) => l.Invoke(c)); + Configuration = hub.Configuration.GetListOfLambdas().Aggregate(new ImportBuilder(workspace, hub.ServiceProvider), (c, l) => l.Invoke(c)); // Don't initialize the import hub in constructor - do it lazily to avoid timing issues logger?.LogDebug("ImportManager constructor completed for hub {HubAddress}", hub.Address); @@ -126,11 +126,117 @@ public async Task ImportInstancesAsync( Activity? activity, CancellationToken cancellationToken) { + // If ExcelImportConfiguration is provided, use ConfiguredExcelImporter directly + if (importRequest.Configuration is ExcelImportConfiguration excelConfig) + { + return await ImportWithConfiguredExcelImporter(importRequest, excelConfig, activity, cancellationToken); + } + var (dataSet, format) = await ReadDataSetAsync(importRequest, activity, cancellationToken); var imported = await format.Import(importRequest, dataSet, activity, cancellationToken); return imported!; } + private async Task ImportWithConfiguredExcelImporter( + ImportRequest importRequest, + ExcelImportConfiguration config, + Activity? activity, + CancellationToken cancellationToken) + { + activity?.LogInformation("Using ConfiguredExcelImporter with TypeName: {TypeName}", config.TypeName); + + // Get the stream provider + var sourceType = importRequest.Source.GetType(); + if (!Configuration.StreamProviders.TryGetValue(sourceType, out var streamProvider)) + throw new ImportException($"Unknown stream type: {sourceType.FullName}"); + + var stream = await streamProvider.Invoke(importRequest); + if (stream == null) + throw new ImportException($"Could not open stream: {importRequest.Source}"); + + // Get the source name for tracking + var sourceName = importRequest.Source switch + { + CollectionSource cs => cs.Path, + _ => "unknown" + }; + + // Resolve the entity type from TypeName + if (string.IsNullOrWhiteSpace(config.TypeName)) + throw new ImportException("TypeName is required in ExcelImportConfiguration"); + + var entityType = ResolveType(config.TypeName); + if (entityType == null) + throw new ImportException($"Could not resolve type: {config.TypeName}"); + + activity?.LogInformation("Resolved entity type: {EntityType}", entityType.FullName); + + // Create entity builder using AutoEntityBuilder.CreateBuilder() generic method + var builderGenericMethod = typeof(AutoEntityBuilder).GetMethods() + .FirstOrDefault(m => m.Name == nameof(AutoEntityBuilder.CreateBuilder) && m.IsGenericMethod); + if (builderGenericMethod == null) + throw new ImportException("Could not find AutoEntityBuilder.CreateBuilder method"); + + var builderMethod = builderGenericMethod.MakeGenericMethod(entityType); + var entityBuilder = builderMethod.Invoke(null, null); + if (entityBuilder == null) + throw new ImportException("Failed to create entity builder"); + + // Create ConfiguredExcelImporter using reflection (since it's generic) + var importerType = typeof(ConfiguredExcelImporter<>).MakeGenericType(entityType); + var importer = Activator.CreateInstance(importerType, entityBuilder); + if (importer == null) + throw new ImportException($"Failed to create ConfiguredExcelImporter<{entityType.Name}>"); + + // Call the Import method + var importMethod = importerType.GetMethod(nameof(ConfiguredExcelImporter.Import), new[] { typeof(Stream), typeof(string), typeof(ExcelImportConfiguration) }); + if (importMethod == null) + throw new ImportException("Could not find Import method on ConfiguredExcelImporter"); + + var importedEntities = importMethod.Invoke(importer, new object[] { stream, sourceName, config }) as System.Collections.IEnumerable; + if (importedEntities == null) + throw new ImportException("Import returned null"); + + // Convert to EntityStore + var entities = importedEntities.Cast().ToArray(); + activity?.LogInformation("Imported {Count} entities of type {TypeName}", entities.Length, config.TypeName); + + var entityStore = new EntityStore(); + var instanceDict = new Dictionary(); + + foreach (var entity in entities) + { + var id = GetEntityId(entity, entityType); + instanceDict[id] = entity; + } + + var collection = new InstanceCollection(instanceDict); + var collectionName = entityType.FullName ?? entityType.Name; + + entityStore = entityStore.WithCollection(collectionName, collection); + return entityStore; + } + + private string GetEntityId(object entity, Type entityType) + { + // Try to get Id property + var idProp = entityType.GetProperty("Id"); + if (idProp != null) + { + var idValue = idProp.GetValue(entity); + if (idValue != null) + return idValue.ToString() ?? Guid.NewGuid().ToString(); + } + + // Fall back to hash code or GUID + return entity.GetHashCode().ToString(); + } + + private Type? ResolveType(string typeName) + { + return Hub.TypeRegistry.GetType(typeName); + } + private async Task<(IDataSet dataSet, ImportFormat format)> ReadDataSetAsync(ImportRequest importRequest, Activity? activity, CancellationToken cancellationToken) @@ -158,7 +264,9 @@ public async Task ImportInstancesAsync( cancellationToken ); activity?.LogInformation("Read data set with {Tables} tables. Will import in format {Format}", dataSet.Tables.Count, format); + format ??= importRequest.Format; + if (format == null) throw new ImportException("Format not specified."); diff --git a/src/MeshWeaver.Import/Implementation/ImportUnpartitionedDataSource.cs b/src/MeshWeaver.Import/Implementation/ImportUnpartitionedDataSource.cs index 173b7e332..422cc7166 100644 --- a/src/MeshWeaver.Import/Implementation/ImportUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Import/Implementation/ImportUnpartitionedDataSource.cs @@ -30,11 +30,11 @@ protected override async Task GetInitialValueAsync(ISynchronization } private ImmutableList< - Func + Func > Configurations { get; init; } = - ImmutableList>.Empty; + ImmutableList>.Empty; public ImportUnpartitionedDataSource WithImportConfiguration( - Func config + Func config ) => this with { Configurations = Configurations.Add(config) }; } diff --git a/src/MeshWeaver.Import/ImportRegistryExtensions.cs b/src/MeshWeaver.Import/ImportRegistryExtensions.cs index 3e83aa5f8..1f22a2e8f 100644 --- a/src/MeshWeaver.Import/ImportRegistryExtensions.cs +++ b/src/MeshWeaver.Import/ImportRegistryExtensions.cs @@ -17,7 +17,7 @@ public static MessageHubConfiguration AddImport(this MessageHubConfiguration con public static MessageHubConfiguration AddImport( this MessageHubConfiguration configuration, - Func importConfiguration + Func importConfiguration ) { var lambdas = configuration.GetListOfLambdas(); @@ -117,10 +117,10 @@ EmbeddedResource source return ret; } - internal static ImmutableList> GetListOfLambdas( + internal static ImmutableList> GetListOfLambdas( this MessageHubConfiguration config ) => - config.Get>>() ?? []; + config.Get>>() ?? []; } public record EmbeddedResource(Assembly Assembly, string Resource) : Source; diff --git a/src/MeshWeaver.Import/ImportRequest.cs b/src/MeshWeaver.Import/ImportRequest.cs index 1185fb653..c22824384 100644 --- a/src/MeshWeaver.Import/ImportRequest.cs +++ b/src/MeshWeaver.Import/ImportRequest.cs @@ -32,6 +32,12 @@ public ImportRequest(Source Source) public string MimeType { get; init; } public string Format { get; init; } = ImportFormat.Default; + + /// + /// Optional import configuration. When provided, this configuration will be used instead of the Format string. + /// + public ImportConfiguration? Configuration { get; init; } + public object? TargetDataSource { get; init; } public UpdateOptions UpdateOptions { get; init; } = UpdateOptions.Default; public DataSetReaderOptions DataSetReaderOptions { get; init; } = new(); diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs index 6919b1b1e..cd4d51265 100644 --- a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -186,4 +186,50 @@ public async Task CollectionPlugin_Import_WithCustomFormat_ShouldImportSuccessfu allData.Should().HaveCount(3); } + + [Fact] + public async Task CollectionPlugin_Import_WithConfiguration_ShouldImportWithoutFormat() + { + // Arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // Create a configuration JSON that is not registered as a format + var configurationJson = @"{ + ""$type"": ""MeshWeaver.Import.Configuration.ImportConfiguration"", + ""name"": ""test-config-not-registered"", + ""entityId"": ""2024"" + }"; + + // Act + var result = await plugin.Import( + path: "test-data.csv", + collection: "TestCollection", + address: new ImportAddress(2024), + format: null, + configuration: configurationJson, + cancellationToken: TestContext.Current.CancellationToken + ); + + // Assert + // The import should succeed even though "test-config-not-registered" is not a registered format + // This demonstrates that when configuration is provided, it bypasses format resolution + result.Should().Contain("succeeded", "import should succeed with configuration even if not registered as format"); + result.Should().NotContain("Error", "there should be no errors"); + result.Should().NotContain("Unknown format", "should not try to resolve configuration as format"); + + // Verify data was imported + var referenceDataHub = Router.GetHostedHub(new ReferenceDataAddress()); + var workspace = referenceDataHub.ServiceProvider.GetRequiredService(); + + var allData = await workspace.GetObservable() + .Timeout(10.Seconds()) + .FirstAsync(x => x.Count >= 3); + + allData.Should().HaveCount(3); + var items = allData.OrderBy(x => x.SystemName).ToList(); + items[0].DisplayName.Should().Be("LoB 1"); + items[1].DisplayName.Should().Be("LoB 2"); + items[2].DisplayName.Should().Be("LoB 3"); + } } From 164ba430834f29126bf87126a1c3d8224770aae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 27 Oct 2025 23:30:05 +0100 Subject: [PATCH 04/57] improving mapping for RiskImportAgent --- .../RiskImportAgent.cs | 141 +++----- src/MeshWeaver.AI/MeshWeaver.AI.csproj | 1 + src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 113 ++++++- .../FileSystemStreamProvider.cs | 2 +- .../CollectionPluginTest.cs | 320 ++++++++++++++++++ 5 files changed, 477 insertions(+), 100 deletions(-) create mode 100644 test/MeshWeaver.AI.Test/CollectionPluginTest.cs diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index 410d9cff3..21a853d77 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -1,15 +1,12 @@ using System.ComponentModel; -using System.Text; using System.Text.Json; using System.Text.Json.Nodes; -using ClosedXML.Excel; using MeshWeaver.AI; using MeshWeaver.AI.Plugins; using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.Insurance.Domain; using MeshWeaver.Messaging; -using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; namespace MeshWeaver.Insurance.AI; @@ -43,9 +40,9 @@ public string Instructions # Importing Risks When the user asks you to import risks, you should: 1) Get the existing risk mapping configuration for the specified file using the function {{{nameof(RiskImportPlugin.GetRiskImportConfiguration)}}} with the filename. - 2) If no import configuration was returned in 1, get a sample of the worksheet using {{{nameof(RiskImportPlugin.GetWorksheetSample)}}} with the filename and extract the table start row as well as the mapping as in the schema provided below. + 2) If no import configuration was returned in 1, get a sample of the worksheet using CollectionPlugin's GetFile function with the collection name "Submissions-{pricingId}", the filename, and numberOfRows=20. Extract the table start row as well as the mapping as in the schema provided below. Consider any input from the user to modify the configuration. Use the {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} function to save the configuration. - 3) Call Import with the filename. The Import function will automatically use the saved configuration. + 3) Call Import with the filename and the configuration you have updated or created. # Updating Risk Import Configuration When the user asks you to update the risk import configuration, you should: @@ -54,10 +51,24 @@ public string Instructions 3) Upload the new configuration using the function {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} with the filename and the updated mapping. # Automatic Risk Import Configuration - - Read the column header from the row which you determine to be the first of the Data Table and map to column numbers. - - Map to the properties of the PropertyRisk type (see schema below). Only these names are allowed for mappings. Read the descriptions contained in the schema to get guidance on which field to map where - - Columns you cannot map ==> ignore. - - Watch out for empty columns at the beginning of the table. In this case, see that you get the column index right. + - Use CollectionPlugin's GetFile with numberOfRows=20 to get a sample of the file. It returns a markdown table with: + - First column: Row numbers (1-based) + - Remaining columns: Labeled A, B, C, D, etc. (Excel column letters) + - Empty cells appear as empty values in the table (not "null") + - Column letters start with A (first data column after Row number). Empty columns are still shown with their letters. + - Row numbers are 1-based. When specifying tableStartRow, use the row number from the Row column (e.g., if headers are on row 1 and data starts on row 2, set tableStartRow=2). + - Look for the header row in the markdown table and map column letters (A, B, C, etc.) to PropertyRisk properties. + - Map to the properties of the PropertyRisk type (see schema below). Only these names are allowed for mappings. Read the descriptions contained in the schema to get guidance on which field to map where. + - IMPORTANT: Each TargetProperty should appear ONLY ONCE in the configuration. If a property maps to multiple columns, use the SourceColumns list (e.g., "sourceColumns": ["A", "B"]) instead of creating multiple entries with the same TargetProperty. + - IMPORTANT: Each column (A, B, C, etc.) should be mapped ONLY ONCE across all mappings. Do not include the same column in multiple targetProperty mappings or sourceColumns lists. + - Columns you cannot map ==> ignore (don't include them in the configuration). + - Empty columns at the beginning still get column letters (A, B, C...). You can see which columns are empty by looking at the markdown table. + + # TsiContent Mapping + - MOST COLUMNS will be mapped to the 'tsiContent' property (Total Sum Insured content breakdown). + - Common column headers for tsiContent include: Stock, Fixtures, Fittings, IT Equipment, Land, Leasehold Improv., Leasehold Improvements, Plant & Equipment, Tooling, Workshop Equipment, Rent Forecast. + - These columns typically represent different categories of insured content and should be mapped to tsiContent using the SourceColumns list. + - Example: If you see columns for "Stock", "Fixtures", "IT Equipment", map them as: "targetProperty": "tsiContent", "sourceColumns": ["E", "F", "G"] Notes: - The agent defaults to ignoring rows where Id or Address is missing (adds "Id == null" and "Address == null" to ignoreRowExpressions). @@ -152,7 +163,33 @@ async Task IInitializableAgent.InitializeAsync() var resp = await hub.AwaitResponse( new GetSchemaRequest("ExcelImportConfiguration"), o => o.WithTarget(new PricingAddress("default"))); - excelImportConfigSchema = resp.Message.Schema; + + // Hard-code TypeName to "PropertyRisk" in the schema + var schema = resp.Message.Schema; + if (!string.IsNullOrEmpty(schema)) + { + // Parse the schema as JSON to modify it + try + { + var schemaJson = JsonNode.Parse(schema) as JsonObject; + if (schemaJson?["anyOf"] is JsonArray array && array.First() is JsonObject obj && obj["properties"] is JsonObject properties) + { + // Set TypeName property to have a constant value of "PropertyRisk" + properties["typeName"] = new JsonObject + { + ["type"] = "string", + ["const"] = "PropertyRisk", + ["description"] = "The fully qualified type name of the entity to import. This is hard-coded to 'PropertyRisk' for risk imports." + }; + schema = schemaJson.ToJsonString(); + } + } + catch + { + // If parsing fails, use original schema + } + } + excelImportConfigSchema = schema; } catch { @@ -292,90 +329,6 @@ private static JsonObject EnsureTypeFirst(JsonObject source, string typeName) return ordered; } - [KernelFunction] - [Description("Gets the first 20 rows for each worksheet in the workbook to help determine the mapping")] - public async Task GetWorksheetSample(string filename) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing first."; - - try - { - var pricingId = chat.Context.Address.Id; - var contentService = hub.ServiceProvider.GetRequiredService(); - var stream = await OpenContentReadStreamAsync(contentService, pricingId, filename); - - if (stream is null) - return $"Content not found: {filename}"; - - await using (stream) - { - using var wb = new XLWorkbook(stream); - var sb = new StringBuilder(); - - foreach (var ws in wb.Worksheets) - { - var used = ws.RangeUsed(); - sb.AppendLine($"Sheet: {ws.Name}"); - if (used is null) - { - sb.AppendLine("(No data)"); - sb.AppendLine(); - continue; - } - - var firstRow = used.FirstRow().RowNumber(); - var lastRow = Math.Min(used.FirstRow().RowNumber() + 19, used.LastRow().RowNumber()); - var firstCol = 1; - var lastCol = used.LastColumn().ColumnNumber(); - - for (var r = firstRow; r <= lastRow; r++) - { - var rowVals = new List(); - for (var c = firstCol; c <= lastCol; c++) - { - var raw = ws.Cell(r, c).GetValue(); - var val = raw?.Replace('\n', ' ').Replace('\r', ' ').Trim(); - rowVals.Add(string.IsNullOrEmpty(val) ? "null" : val); - } - - sb.AppendLine(string.Join('\t', rowVals)); - } - - sb.AppendLine(); - } - - return sb.ToString(); - } - } - catch (Exception e) - { - return $"Error reading sample: {e.Message}"; - } - } - - - private static async Task OpenContentReadStreamAsync( - IContentService contentService, - string pricingId, - string filename) - { - try - { - var collectionName = $"Submissions-{pricingId}"; - - var collection = await contentService.GetCollectionAsync(collectionName, CancellationToken.None); - if (collection is null) - return null; - - return await collection.GetContentAsync(filename); - } - catch - { - return null; - } - } - private string ExtractJson(string json) { return json.Replace("```json", "") diff --git a/src/MeshWeaver.AI/MeshWeaver.AI.csproj b/src/MeshWeaver.AI/MeshWeaver.AI.csproj index 56ee53906..fb026df54 100644 --- a/src/MeshWeaver.AI/MeshWeaver.AI.csproj +++ b/src/MeshWeaver.AI/MeshWeaver.AI.csproj @@ -6,6 +6,7 @@ + diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index 5b74e47d7..f10441767 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -1,6 +1,8 @@ using System.ComponentModel; +using System.Text; using System.Text.Json; using System.Text.Json.Nodes; +using ClosedXML.Excel; using MeshWeaver.ContentCollections; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; @@ -18,8 +20,9 @@ public class CollectionPlugin(IMessageHub hub) [KernelFunction] [Description("Gets the content of a file from a specified collection.")] public async Task GetFile( - [Description("The name of the collection to read from")] string collectionName, + [Description("The name of the collection to read from. If null, uses the default collection.")] string collectionName, [Description("The path to the file within the collection")] string filePath, + [Description("Optional: number of rows to read. If null, reads entire file. For Excel files, reads first N rows from each worksheet.")] int? numberOfRows = null, CancellationToken cancellationToken = default) { try @@ -32,10 +35,32 @@ public async Task GetFile( if (stream == null) return $"File '{filePath}' not found in collection '{collectionName}'."; - using var reader = new StreamReader(stream); - var content = await reader.ReadToEndAsync(cancellationToken); + // Check if this is an Excel file + var extension = Path.GetExtension(filePath).ToLowerInvariant(); + if (extension == ".xlsx" || extension == ".xls") + { + return await ReadExcelFileAsync(stream, filePath, numberOfRows, cancellationToken); + } - return content; + // For non-Excel files, read as text + using var reader = new StreamReader(stream); + if (numberOfRows.HasValue) + { + var sb = new StringBuilder(); + var linesRead = 0; + while (!reader.EndOfStream && linesRead < numberOfRows.Value) + { + var line = await reader.ReadLineAsync(cancellationToken); + sb.AppendLine(line); + linesRead++; + } + return sb.ToString(); + } + else + { + var content = await reader.ReadToEndAsync(cancellationToken); + return content; + } } catch (FileNotFoundException) { @@ -46,6 +71,84 @@ public async Task GetFile( return $"Error reading file '{filePath}' from collection '{collectionName}': {ex.Message}"; } } + + private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows, CancellationToken cancellationToken) + { + try + { + using var wb = new XLWorkbook(stream); + var sb = new StringBuilder(); + + foreach (var ws in wb.Worksheets) + { + var used = ws.RangeUsed(); + sb.AppendLine($"## Sheet: {ws.Name}"); + sb.AppendLine(); + if (used is null) + { + sb.AppendLine("(No data)"); + sb.AppendLine(); + continue; + } + + var firstRow = used.FirstRow().RowNumber(); + var lastRow = numberOfRows.HasValue + ? Math.Min(used.FirstRow().RowNumber() + numberOfRows.Value - 1, used.LastRow().RowNumber()) + : used.LastRow().RowNumber(); + var firstCol = 1; + var lastCol = used.LastColumn().ColumnNumber(); + + // Build markdown table with column letters as headers + var columnHeaders = new List { "Row" }; + for (var c = firstCol; c <= lastCol; c++) + { + // Convert column number to Excel letter (1=A, 2=B, ..., 27=AA, etc.) + columnHeaders.Add(GetExcelColumnLetter(c)); + } + + // Header row + sb.AppendLine("| " + string.Join(" | ", columnHeaders) + " |"); + // Separator row + sb.AppendLine("|" + string.Join("", columnHeaders.Select(_ => "---:|"))); + + // Data rows + for (var r = firstRow; r <= lastRow; r++) + { + var rowVals = new List { r.ToString() }; + for (var c = firstCol; c <= lastCol; c++) + { + var cell = ws.Cell(r, c); + var raw = cell.GetValue(); + var val = raw?.Replace('\n', ' ').Replace('\r', ' ').Replace("|", "\\|").Trim(); + // Empty cells show as empty in table + rowVals.Add(string.IsNullOrEmpty(val) ? "" : val); + } + + sb.AppendLine("| " + string.Join(" | ", rowVals) + " |"); + } + + sb.AppendLine(); + } + + return sb.ToString(); + } + catch (Exception ex) + { + return $"Error reading Excel file '{filePath}': {ex.Message}"; + } + } + + private static string GetExcelColumnLetter(int columnNumber) + { + var columnLetter = ""; + while (columnNumber > 0) + { + var modulo = (columnNumber - 1) % 26; + columnLetter = Convert.ToChar('A' + modulo) + columnLetter; + columnNumber = (columnNumber - 1) / 26; + } + return columnLetter; + } [KernelFunction] [Description("Saves content as a file to a specified collection.")] public async Task SaveFile( @@ -277,7 +380,7 @@ private void EnsureDirectoryExists(object collection, string filePath) } catch (Exception) { - // If we can't create directories through reflection, + // If we can't create directories through reflection, // let the SaveFileAsync method handle any directory creation or fail gracefully } } diff --git a/src/MeshWeaver.ContentCollections/FileSystemStreamProvider.cs b/src/MeshWeaver.ContentCollections/FileSystemStreamProvider.cs index 9da51a7f9..94df466be 100644 --- a/src/MeshWeaver.ContentCollections/FileSystemStreamProvider.cs +++ b/src/MeshWeaver.ContentCollections/FileSystemStreamProvider.cs @@ -16,7 +16,7 @@ public class FileSystemStreamProvider(string basePath) : IStreamProvider public Task GetStreamAsync(string reference, CancellationToken cancellationToken = default) { - var fullPath = Path.IsPathRooted(reference) ? reference : Path.Combine(basePath, reference.TrimStart('/')); + var fullPath = Path.Combine(basePath, reference.TrimStart('/')); if (!File.Exists(fullPath)) { return Task.FromResult(null); diff --git a/test/MeshWeaver.AI.Test/CollectionPluginTest.cs b/test/MeshWeaver.AI.Test/CollectionPluginTest.cs new file mode 100644 index 000000000..ac12d92d6 --- /dev/null +++ b/test/MeshWeaver.AI.Test/CollectionPluginTest.cs @@ -0,0 +1,320 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ClosedXML.Excel; +using FluentAssertions; +using MeshWeaver.AI.Plugins; +using MeshWeaver.ContentCollections; +using MeshWeaver.Fixture; +using MeshWeaver.Messaging; +using Xunit; + +namespace MeshWeaver.AI.Test; + +/// +/// Tests for CollectionPlugin functionality, specifically the GetFile method with Excel support +/// +public class CollectionPluginTest(ITestOutputHelper output) : HubTestBase(output), IAsyncLifetime +{ + private const string TestCollectionName = "test-collection"; + private const string TestExcelFileName = "test.xlsx"; + private const string TestTextFileName = "test.txt"; + private readonly string collectionBasePath = Path.Combine(Path.GetTempPath(), $"CollectionPluginTest_{Guid.NewGuid()}"); + + /// + /// Initialize the test + /// + public override async ValueTask InitializeAsync() + { + await base.InitializeAsync(); + + // Create directory for test files + Directory.CreateDirectory(collectionBasePath); + + // Create test Excel file with empty cells at the start + CreateTestExcelFile(); + + // Create test text file + await CreateTestTextFile(); + } /// + /// Dispose the test + /// + public override async ValueTask DisposeAsync() + { + // Clean up test files and directory + if (Directory.Exists(collectionBasePath)) + { + try + { + Directory.Delete(collectionBasePath, true); + } + catch + { + // Ignore cleanup errors + } + } + + await base.DisposeAsync(); + } + + /// + /// Creates a test Excel file with multiple worksheets and empty cells at the start of rows + /// + private void CreateTestExcelFile() + { + using var wb = new XLWorkbook(); + + // Create first worksheet with data including null cells at the start + var ws1 = wb.Worksheets.Add("Sheet1"); + + // Header row with empty cells at start + ws1.Cell(1, 1).Value = ""; // Empty cell + ws1.Cell(1, 2).Value = ""; // Empty cell + ws1.Cell(1, 3).Value = "ID"; + ws1.Cell(1, 4).Value = "Name"; + ws1.Cell(1, 5).Value = "Value"; + + // Data rows with empty cells + ws1.Cell(2, 1).Value = ""; // Empty + ws1.Cell(2, 2).Value = ""; // Empty + ws1.Cell(2, 3).Value = "1"; + ws1.Cell(2, 4).Value = "Item A"; + ws1.Cell(2, 5).Value = "100"; + + ws1.Cell(3, 1).Value = ""; // Empty + ws1.Cell(3, 2).Value = ""; // Empty + ws1.Cell(3, 3).Value = "2"; + ws1.Cell(3, 4).Value = ""; // Empty cell in middle + ws1.Cell(3, 5).Value = "200"; + + // Add more rows for testing row limiting + for (int i = 4; i <= 30; i++) + { + ws1.Cell(i, 1).Value = ""; + ws1.Cell(i, 2).Value = ""; + ws1.Cell(i, 3).Value = i - 1; + ws1.Cell(i, 4).Value = $"Item {(char)('A' + i - 2)}"; + ws1.Cell(i, 5).Value = i * 100; + } + + // Create second worksheet with minimal data + var ws2 = wb.Worksheets.Add("Sheet2"); + ws2.Cell(1, 1).Value = "Column1"; + ws2.Cell(1, 2).Value = "Column2"; + ws2.Cell(2, 1).Value = "Value1"; + ws2.Cell(2, 2).Value = "Value2"; + + var filePath = Path.Combine(collectionBasePath!, TestExcelFileName); + wb.SaveAs(filePath); + } + + /// + /// Creates a test text file with multiple lines + /// + private async Task CreateTestTextFile() + { + var sb = new StringBuilder(); + for (int i = 1; i <= 50; i++) + { + sb.AppendLine($"Line {i}"); + } + + var filePath = Path.Combine(collectionBasePath!, TestTextFileName); + await File.WriteAllTextAsync(filePath, sb.ToString()); + } + + /// + /// Tests that GetFile preserves null values in Excel files with empty cells at the start of rows + /// + [Fact] + public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // act + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName); + + // assert + result.Should().NotBeNullOrEmpty(); + result.Should().Contain("## Sheet: Sheet1"); + result.Should().Contain("## Sheet: Sheet2"); + + // Verify markdown table structure with column headers + result.Should().Contain("| Row | A | B | C | D | E |"); + + // Verify that empty cells at the start show as empty in the table + var lines = result.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var headerLine = lines.FirstOrDefault(l => l.Contains("ID") && l.Contains("Name")); + headerLine.Should().NotBeNull(); + headerLine.Should().Contain("| 1 | | | ID | Name | Value |"); + + var secondDataLine = lines.FirstOrDefault(l => l.Contains("Item A")); + secondDataLine.Should().NotBeNull(); + secondDataLine.Should().Contain("| 2 | | | 1 | Item A | 100 |"); + + // Verify empty cell in the middle (row 3 has empty Name column) + var thirdDataLine = lines.FirstOrDefault(l => l.Contains("| 3 |")); + thirdDataLine.Should().NotBeNull(); + thirdDataLine.Should().Contain("| 3 | | | 2 | | 200 |"); + } + + /// + /// Tests that GetFile with numberOfRows parameter limits Excel file output + /// + [Fact] + public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + const int rowLimit = 5; + + // act + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, numberOfRows: rowLimit); + + // assert + result.Should().NotBeNullOrEmpty(); + result.Should().Contain("## Sheet: Sheet1"); + + // Count the number of data rows in the markdown table (excluding header and separator) + var lines = result.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var dataLines = lines + .SkipWhile(l => !l.Contains("| Row |")) + .Skip(2) // Skip header and separator + .TakeWhile(l => l.StartsWith("|") && !l.Contains("## Sheet:")) + .ToList(); + + dataLines.Count.Should().Be(rowLimit); + + // Verify it still has the markdown table structure + result.Should().Contain("| Row | A | B | C | D | E |"); + } + + /// + /// Tests that GetFile with numberOfRows parameter limits text file output + /// + [Fact] + public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + const int rowLimit = 10; + + // act + var result = await plugin.GetFile(TestCollectionName, TestTextFileName, numberOfRows: rowLimit); + + // assert + result.Should().NotBeNullOrEmpty(); + + var lines = result.Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()) + .ToArray(); + lines.Length.Should().Be(rowLimit); + + lines[0].Should().Be("Line 1"); + lines[9].Should().Be("Line 10"); + } + + /// + /// Tests that GetFile without numberOfRows parameter reads entire text file + /// + [Fact] + public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // act + var result = await plugin.GetFile(TestCollectionName, TestTextFileName); + + // assert + result.Should().NotBeNullOrEmpty(); + + var lines = result.Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(l => l.Trim()) + .ToArray(); + lines.Length.Should().Be(50); + + lines[0].Should().Be("Line 1"); + lines[49].Should().Be("Line 50"); + } + + /// + /// Tests that GetFile without numberOfRows parameter reads entire Excel file + /// + [Fact] + public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // act + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName); + + // assert + result.Should().NotBeNullOrEmpty(); + result.Should().Contain("## Sheet: Sheet1"); + result.Should().Contain("## Sheet: Sheet2"); + + // Should have all 30 rows from Sheet1 in the markdown table + var lines = result.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var sheet1DataLines = lines + .SkipWhile(l => !l.Contains("## Sheet: Sheet1")) + .SkipWhile(l => !l.Contains("| Row |")) + .Skip(2) // Skip header and separator + .TakeWhile(l => l.StartsWith("|") && !l.Contains("## Sheet:")) + .ToList(); + + sheet1DataLines.Count.Should().Be(30); + } + + /// + /// Tests that GetFile handles non-existent collection + /// + [Fact] + public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // act + var result = await plugin.GetFile("non-existent-collection", "test.xlsx"); + + // assert + result.Should().Contain("Collection 'non-existent-collection' not found"); + } + + /// + /// Tests that GetFile handles non-existent file + /// + [Fact] + public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() + { + // arrange + var client = GetClient(); + var plugin = new CollectionPlugin(client); + + // act + var result = await plugin.GetFile(TestCollectionName, "non-existent.xlsx"); + + // assert + result.Should().Contain("File 'non-existent.xlsx' not found"); + } + + /// + /// Configuration for test client + /// + protected override MessageHubConfiguration ConfigureClient(MessageHubConfiguration configuration) + { + return base.ConfigureClient(configuration) + .AddFileSystemContentCollection(TestCollectionName, _ => collectionBasePath); + } +} From 6326cc257f565a99f8610d82a128b8544bdb9d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 28 Oct 2025 00:42:16 +0100 Subject: [PATCH 05/57] introducing geo coding. --- .../MeshWeaver.Insurance.AI/InsuranceAgent.cs | 4 +- .../RiskImportAgent.cs | 15 +- .../SlipImportAgent.cs | 20 +- .../GeocodingRequest.cs | 34 +++ .../InsuranceApplicationExtensions.cs | 98 +++++++- .../LayoutAreas/PropertyRisksLayoutArea.cs | 39 ++- .../LayoutAreas/RiskMapLayoutArea.cs | 44 +++- .../MeshWeaver.Insurance.Domain.csproj | 1 + .../PropertyRisk.cs | 24 +- .../Services/GoogleGeocodingService.cs | 237 ++++++++++++++++++ .../Services/IGeocodingService.cs | 28 +++ .../InsuranceTestBase.cs | 4 +- .../MeshWeaver.Northwind.AI/NorthwindAgent.cs | 12 +- .../SharedPortalConfiguration.cs | 1 + src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 4 +- .../Client/LayoutClientExtensions.cs | 56 ++--- src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 4 +- .../CollectionPluginTest.cs | 14 +- 18 files changed, 551 insertions(+), 88 deletions(-) create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/GeocodingRequest.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs index c9f2587fd..7c22ae701 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs @@ -149,7 +149,7 @@ async Task IInitializableAgent.InitializeAsync() var typesResponse = await hub.AwaitResponse( new GetDomainTypesRequest(), o => o.WithTarget(new PricingAddress("default"))); - typeDefinitionMap = typesResponse.Message.Types.Select(t => t with { Address = null }).ToDictionary(x => x.Name); + typeDefinitionMap = typesResponse?.Message?.Types?.Select(t => t with { Address = null }).ToDictionary(x => x.Name!); } catch { @@ -161,7 +161,7 @@ async Task IInitializableAgent.InitializeAsync() var layoutAreaResponse = await hub.AwaitResponse( new GetLayoutAreasRequest(), o => o.WithTarget(new PricingAddress("default"))); - layoutAreaMap = layoutAreaResponse.Message.Areas.ToDictionary(x => x.Area); + layoutAreaMap = layoutAreaResponse?.Message?.Areas?.ToDictionary(x => x.Area); } catch { diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index 21a853d77..3870f7eff 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -98,7 +98,7 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); // Add ContentCollectionPlugin for submissions - var submissionPluginConfig = CreateSubmissionPluginConfig(chat); + var submissionPluginConfig = CreateSubmissionPluginConfig(); yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); // Add CollectionPlugin for import functionality @@ -110,7 +110,7 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return KernelPluginFactory.CreateFromObject(plugin); } - private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + private static ContentCollectionPluginConfig CreateSubmissionPluginConfig() { return new ContentCollectionPluginConfig { @@ -151,7 +151,8 @@ async Task IInitializableAgent.InitializeAsync() var typesResponse = await hub.AwaitResponse( new GetDomainTypesRequest(), o => o.WithTarget(new PricingAddress("default"))); - typeDefinitionMap = typesResponse.Message.Types.Select(t => t with { Address = null }).ToDictionary(x => x.Name); + var types = typesResponse?.Message?.Types; + typeDefinitionMap = types?.Select(t => t with { Address = null }).ToDictionary(x => x.Name!); } catch { @@ -165,7 +166,7 @@ async Task IInitializableAgent.InitializeAsync() o => o.WithTarget(new PricingAddress("default"))); // Hard-code TypeName to "PropertyRisk" in the schema - var schema = resp.Message.Schema; + var schema = resp?.Message?.Schema; if (!string.IsNullOrEmpty(schema)) { // Parse the schema as JSON to modify it @@ -201,7 +202,7 @@ async Task IInitializableAgent.InitializeAsync() var resp = await hub.AwaitResponse( new GetSchemaRequest(nameof(PropertyRisk)), o => o.WithTarget(new PricingAddress("default"))); - propertyRiskSchema = resp.Message.Schema; + propertyRiskSchema = resp?.Message?.Schema; } catch { @@ -270,7 +271,7 @@ public async Task GetRiskImportConfiguration(string filename) ); // Serialize the data - var json = JsonSerializer.Serialize(response.Message.Data, hub.JsonSerializerOptions); + var json = JsonSerializer.Serialize(response?.Message?.Data, hub.JsonSerializerOptions); // Parse and ensure $type is set to ExcelImportConfiguration var jsonObject = JsonNode.Parse(json) as JsonObject; @@ -307,7 +308,7 @@ public async Task UpdateRiskImportConfiguration( parsed["entityId"] = pa.Id; parsed["name"] = filename; var response = await hub.AwaitResponse(new DataChangeRequest() { Updates = [parsed] }, o => o.WithTarget(pa)); - return JsonSerializer.Serialize(response.Message, hub.JsonSerializerOptions); + return JsonSerializer.Serialize(response?.Message, hub.JsonSerializerOptions); } catch (Exception e) { diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index 93db5db39..b9424bc0a 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -1,3 +1,7 @@ +using System.ComponentModel; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; using iText.Kernel.Pdf; using iText.Kernel.Pdf.Canvas.Parser; using iText.Kernel.Pdf.Canvas.Parser.Listener; @@ -9,10 +13,6 @@ using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; -using System.ComponentModel; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; namespace MeshWeaver.Insurance.AI; @@ -144,7 +144,7 @@ async Task IInitializableAgent.InitializeAsync() var typesResponse = await hub.AwaitResponse( new GetDomainTypesRequest(), o => o.WithTarget(pricingAddress)); - typeDefinitionMap = typesResponse.Message.Types.Select(t => t with { Address = null }).ToDictionary(x => x.Name); + typeDefinitionMap = typesResponse?.Message?.Types?.Select(t => t with { Address = null }).ToDictionary(x => x.Name!); } catch { @@ -156,7 +156,7 @@ async Task IInitializableAgent.InitializeAsync() var resp = await hub.AwaitResponse( new GetSchemaRequest(nameof(Pricing)), o => o.WithTarget(pricingAddress)); - pricingSchema = resp.Message.Schema; + pricingSchema = resp?.Message?.Schema; } catch { @@ -168,7 +168,7 @@ async Task IInitializableAgent.InitializeAsync() var resp = await hub.AwaitResponse( new GetSchemaRequest(nameof(Structure)), o => o.WithTarget(pricingAddress)); - structureSchema = resp.Message.Schema; + structureSchema = resp?.Message?.Schema; } catch { @@ -279,12 +279,12 @@ public async Task ImportSlipData( // Step 4: Post DataChangeRequest var updateRequest = new DataChangeRequest { Updates = updates }; - var response = await hub.AwaitResponse(updateRequest, o => o.WithTarget(pricingAddress)); + var response = await hub.AwaitResponse(updateRequest, o => o.WithTarget(pricingAddress)); return response.Message.Status switch { DataChangeStatus.Committed => $"Slip data imported successfully. Updated {updates.Count} entities.", - _ => $"Data update failed:\n{string.Join('\n', response.Message.Log.Messages.Select(l => l.LogLevel + ": " + l.Message))}" + _ => $"Data update failed:\n{string.Join('\n', response.Message.Log.Messages?.Select(l => l.LogLevel + ": " + l.Message) ?? Array.Empty())}" }; } catch (Exception e) @@ -301,7 +301,7 @@ public async Task ImportSlipData( new GetDataRequest(new EntityReference(nameof(Pricing), pricingId)), o => o.WithTarget(pricingAddress)); - return response.Message.Data as Pricing; + return response?.Message?.Data as Pricing; } catch { diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/GeocodingRequest.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/GeocodingRequest.cs new file mode 100644 index 000000000..f12dcd3c0 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/GeocodingRequest.cs @@ -0,0 +1,34 @@ +using MeshWeaver.Messaging; + +namespace MeshWeaver.Insurance.Domain; + +/// +/// Request to geocode property risks. +/// +public record GeocodingRequest : IRequest; + +/// +/// Response from geocoding operation. +/// +public record GeocodingResponse +{ + /// + /// Whether the geocoding operation was successful. + /// + public required bool Success { get; init; } + + /// + /// Number of risks successfully geocoded. + /// + public int GeocodedCount { get; init; } + + /// + /// Error message if geocoding failed. + /// + public string? Error { get; init; } + + /// + /// List of updated risks with geocoded locations. + /// + public IReadOnlyList? UpdatedRisks { get; init; } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index bf1a9cdc3..2c1aedf63 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -1,4 +1,5 @@ -using MeshWeaver.ContentCollections; +using System.Reactive.Linq; +using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.Import; using MeshWeaver.Import.Configuration; @@ -16,12 +17,23 @@ namespace MeshWeaver.Insurance.Domain; /// public static class InsuranceApplicationExtensions { + /// + /// Adds Insurance domain services to the service collection. + /// + private static IServiceCollection AddInsuranceDomainServices(this IServiceCollection services) + { + // Register pricing service + services.AddSingleton(); + + return services; + } + /// /// Configures the root Insurance application hub with dimension data and pricing catalog. /// public static MessageHubConfiguration ConfigureInsuranceApplication(this MessageHubConfiguration configuration) => configuration - .WithTypes(typeof(PricingAddress), typeof(ImportConfiguration), typeof(ExcelImportConfiguration), typeof(Structure), typeof(ImportRequest), typeof(CollectionSource)) + .WithTypes(typeof(PricingAddress), typeof(ImportConfiguration), typeof(ExcelImportConfiguration), typeof(Structure), typeof(ImportRequest), typeof(CollectionSource), typeof(GeocodingRequest), typeof(GeocodingResponse)) .AddData(data => { var svc = data.Hub.ServiceProvider.GetRequiredService(); @@ -44,14 +56,15 @@ public static MessageHubConfiguration ConfigureInsuranceApplication(this Message public static MessageHubConfiguration ConfigureSinglePricingApplication(this MessageHubConfiguration configuration) { return configuration + .WithServices(AddInsuranceDomainServices) .AddContentCollection(sp => { var hub = sp.GetRequiredService(); var addressId = hub.Address.Id; - var configuration = sp.GetRequiredService(); + var conf = sp.GetRequiredService(); // Get the global Submissions configuration from appsettings - var globalConfig = configuration.GetSection("Submissions").Get(); + var globalConfig = conf.GetSection("Submissions").Get(); if (globalConfig == null) throw new InvalidOperationException("Submissions collection not found in configuration"); @@ -68,7 +81,7 @@ public static MessageHubConfiguration ConfigureSinglePricingApplication(this Mes var localizedName = GetLocalizedCollectionName("Submissions", addressId); var fullPath = string.IsNullOrEmpty(subPath) ? globalConfig.BasePath ?? "" - : System.IO.Path.Combine(globalConfig.BasePath ?? "", subPath); + : Path.Combine(globalConfig.BasePath ?? "", subPath); return globalConfig with { @@ -106,6 +119,79 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) .WithView(nameof(LayoutAreas.ImportConfigsLayoutArea.ImportConfigs), LayoutAreas.ImportConfigsLayoutArea.ImportConfigs) ) - .AddImport(); + .AddImport() + .WithHandler(HandleGeocodingRequest); + } + + private static async Task HandleGeocodingRequest( + IMessageHub hub, + IMessageDelivery request, + CancellationToken ct) + { + try + { + // Get the geocoding service + var geocodingService = hub.ServiceProvider.GetRequiredService(); + + // Get the current property risks from the workspace + var workspace = hub.GetWorkspace(); + var riskStream = workspace.GetStream(); + if (riskStream == null) + { + var errorResponse = new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = "No property risks found in workspace" + }; + hub.Post(errorResponse, o => o.ResponseFor(request)); + return request.Processed(); + } + + var risks = await riskStream.FirstAsync(); + var riskList = risks?.ToList() ?? new List(); + + if (!riskList.Any()) + { + var errorResponse = new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = "No property risks available to geocode" + }; + hub.Post(errorResponse, o => o.ResponseFor(request)); + return request.Processed(); + } + + // Geocode the risks + var geocodingResponse = await geocodingService.GeocodeRisksAsync(riskList, ct); + + // If successful and we have updated risks, update the workspace + if (geocodingResponse.Success && geocodingResponse.UpdatedRisks != null && geocodingResponse.UpdatedRisks.Any()) + { + // Update the workspace with the geocoded risks + var dataChangeRequest = new DataChangeRequest + { + Updates = geocodingResponse.UpdatedRisks.ToList() + }; + + await hub.AwaitResponse(dataChangeRequest, o => o.WithTarget(hub.Address), ct); + } + + // Post the response + hub.Post(geocodingResponse, o => o.ResponseFor(request)); + } + catch (Exception ex) + { + var errorResponse = new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = $"Geocoding failed: {ex.Message}" + }; + hub.Post(errorResponse, o => o.ResponseFor(request)); + } + + return request.Processed(); } } diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs index 76f11187f..3db9474af 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs @@ -1,10 +1,12 @@ using System.Reactive.Linq; using MeshWeaver.Data; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; +using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; using MeshWeaver.Layout.DataGrid; using MeshWeaver.Utils; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Insurance.Domain.LayoutAreas; @@ -37,7 +39,8 @@ public static IObservable PropertyRisks(LayoutAreaHost host, Renderin return Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "PropertyRisks")) - .WithView(dataGrid); + .WithView(dataGrid) + .WithView(GeocodingArea); }) .StartWith(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "PropertyRisks")) @@ -94,6 +97,40 @@ private static UiControl RenderRisksDataGrid(LayoutAreaHost host, IReadOnlyColle .WithView(Controls.Title("Property Risks", 1)) .WithView(dataGrid); } + + private static IObservable GeocodingArea(LayoutAreaHost host, RenderingContext ctx) + { + var svc = host.Hub.ServiceProvider.GetRequiredService(); + return svc.Progress.Select(p => p is null + ? (UiControl)Controls.Button("Geocode").WithClickAction(ClickGeocoding) + : Controls.Progress($"Processing {p.CurrentRiskName}: {p.ProcessedRisks} of {p.TotalRisks}", + p.TotalRisks == 0 ? 0 : (int)(100.0 * p.ProcessedRisks / p.TotalRisks))); + } + + private static async Task ClickGeocoding(UiActionContext obj) + { + // Show initial progress + obj.Host.UpdateArea(obj.Area, Controls.Progress("Starting geocoding...", 0)); + + try + { + // Start the geocoding process + var response = await obj.Host.Hub.AwaitResponse( + new GeocodingRequest(), + o => o.WithTarget(obj.Hub.Address)); + + // Show completion message + var resultMessage = response?.Message?.Success == true + ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." + : $"❌ Geocoding Failed: {response?.Message?.Error}"; + + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); + } + catch (Exception ex) + { + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); + } + } } /// diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs index be3819ff1..350b3670d 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs @@ -1,7 +1,9 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; +using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Insurance.Domain.LayoutAreas; @@ -35,20 +37,56 @@ public static IObservable RiskMap(LayoutAreaHost host, RenderingConte { return Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown($"# Risk Map\n\n*No geocoded risks found. {riskList.Count} risk(s) available but none have valid coordinates.*")); + .WithView(Controls.Markdown($"# Risk Map\n\n*No geocoded risks found. {riskList.Count} risk(s) available but none have valid coordinates.*")) + .WithView(GeocodingArea); } var mapContent = RenderMapContent(geocodedRisks); return Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown(mapContent)); + .WithView(Controls.Markdown(mapContent)) + .WithView(GeocodingArea); }) .StartWith(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) .WithView(Controls.Markdown("# Risk Map\n\n*Loading...*"))); } + private static IObservable GeocodingArea(LayoutAreaHost host, RenderingContext ctx) + { + var svc = host.Hub.ServiceProvider.GetRequiredService(); + return svc.Progress.Select(p => p is null + ? (UiControl)Controls.Button("Geocode").WithClickAction(ClickGeocoding) + : Controls.Progress($"Processing {p.CurrentRiskName}: {p.ProcessedRisks} of {p.TotalRisks}", + p.TotalRisks == 0 ? 0 : (int)(100.0 * p.ProcessedRisks / p.TotalRisks))); + } + + private static async Task ClickGeocoding(UiActionContext obj) + { + // Show initial progress + obj.Host.UpdateArea(obj.Area, Controls.Progress("Starting geocoding...", 0)); + + try + { + // Start the geocoding process + var response = await obj.Host.Hub.AwaitResponse( + new GeocodingRequest(), + o => o.WithTarget(obj.Hub.Address)); + + // Show completion message + var resultMessage = response?.Message?.Success == true + ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." + : $"❌ Geocoding Failed: {response?.Message?.Error}"; + + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); + } + catch (Exception ex) + { + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); + } + } + private static string RenderMapContent(List geocodedRisks) { var lines = new List diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/MeshWeaver.Insurance.Domain.csproj b/modules/Insurance/MeshWeaver.Insurance.Domain/MeshWeaver.Insurance.Domain.csproj index e1e43d7f3..1bed21874 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/MeshWeaver.Insurance.Domain.csproj +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/MeshWeaver.Insurance.Domain.csproj @@ -10,6 +10,7 @@ + diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs index 7a27187e4..2c6f04867 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs @@ -12,7 +12,7 @@ public record PropertyRisk { /// /// Unique identifier for the property risk record. - /// Synonyms: "Plant code", "Plant ID", "Site Code", "Asset ID", "Code". + /// Synonyms: Plant code, Plant ID, Site Code, Asset ID, Code. /// [Key] public required string Id { get; init; } @@ -41,38 +41,38 @@ public record PropertyRisk /// /// Human-friendly site or facility name. - /// Synonyms: "Plant Description", "Site Name", "Location Name". + /// Synonyms: Plant Description, Site Name, Location Name. /// public string? LocationName { get; init; } /// /// Country; typically ISO code or name (dimension). - /// Synonyms: "Country Code", "Country". + /// Synonyms: Country Code, Country. /// [Dimension] public string? Country { get; init; } /// /// Street address. - /// Synonyms: "Property Address", "Address". + /// Synonyms: Property Address, Address. /// public string? Address { get; init; } /// /// State/region/province. - /// Synonyms: "State/Province", "Region". + /// Synonyms: State/Province, Region. /// public string? State { get; init; } /// /// County/district. - /// Synonyms: "District", "County". + /// Synonyms: District, County. /// public string? County { get; init; } /// /// Postal/ZIP code. - /// Synonyms: "ZIP", "Postcode". + /// Synonyms: ZIP, Postcode. /// public string? ZipCode { get; init; } @@ -83,14 +83,14 @@ public record PropertyRisk /// /// Base currency for the risk. - /// Synonyms: "Currency", "Curr.", "Curr", "CCY". + /// Synonyms: Currency, Curr., Curr, CCY. /// [Dimension] public string? Currency { get; init; } /// /// Sum insured for buildings. - /// Synonyms: "Buildings", "Building Value", "TSI Building(s)". + /// Synonyms: Buildings, Building Value, TSI Building(s). /// public double TsiBuilding { get; init; } @@ -101,7 +101,7 @@ public record PropertyRisk /// /// Sum insured for contents. - /// Synonyms: "Stock", "Fixtures & Fittings", "IT Equipment", "Equipment". + /// Synonyms: Stock, Fixtures, Fittings, IT Equipment, Equipment. /// public double TsiContent { get; init; } @@ -112,7 +112,7 @@ public record PropertyRisk /// /// Business Interruption TSI. - /// Synonyms: "BI", "Business Interruption", "Gross Profit". + /// Synonyms: BI, Business Interruption, Gross Profit. /// public double TsiBi { get; init; } @@ -123,7 +123,7 @@ public record PropertyRisk /// /// Account identifier. - /// Synonyms: "Account #", "Account No". + /// Synonyms: Account #, Account No. /// public string? AccountNumber { get; init; } diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs new file mode 100644 index 000000000..4a0cb3758 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs @@ -0,0 +1,237 @@ +using System.Collections.Concurrent; +using System.Net.Http.Json; +using System.Reactive.Subjects; +using MeshWeaver.GoogleMaps; +using Microsoft.Extensions.Options; + +namespace MeshWeaver.Insurance.Domain.Services; + +/// +/// Google Maps-based geocoding service for property risks. +/// +public class GoogleGeocodingService(IOptions googleConfig) : IGeocodingService +{ + private readonly ReplaySubject progressSubject = InitializeProgress(); + private readonly object progressLock = new(); + private readonly HttpClient http = new(); + + private static ReplaySubject InitializeProgress() + { + var ret = new ReplaySubject(1); + ret.OnNext(null); + return ret; + } + + public IObservable Progress => progressSubject; + + public async Task GeocodeRisksAsync(IReadOnlyCollection risks, CancellationToken cancellationToken = default) + { + try + { + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress(0, 0, null, "Starting geocoding...")); + } + + if (!risks.Any()) + { + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress(0, 0, null, "No risks to geocode", true)); + } + return new GeocodingResponse + { + Success = true, + GeocodedCount = 0, + Error = "No risks found to geocode" + }; + } + + // Check Google Maps API key + if (string.IsNullOrEmpty(googleConfig.Value.ApiKey)) + { + var error = "Google Maps API key not configured"; + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress(0, 0, null, "Configuration error", true)); + } + return new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = error + }; + } + + // Filter risks that need geocoding + var risksToGeocode = risks + .Where(r => r.GeocodedLocation?.Latitude == null || r.GeocodedLocation?.Longitude == null) + .ToList(); + + if (!risksToGeocode.Any()) + { + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress(risks.Count, risks.Count, null, + "All risks already geocoded", true)); + } + return new GeocodingResponse + { + Success = true, + GeocodedCount = 0, + Error = null + }; + } + + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress(risksToGeocode.Count, 0, null, "Initializing geocoding...")); + } + + var geocodedCount = 0; + var updatedRisks = new ConcurrentBag(); + var processedCount = 0; + + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 10), + CancellationToken = cancellationToken + }; + + await Parallel.ForEachAsync(risksToGeocode, parallelOptions, async (risk, ct) => + { + var riskName = risk.LocationName ?? risk.Address ?? $"Risk {risk.Id}"; + + try + { + var geocodedLocation = await GeocodeAsync(risk, ct); + + if (geocodedLocation.Latitude.HasValue && geocodedLocation.Longitude.HasValue) + { + // Update the risk with geocoded data + var updatedRisk = risk with { GeocodedLocation = geocodedLocation }; + updatedRisks.Add(updatedRisk); + Interlocked.Increment(ref geocodedCount); + } + else + { + // Still add the risk with the geocoding attempt result + updatedRisks.Add(risk with { GeocodedLocation = geocodedLocation }); + } + } + catch (Exception) + { + // Add the original risk unchanged + updatedRisks.Add(risk); + } + + // Update progress after processing each risk + var currentProcessed = Interlocked.Increment(ref processedCount); + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress( + risksToGeocode.Count, + currentProcessed, + risk.Id, + $"Processing {currentProcessed}/{risksToGeocode.Count} risks..." + )); + } + }); + + // Final progress update + lock (progressLock) + { + progressSubject.OnNext(new GeocodingProgress( + risksToGeocode.Count, + risksToGeocode.Count, + null, + $"Completed processing {risksToGeocode.Count} risks", + true + )); + } + + return new GeocodingResponse + { + Success = true, + GeocodedCount = geocodedCount, + Error = null, + UpdatedRisks = updatedRisks.ToList() + }; + } + catch (Exception ex) + { + var error = $"Geocoding failed: {ex.Message}"; + return new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = error + }; + } + finally + { + lock (progressLock) + { + progressSubject.OnNext(null); + } + } + } + + private async Task GeocodeAsync(PropertyRisk risk, CancellationToken ct = default) + { + var query = BuildQuery(risk); + var url = $"https://maps.googleapis.com/maps/api/geocode/json?address={Uri.EscapeDataString(query)}&key={googleConfig.Value.ApiKey}"; + + var response = await http.GetFromJsonAsync(url, cancellationToken: ct); + if (response == null) + { + return new GeocodedLocation { Status = "NoResponse" }; + } + + if (response.status != "OK" || response.results == null || response.results.Length == 0) + { + return new GeocodedLocation { Status = response.status }; + } + + var r = response.results[0]; + return new GeocodedLocation + { + Latitude = r.geometry.location.lat, + Longitude = r.geometry.location.lng, + FormattedAddress = r.formatted_address, + PlaceId = r.place_id, + Status = response.status + }; + } + + private static string BuildQuery(PropertyRisk risk) + { + var parts = new[] { risk.LocationName, risk.Address, risk.City, risk.State, risk.ZipCode, risk.Country } + .Where(s => !string.IsNullOrWhiteSpace(s)); + return string.Join(", ", parts); + } + + private sealed class GoogleGeocodeResponse + { + public string status { get; set; } = string.Empty; + public GoogleGeocodeResult[]? results { get; set; } + } + + private sealed class GoogleGeocodeResult + { + public string formatted_address { get; set; } = string.Empty; + public string place_id { get; set; } = string.Empty; + public GoogleGeometry geometry { get; set; } = new(); + } + + private sealed class GoogleGeometry + { + public GoogleLocation location { get; set; } = new(); + } + + private sealed class GoogleLocation + { + public double lat { get; set; } + public double lng { get; set; } + } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs new file mode 100644 index 000000000..bfbd5b3c3 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs @@ -0,0 +1,28 @@ +namespace MeshWeaver.Insurance.Domain.Services; + +/// +/// Service for geocoding property risks. +/// +public interface IGeocodingService +{ + /// + /// Observable stream of geocoding progress. + /// + IObservable Progress { get; } + + /// + /// Geocodes a collection of property risks. + /// + Task GeocodeRisksAsync(IReadOnlyCollection risks, CancellationToken cancellationToken = default); +} + +/// +/// Progress information for geocoding operations. +/// +public record GeocodingProgress( + int TotalRisks, + int ProcessedRisks, + string? CurrentRiskId, + string CurrentRiskName, + bool IsComplete = false +); diff --git a/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs b/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs index aa7543a80..aa6d15db0 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs @@ -27,7 +27,7 @@ protected async Task> GetPropertyRisksAsync(Pr o => o.WithTarget(address), TestContext.Current.CancellationToken); - return (risksResp.Message.Data as IEnumerable)? + return (risksResp?.Message?.Data as IEnumerable)? .Select(x => x as PropertyRisk ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) .Where(x => x != null) .Cast() @@ -43,7 +43,7 @@ protected async Task> GetPricingsAsync() o => o.WithTarget(InsuranceApplicationAttribute.Address), TestContext.Current.CancellationToken); - return (pricingsResp.Message.Data as IEnumerable)? + return (pricingsResp?.Message?.Data as IEnumerable)? .Select(x => x as Pricing ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) .Where(x => x != null) .Cast() diff --git a/modules/Northwind/MeshWeaver.Northwind.AI/NorthwindAgent.cs b/modules/Northwind/MeshWeaver.Northwind.AI/NorthwindAgent.cs index 68da2f9a8..9889b1b76 100644 --- a/modules/Northwind/MeshWeaver.Northwind.AI/NorthwindAgent.cs +++ b/modules/Northwind/MeshWeaver.Northwind.AI/NorthwindAgent.cs @@ -26,20 +26,20 @@ public class NorthwindAgent(IMessageHub hub) : IInitializableAgent, IAgentWithPl public string Instructions => """ You are the NorthwindAgent, specialized in working with Northwind business data. You have access to: - + - Customer data: information about companies, contacts, and addresses - Order data: sales orders, order details, and order history - Product data: product catalog, categories, suppliers, and inventory - Employee data: staff information and territories - Geographic data: regions, territories, and shipping information - + You can help users: - Query and analyze business data - Generate reports and insights - Answer questions about customers, orders, products, and sales - Provide data-driven recommendations - Layout areas (reports, views, charts, dashboards) related to Northwind data - + Use the DataPlugin to access structured domain data and the LayoutAreaPlugin to display visual components. Always provide accurate, data-driven responses based on the available Northwind data. """; @@ -54,13 +54,13 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) } private static readonly Address NorthwindAddress = new ApplicationAddress("Northwind"); - + async Task IInitializableAgent.InitializeAsync() { var typeResponse = await hub.AwaitResponse(new GetDomainTypesRequest(), o => o.WithTarget(NorthwindAddress)); - typeDefinitionMap = typeResponse.Message.Types.ToDictionary(x => x.Name); + typeDefinitionMap = typeResponse?.Message?.Types?.ToDictionary(x => x.Name!); var layoutResponse = await hub.AwaitResponse(new GetLayoutAreasRequest(), o => o.WithTarget(NorthwindAddress)); - layoutDefinitionMap = layoutResponse.Message.Areas.ToDictionary(x => x.Area); + layoutDefinitionMap = layoutResponse?.Message?.Areas?.ToDictionary(x => x.Area); } /// diff --git a/portal/MeshWeaver.Portal.Shared.Web/SharedPortalConfiguration.cs b/portal/MeshWeaver.Portal.Shared.Web/SharedPortalConfiguration.cs index 00054952f..0cbf5fe29 100644 --- a/portal/MeshWeaver.Portal.Shared.Web/SharedPortalConfiguration.cs +++ b/portal/MeshWeaver.Portal.Shared.Web/SharedPortalConfiguration.cs @@ -43,6 +43,7 @@ public static void ConfigureWebPortalServices(this WebApplicationBuilder builder .AddEnvironmentVariables(); var services = builder.Services; + services.AddRazorPages() .AddMicrosoftIdentityUI(); diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index f10441767..50abd6775 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -39,7 +39,7 @@ public async Task GetFile( var extension = Path.GetExtension(filePath).ToLowerInvariant(); if (extension == ".xlsx" || extension == ".xls") { - return await ReadExcelFileAsync(stream, filePath, numberOfRows, cancellationToken); + return await ReadExcelFileAsync(stream, filePath, numberOfRows); } // For non-Excel files, read as text @@ -72,7 +72,7 @@ public async Task GetFile( } } - private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows, CancellationToken cancellationToken) + private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows) { try { diff --git a/src/MeshWeaver.Layout/Client/LayoutClientExtensions.cs b/src/MeshWeaver.Layout/Client/LayoutClientExtensions.cs index 5dd83f799..91fb2b98d 100644 --- a/src/MeshWeaver.Layout/Client/LayoutClientExtensions.cs +++ b/src/MeshWeaver.Layout/Client/LayoutClientExtensions.cs @@ -13,7 +13,7 @@ namespace MeshWeaver.Layout.Client; public static class LayoutClientExtensions { - public static void UpdatePointer(this ISynchronizationStream stream, + public static void UpdatePointer(this ISynchronizationStream stream, object? value, string? dataContext, JsonPointerReference? reference, ModelParameter? model = null) @@ -68,14 +68,14 @@ public static void UpdatePointer(this ISynchronizationStream stream } public static IObservable DataBind(this ISynchronizationStream stream, - JsonPointerReference reference, - string? dataContext = null, - Func? conversion = null, + JsonPointerReference reference, + string? dataContext = null, + Func? conversion = null, T? defaultValue = default(T)) => stream.GetStream(JsonPointer.Parse(GetPointer(reference.Pointer, dataContext ?? ""))) - .Select(x => - conversion is not null - ? conversion.Invoke(x, defaultValue) + .Select(x => + conversion is not null + ? conversion.Invoke(x, defaultValue) : stream.Hub.ConvertSingle(x, null, defaultValue!)) .Where(x => x is not null) .Select(x => (T)x!) @@ -139,7 +139,7 @@ private static string GetPointer(string pointer, string? dataContext) return $"{dataContext}/{pointer.TrimEnd('/')}"; } - public static T? ConvertSingle(this IMessageHub hub, object? value, Func? conversion, T? defaultValue = default(T)) + public static T? ConvertSingle(this IMessageHub hub, object? value, Func? conversion, T? defaultValue = default(T)) { conversion ??= null; if (conversion != null) @@ -169,7 +169,7 @@ private static string GetPointer(string pointer, string? dataContext) // This is a nullable type - check if it has a value var underlyingValue = valueType.GetProperty("Value")?.GetValue(value); var hasValue = (bool)(valueType.GetProperty("HasValue")?.GetValue(value) ?? false); - + if (hasValue && underlyingValue != null) { // Use the underlying value for conversion @@ -182,7 +182,7 @@ private static string GetPointer(string pointer, string? dataContext) } } } - + // Not a nullable type, proceed with normal numeric conversion return ConvertNumericValue(value); } @@ -190,13 +190,13 @@ private static string GetPointer(string pointer, string? dataContext) private static T? ConvertNumericValue(object? value) { var targetType = Nullable.GetUnderlyingType(typeof(T)) ?? typeof(T); - + // Handle numeric conversions more safely if (IsNumericType(targetType)) { return ConvertNumericSafely(value, targetType); } - + // Fall back to Convert.ChangeType for non-numeric types return (T?)Convert.ChangeType(value, typeof(T)); } @@ -205,8 +205,8 @@ private static bool IsNumericType(Type type) { return Type.GetTypeCode(type) switch { - TypeCode.Byte or TypeCode.SByte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 or - TypeCode.Int16 or TypeCode.Int32 or TypeCode.Int64 or + TypeCode.Byte or TypeCode.SByte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 or + TypeCode.Int16 or TypeCode.Int32 or TypeCode.Int64 or TypeCode.Decimal or TypeCode.Double or TypeCode.Single => true, _ => false }; @@ -219,27 +219,27 @@ TypeCode.Int16 or TypeCode.Int32 or TypeCode.Int64 or { if (double.IsNaN(d) || double.IsInfinity(d)) throw new OverflowException($"Cannot convert {d} to {targetType.Name}"); - + // For integer targets, check if the value is within range and truncate if (IsIntegerType(targetType)) { return ConvertDoubleToInteger(d, targetType); } } - - // Handle special float values + + // Handle special float values if (value is float f) { if (float.IsNaN(f) || float.IsInfinity(f)) throw new OverflowException($"Cannot convert {f} to {targetType.Name}"); - + // For integer targets, check if the value is within range and truncate if (IsIntegerType(targetType)) { return ConvertDoubleToInteger(f, targetType); } } - + // Use Convert.ChangeType for other numeric conversions return value is null ? default : (T?)Convert.ChangeType(value, targetType); } @@ -248,7 +248,7 @@ private static bool IsIntegerType(Type type) { return Type.GetTypeCode(type) switch { - TypeCode.Byte or TypeCode.SByte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 or + TypeCode.Byte or TypeCode.SByte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 or TypeCode.Int16 or TypeCode.Int32 or TypeCode.Int64 => true, _ => false }; @@ -259,28 +259,28 @@ private static T ConvertDoubleToInteger(double value, Type targetType) // Check bounds and truncate the value return Type.GetTypeCode(targetType) switch { - TypeCode.Int32 => value > int.MaxValue || value < int.MinValue + TypeCode.Int32 => value > int.MaxValue || value < int.MinValue ? throw new OverflowException($"Value {value} is out of range for Int32") : (T)(object)(int)Math.Truncate(value), - TypeCode.Int16 => value > short.MaxValue || value < short.MinValue + TypeCode.Int16 => value > short.MaxValue || value < short.MinValue ? throw new OverflowException($"Value {value} is out of range for Int16") : (T)(object)(short)Math.Truncate(value), - TypeCode.Int64 => value > long.MaxValue || value < long.MinValue + TypeCode.Int64 => value > long.MaxValue || value < long.MinValue ? throw new OverflowException($"Value {value} is out of range for Int64") : (T)(object)(long)Math.Truncate(value), - TypeCode.Byte => value > byte.MaxValue || value < byte.MinValue + TypeCode.Byte => value > byte.MaxValue || value < byte.MinValue ? throw new OverflowException($"Value {value} is out of range for Byte") : (T)(object)(byte)Math.Truncate(value), - TypeCode.SByte => value > sbyte.MaxValue || value < sbyte.MinValue + TypeCode.SByte => value > sbyte.MaxValue || value < sbyte.MinValue ? throw new OverflowException($"Value {value} is out of range for SByte") : (T)(object)(sbyte)Math.Truncate(value), - TypeCode.UInt16 => value > ushort.MaxValue || value < ushort.MinValue + TypeCode.UInt16 => value > ushort.MaxValue || value < ushort.MinValue ? throw new OverflowException($"Value {value} is out of range for UInt16") : (T)(object)(ushort)Math.Truncate(value), - TypeCode.UInt32 => value > uint.MaxValue || value < uint.MinValue + TypeCode.UInt32 => value > uint.MaxValue || value < uint.MinValue ? throw new OverflowException($"Value {value} is out of range for UInt32") : (T)(object)(uint)Math.Truncate(value), - TypeCode.UInt64 => value > ulong.MaxValue || value < 0 + TypeCode.UInt64 => value > ulong.MaxValue || value < 0 ? throw new OverflowException($"Value {value} is out of range for UInt64") : (T)(object)(ulong)Math.Truncate(value), _ => throw new InvalidOperationException($"Unsupported integer type: {targetType.Name}") diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index d33934dfe..3d03768f9 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -23,9 +23,9 @@ Task> AwaitResponse(IRequest r CancellationToken cancellationToken) => AwaitResponse(request, x => x, x => x, cancellationToken)!; - Task?> AwaitResponse(IRequest request, + async Task> AwaitResponse(IRequest request, Func options, CancellationToken cancellationToken = default) - => AwaitResponse(request, options, o => o, cancellationToken); + => (await AwaitResponse(request, options, o => o, cancellationToken))!; Task AwaitResponse(IRequest request, Func, TResult> selector) => AwaitResponse(request, x => x, selector); diff --git a/test/MeshWeaver.AI.Test/CollectionPluginTest.cs b/test/MeshWeaver.AI.Test/CollectionPluginTest.cs index ac12d92d6..4bd8c9d1b 100644 --- a/test/MeshWeaver.AI.Test/CollectionPluginTest.cs +++ b/test/MeshWeaver.AI.Test/CollectionPluginTest.cs @@ -136,7 +136,7 @@ public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() var plugin = new CollectionPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName); + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -174,7 +174,7 @@ public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() const int rowLimit = 5; // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, numberOfRows: rowLimit); + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -206,7 +206,7 @@ public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() const int rowLimit = 10; // act - var result = await plugin.GetFile(TestCollectionName, TestTextFileName, numberOfRows: rowLimit); + var result = await plugin.GetFile(TestCollectionName, TestTextFileName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -231,7 +231,7 @@ public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new CollectionPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestTextFileName); + var result = await plugin.GetFile(TestCollectionName, TestTextFileName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -256,7 +256,7 @@ public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new CollectionPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName); + var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -286,7 +286,7 @@ public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() var plugin = new CollectionPlugin(client); // act - var result = await plugin.GetFile("non-existent-collection", "test.xlsx"); + var result = await plugin.GetFile("non-existent-collection", "test.xlsx", cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("Collection 'non-existent-collection' not found"); @@ -303,7 +303,7 @@ public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() var plugin = new CollectionPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, "non-existent.xlsx"); + var result = await plugin.GetFile(TestCollectionName, "non-existent.xlsx", cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("File 'non-existent.xlsx' not found"); From 067a70ea09b699e7544654daef8081ab625d298c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 28 Oct 2025 06:33:21 +0100 Subject: [PATCH 06/57] iterating on slip --- .../Insurance/Files/Microsoft/2026/Slip.md | 69 +++--- .../SlipImportAgent.cs | 205 ++++++++++++------ .../InsuranceApplicationExtensions.cs | 9 +- .../ReinsuranceAcceptanceLayoutArea.cs | 174 +++++++++++++++ .../LayoutAreas/RiskMapLayoutArea.cs | 156 ++++++++----- .../LayoutAreas/Shared/PricingLayoutShared.cs | 1 + .../ReinsuranceAcceptance.cs | 86 ++++++++ .../ReinsuranceSection.cs | 50 +++++ .../MeshWeaver.Insurance.Domain/Structure.cs | 110 ---------- .../appsettings.Development.json | 3 - 10 files changed, 598 insertions(+), 265 deletions(-) create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs create mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs delete mode 100644 modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs diff --git a/modules/Insurance/Files/Microsoft/2026/Slip.md b/modules/Insurance/Files/Microsoft/2026/Slip.md index fdb61dab4..739a4bd12 100644 --- a/modules/Insurance/Files/Microsoft/2026/Slip.md +++ b/modules/Insurance/Files/Microsoft/2026/Slip.md @@ -14,69 +14,78 @@ **Address:** One Microsoft Way, Redmond, WA 98052, United States ## Period of Insurance -**From:** 1 January 2026 -**To:** 31 December 2026 +**From:** 1 January 2026 +**To:** 31 December 2026 **Local Standard Time at the address of the Insured** --- +## Reinsurance Terms + +**Estimated Premium Income (EPI):** USD 200,000,000 +**Brokerage:** 10% + +--- + ## Coverage Sections ### 1. Fire Damage - **Layer 1:** - - Attachment Point: USD 0 - - Limit per Occurrence: USD 100,000,000 - - **Annual Aggregate Limit:** USD 300,000,000 - - **Annual Aggregate Deductible:** USD 25,000,000 (applies to Fire only) + - Deductible per Occurrence: USD 5,000,000 + - Limit per Occurrence: USD 100,000,000 + - **Annual Aggregate Deductible:** USD 25,000,000 + - **Annual Aggregate Limit:** USD 300,000,000 - **Layer 2:** - - Attachment Point: USD 100,000,000 - - Limit per Occurrence: USD 150,000,000 - - **Annual Aggregate Limit:** USD 450,000,000 + - Attachment Point: USD 105,000,000 + - Limit per Occurrence: USD 145,000,000 + - **Annual Aggregate Limit:** USD 435,000,000 - **Layer 3:** - - Attachment Point: USD 250,000,000 - - Limit per Occurrence: USD 250,000,000 - - **Annual Aggregate Limit:** USD 600,000,000 + - Attachment Point: USD 250,000,000 + - Limit per Occurrence: USD 250,000,000 + - **Annual Aggregate Limit:** USD 750,000,000 --- ### 2. Natural Catastrophe (Earthquake, Flood, Storm) - **Layer 1:** - - Attachment Point: USD 0 - - Limit per Occurrence: USD 75,000,000 - - **Annual Aggregate Limit:** USD 225,000,000 + - Deductible per Occurrence: USD 5,000,000 + - Limit per Occurrence: USD 100,000,000 + - **Annual Aggregate Deductible:** USD 25,000,000 + - **Annual Aggregate Limit:** USD 300,000,000 - **Layer 2:** - - Attachment Point: USD 75,000,000 - - Limit per Occurrence: USD 125,000,000 - - **Annual Aggregate Limit:** USD 375,000,000 + - Attachment Point: USD 105,000,000 + - Limit per Occurrence: USD 145,000,000 + - **Annual Aggregate Limit:** USD 435,000,000 - **Layer 3:** - - Attachment Point: USD 200,000,000 - - Limit per Occurrence: USD 200,000,000 - - **Annual Aggregate Limit:** USD 500,000,000 + - Attachment Point: USD 250,000,000 + - Limit per Occurrence: USD 250,000,000 + - **Annual Aggregate Limit:** USD 750,000,000 --- ### 3. Business Interruption - **Layer 1:** - - Attachment Point: USD 0 - - Limit per Occurrence: USD 50,000,000 - - **Annual Aggregate Limit:** USD 150,000,000 + - Deductible per Occurrence: USD 5,000,000 + - Limit per Occurrence: USD 100,000,000 + - **Annual Aggregate Deductible:** USD 25,000,000 + - **Annual Aggregate Limit:** USD 300,000,000 - **Layer 2:** - - Attachment Point: USD 50,000,000 - - Limit per Occurrence: USD 100,000,000 - - **Annual Aggregate Limit:** USD 300,000,000 + - Attachment Point: USD 105,000,000 + - Limit per Occurrence: USD 145,000,000 + - **Annual Aggregate Limit:** USD 435,000,000 - **Layer 3:** - - Attachment Point: USD 150,000,000 - - Limit per Occurrence: USD 150,000,000 - - **Annual Aggregate Limit:** USD 450,000,000 + - Attachment Point: USD 250,000,000 + - Limit per Occurrence: USD 250,000,000 + - **Annual Aggregate Limit:** USD 750,000,000 --- diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index b9424bc0a..79dbf68e7 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -20,11 +20,12 @@ public class SlipImportAgent(IMessageHub hub) : IInitializableAgent, IAgentWithP { private Dictionary? typeDefinitionMap; private string? pricingSchema; - private string? structureSchema; + private string? acceptanceSchema; + private string? sectionSchema; public string Name => nameof(SlipImportAgent); - public string Description => "Imports insurance slip documents from PDF files and structures them into Pricing and Structure data models using LLM-based extraction."; + public string Description => "Imports insurance slip documents from PDF or Markdown files and structures them into Pricing and ReinsuranceAcceptance data models using LLM-based extraction."; public string Instructions { @@ -32,41 +33,77 @@ public string Instructions { var baseText = $$$""" - You are a slip import agent that processes PDF documents containing insurance submission slips. + You are a slip import agent that processes insurance submission slip documents in PDF or Markdown format. Your task is to extract structured data and map it to the insurance domain models using the provided schemas. ## Content Collection Context IMPORTANT: The current context is set to pricing/{pricingId} where pricingId follows the format {company}-{uwy}. - - The submission files collection is named "Submissions-{pricingId}" - - All file paths are relative to the root (/) of this collection - - When listing files, you'll see paths like "/slip.pdf", "/submission.pdf" - - When accessing files, use paths starting with "/" (e.g., "/slip.pdf") + - The submission files collection is automatically named "Submissions-{pricingId}" + - Files are stored at the root level of this collection + - When listing files, you'll see filenames like "Slip.pdf", "Slip.md", etc. + - When accessing files with ExtractCompleteText, use just the filename (e.g., "Slip.pdf" or "Slip.md") # Importing Slips When the user asks you to import a slip: 1) First, use {{{nameof(ContentCollectionPlugin.ListFiles)}}}() to see available files in the submissions collection - 2) Use {{{nameof(SlipImportPlugin.ExtractCompleteText)}}} to extract the PDF content (e.g., "slip.pdf" without the leading /) + 2) Use {{{nameof(SlipImportPlugin.ExtractCompleteText)}}} to extract the document content from PDF or Markdown files + - Simply pass the filename (e.g., "Slip.pdf" or "Slip.md") + - The collection name will be automatically resolved to "Submissions-{pricingId}" 3) Review the extracted text and identify data that matches the domain schemas 4) Use {{{nameof(SlipImportPlugin.ImportSlipData)}}} to save the structured data as JSON 5) Provide feedback on what data was successfully imported or if any issues were encountered # Data Mapping Guidelines - Based on the extracted PDF text, create JSON objects that match the schemas provided below: + Based on the extracted document text, create JSON objects that match the schemas provided below: - **Pricing**: Basic pricing information (insured name, broker, dates, premium, country, legal entity) - - **Structure**: Reinsurance layer structure and financial terms (cession, limits, rates, commissions) + - **ReinsuranceAcceptance**: Represents a reinsurance layer (Layer 1, Layer 2, Layer 3) with financial terms + - **ReinsuranceSection**: Represents a coverage type within a layer (Fire Damage, Natural Catastrophe, Business Interruption) + + # Structure Hierarchy + The data structure follows this hierarchy: + 1. **Pricing** (the main insurance program) + 2. **ReinsuranceAcceptance** (the layers: Layer 1, Layer 2, Layer 3, etc.) + 3. **ReinsuranceSection** (the coverage types within each layer: Fire Damage, Natural Catastrophe, Business Interruption, etc.) # Important Rules - - Only extract data that is explicitly present in the PDF text + - Only extract data that is explicitly present in the document text - Use null or default values for missing data points - Ensure all monetary values are properly formatted as numbers - Convert percentages to decimal format (e.g., 25% → 0.25) - Provide clear feedback on what data was successfully extracted - If data is ambiguous or unclear, note it in your response - - For Structure records, generate appropriate LayerId values (e.g., "Layer1", "Layer2") - - Multiple layers can be imported if the slip contains multiple layer structures - # PDF Section Processing + # Creating ReinsuranceAcceptance Records (Layers) + - First, create ReinsuranceAcceptance records for each layer (Layer 1, Layer 2, Layer 3) + - Use IDs like "Layer1", "Layer2", "Layer3" + - Set the Name property to "Layer 1", "Layer 2", "Layer 3" + - Include financial terms like share, cession, rate, commission on the acceptance + - If there is a "Reinsurance Terms" section in the header with properties like EPI and Brokerage, apply these values to ALL ReinsuranceAcceptance records (all layers get the same EPI and Brokerage) + - Convert percentage values to decimals (e.g., 10% → 0.10, 100% → 1.0) + + # Creating ReinsuranceSection Records (Coverage Types) + - Then, create ReinsuranceSection records for each coverage type within each layer + - Use IDs like "Layer1-Fire", "Layer1-NatCat", "Layer1-BI", "Layer2-Fire", etc. + - Set the AcceptanceId to link the section to its parent layer (e.g., "Layer1") + - Set the Type to the coverage type (e.g., "Fire Damage", "Natural Catastrophe", "Business Interruption") + - Set the Name to a descriptive name (e.g., "Fire Damage - Layer 1") + - Include the attachment point (Attach), limit, aggregate deductible (AggAttach), and aggregate limit (AggLimit) + + # Example from a Slip + If the slip shows: + - Fire Damage → Layer 1: Attach 5M, Limit 100M, AAD 25M, AAL 300M + - Fire Damage → Layer 2: Attach 100M, Limit 150M, AAL 450M + - Natural Catastrophe → Layer 1: Attach 10M, Limit 75M, AAD 30M, AAL 225M + + Create: + 1. ReinsuranceAcceptance: Id="Layer1", Name="Layer 1" + 2. ReinsuranceAcceptance: Id="Layer2", Name="Layer 2" + 3. ReinsuranceSection: Id="Layer1-Fire", AcceptanceId="Layer1", Type="Fire Damage", Attach=5000000, Limit=100000000, AggAttach=25000000, AggLimit=300000000 + 4. ReinsuranceSection: Id="Layer2-Fire", AcceptanceId="Layer2", Type="Fire Damage", Attach=100000000, Limit=150000000, AggLimit=450000000 + 5. ReinsuranceSection: Id="Layer1-NatCat", AcceptanceId="Layer1", Type="Natural Catastrophe", Attach=10000000, Limit=75000000, AggAttach=30000000, AggLimit=225000000 + + # Document Section Processing Look for common sections in insurance slips: - Insured information (name, location, industry) - Coverage details (inception/expiration dates, policy terms) @@ -75,14 +112,18 @@ You are a slip import agent that processes PDF documents containing insurance su - Reinsurance terms (commission, brokerage, taxes) Notes: - - When listing files, paths will include "/" prefix (e.g., "/slip.pdf") - - When calling import functions, provide only the filename without "/" (e.g., "slip.pdf") + - When listing files, you may see paths with "/" prefix (e.g., "/Slip.pdf", "/Slip.md") + - When calling ExtractCompleteText, provide only the filename (e.g., "Slip.pdf" or "Slip.md") + - The collection name is automatically determined from the pricing context + - Both PDF and Markdown (.md) files are supported """; if (pricingSchema is not null) baseText += $"\n\n# Pricing Schema\n```json\n{pricingSchema}\n```"; - if (structureSchema is not null) - baseText += $"\n\n# Structure Schema\n```json\n{structureSchema}\n```"; + if (acceptanceSchema is not null) + baseText += $"\n\n# ReinsuranceAcceptance Schema\n```json\n{acceptanceSchema}\n```"; + if (sectionSchema is not null) + baseText += $"\n\n# ReinsuranceSection Schema\n```json\n{sectionSchema}\n```"; return baseText; } @@ -166,13 +207,25 @@ async Task IInitializableAgent.InitializeAsync() try { var resp = await hub.AwaitResponse( - new GetSchemaRequest(nameof(Structure)), + new GetSchemaRequest(nameof(ReinsuranceAcceptance)), o => o.WithTarget(pricingAddress)); - structureSchema = resp?.Message?.Schema; + acceptanceSchema = resp?.Message?.Schema; } catch { - structureSchema = null; + acceptanceSchema = null; + } + + try + { + var resp = await hub.AwaitResponse( + new GetSchemaRequest(nameof(ReinsuranceSection)), + o => o.WithTarget(pricingAddress)); + sectionSchema = resp?.Message?.Schema; + } + catch + { + sectionSchema = null; } } @@ -190,8 +243,10 @@ private JsonSerializerOptions GetJsonOptions() } [KernelFunction] - [Description("Extracts the complete text from a PDF slip document and returns it for LLM processing")] - public async Task ExtractCompleteText(string filename) + [Description("Extracts the complete text from a slip document (PDF or Markdown) and returns it for LLM processing")] + public async Task ExtractCompleteText( + [Description("The filename to extract (e.g., 'Slip.pdf' or 'Slip.md')")] string filename, + [Description("The collection name (optional, defaults to context-based resolution)")] string? collectionName = null) { if (chat.Context?.Address?.Type != PricingAddress.TypeName) return "Please navigate to the pricing first."; @@ -200,14 +255,38 @@ public async Task ExtractCompleteText(string filename) { var pricingId = chat.Context.Address.Id; var contentService = hub.ServiceProvider.GetRequiredService(); - var stream = await OpenContentReadStreamAsync(contentService, pricingId, filename); + + // Get collection name using the same pattern as ContentCollectionPlugin + var resolvedCollectionName = collectionName ?? $"Submissions-{pricingId}"; + + // Use ContentService directly with the correct collection name and simple path + var stream = await contentService.GetContentAsync(resolvedCollectionName, filename, CancellationToken.None); if (stream is null) - return $"Content not found: {filename}"; + return $"Content not found: {filename} in collection {resolvedCollectionName}"; await using (stream) { - var completeText = await ExtractCompletePdfText(stream); + string completeText; + + // Determine file type by extension + if (filename.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + { + // Read markdown file directly as text + using var reader = new StreamReader(stream); + completeText = await reader.ReadToEndAsync(); + } + else if (filename.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase)) + { + // Extract text from PDF + completeText = await ExtractCompletePdfText(stream); + } + else + { + // Try to read as text for unknown file types + using var reader = new StreamReader(stream); + completeText = await reader.ReadToEndAsync(); + } var sb = new StringBuilder(); sb.AppendLine("=== INSURANCE SLIP DOCUMENT TEXT ==="); @@ -219,7 +298,7 @@ public async Task ExtractCompleteText(string filename) } catch (Exception e) { - return $"Error extracting PDF text: {e.Message}"; + return $"Error extracting document text: {e.Message}"; } } @@ -227,7 +306,8 @@ public async Task ExtractCompleteText(string filename) [Description("Imports the structured slip data as JSON into the pricing")] public async Task ImportSlipData( [Description("Pricing data as JSON (optional if updating existing)")] string? pricingJson, - [Description("Array of Structure layer data as JSON (can contain multiple layers)")] string? structuresJson) + [Description("Array of ReinsuranceAcceptance data as JSON (can contain multiple acceptances)")] string? acceptancesJson, + [Description("Array of ReinsuranceSection data as JSON (sections/layers within acceptances)")] string? sectionsJson) { if (chat.Context?.Address?.Type != PricingAddress.TypeName) return "Please navigate to the pricing first."; @@ -254,22 +334,41 @@ public async Task ImportSlipData( } } - // Step 3: Process Structure layers (can be multiple) - if (!string.IsNullOrWhiteSpace(structuresJson)) + // Step 3: Process ReinsuranceAcceptance records (can be multiple) + if (!string.IsNullOrWhiteSpace(acceptancesJson)) { - var structuresData = JsonNode.Parse(ExtractJson(structuresJson)); + var acceptancesData = JsonNode.Parse(ExtractJson(acceptancesJson)); // Handle both array and single object - var structureArray = structuresData is JsonArray arr ? arr : new JsonArray { structuresData }; + var acceptanceArray = acceptancesData is JsonArray arr ? arr : new JsonArray { acceptancesData }; - foreach (var structureData in structureArray) + foreach (var acceptanceData in acceptanceArray) { - if (structureData is JsonObject structureObj) + if (acceptanceData is JsonObject acceptanceObj) { - var processedStructure = EnsureTypeFirst(structureObj, nameof(Structure)); - processedStructure["pricingId"] = pricingId; - RemoveNullProperties(processedStructure); - updates.Add(processedStructure); + var processedAcceptance = EnsureTypeFirst(acceptanceObj, nameof(ReinsuranceAcceptance)); + processedAcceptance["pricingId"] = pricingId; + RemoveNullProperties(processedAcceptance); + updates.Add(processedAcceptance); + } + } + } + + // Step 4: Process ReinsuranceSection records (can be multiple) + if (!string.IsNullOrWhiteSpace(sectionsJson)) + { + var sectionsData = JsonNode.Parse(ExtractJson(sectionsJson)); + + // Handle both array and single object + var sectionArray = sectionsData is JsonArray arr ? arr : new JsonArray { sectionsData }; + + foreach (var sectionData in sectionArray) + { + if (sectionData is JsonObject sectionObj) + { + var processedSection = EnsureTypeFirst(sectionObj, nameof(ReinsuranceSection)); + RemoveNullProperties(processedSection); + updates.Add(processedSection); } } } @@ -277,7 +376,7 @@ public async Task ImportSlipData( if (updates.Count == 0) return "No valid data provided for import."; - // Step 4: Post DataChangeRequest + // Step 5: Post DataChangeRequest var updateRequest = new DataChangeRequest { Updates = updates }; var response = await hub.AwaitResponse(updateRequest, o => o.WithTarget(pricingAddress)); @@ -433,34 +532,6 @@ private async Task ExtractCompletePdfText(Stream stream) return completeText.ToString(); } - private static async Task OpenContentReadStreamAsync( - IContentService contentService, - string pricingId, - string filename) - { - try - { - // Parse pricingId in format {company}-{uwy} - var parts = pricingId.Split('-'); - if (parts.Length != 2) - return null; - - var company = parts[0]; - var uwy = parts[1]; - var contentPath = $"{company}/{uwy}/{filename}"; - - var collection = await contentService.GetCollectionAsync("Submissions", CancellationToken.None); - if (collection is null) - return null; - - return await collection.GetContentAsync(contentPath); - } - catch - { - return null; - } - } - private string ExtractJson(string json) { return json.Replace("```json", "") diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index 2c1aedf63..c1a565599 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -33,7 +33,7 @@ private static IServiceCollection AddInsuranceDomainServices(this IServiceCollec /// public static MessageHubConfiguration ConfigureInsuranceApplication(this MessageHubConfiguration configuration) => configuration - .WithTypes(typeof(PricingAddress), typeof(ImportConfiguration), typeof(ExcelImportConfiguration), typeof(Structure), typeof(ImportRequest), typeof(CollectionSource), typeof(GeocodingRequest), typeof(GeocodingResponse)) + .WithTypes(typeof(PricingAddress), typeof(ImportConfiguration), typeof(ExcelImportConfiguration), typeof(ReinsuranceAcceptance), typeof(ReinsuranceSection), typeof(ImportRequest), typeof(CollectionSource), typeof(GeocodingRequest), typeof(GeocodingResponse)) .AddData(data => { var svc = data.Hub.ServiceProvider.GetRequiredService(); @@ -102,7 +102,8 @@ public static MessageHubConfiguration ConfigureSinglePricingApplication(this Mes })) .WithType(t => t.WithInitialData(async ct => (IEnumerable)await svc.GetRisksAsync(pricingId, ct))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) + .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) + .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) .WithType(t => t.WithInitialData(async ct => await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) ); @@ -116,6 +117,8 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) LayoutAreas.PropertyRisksLayoutArea.PropertyRisks) .WithView(nameof(LayoutAreas.RiskMapLayoutArea.RiskMap), LayoutAreas.RiskMapLayoutArea.RiskMap) + .WithView(nameof(LayoutAreas.ReinsuranceAcceptanceLayoutArea.ReinsuranceAcceptances), + LayoutAreas.ReinsuranceAcceptanceLayoutArea.ReinsuranceAcceptances) .WithView(nameof(LayoutAreas.ImportConfigsLayoutArea.ImportConfigs), LayoutAreas.ImportConfigsLayoutArea.ImportConfigs) ) @@ -175,7 +178,7 @@ private static async Task HandleGeocodingRequest( Updates = geocodingResponse.UpdatedRisks.ToList() }; - await hub.AwaitResponse(dataChangeRequest, o => o.WithTarget(hub.Address), ct); + hub.Post(dataChangeRequest, o => o.WithTarget(hub.Address)); } // Post the response diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs new file mode 100644 index 000000000..45d2a3af9 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs @@ -0,0 +1,174 @@ +using System.Reactive.Linq; +using System.Text; +using MeshWeaver.Data; +using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; + +namespace MeshWeaver.Insurance.Domain.LayoutAreas; + +/// +/// Layout area for displaying reinsurance acceptances associated with a pricing. +/// +public static class ReinsuranceAcceptanceLayoutArea +{ + // Color definitions for diagram elements + private static readonly string PricingColor = "#2c7bb6"; // Blue - dark background, white text + private static readonly string PricingTextColor = "#ffffff"; + private static readonly string AcceptanceColor = "#fdae61"; // Light Orange - light background, dark text + private static readonly string AcceptanceTextColor = "#000000"; + private static readonly string SectionColor = "#abd9e9"; // Light Blue - light background, dark text + private static readonly string SectionTextColor = "#000000"; + + /// + /// Renders the reinsurance acceptances structure for a specific pricing. + /// + public static IObservable ReinsuranceAcceptances(LayoutAreaHost host, RenderingContext ctx) + { + _ = ctx; + var pricingId = host.Hub.Address.Id; + var acceptanceStream = host.Workspace.GetStream()!; + var sectionStream = host.Workspace.GetStream()!; + + return Observable.CombineLatest( + acceptanceStream, + sectionStream, + (acceptances, sections) => (acceptances, sections)) + .Select(data => + { + var acceptanceList = data.acceptances?.ToList() ?? new List(); + var sectionList = data.sections?.ToList() ?? new List(); + + if (!acceptanceList.Any() && !sectionList.Any()) + { + return Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(Controls.Markdown("# Reinsurance Structure\n\n*No reinsurance acceptances loaded. Import or add acceptances to begin.*")); + } + + var diagram = BuildMermaidDiagram(pricingId, acceptanceList, sectionList); + var mermaidControl = new MarkdownControl($"```mermaid\n{diagram}\n```") + .WithStyle(style => style.WithWidth("100%").WithHeight("600px")); + + return Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(Controls.Title("Reinsurance Structure", 1)) + .WithView(mermaidControl); + }) + .StartWith(Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(Controls.Markdown("# Reinsurance Structure\n\n*Loading...*"))); + } + + private static string BuildMermaidDiagram(string pricingId, List acceptances, List sections) + { + var sb = new StringBuilder(); + + // Start with flowchart definition for cards + sb.AppendLine("flowchart TD"); + sb.AppendLine(" classDef leftAlign text-align:left"); + + // Add the pricing node (main node) + sb.AppendLine($" pricing[\"Pricing: {pricingId}\"]"); + sb.AppendLine($" style pricing fill:{PricingColor},color:{PricingTextColor},stroke:#333,stroke-width:1px"); + sb.AppendLine($" class pricing leftAlign"); + + // Group sections by acceptanceId + var sectionsByAcceptance = sections + .Where(s => s.AcceptanceId != null) + .GroupBy(s => s.AcceptanceId) + .ToDictionary(g => g.Key!, g => g.ToList()); + + // Add acceptances and their sections + // Sort acceptances by name to ensure Layer 1, Layer 2, Layer 3 order + foreach (var acceptance in acceptances.OrderBy(a => a.Name ?? a.Id)) + { + RenderAcceptance(sb, acceptance, pricingId); + + // Add sections for this acceptance + // Sort sections by Type first, then by Attach to group coverage types together + if (sectionsByAcceptance.TryGetValue(acceptance.Id, out var acceptanceSections)) + { + foreach (var section in acceptanceSections.OrderBy(s => s.Type).ThenBy(s => s.Attach)) + { + RenderSection(sb, section, acceptance.Id); + } + } + } + + return sb.ToString(); + } + + private static void RenderAcceptance(StringBuilder sb, ReinsuranceAcceptance acceptance, string pricingId) + { + string acceptanceId = SanitizeId(acceptance.Id); + var acceptanceName = acceptance.Name ?? acceptance.Id; + + // Build acceptance content + var acceptanceContent = new StringBuilder(); + acceptanceContent.Append($"{acceptanceName}"); + + if (acceptance.EPI > 0) + acceptanceContent.Append($"
EPI: {acceptance.EPI:N0}"); + + if (acceptance.Rate > 0) + acceptanceContent.Append($"
Rate: {acceptance.Rate:P2}"); + + if (acceptance.Share > 0) + acceptanceContent.Append($"
Share: {acceptance.Share:P2}"); + + if (acceptance.Cession > 0) + acceptanceContent.Append($"
Cession: {acceptance.Cession:P2}"); + + if (acceptance.Brokerage > 0) + acceptanceContent.Append($"
Brokerage: {acceptance.Brokerage:P2}"); + + if (acceptance.Commission > 0) + acceptanceContent.Append($"
Commission: {acceptance.Commission:P2}"); + + sb.AppendLine($" acc_{acceptanceId}[\"{acceptanceContent}\"]"); + sb.AppendLine($" style acc_{acceptanceId} fill:{AcceptanceColor},color:{AcceptanceTextColor},stroke:#333,stroke-width:1px"); + sb.AppendLine($" class acc_{acceptanceId} leftAlign"); + sb.AppendLine($" pricing --> acc_{acceptanceId}"); + } + + private static void RenderSection(StringBuilder sb, ReinsuranceSection section, string acceptanceId) + { + string sectionId = SanitizeId(section.Id); + string sanitizedAcceptanceId = SanitizeId(acceptanceId); + + // Build section content + var sectionContent = new StringBuilder(); + sectionContent.Append($"{section.Name ?? section.Type ?? section.Id}
"); + + if (!string.IsNullOrEmpty(section.Type) && section.Type != section.Name) + { + sectionContent.Append($"Type: {section.Type}
"); + } + + sectionContent.Append($"Attach: {section.Attach:N0}
"); + sectionContent.Append($"Limit: {section.Limit:N0}
"); + + if (section.AggAttach.HasValue && section.AggAttach.Value > 0) + { + sectionContent.Append($"AAD: {section.AggAttach.Value:N0}
"); + } + + if (section.AggLimit.HasValue && section.AggLimit.Value > 0) + { + sectionContent.Append($"AAL: {section.AggLimit.Value:N0}"); + } + + // Create node + sb.AppendLine($" sec_{sectionId}[\"{sectionContent}\"]"); + sb.AppendLine($" style sec_{sectionId} fill:{SectionColor},color:{SectionTextColor},stroke:#333,stroke-width:1px"); + sb.AppendLine($" class sec_{sectionId} leftAlign"); + sb.AppendLine($" acc_{sanitizedAcceptanceId} --> sec_{sectionId}"); + } + + private static string SanitizeId(string id) + { + // Replace characters that might cause issues in Mermaid IDs + return id.Replace("-", "_").Replace(" ", "_").Replace(".", "_"); + } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs index 350b3670d..e68b14622 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs @@ -1,8 +1,12 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using MeshWeaver.Data; +using MeshWeaver.GoogleMaps; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; +using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Insurance.Domain.LayoutAreas; @@ -15,42 +19,50 @@ public static class RiskMapLayoutArea /// /// Renders a map view of property risks for a specific pricing. /// - public static IObservable RiskMap(LayoutAreaHost host, RenderingContext ctx) + public static IObservable RiskMap(LayoutAreaHost host, RenderingContext _) { - _ = ctx; var pricingId = host.Hub.Address.Id; - var riskStream = host.Workspace.GetStream()!; - return riskStream.Select(risks => - { - var riskList = risks?.ToList() ?? new List(); - var geocodedRisks = riskList.Where(r => r.GeocodedLocation?.Latitude != null && r.GeocodedLocation?.Longitude != null).ToList(); - - if (!riskList.Any()) + return host.Workspace.GetStream()! + .Select(risks => { - return Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown("# Risk Map\n\n*No risks loaded. Import or add risks to begin.*")); - } + var riskList = risks?.ToList() ?? new List(); + var geocodedRisks = riskList.Where(r => r.GeocodedLocation?.Latitude != null && r.GeocodedLocation?.Longitude != null).ToList(); + + if (!riskList.Any()) + { + return Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) + .WithView(Controls.Markdown("# Risk Map\n\n*No risks loaded. Import or add risks to begin.*")); + } + + if (!geocodedRisks.Any()) + { + return Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) + .WithView(Controls.Markdown($"# Risk Map\n\n*No geocoded risks found. {riskList.Count} risk(s) available but none have valid coordinates.*")) + .WithView(GeocodingArea); + } + + var mapControl = BuildGoogleMapControl(geocodedRisks); + var riskDetailsSubject = new ReplaySubject(1); + riskDetailsSubject.OnNext(null); + mapControl = mapControl.WithClickAction(ctx => riskDetailsSubject.OnNext(ctx.Payload?.ToString())); - if (!geocodedRisks.Any()) - { return Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown($"# Risk Map\n\n*No geocoded risks found. {riskList.Count} risk(s) available but none have valid coordinates.*")) - .WithView(GeocodingArea); - } - - var mapContent = RenderMapContent(geocodedRisks); - - return Controls.Stack + .WithView(Controls.Title("Risk Map", 2)) + .WithView(mapControl) + .WithView(GeocodingArea) + .WithView(Controls.Title("Risk Details", 3)) + .WithView((h, c) => riskDetailsSubject + .SelectMany(id => id == null ? + Observable.Return(Controls.Html("Click marker to see details.")) : RenderRiskDetails(host.Hub, id)) + ); + }) + .StartWith(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown(mapContent)) - .WithView(GeocodingArea); - }) - .StartWith(Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown("# Risk Map\n\n*Loading...*"))); + .WithView(Controls.Markdown("# Risk Map\n\n*Loading...*"))); } private static IObservable GeocodingArea(LayoutAreaHost host, RenderingContext ctx) @@ -87,36 +99,76 @@ private static async Task ClickGeocoding(UiActionContext obj) } } - private static string RenderMapContent(List geocodedRisks) + private static IObservable RenderRiskDetails(IMessageHub hub, string id) { - var lines = new List - { - "# Risk Map", - "", - $"**Total Geocoded Risks:** {geocodedRisks.Count}", - "", - "## Risk Locations", - "" - }; + return hub.GetWorkspace() + .GetStream(new EntityReference(nameof(PropertyRisk), id))! + .Select(r => BuildRiskDetailsMarkdown(r.Value as PropertyRisk)); + } - foreach (var risk in geocodedRisks.Take(10)) + private static UiControl BuildRiskDetailsMarkdown(PropertyRisk? risk) + { + if (risk is null) + return Controls.Html("Risk not found"); + + return Controls.Stack + .WithView(Controls.Markdown("## Risk Details")) + .WithView(Controls.Markdown($"**ID:** {risk.Id}")) + .WithView(Controls.Markdown($"**Location:** {risk.LocationName ?? "N/A"}")) + .WithView(Controls.Markdown($"**Address:** {risk.GeocodedLocation?.FormattedAddress ?? risk.Address ?? "N/A"}")) + .WithView(Controls.Markdown($"**City:** {risk.City ?? "N/A"}")) + .WithView(Controls.Markdown($"**State:** {risk.State ?? "N/A"}")) + .WithView(Controls.Markdown($"**Country:** {risk.Country ?? "N/A"}")) + .WithView(Controls.Markdown($"**Currency:** {risk.Currency ?? "N/A"}")) + .WithView(Controls.Markdown($"**TSI Building:** {risk.TsiBuilding:N0}")) + .WithView(Controls.Markdown($"**TSI Content:** {risk.TsiContent:N0}")) + .WithView(Controls.Markdown($"**TSI BI:** {risk.TsiBi:N0}")) + .WithView(Controls.Markdown($"**Latitude:** {risk.GeocodedLocation?.Latitude:F6}")) + .WithView(Controls.Markdown($"**Longitude:** {risk.GeocodedLocation?.Longitude:F6}")); + } + + private static GoogleMapControl BuildGoogleMapControl(IReadOnlyCollection risks) + { + var riskList = risks.Where(r => r.GeocodedLocation?.Latitude is not null && r.GeocodedLocation?.Longitude is not null).ToList(); + + // Find center point + LatLng center; + if (riskList.Any()) { - lines.Add($"- **{risk.LocationName ?? "Unknown"}**: {risk.City}, {risk.State}, {risk.Country}"); - lines.Add($" - Coordinates: {risk.GeocodedLocation!.Latitude:F6}, {risk.GeocodedLocation.Longitude:F6}"); - lines.Add($" - TSI Building: {risk.Currency} {risk.TsiBuilding:N0}"); - lines.Add(""); + var avgLat = riskList.Average(r => r.GeocodedLocation!.Latitude!.Value); + var avgLng = riskList.Average(r => r.GeocodedLocation!.Longitude!.Value); + center = new LatLng(avgLat, avgLng); } - - if (geocodedRisks.Count > 10) + else { - lines.Add($"*... and {geocodedRisks.Count - 10} more risk(s)*"); - lines.Add(""); + center = new LatLng(0, 0); } - lines.Add("---"); - lines.Add(""); - lines.Add("*Interactive map visualization coming soon...*"); + // Create markers + var markers = riskList.Select(r => new MapMarker + { + Position = new LatLng(r.GeocodedLocation!.Latitude!.Value, r.GeocodedLocation.Longitude!.Value), + Title = ((r.LocationName ?? r.Address) + " " + (r.City ?? "")).Trim(), + Id = r.Id, + Data = r + }).ToList(); - return string.Join("\n", lines); + var mapOptions = new MapOptions + { + Center = center, + Zoom = riskList.Any() ? 6 : 2, + MapTypeId = "roadmap", + ZoomControl = true, + MapTypeControl = true, + StreetViewControl = false, + FullscreenControl = true + }; + + return new GoogleMapControl() + { + Options = mapOptions, + Markers = markers, + Id = "risk-map" + }.WithStyle(style => style.WithHeight("500px").WithWidth("80%")); } } diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs index 05fc4b5a1..aa4d0a957 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs @@ -18,6 +18,7 @@ string Item(string key, string icon, string text) {Item("Submission", "📎", "Submission")} {Item("PropertyRisks", "📄", "Risks")} {Item("RiskMap", "🗺️", "Map")} +{Item("ReinsuranceAcceptances", "🏦", "Reinsurance")} {Item("ImportConfigs", "⚙️", "Import")} "; diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs new file mode 100644 index 000000000..f9a6faf0d --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs @@ -0,0 +1,86 @@ +using System.ComponentModel.DataAnnotations; + +namespace MeshWeaver.Insurance.Domain; + +/// +/// Represents the reinsurance acceptance with financial terms and coverage sections. +/// +public record ReinsuranceAcceptance +{ + /// + /// Gets or initializes the unique acceptance identifier. + /// + [Key] + public required string Id { get; init; } + + /// + /// Gets or initializes the pricingId this acceptance belongs to. + /// + public string? PricingId { get; init; } + + /// + /// Gets or initializes the acceptance name or description. + /// + public string? Name { get; init; } + + /// + /// Gets or initializes the cession percentage. + /// + public double Cession { get; init; } + + /// + /// Gets or initializes the share percentage. + /// + public double Share { get; init; } + + /// + /// Gets or initializes the collection of reinsurance sections (layers). + /// + public IReadOnlyCollection? Sections { get; init; } + + /// + /// Gets or initializes the Estimated Premium Income (EPI). + /// + public double EPI { get; init; } + + /// + /// Gets or initializes the rate. + /// + public double Rate { get; init; } + + /// + /// Gets or initializes the commission percentage. + /// + public double Commission { get; init; } + + /// + /// Gets or initializes the brokerage percentage. + /// + public double Brokerage { get; init; } + + /// + /// Gets or initializes the tax percentage. + /// + public double Tax { get; init; } + + /// + /// Gets or initializes the reinstatement premium. + /// + public double ReinstPrem { get; init; } + + /// + /// Gets or initializes the no claims bonus percentage. + /// + public double NoClaimsBonus { get; init; } + + /// + /// Gets or initializes the profit commission percentage. + /// + public double ProfitComm { get; init; } + + + /// + /// Gets or initializes the minimum and deposit premium. + /// + public double MDPrem { get; init; } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs new file mode 100644 index 000000000..7e49e4bc6 --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs @@ -0,0 +1,50 @@ +using System.ComponentModel.DataAnnotations; + +namespace MeshWeaver.Insurance.Domain; + +/// +/// Represents a reinsurance coverage section with layer structure and financial terms. +/// +public record ReinsuranceSection +{ + /// + /// Gets or initializes the unique section identifier. + /// + [Key] + public required string Id { get; init; } + + /// + /// Gets or initializes the acceptanceId this section belongs to. + /// + public string? AcceptanceId { get; init; } + + /// + /// Gets or initializes the section name or description. + /// + public string? Name { get; init; } + + /// + /// Gets or initializes the section type (e.g., "Fire Damage", "Natural Catastrophe", "Business Interruption"). + /// + public string? Type { get; init; } + + /// + /// Gets or initializes the attachment point. + /// + public decimal Attach { get; init; } + + /// + /// Gets or initializes the layer limit. + /// + public decimal Limit { get; init; } + + /// + /// Gets or initializes the aggregate attachment point (annual aggregate deductible). + /// + public decimal? AggAttach { get; init; } + + /// + /// Gets or initializes the aggregate limit (annual aggregate limit). + /// + public decimal? AggLimit { get; init; } +} diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs deleted file mode 100644 index 0c1ac7ffe..000000000 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/Structure.cs +++ /dev/null @@ -1,110 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace MeshWeaver.Insurance.Domain; - -/// -/// Represents the reinsurance layer structure and financial terms. -/// -public record Structure -{ - /// - /// Gets or initializes the unique layer identifier. - /// - [Key] - public required string LayerId { get; init; } - - /// - /// Gets or initializes the pricing/contract ID this structure belongs to. - /// - public string? PricingId { get; init; } - - /// - /// Gets or initializes the layer type. - /// - public string? Type { get; init; } - - /// - /// Gets or initializes the cession percentage. - /// - public decimal Cession { get; init; } - - /// - /// Gets or initializes the share percentage. - /// - public decimal Share { get; init; } - - /// - /// Gets or initializes the attachment point. - /// - public decimal Attach { get; init; } - - /// - /// Gets or initializes the layer limit. - /// - public decimal Limit { get; init; } - - /// - /// Gets or initializes the aggregate attachment point. - /// - public decimal AggAttach { get; init; } - - /// - /// Gets or initializes the aggregate limit. - /// - public decimal AggLimit { get; init; } - - /// - /// Gets or initializes the number of reinstatements. - /// - public int NumReinst { get; init; } - - /// - /// Gets or initializes the Estimated Premium Income (EPI). - /// - public decimal EPI { get; init; } - - /// - /// Gets or initializes the rate on line. - /// - public decimal Rate { get; init; } - - /// - /// Gets or initializes the commission percentage. - /// - public decimal Commission { get; init; } - - /// - /// Gets or initializes the brokerage percentage. - /// - public decimal Brokerage { get; init; } - - /// - /// Gets or initializes the tax percentage. - /// - public decimal Tax { get; init; } - - /// - /// Gets or initializes the reinstatement premium. - /// - public decimal ReinstPrem { get; init; } - - /// - /// Gets or initializes the no claims bonus percentage. - /// - public decimal NoClaimsBonus { get; init; } - - /// - /// Gets or initializes the profit commission percentage. - /// - public decimal ProfitComm { get; init; } - - /// - /// Gets or initializes the management expense percentage. - /// - public decimal MgmtExp { get; init; } - - /// - /// Gets or initializes the minimum and deposit premium. - /// - public decimal MDPrem { get; init; } -} diff --git a/portal/MeshWeaver.Portal/appsettings.Development.json b/portal/MeshWeaver.Portal/appsettings.Development.json index 0ea4c9e68..fda758d7c 100644 --- a/portal/MeshWeaver.Portal/appsettings.Development.json +++ b/portal/MeshWeaver.Portal/appsettings.Development.json @@ -10,9 +10,6 @@ "SourceType": "FileSystem", "BasePath": "../../modules/Insurance/Files" }, - "GoogleMaps": { - "ApiKey": "" - }, "EntraId": { "Instance": "https://login.microsoftonline.com/", "Domain": "meshweaverportal.onmicrosoft.com", // Your Entra ID tenant domain From fb270ca673f4544555f5e80b1a0e2c83cb648768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 28 Oct 2025 09:28:08 +0100 Subject: [PATCH 07/57] iterating on indusrance domain --- .../Insurance/Files/Microsoft/2026/Slip.md | 2 +- .../SlipImportAgent.cs | 26 ++++++----- .../InsuranceApplicationExtensions.cs | 4 +- .../LayoutAreas/PricingCatalogLayoutArea.cs | 11 ++--- .../LayoutAreas/PricingOverviewLayoutArea.cs | 9 ++-- .../ReinsuranceAcceptanceLayoutArea.cs | 45 ++++++++++++------- .../LayoutAreas/Shared/PricingLayoutShared.cs | 2 +- .../MeshWeaver.Insurance.Domain/Pricing.cs | 6 +-- .../ReinsuranceSection.cs | 4 +- .../SampleDataProvider.cs | 12 +---- 10 files changed, 63 insertions(+), 58 deletions(-) diff --git a/modules/Insurance/Files/Microsoft/2026/Slip.md b/modules/Insurance/Files/Microsoft/2026/Slip.md index 739a4bd12..dc26a1d3d 100644 --- a/modules/Insurance/Files/Microsoft/2026/Slip.md +++ b/modules/Insurance/Files/Microsoft/2026/Slip.md @@ -49,7 +49,7 @@ --- -### 2. Natural Catastrophe (Earthquake, Flood, Storm) +### 2. Natural Catastrophe (Windstorm, Earthquake) - **Layer 1:** - Deductible per Occurrence: USD 5,000,000 diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index 79dbf68e7..b964ae98b 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -56,7 +56,11 @@ You are a slip import agent that processes insurance submission slip documents i # Data Mapping Guidelines Based on the extracted document text, create JSON objects that match the schemas provided below: - - **Pricing**: Basic pricing information (insured name, broker, dates, premium, country, legal entity) + - **Pricing**: Basic pricing information including: + - Insured name (e.g., "Microsoft Corporation") + - Primary insurance company (labeled as "Primary Insurer" or similar in slip header) - populate the PrimaryInsurance field + - Broker name (labeled as "Broker" in slip header) - populate the BrokerName field + - Dates (inception, expiration), premium, country, legal entity - **ReinsuranceAcceptance**: Represents a reinsurance layer (Layer 1, Layer 2, Layer 3) with financial terms - **ReinsuranceSection**: Represents a coverage type within a layer (Fire Damage, Natural Catastrophe, Business Interruption) @@ -86,7 +90,7 @@ You are a slip import agent that processes insurance submission slip documents i - Then, create ReinsuranceSection records for each coverage type within each layer - Use IDs like "Layer1-Fire", "Layer1-NatCat", "Layer1-BI", "Layer2-Fire", etc. - Set the AcceptanceId to link the section to its parent layer (e.g., "Layer1") - - Set the Type to the coverage type (e.g., "Fire Damage", "Natural Catastrophe", "Business Interruption") + - Set the LineOfBusiness to the coverage type (e.g., "Fire Damage", "Natural Catastrophe", "Business Interruption") - Set the Name to a descriptive name (e.g., "Fire Damage - Layer 1") - Include the attachment point (Attach), limit, aggregate deductible (AggAttach), and aggregate limit (AggLimit) @@ -99,17 +103,19 @@ You are a slip import agent that processes insurance submission slip documents i Create: 1. ReinsuranceAcceptance: Id="Layer1", Name="Layer 1" 2. ReinsuranceAcceptance: Id="Layer2", Name="Layer 2" - 3. ReinsuranceSection: Id="Layer1-Fire", AcceptanceId="Layer1", Type="Fire Damage", Attach=5000000, Limit=100000000, AggAttach=25000000, AggLimit=300000000 - 4. ReinsuranceSection: Id="Layer2-Fire", AcceptanceId="Layer2", Type="Fire Damage", Attach=100000000, Limit=150000000, AggLimit=450000000 - 5. ReinsuranceSection: Id="Layer1-NatCat", AcceptanceId="Layer1", Type="Natural Catastrophe", Attach=10000000, Limit=75000000, AggAttach=30000000, AggLimit=225000000 + 3. ReinsuranceSection: Id="Layer1-Fire", AcceptanceId="Layer1", LineOfBusiness="Fire Damage", Attach=5000000, Limit=100000000, AggAttach=25000000, AggLimit=300000000 + 4. ReinsuranceSection: Id="Layer2-Fire", AcceptanceId="Layer2", LineOfBusiness="Fire Damage", Attach=100000000, Limit=150000000, AggLimit=450000000 + 5. ReinsuranceSection: Id="Layer1-NatCat", AcceptanceId="Layer1", LineOfBusiness="Natural Catastrophe", Attach=10000000, Limit=75000000, AggAttach=30000000, AggLimit=225000000 # Document Section Processing Look for common sections in insurance slips: - - Insured information (name, location, industry) - - Coverage details (inception/expiration dates, policy terms) - - Premium and financial information - - Layer structures (limits, attachments, rates) - - Reinsurance terms (commission, brokerage, taxes) + - **Header section**: Insured name, Primary Insurer, Broker, dates + - **Insured information**: Name, location, industry + - **Coverage details**: Inception/expiration dates, policy terms + - **Premium and financial information**: Premium amounts, currency + - **Reinsurance terms section**: EPI (Estimated Premium Income), Brokerage percentage, Commission, Taxes + - **Layer structures**: Layer 1, Layer 2, Layer 3 with limits, attachments, rates + - **Coverage types within layers**: Fire Damage, Natural Catastrophe, Business Interruption, etc. Notes: - When listing files, you may see paths with "/" prefix (e.g., "/Slip.pdf", "/Slip.md") diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index c1a565599..0a586ef98 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -117,8 +117,8 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) LayoutAreas.PropertyRisksLayoutArea.PropertyRisks) .WithView(nameof(LayoutAreas.RiskMapLayoutArea.RiskMap), LayoutAreas.RiskMapLayoutArea.RiskMap) - .WithView(nameof(LayoutAreas.ReinsuranceAcceptanceLayoutArea.ReinsuranceAcceptances), - LayoutAreas.ReinsuranceAcceptanceLayoutArea.ReinsuranceAcceptances) + .WithView(nameof(LayoutAreas.ReinsuranceAcceptanceLayoutArea.Structure), + LayoutAreas.ReinsuranceAcceptanceLayoutArea.Structure) .WithView(nameof(LayoutAreas.ImportConfigsLayoutArea.ImportConfigs), LayoutAreas.ImportConfigsLayoutArea.ImportConfigs) ) diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs index e6cb6221b..285cb23f1 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs @@ -1,4 +1,4 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; @@ -30,8 +30,8 @@ private static string RenderPricingTable(IReadOnlyCollection pricings) { "# Insurance Pricing Catalog", "", - "| Insured | Line of Business | Country | Legal Entity | Inception | Expiration | Premium | Status |", - "|---------|------------------|---------|--------------|-----------|------------|---------|--------|" + "| Insured | Line of Business | Country | Legal Entity | Inception | Expiration | Status |", + "|---------|------------------|---------|--------------|-----------|------------|--------|" }; lines.AddRange(pricings @@ -41,10 +41,7 @@ private static string RenderPricingTable(IReadOnlyCollection pricings) var link = $"[{p.InsuredName}](/pricing/{p.Id}/Overview)"; var inception = p.InceptionDate?.ToString("yyyy-MM-dd") ?? "-"; var expiration = p.ExpirationDate?.ToString("yyyy-MM-dd") ?? "-"; - var premium = p.Premium.HasValue && p.Currency != null - ? $"{p.Currency} {p.Premium:N0}" - : "-"; - return $"| {link} | {p.LineOfBusiness ?? "-"} | {p.Country ?? "-"} | {p.LegalEntity ?? "-"} | {inception} | {expiration} | {premium} | {p.Status ?? "-"} |"; + return $"| {link} | {p.LineOfBusiness ?? "-"} | {p.Country ?? "-"} | {p.LegalEntity ?? "-"} | {inception} | {expiration} | {p.Status ?? "-"} |"; })); return string.Join("\n", lines); diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingOverviewLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingOverviewLayoutArea.cs index 5987db795..6925fa2fb 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingOverviewLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingOverviewLayoutArea.cs @@ -1,4 +1,4 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; @@ -56,12 +56,9 @@ private static string RenderPricingOverview(Pricing pricing) $"- **Country:** {pricing.Country ?? "N/A"}", $"- **Legal Entity:** {pricing.LegalEntity ?? "N/A"}", "", - "### Financial", - $"- **Premium:** {(pricing.Premium.HasValue && pricing.Currency != null ? $"{pricing.Currency} {pricing.Premium:N2}" : "N/A")}", - $"- **Currency:** {pricing.Currency ?? "N/A"}", - "", "### Parties", - $"- **Broker:** {pricing.BrokerName ?? "N/A"}" + $"- **Broker:** {pricing.BrokerName ?? "N/A"}", + $"- **Primary Insurance:** {pricing.PrimaryInsurance ?? "N/A"}" }; return string.Join("\n", lines); diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs index 45d2a3af9..7631b9dc3 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/ReinsuranceAcceptanceLayoutArea.cs @@ -1,6 +1,5 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using System.Text; -using MeshWeaver.Data; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; @@ -23,44 +22,47 @@ public static class ReinsuranceAcceptanceLayoutArea /// /// Renders the reinsurance acceptances structure for a specific pricing. /// - public static IObservable ReinsuranceAcceptances(LayoutAreaHost host, RenderingContext ctx) + public static IObservable Structure(LayoutAreaHost host, RenderingContext ctx) { _ = ctx; var pricingId = host.Hub.Address.Id; + var pricingStream = host.Workspace.GetStream()!; var acceptanceStream = host.Workspace.GetStream()!; var sectionStream = host.Workspace.GetStream()!; return Observable.CombineLatest( + pricingStream, acceptanceStream, sectionStream, - (acceptances, sections) => (acceptances, sections)) + (pricings, acceptances, sections) => (pricings, acceptances, sections)) .Select(data => { + var pricing = data.pricings?.FirstOrDefault(); var acceptanceList = data.acceptances?.ToList() ?? new List(); var sectionList = data.sections?.ToList() ?? new List(); if (!acceptanceList.Any() && !sectionList.Any()) { return Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "Structure")) .WithView(Controls.Markdown("# Reinsurance Structure\n\n*No reinsurance acceptances loaded. Import or add acceptances to begin.*")); } - var diagram = BuildMermaidDiagram(pricingId, acceptanceList, sectionList); + var diagram = BuildMermaidDiagram(pricingId, pricing, acceptanceList, sectionList); var mermaidControl = new MarkdownControl($"```mermaid\n{diagram}\n```") .WithStyle(style => style.WithWidth("100%").WithHeight("600px")); return Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "Structure")) .WithView(Controls.Title("Reinsurance Structure", 1)) .WithView(mermaidControl); }) .StartWith(Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "ReinsuranceAcceptances")) + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "Structure")) .WithView(Controls.Markdown("# Reinsurance Structure\n\n*Loading...*"))); } - private static string BuildMermaidDiagram(string pricingId, List acceptances, List sections) + private static string BuildMermaidDiagram(string pricingId, Pricing? pricing, List acceptances, List sections) { var sb = new StringBuilder(); @@ -69,7 +71,20 @@ private static string BuildMermaidDiagram(string pricingId, ListPricing: {pricingId}\"]"); + var pricingContent = new StringBuilder(); + pricingContent.Append($"Pricing: {pricingId}"); + + if (pricing?.PrimaryInsurance != null) + { + pricingContent.Append($"
Primary: {pricing.PrimaryInsurance}"); + } + + if (pricing?.BrokerName != null) + { + pricingContent.Append($"
Broker: {pricing.BrokerName}"); + } + + sb.AppendLine($" pricing[\"{pricingContent}\"]"); sb.AppendLine($" style pricing fill:{PricingColor},color:{PricingTextColor},stroke:#333,stroke-width:1px"); sb.AppendLine($" class pricing leftAlign"); @@ -86,10 +101,10 @@ private static string BuildMermaidDiagram(string pricingId, List s.Type).ThenBy(s => s.Attach)) + foreach (var section in acceptanceSections.OrderBy(s => s.LineOfBusiness).ThenBy(s => s.Attach)) { RenderSection(sb, section, acceptance.Id); } @@ -139,11 +154,11 @@ private static void RenderSection(StringBuilder sb, ReinsuranceSection section, // Build section content var sectionContent = new StringBuilder(); - sectionContent.Append($"{section.Name ?? section.Type ?? section.Id}
"); + sectionContent.Append($"{section.Name ?? section.LineOfBusiness ?? section.Id}
"); - if (!string.IsNullOrEmpty(section.Type) && section.Type != section.Name) + if (!string.IsNullOrEmpty(section.LineOfBusiness) && section.LineOfBusiness != section.Name) { - sectionContent.Append($"Type: {section.Type}
"); + sectionContent.Append($"LoB: {section.LineOfBusiness}
"); } sectionContent.Append($"Attach: {section.Attach:N0}
"); diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs index aa4d0a957..f00adbec9 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/Shared/PricingLayoutShared.cs @@ -18,7 +18,7 @@ string Item(string key, string icon, string text) {Item("Submission", "📎", "Submission")} {Item("PropertyRisks", "📄", "Risks")} {Item("RiskMap", "🗺️", "Map")} -{Item("ReinsuranceAcceptances", "🏦", "Reinsurance")} +{Item("Structure", "🏦", "Reinsurance")} {Item("ImportConfigs", "⚙️", "Import")} "; diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs index d724f0c20..0cb52c433 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs @@ -1,4 +1,4 @@ -using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations; using MeshWeaver.Domain; namespace MeshWeaver.Insurance.Domain; @@ -58,9 +58,9 @@ public record Pricing public string? BrokerName { get; init; } /// - /// Premium amount in the pricing currency. + /// Name of the primary insurance company. /// - public decimal? Premium { get; init; } + public string? PrimaryInsurance { get; init; } /// /// Currency code for the premium. diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs index 7e49e4bc6..66969f37b 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs @@ -1,4 +1,4 @@ -using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations; namespace MeshWeaver.Insurance.Domain; @@ -26,7 +26,7 @@ public record ReinsuranceSection /// /// Gets or initializes the section type (e.g., "Fire Damage", "Natural Catastrophe", "Business Interruption"). /// - public string? Type { get; init; } + public string? LineOfBusiness { get; init; } /// /// Gets or initializes the attachment point. diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/SampleDataProvider.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/SampleDataProvider.cs index 23f9f1f96..a4970841a 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/SampleDataProvider.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/SampleDataProvider.cs @@ -1,4 +1,4 @@ -namespace MeshWeaver.Insurance.Domain; +namespace MeshWeaver.Insurance.Domain; /// /// Provides sample data for insurance dimensions and entities. @@ -233,14 +233,12 @@ public static IEnumerable GetSamplePricings() { Id = "PRC-2024-001", InsuredName = "Global Manufacturing Corp", - BrokerName = "Marsh McLennan", InceptionDate = new DateTime(2024, 1, 1), ExpirationDate = new DateTime(2024, 12, 31), UnderwritingYear = 2024, LineOfBusiness = "PROP", Country = "US", LegalEntity = "MW-US", - Premium = 125000m, Currency = "USD", Status = "Bound" }, @@ -248,14 +246,12 @@ public static IEnumerable GetSamplePricings() { Id = "PRC-2024-002", InsuredName = "European Logistics Ltd", - BrokerName = "Aon", InceptionDate = new DateTime(2024, 3, 1), ExpirationDate = new DateTime(2025, 2, 28), UnderwritingYear = 2024, LineOfBusiness = "PROP", Country = "GB", LegalEntity = "MW-UK", - Premium = 85000m, Currency = "GBP", Status = "Quoted" }, @@ -263,14 +259,12 @@ public static IEnumerable GetSamplePricings() { Id = "PRC-2024-003", InsuredName = "Tech Industries GmbH", - BrokerName = "Willis Towers Watson", InceptionDate = new DateTime(2024, 6, 1), ExpirationDate = new DateTime(2025, 5, 31), UnderwritingYear = 2024, LineOfBusiness = "PROP", Country = "DE", LegalEntity = "MW-EU", - Premium = 95000m, Currency = "EUR", Status = "Draft" }, @@ -278,14 +272,10 @@ public static IEnumerable GetSamplePricings() { Id = "Microsoft-2026", InsuredName = "Microsoft", - BrokerName = "Marsh McLennan", - InceptionDate = new DateTime(2026, 1, 1), - ExpirationDate = new DateTime(2026, 12, 31), UnderwritingYear = 2026, LineOfBusiness = "PROP", Country = "US", LegalEntity = "MW-US", - Premium = 2500000m, Currency = "USD", Status = "Bound" } From 182f4444a3871bcbb1652ac4b63d615560a2e779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 28 Oct 2025 11:18:17 +0100 Subject: [PATCH 08/57] iterating on insurance domain --- .../Files/Microsoft/2026/Microsoft.xlsx | Bin 29723 -> 30012 bytes .../MeshWeaver.Insurance.Domain/Dimensions.cs | 4 ++++ .../InsuranceApplicationExtensions.cs | 2 ++ .../LayoutAreas/PricingCatalogLayoutArea.cs | 4 ++-- .../MeshWeaver.Insurance.Domain/Pricing.cs | 1 + .../PropertyRisk.cs | 1 + .../ReinsuranceAcceptance.cs | 1 + .../ReinsuranceSection.cs | 1 + 8 files changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx b/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx index 1513904a46c93963c530727861023994f5519884..ff3f2167e8501fc5c206e7950e0644dea3b363b7 100644 GIT binary patch delta 11309 zcmZX)byOT*@;;2aySoOrpuycOFt`VT1osRR+;w1rCAh=jZVB!doZv12f@}CqKD+zf zJ@4Cp%(;E0yKYsTs(PNfU0Db49Y^prw5T8>@K~@h5**wR4jddV92}gVGmo#Uhm*Cd zs}r}Mi*u!+lk3WBqGw6V7nwN(ug6^F*-;2yNsW1FaLW*t@Ddy%3ga|Bey5^NZD{Tb z@N?a+hDR_-jV*00VcL>7FJ}7P~H?PoU7wh1Mf)4VC1*m^f#~5 z2tNBGsC3M}`QAom%YrPkig@ngKT)WHSkha+X!|9Nba2nu1QF8lQQ1 zNCX+1osLyo%DXm9EA?5_$f5q}+L|fBKSQlsnOt81^!%~MbKm-UAr2qy&QE|A)K!NN zgvoP4lEwX^uvUD$)UJ?humm&#!J@xMLqc(dFU96Pxteky8*cAq|2VQS;sG{z(m71s zPqrQ(C@%FK0k4yQzL2DnrS#cD>hE~H^&RS&akKS0I>K0_d>IOKI&Ua_eXNBhlTH&%el*C2HHyOD0_Z|{QZ%F!Eme~Ec&zvG z?_;S&-`L_Z+L0MWSRW~BxohLHC%9e-uG2)wtUscFt(qu5w`?8`U?Bw1Wz}2+YS?rf z87J+%7W6Oro$vC8$o=JXrV&U;NV2lV#XR~j_d1`g>eL}Qbu_q=&gurV-`sDv;-^IT zHROzQHj?|eZ8A)<@Eh3@&;$wn2XcMz(~Q6yJ>L-JXxTTqW8b2Qc;b?_7-x?ecf>Pk z8{=lFRezyi&Mvp>H^{(`eo$Qq7Ebx#KA&X#Ae4_h5fs{K<~&$+QrBn7_llC)MZT08 zuMd$+nV|>M(pQ!8{QeaO=;BfofSu3qjcR>6GpaP!-RH;m-VWVNCV3|E`-^QU{D-^E zF>$%mrJYK9(^7GpXNQe-N{VJozI93X8y+z4!#UJfh5>v zKx}1b`#o_3q5Y(NcUFs`L{dsu6BvXcdVSBnBbbo%LnJzTpw@!|#G3qfxrgtO30p}R zXkIdTWWF%(pK4+Ria3w)QF--Mtv*l}9s-k>hGHz{Iy!sFLuD~!i(j9-PbhQ9vtoa@ zKAqO}{vgH`CBWgjhwh3qIkaEtQ(#PiTi~pFdo|{F{{A>E@&ww=EL&L;8e7t2`MzPq zyVI2G%B~V}Y0z{W-j3RiL(KkCu{CnC)wT9UB+1TbLS_1&tuLt4UCUh>ysjHbEED!F zOT-HfdhJ!Fl^m}RFUIdjz93mMWHrB3ZDWr~GNSqJ-2eW2-LaLg~z6=}gUkztnHjzk8>Zt}H*&B=9mr~8VZK69F zjW%^#`BC^yX128q_#EbyHdl8pYnaJz*^Mh5STINm><#j=eh=anbS7!dMey0d=apJF z=#*25X{I>?G&xLCD?hQ8C5gU0*Moj3PmP0)<$)wTB^Iu~WM3f6h2?cb(9fi@o@g?y zWD}Q1C_{9RA1bhUDk31P3mYDUT@cq%Fm?v8TUTfR#4xa~^p{aZ(?*o1dW0{xf z84PvpcDI=q|LXBoStnY5N!rJ8eD$1>)x%*XrR}#x`1ihK=Rt3HEw(dY%6YDxKP%wn z3oocU{LpK76H;w`RUyS1sv?+i6VfGhoJ?-YB8eVwc`r!ubNneE^Jspf*Okb?wZhEzi{5a;r85${OvB8T^Zc79;n7DpK8i=BaTV1f7j0nJA$mO`M zREnkJSTewSnf6C4$DfKsfp3zLXZ2P>x@-OP=~Mu%1{=X)umEo)<*g(1*oz!d3>)PH zUoI*>bO;ymlW7ss4LQ=HLG&BCH4`$Bwrww5pzYQ32VMGanB>9u{8%;E+e8S3hdnxX zQoKsvU(;_uA4jncf_mXrc%L*ico=!d%vOy9;1WB94)!jHRabJ|JPS8U^=1v}G-9{v zQ#ORv-aCe9Dg5quQE9C+bJt@B{^(PjQ1#9mYAM6JdE6{ppESq(b#l}km+>G4f~8)Uui(LTHGiHLxS_W3y)74>{bEIqRh{Yh z!Mn(!f&*`D%_YB$puEC{>A0AlmgY#~VQ(&Ss_+-EZFx=jCCS{^%Vl`Wi3=UcSw)Z7 z)O*mupZ1W)Scpp!+Tmq%?mU zyV};uJd+PiZ*oxJdiW)kBl&V)T_Pp*=NO0SLpCpOg`ep6T#^ezz*L>&ben3M? z68XX7z54;TA#C9Za&bS91T{nQ3dtEQ7q2^!xS+uE5no(wb7ml-4X~WniOSC0Vg0Sq%CvZyHZ#aufkETl^lV1@ab;odhU6pqec0m4cwFbV&_yS#W?Kzw9uzS0Dj1Py*Ix*Q`DTho zVGpLo3Yk5Q`?teL5e!K$LVj)^%&JL1mEJPdoa$z zh?l(OgIkVet5oK@BnnMlp-&TE)ELQ`sD<725GAmJbfm$^8mXw9>YkhUIuztNLqZz$ zlR&jcHnpnrp(LBtWy03?gESOdNi+Sg;@y?`lxnhnatdZwE9W_InKpP$J1Qa`rwb6j zb8^K}^8<>el{U)9)fFGOkg54!6W$368foLvz?sje`yuHo@_T3qoO`a%(2rE&jU<78 zQsUl&hA(=p=Bo-H%iHeLd$HIgGOeiRO+PBch-T7#Ab)WuWURdT8; z8Zn_=T`1bP=W{B8k*m>C$jqnXMG?w%BB1^{co>m$9?h=+#^~A+*|sj~k*uS*O5_H1 z2W+HQbpo$OlC7^V)0UmM(UKTzQ8btvdg}H-J7$B~WdOB1gRQ>dt!djDg)J>w=Xg6Y z0X1tuqw_kr4Ps`1lKlG_Aj3ijKiWB&v%yR%J+n~AP8gBRkT_!jrX$T{F1x|v{y})i z#!-HFEQH&*)q%PI9IT*)GeGPV0(K`>@dKI)k+4Fo8;smHsHK?rI`aK*qS!R*wLXDZ z)FvJfB?jI)a^>>x)C5F0Sck1!uoa9?aAtiHp7*BMz$OXg(w3{%~%2 zmJkmC7m=Gws~8?J4+hwdA%IX*b^;v^1E8i23Dd)Gnc}~b0c1Mkn-f_ED>e7x!xrD| z_hD9&X#n~}q&JC=iK0f*!=cQbXUK=<35_Y`ocg>8PWL|p#jb6L%uRNhczA@|90mxFX(N++kj#^L)ICi_MG)!r zbkYq{@p@d;*UAU&oG0z&7sVRP%IXEhi-IfQg0#;qNlTy13l)@l#SR&e({dDx-sBF) zybb!az!{zA6b%wwl-}q+^kkF3F?>g4H*~1=VJe|$Hpssw72!0g##57s>^@yFO$coQf6_#I!Rv+{d7jNHIv&s)k3xXs5HUM{x<5wHAq1B1 z0JTf8y{DqiLZTfY&xrgQu7a!1fD*^eCzk9(S~&B1EX-0)iD9A9$6w6xI`2-I>Fu4E z&RIvXQOV11A%F;TTQ)@0`HVFVF^3X8bpCT$hr^~^*;!N51wQ*%Lv2lo-rHg5#@(d0 z`PU`2rl6b)Yr?5_l(7vkkLcf_m7>+{WG=wL)3B;L!Rnv&S8h2~aaH#ynKxe|`nJim z33}BMuBTku+8j3jI-1*f@W6JW+hMSCh1te5ciHE=tI&lW!GHdXd7E>%) zNcpAuo&hSof4imX^+yo0qBkRd`-w$@fg0w5i9{&#HNuyi&J#Irl1BLL9y z@I*2{38&2wb3FsvC6y8?TgM(_^e4xoaH!GcnTeRQp3kiY5vKOE#{G${*p`S~{~E?2 z<^$g(@$L}cLYLu=G6D)$GSZkFhhttVBaJ7EZ(qBfuO9iLX{!`*s}vv`fY=He_mCob zcZF6uK+y&fr0w^-GzHFEOw??;dA6K$`P!PED~ze1g9XV2n03+4OQQ!cfh ziElR-?>BX@e|nu&xmibKb_t2^0`#muWH2frjK6(ats~CAnb#f5n3!<@Nra($@frSh zjn2J89VK(9yhF?kTllc;YlREvNk#*oCe65z{pXg8nh4~HODMYf5G8hId!HvQ7nAYJ zEU7n?Jpa98B_32i)>|grL=Bc6`5ec?l{OPQN%Po;gmTN)(5-a>z8ULHAbm1Q-BSed z%f=~5F{Qy1{O)Q9t27vdP5X3cvaDq6Nb@9<&_{-LKTp0SAO--a!8a#RO*2E6+KN6M z=l-jfj&nVU?WsH3oYK8E+_Q)pqvhPD8+hFQ#6l@~kI0Mbt|aY^hnNBCt@zYc1aznx z^Q;*_7OXmc+u4Q0NJLTH8+ptuhq%AhY zV4&5>dy6R*&({E9aFQ&%m-lp!f*NVnKbZ0@KbW+>rgiI&Y~QmbIVx|^KK9hRt6fe` z?lFOVd=SPl9LgD*v*;=;)`rMde=1+wD{nm~`g{<}NDyh!?rQ$}2NrEoR7r$`BVo{? zWWKRL*Ichr1O4{0^#1HJZIc#CmDtu0kXj!f=lm)ufm7~?EAw`um1a>wUfrQy>?anJ z;X55WVJ!>0=UpuRh}O?xR026tgoBzxZh_xbL0FzwV3v1#XXP&k!3?!9PdGN)aFql3 z_it@nPFcRmzJ?F!Ctr{*TUVW+?agkY{@%FvbYSceq`#*@UeKK>Im^T>AaU-8l3J9v zX7N+ZL%v-Al%4Fq9JJZSW7Z%$Yrt(iq3>L^s_zWF=*HWX0t`%TJu^##t9G8;tu$9a ziQ0e?b?raRDVGJMu8|}dH}ggjdZ2#bozWMNqUvq2AiJ$HpO(;0-=Q*<{Z}uGTvYO2 z(@Rv&Xt)3295m~EvfutwjIuvQqb%}~ai*Y>rJtKu3FMAsQ zfw1zi`j~@$iHazq%Sr}are|MdP_wgwZ;wO;nP+j&*z;+>2Xt+nz)BDk;tpEg*_Qmo zB0lj4ey@KEUpt!LV|)p;J|hz@RK&mIrufc3_fA%|`MU;Tv58WG18dz6^+PeXzkCQ| z`W@EP4|NzW>sxhdm2>W6JaOU6VIl=lDBc81X{L&6-oWqK?c-~!+zbn`J_6MuzYpVZ zoq39}jv_{qoWhdpX^l|kd>qXc4-8OGs%faJ1hml zAgyuo31K7-*QA2qz%9D{@F@AbVt~Yx{aUN58lY21LX_vabakHiGpXq{ysiWOW zy99N>D-iJCaT>7HlE4m5o+~47erIDl{`UOymrs<`0oV|}I;Kce{6FR%er#hgNGRL>Pn5cF zkwad~V{-cYa=JB@ZF4;!1cp|P01+wdT35p4ID?Lf<^T}vq+UWWTC}6ZBFF*kUG-~=Ur``+2 z!&rw9mo?O(JGK7dJ3mTB;Wc08nwl|gICE8G4}tB0@k{{Guc}C?8TLtn9Jp_K*M4Wt5}{6qJ|Bw(WUBr%lDco zPh>6;2$~mpHB9#)zVb4%HV6}~)_{p?jotH{`mS{Zs^&n_%-%3(Nq}xZ0vB&0)xTQ+ z44uHtIAf`dogclZ!;6kk?R5hBx)3|6?;^vff*K;jgFV?hh_95eVex(sYvD?3Zx93u z0ru9K%6xT&0{kr~*>pAg!|T3douEUk?;AEOLz|GtBL+ z7puWV&Sx~u-bMU5hV0j#s82D?Dl0mEwjw4EVfUvW{D~-P6y&fY8{^%w1*;-e!^!pa zYm*8XC6@p8X7Vm9=us$mO5KpMDweBv^n+^R+MeFnKfsls+dt@M>H$C5`7o_#gN|-p zynVxxdI{2ZgKe-1fgC~HAE-t1_Y#u+5A|F-2T~8I>Br3Zn)XI+IFf&bGRJUfp2KSr zYMz$~+TX!BIrqPXiM<1wxXOmrDvrV0yYTWS;Q6=Oa3W6+Bn;7^vb1)(U_FDg|8yl9 z{zAp-10-LvK)s8Z96zh&TW}$4O(JEeElU@ZT2XP_4ndm`8$Wy7^Og?ZTt^3ue zaFtGjEqj7#ShR(n6*l#M93wm}2S(0eoobjEW2j{I+}F5Hy8am@kHVp=gRdRFqY!S; zznazUg%K0+iL_HghE>#9LO^cy=;VRy=AI{{!z)hB$uiGXo0Pw13aF?V+s zW;i%Z3J0zRh2#O^OtbS|jco|@P(i+@;G~7d{%il8NKNk7S&>t>yny9UO9_qu>jz@$ zp|FEAuGPAKL~_ZBSmc?E&+VHqyfdv;=kAWCyt5f#mWsi2>tQ&?x5VbW8b<)G&q_L0 z_NtNV9&oDM8guE!`BqI10n!giM(z{`z7~*@*GeRJyTNS#YdQ$lsPA?LHGSonY?eo> zv?P~{-1V2=ZFYm#B=|!F_l*FslcjgxRR*-IipQBuvhI?LB{~kQ?Bngb#ni359q}l1 zvuypJUcho%yf>+%djIH>3m*~l>OL&3N}GQ?q5@b#05D<#J#W^U8xuK?J2SD^ z4M2%GNizwKf}{im@mhtZNi*eOL1G3ugwZ!D|Lp}3Vw>>)R?pu~I@_Uk`i%qzjc}T~ zq^@Ox6~=H*rm*10)LeUP>gmG0c^8qoSE9sJCP`oPj6^y}jBzn=tHG^52?PXrSO$}# zbK-JMy-E8|lYuJ(5)qvbfRd~exc9{%|E6}mL<$nLtR=#AyY!YcCPt_X4(}56Kg$xs z0t`r}&@;c+s<9!2l@l_14{OeQZsqqDz}=RMs|OCq{?FC_P(Bs%QOt@~{wHJGI~dmC z;8FxiXN_xhWTAc{Tq_hssI)?U)$!EkvaWxIgapzZBg9j#)s&{|m4KJI2C&j4F>I%C z^tapnn?H7YAy`vaoNpFx&YsCP$x~HZ(|-Yd2&BE+_G6ZE!F2OHO&+XhX>dMEa#dN? znQijHd926~sShu$O@f`yJbH6XR7u!Lu%Zq~1bVLkS{(jT^!c(y7$}VvOiW&Zs%%zQ zu{oz#VJ&Q*^(yt0e$X<>#@=Nh6`|mw>pyUD zFwRtZ---Vp9epxbu-9PnsG2)^1&;xrd~HA#dLaeA#4zhMiG(oaJ?;;U*|N_KuJJA7 zxnRlkF;JuVe&Wf$6HAnzbq3vkdoa9|;5Zxdmy6XGK=&I=|3!1?9bFa{`0O6CN#51* zI=fh-uVGd}(ELBvs7I3ImOTNuzKv^{O{JpMgpu6uNp!h7<`iZPQYG~ygX~0dWiZKC zt0}Z;&J2>lp*PobUjIjR9ByrH0sho_ESG++l0W+hKB85wX8}|oFAfI6tGaxF_uC+o zl~rNTpZ!azm%CfFKNq{ZIoOA%n;N%_ja7*QNk2P}Gy;XFXT9`)fHS|&rIsBQ7T0HT zJC4kGz17Pr*3|%|WjgY13Q$+f{;BF-Jv0}N|2)araUv`{GewOmM2)H!aNNKu^ai)s z?8g$G#XrZlP8$nv)$R-8BYmYzUv4`5pF3J#9qN*vpk2N~{a%5#~2@Bt0rZ+_IQ8VB;@NSN9S`Pp=a znT)-1+>P1IPAT)A?~Dy+A?nC#Ffqx&hvv;h;;CHkc?)!5CpUCz9df<;m~L&=<)8k* zy66AnxZehv_W!f*AM>sUVt)H|YGwVF&B^MPo1nE2Gtul3jooE{n1VPBYl*Dc`*ov> z61e=n7s3Fp<;Uh$tnKLgUPwKF+j_X)gIJ9oqRW{EI9d!LRVgj-kJuHjv^wpwUwsW!Gcmlyep{Q}=Rq>6z}X6bTC?}*#uO2#aY4=~ zMtK#wz!iVYIi;d@K}xkxOpu)Zi&#Jq@#uCzX4hM=&-}KGk&+NLP)t%dQLaR>SRE5* zm7+ExC|+G|P_a1Ao)LEHmi+!;Wk}HLgDpjXFhBMxWrpm_7TjIw>Qm|S6B>1VBEhJV zg?}nQK494ka;I~|`MaPGqG#c_lilxuuf`3j0ASRnT9@^KbYpkh2lypbKhCY+KGnJg z;f*F*->Yy+#qN^UKD~nM>1?6A392aW%*1gND}R5W=a*VI@K+y<#1tYbF%lYPD{M2& z#FUzdiEt$qfZT0Q0aK-KfXwpbcmWY{pHlQ(UA2U2WyTv9F|{W)Z$3g}GaGVR=B}f&z@R5KzI~<5|N2GklFi+2ynyL{wdeyDBS(IK z?TLXy{m=f;fA-)1yMJEbrVhcpcqiSj{XLx+jGlr9JMSrADve1%2lwJFWUdtX3UaV{ z#5LW^41$0G6V|O!%rIU4SC!T`rXI9iq@%dj_d1+X(YyAwPYe)?{)hJ=>T?hJ*V9`&Nzpui+R^FMnq{&%dKFKaGw* zz(!;KNLilIm42VX(etWhQY-s4CJc|bv@Jq=$dQ0?aB5gnE%cUIw$3oP*^g^>a2_b#tBK(XkueG__Dssb%|}w4ei9XW{~1Lo_~QK%tRgr?s@l!8~%Vgu9P#x^E2sb|A^d|0*;^u`ZIun48A^J zoMgB=lF^!X{Ji(M?|2Gm-LKfm9nQJ*{Jyb(%cI{ZZxCwLW+Ji%%K7+M?jP}D-`X>( zR&}%Zt#64)k^A{q#S_;srNe4y19sRi7wj%6(P?Mc$x?@-pPK7jB&B{a>HLZJJEp3* zqhOtT>1r_00NP8IKffcwNI+Q5izdliPNvd}G)&&hJ3tt7-(Zx!;l~`wB}uo{nEu%^ zibk?@l=!R97j&f`AWkrS~P+b@1{dAkln?2kR2)Lw3{0byx9({q!!o-W;?O6%ZfL#dN4h!okV<98VJE z!V1YndLsk6@87k`?_$CfuOu(M4~9}8MOcwT(e76&nZr=Twa~uE>gat4dwl@FxfbKg z`^bh>n;Yip+iQR!@H$?BZ(ofr$$WWDX${igVuq?%AyAa26>5(V=iLdE zhLRddK;N#h^HKf2MJrUSGccYwW$uB;6IXi$ns##!hFCJ!QXYsOlQv${gMRAvW=Ukw zQA7pB*LQ_`VAy>mJ=)B8bm!9bjTKj0bHrNXsvq+ENZ|<}W;>WD|A|~N`Y9B>kx3_H z%Uf@k&kmH*Lc9}kE_QOT^Cwn)^{J*2BJf?Nsk*wfFlQ3@S^l za5w!eQHR&ls?wL*P9v|4>%2?y&bLdUVq@&#D{*f#W?Mb?L?Y^m-SN0J|BReM>HK>~ z*ISr2S{oVgevREwq(~dNe(ccCw-=4jJUqo8BHHzslmS z`14k|=T0d4zOqQy8o%k@Q{pHdMGocO z^+wsJKd;y!3e_A4ZIYruC84^;l^Fj(uXOnZ^j+EWWYba*f=6gT9>9Z&Kv3TLyf35 ztZt5rU&PH1$=cqJjMC$*iQ0(B@M)@+>-h9qxXqtIW?5CnL;FFd0DoxDJV(^f+S zvkPL@SBj3)fa{HEz@GtJ^?!`r;lbfXT=m(eA@E>-01bHBmFcm{zo^3_l zQ}P!zm}qloXj|KYT(VA(KTge8s_@w`Au?9Y6q|!-SK2?S)P3q$J{FJ>p#l~H> z8|4jrJ2cDtl8Nz%=iuzII!n+II=+#Hq+6A`Q(7DBsE{~$htb}xMdz-rCBvzVi%?bl z8ZM_U2drdC585`S6)er_diapl^5f(kYS(=9DYoU^WN$3ZsrAzt8GA-H6>;cDzVJt{ z3+;QM?&rEr#<#;G3KJvQz_mJ#u9mIKQK?>l{d1_HP5vt8lhn3?c}<7uhouxANQa1> z1fKq(1hT9PP-^6}UMly3rNkF(jGw=LrQUp*Jp>xQz=7!miRvFMeX!C0Z_X7Q@IezP z9TN@?Jb_68ZuP{Y{O{x$92^-8<^SgbF-h%+~FpHV5$y&aFasxGK2%W8xeuE-HE{`?i2`d zM&Mv~W-ylr1tPP_-@nMgKi%=Hm#q$-?gX2Fp YMBr*qK_qyWf8RWMQlOx@{uTBA02+9lk^lez delta 10993 zcmZvC1yo#1*CjN;CAhm=3Q$IzrlMj*_j(;m=<}Z;3s3iuRT*yz*?+iAXoKR@Zg z0M;Yn(3g!m#&eigKahO!YO#K@m|PAA!0GU6d(s@gS<2yKVsKS6P-P`H;4-y@R}HQc zx}WWwbc<1{{pO;f(t?$eq4BNC1JE>{!!w(wyj7h&EoXW@ZE4oYa9fb_CP$iNC6JshNvN5eG4<C$Gs zSHcK)eSjv2OE*NuV;@pg9J$(pw)vH7Iouz{8lh4D#2$5`fS2ZQMZ_Goxdi2_U z@b&!KCyg_9@UOT1oJ?~!kD|$AdSH0(^?f%JnmyxK(@FZ$N0-L!veiKRpK z>eBWQt?mKG`W0}(bP=4Gn{1OHZ4@U3_t}`(;nQAHvpc?g^3S2|p={H7~1^82gQ^!Ic++o53NG&5_H{6lf<2-F${#v&|pUqiBsmCEw5IRj~%&u{g& zgk3*i4B-H21=RDA>@ z*&RQ)`vyVXj&47@pyQ+U)mL$Zj)Dd)-e+%2p{hSyJnPSwi@)hb>Ttyr)8(|jxctEM zGlPgTF|`J&k%QZ_P0xhycs@;@rvx!2)%Mvpy*$lW^E`Osk-p4u7Q8^FZ=$|AF==&X zjNY3;FN_HqOVRBCXxTV?amti%aCc=;Vk{1@3<@f(c4U}tMRuk4`X^QQ9!FfemUi#& zzMJ(>IGxf|oiOs9Z&L?Xk{0hhM|hp&Bdx!Fz-ckDo}t}l)2+6;ehQjFPi7!s(WT!( zOqfYl^5VUJa`<}iL)Mj9hY%Z$q$ojiZ^Xm8p+GoUfYSk)g#nt9?!Lw+ zu<)l7BsDetP}V*Lut8TU2ziM(o=_nljmCn?nX;WtxbW9)x5N-2L zH(Gd*{4~WJqFB+_r1l78rgYz^8%Y#&uo~A#4({&_I`P4GM9=1=s&<_gl9_i;!eXVN zlBYnzMtgU`*QLGhZ;@4p+%F}#ut$tpQ4QMNotbw;`y+ zuXah*jU`hF5nK{^IWCkRv^d_H!=`}v5up9d(>U`2qXAi+&F*0x z9w>t~bk5dbC*$&dDZfPISl}u-&6(k?nb}4N>nr<7>K=g)ho>e9A%P;;`F!DUMs)7> zuitZ`_0hc55fycHkHklwFFt!`B}C?BsM+=Z9AXp z(@pSYun>QJ@OtL$3pyO!3-|@5&o3{zCaWo{LJdy@@{(s=)j9L)7NUjnnu09vzQ!Ls z?`K-&Wqnxxh-X!IclVkjl9827k#|V_Oa)&72Q?o8SF)5B{CKhb>*&wHotxRq{V>?< z>3+YqE%5T_W#i=$`go#K-yL|lYInEvOqKJcwGMpqB-quH^QC_s9W4JIbePe4S7#RN z|M;+1r}=|e8UN0g6On|r;kHNF_v1lan%S33&WnB543p}RxIx@NUplHytS?tL-;$ow zZ}0YO!9SmmW@Xr2+^lb!$u1GE6WWw3)G3sIligqWXw~{eetED(UP_8Qy;(B*(#4ry z*By2;j5{bG7UR3Q`3&}9H>1!4XCOCpno+cz$7G7eTHGcIxXX7xAIu#{KHvNv&U*>? zNwA%xiQgERaxOs;u~!3mpIbHRuw$U;^joEPM2<)}nbyc;S0(n%7qJxDK$CBHETCyw zR?T8K#Kme1`-i@U5W=R21yc}HLmTk30-l9O-m75ZUFcgYfk_?qarIo1AI8;5zWiL39JPwfht?uDd zpsFwHN;ln zBwFs_>`xxm^TGLG9+gB_JK1er)Ox4W#OCJO`lf8>sTug@(XJC;BP%6N*_m*3FhdJ8 z%1g_evTI!*mIT%;5DMHYPL8W38o2g?Bu8M;C#QzKx~2`4i0s7z$E<~-GHPBNWq%k$ zDd6(Q%B3wbje6 zeWAsMSia=v2elHfKuRR5?a6(1wQ*6g8XD+y>H*5*yE;n7)#k^-NAV=39~vzu{NQ!^ z=dX7Umud9(X{paaYH|7N%Huqh$@3*Fq+=!<;joKj$YKO&su@ zDh!%flg*&QjN$yvv2%pggpabI%&&Je31UBj;MSBKL_9G=b5rC)r=+Kt4+T_?=^D}B zQh~9uvfhDO^euD^cdwCBA{%Wj1Ol|3>-z@gXH2Qs*~%l>6xo*gl5zHEwNuFxNiRVv z2?+yIRvSWVptc?Nsco7bs39Dx6HERjFH7Wj84in(wH%u839jiMICWVyH*POsX2DQYjg=Fz5p_iaC)2BPJt z9}cWoxZs}yt`J9Zmh+-TkC)Tf#WgGPdj^D`=z!}z47m})Yrhv~ZwNhKwQN<`(i>Q6 ze5r~e8J4un6^JN|{b1Z)?~$PGAQwe!2~t7x{hIQrUnYcUiGnNG0q94^F8G5<9vpR; zY4a)lSI9vVq!F=JTb!QA<`v~~M2AE%0m<9iKuZLLtJcFLD6U3kXC zh+DBHI~=_l@*Sb?CJhqP(6JtwoxveBk#EVERN^w;Hmq&27Qc(k%oHoY$AjuDU!jSH z8eDxxC(00TXOf{V8;THW6I=X&*zf*}&qXvsBRA;}J%zUcDu?WY;w^mh` zWTKnGDPwGb{0ie1;zVObuSUgv@unfXYGCl~z{uNp)%2A$`q^>XwTs=7NAPvlva|3j z5YM-@vuhiNe0rAD?z&^9JI>fIl7%T z__;st>IadaYO-$IE4khaLSWZ73z=RbBGqygMl`#zH$Mzz``T#8Z@O!x)2j~zSU|(< zYBcZ2$%D!nau|c4U%-IbOAyHRC2Ls6zmmf~%H2rg@?~-DH0*4-~0dcyKwckI` zb-Rat%VHur9h6$_#?=%sv(S4hTPS2%GhYJ{A{TnK5siv5gt=WlJPNKlfg!68l#aqu zZftpKg1H2S9AP1*MkSI+3gIFrFxmp_79Rr7J5DMSxKf8>j!vQIpH!GxX^uo%bd#Cm z^_|ubqGe+T?mdOuMqDC{8q;FkZAbD2Y=@O$6S{t38GEGLIyhr&Ef+6}TQKObk;@%= zUA_#!n}8O!9)j&z7Be?B_)7~CvHG^=6zT8ibEP)Q5R4>)oltsbOJt}@~^7`)^DH-*Kcqv6y`bE~+j#lQd{ zqK)UIV()UU&AC82V}*t({UjC>!B^QFt$JOw4o9emTq7g_kXGeq9B-~fqsu64M`KaN zgOb`JgX3=3*U zE*J(U$DD!ZeZ3nd6Rb$#MuWa@x5$c%j8u`PMznU4dhFA|6t(-rNIu2s;QcWor1s%D z=Jyy~uG}YgF{^=julo5*_Jp0IWbOp6IoV0VI%?=T@q!j8z=YX!2b z78QB#PBc=Z+A>t!^OV9pT!YwM! zL-%@5!FA?TyLaRhfzw_=m4Jb*i00@goxMt|Rg3HRNQl=$fFiw-qH$z=;4j8a{OB}Q zDZv=NtF2%r&UN&0@tQYk-w8b`Kc)<8b~o|`m&x1i7GN2lt2AWI8C{F&EP!<>VG(d1 z>y__9!RdzjXE%ST&X`Q3_>RZ%iFt@#CSFybK@eB}Gw?A7B1g9NbGIZgw4zAh6-ZhK52(A5rnW5< zE8cS--ZnMwa1ne3nr+OaU`@}?WwQbWSd+85ud37!VzsUlB!Ce{4eEW$H5d=Dpc+Kv z0*gmS>@-$oJlP|3@9>g~+>#YaY>Pk8$dYe*2MNZ+kO0-_zV;>d2R*3FzhLb)MRVJQ z)TKY<7UWa1q)s_B6<~Rsb0OR;B+wx~usiIHpY7uK}p>Cup#aQGt$xF@9QmOeJnp8{U>M3H{b@X2G+Un=!13?q$QSku)Ap zIaaoolgeAhjVN4I=`vPPCBRaandVhI57J4>XHs~2P`LmR^T#JcP#qvxWjR?pw=$^DDg|BQo-k)#2m6J=On ztgde+oIC*%*r8D}Y0E|h3!Xlj~lakArhEmcK8glKbB4x-BnJ8IOkhiv~Bu z1ck`A*PL-1mi+RMIHlN>vws&yoS#qJZ>@Wec(ZWpQS!m<#?EB4WgG%y zh6wT00n1{gltBry3Wtfi27*Zxn!`L~IG;4aN|fqrhn)Fs?2m-#roZZpJ}%%M-J_a= z`<-P*re9Ou(}i3$Dc-V2r`*!yOGr^g_@#cjWexE>%S@mOlZiVv!sBg28)V8l?Le^^x0(~t;^j~ZJuqIy(gX@W0di3qrcTb&J5$;+tcbXeUYxckoG{_k-Y|Xs!lIA4s^{Y_e)6Yfz|oh11ri+E-3W(ruW_GxFL! z{kXuwrNK38yP2%ZyP3jHb7&^bqufw7MZ<0oRro%EKL#UVUt|Q9r;4EoonAqi<8O-p z4)(!ousT$j@z9ikFsDQS1~;&k?2+3;u2pXnDqM;+1T*&eFtrneON)X68&y9j3AGrQ zpv%vO#A(=!9KU&8F?L}avroDW&SzTtmwo2~NqF6&UD5+m3-Z&$Kr|kxe%J6T$;c`^ z1ZPoNf3T;pYgXjVxxaaNVIwrtMikq+)zDhJ$4)#;GSwxR!~RMM+{Z8mPiYU_zK_FU@D09SatiLz+*>DlIAlI@8Q#Yj888 zdjPW{C#sxYl@bcPcKEv^A<7Kq)ZHotI*VsH3bXSA1&5lh6&R7nIsd|#N%Jvg#Eq&Hi5)G5v`0<;k^iu2TeTxH{0@btV%K;2A6nq@t^^3OAPX@U?tKQ& zx2zGb&eCO(bCqc~d3xUP3QO@ZrojL;h8WJHB-+h6k#D_q(%j`R4NzjGoRwQ9x0?9@ zL>6Y`gFP(Rp^wUnw45vZ9L*9_QrwI z>x^abYF1)um^G%bCHz^Up@I(>`!Pl6N0e!#m09s3%CH*yS_H4bs~1gG(+>5#4;q8KwamUO>GZNFOQ@;WY+Js0`n*5C5l`{6(ip(Km^eU zc-D&V(B-4T<1{W*v9^~tbcAk61L<)Kooo^M+t-kGK2HPkpS zyuYL-IK?X3wxmmHF`V|!dC!v#J#BllS4-CP<{dDP|3qNan(t;Aw;tvp zYIxRiU!-8;J9O}sq0q|E*HTtx$;EthsXv1`5t&(G-3pV+|5?4x=H6STOlbP@;Lopw zi)c9Pqu9`7=K-!368xDTbnIk}fQKD3pS*gzI#&!rm4ud&6Wy*+5vgT~5W7yw?28b~$#@mz z1A)#fSZ(gtHx%2;zIqI?LJ+;}<*Y|Jp?N^G(cd!&VX;?!n&=m|$zv3)%Il*V8ctH} zP(%?d3G>w0`voc(ng~{Gi{-Sp)zx8GC-L(3zLL-)sM+0OIz*B7-L5iIZ66{h3V0u- zAfE6yi_-s$i7IwrOl}ei$ZSWGJY{ zko(AL#@oKs7x^>j4NI|1>g}H`r^>5OyXbu+z-hPkf8W{7ZgLlWwM5ODxqLWs4NEMa z^X!{lKY%nD9P6_=1UNzYCy?B}UcDCj)In{ywJ1zvH?mIKDBg+|)-~<~9ADpu_XC<) z+;BrH=>AIA->roM=QxW+1!_A#W3l|FRnjl}@9JIc5XBT^i4Ixw2wc+g#T0(!NO1Up zIJJa2GFZr0zv0K8+my1dn3lTRYH!KWbFO=U+YTg88)w^<_v)LtNfMziztem-Tz26;Fn%KK$syK^K|9#lv|L}ZqGb^ z*v0i_TY9^EdG@I|sc`)|0md103THUS0;5Hlao4HgaV4M;D-EpVB1!VGzl3JSKloqw zO1lraVJBEvPGDl5W^&L^4_&OGz_op>{Iu{ZT?;u_M6;4D#>!*oe^_!giVPqa zR4eu6!qFY2+cqy4-CYf{AYX}2oTmn9FnymX0RODZgm52CmW=<+dZwJn_o`4=eY9xH z5t%pW(t~w$h`@{Q+4#7L#wQT6|1!DI6D~s{Mre)--2r2)iR+sHFz{(VyH)?CzvSA( z<6Y42KVSr(yB|p}@1GBTzC2IA|9!QcR|^T;x-9dTrodwUX^Q66G(9S9hc0e-I4YeM z>4Di0(~z4CG^0@aNfF3|@$)?M4>=`uKd?TDD$Qw*F_+to!Nkv1s+IEHDy!6O6{+ zPIf1G)gcMl(B_OWsqr27-Oi3C?%c1ewS2Z$kf9hkSa`C#{vEqorS zop{#~GDIyH-h_07F-vAWg7YV|_c4X$(jnRnRS&5G6U05TK%F4visXPfOHwR20-<}5 z39Enx=LZsrgL7$f%x4xHw6dt-OxOYt&Re_InHeh zjWK75cWohy)E}h+!ZDEU7ik<~-`h#t_#)kJP)ic;%7;H$L-H`M;-n}k*{>5Kovp-k zohuq6HFFaC2W1@RF_dbur78?~nVX%EN}?w6=Nm{;!Px@gnya=Nj%%>i9H|O_-jUWi zN;oQKUdd&jU=7TgFSrk^W=BegwPNSADZV8ic1=)|4GV{O{(tUR>Gi4q`It(zx*r(+ zWDcRiyn;#v3yg3gd6=NO9d47MQvkghrdOmZRCV-t^s1fB9d*GAX)5N9`mkrkd{ZS` zmS%+qtgS@zS7oiWQfDL8NK501l0p8t(xGBd30nd0ON&48Cl zs_V_5#mZlEtW>J)X#9^yU;lda^Ax$^?6gCZu)JedLh{|=$B32MYwXyAmW_81kK}7+ zg~j$)qaqnnj^V}_AbtfaNRE_YxEIn9CJ*thQM8yZ$1n0O-jO?``(A2E(p{$TCufKm z#+5nQ{x+Ert0KQHV+9oG-J;<*_tkSnGc8iHasT7DbS2CjUk)QyewU4Q=Zg6@|APit zO#U_m*wL&`zOF2=ItaKwYJE*5k+%U3ez|>tZ;WDBKmmswNJ*+9!olg1z`Xg59D^geV~kEKwII06$ROhsLCz&Vw_Q3x|<~@ zGf%J%!+{hbBI?A2!uk*6AkpXNFBX%ns9hbsnQT3@-{g-m4^BdB54JJY!kS z$7VzNAvM?UC)~03EZ=4SQ-4AVQ2`2L%wFI#I`l{oIsuE1M7FFIar;_RGAfI=x7JyU z*!3CVyu+)hwFO2jN*gS07(uri5-g}B;0cFu&d@rwoy>^e62|Ix*VN8J@CUzyXp98M z#a9GRRn*caFMv4vTEqR-3V%`$loaCPljb*El~}sJ#`O#(mavxS*zy6y|2+3Eab`RD zflZ&hJS>b!hjJvF7ON_k-rr>|KdJ;jRpmcTEOG8Nnr_rD3U}zT&$BrU^=H~9fHgQ! z7#N58pP(QzdTJcpT^dzpEg>PQ5qi4@>OqD4?94SI%7R8pmCKkrxZe1@H_@Bvv z8KPg$C6q%;jTmJdf3@2*nCkil@^oI9MtjLOpty0hbw1{P|H8R&^7#FI&Ya36y6NXP zS(*=*`MxJ5=-=eAHq=wBGbo*L6v15Fl+uiQnKlv_smFf;zC;O)Q6FVoQXeI1)aieg zEe(1I%Vng;W!7!rS_u!ehyf{H>9agF zmHagFD-MwH(oHpG1A7*qDK@+jP`>gfS(Jki&n{JC20rQO*JAcCnNA55_Gz$(Fh3=u z(0)F1ri-c$#R9!xF+=9sf*+2$-1o!hK?O=ohpO0SQVRNhp$6-N( zf0%0qsA2@5azV$yneI7jGIu- zIlx9Aciepa5d>%C4a8j-C?WzEL=zsltd-yc;UVR;l$@arF-ftrz!%qM-0G!KpA?8r zx@GbKc77;s2|24^YO;F1>buns?wYcplZwnkG}Ai`Z&wD?ako)E1RAk|8_k0f1aWH> zUfY3k?h5ZWnkoWz5(M7iF(_bl;5 zuY`nc6f2T!Vc7J*hhk&}X%reeP^;}F-zkhxY{%3fsGw-)_8#RtcBsB8T^c-vyOV-w$yrMkWuvp?(63&G12E}lfC%2zay$? z>vYDd0VW#qs@Iy6>?fCCtt!mjCZ4TP*{Q0Jw}BUisH9J~WNOpWd(kJm$VKPH+KV=S z!X2d2mozk5AHqA&&GmxVe%M+U2V>uuXdWQpcpxg~k8Y;TMH4Z9fy&PVoHeiyxt_CB z$iJCgle@GWu4MBY5P!#C@tpWL#n{;{n0MVdYUf|4bH@?LjNfPhX8XP5oFH2#*$Al; zZj3202&yVmRhe6w##kxk{Ygf?iYn9O0*Dx>JZ`+-&T)r|dh=IB zyG%5ec}}$0&fTxM+--?i+_rn8K1Z;*``f+lL~cOx2N3O5_Up=}00sD?=|RM!aDC}9c@56w29gwmM+ z8}|%6;GtkoL@1#tb7Qzs82qbo%?H^=88aq$aIfns7e?pnWl3U@spQipMJ9TVX-x5& zp3l}N(>aXwwYo;Q^rf{g!40`WqZ-$YDM{){NyaAKEvmNw345o-;!N(8&}3Z)t-8^v zGnZi5hLAqy8GQt&SsCi3h2ht1M{P}%U4bV(MNtr0M!YsJr>lrVp%N(xDYia#S`~H) zu;C_Jd_i3W-j%`p8|5LwCPaLt{Cot9b4Iy_GHp zK{Cy~S1XRECblgxNfA{lBF@S&kmWMO#!iKKE8} z&zvK6P5EUJv2$jurg*e$4ep`tgyt(9uwXl$JihD7E%a(f+BtI8*m89%YjP`A`}ySo z8G31q-$-uhgNgdT>o%b|ZffwKp~r4O@_);v;NXa1g8J`8j0GjfB895DvjVc%q3S>a zXrend+5aoR|Mq8uw!4b}J_$o_+}{A| /// Line of business dimension for insurance classification. /// +[Display(GroupName = "Reference Data")] public record LineOfBusiness { /// @@ -27,6 +28,7 @@ public record LineOfBusiness /// /// Country dimension for geographic classification. /// +[Display(GroupName = "Reference Data")] public record Country { /// @@ -54,6 +56,7 @@ public record Country /// /// Legal entity dimension for organizational structure. /// +[Display(GroupName = "Reference Data")] public record LegalEntity { /// @@ -81,6 +84,7 @@ public record LegalEntity /// /// Currency dimension for monetary values. /// +[Display(GroupName = "Reference Data")] public record Currency { /// diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index 0a586ef98..cddab8717 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -5,6 +5,7 @@ using MeshWeaver.Import.Configuration; using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; +using MeshWeaver.Layout.Domain; using MeshWeaver.Messaging; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -121,6 +122,7 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) LayoutAreas.ReinsuranceAcceptanceLayoutArea.Structure) .WithView(nameof(LayoutAreas.ImportConfigsLayoutArea.ImportConfigs), LayoutAreas.ImportConfigsLayoutArea.ImportConfigs) + .AddDomainViews() ) .AddImport() .WithHandler(HandleGeocodingRequest); diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs index 285cb23f1..7223810f7 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PricingCatalogLayoutArea.cs @@ -28,14 +28,14 @@ private static string RenderPricingTable(IReadOnlyCollection pricings) var lines = new List { - "# Insurance Pricing Catalog", + "# Insurance Pricing Catalog | [Data Model](/pricing/default/DataModel)", "", "| Insured | Line of Business | Country | Legal Entity | Inception | Expiration | Status |", "|---------|------------------|---------|--------------|-----------|------------|--------|" }; lines.AddRange(pricings - .OrderByDescending(p => p.InceptionDate) + .OrderByDescending(p => p.InceptionDate ?? DateTime.MaxValue) .Select(p => { var link = $"[{p.InsuredName}](/pricing/{p.Id}/Overview)"; diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs index 0cb52c433..bb126a234 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/Pricing.cs @@ -6,6 +6,7 @@ namespace MeshWeaver.Insurance.Domain; /// /// Represents an insurance pricing entity with dimension-based classification. /// +[Display(GroupName = "Structure")] public record Pricing { /// diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs index 2c6f04867..d47e9884d 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/PropertyRisk.cs @@ -8,6 +8,7 @@ namespace MeshWeaver.Insurance.Domain; /// Represents a property risk within an insurance pricing. /// Contains location details, values, and dimensions for property insurance underwriting. /// +[Display(GroupName = "Risk")] public record PropertyRisk { /// diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs index f9a6faf0d..7300dabf2 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceAcceptance.cs @@ -5,6 +5,7 @@ namespace MeshWeaver.Insurance.Domain; /// /// Represents the reinsurance acceptance with financial terms and coverage sections. /// +[Display(GroupName = "Structure")] public record ReinsuranceAcceptance { /// diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs index 66969f37b..42c21572c 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/ReinsuranceSection.cs @@ -5,6 +5,7 @@ namespace MeshWeaver.Insurance.Domain; /// /// Represents a reinsurance coverage section with layer structure and financial terms. /// +[Display(GroupName = "Structure")] public record ReinsuranceSection { /// From 6e3a34360e7b67bbfbf97a2a1e31822a5609409c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Wed, 29 Oct 2025 19:44:28 +0100 Subject: [PATCH 09/57] iterating on CollectionPlugin --- Directory.Packages.props | 1 + .../Files/Microsoft/2026/Microsoft.xlsx | Bin 30012 -> 30003 bytes src/MeshWeaver.AI/MeshWeaver.AI.csproj | 2 + src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 115 +++++++++++++++++- .../ImportRegistryExtensions.cs | 1 - 5 files changed, 116 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index c4e7ecd92..2ab4e7b80 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -143,6 +143,7 @@ + diff --git a/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx b/modules/Insurance/Files/Microsoft/2026/Microsoft.xlsx index ff3f2167e8501fc5c206e7950e0644dea3b363b7..ab1b522feb07c57b99f4b922cd3e106d371f3dae 100644 GIT binary patch delta 14272 zcma*Ob8ux{(>@&AwkF2Jwmq@UiEW+OKEcGcZQHi3iIa&lU+(*Tp8CD-f8VJ(yH?fS zyI1#WUEO=F&U4U)bI@86XzCnGnB3Mcmkk+bSU}6t?Q>8Z@6;GsM!>1=~G`Ujfsu?(ZSEmIQ zA*}GpQB#cxT?yCeCURm{mUV`j&z3*Af{y{tkCvxbws5jL(aQchuq zam+Jo!JPL~A)$%wj$JEHv~2_#`u667-Yi&s0uINb> z8kox#cmqj8e|n~|;gUwiP zm*1y8A<4|cE_&s(_7#r{DM&xx^tTCRp z0WQy|D!~NP022#>!a`4<>95OS3OkQ><{*EfQOe49{RNzwcC5lSM0tJ7PKyjd@y(tN z7u%h|;Gt%B*|B=RrWk6O3SMpUBPS97oz$<77hA_Qj_tYNl3iYS?pZUvr?lBH44iKp zb>i_%dQAAEjQk6%kA5gLN~MyKxDVfk;J+0H>OZ0lf5v-FUBIEVaL!Od&7~ggPvIHESLU;L|-G z`R;?H9#jg$H1gC+Np^0Wx#e=hRFgyriz+SA{r;>FEa1m=KOOJQ!F_>?eLk%zi1+*y zI(#WI*`nb8F+m1-^*PW58Hj2e6+A3Evq3 zn$Xd@CZzDmWpv6%;ZS$e%C)tS2hF9OM=y=Q$r+_EVkB0 zvAmB=xuFCcp;MB_iL}Pyo6wZG_GqWED6m{uR~dojIZZ{)}C_(lrXgC4jbaLHMQ1xP+XUT2(i|3}JEmf1O^q9hngQy|4>-;CGUr98-~jE70BIaXyaK*1YlL={k5wmWp@R`Oze>} zthG59=Ogy!N5dZwQ+jh@a1~DO-jGME(xI{v%owm+rpvdSr}Ds%3FVJ{K0u;GC)fdsg}ZV|*E zz}GA45nITePfWSV2;jdH_QJLbMf|Z@!0wH44Dp9)Afy&T3LIWC3Jfz)99BNeBz#Q$ zZE=iBic?W>imUAy`29w<&)uD0po(l2H3?|At}q@Er55s^E6Ja?l+)99Rh0Oy0m{`j z8k?#zjOnX^4a$1dEOxZPqJlh+Z);6}+3D(Tv&qH3kRTv`|APE;$<*6O+~lZ##~k6i z?5+&`-64`5e(R#JvnoG-_aVQ>%HuY8^%`8b^!Z6DRU0u$pMtZzP|BhN5<#XY<^P*A z#}DB7(&_v6uF}JKHden7DI=gfznzmH* z2*1H2zTYU;+;P?j1WQz1-w(ZZzl1%s7g)=h8(2$|7|OC(Nsn0Q+#MMu%&2Rrd>D%( zo@%c9zpUV)z^wWoYwtC?M+0ouwjs%BcxL>XSi`t9$V9gIK`B_6?$_gA&e6+dRr@ENv+`~Q7C64 z!ntm3=T^>5^yS*bJF!vYil2?`E4F4aAc>0^Ii4aJ&?9Z_R}AHO>jJb1+Q*WI_Miv% zXKg?2Xue6aiuJY*^Al8yMZY8@D&)e%f3~| ztd$e$WmpX`4TGA&>fbuh@)3o+nFn9;}>E(HxJ<*?l!HaF}YYPxGB`l zJj?w1RfNF&JYWV6{Q!^c+K&UKfQ~R_CKGn_okA{nzVHT@;Uq5EKKc?WV|ekixy?Q~ zxzyKazcZS3ciU)3bGHvr>W1nG3qICPk^_invyBoAquy|S|0FQ#w53FN<#rq~MvdIr zE!tb-n6T=8w`oL~EJcAv?)mF$!S45&Ely$sfzGaqm6{7RR=}3;>%ryLhaR@i#mdqv zmJjYr@al$KbXRZCsx?B3jlA(cf}2+`U>sSpLpsOu-|&8O*=f4v7)}!>)~ zZ*@@e0E))``r|{rlFT~IiD&iw!Lju@i&~0x1%&2dmh3b$n)E3TEs&eQYTqzw`SzK^ zbPXs_qQEJcbR?;<7!q!>A)LD(qo;xCBdivYn13L>^6(uU=l)%iHjAwMf&g&^q+v|i zzyil)Y7gdWyeM>yFN$18Gs+&q_rH;p(gEfCGDxW-;mV)?aYY;xtCcmG0aG{C9fGY%lYWcm3?G7BsX{_)g4Le# zAOJV76x)FJiW$pchN?V1LN=a7kPiy9%XgK7^+zT6{U0 zNISZcWMiAG;$_Cd)b-U4Pm2W5vf%Qx$ag;0@A#d@7miUZTS`g(Y_6&B0X_Ip)M6V& z`%;*%vB;*E~<2A1DF5&Nu5EXB?V2%+DwVGNM}&l?GBLKY=rYL*u(cuCE%uU;#EP zvxDI^eO}lk){t(@j;$ylq_k~Dy(Du^NXKhT3kj}=HKbW=RzJuL$BkOt3ykex(ii?F zP>tDTDnO!|`y!4Z=FE$8GCm62$$S~dsqtMgmvkt}VP=#WIy&&Zl0Jc$o3F6V;w&Gt zX*S6~W%`uXpBRI01(LQ0+j4BI_DB1u@KK4<8SHE{a<(m~{=6?>BhNa5ZB=M@=u7~E zD3pyg5@|Goiqm3b{%#-vN-eD%$7_!ian6-DY$2OMc@lK0s=#F~@+vr4e39lS{p9lY zU|P`lwRl?Td)_Hiwi4VQ2#eu~F@|1Qf#;xVigBT%lgGjCuELD*9G$02|IA5cp4b%P zSeE$djslLxLor6c-VF&OKU|@v)hHwl%P&tjzq6F8(;u3$_dG}MMO}pihrW#CV%x~# zVblj9RJpi6Uv>Ob=?C(yBUo0scQ@^J^yg8`j`h);mLm^{#!}-WS1K42r1HotwHv1QnsBfO9o}drx2(^-D#RftQbS*!ayTfy0hHt0`uva z5wXCe3b>7NHaRt2rX(ZKXgx7hgnJN4aQ@eR-%J()guVl0G&P8(?N1kpWU-V*1i2xRwi5gl z;cXIY`?u|^cKnIn;_;8R(T~3ULA=oq*_5|DrUG6e_T^djoc#!fOB28pp9Vl^ev^;Y zoM&J2ThTVbqJw=O9vfU`g=f?>7kCsw1iQwq`-FPqmz(Z0ipTW!PpsF=o7M|*gn3vW6B%)#5ZSwefeYsv?K*m)U=8s9mg%)r`?&B2h zW(lwx#Pe?XQf;N_2>oP6!zUF)XB*8@gDXN)y56_*$S?%ZJ)?D|BSPTrVH~VyyDZ_u`i$ zyuXRA+m!F`pfS%Y@A*vBnt-Z+qV)s11C`vs%UFX?#9CvAWQ`n|_QV=~&P%~6G`e_7o zO5NuU+K_19&HtPPisf&l2I$u^G-4fdYaY79RYBl{pJWje4f-2*`$n*j;XonP+m-jr zd_T_~e)zZ3s3U+2J_a4$s$f6hc_J<)pqim?a^woT&Nc9#M=z*ujVHk;aPVDh(x;boXD|jc`HIXEfD4{luJ$@>EBqIo1*{l#)4y`?Op8N*$i+%wH-? zd)tb7QSAcsQ1n}1(TgVCT0mJe@BJn=)Th4!Nv8g`lKC-`@`tvPCYdyhm1P2a2wS-w z(Jq~7iw91OHzCVhW~J|c(8ZLT>z4_X*KSj(9jicoriAA~f(g3LsO%!6AB6(HjBBk8 zTs6_xX==>R#!`@LE$_(p$BBjK?rc9l$bl??=I2BID_#B;2SixP%4o$!x62&dTudFm zQc2RBD|vEonWKQ(j(r47LWr7r_m-0@>6&E3VMRUW4V(K)&lS) zGZFlKos|%KqhW*Lz?!ek*`;C}b1p&rw>(H~-zOq$WNoQBH;RAcuDZt6)rHo;X3%RC zS>z#6E=h8{g%{@~Xe_pRb)I_&G7G22q6V4b$eb}T5&(h!;;@sZx90A*iR{~GeYcXk zP^=gh++!7VEPWx)!TMTNL9RNWxoct}uVnr|cB-9oUa2zv12Tmu$sT@IQV^>@5k=%2`!1>O7bU zmH|U1b9e6!~=@+eqGH*4zaF5T|zjTq`Q;%mbh7MMo&1m7kE}+sZb4Q^o zMI(!6B^bj35FwQ-6}w16%)bB6vc9KFWI27NV==2W=1gdP7~%%X&A(zyT~kzr^(QWy z7$LEUQB+CJZF@Z<-&)yK$b1us9jy}(^ZoK8i{D>H4O56MPo==TvBzMWe9< zYCTb7-^gXS^NAzuSVjCTC45}0JjiX+;ZF{gP_-r>=-ocso1~ z+=+aw#QA=h3au=WL&2PT2{;e$Vo1d8@}D|D?9!6O^1~ug0hyTGB&tlrS2lPQF2xAq zfSh=i@9P5ZoQPaXz)rrBc{q|XJX-$;3|XpfdXON&*i@Rz)WtqQ(Lss-GJ+IPh#|En zw4eZV~!IblT!%Lp(Z;#p#vH7^7>NH78^ttW5t|&}!{0Cy9gT6%{UjdCW^} zCu%`WD6P+IbYK7bXmo%t?NpjTFAH~~{q%#0&K4r+;mzTfcjADX?YtwE=tu^Xwvit4 zw6_0)Y4eXu^i{^6Ls2M+aCB7E-a<;+j`}ngr&zFA@eqUYA zKr*67MIKF96;b*W@Yr1yk)b2M-Gsr|wkcG%3k3Ti-L)|a__tOgwJNWmF=J~!T_qS^ zR^9R=ZTGDuuN&g#@AVw!=1)E@Br=R)P`rA984_q_A5M9!TLM=mZxmv-Cv-5)v$H3d z`KT+~fvj#dV+}Nk#COYZrekgNUG^k2c9s7YSEw$t} z_1_RmBU6}J$8?ZmJP2x29^He2E2wD~KrXldWid^RwgHy8bw2_QvRq`M?VK$eiag%7 zR2SyK{=v*-V&owf$t;o;FT68lg*=>{U<~lxEfI>8i@WOcZm~JbPmWrq;o;~Ao8Y&+PWcK`PIElsBUBQMwZ-L+@Jbg!L$LJmi$_15^BnHeRDU-R_5$!U zPj%`&6-yZMc2R1IbFixzNDCVcGWL1{y_^mj2hz?hX98|3Pc6@|T}aULHJV)})W2(& zoVLR3`2^cBt7Y3q{BkU!ZTk_Dn>S&<59K}};?KTJ*o&vRtK)~w$-yoWH};Fgq**@s0uL6~KMOT$k};+BdY%4H z@n3@=de@yvJJTYDKp%g?s@^OTKzem}Vh=j4*|h^&nGn(q;P#HQjY%-;Qg;@LTsn{} zY%IW=3K^{RQ--3c;*nM%Wg)NK40^Cz`>Or1t5jU*Yc23W53I`ZdN9}pG!w_7)foJp zU;1=tr^>N<@iZ)#UZR0dtLm#yecOB_6xMMABuC*%f8ccqG^j`Mb2zzG_@9hk#sVPq z$!qx@BCpB-|5dslRR36Aib~USO87T*+HiZ145}L-XjGAhKdbt=JBYPmp8bz)6O}j6 z2^^`f-p4xlZvst{Li}iSZsru9pM{O~-A=lmULTMX&R19eAk}}AR2qF0vZRXZN5kS% zy9pKs9YZDL{9>!)&$?Z>23lzZN!$4jXtlK|cLxEZIR=F()PyF9HzM-S^{NC%kxIB7 zPu}0E{|_kx9*{aYwP%M}hu=TRzF?k48P;@9^9RQBfCoj74ZOo}haj)NnRl3v?0IZ{ zbCWYp4dpPA{WXad{oz-C#%~i2I=?|jTo@_Z^UwZ&qsemKDi5G2dZJ*u=7B+FYYUNn zZcB8|XmH=)6of6+pc%52TS1ogYgsjOJ7y@Ha~%A-yRJ6uhonq#iMYDJ#y7dByxcCR zQS>yaPW1kpmi=13;MmY|pqm-9uR@VKXS0}>K`~gX&k512uHOdM?(sw{fliry3`6|( zNn+Olpv?1rQN0xK{u7AWv`xxKI`k@h-SAJ?dwEp-FT`Q*vHofi)UBp3tbAu`R({-y zVNlvhErY7AgM@2)k7mHIL$ipjYHS?FdQE{KQggho^(oznU>E?8(-C3@{2lrSj=XP% z|F?#4OQS^G{;DDfTRH~-Y!j;f?j{u~fj|w+FX4Y+u)~Wvy&P#%XWptmB7KFu8wNqW z4gCe%ZdQQawLd```k8;{?S_bIgzJxqFT!k`hxJkarL4X2=qiINSQ3&(kDybb8u%nq zK_Nm!On;CEY4N>B?MqEmm~J4l776y#2PP^eiFQN)XCXMrls$ha22(b8UDWbvJ4{G! ziyfqXp6}H`{}Y?nKbHB`^}pa4-*yJFMi~S#*u@j2di{%FI=gy{DqrXOZBNzZRfgc- z` z;HA7!wkqANi>AkQy&?Tim*tiV$Kc9H zDzg1G-m~kH@~6eY-WGCd&!tAqQ^i|PkLTmw-c)MXv0{QJp5fwk$+F7A4I}c@K|3>1 z>p@a|!^^lLvH~#DF6HuKv~fE}*`<3OfFl_kMI)Z^W=i1EDra>w?gjnCQ&yTxm!cwb zysY)~9V=Zi-a)$EwV|<_OjooBnKbcy>B8LS;+v&!&VsD!GgDE&&pp!+m3dF3*KNG@ z{}Jy0$@W+rdF#+MQ?V!VG_^x|qQIENAi0-0=!6B3B~6LatTv_@QjRG4!QjzXXu8eQ zssg?cWjf(FlmgV8Hl`VLLY+{fFVdu~TP%ZQOjK?1NQkST#-l)|pRiiVW`!qz826IV zgo%~uQ=n5<%PuhcMf+!NetF4PM2%ju?w?uy_as;o?lDCni|%&D=iZ#mk6RhPME>rBJeO|0mwvp%5XQuzO-LGeCTWU!>AR*pD4o-v6%3}B z(6c-IFy@XbLYWY!*QiS}29ymdM;&zxv+&9IEbT~s&T5oVanc9 zI|dQ;sQt%+%C1#Bk_v}|uzq)75>e2sy3_N?^q+TN5m6XDX2HiL*1leaNJ?v7KLj@B zWJuD`gvX&(nEk^BtM~t5BUwOZk{iAR1^YjI{9iWwUV*uEH~=bk+oeVom7S6s>WH`h z-&4ZG&8II3KPh?HoSn7r(r9A^VWBXL4Pbu#DN+hF|Bv|pnbyg>^-#d)OCA458w!y} zO1#InI%bbq$-#^2TD(0HWKw*Tez!``UP>BA>~p_QC!fS;O_-t4wS=4hD%4xz<5Yz#LQUIu z*;WJeAh9W^v0xQ1Fj3N+m(wMcrFIbmPEBnv3o7t~e}U1@S5|B@HMh<_(zgps|H=YF z%Ir-<&yt(o$eegJ19GV|%`>SZTql@x`C%Q)WstC?T@Zleg~azESP7=6EPQ>;=5&3s zYKf=&OMDb;nS&s%mg`1^^zy0N!-4$d;)h;phAft92 zuI((}k9^P2zdzghC!Vw~>jEMgXrgsrFR)*95;> zq)jb!OABx77Rd>!uudrpinu6ujy3MsvGjf{H+}>o0Q|EA?W7}OZSc&!3?wSM5rL@D$w|K zy)|Ceb9Q-fm(&O0Z~$iAzn4{S=H92nY`s|uFD~4Fno5ry24R3j7xnH*OBM#vEgOX= zB)@nQ0Ao8_M{w?H%x02HA9=$0o|*vpo`W)McyR4n%#{gbi*DnQjpU)g5LJ9~QF8-# zl}&RQ$V?zRInVU%``FbD5EkO%!Zb{wMxIXY=c}YwR{1OX3z35{8o3%HkE?rM;;%{b zp2CDyReC^u+I4bg_kJ9T=TN}|;R~27JL)Y9*zq zi+qGJ{``G$fc5D>yYLE&3Ru8J0g4SA4 ze_Z4RP!_XWoDg%I>bH=*a0ll`xH?#Rn6^suTw)b3VhFLM>yptAnoaY{@xti>su5;D zW>t8QgX5isn>*wfzW(Ph1?Wr_W`M=d=9PGeaj%1eBDAhiW|zRf4{z0k(K^)}1T6$p zS8<58hCJV<@Dbmt5(Lj%K0Z&(F(TFKj^1UPhg+A3){9Hp2D+=(Y&(h`8R%vw^UgGf zbFVekWoxUn85P%Dv>t0j(1vTI#eQ}G)X7G9#T%I7)*fF?G^(-zqAHnG{zbnYv3AfX zHC>j@U)Tk;nf1QoxX{$Fav=v8P?X@fun;wV%yHH3mI8Jy-Qc;?uMs5DO^~4rly(Q| zW^SA>>yPoPtp>Ny8qJG+6NVh}MB$C#Z#0%PHLAL#p7(_i4Oi;kQmh{c7iI(bQjDyc z@x9Vd0?oij8YQavad-n%2CgzaicC(6X&A$}JWc}lAcnXYY}K?&1>htCS9pVGWo1v7 zhV-rtfH%6BkecS{*L%Qu9$vtO@m|8^puKGwgS;mj!!em8A1M5g84wYJelIb8%6K#s zg(ILSKNyVYooaVV$7~EX^>3|#Y<;Ej49rS*sqxtZr*Her%QW}S;Up{e1*Ij+26L;c z8?6&9pZG7eW6Xi_>?~~t2I_m2LB}$+Jj#~UwK6Q{;%x4xx1B6Uf>eNE)lp_vd7ItA z*_maDu8He;=02kNUMhp#N?DZP<$2>mEp8PT`|xHvHz8u!&ytJ9+G43R?-Xx0TC6xZ z)ESoS`(Mj{DS6Oyc-)7Xr?VVql3Qivl^a2AIv5fQUA4_jW73IuSueABSs5lJQ&U^URID0gQcA@!ly~LP5G73j1TAtNp)__%ftpO;+lgw&h%LHq zf7TneR_ACHiz&}|N)$}fyA$w+dBk^{EE=)fsY?}a*LvP3^(+8B!l1}fxj5@GZksMP zx81BdQ9CIzg)=J+YfJ++D&+_*7g`VWNn%WDv_mIo$)iTpJ$&C+j;a_6==D^x_1MOJ zHq{HTs%CHPSvf=trB0eg<#PcZ>vBt%VthpB1_mrdzL(j>Bt!be*NotEJ4?6DPIvDPXo!{2KO>-%bz zUYHg(%M08K-&3E&6VuS*os5}w03<1|gMCl+M% zO{}1JZ;GrZ(iF%)TOK}JOQfk6L7y!`xMH@EHq4K< z1>&jwo@aS3^d(>%#c#BO;O$DMID>8RjYf~Z7=ijc^L_UQp^F|CGBl5SCZ0qnkmwd; zivC;J0!s@JSbHgRg?`e_djzJv&9B`~Bdxph68l9>Ug)-a-JIU{^uBcFYxn;69F#;x$|o9hE?{HI?FVM*OQ0|||H{=g ze`PwA;zN>KsDf)aqf9Q3x6$4W)g6$Se+$p51MLE+)%{ZKlX&0H)&XZIcnW%CWHfj? z3TjpzjJ2hV{}DGNE~MKQ_6vrCU>S5(;N{`N?ro>9S&{;|SPNM#6?T_*sj0{wsdz^!)UaUna>hS7d)D-Ljf3hjQ9Zvf1Q$|aK`bLNBU8owqLEUmo zh7iySj${{+*6}Qtq#Q;hu^j)8{o;_mVyO;?jK4$7N-3luC%+Eu&1?8saFCPDxSUY( zO&LebajRwVPZWgs@@Bm6%YPj%Mp6MjusJn=Plvx@8wY>*D-7d*DxqCUb8oZ4=;zd) z-G|Ul|=V)Ld3rycH{y!cvUCY1N3xv z$Z(9TvDNY);wk(O_-UTb?< zC>*tDqf{J~p%8;=Pzk4hC9F@GHe_@@Q1DNeP&S3Eg5tz)%GSu0XXXu&qC9CJ)?+Xd z6u>5iQdVFN+C5=Gg9Sdu=N=aTwBTa1pWq9lBYmeP0x*VY@`CMI7k(3J%n!RpNYja7 z6~-`7CT-TYHZ3FuNv==cbUNdVv=vgDD$6*r=LLV;a|||v4>9_MAAd3)p7-g)YS&(y zkD<2dDb`-IYI#>FiKu)_B7U{yhA>L~U2AXCNZy#qaXmg;`C*;~q>y~V3x$v~0fjvB6xEADRajaB zUsqijNYKx0cg<8fA&RL1@IiMm2nv$6O8n-v=ASq<*kl8g;^W(6Q#KUE%*0DT(<=z1 zaj2JVJ!YVV*uUGB?}+ALL3+At1ZxxL*$hmSVgUUyz7+Il;A-TFc&C$Rq3wAv7ATy)HRZ+ty0prXURbGFUr@iqt` zY3{voLmZhJDRTy9$37Nfhyg#ajVQ>h?d4k|DJbs&e)sOQ+5b z5ak1bnhz= zr^5YPIPxXu>f_@i_49FIB?o9=(Drb#w3Q?1wRe)SB4r8%csZznOn9vk~sHQ~=gFtg(pxh1))P}l#l_EovglE<& z^iWqzoUnB9PwbFBrB{?1XNNEFZCa|x27>P^Z<|$~(4OmVmB%d?n@M95xoh`nSg@7yxq8L?vlCQY5F_wYmE~=-MS!}fwxnK63y0)9*r1B> z)d_+%hbq)FjyYE&M!IYSQtz0{-eEt?`&j`vo_@0Xy;7-4D~>hFt3zaUFcD(=hN%VR z&EN6pJM)HJsJ1VSly5S)mS-_2*lAw(Z#^6TL{UDC`wP2XG}##Mt7hSe@f`k5C*I^) zk0_)Jk)uay#2CZH8Ipr92hXgk!wNbKn_IH~p~@L)oS!K2%?Ynii=-S@W`2UbQgae_6RbGgv$u z>#YHLGlg$sZL~mH7^%3}e6ee1uzNbfJmERw`m33w`e(1Pwi1GylWd%0S$j- zXRv~sSuP?RHr{r&*UC6NPIlw;_x6?F3#Rt)0Nvly?VdT`C;gM*omt4|NFK1_MmU^^ z-`Sg<3iMqMFdkWwD`F!r`x_P}z7)j0h1d)J2~HQcR6Fj(-E?@X?gso8J+Ye0VvYTN zRw&wg{bXFM-`&|SJGL(DBdR(*U&g95_Qx~9u6l_-l~Mh%)Rk@zoPIjL1buz#1}l9+ zHgwhRiH6eaeW{R6hkk7suPPZU&*qzswtTXULR0h zhoM109~2aru0;g=rpM55q~!<-1UN$ivGl1Mf^-5wAyUI%1{y>Ss6YXIb{k9xUC66~ zI6N~6zg;Y;$iL4b@mKZMnp)3g(AHIJ>!6VrSO4{^&*U0Y{7sn{rw|vXtKZ!s{|8FI z!dkuX7jt4jg1V(j&FJ-%yq|Ere?Rq{COF-^C`szliGu7|Ya@}L*F|qZXuLQjdaDcF zUGynek+6U$T|YgU9K8TQYX>emuci$BPHj<0b{MM>5<@087u@KIQlh><)hXXaXNMe& zf`)B2uumvTOBh)bVkZtNU&RY2q1^gYMB=PXjjQ989Zt?dS}o6%29}>Y9KSPjvhH8@jc_gs0XJ51jl2PWf4-+< z(=#%1Kpq?gvbqhDwo;YOd%^{;zxpLHmQd?e_j&MGW3+Cq(Og289lTY4g8h2{g}~Sy z@!xZ4DfR*X>mUk=g9aES15kkh6Bz4&g8y%e@M}Z{#5I|PTt7bmy`cbCOof&_P22=1~#(BK4J+%34f1qkj=kRXBZc z-umj>Kc-I2%y#$bYPqhiGYu!OohPuh)X4CVi4X`KG6!rrL>6*M%#Wo@l z#L)D7e0`Y5o(}Nu(RRWOe4Rxs-^DI{^vaS#&(7-esFO) zN>{R3Q%$eAeCgN-KA}Ua_28LzeSHh)H!I0pf~=HT^%5(4)V(UN!??;94h|w-x)z61 z#ny`MEu$)J1|?FcZ-%;RO32Spi#B>^7Lb-N=48H(w+H+v_+)=+YFSxv1dg9H?{l)S zPZY+Qmxt0Vf+YrzDkwzo*ZAj9Z2lXeC3j$RUiy~HN6EkTOmsM)EshLUBiFN?KTl*g z+V-ILNrB$!N#9m77LF-;arzrNm9yd&8Z*7hjOH-NF2f>-|}u!Mts=(g9I^ee)pwy`*;Wg&W}2~_S#>?vh&0s>EJ!D zZ^^Ghr@sWQuNQMof%y1DYa2ko_|xM1LZ<2qyX4gI5O5W>`R~J)LF+XiIsC7mFWD9% z*-zW2!^DceldJ?9BB0JAHiSIS@rY`9f0mDy6xE#g9!h;yduwBMa;GjUWiKkoG(I=5AJ0S*)wqd~biJW){6PJ?Z21t~l<~!}f%*)WynP zm90^ku;q*0*5*617IdynG1%W65YDG7sP|z|W?%$?%nuDF$R~-?Al#4$1}l#}v8vFSvHpp!aeQ#;K7eP;F;hNfJ?75hqIzjKV)$q})}3 zyT6f2%*NsDDec+C9IljFASRpNsc%IR?Tsf?W&GXAMV{?m?N;G*-b!MavUOS^T(;Bd zs5Yu%eSdsC`8bw~U_qPR@>;!%IUz=e?3M5d`|Jozsy2-Ly9@2xO~=v9RDT^Y2KJ?R zj&cN8wO_oQW`PQ=-NeP!$*IEYecSUpNUt10Ln*o)94fOe_rwb&42%?n7>gR5X6U@i zNie+0crGj*#26hqiilew`Js|m#m>0J7N5K}Yar8-{k-wA+cFZNSvghrbI#U}o*VHN z&UV4Q%qGkFox&*GW@D>5T3l9>D$Cmkrw#OE9cF_nI|ekOB3s>p>|ev!MO{g1ixFJb zumxonjT#l?LaJ$wAXQew)T&@GV|kLG&XpE4w<0wTI#D3vF0y==n{y4f7*^03K{J=i zc&19fmP1$(A)l^+_*99>Q5liWxV+`YlGkj_f$NJJ1(W@ zt$i6mueqE(;m01M+vznHx0T|Ip$fd2zdv`2pC*$UF^Hl1-8}M={hWM`7SH}GyPJFr z2`5h^FjkdeC@#rw$R~iUC*aVIGZKy}WEuz}iNHO!loZ*ElT-C~8!S-puwWaVBU#_s zKDRtN5w#ijp8CDG(uMdHObHso8qJQwr{}da^)T$G^)ffR zWnv5Wnd|fyIR5;ULNwtQ#8gmVvG*F<4#2IqD||=*nwiF>OEM=rnA)rou$5?b*7n)) zrFamKJ>4IWX`&k;Y<+wBD7kp=#L@_503LEpPdc&}H(4;R+o08hpF?XF7-S15eExpF zM=JCEt#c;M)x9CXoAINTy_sbf@y~=8TWF+^=@ut0sTBcKck1pB!cY`t-1@TBbc7%7 zaz5)_m0T<}>xwSU>#Q$)1@25F5^VDaX+}>ugon1^;0qp% z5GK+Yu2fWf=m<7EIM}EJ;WsJ558Y@{>J38@b*p|Df2-S+8Rl9OJc=HG%r)wCJGKKz;u5j{b`%N`>`Q>0jJm|72UL{+WJiYAi-)z4e-eWt9{e9eY7VGX(y6Aqq1z*0tls}$& z)_46mxSs}3ny!JXE+{viiLRXCx6h43dB1-fdfMOqJ6Xwr?QHV0#PgfN()+%Ge<9bL z6r(1~{*z~kX(cPp#)ea2J6=VlCH-kB4He~y%G1GOKt zf3V!ree8OB8(^*wn$c`01AICc&l7untobG`{`WKxC3+?^aHJ}FrLlpF%ao2e)j?!k zZoQ+}ft!i7m>gq_j9D&J^5xn}z4>T=CI2ibg)}ux^2*AUsbZQaa_8K_m5JUhR;uN} z-t`DfNlX;^$?c=-5xd^o!rjUBNDSPPUQkR*XZB;~=rpo$_`ARVvaCOU&S2|XvjD}{EnU6MpA(xh~TL+J!Iaoa6NVR4d0Vh>B* zG`epTer=(%S-X@=jnBRDpRmn zXIEuQ&8Z5{X5iEA`TC17alqR0;%}k=)W@(NYm;$Z-$Q>mV6<4NSa2Xg5Z58_1pB^x zNPyon95M%QR;-cOV!3`ljub!>^T6k3r}UhXSka$F#%PBX41UQEk8hFG)xC7{b5q7z8XogTwld+`#4NVwv#kB$P(`=&m~uz^OKD?r$)ZO4rtWqF>5ai zf11HVXyf3Fq2v=Nm{! amp^QW=wtj zJL@VoWm$cCl-p}rOV>g+SqNecN(za|Y`h>`g+>I_R|5wva?!1Y1!RD#9+6|^q#VgO zjx9&3YkkB-d|S`MGL~#{cLPpabznzHqN_tv0W|j2AM6-`2^D+-jrfQdp%#t$uHY>SaeA)KLf_v} zOezg(!3;`MPw*l`I`+VP?!8*S2s?|g^-LD7NDjM5<&+Ss;ombB(yW+knkY1(@-R*EDn$vuT1R?Jg< zLByVSY<&?C27iW>keWy+=$!zD{j4V7kW=>j?T-DRMvV!xqdN5Qy(FLyUGXi648v8b z2k~J)Yz_y|t4LHp0|FA;gr@{iV;SL4K-VSWu}MNxN(GxXXMzK_O#!kyErQ~g@+;}1 z#bitLD%W2p@9SM==VVx=t33)5n6;9d@*|1pSVx*$z$5m5c}VqhyWh#_vbB6$5dLG+ z&t@RN4k$@9-w+X&G*>yEZ7ROy3AF`a#D0*NJx`6PNJKqJ-PRi zO?>LoQQBV8q_wYRV=4BC1=%}NxHIg=$69kDd`fjt(meu37KHkr>IP#OvChasz7Fc|Ntmd7CRGDnX zqnkiO8vBz-&xuUs8~%4jG?w{@$6mVmo@CDCcKP4uXPVR`KHCcJz+5TPbGZORBA|e2 zH6 zfq)>$7R@O-=?-~Z;7mQH5t(;;;sxd^>*5vH*gZelVe08m`tM^u{t0zNQa7R{61V#2 znWzW?t-dbmVRBBl>xMe%puMZ4gTj(n-33Xlpm;$@B}|a|l^Jo_vq`axT))sUEn-?8 zShhqoe>6rXD0rDIy1*ft_lLyR;ITWC2$r4=f%V9-+NYU>lKn_J%nEKDuG>B-vcfel ztZ$-9#ToEfhH}S+gzKoRv1OK1+8It=(L9K0o3Y#)k5aR)B1QBL*miy6qwwDy#2!MS zfH8^v!@B`@e+B+D@umyIh@1gUA=Pv$Z~}MIR7cVK#yx3{?OSRNQMY1{{Rkm5cZM%o z&~f@(zJru*gbtodx{3+*0y)N{H?U=#HHPF^e+M(<98~Yv5AcGjsH0Z#B11G z0BCF-=&u;ZF_B3t?$bdLCRR-F$V-_UtU`9*v{1RPB<+ry^CcIIjF!1hfC)z&OaDOZ$fE)*&9(3w59cbL> zHY+6iZ8_y!@gpr%;_!Y)(c>>~`kL09^nEac2rb2%3#JmGfIGO{ywGbwf&{z(cX(7m z1x5rHE7BmJ5y==q)^>aGX3J1`iH?MJ+CkN;E+It^iDbjs+}h=2aPsRYLLOG&))C>t zp5Pd0b#yA3n~2Tw1kgZ>azieM%+z^67aeSW5)ReBxil1V)bhGlAwXB3Re3zK650`v z8r(oTMt|a(Cfpz4TJF}{lZQhBCL>HpvDz23(NVfHcn@@Fd25mWFlrMgY!e6NfZ*G} zlWyV!Hn%8cLuBpXbY7}K_Zy?YCDWrdFWBaFT_DN7lbeJ zRc$dnhRu=-L>87)#Ue}l0Ij1R?l!>w>8Co$fR2{`zM|2?!?D^ReNooPj;%^M`~15|qa!(iI*d5J(BOC4qbv z09|P>2|msLuUa|@d=@%Twzs^Xey?|68Z|-1wo5(qXE&JPo!BEhC$_7cgeMMsCb+*c zxVt3Ku6n|~cI24S5S0^WT3B2p3U)By51s=!g$sk6eqB2}&1hDtWom0>uDax{xm<*| zMVmNz z68`X57<43=Yr@|--(y@?yQ4f~FsEGIu=SlnY};qB(f}9R(mFAoL;jdEU^mfRwIrdS z{#YwEn89%LLBpC~&D8p3AA>sr-1a4moF`8le^_mnF(K6{o2BbDAl7n+~3j4BUdW4^&8tda}e8=m9(; zS3XFoB?TL%KZV?+J9xl3$-b*W+wC02jgkwx>=sknj@9ehj?n8KoPBXHXlQ2V1t0;b z-g|a6S6xd~2Ypjk|J#ysQ&i?0Nrd)$Nk2jhJQ(<(p9_{%yf5Wtwvy*kS3CXOxka>LN%VX<4NIOs>O2&7#}oR=>hBnkXByi_+G-l5=D&ax&<*!e0wgaMS9DP z9I9JqU8mJX*3%lKmIAOH9ll_CK3F!!Q|M5VPv2i%(5<;NVmtZ?Q7B2^W=t2f6r><> zpgcr6PSuK(!TQJ{eT|)NNdIfC-CqKdTLrt**C{C=X&%p$^z2=aBq6vdQu)Jl636yH zUuMt4meO}9tYV@e=4eo)GK%1)idK{U#Tyaa;wa;Fbk^PFp(o}Ny zJRQMpQk^m}M*bHH&I0SXL*J5i%={vsPjA+A47~DU)`mEp?hSxd@yc1ZG#AswEQ=s= zBsvr5cuRq>^tL$ZgfJqzyLEpf5j&c})ijTPBzD`ncv29s8?@Vnmw8GQCos?N5XX24 z*iA0peknYDt6chQ>M*Of>>r;%8Q9<9Dv}YM(SEv%ZMJwsv7Ns2Bz2>&S1)rGb+H1| z49<1G#i_mZT89XQA4}kW$En{+YXUPQd9j?d#m3TVQs?SRZtyz_Kg@KldiqFY+`lGn zK1>ra2(3#alFKRmp)*chi5YjqJTw)wt3q3?eJXIfgo&|+G6{C|SjnTV`zLqMKmWo% zm8UmvD&hz$!*N0nD(;lP8mFQAM0!>5XRs=lfRM;O54A*4M&cR7L1F8|-ur2Y&C8-s z(JHFofgnZk4_0Y4x9Qp5Rln1B%m>MV#kDj0{~!=|EvAEKK12(sOAo=e7-)!T_eD-( z-e2LMfU^e@8x9!TsXy+bzhN57=7_k}2@#4E+uta2x^Ed-NL#k5lo;*q`Kjsia! zv~i>Va_S>r{2S`v16n}sdQuyndbyD@i|@FyHk1so!vWQieRx(!26KJ{=hczor!@!7 z6^&xFapb2`3%13e$RT!ZNE+I~$Vzh}qFM&jE)mO$&0WUdJ|E~SL&;$KRBz^jj{DbFNM5pb&41-0~#TovUc?dkMXYD0$3wbn0-%KH`($f`q0a|fer-~2TF z5`dhEOWhP_X><6^Qs7p4)^8k!L!I=zQz0F z8-**Y`whoa46?OQmFKE2=HZ@uuV1^yx^{CYxoH6r8+V6OvA6_N5vA2`{K0WVa_7pAwA1`A+#1f-8kTrQyp!llpM?ooE45fbhJ!y zyagymh;^JWV{s$0oUd1y-^Avt6yqeRV&E$-Gal$d!@tGxxG*htiAzY7%P)ja4{921 zM8&3R7E;RIlZ_q@2?L8`!PFuVJ*7KR?tN5Z8hNTDJ&;ZoqY z8fqrO>SyspNHG$2lm=X{|5qeXQoyvpaB}g`kk-biPMN(Y`rU)&5I{Tz-KCHA6xR%s z?RF9_aC1S-zN%k|RP%^U;obnK8RuO+Jp#%&CK`K?9r_9`A||a9N$zp^-@S+IcLgR&a$IdG4z zS#>|=R_tQd_J6p5*{pDXQfJNK$qf(}9{u()EUj9dnyem_hb*7~T7d)6fzboqe{Zxj zC9<7%WnnPuCgvs0CD`*4;}ylL6&oeZRX}(NX{F%CMHT+T1>juCzf*)OV<4<#6H}-Fvh}45`a`fe5G$k(x#KVMW*F*Oz?Ap_T zpddH15Mop|Fg9>TH0^&_2KEO5!6iQ^(I%ejKn&t93g;VyAYQXN0&JHXPcZ{RxT=uw zZb9G6Y#|KLkVFa%;Gc41Mh?c`CKNUz<#&` zF;XYd?xwL0c3A&6e(dppv!JNF+AiK+xRh>|Ca?0H_gn+D+JXPU=u724-n!Cw z(bV*9wdMLcCfn@#8-*RPSf`lL4q7GJI=JyC2h}-avdyf2NokuFkk~Z2KOeSWf~rMj z$sH!w^J2xA{DbxMmS%mIKm5xCqgFZe-X=*u3}&HhrI&TL7endtm!}{YQ+dY_5q9xU z=;7J)k(TJiVDIY}gz)(it4klp4Ur~7Bjwjm+=WX@VQ-D3=P1zit=aR5=bijF#?iRx zw|PT2G1Y*;3ph(w-OcqMcz1ym4qIe~`AOrKKP0#534yHa)w^#Ag+Qs}deazCwiylV zhvz^KvG@xU_lB9u9=mY=$40MA1~BHvn?UlG&VJrgP_VaUrPk*nZ^F0Ts}k^`OS?TD z8!+Wu>E7X*#RK1xDZpQ)<#FoSw+lm%n{f`+cXv3v4DU22{U0pWPy{_}H2N=?Lu2o> zyv$|<W}w&*|u7fx3ZSosQxgP^%nClwbY~0)C%VGoDFBsSN`YJsya3JKN_KXAhGn z^(9*i~j*XtP(sPJxx@Fa5w=o7}>l}P2$C-8cA;>4~rSZ`ld8ffy#YY2BcK(%@@ zkQ+d6!L`nB@5EY-f9y|h0I^$)4!RL4(WGm#rGZXM3m zj6YH8n{25vI@UT_T|`j4ved9_X@M=>+te-j_JOR zAs`nbq6C;@8lxws)J9B&%PD}Q?)!@9tGxX_tWHlB5fBc@MK9LZh$vQNiaH4?Ju`Wx zKY%qX#>+Vf#T6&oP#B5Fb=bl|ignr+<%8G8mZVgG?vqRZpl2qoL%Hq$_(kc4$<=zY zi2natF%Vda82RZOcHPe}VtD2Huiz7xnq!fkq5kUs0pA=S*NKrJ-LZ zJNUi9`Pqf} zG&id;kE)=kBl2|d8^)%oi5S%eIOz526{f+LO&$d?YW#9*0W!q<9?WAV!9!55EG!L} zEh(G0nE7^`eLLzUub3X7ab~o9E@u(FBltX(c9@iVV~xkU9A%SHY`iGKr;S*Go4Mn$ zp#$zg;Z@V&CMuohWMDq2U`;JQ?4kl6Sve9GdF!EYA%cqa^|~w^T}w~(J;GBsg#qZP zeP2im6h^kvrtTa+fHedr1vKzZ)6HA`@OHG*$fr_{NEj64#ShSdw4=0EKA$ZobU%No zpr2i-^i{MIh=x6xYwVZT&th*%1ptY0i-b?<1q$iuc)ZX8Jh`*CQO^7BJWa8SV(?wM z-!ab5Vme6u26WLe8h$$>bH6F?Qq5xg(8mp+^&M-kXrR>G1Aoz*huW`I(8hDqYpKsi z9*PH!q5AvMf`fFwzFePWy4sUanRNbq^m^=k_G>$=+{+)$yK(Q`TE^zk?vmCGHE%Z* z*vJd`Q{fx&YTMQ~u2lW|$M=C10$KK#^U7!7=sUah&_>L#b0^Ghalu)~w<%Gly^oUf zV&psRQsSjEPp}Prb=*ma#-l_HBv2RHPg1zFCqRdXU%`nY##uq4(2p=m+Rr(JAM@C# zpRwfwhy;pJ?=)q6F^i%UD;p>L>XnNsH_ry4srog;y^>hjftg9^TOHWme@p+ zsiE~L%;fzM2jw56G+*_L&$nD>f}AUkR^T^ z*d@pVzaZFOhh==_L+<-`7AimhRKXmMtSqfqeT=Z+fu}5#SYHIHv%$|{zcfU4mzX{u{;eO+q z%X}mqofm2-D@|N4Xjjl!qUMNr5Vyn9sKFX`ve6e?Apwq zpS0euzZ6Qa|51x|v#r^<&^lS~?-fNOH&Y9D(cTfXdq1lvaii`q_THf0^IO5yZW&Z) zf;oIGPA7Ar&HX?i0^C68io>q@ckBX6?b|=T*-F3F)>pi{fhRxm>K z>N04vDb#g*F=98pXyv7@08`nv*tf{MhwobMrFrvpE>5zJr8x&Yx;0pJ zufwUOvD+FkOOTghoaB~b+#a=A=M<48o4xX{j(pz(U-XgVGFTD8_nRHj0aphId{(h| z<7^jmgHTJ*u(9LNhY$-j5bZgTV=Wy=3}xTsmu6S5_vi~WyWVUuH$_K#!ZG*}ygmI5ERRFw?V zh$<34CvV*$(cI4A`FXl7|3Uueh!5iG_KwgdWLQWTY?u-bSc~3V)igZ0wlkQJN$`Oq ziWz5k#{}vF&}0uLVbVClDaoKBiae-&&u$q@!DoLp?@6~>Dfx;E4b;%jnm65?dYls6 ze3;-NK4tjGy|&#aB#bLa_?3EsB^Irv!j@52(S7v{!PUrKr;M4+%d2oOC)#%E<*&Cb z`7nS4>>y~%Xb)ZU@pGGFUIyg~eUlE=_o-(-6Y~7E2vrrr%$|+WvYe>ShRnM7_|>bn z@qHNUYxvoKU*}`>*@p-`WhwjmM^b*$tkHHnGxM;TCEN#Ir*2fGAll=`@5w-< z`)7Yp$pP^x`S%G~aonE^Phqp4Lzdow8Qgz*zD#ffQv9ZN$IGK-uTor)TdG|n+yB&*AhHXs zs2uj4N4l^l(7PvJE-j78wOzByH7__ZFAdiL#6GA$1YbLOUWgTlAXaGfF$|A4y9|^n zVT45=lr8nQwPg4Q!s9IpKNq4ervn(N_3k>2O&E(49KSE`cfow0^O5AOCO313D%w+ z5)txx<})ePu`Zw=h*u z?8vE6d;9Cr&6YnY*+t9dQP9g4Lc}`qz3_53+WRw;mty~Ocjfi2An@6ORR7deHREL< zQ5*JC~cA_+HhY`5{vaHC0qUazm4OvUV270yO%Fph6QMnVH={#Vgv{pOUx=Mc5c%jQ=K zNB@j#v-TsJ%A_jfJ}RleVKIHcdNAMt-k3O_TVT{A1``_N6^M?A+>#9f$+P0j1E-A8 z3o4Uu!9Bj&Bu(ob^uyJwM^TI2>m*qVS}?HA>fuVrs2Y*@DcZSfJ;}uWss+xg^000W zg|fgi_x^@71XB?kp@V$#Y;JsYK){L0207p4Eyib9%|+e*!3Qevr_h%Yy}S7D1HX$w16U6$n!Rrl7RG~ z9g?gZ#`)(04D*;L*}Q0Joz@P1IVWu3kXe4Gm75e3X{%bZoZ!cb!Fx`byK$E|{#LyL zLQQk}7XP&xUhyHYyM@1_tD9BZ*f~o}Z7=>Js{>F2c`=XEboJRcL`eYmTEi{=tK)a_ z7QYD|q(C#g03Kg+XRQ?nWL?+5ShKL*WZ%AcEcfx~seO<*4Dn3Yb-Jm;mY%N2bN9_w z(N-Gvc+7=^v)JQ!3nN!&kT=_oM$^^7{#07-)JqRiD3%i#TdZsDQ4_Tyt*|*q$$s0# zihvewNzBn?$&S_IM{C$fmY6+ys?kS6CJjV@h-f%FEZZoXHXKa~#2#~14eswI7H&T_ zE4UX%SGhMP39qKegkwxLI()<&IkboUfo*igJvjQ0W87l+Bn=$EX#-)ZcDnaE^^T;y zx#AF(mk$lhi=Rt-V^^E~6XBd-9S0%r!!SNAs?;`zF)I)VZ|8SA>WHwYhmP`o4~pXv zZlXm%tK^$!d|XnM&YOCg^)*ve1uZV!0WX|QPts~eIDDjYljhwq#u7J=5Pb3b6xwkn zzOa?DJ1@l5)GnOUuBvQwrBu^cHXX( z)e5lcL9jH`6fIznRvFw@j||M*S^ek~DDz1Rwb^>&!VPstAZ|d?YQ3L=1)Zg|(c&X2 zjA9bd4~K#u2ZtW*GT9HH{Xs_WossUG`ui}j<%+lf`BWNmLGa(D5r;uuNtVK}gY4$NI5q|#i?jXD5PI&o9+mimw^e%!#%rlhL4F$hRz#7aKtXli!u8Fy4&EZ|JlW5I z9&@A-Ou;cHbb#=y=1!m0jtH;}sx}Mvbyzaj_&rf3|rZpGf=jT~; z@y$+ITCFH5uu6VoX&M@{88d9{>&s;5<)?o4d#bMamz~qh;mgh7hKCrqV;NwV05%Q) zPDnU!l1n`?0h2sOYd#QaX37OXt55l@;6cXroy}egPwKsQuRnpD zh^hutaTB>6E4h}11(Dlm4l@Gydhd2ZJL3pgt~8=J(f{&?6|1B%9b|-%g-4)O#u2lj zJ6Xx!rmTH^{a1~d7P3G@q?953>yGbYfjNcf)~_`}Z~MAhmPtMygb19Z`)nCg$#{i# zV4=U|nHrv=obK24J!Cm?k=|qP`DWyiYw*sWTe>Hwj$|QVz=9*PY;^$K4h8N3B6c3R zik<2sEO2GeN2MyLi1K&oqm^}8tu+jXQ`H41A|F1?@iOL+ImPrVq2d8FZ?X#Lpq7f{ z1gA58>PguQ8)2UfkS?F(Y4PMxxlD?EbeB_=EH+f%EOl7vK&A5#qBH0pYdAF)VLecvN}#U|ob4+ZFchjj&9 zaO>U?DCjo_K;m*cwtl>k{M|haO9dsi5R4tg5mEwG7gNjjI` z*?p|=NGsKP^`i+|-u5`hIIopKE)GnLDG@u|bpkV*^y9NJnMB&ij?F^`7%5Z+dOczq zrW_*)imve^ACtf~Uym4=%0bX=H(ZMM2;J;?omK+s50u~fRF$U>U-dFDeI_IsDE|Q$p zmpB9~u91m&(CX{7l>8c%7J6;1X-Ykl0*)cM61(*i4z z*6YNJmv`&lm!6-VF?E7#%IM4$S%Wo3E=F_j_hyt5gO`0hRCIGVy5qT*^sZr*{lqXy z0L_%(>p8!5Ye zPrnyMB*`$~w|Y2SM;!^2d=Qgl?PH@=W)lO0Hqqh=YRd7i^%o?RhS)!%Go?y=K<@jg z5@)4Wz9)U`)vbV_sA-pv7#c#<@R_m>r5p3|eeoD-{o;=ffq9b)=9UyoSIHhEQP>V> zw&x8!-3iCh<)4~tUVG@|RvLn4b?RPOU5LF*;`9SrM~@n{tFqb$HhFBg>YDd3dG&c< zgNy~A5v(9_sZISF$urSdi-ynCU0x(D0^E53+IC#n%UuI1-m>iOr%&0f^Jg~5-AmCI zm}U>t{jro67SER?%$YglgrQ@F`~e=<>W_RqFZEq?I-_GUQ)4-S8}+Q+tvfg4;{71o zmry;+!gcg#@m(2{+D@ZSD=8f5odVX>A~@Q|B8ZYsf#PFdv{KoZ%|voB(SDwvr~b}e zI1bc%h0vH1LdM*&8fYv&V*dLTTfB3~|NU&mMGuw-;_Slk?%z=%A`HymcQ7!RZ>RSE z^ZO2|b)hEzmkeXTz>w3!z!3cB;oGW52r?r?1POP+fh@}5p#SID3Xb=id93&Lyf0X|}yncCq diff --git a/src/MeshWeaver.AI/MeshWeaver.AI.csproj b/src/MeshWeaver.AI/MeshWeaver.AI.csproj index fb026df54..546a6ca40 100644 --- a/src/MeshWeaver.AI/MeshWeaver.AI.csproj +++ b/src/MeshWeaver.AI/MeshWeaver.AI.csproj @@ -7,10 +7,12 @@ + + diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs index 50abd6775..e538e2785 100644 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs @@ -3,10 +3,13 @@ using System.Text.Json; using System.Text.Json.Nodes; using ClosedXML.Excel; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Wordprocessing; using MeshWeaver.ContentCollections; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; +using UglyToad.PdfPig; namespace MeshWeaver.AI.Plugins; @@ -35,14 +38,22 @@ public async Task GetFile( if (stream == null) return $"File '{filePath}' not found in collection '{collectionName}'."; - // Check if this is an Excel file + // Check file type and read accordingly var extension = Path.GetExtension(filePath).ToLowerInvariant(); if (extension == ".xlsx" || extension == ".xls") { return await ReadExcelFileAsync(stream, filePath, numberOfRows); } + else if (extension == ".docx") + { + return await ReadWordFileAsync(stream, filePath, numberOfRows); + } + else if (extension == ".pdf") + { + return await ReadPdfFileAsync(stream, filePath, numberOfRows); + } - // For non-Excel files, read as text + // For other files, read as text using var reader = new StreamReader(stream); if (numberOfRows.HasValue) { @@ -149,6 +160,106 @@ private static string GetExcelColumnLetter(int columnNumber) } return columnLetter; } + + private async Task ReadWordFileAsync(Stream stream, string filePath, int? numberOfRows) + { + try + { + using var wordDoc = WordprocessingDocument.Open(stream, false); + var body = wordDoc.MainDocumentPart?.Document?.Body; + + if (body == null) + return $"Word document '{filePath}' has no readable content."; + + var sb = new StringBuilder(); + sb.AppendLine($"# Document: {Path.GetFileName(filePath)}"); + sb.AppendLine(); + + var paragraphs = body.Elements().ToList(); + var paragraphsToRead = numberOfRows.HasValue + ? paragraphs.Take(numberOfRows.Value).ToList() + : paragraphs; + + foreach (var paragraph in paragraphsToRead) + { + var text = paragraph.InnerText; + if (!string.IsNullOrWhiteSpace(text)) + { + sb.AppendLine(text); + sb.AppendLine(); + } + } + + // Also handle tables + var tables = body.Elements().ToList(); + foreach (var table in tables) + { + sb.AppendLine("## Table"); + sb.AppendLine(); + + var rows = table.Elements().ToList(); + var rowsToRead = numberOfRows.HasValue + ? rows.Take(numberOfRows.Value).ToList() + : rows; + + foreach (var row in rowsToRead) + { + var cells = row.Elements().ToList(); + var cellTexts = cells.Select(c => c.InnerText.Replace('|', '\\').Trim()).ToList(); + sb.AppendLine("| " + string.Join(" | ", cellTexts) + " |"); + } + + sb.AppendLine(); + } + + return sb.ToString(); + } + catch (Exception ex) + { + return $"Error reading Word document '{filePath}': {ex.Message}"; + } + } + + private async Task ReadPdfFileAsync(Stream stream, string filePath, int? numberOfRows) + { + try + { + using var pdfDocument = PdfDocument.Open(stream); + var sb = new StringBuilder(); + sb.AppendLine($"# PDF Document: {Path.GetFileName(filePath)}"); + sb.AppendLine($"Total pages: {pdfDocument.NumberOfPages}"); + sb.AppendLine(); + + var pagesToRead = numberOfRows.HasValue + ? Math.Min(numberOfRows.Value, pdfDocument.NumberOfPages) + : pdfDocument.NumberOfPages; + + for (int pageNum = 1; pageNum <= pagesToRead; pageNum++) + { + var page = pdfDocument.GetPage(pageNum); + sb.AppendLine($"## Page {pageNum}"); + sb.AppendLine(); + + var text = page.Text; + if (!string.IsNullOrWhiteSpace(text)) + { + sb.AppendLine(text); + } + else + { + sb.AppendLine("(No text content)"); + } + sb.AppendLine(); + } + + return sb.ToString(); + } + catch (Exception ex) + { + return $"Error reading PDF document '{filePath}': {ex.Message}"; + } + } + [KernelFunction] [Description("Saves content as a file to a specified collection.")] public async Task SaveFile( diff --git a/src/MeshWeaver.Import/ImportRegistryExtensions.cs b/src/MeshWeaver.Import/ImportRegistryExtensions.cs index 1f22a2e8f..c6d91afe2 100644 --- a/src/MeshWeaver.Import/ImportRegistryExtensions.cs +++ b/src/MeshWeaver.Import/ImportRegistryExtensions.cs @@ -116,7 +116,6 @@ EmbeddedResource source return ret; } - internal static ImmutableList> GetListOfLambdas( this MessageHubConfiguration config ) => From cd648ffde2d281d1f9a69f7dc9449935ec8bb4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Wed, 29 Oct 2025 20:31:12 +0100 Subject: [PATCH 10/57] deferring messages until data context is initialized --- src/MeshWeaver.Data/DataContext.cs | 7 +++- .../GenericUnpartitionedDataSource.cs | 37 +++++++++++-------- .../Serialization/SynchronizationStream.cs | 5 +-- .../MessageHubConfiguration.cs | 2 +- .../MessageService.cs | 4 +- .../SerializationTest.cs | 14 +++---- 6 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index ea78c5c79..7435cba1a 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -1,9 +1,9 @@ using System.Collections.Immutable; -using Microsoft.Extensions.DependencyInjection; using MeshWeaver.Domain; using MeshWeaver.Messaging; using MeshWeaver.Messaging.Serialization; using MeshWeaver.Reflection; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Data; @@ -24,8 +24,11 @@ public DataContext(IWorkspace workspace) type.GetProperties().Where(x => x.HasAttribute()).ToArray() ) ?? null ); + deferral = Hub.Defer(x => x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }); } + private readonly IDisposable deferral; + private Dictionary TypeSourcesByType { get; set; } = new(); public IEnumerable DataSources => DataSourcesById.Values; @@ -87,6 +90,8 @@ public void Initialize() foreach (var typeSource in TypeSources.Values) TypeRegistry.WithType(typeSource.TypeDefinition.Type, typeSource.TypeDefinition.CollectionName); + Task.WhenAll(DataSourcesById.Values.Select(d => d.Initialized)) + .ContinueWith(_ => deferral.Dispose()); } public IEnumerable MappedTypes => DataSourcesByType.Keys; diff --git a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs index 6dc3ea206..e44dcdc93 100644 --- a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs @@ -27,6 +27,7 @@ public interface IDataSource : IDisposable ISynchronizationStream? GetStreamForPartition(object? partition); IEnumerable TypeSources { get; } + Task Initialized { get; } } public interface IUnpartitionedDataSource : IDataSource @@ -110,8 +111,16 @@ This with - protected readonly Dictionary?> Streams = new(); + protected readonly Dictionary> Streams = new(); + public Task Initialized + { + get + { + lock (Streams) + return Task.WhenAll(Streams.Values.Select(s => s.Hub.Started)); + } + } public CollectionsReference Reference => GetReference(); protected virtual CollectionsReference GetReference() => @@ -120,7 +129,7 @@ protected virtual CollectionsReference GetReference() => public virtual void Dispose() { foreach (var stream in Streams.Values) - stream?.Dispose(); + stream.Dispose(); if (changesSubscriptions != null) foreach (var subscription in changesSubscriptions) @@ -129,10 +138,10 @@ public virtual void Dispose() public virtual ISynchronizationStream GetStream(WorkspaceReference reference) { var stream = GetStreamForPartition(reference is IPartitionedWorkspaceReference partitioned ? partitioned.Partition : null); - return stream?.Reduce(reference) ?? throw new InvalidOperationException("Unable to create stream"); + return stream.Reduce(reference) ?? throw new InvalidOperationException("Unable to create stream"); } - public ISynchronizationStream? GetStreamForPartition(object? partition) + public ISynchronizationStream GetStreamForPartition(object? partition) { var identity = new StreamIdentity(new DataSourceAddress(Id.ToString() ?? ""), partition); lock (Streams) @@ -144,13 +153,13 @@ public virtual ISynchronizationStream GetStream(WorkspaceReference< } } - protected abstract ISynchronizationStream? CreateStream(StreamIdentity identity); + protected abstract ISynchronizationStream CreateStream(StreamIdentity identity); - protected virtual ISynchronizationStream? CreateStream(StreamIdentity identity, + protected virtual ISynchronizationStream CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) => SetupDataSourceStream(identity, config); - protected virtual ISynchronizationStream? SetupDataSourceStream(StreamIdentity identity, + protected virtual ISynchronizationStream SetupDataSourceStream(StreamIdentity identity, Func, StreamConfiguration> config) { var reference = GetReference(); @@ -175,7 +184,7 @@ public virtual void Initialize() public record GenericUnpartitionedDataSource(object Id, IWorkspace Workspace) : GenericUnpartitionedDataSource(Id, Workspace) { - public ISynchronizationStream? GetStream() + public ISynchronizationStream GetStream() => GetStreamForPartition(null); } @@ -193,7 +202,7 @@ public TDataSource WithType(Func, TypeSourceWithType public abstract record GenericPartitionedDataSource(object Id, IWorkspace Workspace) : GenericPartitionedDataSource, TPartition>(Id, Workspace) { - public ISynchronizationStream? GetStream() + public ISynchronizationStream GetStream() => GetStreamForPartition(null); } @@ -258,7 +267,7 @@ protected virtual async Task GetInitialValueAsync(ISynchronizationS } - protected override ISynchronizationStream? CreateStream(StreamIdentity identity) + protected override ISynchronizationStream CreateStream(StreamIdentity identity) { return CreateStream(identity, config => config.WithInitialization(GetInitialValueAsync).WithExceptionCallback(LogException)); @@ -270,10 +279,9 @@ private Task LogException(Exception exception) return Task.CompletedTask; } - protected override ISynchronizationStream? SetupDataSourceStream(StreamIdentity identity, Func, StreamConfiguration> config) + protected override ISynchronizationStream SetupDataSourceStream(StreamIdentity identity, Func, StreamConfiguration> config) { var stream = base.SetupDataSourceStream(identity, config); - if (stream == null) return null; var isFirst = true; stream.RegisterForDisposal( @@ -347,15 +355,14 @@ protected virtual async Task return initial; } - protected override ISynchronizationStream? CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) + protected override ISynchronizationStream CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) { return SetupDataSourceStream(identity, config); } - protected override ISynchronizationStream? SetupDataSourceStream(StreamIdentity identity, Func, StreamConfiguration> config) + protected override ISynchronizationStream SetupDataSourceStream(StreamIdentity identity, Func, StreamConfiguration> config) { var stream = base.SetupDataSourceStream(identity, config); - if (stream == null) return null; // Always use async initialization to call GetInitialValueAsync properly diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 546f48d80..45243df0e 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -272,9 +272,8 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat } - private static readonly Predicate StartupDeferrable = x => - x.Message is not InitializeHubRequest - && x.Message is not SetCurrentRequest && + private static bool StartupDeferrable(IMessageDelivery x) => + x.Message is not SetCurrentRequest && x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }; private async Task InitializeAsync(CancellationToken ct) diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 8e991a0fb..e72666220 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -23,7 +23,7 @@ public MessageHubConfiguration(IServiceProvider? parentServiceProvider, Address internal Predicate StartupDeferral { get; init; } = x => x.Message is not InitializeHubRequest; public MessageHubConfiguration WithStartupDeferral(Predicate startupDeferral) - => this with { StartupDeferral = startupDeferral }; + => this with { StartupDeferral = x => startupDeferral(x) && StartupDeferral(x) }; public IMessageHub? ParentHub { diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index dd1b76e71..e90676939 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -149,7 +149,7 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { logger.LogTrace("MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", name, Address, delivery.Id); - return await deliveryPipeline.Invoke(delivery, cancellationToken); + delivery = await deliveryPipeline.Invoke(delivery, cancellationToken); } return delivery; @@ -195,6 +195,8 @@ private IMessageDelivery ScheduleExecution(IMessageDelivery delivery) { delivery = await hub.HandleMessageAsync(delivery, cancellationTokenSource.Token); + if (delivery.State == MessageDeliveryState.Ignored) + ReportFailure(delivery.WithProperty("Error", $"No handler found for delivery {delivery.Message.GetType().FullName}")); } else { diff --git a/test/MeshWeaver.Serialization.Test/SerializationTest.cs b/test/MeshWeaver.Serialization.Test/SerializationTest.cs index cde5e4b04..31c46c2ec 100644 --- a/test/MeshWeaver.Serialization.Test/SerializationTest.cs +++ b/test/MeshWeaver.Serialization.Test/SerializationTest.cs @@ -300,21 +300,21 @@ public void TestGenericPolymorphicTypeSerialization() public async Task TestSerializationFailureHandling() { Output.WriteLine("Testing serialization failure handling..."); - + // This test verifies that when no handler exists for a request message type, // AwaitResponse should throw DeliveryFailureException instead of hanging - + var client = Router.GetHostedHub(new ClientAddress(), ConfigureClient); - + // Send an UnknownRequest to the host // The host has no handler for this type at all // This should result in a DeliveryFailure being sent back to the client var unknownRequest = new GetDataRequest(new EntityReference("collection", "id")); Output.WriteLine("Sending UnknownRequest to host (no handler exists for this type)..."); - + // AwaitResponse should now throw DeliveryFailureException due to no handler being found - var exception = await Assert.ThrowsAsync(() => + var exception = await Assert.ThrowsAsync(() => client.AwaitResponse( unknownRequest, o => o.WithTarget(new HostAddress()), @@ -326,10 +326,10 @@ public async Task TestSerializationFailureHandling() exception.Should().NotBeNull(); exception.Message.Should().NotBeEmpty(); Output.WriteLine($"Exception message: {exception.Message}"); - + // The message should indicate no handler was found var message = exception.Message.ToLowerInvariant(); - message.Should().Contain("could not deserialize"); + message.Should().Contain("no handler found"); } } From d7156d2c738d24782f1a0d8e5f09d7a7bedcb27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Wed, 29 Oct 2025 20:45:43 +0100 Subject: [PATCH 11/57] fixing nullability issues. --- src/MeshWeaver.Data/IWorkspace.cs | 4 ++-- src/MeshWeaver.Data/Persistence/HubDataSource.cs | 4 ++-- .../Persistence/PartitionedHubDataSource.cs | 8 ++++---- src/MeshWeaver.Data/Workspace.cs | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/MeshWeaver.Data/IWorkspace.cs b/src/MeshWeaver.Data/IWorkspace.cs index 665850108..c87628e10 100644 --- a/src/MeshWeaver.Data/IWorkspace.cs +++ b/src/MeshWeaver.Data/IWorkspace.cs @@ -19,12 +19,12 @@ public interface IWorkspace : IAsyncDisposable ISynchronizationStream GetStream(params Type[] types); ReduceManager ReduceManager { get; } - ISynchronizationStream? GetRemoteStream( + ISynchronizationStream GetRemoteStream( Address owner, WorkspaceReference reference ); ISynchronizationStream? GetStream( - WorkspaceReference reference, + WorkspaceReference reference, Func, StreamConfiguration>? configuration = null); IObservable>? GetRemoteStream(Address address); diff --git a/src/MeshWeaver.Data/Persistence/HubDataSource.cs b/src/MeshWeaver.Data/Persistence/HubDataSource.cs index 623912f8e..96c8d2016 100644 --- a/src/MeshWeaver.Data/Persistence/HubDataSource.cs +++ b/src/MeshWeaver.Data/Persistence/HubDataSource.cs @@ -15,9 +15,9 @@ public UnpartitionedHubDataSource WithType( Func, TypeSourceWithType> typeSource ) => WithTypeSource(typeof(T), typeSource.Invoke(new TypeSourceWithType(Workspace, Id))); - protected override ISynchronizationStream? CreateStream(StreamIdentity identity) => + protected override ISynchronizationStream CreateStream(StreamIdentity identity) => CreateStream(identity, x => x); - protected override ISynchronizationStream? CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) => + protected override ISynchronizationStream CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) => Workspace.GetRemoteStream(Address, GetReference()); } diff --git a/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs b/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs index 475872fcc..feec1fd59 100644 --- a/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs +++ b/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs @@ -6,13 +6,13 @@ public record PartitionedHubDataSource(object Id, IWorkspace Workspa : PartitionedDataSource, IPartitionedTypeSource, TPartition>(Id, Workspace) { public override PartitionedHubDataSource WithType(Func partitionFunction, Func? config = null) -=> WithTypeSource( +=> WithTypeSource( typeof(T), (config ?? (x => x)).Invoke( new PartitionedTypeSourceWithType(Workspace, partitionFunction, Id) ) ); - + public PartitionedHubDataSource InitializingPartitions(IEnumerable partitions) => @@ -24,10 +24,10 @@ this with private object[] InitializePartitions { get; init; } = []; - protected override ISynchronizationStream? CreateStream(StreamIdentity identity) + protected override ISynchronizationStream CreateStream(StreamIdentity identity) { if (identity.Partition is not Address partition) - return null; + throw new NotSupportedException($"Partition {identity.Partition} must be of type Address"); var reference = GetReference(); var partitionedReference = new PartitionedWorkspaceReference( partition, diff --git a/src/MeshWeaver.Data/Workspace.cs b/src/MeshWeaver.Data/Workspace.cs index bc2b67810..21a66b0e1 100644 --- a/src/MeshWeaver.Data/Workspace.cs +++ b/src/MeshWeaver.Data/Workspace.cs @@ -46,14 +46,14 @@ public Workspace(IMessageHub hub, ILogger logger) .Select(x => x.Value?.Collections.SingleOrDefault().Value?.Instances.Values.Cast().ToArray()); } - public ISynchronizationStream? GetRemoteStream( + public ISynchronizationStream GetRemoteStream( Address id, WorkspaceReference reference ) => - (ISynchronizationStream?) + (ISynchronizationStream) GetSynchronizationStreamMethod .MakeGenericMethod(typeof(TReduced), reference.GetType()) - .Invoke(this, [id, reference]); + .Invoke(this, [id, reference])!; private static readonly MethodInfo GetSynchronizationStreamMethod = From 364f54bb8171317a446a07f10c11a74a52de84c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 09:44:49 +0100 Subject: [PATCH 12/57] consolidating CollectionPlugin and ContentCollectionPlugin into ContentPlugin --- .../MeshWeaver.Insurance.AI/InsuranceAgent.cs | 32 +- .../RiskImportAgent.cs | 158 +---- .../SlipImportAgent.cs | 345 +---------- src/MeshWeaver.AI/Plugins/CollectionPlugin.cs | 498 --------------- ...ntCollectionPlugin.cs => ContentPlugin.cs} | 583 ++++++++++++++++-- ...tionPluginTest.cs => ContentPluginTest.cs} | 18 +- .../CollectionPluginImportTest.cs | 14 +- 7 files changed, 585 insertions(+), 1063 deletions(-) delete mode 100644 src/MeshWeaver.AI/Plugins/CollectionPlugin.cs rename src/MeshWeaver.AI/Plugins/{ContentCollectionPlugin.cs => ContentPlugin.cs} (54%) rename test/MeshWeaver.AI.Test/{CollectionPluginTest.cs => ContentPluginTest.cs} (95%) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs index 7c22ae701..3195c900c 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs @@ -64,24 +64,24 @@ public IEnumerable Delegations CRITICAL: When users ask about submission files, documents, or content: - DO NOT call {{{nameof(DataPlugin.GetData)}}} for Pricing or any other data first - DO NOT try to verify the pricing exists before accessing files - - The SubmissionPlugin is already configured for the current pricing context - - Simply call the SubmissionPlugin functions directly + - The ContentPlugin is already configured for the current pricing context + - Simply call the ContentPlugin functions directly - All file paths should start with "/" (e.g., "/slip.pdf", "/risks.xlsx") - Available SubmissionPlugin functions (all collectionName parameters are optional): - - {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - List all files in the current pricing's submissions - - {{{nameof(ContentCollectionPlugin.ListFolders)}}}() - List all folders - - {{{nameof(ContentCollectionPlugin.ListCollectionItems)}}}() - List both files and folders - - {{{nameof(ContentCollectionPlugin.GetDocument)}}}(documentPath) - Get document content (use path like "/Slip.md") - - {{{nameof(ContentCollectionPlugin.SaveDocument)}}}(documentPath, content) - Save a document - - {{{nameof(ContentCollectionPlugin.DeleteFile)}}}(filePath) - Delete a file - - {{{nameof(ContentCollectionPlugin.CreateFolder)}}}(folderPath) - Create a folder - - {{{nameof(ContentCollectionPlugin.DeleteFolder)}}}(folderPath) - Delete a folder + Available ContentPlugin functions (all collectionName parameters are optional): + - {{{nameof(ContentPlugin.ListFiles)}}}() - List all files in the current pricing's submissions + - {{{nameof(ContentPlugin.ListFolders)}}}() - List all folders + - {{{nameof(ContentPlugin.ListCollectionItems)}}}() - List both files and folders + - {{{nameof(ContentPlugin.GetDocument)}}}(documentPath) - Get document content (use path like "/Slip.md") + - {{{nameof(ContentPlugin.SaveFile)}}}(documentPath, content) - Save a document + - {{{nameof(ContentPlugin.DeleteFile)}}}(filePath) - Delete a file + - {{{nameof(ContentPlugin.CreateFolder)}}}(folderPath) - Create a folder + - {{{nameof(ContentPlugin.DeleteFolder)}}}(folderPath) - Delete a folder Examples: - - User: "Show me the submission files" → You: Call {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - - User: "What files are in the submissions?" → You: Call {{{nameof(ContentCollectionPlugin.ListFiles)}}}() - - User: "Read the slip document" → You: Call {{{nameof(ContentCollectionPlugin.GetDocument)}}}("/Slip.md") + - User: "Show me the submission files" → You: Call {{{nameof(ContentPlugin.ListFiles)}}}() + - User: "What files are in the submissions?" → You: Call {{{nameof(ContentPlugin.ListFiles)}}}() + - User: "Read the slip document" → You: Call {{{nameof(ContentPlugin.GetDocument)}}}("/Slip.md") ## Working with Pricing Data @@ -102,9 +102,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); yield return new LayoutAreaPlugin(hub, chat, layoutAreaMap).CreateKernelPlugin(); - // Always provide ContentCollectionPlugin - it will use ContextToConfigMap to determine the collection + // Always provide ContentPlugin - it will use ContextToConfigMap to determine the collection var submissionPluginConfig = CreateSubmissionPluginConfig(chat); - yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); + yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index 3870f7eff..f0b4529d7 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -1,5 +1,4 @@ -using System.ComponentModel; -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Nodes; using MeshWeaver.AI; using MeshWeaver.AI.Plugins; @@ -39,19 +38,19 @@ public string Instructions # Importing Risks When the user asks you to import risks, you should: - 1) Get the existing risk mapping configuration for the specified file using the function {{{nameof(RiskImportPlugin.GetRiskImportConfiguration)}}} with the filename. - 2) If no import configuration was returned in 1, get a sample of the worksheet using CollectionPlugin's GetFile function with the collection name "Submissions-{pricingId}", the filename, and numberOfRows=20. Extract the table start row as well as the mapping as in the schema provided below. - Consider any input from the user to modify the configuration. Use the {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} function to save the configuration. - 3) Call Import with the filename and the configuration you have updated or created. + 1) Get the existing risk mapping configuration for the specified file using DataPlugin's GetData function with type="ExcelImportConfiguration" and entityId=filename. + 2) If no import configuration was returned in 1, get a sample of the worksheet using ContentPlugin's GetFile function with the collection name "Submissions-{pricingId}", the filename, and numberOfRows=20. Extract the table start row as well as the mapping as in the schema provided below. + Consider any input from the user to modify the configuration. Ensure the JSON includes "name" field set to the filename. Use DataPlugin's UpdateData function with type="ExcelImportConfiguration" to save the configuration. + 3) Call ContentPlugin's Import function with path=filename, collection="Submissions-{pricingId}", address=PricingAddress, and configuration=the JSON configuration you created or retrieved. # Updating Risk Import Configuration When the user asks you to update the risk import configuration, you should: - 1) Get the existing risk mapping configuration for the specified file using the function {{{nameof(RiskImportPlugin.GetRiskImportConfiguration)}}} with the filename. + 1) Get the existing risk mapping configuration for the specified file using DataPlugin's GetData function with type="ExcelImportConfiguration" and entityId=filename. 2) Modify it according to the user's input, ensuring it follows the schema provided below. - 3) Upload the new configuration using the function {{{nameof(RiskImportPlugin.UpdateRiskImportConfiguration)}}} with the filename and the updated mapping. + 3) Upload the new configuration using DataPlugin's UpdateData function with type="ExcelImportConfiguration" and the updated JSON (ensure "name" field is set to filename). # Automatic Risk Import Configuration - - Use CollectionPlugin's GetFile with numberOfRows=20 to get a sample of the file. It returns a markdown table with: + - Use ContentPlugin's GetFile with numberOfRows=20 to get a sample of the file. It returns a markdown table with: - First column: Row numbers (1-based) - Remaining columns: Labeled A, B, C, D, etc. (Excel column letters) - Empty cells appear as empty values in the table (not "null") @@ -76,7 +75,7 @@ public string Instructions IMPORTANT OUTPUT RULES: - do not output JSON to the user. - - When the user asks you to import, your job is not finished by creating the risk import configuration. You will actually have to call import. + - When the user asks you to import, your job is not finished by creating the risk import configuration. You will actually have to call ContentPlugin's Import function. """; if (excelImportConfigSchema is not null) @@ -97,17 +96,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) { yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); - // Add ContentCollectionPlugin for submissions + // Add ContentPlugin for submissions and import functionality var submissionPluginConfig = CreateSubmissionPluginConfig(); - yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); - - // Add CollectionPlugin for import functionality - var collectionPlugin = new CollectionPlugin(hub); - yield return KernelPluginFactory.CreateFromObject(collectionPlugin); - - // Add risk import specific plugin - var plugin = new RiskImportPlugin(hub, chat); - yield return KernelPluginFactory.CreateFromObject(plugin); + yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } private static ContentCollectionPluginConfig CreateSubmissionPluginConfig() @@ -210,130 +201,3 @@ async Task IInitializableAgent.InitializeAsync() } } } - -public class RiskImportPlugin(IMessageHub hub, IAgentChat chat) -{ - private JsonSerializerOptions GetJsonOptions() - { - return hub.JsonSerializerOptions; - } - - [KernelFunction] - [Description("Imports a file with filename")] - public async Task Import(string filename) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing for which you want to import risks."; - - var pricingId = chat.Context.Address.Id; - var collectionName = $"Submissions-{pricingId}"; - var address = new PricingAddress(pricingId); - - // Try to get the saved configuration for this file - string? configuration = null; - try - { - var configJson = await GetRiskImportConfiguration(filename); - // Check if we got a valid configuration (not an error message) - if (!configJson.StartsWith("Error") && !configJson.StartsWith("Please navigate")) - { - configuration = configJson; - } - } - catch - { - // If we can't get the configuration, fall back to format-based import - } - - // Delegate to CollectionPlugin's Import method - var collectionPlugin = new CollectionPlugin(hub); - return await collectionPlugin.Import( - path: filename, - collection: collectionName, - address: address, - format: configuration != null ? null : "PropertyRiskImport", // Use format only if no configuration - configuration: configuration // Pass configuration if available - ); - } - - [KernelFunction] - [Description("Gets the risk configuration for a particular file")] - public async Task GetRiskImportConfiguration(string filename) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing for which you want to create a risk import mapping."; - - try - { - var response = await hub.AwaitResponse( - new GetDataRequest(new EntityReference("ExcelImportConfiguration", filename)), - o => o.WithTarget(new PricingAddress(chat.Context.Address.Id)) - ); - - // Serialize the data - var json = JsonSerializer.Serialize(response?.Message?.Data, hub.JsonSerializerOptions); - - // Parse and ensure $type is set to ExcelImportConfiguration - var jsonObject = JsonNode.Parse(json) as JsonObject; - if (jsonObject != null) - { - var withType = EnsureTypeFirst(jsonObject, "ExcelImportConfiguration"); - return JsonSerializer.Serialize(withType, hub.JsonSerializerOptions); - } - - return json; - } - catch (Exception e) - { - return $"Error processing file '{filename}': {e.Message}"; - } - } - - [KernelFunction] - [Description("Updates the mapping configuration for risks import")] - public async Task UpdateRiskImportConfiguration( - string filename, - [Description("Needs to follow the schema provided in the system prompt")] string mappingJson) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing for which you want to update the risk import configuration."; - - var pa = new PricingAddress(chat.Context.Address.Id); - if (string.IsNullOrWhiteSpace(mappingJson)) - return "Json mapping is empty. Please provide valid JSON."; - - try - { - var parsed = EnsureTypeFirst((JsonObject)JsonNode.Parse(ExtractJson(mappingJson))!, "ExcelImportConfiguration"); - parsed["entityId"] = pa.Id; - parsed["name"] = filename; - var response = await hub.AwaitResponse(new DataChangeRequest() { Updates = [parsed] }, o => o.WithTarget(pa)); - return JsonSerializer.Serialize(response?.Message, hub.JsonSerializerOptions); - } - catch (Exception e) - { - return $"Mapping JSON is invalid. Please provide valid JSON. Exception: {e.Message}"; - } - } - - private static JsonObject EnsureTypeFirst(JsonObject source, string typeName) - { - var ordered = new JsonObject - { - ["$type"] = typeName - }; - foreach (var kv in source) - { - if (string.Equals(kv.Key, "$type", StringComparison.Ordinal)) continue; - ordered[kv.Key] = kv.Value?.DeepClone(); - } - return ordered; - } - - private string ExtractJson(string json) - { - return json.Replace("```json", "") - .Replace("```", "") - .Trim(); - } -} diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index b964ae98b..0139f0a97 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -1,17 +1,9 @@ -using System.ComponentModel; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using iText.Kernel.Pdf; -using iText.Kernel.Pdf.Canvas.Parser; -using iText.Kernel.Pdf.Canvas.Parser.Listener; -using MeshWeaver.AI; +using MeshWeaver.AI; using MeshWeaver.AI.Plugins; using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.Insurance.Domain; using MeshWeaver.Messaging; -using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; namespace MeshWeaver.Insurance.AI; @@ -46,13 +38,18 @@ You are a slip import agent that processes insurance submission slip documents i # Importing Slips When the user asks you to import a slip: - 1) First, use {{{nameof(ContentCollectionPlugin.ListFiles)}}}() to see available files in the submissions collection - 2) Use {{{nameof(SlipImportPlugin.ExtractCompleteText)}}} to extract the document content from PDF or Markdown files - - Simply pass the filename (e.g., "Slip.pdf" or "Slip.md") - - The collection name will be automatically resolved to "Submissions-{pricingId}" + 1) First, use ContentCollectionPlugin's ListFiles() to see available files in the submissions collection + 2) Use ContentPlugin's GetFile function to extract the document content from PDF or Markdown files + - Pass collectionName="Submissions-{pricingId}" and filePath=filename (e.g., "Slip.pdf" or "Slip.md") + - For PDFs, this will extract all pages of text 3) Review the extracted text and identify data that matches the domain schemas - 4) Use {{{nameof(SlipImportPlugin.ImportSlipData)}}} to save the structured data as JSON - 5) Provide feedback on what data was successfully imported or if any issues were encountered + 4) Create JSON objects for each entity type following the schemas below + 5) Import the data using DataPlugin's UpdateData function: + - First, retrieve existing Pricing data using DataPlugin's GetData with type="Pricing" and entityId=pricingId + - Merge new pricing fields with existing data and call DataPlugin's UpdateData with type="Pricing" + - For each ReinsuranceAcceptance (layer), create JSON and call DataPlugin's UpdateData with type="ReinsuranceAcceptance" + - For each ReinsuranceSection (coverage within layer), create JSON and call DataPlugin's UpdateData with type="ReinsuranceSection" + 6) Provide feedback on what data was successfully imported or if any issues were encountered # Data Mapping Guidelines Based on the extracted document text, create JSON objects that match the schemas provided below: @@ -119,9 +116,10 @@ You are a slip import agent that processes insurance submission slip documents i Notes: - When listing files, you may see paths with "/" prefix (e.g., "/Slip.pdf", "/Slip.md") - - When calling ExtractCompleteText, provide only the filename (e.g., "Slip.pdf" or "Slip.md") - - The collection name is automatically determined from the pricing context + - When calling ContentPlugin's GetFile, use collectionName="Submissions-{pricingId}" and provide the filename - Both PDF and Markdown (.md) files are supported + - When updating data, ensure each JSON object has the correct $type field and required ID fields (id, pricingId, acceptanceId, etc.) + - Remove null-valued properties from JSON before calling UpdateData """; if (pricingSchema is not null) @@ -139,13 +137,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) { yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); - // Add ContentCollectionPlugin for submissions + // Add ContentPlugin for submissions and file reading functionality var submissionPluginConfig = CreateSubmissionPluginConfig(chat); - yield return new ContentCollectionPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); - - // Add slip import specific plugin - var plugin = new SlipImportPlugin(hub, chat); - yield return KernelPluginFactory.CreateFromObject(plugin); + yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) @@ -240,308 +234,3 @@ public bool Matches(AgentContext? context) return context?.Address?.Type == PricingAddress.TypeName; } } - -public class SlipImportPlugin(IMessageHub hub, IAgentChat chat) -{ - private JsonSerializerOptions GetJsonOptions() - { - return hub.JsonSerializerOptions; - } - - [KernelFunction] - [Description("Extracts the complete text from a slip document (PDF or Markdown) and returns it for LLM processing")] - public async Task ExtractCompleteText( - [Description("The filename to extract (e.g., 'Slip.pdf' or 'Slip.md')")] string filename, - [Description("The collection name (optional, defaults to context-based resolution)")] string? collectionName = null) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing first."; - - try - { - var pricingId = chat.Context.Address.Id; - var contentService = hub.ServiceProvider.GetRequiredService(); - - // Get collection name using the same pattern as ContentCollectionPlugin - var resolvedCollectionName = collectionName ?? $"Submissions-{pricingId}"; - - // Use ContentService directly with the correct collection name and simple path - var stream = await contentService.GetContentAsync(resolvedCollectionName, filename, CancellationToken.None); - - if (stream is null) - return $"Content not found: {filename} in collection {resolvedCollectionName}"; - - await using (stream) - { - string completeText; - - // Determine file type by extension - if (filename.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) - { - // Read markdown file directly as text - using var reader = new StreamReader(stream); - completeText = await reader.ReadToEndAsync(); - } - else if (filename.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase)) - { - // Extract text from PDF - completeText = await ExtractCompletePdfText(stream); - } - else - { - // Try to read as text for unknown file types - using var reader = new StreamReader(stream); - completeText = await reader.ReadToEndAsync(); - } - - var sb = new StringBuilder(); - sb.AppendLine("=== INSURANCE SLIP DOCUMENT TEXT ==="); - sb.AppendLine(); - sb.AppendLine(completeText); - - return sb.ToString(); - } - } - catch (Exception e) - { - return $"Error extracting document text: {e.Message}"; - } - } - - [KernelFunction] - [Description("Imports the structured slip data as JSON into the pricing")] - public async Task ImportSlipData( - [Description("Pricing data as JSON (optional if updating existing)")] string? pricingJson, - [Description("Array of ReinsuranceAcceptance data as JSON (can contain multiple acceptances)")] string? acceptancesJson, - [Description("Array of ReinsuranceSection data as JSON (sections/layers within acceptances)")] string? sectionsJson) - { - if (chat.Context?.Address?.Type != PricingAddress.TypeName) - return "Please navigate to the pricing first."; - - var pricingId = chat.Context.Address.Id; - var pricingAddress = new PricingAddress(pricingId); - - try - { - // Step 1: Retrieve existing Pricing data - var existingPricing = await GetExistingPricingAsync(pricingAddress, pricingId); - - var updates = new List(); - - // Step 2: Update Pricing if provided - if (!string.IsNullOrWhiteSpace(pricingJson)) - { - var newPricingData = JsonNode.Parse(ExtractJson(pricingJson)); - if (newPricingData is JsonObject newPricingObj) - { - var mergedPricing = MergeWithExistingPricing(existingPricing, newPricingObj, pricingId); - RemoveNullProperties(mergedPricing); - updates.Add(mergedPricing); - } - } - - // Step 3: Process ReinsuranceAcceptance records (can be multiple) - if (!string.IsNullOrWhiteSpace(acceptancesJson)) - { - var acceptancesData = JsonNode.Parse(ExtractJson(acceptancesJson)); - - // Handle both array and single object - var acceptanceArray = acceptancesData is JsonArray arr ? arr : new JsonArray { acceptancesData }; - - foreach (var acceptanceData in acceptanceArray) - { - if (acceptanceData is JsonObject acceptanceObj) - { - var processedAcceptance = EnsureTypeFirst(acceptanceObj, nameof(ReinsuranceAcceptance)); - processedAcceptance["pricingId"] = pricingId; - RemoveNullProperties(processedAcceptance); - updates.Add(processedAcceptance); - } - } - } - - // Step 4: Process ReinsuranceSection records (can be multiple) - if (!string.IsNullOrWhiteSpace(sectionsJson)) - { - var sectionsData = JsonNode.Parse(ExtractJson(sectionsJson)); - - // Handle both array and single object - var sectionArray = sectionsData is JsonArray arr ? arr : new JsonArray { sectionsData }; - - foreach (var sectionData in sectionArray) - { - if (sectionData is JsonObject sectionObj) - { - var processedSection = EnsureTypeFirst(sectionObj, nameof(ReinsuranceSection)); - RemoveNullProperties(processedSection); - updates.Add(processedSection); - } - } - } - - if (updates.Count == 0) - return "No valid data provided for import."; - - // Step 5: Post DataChangeRequest - var updateRequest = new DataChangeRequest { Updates = updates }; - var response = await hub.AwaitResponse(updateRequest, o => o.WithTarget(pricingAddress)); - - return response.Message.Status switch - { - DataChangeStatus.Committed => $"Slip data imported successfully. Updated {updates.Count} entities.", - _ => $"Data update failed:\n{string.Join('\n', response.Message.Log.Messages?.Select(l => l.LogLevel + ": " + l.Message) ?? Array.Empty())}" - }; - } - catch (Exception e) - { - return $"Import failed: {e.Message}"; - } - } - - private async Task GetExistingPricingAsync(Address pricingAddress, string pricingId) - { - try - { - var response = await hub.AwaitResponse( - new GetDataRequest(new EntityReference(nameof(Pricing), pricingId)), - o => o.WithTarget(pricingAddress)); - - return response?.Message?.Data as Pricing; - } - catch - { - return null; - } - } - - private JsonObject MergeWithExistingPricing(Pricing? existing, JsonObject newData, string pricingId) - { - JsonObject baseData; - if (existing != null) - { - var existingJson = JsonSerializer.Serialize(existing, GetJsonOptions()); - baseData = JsonNode.Parse(existingJson) as JsonObject ?? new JsonObject(); - } - else - { - baseData = new JsonObject(); - } - - var merged = MergeJsonObjects(baseData, newData); - merged = EnsureTypeFirst(merged, nameof(Pricing)); - merged["id"] = pricingId; - return merged; - } - - private static JsonObject MergeJsonObjects(JsonObject? existing, JsonObject? newData) - { - if (existing == null) - return newData?.DeepClone() as JsonObject ?? new JsonObject(); - - if (newData == null) - return existing.DeepClone() as JsonObject ?? new JsonObject(); - - var merged = existing.DeepClone() as JsonObject ?? new JsonObject(); - - foreach (var kvp in newData) - { - var isNullValue = kvp.Value == null || - (kvp.Value?.GetValueKind() == System.Text.Json.JsonValueKind.String && - kvp.Value.GetValue() == "null"); - - if (!isNullValue) - { - if (merged.ContainsKey(kvp.Key) && - merged[kvp.Key] is JsonObject existingObj && - kvp.Value is JsonObject newObj) - { - merged[kvp.Key] = MergeJsonObjects(existingObj, newObj); - } - else - { - merged[kvp.Key] = kvp.Value?.DeepClone(); - } - } - } - - return merged; - } - - private static JsonObject EnsureTypeFirst(JsonObject source, string typeName) - { - var ordered = new JsonObject - { - ["$type"] = typeName - }; - foreach (var kv in source) - { - if (string.Equals(kv.Key, "$type", StringComparison.Ordinal)) continue; - ordered[kv.Key] = kv.Value?.DeepClone(); - } - return ordered; - } - - private static void RemoveNullProperties(JsonNode? node) - { - if (node is JsonObject obj) - { - foreach (var kvp in obj.ToList()) - { - var value = kvp.Value; - if (value is null) - { - obj.Remove(kvp.Key); - } - else - { - RemoveNullProperties(value); - } - } - } - else if (node is JsonArray arr) - { - foreach (var item in arr) - { - RemoveNullProperties(item); - } - } - } - - private async Task ExtractCompletePdfText(Stream stream) - { - var completeText = new StringBuilder(); - - try - { - using var pdfReader = new PdfReader(stream); - using var pdfDocument = new PdfDocument(pdfReader); - - for (int pageNum = 1; pageNum <= pdfDocument.GetNumberOfPages(); pageNum++) - { - var page = pdfDocument.GetPage(pageNum); - var strategy = new SimpleTextExtractionStrategy(); - var pageText = PdfTextExtractor.GetTextFromPage(page, strategy); - - if (!string.IsNullOrWhiteSpace(pageText)) - { - completeText.AppendLine($"=== PAGE {pageNum} ==="); - completeText.AppendLine(pageText.Trim()); - completeText.AppendLine(); - } - } - } - catch (Exception ex) - { - completeText.AppendLine($"Error extracting PDF: {ex.Message}"); - } - - return completeText.ToString(); - } - - private string ExtractJson(string json) - { - return json.Replace("```json", "") - .Replace("```", "") - .Trim(); - } -} diff --git a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs deleted file mode 100644 index e538e2785..000000000 --- a/src/MeshWeaver.AI/Plugins/CollectionPlugin.cs +++ /dev/null @@ -1,498 +0,0 @@ -using System.ComponentModel; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using ClosedXML.Excel; -using DocumentFormat.OpenXml.Packaging; -using DocumentFormat.OpenXml.Wordprocessing; -using MeshWeaver.ContentCollections; -using MeshWeaver.Messaging; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.SemanticKernel; -using UglyToad.PdfPig; - -namespace MeshWeaver.AI.Plugins; - -/// -/// Generalized plugin for reading and writing files to configured collections -/// -public class CollectionPlugin(IMessageHub hub) -{ - private readonly IContentService contentService = hub.ServiceProvider.GetRequiredService(); - - [KernelFunction] - [Description("Gets the content of a file from a specified collection.")] - public async Task GetFile( - [Description("The name of the collection to read from. If null, uses the default collection.")] string collectionName, - [Description("The path to the file within the collection")] string filePath, - [Description("Optional: number of rows to read. If null, reads entire file. For Excel files, reads first N rows from each worksheet.")] int? numberOfRows = null, - CancellationToken cancellationToken = default) - { - try - { - var collection = await contentService.GetCollectionAsync(collectionName, cancellationToken); - if (collection == null) - return $"Collection '{collectionName}' not found."; - - await using var stream = await collection.GetContentAsync(filePath, cancellationToken); - if (stream == null) - return $"File '{filePath}' not found in collection '{collectionName}'."; - - // Check file type and read accordingly - var extension = Path.GetExtension(filePath).ToLowerInvariant(); - if (extension == ".xlsx" || extension == ".xls") - { - return await ReadExcelFileAsync(stream, filePath, numberOfRows); - } - else if (extension == ".docx") - { - return await ReadWordFileAsync(stream, filePath, numberOfRows); - } - else if (extension == ".pdf") - { - return await ReadPdfFileAsync(stream, filePath, numberOfRows); - } - - // For other files, read as text - using var reader = new StreamReader(stream); - if (numberOfRows.HasValue) - { - var sb = new StringBuilder(); - var linesRead = 0; - while (!reader.EndOfStream && linesRead < numberOfRows.Value) - { - var line = await reader.ReadLineAsync(cancellationToken); - sb.AppendLine(line); - linesRead++; - } - return sb.ToString(); - } - else - { - var content = await reader.ReadToEndAsync(cancellationToken); - return content; - } - } - catch (FileNotFoundException) - { - return $"File '{filePath}' not found in collection '{collectionName}'."; - } - catch (Exception ex) - { - return $"Error reading file '{filePath}' from collection '{collectionName}': {ex.Message}"; - } - } - - private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows) - { - try - { - using var wb = new XLWorkbook(stream); - var sb = new StringBuilder(); - - foreach (var ws in wb.Worksheets) - { - var used = ws.RangeUsed(); - sb.AppendLine($"## Sheet: {ws.Name}"); - sb.AppendLine(); - if (used is null) - { - sb.AppendLine("(No data)"); - sb.AppendLine(); - continue; - } - - var firstRow = used.FirstRow().RowNumber(); - var lastRow = numberOfRows.HasValue - ? Math.Min(used.FirstRow().RowNumber() + numberOfRows.Value - 1, used.LastRow().RowNumber()) - : used.LastRow().RowNumber(); - var firstCol = 1; - var lastCol = used.LastColumn().ColumnNumber(); - - // Build markdown table with column letters as headers - var columnHeaders = new List { "Row" }; - for (var c = firstCol; c <= lastCol; c++) - { - // Convert column number to Excel letter (1=A, 2=B, ..., 27=AA, etc.) - columnHeaders.Add(GetExcelColumnLetter(c)); - } - - // Header row - sb.AppendLine("| " + string.Join(" | ", columnHeaders) + " |"); - // Separator row - sb.AppendLine("|" + string.Join("", columnHeaders.Select(_ => "---:|"))); - - // Data rows - for (var r = firstRow; r <= lastRow; r++) - { - var rowVals = new List { r.ToString() }; - for (var c = firstCol; c <= lastCol; c++) - { - var cell = ws.Cell(r, c); - var raw = cell.GetValue(); - var val = raw?.Replace('\n', ' ').Replace('\r', ' ').Replace("|", "\\|").Trim(); - // Empty cells show as empty in table - rowVals.Add(string.IsNullOrEmpty(val) ? "" : val); - } - - sb.AppendLine("| " + string.Join(" | ", rowVals) + " |"); - } - - sb.AppendLine(); - } - - return sb.ToString(); - } - catch (Exception ex) - { - return $"Error reading Excel file '{filePath}': {ex.Message}"; - } - } - - private static string GetExcelColumnLetter(int columnNumber) - { - var columnLetter = ""; - while (columnNumber > 0) - { - var modulo = (columnNumber - 1) % 26; - columnLetter = Convert.ToChar('A' + modulo) + columnLetter; - columnNumber = (columnNumber - 1) / 26; - } - return columnLetter; - } - - private async Task ReadWordFileAsync(Stream stream, string filePath, int? numberOfRows) - { - try - { - using var wordDoc = WordprocessingDocument.Open(stream, false); - var body = wordDoc.MainDocumentPart?.Document?.Body; - - if (body == null) - return $"Word document '{filePath}' has no readable content."; - - var sb = new StringBuilder(); - sb.AppendLine($"# Document: {Path.GetFileName(filePath)}"); - sb.AppendLine(); - - var paragraphs = body.Elements().ToList(); - var paragraphsToRead = numberOfRows.HasValue - ? paragraphs.Take(numberOfRows.Value).ToList() - : paragraphs; - - foreach (var paragraph in paragraphsToRead) - { - var text = paragraph.InnerText; - if (!string.IsNullOrWhiteSpace(text)) - { - sb.AppendLine(text); - sb.AppendLine(); - } - } - - // Also handle tables - var tables = body.Elements
().ToList(); - foreach (var table in tables) - { - sb.AppendLine("## Table"); - sb.AppendLine(); - - var rows = table.Elements().ToList(); - var rowsToRead = numberOfRows.HasValue - ? rows.Take(numberOfRows.Value).ToList() - : rows; - - foreach (var row in rowsToRead) - { - var cells = row.Elements().ToList(); - var cellTexts = cells.Select(c => c.InnerText.Replace('|', '\\').Trim()).ToList(); - sb.AppendLine("| " + string.Join(" | ", cellTexts) + " |"); - } - - sb.AppendLine(); - } - - return sb.ToString(); - } - catch (Exception ex) - { - return $"Error reading Word document '{filePath}': {ex.Message}"; - } - } - - private async Task ReadPdfFileAsync(Stream stream, string filePath, int? numberOfRows) - { - try - { - using var pdfDocument = PdfDocument.Open(stream); - var sb = new StringBuilder(); - sb.AppendLine($"# PDF Document: {Path.GetFileName(filePath)}"); - sb.AppendLine($"Total pages: {pdfDocument.NumberOfPages}"); - sb.AppendLine(); - - var pagesToRead = numberOfRows.HasValue - ? Math.Min(numberOfRows.Value, pdfDocument.NumberOfPages) - : pdfDocument.NumberOfPages; - - for (int pageNum = 1; pageNum <= pagesToRead; pageNum++) - { - var page = pdfDocument.GetPage(pageNum); - sb.AppendLine($"## Page {pageNum}"); - sb.AppendLine(); - - var text = page.Text; - if (!string.IsNullOrWhiteSpace(text)) - { - sb.AppendLine(text); - } - else - { - sb.AppendLine("(No text content)"); - } - sb.AppendLine(); - } - - return sb.ToString(); - } - catch (Exception ex) - { - return $"Error reading PDF document '{filePath}': {ex.Message}"; - } - } - - [KernelFunction] - [Description("Saves content as a file to a specified collection.")] - public async Task SaveFile( - [Description("The name of the collection to save to")] string collectionName, - [Description("The path where the file should be saved within the collection")] string filePath, - [Description("The content to save to the file")] string content, - CancellationToken cancellationToken = default) - { - try - { - var collection = await contentService.GetCollectionAsync(collectionName, cancellationToken); - if (collection == null) - return $"Collection '{collectionName}' not found."; // Ensure directory structure exists if the collection has a base path - EnsureDirectoryExists(collection, filePath); - - // Extract directory and filename components - var directoryPath = Path.GetDirectoryName(filePath) ?? ""; - var fileName = Path.GetFileName(filePath); - - await using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); - await collection.SaveFileAsync(directoryPath, fileName, stream); - - return $"File '{filePath}' successfully saved to collection '{collectionName}'. Full path: {directoryPath}/{fileName}"; - } - catch (Exception ex) - { - return $"Error saving file '{filePath}' to collection '{collectionName}': {ex.Message}"; - } - } - - [KernelFunction] - [Description("Lists all files in a specified collection.")] - public async Task ListFiles( - [Description("The name of the collection to list files from")] string collectionName, - [Description("The path for which to load files.")] string path = "/", - CancellationToken cancellationToken = default) - { - try - { - var collection = await contentService.GetCollectionAsync(collectionName, cancellationToken); - if (collection == null) - return $"Collection '{collectionName}' not found."; - - var files = await collection.GetFilesAsync(path); - var fileList = files.Select(f => new { f.Name, f.Path }).ToList(); - - if (!fileList.Any()) - return $"No files found in collection '{collectionName}'."; - - return string.Join("\n", fileList.Select(f => $"- {f.Name} ({f.Path})")); - } - catch (Exception ex) - { - return $"Error listing files in collection '{collectionName}': {ex.Message}"; - } - } - - [KernelFunction] - [Description("Checks if a specific file exists in a collection.")] - public async Task FileExists( - [Description("The name of the collection to check")] string collectionName, - [Description("The path to the file within the collection")] string filePath, - CancellationToken cancellationToken = default) - { - try - { - var collection = await contentService.GetCollectionAsync(collectionName, cancellationToken); - if (collection == null) - return $"Collection '{collectionName}' not found."; - - await using var stream = await collection.GetContentAsync(filePath, cancellationToken); - if (stream == null) - return $"File '{filePath}' does not exist in collection '{collectionName}'."; - - return $"File '{filePath}' exists in collection '{collectionName}'."; - } - catch (FileNotFoundException) - { - return $"File '{filePath}' does not exist in collection '{collectionName}'."; - } - catch (Exception ex) - { - return $"Error checking file '{filePath}' in collection '{collectionName}': {ex.Message}"; - } - } - - [KernelFunction] - [Description("Generates a unique filename with timestamp for saving temporary files.")] - public string GenerateUniqueFileName( - [Description("The base name for the file (without extension)")] string baseName, - [Description("The file extension (e.g., 'json', 'txt')")] string extension) - { - var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss_fff"); - return $"{baseName}_{timestamp}.{extension.TrimStart('.')}"; - } - - [KernelFunction] - [Description("Imports data from a file in a collection to a specified address.")] - public async Task Import( - [Description("The path to the file to import")] string path, - [Description("The name of the collection containing the file (optional if default collection is configured)")] string? collection = null, - [Description("The target address for the import (optional if default address is configured), can be a string like 'AddressType/id' or an Address object")] object? address = null, - [Description("The import format to use (optional, defaults to 'Default')")] string? format = null, - [Description("Optional import configuration as JSON string. When provided, this will be used instead of the format parameter.")] string? configuration = null, - CancellationToken cancellationToken = default) - { - try - { - if (string.IsNullOrWhiteSpace(collection)) - return "Collection name is required."; - - if (address == null) - return "Target address is required."; - - // Parse the address - handle both string and Address types - Address targetAddress; - if (address is string addressString) - { - targetAddress = hub.GetAddress(addressString); - } - else if (address is Address addr) - { - targetAddress = addr; - } - else - { - return $"Invalid address type: {address.GetType().Name}. Expected string or Address."; - } - - // Build ImportRequest JSON structure - var importRequestJson = new JsonObject - { - ["$type"] = "MeshWeaver.Import.ImportRequest", - ["source"] = new JsonObject - { - ["$type"] = "MeshWeaver.Import.CollectionSource", - ["collection"] = collection, - ["path"] = path - }, - ["format"] = format ?? "Default" - }; - - // Add configuration if provided - if (!string.IsNullOrWhiteSpace(configuration)) - { - var configNode = JsonNode.Parse(configuration); - if (configNode != null) - { - importRequestJson["configuration"] = configNode; - } - } - - // Serialize and deserialize through hub's serializer to get proper type - var jsonString = importRequestJson.ToJsonString(); - var importRequestObj = JsonSerializer.Deserialize(jsonString, hub.JsonSerializerOptions)!; - - // Post the request to the hub - var responseMessage = await hub.AwaitResponse( - importRequestObj, - o => o.WithTarget(targetAddress), - cancellationToken - ); - - // Serialize the response back to JSON for processing - var responseJson = JsonSerializer.Serialize(responseMessage, hub.JsonSerializerOptions); - var responseObj = JsonNode.Parse(responseJson)!; - - var log = responseObj["log"] as JsonObject; - var status = log?["status"]?.ToString() ?? "Unknown"; - var messages = log?["messages"] as JsonArray ?? new JsonArray(); - - var result = $"Import {status.ToLower()}.\n"; - if (messages.Count > 0) - { - result += "Log messages:\n"; - foreach (var msg in messages) - { - if (msg is JsonObject msgObj) - { - var level = msgObj["logLevel"]?.ToString() ?? "Info"; - var message = msgObj["message"]?.ToString() ?? ""; - result += $" [{level}] {message}\n"; - } - } - } - - return result; - } - catch (Exception ex) - { - return $"Error importing file '{path}' from collection '{collection}' to address '{address}': {ex.Message}"; - } - } - - /// - /// Ensures that the directory structure exists for the given file path within the collection. - /// - /// The collection to check - /// The file path that may contain directories - private void EnsureDirectoryExists(object collection, string filePath) - { - try - { - // Normalize path separators and get the directory path from the file path - var normalizedPath = filePath.Replace('/', Path.DirectorySeparatorChar); - var directoryPath = Path.GetDirectoryName(normalizedPath); - - if (string.IsNullOrEmpty(directoryPath) || directoryPath == "." || directoryPath == Path.DirectorySeparatorChar.ToString()) - { - // No directory structure needed, file is in root - return; - } - - // Try to get the collection's base path using reflection if available - var collectionType = collection.GetType(); - var basePathProperty = collectionType.GetProperty("BasePath") ?? - collectionType.GetProperty("Path") ?? - collectionType.GetProperty("RootPath"); - - if (basePathProperty != null) - { - var basePath = basePathProperty.GetValue(collection) as string; - if (!string.IsNullOrEmpty(basePath)) - { - var fullDirectoryPath = Path.Combine(basePath, directoryPath); - Directory.CreateDirectory(fullDirectoryPath); - } - } - } - catch (Exception) - { - // If we can't create directories through reflection, - // let the SaveFileAsync method handle any directory creation or fail gracefully - } - } -} diff --git a/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs similarity index 54% rename from src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs rename to src/MeshWeaver.AI/Plugins/ContentPlugin.cs index e47fb7f4f..34480d38d 100644 --- a/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs @@ -2,16 +2,22 @@ using System.Reactive.Linq; using System.Reflection; using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ClosedXML.Excel; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Wordprocessing; using MeshWeaver.ContentCollections; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.SemanticKernel; +using UglyToad.PdfPig; namespace MeshWeaver.AI.Plugins; /// -/// Plugin for managing documents and files in content collections. -/// Provides functions for listing, loading, saving, and deleting documents. +/// Generalized plugin for reading and writing files to configured collections. +/// Supports context resolution via LayoutAreaReference and dynamic collection configuration. /// /// Context Resolution: /// - When LayoutAreaReference.Area is "Content" or "Collection" @@ -31,14 +37,27 @@ namespace MeshWeaver.AI.Plugins; /// - Parsed collection: "Documents" /// - Parsed path: "/" (root) /// -public class ContentCollectionPlugin +public class ContentPlugin { + private readonly IMessageHub hub; private readonly IContentService contentService; private readonly ContentCollectionPluginConfig config; - private readonly IAgentChat chat; + private readonly IAgentChat? chat; - public ContentCollectionPlugin(IMessageHub hub, ContentCollectionPluginConfig config, IAgentChat chat) + /// + /// Creates a ContentPlugin with basic functionality (no context resolution). + /// + public ContentPlugin(IMessageHub hub) + : this(hub, new ContentCollectionPluginConfig { Collections = [] }, null!) + { + } + + /// + /// Creates a ContentPlugin with context resolution and dynamic collection configuration. + /// + public ContentPlugin(IMessageHub hub, ContentCollectionPluginConfig config, IAgentChat chat) { + this.hub = hub; this.config = config; this.chat = chat; contentService = hub.ServiceProvider.GetRequiredService(); @@ -62,6 +81,9 @@ public ContentCollectionPlugin(IMessageHub hub, ContentCollectionPluginConfig co if (!string.IsNullOrEmpty(collectionName)) return collectionName; + if (chat == null) + return null; + // Only parse from LayoutAreaReference.Id when area is "Content" or "Collection" if (chat.Context?.LayoutArea != null) { @@ -105,7 +127,7 @@ public ContentCollectionPlugin(IMessageHub hub, ContentCollectionPluginConfig co /// private string? GetPathFromContext() { - if (chat.Context?.LayoutArea == null) + if (chat?.Context?.LayoutArea == null) return null; var area = chat.Context.LayoutArea.Area?.ToString(); @@ -127,31 +149,292 @@ public ContentCollectionPlugin(IMessageHub hub, ContentCollectionPluginConfig co } [KernelFunction] - [Description("Lists all available collections with their configurations.")] - public Task GetCollections() + [Description("Gets the content of a file from a specified collection. Supports Excel, Word, PDF, and text files. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] + public async Task GetFile( + [Description("The path to the file within the collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/'); else null.")] string? filePath = null, + [Description("The name of the collection to read from. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, + [Description("Optional: number of rows to read. If null, reads entire file. For Excel files, reads first N rows from each worksheet.")] int? numberOfRows = null, + CancellationToken cancellationToken = default) { + var resolvedCollectionName = GetCollectionName(collectionName); + if (string.IsNullOrEmpty(resolvedCollectionName)) + return "No collection specified and no default collection configured."; + + var resolvedFilePath = filePath ?? GetPathFromContext(); + if (string.IsNullOrEmpty(resolvedFilePath)) + return "No file path specified and no path found in context."; + try { - if (config.Collections.Count == 0) - return Task.FromResult("No collections configured."); + var collection = await contentService.GetCollectionAsync(resolvedCollectionName, cancellationToken); + if (collection == null) + return $"Collection '{resolvedCollectionName}' not found."; - var collectionList = config.Collections.Select(c => new + await using var stream = await collection.GetContentAsync(resolvedFilePath, cancellationToken); + if (stream == null) + return $"File '{resolvedFilePath}' not found in collection '{resolvedCollectionName}'."; + + // Check file type and read accordingly + var extension = Path.GetExtension(resolvedFilePath).ToLowerInvariant(); + if (extension == ".xlsx" || extension == ".xls") { - c.Name, - DisplayName = c.DisplayName ?? c.Name, - Address = c.Address?.ToString() ?? "No address", - c.SourceType, - BasePath = c.BasePath ?? "Not specified" - }).ToList(); + return await ReadExcelFileAsync(stream, resolvedFilePath, numberOfRows); + } + else if (extension == ".docx") + { + return await ReadWordFileAsync(stream, resolvedFilePath, numberOfRows); + } + else if (extension == ".pdf") + { + return await ReadPdfFileAsync(stream, resolvedFilePath, numberOfRows); + } - var result = string.Join("\n", collectionList.Select(c => - $"- {c.DisplayName} (Name: {c.Name}, Type: {c.SourceType}, Path: {c.BasePath}, Address: {c.Address})")); + // For other files, read as text + using var reader = new StreamReader(stream); + if (numberOfRows.HasValue) + { + var sb = new StringBuilder(); + var linesRead = 0; + while (!reader.EndOfStream && linesRead < numberOfRows.Value) + { + var line = await reader.ReadLineAsync(cancellationToken); + sb.AppendLine(line); + linesRead++; + } + return sb.ToString(); + } + else + { + var content = await reader.ReadToEndAsync(cancellationToken); + return content; + } + } + catch (FileNotFoundException) + { + return $"File '{resolvedFilePath}' not found in collection '{resolvedCollectionName}'."; + } + catch (Exception ex) + { + return $"Error reading file '{resolvedFilePath}' from collection '{resolvedCollectionName}': {ex.Message}"; + } + } - return Task.FromResult(result); + private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows) + { + try + { + using var wb = new XLWorkbook(stream); + var sb = new StringBuilder(); + + foreach (var ws in wb.Worksheets) + { + var used = ws.RangeUsed(); + sb.AppendLine($"## Sheet: {ws.Name}"); + sb.AppendLine(); + if (used is null) + { + sb.AppendLine("(No data)"); + sb.AppendLine(); + continue; + } + + var firstRow = used.FirstRow().RowNumber(); + var lastRow = numberOfRows.HasValue + ? Math.Min(used.FirstRow().RowNumber() + numberOfRows.Value - 1, used.LastRow().RowNumber()) + : used.LastRow().RowNumber(); + var firstCol = 1; + var lastCol = used.LastColumn().ColumnNumber(); + + // Build markdown table with column letters as headers + var columnHeaders = new List { "Row" }; + for (var c = firstCol; c <= lastCol; c++) + { + // Convert column number to Excel letter (1=A, 2=B, ..., 27=AA, etc.) + columnHeaders.Add(GetExcelColumnLetter(c)); + } + + // Header row + sb.AppendLine("| " + string.Join(" | ", columnHeaders) + " |"); + // Separator row + sb.AppendLine("|" + string.Join("", columnHeaders.Select(_ => "---:|"))); + + // Data rows + for (var r = firstRow; r <= lastRow; r++) + { + var rowVals = new List { r.ToString() }; + for (var c = firstCol; c <= lastCol; c++) + { + var cell = ws.Cell(r, c); + var raw = cell.GetValue(); + var val = raw?.Replace('\n', ' ').Replace('\r', ' ').Replace("|", "\\|").Trim(); + // Empty cells show as empty in table + rowVals.Add(string.IsNullOrEmpty(val) ? "" : val); + } + + sb.AppendLine("| " + string.Join(" | ", rowVals) + " |"); + } + + sb.AppendLine(); + } + + return sb.ToString(); } catch (Exception ex) { - return Task.FromResult($"Error retrieving collections: {ex.Message}"); + return $"Error reading Excel file '{filePath}': {ex.Message}"; + } + } + + private static string GetExcelColumnLetter(int columnNumber) + { + var columnLetter = ""; + while (columnNumber > 0) + { + var modulo = (columnNumber - 1) % 26; + columnLetter = Convert.ToChar('A' + modulo) + columnLetter; + columnNumber = (columnNumber - 1) / 26; + } + return columnLetter; + } + + private async Task ReadWordFileAsync(Stream stream, string filePath, int? numberOfRows) + { + try + { + using var wordDoc = WordprocessingDocument.Open(stream, false); + var body = wordDoc.MainDocumentPart?.Document?.Body; + + if (body == null) + return $"Word document '{filePath}' has no readable content."; + + var sb = new StringBuilder(); + sb.AppendLine($"# Document: {Path.GetFileName(filePath)}"); + sb.AppendLine(); + + var paragraphs = body.Elements().ToList(); + var paragraphsToRead = numberOfRows.HasValue + ? paragraphs.Take(numberOfRows.Value).ToList() + : paragraphs; + + foreach (var paragraph in paragraphsToRead) + { + var text = paragraph.InnerText; + if (!string.IsNullOrWhiteSpace(text)) + { + sb.AppendLine(text); + sb.AppendLine(); + } + } + + // Also handle tables + var tables = body.Elements
().ToList(); + foreach (var table in tables) + { + sb.AppendLine("## Table"); + sb.AppendLine(); + + var rows = table.Elements().ToList(); + var rowsToRead = numberOfRows.HasValue + ? rows.Take(numberOfRows.Value).ToList() + : rows; + + foreach (var row in rowsToRead) + { + var cells = row.Elements().ToList(); + var cellTexts = cells.Select(c => c.InnerText.Replace('|', '\\').Trim()).ToList(); + sb.AppendLine("| " + string.Join(" | ", cellTexts) + " |"); + } + + sb.AppendLine(); + } + + return sb.ToString(); + } + catch (Exception ex) + { + return $"Error reading Word document '{filePath}': {ex.Message}"; + } + } + + private async Task ReadPdfFileAsync(Stream stream, string filePath, int? numberOfRows) + { + try + { + using var pdfDocument = PdfDocument.Open(stream); + var sb = new StringBuilder(); + sb.AppendLine($"# PDF Document: {Path.GetFileName(filePath)}"); + sb.AppendLine($"Total pages: {pdfDocument.NumberOfPages}"); + sb.AppendLine(); + + var pagesToRead = numberOfRows.HasValue + ? Math.Min(numberOfRows.Value, pdfDocument.NumberOfPages) + : pdfDocument.NumberOfPages; + + for (int pageNum = 1; pageNum <= pagesToRead; pageNum++) + { + var page = pdfDocument.GetPage(pageNum); + sb.AppendLine($"## Page {pageNum}"); + sb.AppendLine(); + + var text = page.Text; + if (!string.IsNullOrWhiteSpace(text)) + { + sb.AppendLine(text); + } + else + { + sb.AppendLine("(No text content)"); + } + sb.AppendLine(); + } + + return sb.ToString(); + } + catch (Exception ex) + { + return $"Error reading PDF document '{filePath}': {ex.Message}"; + } + } + + [KernelFunction] + [Description("Saves content as a file to a specified collection. If collection not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] + public async Task SaveFile( + [Description("The path where the file should be saved within the collection")] string filePath, + [Description("The content to save to the file")] string content, + [Description("The name of the collection to save to. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, + CancellationToken cancellationToken = default) + { + var resolvedCollectionName = GetCollectionName(collectionName); + if (string.IsNullOrEmpty(resolvedCollectionName)) + return "No collection specified and no default collection configured."; + + if (string.IsNullOrEmpty(filePath)) + return "File path is required."; + + try + { + var collection = await contentService.GetCollectionAsync(resolvedCollectionName, cancellationToken); + if (collection == null) + return $"Collection '{resolvedCollectionName}' not found."; + + // Ensure directory structure exists if the collection has a base path + EnsureDirectoryExists(collection, filePath); + + // Extract directory and filename components + var directoryPath = Path.GetDirectoryName(filePath) ?? ""; + var fileName = Path.GetFileName(filePath); + + if (string.IsNullOrEmpty(fileName)) + return $"Invalid file path: '{filePath}'. Must include a filename."; + + await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + await collection.SaveFileAsync(directoryPath, fileName, stream); + + return $"File '{filePath}' successfully saved to collection '{resolvedCollectionName}'."; + } + catch (Exception ex) + { + return $"Error saving file '{filePath}' to collection '{resolvedCollectionName}': {ex.Message}"; } } @@ -186,6 +469,35 @@ public async Task ListFiles( } } + [KernelFunction] + [Description("Lists all available collections with their configurations.")] + public Task GetCollections() + { + try + { + if (config.Collections.Count == 0) + return Task.FromResult("No collections configured."); + + var collectionList = config.Collections.Select(c => new + { + c.Name, + DisplayName = c.DisplayName ?? c.Name, + Address = c.Address?.ToString() ?? "No address", + c.SourceType, + BasePath = c.BasePath ?? "Not specified" + }).ToList(); + + var result = string.Join("\n", collectionList.Select(c => + $"- {c.DisplayName} (Name: {c.Name}, Type: {c.SourceType}, Path: {c.BasePath}, Address: {c.Address})")); + + return Task.FromResult(result); + } + catch (Exception ex) + { + return Task.FromResult($"Error retrieving collections: {ex.Message}"); + } + } + [KernelFunction] [Description("Lists all folders in a specified collection at a given path. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] public async Task ListFolders( @@ -253,7 +565,7 @@ public async Task ListCollectionItems( } [KernelFunction] - [Description("Gets the content of a specific document from a collection. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] + [Description("Gets the content of a specific document from a collection (simple text reading). If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] public async Task GetDocument( [Description("Document path in collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/', e.g., 'Slip.md' from 'Submissions-Microsoft-2026/Slip.md'); else null.")] string? documentPath = null, [Description("Collection name. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, @@ -290,41 +602,6 @@ public async Task GetDocument( } } - [KernelFunction] - [Description("Saves content as a document to a specified collection. If collection not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] - public async Task SaveDocument( - [Description("The path where the document should be saved within the collection")] string documentPath, - [Description("The content to save to the document")] string content, - [Description("Collection name. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, - CancellationToken cancellationToken = default) - { - var resolvedCollectionName = GetCollectionName(collectionName); - if (string.IsNullOrEmpty(resolvedCollectionName)) - return "No collection specified and no default collection configured."; - - try - { - var collection = await contentService.GetCollectionAsync(resolvedCollectionName, cancellationToken); - if (collection == null) - return $"Collection '{resolvedCollectionName}' not found."; - - var directoryPath = Path.GetDirectoryName(documentPath) ?? ""; - var fileName = Path.GetFileName(documentPath); - - if (string.IsNullOrEmpty(fileName)) - return $"Invalid document path: '{documentPath}'. Must include a filename."; - - await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - await collection.SaveFileAsync(directoryPath, fileName, stream); - - return $"Document '{documentPath}' successfully saved to collection '{resolvedCollectionName}'."; - } - catch (Exception ex) - { - return $"Error saving document '{documentPath}' to collection: {ex.Message}"; - } - } - [KernelFunction] [Description("Deletes a file from a specified collection. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] public async Task DeleteFile( @@ -532,14 +809,204 @@ public async Task GetContentType( } } + [KernelFunction] + [Description("Checks if a specific file exists in a collection. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] + public async Task FileExists( + [Description("The path to the file within the collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/'); else null.")] string? filePath = null, + [Description("The name of the collection to check. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, + CancellationToken cancellationToken = default) + { + var resolvedCollectionName = GetCollectionName(collectionName); + if (string.IsNullOrEmpty(resolvedCollectionName)) + return "No collection specified and no default collection configured."; + + var resolvedFilePath = filePath ?? GetPathFromContext(); + if (string.IsNullOrEmpty(resolvedFilePath)) + return "No file path specified and no path found in context."; + + try + { + var collection = await contentService.GetCollectionAsync(resolvedCollectionName, cancellationToken); + if (collection == null) + return $"Collection '{resolvedCollectionName}' not found."; + + await using var stream = await collection.GetContentAsync(resolvedFilePath, cancellationToken); + if (stream == null) + return $"File '{resolvedFilePath}' does not exist in collection '{resolvedCollectionName}'."; + + return $"File '{resolvedFilePath}' exists in collection '{resolvedCollectionName}'."; + } + catch (FileNotFoundException) + { + return $"File '{resolvedFilePath}' does not exist in collection '{resolvedCollectionName}'."; + } + catch (Exception ex) + { + return $"Error checking file '{resolvedFilePath}' in collection '{resolvedCollectionName}': {ex.Message}"; + } + } + + [KernelFunction] + [Description("Generates a unique filename with timestamp for saving temporary files.")] + public string GenerateUniqueFileName( + [Description("The base name for the file (without extension)")] string baseName, + [Description("The file extension (e.g., 'json', 'txt')")] string extension) + { + var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss_fff"); + return $"{baseName}_{timestamp}.{extension.TrimStart('.')}"; + } + + [KernelFunction] + [Description("Imports data from a file in a collection to a specified address.")] + public async Task Import( + [Description("The path to the file to import")] string path, + [Description("The name of the collection containing the file (optional if default collection is configured)")] string? collection = null, + [Description("The target address for the import (optional if default address is configured), can be a string like 'AddressType/id' or an Address object")] object? address = null, + [Description("The import format to use (optional, defaults to 'Default')")] string? format = null, + [Description("Optional import configuration as JSON string. When provided, this will be used instead of the format parameter.")] string? configuration = null, + CancellationToken cancellationToken = default) + { + try + { + if (string.IsNullOrWhiteSpace(collection)) + return "Collection name is required."; + + if (address == null) + return "Target address is required."; + + // Parse the address - handle both string and Address types + Address targetAddress; + if (address is string addressString) + { + targetAddress = hub.GetAddress(addressString); + } + else if (address is Address addr) + { + targetAddress = addr; + } + else + { + return $"Invalid address type: {address.GetType().Name}. Expected string or Address."; + } + + // Build ImportRequest JSON structure + var importRequestJson = new JsonObject + { + ["$type"] = "MeshWeaver.Import.ImportRequest", + ["source"] = new JsonObject + { + ["$type"] = "MeshWeaver.Import.CollectionSource", + ["collection"] = collection, + ["path"] = path + }, + ["format"] = format ?? "Default" + }; + + // Add configuration if provided + if (!string.IsNullOrWhiteSpace(configuration)) + { + var configNode = JsonNode.Parse(configuration); + if (configNode != null) + { + importRequestJson["configuration"] = configNode; + } + } + + // Serialize and deserialize through hub's serializer to get proper type + var jsonString = importRequestJson.ToJsonString(); + var importRequestObj = JsonSerializer.Deserialize(jsonString, hub.JsonSerializerOptions)!; + + // Post the request to the hub + var responseMessage = await hub.AwaitResponse( + importRequestObj, + o => o.WithTarget(targetAddress), + cancellationToken + ); + + // Serialize the response back to JSON for processing + var responseJson = JsonSerializer.Serialize(responseMessage, hub.JsonSerializerOptions); + var responseObj = JsonNode.Parse(responseJson)!; + + var log = responseObj["log"] as JsonObject; + var status = log?["status"]?.ToString() ?? "Unknown"; + var messages = log?["messages"] as JsonArray ?? new JsonArray(); + + var result = $"Import {status.ToLower()}.\n"; + if (messages.Count > 0) + { + result += "Log messages:\n"; + foreach (var msg in messages) + { + if (msg is JsonObject msgObj) + { + var level = msgObj["logLevel"]?.ToString() ?? "Info"; + var message = msgObj["message"]?.ToString() ?? ""; + result += $" [{level}] {message}\n"; + } + } + } + + return result; + } + catch (Exception ex) + { + return $"Error importing file '{path}' from collection '{collection}' to address '{address}': {ex.Message}"; + } + } + + /// + /// Creates a KernelPlugin from this instance using reflection. + /// public KernelPlugin CreateKernelPlugin() { var plugin = KernelPluginFactory.CreateFromFunctions( - nameof(ContentCollectionPlugin), + nameof(ContentPlugin), GetType().GetMethods(BindingFlags.Instance | BindingFlags.Public) .Where(m => m.GetCustomAttribute() != null) .Select(m => KernelFunctionFactory.CreateFromMethod(m, this)) ); return plugin; } + + /// + /// Ensures that the directory structure exists for the given file path within the collection. + /// + /// The collection to check + /// The file path that may contain directories + private void EnsureDirectoryExists(object collection, string filePath) + { + try + { + // Normalize path separators and get the directory path from the file path + var normalizedPath = filePath.Replace('/', Path.DirectorySeparatorChar); + var directoryPath = Path.GetDirectoryName(normalizedPath); + + if (string.IsNullOrEmpty(directoryPath) || directoryPath == "." || directoryPath == Path.DirectorySeparatorChar.ToString()) + { + // No directory structure needed, file is in root + return; + } + + // Try to get the collection's base path using reflection if available + var collectionType = collection.GetType(); + var basePathProperty = collectionType.GetProperty("BasePath") ?? + collectionType.GetProperty("Path") ?? + collectionType.GetProperty("RootPath"); + + if (basePathProperty != null) + { + var basePath = basePathProperty.GetValue(collection) as string; + if (!string.IsNullOrEmpty(basePath)) + { + var fullDirectoryPath = Path.Combine(basePath, directoryPath); + Directory.CreateDirectory(fullDirectoryPath); + } + } + } + catch (Exception) + { + // If we can't create directories through reflection, + // let the SaveFileAsync method handle any directory creation or fail gracefully + } + } } diff --git a/test/MeshWeaver.AI.Test/CollectionPluginTest.cs b/test/MeshWeaver.AI.Test/ContentPluginTest.cs similarity index 95% rename from test/MeshWeaver.AI.Test/CollectionPluginTest.cs rename to test/MeshWeaver.AI.Test/ContentPluginTest.cs index 4bd8c9d1b..720922dc2 100644 --- a/test/MeshWeaver.AI.Test/CollectionPluginTest.cs +++ b/test/MeshWeaver.AI.Test/ContentPluginTest.cs @@ -14,9 +14,9 @@ namespace MeshWeaver.AI.Test; /// -/// Tests for CollectionPlugin functionality, specifically the GetFile method with Excel support +/// Tests for ContentPlugin functionality, specifically the GetFile method with Excel support /// -public class CollectionPluginTest(ITestOutputHelper output) : HubTestBase(output), IAsyncLifetime +public class ContentPluginTest(ITestOutputHelper output) : HubTestBase(output), IAsyncLifetime { private const string TestCollectionName = "test-collection"; private const string TestExcelFileName = "test.xlsx"; @@ -133,7 +133,7 @@ public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // act var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); @@ -170,7 +170,7 @@ public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); const int rowLimit = 5; // act @@ -202,7 +202,7 @@ public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); const int rowLimit = 10; // act @@ -228,7 +228,7 @@ public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // act var result = await plugin.GetFile(TestCollectionName, TestTextFileName, cancellationToken: TestContext.Current.CancellationToken); @@ -253,7 +253,7 @@ public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // act var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); @@ -283,7 +283,7 @@ public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // act var result = await plugin.GetFile("non-existent-collection", "test.xlsx", cancellationToken: TestContext.Current.CancellationToken); @@ -300,7 +300,7 @@ public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() { // arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // act var result = await plugin.GetFile(TestCollectionName, "non-existent.xlsx", cancellationToken: TestContext.Current.CancellationToken); diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs index cd4d51265..1b2c24598 100644 --- a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -47,7 +47,7 @@ public async Task CollectionPlugin_Import_ShouldImportSuccessfully() { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -82,7 +82,7 @@ public async Task CollectionPlugin_Import_WithNonExistentFile_ShouldFail() { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -102,7 +102,7 @@ public async Task CollectionPlugin_Import_WithNonExistentCollection_ShouldFail() { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -122,7 +122,7 @@ public async Task CollectionPlugin_Import_WithMissingCollection_ShouldReturnErro { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -142,7 +142,7 @@ public async Task CollectionPlugin_Import_WithMissingAddress_ShouldReturnError() { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -162,7 +162,7 @@ public async Task CollectionPlugin_Import_WithCustomFormat_ShouldImportSuccessfu { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Act var result = await plugin.Import( @@ -192,7 +192,7 @@ public async Task CollectionPlugin_Import_WithConfiguration_ShouldImportWithoutF { // Arrange var client = GetClient(); - var plugin = new CollectionPlugin(client); + var plugin = new ContentPlugin(client); // Create a configuration JSON that is not registered as a format var configurationJson = @"{ From 91c67087488d16b3470286b1c306aed6dc0ac702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 09:46:01 +0100 Subject: [PATCH 13/57] Treating callbacks as delivery finished --- src/MeshWeaver.Data/DataContext.cs | 10 +++++++++- .../Serialization/SynchronizationStream.cs | 2 +- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index 7435cba1a..d3ea667a3 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -4,6 +4,7 @@ using MeshWeaver.Messaging.Serialization; using MeshWeaver.Reflection; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace MeshWeaver.Data; @@ -14,6 +15,7 @@ public sealed record DataContext : IDisposable public DataContext(IWorkspace workspace) { Hub = workspace.Hub; + logger = Hub.ServiceProvider.GetRequiredService>(); Workspace = workspace; ReduceManager = Hub.CreateReduceManager(); @@ -27,6 +29,7 @@ public DataContext(IWorkspace workspace) deferral = Hub.Defer(x => x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }); } + private readonly ILogger logger; private readonly IDisposable deferral; private Dictionary TypeSourcesByType { get; set; } = new(); @@ -76,6 +79,7 @@ Func, ReduceManager> change public void Initialize() { + logger.LogDebug("Starting initialization of DataContext for {Address}", Hub.Address); DataSourcesById = DataSourceBuilders.Select(x => x.Invoke(Hub)).ToImmutableDictionary(x => x.Id); DataSourcesByType = DataSourcesById.Values @@ -91,7 +95,11 @@ public void Initialize() TypeRegistry.WithType(typeSource.TypeDefinition.Type, typeSource.TypeDefinition.CollectionName); Task.WhenAll(DataSourcesById.Values.Select(d => d.Initialized)) - .ContinueWith(_ => deferral.Dispose()); + .ContinueWith(_ => + { + logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); + deferral.Dispose(); + }); } public IEnumerable MappedTypes => DataSourcesByType.Keys; diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 45243df0e..11ee55317 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -128,7 +128,7 @@ private void SetCurrent(ChangeItem? value) current = value; try { - logger.LogDebug("Setting value for {StreamId} to {Value}", StreamId, JsonSerializer.Serialize(value, Host.JsonSerializerOptions)); + logger.LogDebug("Setting value for {StreamId}", StreamId); Store.OnNext(value); } catch (Exception e) diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index cf68dc577..6f228abcb 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -517,7 +517,7 @@ CancellationToken cancellationToken logger.LogTrace("MESSAGE_FLOW: HUB_CALLBACKS_COMPLETE | {MessageType} | Hub: {Address} | MessageId: {MessageId}", delivery.Message.GetType().Name, Address, delivery.Id); - return delivery; + return delivery.Processed(); } Address IMessageHub.Address => Address; From 481fd01d16d16c6db1364d8dcf3e6816fdd872bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 10:04:45 +0100 Subject: [PATCH 14/57] resolving deadlocks of content collection creation --- .../ContentService.cs | 21 ++++++++----------- .../EmbeddedResourceStreamProviderFactory.cs | 4 ++-- .../FileSystemStreamProviderFactory.cs | 4 ++-- .../HubStreamProviderFactory.cs | 15 ++++++------- .../IStreamProviderFactory.cs | 3 ++- .../ArticleConfigurationExtensions.cs | 4 ++-- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/MeshWeaver.ContentCollections/ContentService.cs b/src/MeshWeaver.ContentCollections/ContentService.cs index 6228fc7e9..c5a896a70 100644 --- a/src/MeshWeaver.ContentCollections/ContentService.cs +++ b/src/MeshWeaver.ContentCollections/ContentService.cs @@ -59,8 +59,8 @@ public ContentService(IMessageHub hub, AccessService accessService) if (factory is null) throw new ArgumentException($"Unknown source type {config.SourceType}"); - // Create provider using the factory - var provider = factory.Create(config); + // Create provider using the factory (now properly async) + var provider = await factory.CreateAsync(config, cancellationToken); // Create and initialize the collection var collection = new ContentCollection(config, provider, hub); @@ -104,18 +104,15 @@ public ContentService(IMessageHub hub, AccessService accessService) if (collections.TryGetValue(config.Name, out var existing)) return existing; - else + lock (initializeLock) { - lock (initializeLock) - { - if (collections.TryGetValue(config.Name, out existing)) - return existing; + if (collections.TryGetValue(config.Name, out existing)) + return existing; - // Create a new initialization task - initTask = InstantiateCollectionAsync(config, cancellationToken); - collections[config.Name] = initTask; - return initTask; - } + // Create a new initialization task + initTask = InstantiateCollectionAsync(config, cancellationToken); + collections[config.Name] = initTask; + return initTask; } } diff --git a/src/MeshWeaver.ContentCollections/EmbeddedResourceStreamProviderFactory.cs b/src/MeshWeaver.ContentCollections/EmbeddedResourceStreamProviderFactory.cs index 035aa88c2..516ee1e34 100644 --- a/src/MeshWeaver.ContentCollections/EmbeddedResourceStreamProviderFactory.cs +++ b/src/MeshWeaver.ContentCollections/EmbeddedResourceStreamProviderFactory.cs @@ -5,7 +5,7 @@ namespace MeshWeaver.ContentCollections; /// public class EmbeddedResourceStreamProviderFactory : IStreamProviderFactory { - public IStreamProvider Create(ContentCollectionConfig config) + public Task CreateAsync(ContentCollectionConfig config, CancellationToken cancellationToken = default) { var assemblyName = config.Settings?.GetValueOrDefault("AssemblyName") ?? throw new ArgumentException("AssemblyName required for EmbeddedResource"); @@ -16,6 +16,6 @@ public IStreamProvider Create(ContentCollectionConfig config) .FirstOrDefault(a => a.GetName().Name == assemblyName) ?? throw new InvalidOperationException($"Assembly not found: {assemblyName}"); - return new EmbeddedResourceStreamProvider(assembly, resourcePrefix); + return Task.FromResult(new EmbeddedResourceStreamProvider(assembly, resourcePrefix)); } } diff --git a/src/MeshWeaver.ContentCollections/FileSystemStreamProviderFactory.cs b/src/MeshWeaver.ContentCollections/FileSystemStreamProviderFactory.cs index f3dc7fc4c..479b299ba 100644 --- a/src/MeshWeaver.ContentCollections/FileSystemStreamProviderFactory.cs +++ b/src/MeshWeaver.ContentCollections/FileSystemStreamProviderFactory.cs @@ -5,10 +5,10 @@ /// public class FileSystemStreamProviderFactory : IStreamProviderFactory { - public IStreamProvider Create(ContentCollectionConfig config) + public Task CreateAsync(ContentCollectionConfig config, CancellationToken cancellationToken = default) { var basePath = config.BasePath ?? config.Settings?.GetValueOrDefault("BasePath") ?? ""; - return new FileSystemStreamProvider(basePath); + return Task.FromResult(new FileSystemStreamProvider(basePath)); } } diff --git a/src/MeshWeaver.ContentCollections/HubStreamProviderFactory.cs b/src/MeshWeaver.ContentCollections/HubStreamProviderFactory.cs index 333538391..ff71fd424 100644 --- a/src/MeshWeaver.ContentCollections/HubStreamProviderFactory.cs +++ b/src/MeshWeaver.ContentCollections/HubStreamProviderFactory.cs @@ -11,18 +11,19 @@ public class HubStreamProviderFactory(IMessageHub hub) : IStreamProviderFactory { public const string SourceType = "Hub"; - public IStreamProvider Create(ContentCollectionConfig config) + public async Task CreateAsync(ContentCollectionConfig config, CancellationToken cancellationToken = default) { if (config.Address == null) throw new ArgumentException("Address is required for Hub source type"); var collectionName = config.Settings?.GetValueOrDefault("CollectionName") ?? config.Name; - // Query the remote hub for the collection configuration - var response = hub.AwaitResponse( + // Query the remote hub for the collection configuration (now properly async) + var response = await hub.AwaitResponse( new GetContentCollectionRequest([collectionName]), - o => o.WithTarget(config.Address) - ).GetAwaiter().GetResult(); + o => o.WithTarget(config.Address), + cancellationToken + ); var remoteConfig = response.Message.Collections.FirstOrDefault(); if (remoteConfig == null) @@ -33,7 +34,7 @@ public IStreamProvider Create(ContentCollectionConfig config) if (factory == null) throw new InvalidOperationException($"Unknown provider type '{remoteConfig.SourceType}'"); - // Create provider using the factory with the remote config - return factory.Create(remoteConfig); + // Create provider using the factory with the remote config (now properly async) + return await factory.CreateAsync(remoteConfig, cancellationToken); } } diff --git a/src/MeshWeaver.ContentCollections/IStreamProviderFactory.cs b/src/MeshWeaver.ContentCollections/IStreamProviderFactory.cs index ea84c9dd0..bd24bf5d8 100644 --- a/src/MeshWeaver.ContentCollections/IStreamProviderFactory.cs +++ b/src/MeshWeaver.ContentCollections/IStreamProviderFactory.cs @@ -9,6 +9,7 @@ public interface IStreamProviderFactory /// Creates a stream provider from the given configuration /// /// Content collection configuration + /// Cancellation token /// The created stream provider - IStreamProvider Create(ContentCollectionConfig config); + Task CreateAsync(ContentCollectionConfig config, CancellationToken cancellationToken = default); } diff --git a/src/MeshWeaver.Hosting.AzureBlob/ArticleConfigurationExtensions.cs b/src/MeshWeaver.Hosting.AzureBlob/ArticleConfigurationExtensions.cs index 4985fb001..a6eb17107 100644 --- a/src/MeshWeaver.Hosting.AzureBlob/ArticleConfigurationExtensions.cs +++ b/src/MeshWeaver.Hosting.AzureBlob/ArticleConfigurationExtensions.cs @@ -67,7 +67,7 @@ public class AzureBlobStreamProviderFactory(IServiceProvider serviceProvider) : { public const string SourceType = "AzureBlob"; - public IStreamProvider Create(ContentCollectionConfig config) + public Task CreateAsync(ContentCollectionConfig config, CancellationToken cancellationToken = default) { if (config.Settings == null) throw new ArgumentException("Settings are required for AzureBlob source type"); @@ -81,6 +81,6 @@ public IStreamProvider Create(ContentCollectionConfig config) var clientName = config.Settings.GetValueOrDefault("ClientName", "default"); var blobServiceClient = factory.CreateClient(clientName); - return new AzureBlobStreamProvider(blobServiceClient, containerName); + return Task.FromResult(new AzureBlobStreamProvider(blobServiceClient, containerName)); } } From 379736e614074ec4ba68adc9d918e9d8406f69d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 10:28:13 +0100 Subject: [PATCH 15/57] excluding large properties from logging. --- portal/MeshWeaver.Portal/appsettings.json | 2 +- .../SerilogExtensions.cs | 4 +- src/MeshWeaver.Data.Contract/Messages.cs | 2 +- src/MeshWeaver.Data/ChangeItem.cs | 3 +- .../Serialization/SynchronizationStream.cs | 2 +- .../PreventLoggingAttribute.cs | 10 ++++ .../MessageService.cs | 5 +- .../Serialization/LoggingTypeInfoResolver.cs | 51 +++++++++++++++++++ .../SerializationExtensions.cs | 21 ++++++++ 9 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 src/MeshWeaver.Messaging.Contract/PreventLoggingAttribute.cs create mode 100644 src/MeshWeaver.Messaging.Hub/Serialization/LoggingTypeInfoResolver.cs diff --git a/portal/MeshWeaver.Portal/appsettings.json b/portal/MeshWeaver.Portal/appsettings.json index 6a12f33d3..0bc0edbe3 100644 --- a/portal/MeshWeaver.Portal/appsettings.json +++ b/portal/MeshWeaver.Portal/appsettings.json @@ -4,7 +4,7 @@ "Default": "Warning", "Microsoft.AspNetCore": "Warning", "MeshWeaver": "Warning", - "MeshWeaver.Messaging.MessageService": "Warning" + "MeshWeaver.Messaging.MessageService": "Information" }, "Console": { "IncludeScopes": true, diff --git a/portal/aspire/MeshWeaver.Portal.ServiceDefaults/SerilogExtensions.cs b/portal/aspire/MeshWeaver.Portal.ServiceDefaults/SerilogExtensions.cs index b7f932ffe..fd350cc73 100644 --- a/portal/aspire/MeshWeaver.Portal.ServiceDefaults/SerilogExtensions.cs +++ b/portal/aspire/MeshWeaver.Portal.ServiceDefaults/SerilogExtensions.cs @@ -21,7 +21,7 @@ public static MeshHostApplicationBuilder AddEfCoreSerilog(this MeshHostApplicati builder.ConfigureHub(h => h.WithInitialization(hub => { - messageDeliveryPolicy.JsonOptions = hub.JsonSerializerOptions; + messageDeliveryPolicy.JsonOptions = hub.CreateLoggingSerializerOptions(); sink.Initialize(hub.ServiceProvider); })); @@ -55,7 +55,7 @@ public static MeshHostApplicationBuilder AddEfCoreMessageLog(this MeshHostApplic builder.ConfigureHub(h => h.WithInitialization(hub => { - messageDeliveryPolicy.JsonOptions = hub.JsonSerializerOptions; + messageDeliveryPolicy.JsonOptions = hub.CreateLoggingSerializerOptions(); sink.Initialize(hub.ServiceProvider); })); diff --git a/src/MeshWeaver.Data.Contract/Messages.cs b/src/MeshWeaver.Data.Contract/Messages.cs index 8febdafb1..6d43a6ba0 100644 --- a/src/MeshWeaver.Data.Contract/Messages.cs +++ b/src/MeshWeaver.Data.Contract/Messages.cs @@ -57,7 +57,7 @@ public abstract record StreamMessage(string StreamId); public abstract record JsonChange( string StreamId, long Version, - RawJson Change, + [property: PreventLogging] RawJson Change, ChangeType ChangeType, string? ChangedBy ) : StreamMessage(StreamId); diff --git a/src/MeshWeaver.Data/ChangeItem.cs b/src/MeshWeaver.Data/ChangeItem.cs index 3214a3707..35ab48b4d 100644 --- a/src/MeshWeaver.Data/ChangeItem.cs +++ b/src/MeshWeaver.Data/ChangeItem.cs @@ -1,4 +1,5 @@ using MeshWeaver.Data; +using MeshWeaver.Messaging; namespace MeshWeaver.Data; @@ -12,7 +13,7 @@ public interface IChangeItem public record ChangeItem( - TStream? Value, + [property: PreventLogging] TStream? Value, string? ChangedBy, string? StreamId, ChangeType ChangeType, diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 11ee55317..45243df0e 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -128,7 +128,7 @@ private void SetCurrent(ChangeItem? value) current = value; try { - logger.LogDebug("Setting value for {StreamId}", StreamId); + logger.LogDebug("Setting value for {StreamId} to {Value}", StreamId, JsonSerializer.Serialize(value, Host.JsonSerializerOptions)); Store.OnNext(value); } catch (Exception e) diff --git a/src/MeshWeaver.Messaging.Contract/PreventLoggingAttribute.cs b/src/MeshWeaver.Messaging.Contract/PreventLoggingAttribute.cs new file mode 100644 index 000000000..aa62361d2 --- /dev/null +++ b/src/MeshWeaver.Messaging.Contract/PreventLoggingAttribute.cs @@ -0,0 +1,10 @@ +namespace MeshWeaver.Messaging; + +/// +/// Marks a property or field to be excluded from serialization when logging. +/// This is useful for properties with large payloads that would clutter logs. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, Inherited = true)] +public class PreventLoggingAttribute : Attribute +{ +} diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index e90676939..44ac0f3d9 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -23,7 +23,10 @@ public class MessageService : IMessageService private readonly CancellationTokenSource hangDetectionCts = new(); private readonly TaskCompletionSource startupCompletionSource = new(); //private volatile int pendingStartupMessages; + private JsonSerializerOptions? loggingSerializerOptions; + private JsonSerializerOptions LoggingSerializerOptions => + loggingSerializerOptions ??= hub.CreateLoggingSerializerOptions(); public MessageService( Address address, @@ -255,7 +258,7 @@ private IMessageDelivery ScheduleExecution(IMessageDelivery delivery) var ret = PostImpl(message, opt); if (!ExcludedFromLogging.Contains(message.GetType())) logger.LogInformation("Posting message {Delivery} (ID: {MessageId}) in {Address}", - JsonSerializer.Serialize(ret, hub.JsonSerializerOptions), ret.Id, Address); + JsonSerializer.Serialize(ret, LoggingSerializerOptions), ret.Id, Address); return ret; } } diff --git a/src/MeshWeaver.Messaging.Hub/Serialization/LoggingTypeInfoResolver.cs b/src/MeshWeaver.Messaging.Hub/Serialization/LoggingTypeInfoResolver.cs new file mode 100644 index 000000000..e9d44b02d --- /dev/null +++ b/src/MeshWeaver.Messaging.Hub/Serialization/LoggingTypeInfoResolver.cs @@ -0,0 +1,51 @@ +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace MeshWeaver.Messaging.Serialization; + +/// +/// A custom JSON type info resolver that filters out properties marked with [PreventLogging] attribute. +/// This resolver wraps an existing resolver and removes properties that should not appear in logs. +/// +public class LoggingTypeInfoResolver : IJsonTypeInfoResolver +{ + private readonly IJsonTypeInfoResolver _innerResolver; + + public LoggingTypeInfoResolver(IJsonTypeInfoResolver innerResolver) + { + _innerResolver = innerResolver ?? throw new ArgumentNullException(nameof(innerResolver)); + } + + public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options) + { + var typeInfo = _innerResolver.GetTypeInfo(type, options); + + if (typeInfo?.Kind == JsonTypeInfoKind.Object && typeInfo.Properties.Count > 0) + { + // Find properties to remove (can't modify during enumeration) + var propertiesToRemove = typeInfo.Properties + .Where(ShouldExcludeFromLogging) + .ToList(); + + // Remove properties marked with [PreventLogging] + foreach (var property in propertiesToRemove) + { + typeInfo.Properties.Remove(property); + } + } + + return typeInfo; + } + + private static bool ShouldExcludeFromLogging(JsonPropertyInfo propertyInfo) + { + // Check if the underlying property/field has [PreventLogging] attribute + if (propertyInfo.AttributeProvider is MemberInfo memberInfo) + { + return memberInfo.GetCustomAttribute(inherit: true) != null; + } + + return false; + } +} diff --git a/src/MeshWeaver.Messaging.Hub/SerializationExtensions.cs b/src/MeshWeaver.Messaging.Hub/SerializationExtensions.cs index 24ee2fac9..12d76242e 100644 --- a/src/MeshWeaver.Messaging.Hub/SerializationExtensions.cs +++ b/src/MeshWeaver.Messaging.Hub/SerializationExtensions.cs @@ -2,6 +2,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using MeshWeaver.Domain; using MeshWeaver.Messaging.Serialization; using Microsoft.Extensions.DependencyInjection; @@ -87,4 +88,24 @@ private static SerializationConfiguration CreateSerializationConfiguration(IMess }); } + /// + /// Creates a JsonSerializerOptions configured for logging purposes. + /// This wraps the hub's standard serializer options with a LoggingTypeInfoResolver + /// that filters out properties marked with [PreventLogging] attribute. + /// + public static JsonSerializerOptions CreateLoggingSerializerOptions(this IMessageHub hub) + { + var baseOptions = hub.JsonSerializerOptions; + + // Create new options that copy settings from base options + var loggingOptions = new JsonSerializerOptions(baseOptions); + + // Wrap the existing TypeInfoResolver with LoggingTypeInfoResolver + loggingOptions.TypeInfoResolver = new LoggingTypeInfoResolver( + baseOptions.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver() + ); + + return loggingOptions; + } + } From 56aaa044306d1df8677c95bf78cd400123a26225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 11:17:38 +0100 Subject: [PATCH 16/57] improving ContentPlugin --- .../RiskImportAgent.cs | 14 +++++------- .../SlipImportAgent.cs | 14 +++++------- .../InsuranceApplicationExtensions.cs | 22 ++++++++++++++----- src/MeshWeaver.AI/Plugins/ContentPlugin.cs | 15 ++++++++----- test/MeshWeaver.AI.Test/ContentPluginTest.cs | 14 ++++++------ 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index f0b4529d7..7a0ffdfcd 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -101,9 +101,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } - private static ContentCollectionPluginConfig CreateSubmissionPluginConfig() + private static ContentPluginConfig CreateSubmissionPluginConfig() { - return new ContentCollectionPluginConfig + return new ContentPluginConfig { Collections = [], ContextToConfigMap = context => @@ -119,17 +119,13 @@ private static ContentCollectionPluginConfig CreateSubmissionPluginConfig() if (parts.Length != 2) return null!; - var company = parts[0]; - var uwy = parts[1]; - var subPath = $"{company}/{uwy}"; - - // Create Hub-based collection config pointing to the pricing address + // Use Hub-based collection config pointing to the pricing address + // This allows the ContentPlugin to query the pricing hub for the actual collection configuration return new ContentCollectionConfig { SourceType = HubStreamProviderFactory.SourceType, Name = $"Submissions-{pricingId}", - Address = context.Address, - BasePath = subPath + Address = context.Address }; } }; diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index 0139f0a97..93225a365 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -142,9 +142,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } - private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + private static ContentPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) { - return new ContentCollectionPluginConfig + return new ContentPluginConfig { Collections = [], ContextToConfigMap = context => @@ -160,17 +160,13 @@ private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgent if (parts.Length != 2) return null!; - var company = parts[0]; - var uwy = parts[1]; - var subPath = $"{company}/{uwy}"; - - // Create Hub-based collection config pointing to the pricing address + // Use Hub-based collection config pointing to the pricing address + // This allows the ContentPlugin to query the pricing hub for the actual collection configuration return new ContentCollectionConfig { SourceType = HubStreamProviderFactory.SourceType, Name = $"Submissions-{pricingId}", - Address = context.Address, - BasePath = subPath + Address = context.Address }; } }; diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index cddab8717..32c75887a 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -64,11 +64,6 @@ public static MessageHubConfiguration ConfigureSinglePricingApplication(this Mes var addressId = hub.Address.Id; var conf = sp.GetRequiredService(); - // Get the global Submissions configuration from appsettings - var globalConfig = conf.GetSection("Submissions").Get(); - if (globalConfig == null) - throw new InvalidOperationException("Submissions collection not found in configuration"); - // Parse addressId in format {company}-{uwy} var parts = addressId.Split('-'); if (parts.Length != 2) @@ -78,6 +73,23 @@ public static MessageHubConfiguration ConfigureSinglePricingApplication(this Mes var uwy = parts[1]; var subPath = $"{company}/{uwy}"; + // Get the global Submissions configuration from appsettings, or create a default one + var globalConfig = conf.GetSection("Submissions").Get(); + + // If no configuration exists, create a default FileSystem-based collection + if (globalConfig == null) + { + // Default to a "Submissions" folder in the current directory + var defaultBasePath = Path.Combine(Directory.GetCurrentDirectory(), "Submissions"); + globalConfig = new ContentCollectionConfig + { + SourceType = FileSystemStreamProvider.SourceType, + Name = "Submissions", + BasePath = defaultBasePath, + DisplayName = "Submission Files" + }; + } + // Create localized config with modified name and basepath var localizedName = GetLocalizedCollectionName("Submissions", addressId); var fullPath = string.IsNullOrEmpty(subPath) diff --git a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs index 34480d38d..7d306ddc5 100644 --- a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs @@ -41,21 +41,21 @@ public class ContentPlugin { private readonly IMessageHub hub; private readonly IContentService contentService; - private readonly ContentCollectionPluginConfig config; + private readonly ContentPluginConfig config; private readonly IAgentChat? chat; /// /// Creates a ContentPlugin with basic functionality (no context resolution). /// public ContentPlugin(IMessageHub hub) - : this(hub, new ContentCollectionPluginConfig { Collections = [] }, null!) + : this(hub, new ContentPluginConfig { Collections = [] }, null!) { } /// /// Creates a ContentPlugin with context resolution and dynamic collection configuration. /// - public ContentPlugin(IMessageHub hub, ContentCollectionPluginConfig config, IAgentChat chat) + public ContentPlugin(IMessageHub hub, ContentPluginConfig config, IAgentChat chat) { this.hub = hub; this.config = config; @@ -151,9 +151,12 @@ public ContentPlugin(IMessageHub hub, ContentCollectionPluginConfig config, IAge [KernelFunction] [Description("Gets the content of a file from a specified collection. Supports Excel, Word, PDF, and text files. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] public async Task GetFile( - [Description("The path to the file within the collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/'); else null.")] string? filePath = null, - [Description("The name of the collection to read from. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] string? collectionName = null, - [Description("Optional: number of rows to read. If null, reads entire file. For Excel files, reads first N rows from each worksheet.")] int? numberOfRows = null, + [Description("The path to the file within the collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/'); else null.")] + string? filePath = null, + [Description("The name of the collection to read from. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] + string? collectionName = null, + [Description("Optional: number of rows to read. If null, reads entire file. For Excel files, reads first N rows from each worksheet.")] + int? numberOfRows = null, CancellationToken cancellationToken = default) { var resolvedCollectionName = GetCollectionName(collectionName); diff --git a/test/MeshWeaver.AI.Test/ContentPluginTest.cs b/test/MeshWeaver.AI.Test/ContentPluginTest.cs index 720922dc2..28bc698bd 100644 --- a/test/MeshWeaver.AI.Test/ContentPluginTest.cs +++ b/test/MeshWeaver.AI.Test/ContentPluginTest.cs @@ -136,7 +136,7 @@ public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -174,7 +174,7 @@ public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() const int rowLimit = 5; // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -206,7 +206,7 @@ public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() const int rowLimit = 10; // act - var result = await plugin.GetFile(TestCollectionName, TestTextFileName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile(TestTextFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -231,7 +231,7 @@ public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestTextFileName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile(TestTextFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -256,7 +256,7 @@ public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, TestExcelFileName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -286,7 +286,7 @@ public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile("non-existent-collection", "test.xlsx", cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile("test.xlsx", "non-existent-collection", cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("Collection 'non-existent-collection' not found"); @@ -303,7 +303,7 @@ public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestCollectionName, "non-existent.xlsx", cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetFile("non-existent.xlsx", TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("File 'non-existent.xlsx' not found"); From 97f5a217b9801bb0c2d7dbd683b8e03379d2b458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 11:18:00 +0100 Subject: [PATCH 17/57] renaming config --- modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs | 4 ++-- ...ontentCollectionPluginConfig.cs => ContentPluginConfig.cs} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/MeshWeaver.AI/Plugins/{ContentCollectionPluginConfig.cs => ContentPluginConfig.cs} (93%) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs index 3195c900c..2f4e0a930 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs @@ -107,9 +107,9 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } - private static ContentCollectionPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + private static ContentPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) { - return new ContentCollectionPluginConfig + return new ContentPluginConfig { Collections = [], ContextToConfigMap = context => diff --git a/src/MeshWeaver.AI/Plugins/ContentCollectionPluginConfig.cs b/src/MeshWeaver.AI/Plugins/ContentPluginConfig.cs similarity index 93% rename from src/MeshWeaver.AI/Plugins/ContentCollectionPluginConfig.cs rename to src/MeshWeaver.AI/Plugins/ContentPluginConfig.cs index d42fe7d2b..09416906b 100644 --- a/src/MeshWeaver.AI/Plugins/ContentCollectionPluginConfig.cs +++ b/src/MeshWeaver.AI/Plugins/ContentPluginConfig.cs @@ -6,7 +6,7 @@ namespace MeshWeaver.AI.Plugins; /// /// Configuration for the SubmissionPlugin. /// -public class ContentCollectionPluginConfig +public class ContentPluginConfig { /// /// Collection of content collection configurations. From 14ff811c41d4f219cd325d987796ae65e684ddf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 11:38:09 +0100 Subject: [PATCH 18/57] improving exception handling around messagehub start --- src/MeshWeaver.Data/DataContext.cs | 26 ++++++++++++++++--- .../Serialization/SynchronizationStream.cs | 9 +++++++ src/MeshWeaver.Messaging.Hub/MessageHub.cs | 23 ++++++++++------ 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index d3ea667a3..fddfdc4ba 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -95,11 +95,29 @@ public void Initialize() TypeRegistry.WithType(typeSource.TypeDefinition.Type, typeSource.TypeDefinition.CollectionName); Task.WhenAll(DataSourcesById.Values.Select(d => d.Initialized)) - .ContinueWith(_ => + .ContinueWith(task => { - logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); - deferral.Dispose(); - }); + try + { + if (task.IsFaulted) + { + logger.LogError(task.Exception, "DataContext initialization failed for {Address}, disposing deferral anyway", Hub.Address); + } + else if (task.IsCanceled) + { + logger.LogWarning("DataContext initialization was canceled for {Address}, disposing deferral anyway", Hub.Address); + } + else + { + logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); + } + } + finally + { + // Always dispose deferral to release buffered messages, even if initialization fails + deferral.Dispose(); + } + }, TaskScheduler.Default); } public IEnumerable MappedTypes => DataSourcesByType.Keys; diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 45243df0e..ce5e649c3 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -279,7 +279,16 @@ x.Message is not SetCurrentRequest && private async Task InitializeAsync(CancellationToken ct) { if (Configuration.Initialization is null) + { + // If no custom initialization, immediately dispose startup deferrable to release buffered messages + if (startupDeferrable is not null) + { + logger.LogDebug("No initialization configured, disposing startup deferrable for Stream {StreamId}", StreamId); + startupDeferrable.Dispose(); + startupDeferrable = null; + } return; + } var init = await Configuration.Initialization(this, ct); SetCurrent(new ChangeItem(init, StreamId, Host.Version)); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 6f228abcb..823fc7cf4 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -96,16 +96,23 @@ private async Task HandleInitialize(IMessageDelivery Date: Thu, 30 Oct 2025 12:04:55 +0100 Subject: [PATCH 19/57] making deferrables more robust --- src/MeshWeaver.Data/DataContext.cs | 13 ++++--- src/MeshWeaver.Data/DataExtensions.cs | 1 + .../Serialization/SynchronizationStream.cs | 27 +++----------- src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 7 ++++ src/MeshWeaver.Messaging.Hub/MessageHub.cs | 37 +++++++++++++++++++ .../MessageHubConfiguration.cs | 16 ++++++++ 6 files changed, 74 insertions(+), 27 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index fddfdc4ba..b4ea42936 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -10,6 +10,8 @@ namespace MeshWeaver.Data; public sealed record DataContext : IDisposable { + public const string DataContextDeferralName = "DataContextInitialization"; + public ITypeRegistry TypeRegistry { get; } public DataContext(IWorkspace workspace) @@ -26,11 +28,10 @@ public DataContext(IWorkspace workspace) type.GetProperties().Where(x => x.HasAttribute()).ToArray() ) ?? null ); - deferral = Hub.Defer(x => x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }); + // Named deferral is now created in MessageHubConfiguration via AddData() } private readonly ILogger logger; - private readonly IDisposable deferral; private Dictionary TypeSourcesByType { get; set; } = new(); @@ -101,11 +102,11 @@ public void Initialize() { if (task.IsFaulted) { - logger.LogError(task.Exception, "DataContext initialization failed for {Address}, disposing deferral anyway", Hub.Address); + logger.LogError(task.Exception, "DataContext initialization failed for {Address}, releasing deferral anyway", Hub.Address); } else if (task.IsCanceled) { - logger.LogWarning("DataContext initialization was canceled for {Address}, disposing deferral anyway", Hub.Address); + logger.LogWarning("DataContext initialization was canceled for {Address}, releasing deferral anyway", Hub.Address); } else { @@ -114,8 +115,8 @@ public void Initialize() } finally { - // Always dispose deferral to release buffered messages, even if initialization fails - deferral.Dispose(); + // Always release deferral to process buffered messages, even if initialization fails + Hub.ReleaseDeferral(DataContextDeferralName); } }, TaskScheduler.Default); } diff --git a/src/MeshWeaver.Data/DataExtensions.cs b/src/MeshWeaver.Data/DataExtensions.cs index 238695b19..53d53a2dd 100644 --- a/src/MeshWeaver.Data/DataExtensions.cs +++ b/src/MeshWeaver.Data/DataExtensions.cs @@ -33,6 +33,7 @@ Func dataPluginConfiguration if (existingLambdas.Any()) return ret; return ret + .WithNamedDeferral(DataContext.DataContextDeferralName, x => x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }) .WithInitialization(h => h.GetWorkspace()) .WithRoutes(routes => routes.WithHandler((delivery, _) => RouteStreamMessage(routes.Hub, delivery))) .WithServices(sc => sc.AddScoped(sp => diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index ce5e649c3..721987134 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -111,12 +111,8 @@ public ChangeItem? Current private void SetCurrent(ChangeItem? value) { - if (startupDeferrable is not null) - { - logger.LogDebug("Disposing startup deferrable for Stream {StreamId}", StreamId); - startupDeferrable.Dispose(); - startupDeferrable = null; - } + // Release startup deferral if it hasn't been released yet + Hub.ReleaseDeferral(StartupDeferralName); if (isDisposed || value == null) { @@ -197,15 +193,9 @@ public SynchronizationStream( logger.LogInformation("Creating Synchronization Stream {StreamId} for Host {Host} and {StreamIdentity} and {Reference}", StreamId, Host.Address, StreamIdentity, Reference); Hub = Host.GetHostedHub(new SynchronizationAddress(ClientId), c => ConfigureSynchronizationHub(c)); - if (Configuration.Initialization is null) - startupDeferrable = Hub.Defer(StartupDeferrable); - - - - } - private IDisposable? startupDeferrable; + private const string StartupDeferralName = "SynchronizationStreamStartup"; private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfiguration config) { @@ -267,7 +257,7 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat throw new SynchronizationException("An error occurred during synchronization", ex); } return request.Processed(); - }).WithStartupDeferral(StartupDeferrable) + }).WithNamedDeferral(StartupDeferralName, StartupDeferrable) .WithInitialization((_, ct) => InitializeAsync(ct)); } @@ -280,13 +270,8 @@ private async Task InitializeAsync(CancellationToken ct) { if (Configuration.Initialization is null) { - // If no custom initialization, immediately dispose startup deferrable to release buffered messages - if (startupDeferrable is not null) - { - logger.LogDebug("No initialization configured, disposing startup deferrable for Stream {StreamId}", StreamId); - startupDeferrable.Dispose(); - startupDeferrable = null; - } + // If no custom initialization, immediately release startup deferral to process buffered messages + Hub.ReleaseDeferral(StartupDeferralName); return; } diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index 3d03768f9..6e4df65ef 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -110,6 +110,13 @@ IMessageHub GetHostedHub(TAddress address, Func deferredFilter); + /// + /// Releases a named deferral, allowing deferred messages to be processed. + /// + /// The name of the deferral to release + /// True if the deferral was found and released, false otherwise + bool ReleaseDeferral(string name); + internal Task HandleMessageAsync( IMessageDelivery delivery, diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 823fc7cf4..3fd34b2dc 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -25,6 +25,7 @@ public void InvokeAsync(Func action, Func> callbacks = new(); private readonly HashSet pendingCallbackCancellations = new(); + private readonly ConcurrentDictionary namedDeferrals = new(); private readonly ILogger logger; public MessageHubConfiguration Configuration { get; } @@ -83,6 +84,22 @@ public MessageHub( Register(HandleCallbacks); messageService.Start(); + + // Create all named deferrals before starting message processing + foreach (var (name, predicate) in configuration.NamedDeferrals) + { + var deferral = Defer(predicate); + if (!namedDeferrals.TryAdd(name, deferral)) + { + logger.LogWarning("Duplicate named deferral '{Name}' for hub {Address}", name, Address); + deferral.Dispose(); + } + else + { + logger.LogDebug("Created named deferral '{Name}' for hub {Address}", name, Address); + } + } + Post(new InitializeHubRequest(Defer(Configuration.StartupDeferral))); } @@ -637,6 +654,14 @@ public void Dispose() private void DisposeImpl() { + // Dispose all remaining named deferrals + foreach (var (name, deferral) in namedDeferrals) + { + logger.LogDebug("Disposing remaining named deferral '{Name}' during hub disposal for {Address}", name, Address); + deferral.Dispose(); + } + namedDeferrals.Clear(); + while (disposeActions.TryTake(out var disposeAction)) disposeAction.Invoke(this); @@ -831,6 +856,18 @@ private void CancelCallbacks() public IDisposable Defer(Predicate deferredFilter) => messageService.Defer(deferredFilter); + public bool ReleaseDeferral(string name) + { + if (namedDeferrals.TryRemove(name, out var deferral)) + { + logger.LogDebug("Releasing named deferral '{Name}' for hub {Address}", name, Address); + deferral.Dispose(); + return true; + } + logger.LogWarning("Named deferral '{Name}' not found in hub {Address}", name, Address); + return false; + } + private readonly ConcurrentDictionary<(string Conext, Type Type), object?> properties = new(); public void Set(T obj, string context = "") diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index e72666220..bb18626e1 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -25,6 +25,22 @@ public MessageHubConfiguration(IServiceProvider? parentServiceProvider, Address public MessageHubConfiguration WithStartupDeferral(Predicate startupDeferral) => this with { StartupDeferral = x => startupDeferral(x) && StartupDeferral(x) }; + /// + /// Named deferrals that are created during hub initialization and can be released by name. + /// The key is the deferral name, the value is the predicate that determines which messages to defer. + /// + internal ImmutableDictionary> NamedDeferrals { get; init; } = ImmutableDictionary>.Empty; + + /// + /// Adds a named deferral that will be created during hub initialization. + /// This ensures the deferral is in place before any messages are processed. + /// + /// Unique name for this deferral + /// Predicate that determines which messages to defer + /// Updated configuration + public MessageHubConfiguration WithNamedDeferral(string name, Predicate predicate) + => this with { NamedDeferrals = NamedDeferrals.SetItem(name, predicate) }; + public IMessageHub? ParentHub { get From f358ff65f8ed00acd9eb906a21be99a4fe4da22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 13:18:17 +0100 Subject: [PATCH 20/57] rename GetFile => GetContent --- .../RiskImportAgent.cs | 7 ++--- .../SlipImportAgent.cs | 4 +-- src/MeshWeaver.AI/Plugins/ContentPlugin.cs | 2 +- test/MeshWeaver.AI.Test/ContentPluginTest.cs | 30 +++++++++---------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs index 7a0ffdfcd..fbd697eec 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/RiskImportAgent.cs @@ -1,5 +1,4 @@ -using System.Text.Json; -using System.Text.Json.Nodes; +using System.Text.Json.Nodes; using MeshWeaver.AI; using MeshWeaver.AI.Plugins; using MeshWeaver.ContentCollections; @@ -39,7 +38,7 @@ public string Instructions # Importing Risks When the user asks you to import risks, you should: 1) Get the existing risk mapping configuration for the specified file using DataPlugin's GetData function with type="ExcelImportConfiguration" and entityId=filename. - 2) If no import configuration was returned in 1, get a sample of the worksheet using ContentPlugin's GetFile function with the collection name "Submissions-{pricingId}", the filename, and numberOfRows=20. Extract the table start row as well as the mapping as in the schema provided below. + 2) If no import configuration was returned in 1, get a sample of the worksheet using ContentPlugin's GetContent function with the collection name "Submissions-{pricingId}", the filename, and numberOfRows=20. Extract the table start row as well as the mapping as in the schema provided below. Consider any input from the user to modify the configuration. Ensure the JSON includes "name" field set to the filename. Use DataPlugin's UpdateData function with type="ExcelImportConfiguration" to save the configuration. 3) Call ContentPlugin's Import function with path=filename, collection="Submissions-{pricingId}", address=PricingAddress, and configuration=the JSON configuration you created or retrieved. @@ -50,7 +49,7 @@ public string Instructions 3) Upload the new configuration using DataPlugin's UpdateData function with type="ExcelImportConfiguration" and the updated JSON (ensure "name" field is set to filename). # Automatic Risk Import Configuration - - Use ContentPlugin's GetFile with numberOfRows=20 to get a sample of the file. It returns a markdown table with: + - Use ContentPlugin's GetContent with numberOfRows=20 to get a sample of the file. It returns a markdown table with: - First column: Row numbers (1-based) - Remaining columns: Labeled A, B, C, D, etc. (Excel column letters) - Empty cells appear as empty values in the table (not "null") diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index 93225a365..9c951c067 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -39,7 +39,7 @@ You are a slip import agent that processes insurance submission slip documents i # Importing Slips When the user asks you to import a slip: 1) First, use ContentCollectionPlugin's ListFiles() to see available files in the submissions collection - 2) Use ContentPlugin's GetFile function to extract the document content from PDF or Markdown files + 2) Use ContentPlugin's GetContent function to extract the document content from PDF or Markdown files - Pass collectionName="Submissions-{pricingId}" and filePath=filename (e.g., "Slip.pdf" or "Slip.md") - For PDFs, this will extract all pages of text 3) Review the extracted text and identify data that matches the domain schemas @@ -116,7 +116,7 @@ You are a slip import agent that processes insurance submission slip documents i Notes: - When listing files, you may see paths with "/" prefix (e.g., "/Slip.pdf", "/Slip.md") - - When calling ContentPlugin's GetFile, use collectionName="Submissions-{pricingId}" and provide the filename + - When calling ContentPlugin's GetContent, use collectionName="Submissions-{pricingId}" and provide the filename - Both PDF and Markdown (.md) files are supported - When updating data, ensure each JSON object has the correct $type field and required ID fields (id, pricingId, acceptanceId, etc.) - Remove null-valued properties from JSON before calling UpdateData diff --git a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs index 7d306ddc5..03400c16b 100644 --- a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs @@ -150,7 +150,7 @@ public ContentPlugin(IMessageHub hub, ContentPluginConfig config, IAgentChat cha [KernelFunction] [Description("Gets the content of a file from a specified collection. Supports Excel, Word, PDF, and text files. If collection/path not provided: when Area='Content' or 'Collection', parses from LayoutAreaReference.Id ('{collection}/{path}'); otherwise uses ContextToConfigMap or plugin config.")] - public async Task GetFile( + public async Task GetContent( [Description("The path to the file within the collection. If omitted: when Area='Content'/'Collection', extracts from Id (after first '/'); else null.")] string? filePath = null, [Description("The name of the collection to read from. If omitted: when Area='Content'/'Collection', extracts from Id (before '/'); else uses ContextToConfigMap/config.")] diff --git a/test/MeshWeaver.AI.Test/ContentPluginTest.cs b/test/MeshWeaver.AI.Test/ContentPluginTest.cs index 28bc698bd..e724aad90 100644 --- a/test/MeshWeaver.AI.Test/ContentPluginTest.cs +++ b/test/MeshWeaver.AI.Test/ContentPluginTest.cs @@ -14,7 +14,7 @@ namespace MeshWeaver.AI.Test; /// -/// Tests for ContentPlugin functionality, specifically the GetFile method with Excel support +/// Tests for ContentPlugin functionality, specifically the GetContent method with Excel support /// public class ContentPluginTest(ITestOutputHelper output) : HubTestBase(output), IAsyncLifetime { @@ -126,7 +126,7 @@ private async Task CreateTestTextFile() } /// - /// Tests that GetFile preserves null values in Excel files with empty cells at the start of rows + /// Tests that GetContent preserves null values in Excel files with empty cells at the start of rows /// [Fact] public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() @@ -136,7 +136,7 @@ public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -163,7 +163,7 @@ public async Task GetFile_ExcelWithEmptyCellsAtStart_ShouldPreserveNulls() } /// - /// Tests that GetFile with numberOfRows parameter limits Excel file output + /// Tests that GetContent with numberOfRows parameter limits Excel file output /// [Fact] public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() @@ -174,7 +174,7 @@ public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() const int rowLimit = 5; // act - var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent(TestExcelFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -195,7 +195,7 @@ public async Task GetFile_ExcelWithNumberOfRows_ShouldLimitRows() } /// - /// Tests that GetFile with numberOfRows parameter limits text file output + /// Tests that GetContent with numberOfRows parameter limits text file output /// [Fact] public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() @@ -206,7 +206,7 @@ public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() const int rowLimit = 10; // act - var result = await plugin.GetFile(TestTextFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent(TestTextFileName, TestCollectionName, numberOfRows: rowLimit, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -221,7 +221,7 @@ public async Task GetFile_TextWithNumberOfRows_ShouldLimitRows() } /// - /// Tests that GetFile without numberOfRows parameter reads entire text file + /// Tests that GetContent without numberOfRows parameter reads entire text file /// [Fact] public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() @@ -231,7 +231,7 @@ public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestTextFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent(TestTextFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -246,7 +246,7 @@ public async Task GetFile_TextWithoutNumberOfRows_ShouldReadEntireFile() } /// - /// Tests that GetFile without numberOfRows parameter reads entire Excel file + /// Tests that GetContent without numberOfRows parameter reads entire Excel file /// [Fact] public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() @@ -256,7 +256,7 @@ public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent(TestExcelFileName, TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().NotBeNullOrEmpty(); @@ -276,7 +276,7 @@ public async Task GetFile_ExcelWithoutNumberOfRows_ShouldReadEntireFile() } /// - /// Tests that GetFile handles non-existent collection + /// Tests that GetContent handles non-existent collection /// [Fact] public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() @@ -286,14 +286,14 @@ public async Task GetFile_NonExistentCollection_ShouldReturnErrorMessage() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile("test.xlsx", "non-existent-collection", cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent("test.xlsx", "non-existent-collection", cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("Collection 'non-existent-collection' not found"); } /// - /// Tests that GetFile handles non-existent file + /// Tests that GetContent handles non-existent file /// [Fact] public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() @@ -303,7 +303,7 @@ public async Task GetFile_NonExistentFile_ShouldReturnErrorMessage() var plugin = new ContentPlugin(client); // act - var result = await plugin.GetFile("non-existent.xlsx", TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); + var result = await plugin.GetContent("non-existent.xlsx", TestCollectionName, cancellationToken: TestContext.Current.CancellationToken); // assert result.Should().Contain("File 'non-existent.xlsx' not found"); From f4e7d10e243af7af4c5b9e87ee89123d8547b08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 13:18:31 +0100 Subject: [PATCH 21/57] Refactoring deferrals. --- src/MeshWeaver.Data/DataContext.cs | 5 ++- src/MeshWeaver.Data/DataExtensions.cs | 1 - .../Serialization/SynchronizationStream.cs | 15 ++------ src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 8 ++--- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 36 +++++++++---------- .../MessageHubConfiguration.cs | 21 ++++++----- 6 files changed, 38 insertions(+), 48 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index b4ea42936..b9a015e52 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -10,7 +10,7 @@ namespace MeshWeaver.Data; public sealed record DataContext : IDisposable { - public const string DataContextDeferralName = "DataContextInitialization"; + public const string InitializationGateName = "DataContextInit"; public ITypeRegistry TypeRegistry { get; } @@ -115,8 +115,7 @@ public void Initialize() } finally { - // Always release deferral to process buffered messages, even if initialization fails - Hub.ReleaseDeferral(DataContextDeferralName); + // Initialization complete } }, TaskScheduler.Default); } diff --git a/src/MeshWeaver.Data/DataExtensions.cs b/src/MeshWeaver.Data/DataExtensions.cs index 53d53a2dd..238695b19 100644 --- a/src/MeshWeaver.Data/DataExtensions.cs +++ b/src/MeshWeaver.Data/DataExtensions.cs @@ -33,7 +33,6 @@ Func dataPluginConfiguration if (existingLambdas.Any()) return ret; return ret - .WithNamedDeferral(DataContext.DataContextDeferralName, x => x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }) .WithInitialization(h => h.GetWorkspace()) .WithRoutes(routes => routes.WithHandler((delivery, _) => RouteStreamMessage(routes.Hub, delivery))) .WithServices(sc => sc.AddScoped(sp => diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 721987134..36d9ff7e6 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -111,9 +111,6 @@ public ChangeItem? Current private void SetCurrent(ChangeItem? value) { - // Release startup deferral if it hasn't been released yet - Hub.ReleaseDeferral(StartupDeferralName); - if (isDisposed || value == null) { logger.LogWarning("Not setting {StreamId} to {Value} because the stream is disposed or value is null.", StreamId, value); @@ -195,8 +192,6 @@ public SynchronizationStream( Hub = Host.GetHostedHub(new SynchronizationAddress(ClientId), c => ConfigureSynchronizationHub(c)); } - private const string StartupDeferralName = "SynchronizationStreamStartup"; - private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfiguration config) { return config @@ -257,21 +252,15 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat throw new SynchronizationException("An error occurred during synchronization", ex); } return request.Processed(); - }).WithNamedDeferral(StartupDeferralName, StartupDeferrable) - .WithInitialization((_, ct) => InitializeAsync(ct)); + }).WithInitialization((_, ct) => InitializeAsync(ct)); } - private static bool StartupDeferrable(IMessageDelivery x) => - x.Message is not SetCurrentRequest && - x.Message is not DataChangedEvent { ChangeType: ChangeType.Full }; - private async Task InitializeAsync(CancellationToken ct) { if (Configuration.Initialization is null) { - // If no custom initialization, immediately release startup deferral to process buffered messages - Hub.ReleaseDeferral(StartupDeferralName); + // No custom initialization return; } diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index 6e4df65ef..4564d2b03 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -111,11 +111,11 @@ IMessageHub GetHostedHub(TAddress address, Func deferredFilter); /// - /// Releases a named deferral, allowing deferred messages to be processed. + /// Opens a named initialization gate, allowing all deferred messages to be processed. /// - /// The name of the deferral to release - /// True if the deferral was found and released, false otherwise - bool ReleaseDeferral(string name); + /// The name of the gate to open + /// True if the gate was found and opened, false if already opened or not found + bool OpenGate(string name); internal Task HandleMessageAsync( diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 3fd34b2dc..51f74f1b6 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -25,7 +25,7 @@ public void InvokeAsync(Func action, Func> callbacks = new(); private readonly HashSet pendingCallbackCancellations = new(); - private readonly ConcurrentDictionary namedDeferrals = new(); + private readonly ConcurrentDictionary initializationGates = new(); private readonly ILogger logger; public MessageHubConfiguration Configuration { get; } @@ -85,18 +85,20 @@ public MessageHub( messageService.Start(); - // Create all named deferrals before starting message processing - foreach (var (name, predicate) in configuration.NamedDeferrals) + // Create all initialization gates before starting message processing + // The gate defers all messages EXCEPT those matching the predicate + foreach (var (name, allowDuringInit) in configuration.InitializationGates) { - var deferral = Defer(predicate); - if (!namedDeferrals.TryAdd(name, deferral)) + // Invert the predicate: defer everything that is NOT allowed + var gate = Defer(delivery => !allowDuringInit(delivery)); + if (!initializationGates.TryAdd(name, gate)) { - logger.LogWarning("Duplicate named deferral '{Name}' for hub {Address}", name, Address); - deferral.Dispose(); + logger.LogWarning("Duplicate initialization gate '{Name}' for hub {Address}", name, Address); + gate.Dispose(); } else { - logger.LogDebug("Created named deferral '{Name}' for hub {Address}", name, Address); + logger.LogDebug("Created initialization gate '{Name}' for hub {Address}", name, Address); } } @@ -654,13 +656,11 @@ public void Dispose() private void DisposeImpl() { - // Dispose all remaining named deferrals - foreach (var (name, deferral) in namedDeferrals) + // Open all remaining initialization gates to release any buffered messages + foreach (var gateName in initializationGates.Keys.ToArray()) { - logger.LogDebug("Disposing remaining named deferral '{Name}' during hub disposal for {Address}", name, Address); - deferral.Dispose(); + OpenGate(gateName); } - namedDeferrals.Clear(); while (disposeActions.TryTake(out var disposeAction)) disposeAction.Invoke(this); @@ -856,15 +856,15 @@ private void CancelCallbacks() public IDisposable Defer(Predicate deferredFilter) => messageService.Defer(deferredFilter); - public bool ReleaseDeferral(string name) + public bool OpenGate(string name) { - if (namedDeferrals.TryRemove(name, out var deferral)) + if (initializationGates.TryRemove(name, out var gate)) { - logger.LogDebug("Releasing named deferral '{Name}' for hub {Address}", name, Address); - deferral.Dispose(); + logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); + gate.Dispose(); return true; } - logger.LogWarning("Named deferral '{Name}' not found in hub {Address}", name, Address); + logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, Address); return false; } diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index bb18626e1..999cdce1f 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -26,20 +26,23 @@ public MessageHubConfiguration WithStartupDeferral(Predicate s => this with { StartupDeferral = x => startupDeferral(x) && StartupDeferral(x) }; /// - /// Named deferrals that are created during hub initialization and can be released by name. - /// The key is the deferral name, the value is the predicate that determines which messages to defer. + /// Named initialization gates that are created during hub initialization and can be opened by name. + /// The key is the gate name, the value is the predicate that determines which messages are allowed during initialization. + /// All other messages are deferred until the gate is opened. /// - internal ImmutableDictionary> NamedDeferrals { get; init; } = ImmutableDictionary>.Empty; + internal ImmutableDictionary> InitializationGates { get; init; } = ImmutableDictionary>.Empty; /// - /// Adds a named deferral that will be created during hub initialization. - /// This ensures the deferral is in place before any messages are processed. + /// Adds a named initialization gate that will be created during hub initialization. + /// This ensures the gate is in place before any messages are processed. + /// Only messages matching the predicate will be allowed through during initialization. + /// All other messages will be deferred until the gate is opened via OpenGate(). /// - /// Unique name for this deferral - /// Predicate that determines which messages to defer + /// Unique name for this initialization gate + /// Predicate that determines which messages are allowed during initialization (e.g. InitializeHubRequest, SetCurrentRequest) /// Updated configuration - public MessageHubConfiguration WithNamedDeferral(string name, Predicate predicate) - => this with { NamedDeferrals = NamedDeferrals.SetItem(name, predicate) }; + public MessageHubConfiguration WithInitializationGate(string name, Predicate allowDuringInit) + => this with { InitializationGates = InitializationGates.SetItem(name, allowDuringInit) }; public IMessageHub? ParentHub { From 4d7f331d461ccb3890ca9251b44fdb2d07c64a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 13:54:14 +0100 Subject: [PATCH 22/57] updating deferral conditions --- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 51f74f1b6..6cf20cbf4 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -90,7 +90,11 @@ public MessageHub( foreach (var (name, allowDuringInit) in configuration.InitializationGates) { // Invert the predicate: defer everything that is NOT allowed - var gate = Defer(delivery => !allowDuringInit(delivery)); + // IMPORTANT: Never defer response messages (messages with RequestId property) + // as they need to flow back to awaiting requests immediately + var gate = Defer(delivery => + !allowDuringInit(delivery) && + !delivery.Properties.ContainsKey(PostOptions.RequestId)); if (!initializationGates.TryAdd(name, gate)) { logger.LogWarning("Duplicate initialization gate '{Name}' for hub {Address}", name, Address); From ca30dcb6bf30ac4e09bd0623addb54f8546e162f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Thu, 30 Oct 2025 21:28:45 +0100 Subject: [PATCH 23/57] fixing hub initialization --- .../MeshWeaver.Insurance.AI/InsuranceAgent.cs | 4 +- .../SlipImportAgent.cs | 4 +- src/MeshWeaver.Data/DataContext.cs | 29 ++++---- .../GenericUnpartitionedDataSource.cs | 3 +- .../Persistence/HubDataSource.cs | 6 ++ .../Serialization/SynchronizationStream.cs | 8 ++- src/MeshWeaver.Data/WorkspaceOperations.cs | 2 +- .../Implementation/ImportManager.cs | 6 +- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 68 ++++++++++++------- .../MessageHubConfiguration.cs | 11 ++- 10 files changed, 84 insertions(+), 57 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs index 2f4e0a930..4d9d5feaa 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/InsuranceAgent.cs @@ -103,11 +103,11 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new LayoutAreaPlugin(hub, chat, layoutAreaMap).CreateKernelPlugin(); // Always provide ContentPlugin - it will use ContextToConfigMap to determine the collection - var submissionPluginConfig = CreateSubmissionPluginConfig(chat); + var submissionPluginConfig = CreateSubmissionPluginConfig(); yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } - private static ContentPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + private static ContentPluginConfig CreateSubmissionPluginConfig() { return new ContentPluginConfig { diff --git a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs index 9c951c067..e448a6371 100644 --- a/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs +++ b/modules/Insurance/MeshWeaver.Insurance.AI/SlipImportAgent.cs @@ -138,11 +138,11 @@ IEnumerable IAgentWithPlugins.GetPlugins(IAgentChat chat) yield return new DataPlugin(hub, chat, typeDefinitionMap).CreateKernelPlugin(); // Add ContentPlugin for submissions and file reading functionality - var submissionPluginConfig = CreateSubmissionPluginConfig(chat); + var submissionPluginConfig = CreateSubmissionPluginConfig(); yield return new ContentPlugin(hub, submissionPluginConfig, chat).CreateKernelPlugin(); } - private static ContentPluginConfig CreateSubmissionPluginConfig(IAgentChat chat) + private static ContentPluginConfig CreateSubmissionPluginConfig() { return new ContentPluginConfig { diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index b9a015e52..6e9c43d39 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -11,6 +11,7 @@ namespace MeshWeaver.Data; public sealed record DataContext : IDisposable { public const string InitializationGateName = "DataContextInit"; + private const string DataContextGateName = InitializationGateName; public ITypeRegistry TypeRegistry { get; } @@ -28,7 +29,6 @@ public DataContext(IWorkspace workspace) type.GetProperties().Where(x => x.HasAttribute()).ToArray() ) ?? null ); - // Named deferral is now created in MessageHubConfiguration via AddData() } private readonly ILogger logger; @@ -95,27 +95,24 @@ public void Initialize() foreach (var typeSource in TypeSources.Values) TypeRegistry.WithType(typeSource.TypeDefinition.Type, typeSource.TypeDefinition.CollectionName); + // Initialize each data source + foreach (var dataSource in DataSourcesById.Values) + dataSource.Initialize(); + Task.WhenAll(DataSourcesById.Values.Select(d => d.Initialized)) .ContinueWith(task => { - try + if (task.IsFaulted) + { + logger.LogError(task.Exception, "DataContext initialization failed for {Address}", Hub.Address); + } + else if (task.IsCanceled) { - if (task.IsFaulted) - { - logger.LogError(task.Exception, "DataContext initialization failed for {Address}, releasing deferral anyway", Hub.Address); - } - else if (task.IsCanceled) - { - logger.LogWarning("DataContext initialization was canceled for {Address}, releasing deferral anyway", Hub.Address); - } - else - { - logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); - } + logger.LogWarning("DataContext initialization was canceled for {Address}", Hub.Address); } - finally + else { - // Initialization complete + logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); } }, TaskScheduler.Default); } diff --git a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs index e44dcdc93..6bbfcda62 100644 --- a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs @@ -27,7 +27,8 @@ public interface IDataSource : IDisposable ISynchronizationStream? GetStreamForPartition(object? partition); IEnumerable TypeSources { get; } - Task Initialized { get; } + internal Task Initialized { get; } + internal void Initialize(); } public interface IUnpartitionedDataSource : IDataSource diff --git a/src/MeshWeaver.Data/Persistence/HubDataSource.cs b/src/MeshWeaver.Data/Persistence/HubDataSource.cs index 96c8d2016..f3da2c7a4 100644 --- a/src/MeshWeaver.Data/Persistence/HubDataSource.cs +++ b/src/MeshWeaver.Data/Persistence/HubDataSource.cs @@ -20,4 +20,10 @@ protected override ISynchronizationStream CreateStream(StreamIdenti protected override ISynchronizationStream CreateStream(StreamIdentity identity, Func, StreamConfiguration> config) => Workspace.GetRemoteStream(Address, GetReference()); + + public override void Initialize() + { + base.Initialize(); + GetStream(GetReference()); + } } diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 36d9ff7e6..d57f08752 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -123,6 +123,7 @@ private void SetCurrent(ChangeItem? value) { logger.LogDebug("Setting value for {StreamId} to {Value}", StreamId, JsonSerializer.Serialize(value, Host.JsonSerializerOptions)); Store.OnNext(value); + Hub.OpenGate(SynchronizationGate); } catch (Exception e) { @@ -130,6 +131,7 @@ private void SetCurrent(ChangeItem? value) } } + private const string SynchronizationGate = nameof(SynchronizationGate); public void Update(Func?> update, Func exceptionCallback) => Hub.Post(new UpdateStreamRequest((stream, _) => Task.FromResult(update.Invoke(stream)), exceptionCallback)); @@ -189,7 +191,7 @@ public SynchronizationStream( logger = Host.ServiceProvider.GetRequiredService>>(); logger.LogInformation("Creating Synchronization Stream {StreamId} for Host {Host} and {StreamIdentity} and {Reference}", StreamId, Host.Address, StreamIdentity, Reference); - Hub = Host.GetHostedHub(new SynchronizationAddress(ClientId), c => ConfigureSynchronizationHub(c)); + Hub = Host.GetHostedHub(new SynchronizationAddress(ClientId), ConfigureSynchronizationHub); } private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfiguration config) @@ -252,7 +254,9 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat throw new SynchronizationException("An error occurred during synchronization", ex); } return request.Processed(); - }).WithInitialization((_, ct) => InitializeAsync(ct)); + }) + .WithInitialization((_, ct) => InitializeAsync(ct)) + .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full }); } diff --git a/src/MeshWeaver.Data/WorkspaceOperations.cs b/src/MeshWeaver.Data/WorkspaceOperations.cs index 5d57cbe9c..2362a8c2b 100644 --- a/src/MeshWeaver.Data/WorkspaceOperations.cs +++ b/src/MeshWeaver.Data/WorkspaceOperations.cs @@ -196,7 +196,7 @@ private static void UpdateStreams(this IWorkspace workspace, DataChangeRequest c throw new NotSupportedException($"Operation {g.Key.Op} not supported"); }); subActivity?.LogInformation("Applying changes to Data Stream {Stream}", stream.StreamIdentity); - logger?.LogInformation("Applying changes to Data Stream {Stream}", stream.StreamIdentity); + logger.LogInformation("Applying changes to Data Stream {Stream}", stream.StreamIdentity); // Complete sub-activity - this would need proper sub-activity tracking to work correctly return stream.ApplyChanges(updates); } diff --git a/src/MeshWeaver.Import/Implementation/ImportManager.cs b/src/MeshWeaver.Import/Implementation/ImportManager.cs index 7589ee18a..7c21163ae 100644 --- a/src/MeshWeaver.Import/Implementation/ImportManager.cs +++ b/src/MeshWeaver.Import/Implementation/ImportManager.cs @@ -106,6 +106,7 @@ private async Task ImportImpl(IMessageDelivery request, Cancellat request ); activity.LogInformation("Finished import {ActivityId} for request {RequestId}", activity.Id, request.Id); + Hub.Post(new ImportResponse(Hub.Version, log), o => o.ResponseFor(request)); }); @@ -129,7 +130,7 @@ public async Task ImportInstancesAsync( // If ExcelImportConfiguration is provided, use ConfiguredExcelImporter directly if (importRequest.Configuration is ExcelImportConfiguration excelConfig) { - return await ImportWithConfiguredExcelImporter(importRequest, excelConfig, activity, cancellationToken); + return await ImportWithConfiguredExcelImporter(importRequest, excelConfig, activity); } var (dataSet, format) = await ReadDataSetAsync(importRequest, activity, cancellationToken); @@ -140,8 +141,7 @@ public async Task ImportInstancesAsync( private async Task ImportWithConfiguredExcelImporter( ImportRequest importRequest, ExcelImportConfiguration config, - Activity? activity, - CancellationToken cancellationToken) + Activity? activity) { activity?.LogInformation("Using ConfiguredExcelImporter with TypeName: {TypeName}", config.TypeName); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 6cf20cbf4..bcda60ad7 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -85,28 +85,34 @@ public MessageHub( messageService.Start(); - // Create all initialization gates before starting message processing - // The gate defers all messages EXCEPT those matching the predicate - foreach (var (name, allowDuringInit) in configuration.InitializationGates) - { - // Invert the predicate: defer everything that is NOT allowed - // IMPORTANT: Never defer response messages (messages with RequestId property) - // as they need to flow back to awaiting requests immediately - var gate = Defer(delivery => - !allowDuringInit(delivery) && - !delivery.Properties.ContainsKey(PostOptions.RequestId)); - if (!initializationGates.TryAdd(name, gate)) - { - logger.LogWarning("Duplicate initialization gate '{Name}' for hub {Address}", name, Address); - gate.Dispose(); - } - else + // Create a single deferral that combines all allow-list predicates + // Messages are allowed if they match InitializationAllowList OR any of the InitializationGates + var singleDeferral = Defer(delivery => + { + // Check all gate predicates + foreach (var (name, allowDuringInit) in Configuration.InitializationGates) { - logger.LogDebug("Created initialization gate '{Name}' for hub {Address}", name, Address); + if (allowDuringInit(delivery)) + { + logger.LogDebug("Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, name, Address); + return false; // Don't defer - message is allowed by this gate + } } + + // Message doesn't match any allow-list - defer it + return true; + }); + + // Store gate names from configuration for tracking which gates are still open + // All gates reference the same single deferral + foreach (var (name, _) in configuration.InitializationGates) + { + initializationGates[name] = singleDeferral; + logger.LogDebug("Registered initialization gate '{Name}' for hub {Address}", name, Address); } - Post(new InitializeHubRequest(Defer(Configuration.StartupDeferral))); + Post(new InitializeHubRequest(singleDeferral)); } private IMessageDelivery HandlePingRequest(IMessageDelivery request) @@ -125,10 +131,11 @@ private async Task HandleInitialize(IMessageDelivery StartupDeferral { get; init; } = x => x.Message is not InitializeHubRequest; - - public MessageHubConfiguration WithStartupDeferral(Predicate startupDeferral) - => this with { StartupDeferral = x => startupDeferral(x) && StartupDeferral(x) }; - /// /// Named initialization gates that are created during hub initialization and can be opened by name. /// The key is the gate name, the value is the predicate that determines which messages are allowed during initialization. /// All other messages are deferred until the gate is opened. + /// The Initialize gate doesn't allow any additional messages - it's just a marker for when BuildupActions complete. /// - internal ImmutableDictionary> InitializationGates { get; init; } = ImmutableDictionary>.Empty; + internal ImmutableDictionary> InitializationGates { get; init; } = ImmutableDictionary>.Empty + .Add(InitializeGateName, d => d.Message is InitializeHubRequest); // Initialize gate doesn't allow any messages - just marks completion of BuildupActions /// /// Adds a named initialization gate that will be created during hub initialization. From 52d8d036a247a469632da9e29d8de0bfe2c6157c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 10:39:41 +0100 Subject: [PATCH 24/57] improving Initialization for importhub --- src/MeshWeaver.Data/DataContext.cs | 7 ++++-- .../Persistence/PartitionedHubDataSource.cs | 2 +- .../Serialization/SynchronizationStream.cs | 22 +++++++++---------- .../TestHubSetup.cs | 4 +++- .../TransactionalData.cs | 4 ++-- test/MeshWeaver.Import.Test/ImportTest.cs | 12 +++++----- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index 6e9c43d39..1c927f9b9 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -97,9 +97,12 @@ public void Initialize() // Initialize each data source foreach (var dataSource in DataSourcesById.Values) + { dataSource.Initialize(); + tasks.Add(dataSource.Initialized); + } - Task.WhenAll(DataSourcesById.Values.Select(d => d.Initialized)) + Task.WhenAll(tasks) .ContinueWith(task => { if (task.IsFaulted) @@ -118,7 +121,7 @@ public void Initialize() } public IEnumerable MappedTypes => DataSourcesByType.Keys; - + private readonly List tasks = new(); public void Dispose() { foreach (var dataSource in DataSourcesById.Values) diff --git a/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs b/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs index feec1fd59..adf9d3eea 100644 --- a/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs +++ b/src/MeshWeaver.Data/Persistence/PartitionedHubDataSource.cs @@ -15,7 +15,7 @@ public override PartitionedHubDataSource WithType(Func InitializingPartitions(IEnumerable partitions) => + public PartitionedHubDataSource InitializingPartitions(params IEnumerable partitions) => this with { InitializePartitions = InitializePartitions.Concat(partitions).ToArray() diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index d57f08752..74fe3cbe6 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -109,7 +109,7 @@ public ChangeItem? Current public ReduceManager ReduceManager { get; init; } - private void SetCurrent(ChangeItem? value) + private void SetCurrent(IMessageHub hub, ChangeItem? value) { if (isDisposed || value == null) { @@ -123,7 +123,7 @@ private void SetCurrent(ChangeItem? value) { logger.LogDebug("Setting value for {StreamId} to {Value}", StreamId, JsonSerializer.Serialize(value, Host.JsonSerializerOptions)); Store.OnNext(value); - Hub.OpenGate(SynchronizationGate); + hub.OpenGate(SynchronizationGate); } catch (Exception e) { @@ -221,7 +221,7 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat { hub.Dispose(); return delivery.Processed(); - }).WithHandler(async (_, request, ct) => + }).WithHandler(async (hub, request, ct) => { var update = request.Message.UpdateAsync; var exceptionCallback = request.Message.ExceptionCallback; @@ -236,18 +236,18 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat // SetCurrent will be called with the computed result // The Message Hub serializes these messages, so only one UpdateStreamRequest // is processed at a time per stream, preventing race conditions - SetCurrent(newChangeItem); + SetCurrent(hub, newChangeItem); } catch (Exception e) { await exceptionCallback.Invoke(e); } return request.Processed(); - }).WithHandler((_, request) => + }).WithHandler((hub, request) => { try { - SetCurrent(request.Message.Value); + SetCurrent(hub, request.Message.Value); } catch (Exception ex) { @@ -255,12 +255,12 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat } return request.Processed(); }) - .WithInitialization((_, ct) => InitializeAsync(ct)) + .WithInitialization(InitializeAsync) .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full }); } - private async Task InitializeAsync(CancellationToken ct) + private async Task InitializeAsync(IMessageHub hub, CancellationToken ct) { if (Configuration.Initialization is null) { @@ -269,7 +269,7 @@ private async Task InitializeAsync(CancellationToken ct) } var init = await Configuration.Initialization(this, ct); - SetCurrent(new ChangeItem(init, StreamId, Host.Version)); + SetCurrent(hub, new ChangeItem(init, StreamId, Host.Version)); } @@ -284,7 +284,7 @@ private void UpdateStream(IMessageDelivery delivery, IMessageH currentJson = JsonSerializer.Deserialize(delivery.Message.Change.Content); try { - SetCurrent(new ChangeItem( + SetCurrent(hub, new ChangeItem( currentJson.Value.Deserialize(Host.JsonSerializerOptions)!, StreamId, Host.Version)); @@ -301,7 +301,7 @@ private void UpdateStream(IMessageDelivery delivery, IMessageH (currentJson, var patch) = delivery.Message.UpdateJsonElement(currentJson, hub.JsonSerializerOptions); try { - SetCurrent(this.ToChangeItem(Current!.Value!, + SetCurrent(hub, this.ToChangeItem(Current!.Value!, currentJson.Value, patch, delivery.Message.ChangedBy ?? ClientId)); diff --git a/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs b/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs index 82e025175..2d309a512 100644 --- a/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs +++ b/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs @@ -60,9 +60,11 @@ this MessageHubConfiguration config data.AddPartitionedHubSource( c => c.WithType(td => new TransactionalDataAddress(td.Year, td.BusinessUnit)) + .InitializingPartitions(new TransactionalDataAddress(2024, "1"), new TransactionalDataAddress(2024, "2")) ) .AddPartitionedHubSource( c => c.WithType(cd => new(cd.Year, cd.BusinessUnit)) + .InitializingPartitions(new ComputedDataAddress(2024, "1"), new ComputedDataAddress(2024, "2")) ) .AddHubSource( new ReferenceDataAddress(), @@ -79,7 +81,7 @@ this MessageHubConfiguration config format => format.WithAutoMappings().WithImportFunction(ImportFunction) ) ); - + private static EntityStore ImportFunction( ImportRequest request, diff --git a/test/MeshWeaver.Data.TestDomain/TransactionalData.cs b/test/MeshWeaver.Data.TestDomain/TransactionalData.cs index 81f9a3f5e..c4f24aa52 100644 --- a/test/MeshWeaver.Data.TestDomain/TransactionalData.cs +++ b/test/MeshWeaver.Data.TestDomain/TransactionalData.cs @@ -22,5 +22,5 @@ public record BusinessUnit([property: Key] string SystemName, string DisplayName public record ImportAddress(int Year) : Address(nameof(ImportAddress), Year.ToString()); public record ReferenceDataAddress() : Address(nameof(ReferenceDataAddress), "1"); -public record ComputedDataAddress(int Year, string BusinessUnit) : Address(nameof(ComputedDataAddress), $"{Year}/{BusinessUnit}"); -public record TransactionalDataAddress(int Year, string BusinessUnit) : Address(nameof(TransactionalData), $"{Year}/{BusinessUnit}"); +public record ComputedDataAddress(int Year, string BusinessUnit) : Address(nameof(ComputedDataAddress), $"{Year}-{BusinessUnit}"); +public record TransactionalDataAddress(int Year, string BusinessUnit) : Address(nameof(TransactionalData), $"{Year}-{BusinessUnit}"); diff --git a/test/MeshWeaver.Import.Test/ImportTest.cs b/test/MeshWeaver.Import.Test/ImportTest.cs index 014b21db4..2581b49c2 100644 --- a/test/MeshWeaver.Import.Test/ImportTest.cs +++ b/test/MeshWeaver.Import.Test/ImportTest.cs @@ -53,7 +53,6 @@ public async Task DistributedImportTest() { // arrange var client = GetClient(); - var timeout = 20.Seconds(); var importRequest = new ImportRequest(VanillaDistributedCsv) { Format = TestHubSetup.CashflowImportFormat, @@ -80,11 +79,12 @@ public async Task DistributedImportTest() importResponse.Message.Log.Status.Should().Be(ActivityStatus.Succeeded); Logger.LogInformation("DistributedImportTest {TestId}: Getting transactional workspace", testId); - var transactionalItems1 = await (GetWorkspace( - Router.GetHostedHub(new TransactionalDataAddress(2024, "1")) - )) + var workspace = GetWorkspace( + Router.GetHostedHub(new TransactionalDataAddress(2024, "1")) + ); + var transactionalItems1 = await workspace .GetObservable() - .Timeout(timeout) + .Timeout(5.Seconds()) .FirstAsync(x => x.Count > 1); Logger.LogInformation("DistributedImportTest {TestId}: Got {Count} transactional items", testId, transactionalItems1.Count); @@ -93,7 +93,7 @@ public async Task DistributedImportTest() Router.GetHostedHub(new ComputedDataAddress(2024, "1")) )) .GetObservable() - .Timeout(timeout) + .Timeout(5.Seconds()) .FirstAsync(x => x is { Count: > 0 }); Logger.LogInformation("DistributedImportTest {TestId}: Got {Count} computed items", testId, computedItems1.Count); From 80f4ea6db2d967cc0195670ca774c9e986bec7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 13:05:49 +0100 Subject: [PATCH 25/57] correcting startup mechanissm --- src/MeshWeaver.Data/DataContext.cs | 4 +- src/MeshWeaver.Data/DataExtensions.cs | 1 + .../GenericUnpartitionedDataSource.cs | 1 + src/MeshWeaver.Data/WorkspaceOperations.cs | 6 + src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 1 - .../IMessageService.cs | 6 +- .../InitializeHubRequest.cs | 2 +- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 104 ++++++++---------- .../MessageHubConfiguration.cs | 4 +- .../MessageService.cs | 24 +++- .../CollectionPluginImportTest.cs | 13 ++- .../CollectionSourceImportTest.cs | 7 +- 12 files changed, 98 insertions(+), 75 deletions(-) diff --git a/src/MeshWeaver.Data/DataContext.cs b/src/MeshWeaver.Data/DataContext.cs index 1c927f9b9..bbdf686bf 100644 --- a/src/MeshWeaver.Data/DataContext.cs +++ b/src/MeshWeaver.Data/DataContext.cs @@ -11,7 +11,6 @@ namespace MeshWeaver.Data; public sealed record DataContext : IDisposable { public const string InitializationGateName = "DataContextInit"; - private const string DataContextGateName = InitializationGateName; public ITypeRegistry TypeRegistry { get; } @@ -100,6 +99,7 @@ public void Initialize() { dataSource.Initialize(); tasks.Add(dataSource.Initialized); + initialized.Add(dataSource.Reference); } Task.WhenAll(tasks) @@ -115,6 +115,7 @@ public void Initialize() } else { + Hub.OpenGate(InitializationGateName); logger.LogDebug("Finished initialization of DataContext for {Address}", Hub.Address); } }, TaskScheduler.Default); @@ -122,6 +123,7 @@ public void Initialize() public IEnumerable MappedTypes => DataSourcesByType.Keys; private readonly List tasks = new(); + private readonly List initialized = new(); public void Dispose() { foreach (var dataSource in DataSourcesById.Values) diff --git a/src/MeshWeaver.Data/DataExtensions.cs b/src/MeshWeaver.Data/DataExtensions.cs index 238695b19..8a6a70d68 100644 --- a/src/MeshWeaver.Data/DataExtensions.cs +++ b/src/MeshWeaver.Data/DataExtensions.cs @@ -90,6 +90,7 @@ Func dataPluginConfiguration .WithType(typeof(ActivityAddress), ActivityAddress.TypeName) .WithType(typeof(ActivityLog), nameof(ActivityLog)) .RegisterDataEvents() + .WithInitializationGate(DataContext.InitializationGateName) ; } diff --git a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs index 6bbfcda62..06cca06c0 100644 --- a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs @@ -149,6 +149,7 @@ public ISynchronizationStream GetStreamForPartition(object? partiti { if (Streams.TryGetValue(partition ?? Id, out var ret)) return ret; + Logger.LogDebug("Creating new stream for Id {Id} and Partition {Partition}", Id, partition); Streams[partition ?? Id] = ret = CreateStream(identity); return ret; } diff --git a/src/MeshWeaver.Data/WorkspaceOperations.cs b/src/MeshWeaver.Data/WorkspaceOperations.cs index 2362a8c2b..b6ea00238 100644 --- a/src/MeshWeaver.Data/WorkspaceOperations.cs +++ b/src/MeshWeaver.Data/WorkspaceOperations.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using System.ComponentModel.DataAnnotations; +using System.Data; using Json.Patch; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; @@ -130,6 +131,11 @@ private static void UpdateStreams(this IWorkspace workspace, DataChangeRequest c } var stream = group.Key.DataSource.GetStreamForPartition(group.Key.Partition); + if (stream is null) + throw new DataException($"Data source {group.Key.DataSource.Reference} does not have a stream for partition {group.Key.Partition}"); + if (!stream.Hub.Started.IsCompleted) + throw new DataException($"Data source {group.Key.DataSource.Reference} for partition {group.Key.Partition} is not initialized."); + // Start sub-activity for data update var subActivity = activity?.StartSubActivity(ActivityCategory.DataUpdate); diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index 4564d2b03..50e6614f1 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -108,7 +108,6 @@ IMessageHub GetHostedHub(TAddress address, Func disposeAction); JsonSerializerOptions JsonSerializerOptions { get; } MessageHubRunLevel RunLevel { get; } - IDisposable Defer(Predicate deferredFilter); /// /// Opens a named initialization gate, allowing all deferred messages to be processed. diff --git a/src/MeshWeaver.Messaging.Hub/IMessageService.cs b/src/MeshWeaver.Messaging.Hub/IMessageService.cs index 747062dfb..d7c50f41a 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageService.cs @@ -1,12 +1,10 @@ -using System.Text.Json; - -namespace MeshWeaver.Messaging; +namespace MeshWeaver.Messaging; internal interface IMessageService : IAsyncDisposable { Address Address { get; } - public IDisposable Defer(Predicate deferredFilter); IMessageDelivery RouteMessageAsync(IMessageDelivery message, CancellationToken cancellationToken); IMessageDelivery? Post(TMessage message, PostOptions opt); internal void Start(); + internal IDisposable Defer(Predicate predicate); } diff --git a/src/MeshWeaver.Messaging.Hub/InitializeHubRequest.cs b/src/MeshWeaver.Messaging.Hub/InitializeHubRequest.cs index d8517441a..9519f3ff7 100644 --- a/src/MeshWeaver.Messaging.Hub/InitializeHubRequest.cs +++ b/src/MeshWeaver.Messaging.Hub/InitializeHubRequest.cs @@ -4,4 +4,4 @@ /// Request to initialize a message hub during startup. /// Used to defer messages until initialization is complete. /// -public record InitializeHubRequest(IDisposable Deferral); +public record InitializeHubRequest(); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index bcda60ad7..739231a25 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -25,7 +25,6 @@ public void InvokeAsync(Func action, Func> callbacks = new(); private readonly HashSet pendingCallbackCancellations = new(); - private readonly ConcurrentDictionary initializationGates = new(); private readonly ILogger logger; public MessageHubConfiguration Configuration { get; } @@ -38,7 +37,8 @@ public void InvokeAsync(Func action, Func rules = new(); private readonly Lock messageHandlerRegistrationLock = new(); private readonly Lock typeRegistryLock = new(); - + private readonly IDisposable startupDeferral; + private readonly ConcurrentDictionary> gates; public MessageHub( IServiceProvider serviceProvider, HostedHubsCollection hostedHubs, @@ -82,19 +82,21 @@ public MessageHub( } Register(ExecuteRequest); Register(HandleCallbacks); - - messageService.Start(); + // Store gate names from configuration for tracking which gates are still open + // All gates reference the same single deferral + gates = new(Configuration.InitializationGates); // Create a single deferral that combines all allow-list predicates // Messages are allowed if they match InitializationAllowList OR any of the InitializationGates - var singleDeferral = Defer(delivery => + startupDeferral = messageService.Defer(delivery => { // Check all gate predicates - foreach (var (name, allowDuringInit) in Configuration.InitializationGates) + foreach (var (name, allowDuringInit) in gates) { if (allowDuringInit(delivery)) { - logger.LogDebug("Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", delivery.Message.GetType().Name, delivery.Id, name, Address); return false; // Don't defer - message is allowed by this gate } @@ -103,16 +105,9 @@ public MessageHub( // Message doesn't match any allow-list - defer it return true; }); + messageService.Start(); - // Store gate names from configuration for tracking which gates are still open - // All gates reference the same single deferral - foreach (var (name, _) in configuration.InitializationGates) - { - initializationGates[name] = singleDeferral; - logger.LogDebug("Registered initialization gate '{Name}' for hub {Address}", name, Address); - } - - Post(new InitializeHubRequest(singleDeferral)); + Post(new InitializeHubRequest()); } private IMessageDelivery HandlePingRequest(IMessageDelivery request) @@ -125,24 +120,16 @@ private async Task HandleInitialize(IMessageDelivery IMessageHub.HandleMessageAsync( IMessageDelivery delivery, CancellationToken cancellationToken @@ -669,7 +684,7 @@ public void Dispose() private void DisposeImpl() { // Open all remaining initialization gates to release any buffered messages - foreach (var gateName in initializationGates.Keys.ToArray()) + foreach (var gateName in gates.Keys.ToArray()) { OpenGate(gateName); } @@ -865,32 +880,7 @@ private void CancelCallbacks() private readonly ConcurrentBag> asyncDisposeActions = new(); private readonly ConcurrentBag> disposeActions = new(); - public IDisposable Defer(Predicate deferredFilter) => - messageService.Defer(deferredFilter); - - public bool OpenGate(string name) - { - if (initializationGates.TryRemove(name, out var gate)) - { - logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); - // If this was the last gate, dispose the single deferral and mark hub as started - if (initializationGates.IsEmpty) - { - gate.Dispose(); // Dispose the single deferral when last gate opens - if (RunLevel < MessageHubRunLevel.Started) - { - RunLevel = MessageHubRunLevel.Started; - hasStarted.SetResult(); - logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); - } - } - - return true; - } - logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, Address); - return false; - } private readonly ConcurrentDictionary<(string Conext, Type Type), object?> properties = new(); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 52abef918..c545576d8 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -40,8 +40,8 @@ public MessageHubConfiguration(IServiceProvider? parentServiceProvider, Address /// Unique name for this initialization gate /// Predicate that determines which messages are allowed during initialization (e.g. InitializeHubRequest, SetCurrentRequest) /// Updated configuration - public MessageHubConfiguration WithInitializationGate(string name, Predicate allowDuringInit) - => this with { InitializationGates = InitializationGates.SetItem(name, allowDuringInit) }; + public MessageHubConfiguration WithInitializationGate(string name, Predicate? allowDuringInit = null) + => this with { InitializationGates = InitializationGates.SetItem(name, allowDuringInit ?? (_ => false)) }; public IMessageHub? ParentHub { diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 44ac0f3d9..be1d5e584 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -21,7 +21,9 @@ public class MessageService : IMessageService private readonly AsyncDelivery deliveryPipeline; private readonly DeferralContainer deferralContainer; private readonly CancellationTokenSource hangDetectionCts = new(); + private readonly TaskCompletionSource startupCompletionSource = new(); + //private volatile int pendingStartupMessages; private JsonSerializerOptions? loggingSerializerOptions; @@ -33,7 +35,7 @@ public MessageService( ILogger logger, IMessageHub hub, IMessageHub? parentHub - ) + ) { Address = address; ParentHub = parentHub; @@ -41,12 +43,19 @@ public MessageService( this.hub = hub; deferralContainer = new DeferralContainer(ScheduleExecution, ReportFailure); + + deliveryAction = new(x => x.Invoke()); - postPipeline = hub.Configuration.PostPipeline.Aggregate(new SyncPipelineConfig(hub, d => d), (p, c) => c.Invoke(p)).SyncDelivery; + postPipeline = hub.Configuration.PostPipeline + .Aggregate(new SyncPipelineConfig(hub, d => d), (p, c) => c.Invoke(p)).SyncDelivery; hierarchicalRouting = new HierarchicalRouting(hub, parentHub); - deliveryPipeline = hub.Configuration.DeliveryPipeline.Aggregate(new AsyncPipelineConfig(hub, (d, _) => Task.FromResult(deferralContainer.DeliverMessage(d))), (p, c) => c.Invoke(p)).AsyncDelivery; + deliveryPipeline = hub.Configuration.DeliveryPipeline + .Aggregate(new AsyncPipelineConfig(hub, (d, _) => Task.FromResult(deferralContainer.DeliverMessage(d))), + (p, c) => c.Invoke(p)).AsyncDelivery; } + + void IMessageService.Start() { // Ensure the execution buffer is linked before we start processing @@ -56,6 +65,13 @@ void IMessageService.Start() buffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = true }); } + + public IDisposable Defer(Predicate predicate) + { + return deferralContainer.Defer(predicate); + } + + private IMessageDelivery ReportFailure(IMessageDelivery delivery) { logger.LogWarning("An exception occurred processing {MessageType} (ID: {MessageId}) in {Address}", @@ -87,8 +103,6 @@ private IMessageDelivery ReportFailure(IMessageDelivery delivery) public Address Address { get; } public IMessageHub? ParentHub { get; } - public IDisposable Defer(Predicate deferredFilter) => - deferralContainer.Defer(deferredFilter); IMessageDelivery IMessageService.RouteMessageAsync(IMessageDelivery delivery, CancellationToken cancellationToken) => ScheduleNotify(delivery, cancellationToken); diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs index 1b2c24598..fa417d9c7 100644 --- a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Linq; using System.Reactive.Linq; @@ -38,7 +38,16 @@ protected override MessageHubConfiguration ConfigureRouter(MessageHubConfigurati .WithRoutes(forward => forward .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) - .RouteAddressToHostedHub(c => c.ConfigureImportHub()) + .RouteAddressToHostedHub(c => c + .AddData(data => data.AddHubSource( + new ReferenceDataAddress(), + dataSource => + dataSource.WithType().WithType() + ) + .AddSource(dataSource => dataSource.WithType(t => t) + ) + ) + ) ); } diff --git a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs index c11a6ebc2..521c25a3a 100644 --- a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.IO; using System.Linq; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using FluentAssertions.Extensions; @@ -84,10 +85,12 @@ public async Task ImportFromCollectionSource_WithSubfolder_ShouldResolveAndImpor var importRequest = new ImportRequest(new CollectionSource("TestCollection", "test-data.csv")); // Act + var token = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken, + new CancellationTokenSource(5.Seconds()).Token).Token; var importResponse = await client.AwaitResponse( importRequest, o => o.WithTarget(new ImportAddress(2024)), - TestContext.Current.CancellationToken + token ); // Assert From b4a0b815db950438b24b336920ec1ddeeb9c6c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 13:23:17 +0100 Subject: [PATCH 26/57] fixing import unit test setup --- src/MeshWeaver.Messaging.Hub/IMessageService.cs | 1 + src/MeshWeaver.Messaging.Hub/MessageHub.cs | 10 ++++++++++ .../MessageHubConfiguration.cs | 4 ++-- src/MeshWeaver.Messaging.Hub/MessageService.cs | 7 +++++++ test/MeshWeaver.Data.TestDomain/TestHubSetup.cs | 9 +++++++++ .../CollectionPluginImportTest.cs | 15 +-------------- .../CollectionSourceImportTest.cs | 6 +----- test/MeshWeaver.Import.Test/ImportTest.cs | 9 +-------- test/appsettings.json | 6 +++--- 9 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/IMessageService.cs b/src/MeshWeaver.Messaging.Hub/IMessageService.cs index d7c50f41a..3456ddefe 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageService.cs @@ -7,4 +7,5 @@ internal interface IMessageService : IAsyncDisposable IMessageDelivery? Post(TMessage message, PostOptions opt); internal void Start(); internal IDisposable Defer(Predicate predicate); + internal void NotifyStartupFailure(); } diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 739231a25..a6b05bc92 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -106,10 +106,19 @@ public MessageHub( return true; }); messageService.Start(); + startupTimer = new(StartupTimeout, null, Configuration.StartupTimeout, Timeout.InfiniteTimeSpan); Post(new InitializeHubRequest()); + } + public void StartupTimeout(object? _) + { + messageService.NotifyStartupFailure(); + } + + + private readonly Timer startupTimer; private IMessageDelivery HandlePingRequest(IMessageDelivery request) { Post(new PingResponse(), o => o.ResponseFor(request)); @@ -292,6 +301,7 @@ public bool OpenGate(string name) { if (RunLevel < MessageHubRunLevel.Started) { + startupTimer.Dispose(); RunLevel = MessageHubRunLevel.Started; hasStarted.SetResult(); startupDeferral.Dispose(); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index c545576d8..576c85e4b 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -203,9 +203,9 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli }); } internal ImmutableList> DeliveryPipeline { get; set; } - internal long StartupTimeout { get; init; } + internal TimeSpan StartupTimeout { get; init; } = new(0, 0, 10); // Default 10 seconds - public MessageHubConfiguration WithStartupTimeout(long timeout) => this with { StartupTimeout = timeout }; + public MessageHubConfiguration WithStartupTimeout(TimeSpan timeout) => this with { StartupTimeout = timeout }; public MessageHubConfiguration AddDeliveryPipeline(Func pipeline) => this with { DeliveryPipeline = DeliveryPipeline.Add(pipeline) }; private AsyncPipelineConfig UserServiceDeliveryPipeline(AsyncPipelineConfig asyncPipeline) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index be1d5e584..24bac7e0b 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -71,6 +71,13 @@ public IDisposable Defer(Predicate predicate) return deferralContainer.Defer(predicate); } + public void NotifyStartupFailure() + { + // TODO V10: See that we respond to each message (31.10.2025, Roland Buergi) + throw new DeliveryFailureException( + $"Message hub {Address} failed to initialize in {hub.Configuration.StartupTimeout}"); + } + private IMessageDelivery ReportFailure(IMessageDelivery delivery) { diff --git a/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs b/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs index 2d309a512..f0e98e6e3 100644 --- a/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs +++ b/test/MeshWeaver.Data.TestDomain/TestHubSetup.cs @@ -52,6 +52,15 @@ this MessageHubConfiguration configuration public const string CashflowImportFormat = nameof(CashflowImportFormat); + public static MessageHubConfiguration ConfigureImportRouter(this MessageHubConfiguration config) + => config.WithRoutes(forward => + forward + .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) + .RouteAddressToHostedHub(c => + c.ConfigureTransactionalModel((TransactionalDataAddress)c.Address)) + .RouteAddressToHostedHub(c => c.ConfigureComputedModel()) + .RouteAddressToHostedHub(c => c.ConfigureImportHub()) + ); public static MessageHubConfiguration ConfigureImportHub( this MessageHubConfiguration config ) => diff --git a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs index fa417d9c7..88a37519b 100644 --- a/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionPluginImportTest.cs @@ -35,20 +35,7 @@ protected override MessageHubConfiguration ConfigureRouter(MessageHubConfigurati .WithTypes(typeof(ImportAddress), typeof(ImportRequest), typeof(CollectionSource)) .AddContentCollections() .AddFileSystemContentCollection("TestCollection", _ => _testFilesPath) - .WithRoutes(forward => - forward - .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) - .RouteAddressToHostedHub(c => c - .AddData(data => data.AddHubSource( - new ReferenceDataAddress(), - dataSource => - dataSource.WithType().WithType() - ) - .AddSource(dataSource => dataSource.WithType(t => t) - ) - ) - ) - ); + .ConfigureImportRouter(); } [Fact] diff --git a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs index 521c25a3a..46d7e9f94 100644 --- a/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs +++ b/test/MeshWeaver.Import.Test/CollectionSourceImportTest.cs @@ -34,11 +34,7 @@ protected override MessageHubConfiguration ConfigureRouter(MessageHubConfigurati return base.ConfigureRouter(configuration) .AddContentCollections() .AddFileSystemContentCollection("TestCollection", _ => _testFilesPath) - .WithRoutes(forward => - forward - .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) - .RouteAddressToHostedHub(c => c.ConfigureImportHub()) - ); + .ConfigureImportRouter(); } [Fact] diff --git a/test/MeshWeaver.Import.Test/ImportTest.cs b/test/MeshWeaver.Import.Test/ImportTest.cs index 2581b49c2..bd15cbd2f 100644 --- a/test/MeshWeaver.Import.Test/ImportTest.cs +++ b/test/MeshWeaver.Import.Test/ImportTest.cs @@ -23,14 +23,7 @@ public class ImportTest(ITestOutputHelper output) : HubTestBase(output) protected override MessageHubConfiguration ConfigureRouter( MessageHubConfiguration configuration) { - return base.ConfigureRouter(configuration) - .WithRoutes(forward => - forward - .RouteAddressToHostedHub(c => c.ConfigureReferenceDataModel()) - .RouteAddressToHostedHub(c => c.ConfigureTransactionalModel((TransactionalDataAddress)c.Address)) - .RouteAddressToHostedHub(c => c.ConfigureComputedModel()) - .RouteAddressToHostedHub(c => c.ConfigureImportHub()) - ); + return base.ConfigureRouter(configuration).ConfigureImportRouter(); } diff --git a/test/appsettings.json b/test/appsettings.json index 7077e231d..03dcfca3d 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -2,9 +2,9 @@ "Logging": { "LogLevel": { "Default": "Information", - "MeshWeaver.Import": "Warning", - "MeshWeaver.Data": "Warning", - "MeshWeaver.Messaging": "Warning", + "MeshWeaver": "Debug", + "MeshWeaver.Data": "Debug", + "MeshWeaver.Messaging": "Debug", "Microsoft": "Warning", "System": "Warning" }, From c46aa1f3f57ce28b262aa2fd87dc0606b79c59b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 13:23:57 +0100 Subject: [PATCH 27/57] undo appsettings --- test/appsettings.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/appsettings.json b/test/appsettings.json index 03dcfca3d..7077e231d 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -2,9 +2,9 @@ "Logging": { "LogLevel": { "Default": "Information", - "MeshWeaver": "Debug", - "MeshWeaver.Data": "Debug", - "MeshWeaver.Messaging": "Debug", + "MeshWeaver.Import": "Warning", + "MeshWeaver.Data": "Warning", + "MeshWeaver.Messaging": "Warning", "Microsoft": "Warning", "System": "Warning" }, From c19bf2ea58d5edf7fe9f36e55914e3926164b812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 14:07:55 +0100 Subject: [PATCH 28/57] trying to make linking more robust --- src/MeshWeaver.Messaging.Hub/DeferralItem.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs b/src/MeshWeaver.Messaging.Hub/DeferralItem.cs index 468b8afd6..7eea6113d 100644 --- a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs +++ b/src/MeshWeaver.Messaging.Hub/DeferralItem.cs @@ -68,7 +68,18 @@ public void Dispose() // Link OUTSIDE the lock to avoid deadlock if (shouldLink) + { + // Link the deferral to execution buffer to start processing deferred messages deferral.LinkTo(executionBuffer, new DataflowLinkOptions { PropagateCompletion = true }); + + // Manually drain any messages that might be buffered + // TPL Dataflow's LinkTo doesn't retroactively pull buffered messages in all timing scenarios + // This ensures no messages are left stuck in the deferral buffer + while (deferral.TryReceive(out var message)) + { + executionBuffer.Post(message); + } + } } public void Release() From bcb25e216f70d7350554f029ddbb0d3db2288c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 17:49:36 +0100 Subject: [PATCH 29/57] Moving all gate logic to MessageService --- src/MeshWeaver.Messaging.Hub/DeferralItem.cs | 11 --- src/MeshWeaver.Messaging.Hub/IMessageHub.cs | 10 +-- .../IMessageService.cs | 6 +- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 72 +++---------------- .../MessageHubConfiguration.cs | 12 ++++ .../MessageService.cs | 66 ++++++++++++++++- 6 files changed, 91 insertions(+), 86 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs b/src/MeshWeaver.Messaging.Hub/DeferralItem.cs index 7eea6113d..468b8afd6 100644 --- a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs +++ b/src/MeshWeaver.Messaging.Hub/DeferralItem.cs @@ -68,18 +68,7 @@ public void Dispose() // Link OUTSIDE the lock to avoid deadlock if (shouldLink) - { - // Link the deferral to execution buffer to start processing deferred messages deferral.LinkTo(executionBuffer, new DataflowLinkOptions { PropagateCompletion = true }); - - // Manually drain any messages that might be buffered - // TPL Dataflow's LinkTo doesn't retroactively pull buffered messages in all timing scenarios - // This ensures no messages are left stuck in the deferral buffer - while (deferral.TryReceive(out var message)) - { - executionBuffer.Post(message); - } - } } public void Release() diff --git a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs index 50e6614f1..b558f1e6f 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageHub.cs @@ -14,7 +14,7 @@ public interface IMessageHub : IMessageHandlerRegistry, IDisposable IServiceProvider ServiceProvider { get; } Task> AwaitResponse(IRequest request) => - AwaitResponse(request, new CancellationTokenSource(DefaultTimeout).Token); + AwaitResponse(request, new CancellationTokenSource(Configuration.RequestTimeout).Token); async Task> AwaitResponse(IMessageDelivery> request, CancellationToken cancellationToken) => (IMessageDelivery)(await AwaitResponse(request, o => o, o => o, cancellationToken))!; @@ -124,12 +124,6 @@ CancellationToken cancellationToken Task? Disposal { get; } ITypeRegistry TypeRegistry { get; } -#if DEBUG - - internal static TimeSpan DefaultTimeout => TimeSpan.FromSeconds(3000); -#else - internal static TimeSpan DefaultTimeout => TimeSpan.FromSeconds(30); - -#endif + internal void Start(); } diff --git a/src/MeshWeaver.Messaging.Hub/IMessageService.cs b/src/MeshWeaver.Messaging.Hub/IMessageService.cs index 3456ddefe..1cf710280 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageService.cs @@ -5,7 +5,7 @@ internal interface IMessageService : IAsyncDisposable Address Address { get; } IMessageDelivery RouteMessageAsync(IMessageDelivery message, CancellationToken cancellationToken); IMessageDelivery? Post(TMessage message, PostOptions opt); - internal void Start(); - internal IDisposable Defer(Predicate predicate); - internal void NotifyStartupFailure(); + void Start(); + IDisposable Defer(Predicate predicate); + bool OpenGate(string name); } diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index a6b05bc92..7e685c1a3 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -34,11 +34,18 @@ public void InvokeAsync(Func action, Func rules = new(); private readonly Lock messageHandlerRegistrationLock = new(); private readonly Lock typeRegistryLock = new(); - private readonly IDisposable startupDeferral; - private readonly ConcurrentDictionary> gates; public MessageHub( IServiceProvider serviceProvider, HostedHubsCollection hostedHubs, @@ -82,43 +89,11 @@ public MessageHub( } Register(ExecuteRequest); Register(HandleCallbacks); - // Store gate names from configuration for tracking which gates are still open - // All gates reference the same single deferral - gates = new(Configuration.InitializationGates); - - // Create a single deferral that combines all allow-list predicates - // Messages are allowed if they match InitializationAllowList OR any of the InitializationGates - startupDeferral = messageService.Defer(delivery => - { - // Check all gate predicates - foreach (var (name, allowDuringInit) in gates) - { - if (allowDuringInit(delivery)) - { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, name, Address); - return false; // Don't defer - message is allowed by this gate - } - } - - // Message doesn't match any allow-list - defer it - return true; - }); messageService.Start(); - startupTimer = new(StartupTimeout, null, Configuration.StartupTimeout, Timeout.InfiniteTimeSpan); - Post(new InitializeHubRequest()); } - public void StartupTimeout(object? _) - { - messageService.NotifyStartupFailure(); - } - - - private readonly Timer startupTimer; private IMessageDelivery HandlePingRequest(IMessageDelivery request) { Post(new PingResponse(), o => o.ResponseFor(request)); @@ -292,29 +267,7 @@ CancellationToken cancellationToken public bool OpenGate(string name) { - if (gates.TryRemove(name, out var gate)) - { - logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); - - // If this was the last gate, dispose the single deferral and mark hub as started - if (gates.IsEmpty) - { - if (RunLevel < MessageHubRunLevel.Started) - { - startupTimer.Dispose(); - RunLevel = MessageHubRunLevel.Started; - hasStarted.SetResult(); - startupDeferral.Dispose(); - logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); - } - } - - return true; - } - - logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, - Address); - return false; + return messageService.OpenGate(name); } @@ -693,11 +646,6 @@ public void Dispose() private void DisposeImpl() { - // Open all remaining initialization gates to release any buffered messages - foreach (var gateName in gates.Keys.ToArray()) - { - OpenGate(gateName); - } while (disposeActions.TryTake(out var disposeAction)) disposeAction.Invoke(this); diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 576c85e4b..4db364cb7 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -204,9 +204,21 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli } internal ImmutableList> DeliveryPipeline { get; set; } internal TimeSpan StartupTimeout { get; init; } = new(0, 0, 10); // Default 10 seconds + internal TimeSpan RequestTimeout { get; init; } = new(0, 0, 30); + /// + /// Sets the timeout allowed for startup + /// + /// + /// public MessageHubConfiguration WithStartupTimeout(TimeSpan timeout) => this with { StartupTimeout = timeout }; + /// + /// Sets the timeout for callbacks (AwaitResponse) + /// + /// + /// + public MessageHubConfiguration WithRequestTimeout(TimeSpan timeout) => this with { RequestTimeout = timeout }; public MessageHubConfiguration AddDeliveryPipeline(Func pipeline) => this with { DeliveryPipeline = DeliveryPipeline.Add(pipeline) }; private AsyncPipelineConfig UserServiceDeliveryPipeline(AsyncPipelineConfig asyncPipeline) { diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 24bac7e0b..3520fe57a 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Reflection; using System.Text.Json; using System.Text.Json.Nodes; @@ -21,8 +22,10 @@ public class MessageService : IMessageService private readonly AsyncDelivery deliveryPipeline; private readonly DeferralContainer deferralContainer; private readonly CancellationTokenSource hangDetectionCts = new(); + private readonly ConcurrentDictionary> gates; private readonly TaskCompletionSource startupCompletionSource = new(); + private readonly IDisposable startupDeferral; //private volatile int pendingStartupMessages; private JsonSerializerOptions? loggingSerializerOptions; @@ -53,9 +56,36 @@ public MessageService( deliveryPipeline = hub.Configuration.DeliveryPipeline .Aggregate(new AsyncPipelineConfig(hub, (d, _) => Task.FromResult(deferralContainer.DeliverMessage(d))), (p, c) => c.Invoke(p)).AsyncDelivery; + // Store gate names from configuration for tracking which gates are still open + // All gates reference the same single deferral + gates = new(hub.Configuration.InitializationGates); + startupTimer = new(NotifyStartupFailure, null, hub.Configuration.StartupTimeout, Timeout.InfiniteTimeSpan); + + // Create a single deferral that combines all allow-list predicates + // Messages are allowed if they match InitializationAllowList OR any of the InitializationGates + startupDeferral = Defer(delivery => + { + // Check all gate predicates + foreach (var (name, allowDuringInit) in gates) + { + if (allowDuringInit(delivery)) + { + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, name, Address); + return false; // Don't defer - message is allowed by this gate + } + } + + // Message doesn't match any allow-list - defer it + return true; + }); } + private readonly Timer startupTimer; + + void IMessageService.Start() { // Ensure the execution buffer is linked before we start processing @@ -71,13 +101,40 @@ public IDisposable Defer(Predicate predicate) return deferralContainer.Defer(predicate); } - public void NotifyStartupFailure() + private void NotifyStartupFailure(object? _) { // TODO V10: See that we respond to each message (31.10.2025, Roland Buergi) throw new DeliveryFailureException( $"Message hub {Address} failed to initialize in {hub.Configuration.StartupTimeout}"); } + public bool OpenGate(string name) + { + if (gates.TryRemove(name, out _)) + { + logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); + + // If this was the last gate, dispose the single deferral and mark hub as started + if (gates.IsEmpty) + { + if (hub.RunLevel < MessageHubRunLevel.Started) + { + startupTimer.Dispose(); + hub.Start(); + startupDeferral.Dispose(); + logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); + } + } + + return true; + } + + logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, + Address); + return false; + + } + private IMessageDelivery ReportFailure(IMessageDelivery delivery) { @@ -345,6 +402,11 @@ public async ValueTask DisposeAsync() { var totalStopwatch = Stopwatch.StartNew(); logger.LogInformation("Starting disposal of message service in {Address}", Address); + // Open all remaining initialization gates to release any buffered messages + foreach (var gateName in gates.Keys.ToArray()) + { + OpenGate(gateName); + } // Dispose hang detection timer first var hangDetectionStopwatch = Stopwatch.StartNew(); From 0433e58bc6d02cc79acca36bb173e348bd7943af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Fri, 31 Oct 2025 18:13:58 +0100 Subject: [PATCH 30/57] simplifying setup around DeferralContainer --- .../DeferralContainer.cs | 31 ------ src/MeshWeaver.Messaging.Hub/DeferralItem.cs | 98 ----------------- .../IMessageService.cs | 1 - .../MessageService.cs | 103 ++++++++---------- 4 files changed, 47 insertions(+), 186 deletions(-) delete mode 100644 src/MeshWeaver.Messaging.Hub/DeferralContainer.cs delete mode 100644 src/MeshWeaver.Messaging.Hub/DeferralItem.cs diff --git a/src/MeshWeaver.Messaging.Hub/DeferralContainer.cs b/src/MeshWeaver.Messaging.Hub/DeferralContainer.cs deleted file mode 100644 index bd460cc0f..000000000 --- a/src/MeshWeaver.Messaging.Hub/DeferralContainer.cs +++ /dev/null @@ -1,31 +0,0 @@ -using MeshWeaver.Utils; - -namespace MeshWeaver.Messaging; - -public class DeferralContainer : IAsyncDisposable -{ - private readonly LinkedList deferralChain = new(); - - public DeferralContainer(SyncDelivery asyncDelivery, SyncDelivery failure) - { - deferralChain.AddFirst(new DeferralItem(_ => false, asyncDelivery, failure)); - } - - public IDisposable Defer(Predicate deferredFilter) - { - var deferralItem = deferralChain.First!; - - var deliveryLink = new DeferralItem(deferredFilter, deferralItem.Value.DeliverMessage, deferralItem.Value.Failure); - deferralChain.AddFirst(deliveryLink); - return new AnonymousDisposable(deliveryLink.Release); - } - - public IMessageDelivery DeliverMessage(IMessageDelivery delivery) => - deferralChain.First!.Value.DeliverMessage(delivery); - - public async ValueTask DisposeAsync() - { - foreach (var deferralItem in deferralChain) - await deferralItem.DisposeAsync(); - } -} diff --git a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs b/src/MeshWeaver.Messaging.Hub/DeferralItem.cs deleted file mode 100644 index 468b8afd6..000000000 --- a/src/MeshWeaver.Messaging.Hub/DeferralItem.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System.Threading.Tasks.Dataflow; - -namespace MeshWeaver.Messaging; - -public record DeferralItem : IAsyncDisposable, IDisposable -{ - private readonly SyncDelivery syncDelivery; - private readonly SyncDelivery failure; - private readonly ActionBlock executionBuffer; - private readonly BufferBlock deferral = new(); - private bool isReleased; - - public DeferralItem(Predicate Filter, SyncDelivery syncDelivery, SyncDelivery failure) - { - this.syncDelivery = syncDelivery; - this.failure = failure; - executionBuffer = new ActionBlock(d => syncDelivery(d)); - this.Filter = Filter; - } - - public IMessageDelivery Failure( - IMessageDelivery delivery - ) - => failure.Invoke(delivery); - - public IMessageDelivery DeliverMessage( - IMessageDelivery delivery - ) - { - if (Filter(delivery)) - { - deferral.Post(delivery); - return null!; - } - - try - { - // TODO V10: Add logging here. (30.07.2024, Roland Bürgi) - var ret = syncDelivery.Invoke(delivery); - if(ret is null) - return null!; - if (ret.State == MessageDeliveryState.Failed) - return failure(ret); - return ret; - } - catch (Exception e) - { - // TODO V10: Add logging here. (30.07.2024, Roland Bürgi) - var ret = delivery.Failed(e.Message); - failure.Invoke(ret); - return ret; - } - } - - private bool isLinked; - private readonly object locker = new object(); - - public void Dispose() - { - bool shouldLink; - lock (locker) - { - if (isLinked) - return; - isLinked = true; - shouldLink = true; - } - - // Link OUTSIDE the lock to avoid deadlock - if (shouldLink) - deferral.LinkTo(executionBuffer, new DataflowLinkOptions { PropagateCompletion = true }); - } - - public void Release() - { - bool shouldLink; - lock (locker) - { - if (isReleased) - return; - isReleased = true; - shouldLink = true; - } - - // Link OUTSIDE the lock to avoid deadlock - if (shouldLink) - deferral.LinkTo(executionBuffer); - } - - public async ValueTask DisposeAsync() - { - Dispose(); - deferral.Complete(); - await executionBuffer.Completion; - } - - public Predicate Filter { get; init; } -} diff --git a/src/MeshWeaver.Messaging.Hub/IMessageService.cs b/src/MeshWeaver.Messaging.Hub/IMessageService.cs index 1cf710280..878e3862f 100644 --- a/src/MeshWeaver.Messaging.Hub/IMessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/IMessageService.cs @@ -6,6 +6,5 @@ internal interface IMessageService : IAsyncDisposable IMessageDelivery RouteMessageAsync(IMessageDelivery message, CancellationToken cancellationToken); IMessageDelivery? Post(TMessage message, PostOptions opt); void Start(); - IDisposable Defer(Predicate predicate); bool OpenGate(string name); } diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 3520fe57a..7f2d435ac 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -14,18 +14,17 @@ public class MessageService : IMessageService private readonly ILogger logger; private readonly IMessageHub hub; private readonly BufferBlock>> buffer = new(); + private readonly BufferBlock>> deferredBuffer = new(); private readonly ActionBlock>> deliveryAction; private readonly BufferBlock> executionBuffer = new(); private readonly ActionBlock> executionBlock = new(f => f.Invoke(default)); private readonly HierarchicalRouting hierarchicalRouting; private readonly SyncDelivery postPipeline; private readonly AsyncDelivery deliveryPipeline; - private readonly DeferralContainer deferralContainer; private readonly CancellationTokenSource hangDetectionCts = new(); private readonly ConcurrentDictionary> gates; private readonly TaskCompletionSource startupCompletionSource = new(); - private readonly IDisposable startupDeferral; //private volatile int pendingStartupMessages; private JsonSerializerOptions? loggingSerializerOptions; @@ -45,41 +44,16 @@ public MessageService( this.logger = logger; this.hub = hub; - deferralContainer = new DeferralContainer(ScheduleExecution, ReportFailure); - - - deliveryAction = - new(x => x.Invoke()); + deliveryAction = new(x => x.Invoke()); postPipeline = hub.Configuration.PostPipeline .Aggregate(new SyncPipelineConfig(hub, d => d), (p, c) => c.Invoke(p)).SyncDelivery; hierarchicalRouting = new HierarchicalRouting(hub, parentHub); deliveryPipeline = hub.Configuration.DeliveryPipeline - .Aggregate(new AsyncPipelineConfig(hub, (d, _) => Task.FromResult(deferralContainer.DeliverMessage(d))), + .Aggregate(new AsyncPipelineConfig(hub, (d, _) => Task.FromResult(ScheduleExecution(d))), (p, c) => c.Invoke(p)).AsyncDelivery; // Store gate names from configuration for tracking which gates are still open - // All gates reference the same single deferral gates = new(hub.Configuration.InitializationGates); startupTimer = new(NotifyStartupFailure, null, hub.Configuration.StartupTimeout, Timeout.InfiniteTimeSpan); - - // Create a single deferral that combines all allow-list predicates - // Messages are allowed if they match InitializationAllowList OR any of the InitializationGates - startupDeferral = Defer(delivery => - { - // Check all gate predicates - foreach (var (name, allowDuringInit) in gates) - { - if (allowDuringInit(delivery)) - { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, name, Address); - return false; // Don't defer - message is allowed by this gate - } - } - - // Message doesn't match any allow-list - defer it - return true; - }); } @@ -91,14 +65,9 @@ void IMessageService.Start() // Ensure the execution buffer is linked before we start processing executionBuffer.LinkTo(executionBlock, new DataflowLinkOptions { PropagateCompletion = true }); - // Link the delivery buffer to the action block immediately to avoid race conditions + // Link only the main buffer to the action block initially + // The deferred buffer will be linked when all gates are opened buffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = true }); - - } - - public IDisposable Defer(Predicate predicate) - { - return deferralContainer.Defer(predicate); } private void NotifyStartupFailure(object? _) @@ -114,14 +83,17 @@ public bool OpenGate(string name) { logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); - // If this was the last gate, dispose the single deferral and mark hub as started + // If this was the last gate, link the deferred buffer and mark hub as started if (gates.IsEmpty) { if (hub.RunLevel < MessageHubRunLevel.Started) { startupTimer.Dispose(); hub.Start(); - startupDeferral.Dispose(); + // Link the deferred buffer to process all deferred messages + deferredBuffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = false }); + // Complete the deferred buffer so no new messages go there + deferredBuffer.Complete(); logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); } } @@ -132,7 +104,6 @@ public bool OpenGate(string name) logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, Address); return false; - } @@ -177,11 +148,41 @@ private IMessageDelivery ScheduleNotify(IMessageDelivery delivery, CancellationT delivery.Message.GetType().Name, Address, delivery.Id, delivery.Target); logger.LogDebug("Buffering message {MessageType} (ID: {MessageId}) in {Address}", - delivery.Message.GetType().Name, delivery.Id, Address); // Reset hang detection timer on activity (if not debugging and not already triggered) + delivery.Message.GetType().Name, delivery.Id, Address); logger.LogTrace("MESSAGE_FLOW: POSTING_TO_DELIVERY_PIPELINE | {MessageType} | Hub: {Address} | MessageId: {MessageId}", delivery.Message.GetType().Name, Address, delivery.Id); - buffer.Post(() => NotifyAsync(delivery, cancellationToken)); + + // Determine which buffer to post to based on gate predicates + var shouldDefer = !gates.IsEmpty; + if (shouldDefer) + { + // Check all gate predicates + foreach (var (name, allowDuringInit) in gates) + { + if (allowDuringInit(delivery)) + { + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, name, Address); + shouldDefer = false; + break; + } + } + } + + // Post to appropriate buffer + if (shouldDefer) + { + logger.LogDebug("Deferring message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); + } + else + { + buffer.Post(() => NotifyAsync(delivery, cancellationToken)); + } + logger.LogTrace("MESSAGE_FLOW: SCHEDULE_NOTIFY_END | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: Forwarded", delivery.Message.GetType().Name, Address, delivery.Id); return delivery.Forwarded(); @@ -428,6 +429,7 @@ public async ValueTask DisposeAsync() var bufferStopwatch = Stopwatch.StartNew(); logger.LogDebug("Completing buffers for message service in {Address}", Address); buffer.Complete(); + deferredBuffer.Complete(); executionBuffer.Complete(); logger.LogDebug("Buffers completed in {elapsed}ms for {Address}", bufferStopwatch.ElapsedMilliseconds, Address); @@ -449,24 +451,13 @@ public async ValueTask DisposeAsync() { logger.LogError(ex, "Error during delivery completion after {elapsed}ms in {Address}", deliveryStopwatch.ElapsedMilliseconds, Address); - } // Don't wait for execution completion during disposal as this disposal itself + } + + // Don't wait for execution completion during disposal as this disposal itself // runs as an execution and might cause deadlocks waiting for itself logger.LogDebug("Skipping execution completion wait during disposal for {Address}", Address); - // Wait for startup processing to complete before disposing deferrals - var deferralsStopwatch = Stopwatch.StartNew(); - try - { - logger.LogDebug("Awaiting finishing deferrals in {Address}", Address); - await deferralContainer.DisposeAsync(); - logger.LogDebug("Deferrals completed successfully in {elapsed}ms for {Address}", - deferralsStopwatch.ElapsedMilliseconds, Address); - } - catch (Exception ex) - { - logger.LogError(ex, "Error during deferrals disposal after {elapsed}ms in {Address}", - deferralsStopwatch.ElapsedMilliseconds, Address); - } // Complete the startup task if it's still pending + // Complete the startup task if it's still pending try { if (!startupCompletionSource.Task.IsCompleted) From 20da20509c9266e1769a9327ac31863e5cb3238a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 01:20:28 +0100 Subject: [PATCH 31/57] Update modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../LayoutAreas/RiskMapLayoutArea.cs | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs index e68b14622..e7467f59f 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs @@ -45,20 +45,28 @@ public static IObservable RiskMap(LayoutAreaHost host, RenderingConte } var mapControl = BuildGoogleMapControl(geocodedRisks); - var riskDetailsSubject = new ReplaySubject(1); - riskDetailsSubject.OnNext(null); - mapControl = mapControl.WithClickAction(ctx => riskDetailsSubject.OnNext(ctx.Payload?.ToString())); - - return Controls.Stack - .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Title("Risk Map", 2)) - .WithView(mapControl) - .WithView(GeocodingArea) - .WithView(Controls.Title("Risk Details", 3)) - .WithView((h, c) => riskDetailsSubject - .SelectMany(id => id == null ? - Observable.Return(Controls.Html("Click marker to see details.")) : RenderRiskDetails(host.Hub, id)) - ); + + return Observable.Using( + () => new ReplaySubject(1), + riskDetailsSubject => + { + riskDetailsSubject.OnNext(null); + var mapControlWithClick = mapControl.WithClickAction(ctx => riskDetailsSubject.OnNext(ctx.Payload?.ToString())); + + return Observable.Return( + Controls.Stack + .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) + .WithView(Controls.Title("Risk Map", 2)) + .WithView(mapControlWithClick) + .WithView(GeocodingArea) + .WithView(Controls.Title("Risk Details", 3)) + .WithView((h, c) => riskDetailsSubject + .SelectMany(id => id == null ? + Observable.Return(Controls.Html("Click marker to see details.")) : RenderRiskDetails(host.Hub, id)) + ) + ); + } + ); }) .StartWith(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) From b5d9ef8a38a35bbad5c243895ee9c7155163dd22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 01:37:30 +0100 Subject: [PATCH 32/57] fixing build error --- .../LayoutAreas/RiskMapLayoutArea.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs index e7467f59f..f35463e38 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs @@ -24,24 +24,24 @@ public static IObservable RiskMap(LayoutAreaHost host, RenderingConte var pricingId = host.Hub.Address.Id; return host.Workspace.GetStream()! - .Select(risks => + .SelectMany(risks => { var riskList = risks?.ToList() ?? new List(); var geocodedRisks = riskList.Where(r => r.GeocodedLocation?.Latitude != null && r.GeocodedLocation?.Longitude != null).ToList(); if (!riskList.Any()) { - return Controls.Stack + return Observable.Return(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) - .WithView(Controls.Markdown("# Risk Map\n\n*No risks loaded. Import or add risks to begin.*")); + .WithView(Controls.Markdown("# Risk Map\n\n*No risks loaded. Import or add risks to begin.*"))); } if (!geocodedRisks.Any()) { - return Controls.Stack + return Observable.Return(Controls.Stack .WithView(PricingLayoutShared.BuildToolbar(pricingId, "RiskMap")) .WithView(Controls.Markdown($"# Risk Map\n\n*No geocoded risks found. {riskList.Count} risk(s) available but none have valid coordinates.*")) - .WithView(GeocodingArea); + .WithView(GeocodingArea)); } var mapControl = BuildGoogleMapControl(geocodedRisks); From a1cdac505df094446f432a3cde58834baad1af29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 01:51:32 +0100 Subject: [PATCH 33/57] increasing timeouts --- .../NotebookConnectionTest.cs | 5 +++-- .../DataChangeStreamUpdateTest.cs | 20 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test/MeshWeaver.Hosting.Monolith.Test/NotebookConnectionTest.cs b/test/MeshWeaver.Hosting.Monolith.Test/NotebookConnectionTest.cs index c2a0f5ae5..f143fdf31 100644 --- a/test/MeshWeaver.Hosting.Monolith.Test/NotebookConnectionTest.cs +++ b/test/MeshWeaver.Hosting.Monolith.Test/NotebookConnectionTest.cs @@ -86,8 +86,9 @@ public async Task LayoutAreas() var control = await stream .GetControlStream(area.ToString()!) - .Timeout(5.Seconds()) - .FirstAsync(x => x != null); + .Where(x => x != null) + .Timeout(10.Seconds()) + .FirstAsync(); var md = control.Should().BeOfType() diff --git a/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs b/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs index aef0934e3..262798425 100644 --- a/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs +++ b/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs @@ -290,24 +290,28 @@ await stream .FirstAsync(); // Update multiple tasks simultaneously - var updatedTasks = tasksData.Select(task => task with - { - Status = "Completed", - UpdatedAt = DateTime.UtcNow + var updatedTasks = tasksData.Select(task => task with + { + Status = "Completed", + UpdatedAt = DateTime.UtcNow }).Cast().ToArray(); var changeRequest = new DataChangeRequest().WithUpdates(updatedTasks); - + Output.WriteLine($"📤 Sending DataChangeRequest to complete all {updatedTasks.Length} tasks"); - client.Post(changeRequest, o => o.WithTarget(new HostAddress())); - // Verify all tasks are now completed - var allCompletedControl = await stream + // Set up the completion watch BEFORE posting the change to avoid race condition + var allCompletedTask = stream .GetControlStream(TaskCountArea) .Where(x => x != null && x.ToString().Contains("✅ **Completed:** 3")) .Timeout(10.Seconds()) .FirstAsync(); + client.Post(changeRequest, o => o.WithTarget(new HostAddress())); + + // Verify all tasks are now completed + var allCompletedControl = await allCompletedTask; + allCompletedControl.Should().NotBeNull(); var content = allCompletedControl.ToString(); content.Should().Contain("✅ **Completed:** 3"); From 229a1ed6b6b81cd544751653bc5365bc210eb246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 16:31:46 +0100 Subject: [PATCH 34/57] Improving threadsafety of MessageService --- src/MeshWeaver.AI/Plugins/ContentPlugin.cs | 27 +++--- .../MessageService.cs | 84 ++++++++++++------- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs index 03400c16b..c077df234 100644 --- a/src/MeshWeaver.AI/Plugins/ContentPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/ContentPlugin.cs @@ -87,7 +87,7 @@ public ContentPlugin(IMessageHub hub, ContentPluginConfig config, IAgentChat cha // Only parse from LayoutAreaReference.Id when area is "Content" or "Collection" if (chat.Context?.LayoutArea != null) { - var area = chat.Context.LayoutArea.Area?.ToString(); + var area = chat.Context.LayoutArea.Area; if (area == "Content" || area == "Collection") { var id = chat.Context.LayoutArea.Id?.ToString(); @@ -106,12 +106,9 @@ public ContentPlugin(IMessageHub hub, ContentPluginConfig config, IAgentChat cha if (config.ContextToConfigMap != null && chat.Context != null) { var contextConfig = config.ContextToConfigMap(chat.Context); - if (contextConfig != null) - { - // Add the dynamically created config to IContentService - contentService.AddConfiguration(contextConfig); - return contextConfig.Name; - } + // Add the dynamically created config to IContentService + contentService.AddConfiguration(contextConfig); + return contextConfig.Name; } // Fall back to the first collection from config as default @@ -130,7 +127,7 @@ public ContentPlugin(IMessageHub hub, ContentPluginConfig config, IAgentChat cha if (chat?.Context?.LayoutArea == null) return null; - var area = chat.Context.LayoutArea.Area?.ToString(); + var area = chat.Context.LayoutArea.Area; if (area != "Content" && area != "Collection") return null; @@ -181,15 +178,15 @@ public async Task GetContent( var extension = Path.GetExtension(resolvedFilePath).ToLowerInvariant(); if (extension == ".xlsx" || extension == ".xls") { - return await ReadExcelFileAsync(stream, resolvedFilePath, numberOfRows); + return ReadExcelFile(stream, resolvedFilePath, numberOfRows); } else if (extension == ".docx") { - return await ReadWordFileAsync(stream, resolvedFilePath, numberOfRows); + return ReadWordFile(stream, resolvedFilePath, numberOfRows); } else if (extension == ".pdf") { - return await ReadPdfFileAsync(stream, resolvedFilePath, numberOfRows); + return ReadPdfFile(stream, resolvedFilePath, numberOfRows); } // For other files, read as text @@ -222,7 +219,7 @@ public async Task GetContent( } } - private async Task ReadExcelFileAsync(Stream stream, string filePath, int? numberOfRows) + private string ReadExcelFile(Stream stream, string filePath, int? numberOfRows) { try { @@ -300,12 +297,12 @@ private static string GetExcelColumnLetter(int columnNumber) return columnLetter; } - private async Task ReadWordFileAsync(Stream stream, string filePath, int? numberOfRows) + private string ReadWordFile(Stream stream, string filePath, int? numberOfRows) { try { using var wordDoc = WordprocessingDocument.Open(stream, false); - var body = wordDoc.MainDocumentPart?.Document?.Body; + var body = wordDoc.MainDocumentPart?.Document.Body; if (body == null) return $"Word document '{filePath}' has no readable content."; @@ -359,7 +356,7 @@ private async Task ReadWordFileAsync(Stream stream, string filePath, int } } - private async Task ReadPdfFileAsync(Stream stream, string filePath, int? numberOfRows) + private string ReadPdfFile(Stream stream, string filePath, int? numberOfRows) { try { diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 7f2d435ac..c18085066 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -23,6 +23,7 @@ public class MessageService : IMessageService private readonly AsyncDelivery deliveryPipeline; private readonly CancellationTokenSource hangDetectionCts = new(); private readonly ConcurrentDictionary> gates; + private readonly Lock gateStateLock = new(); private readonly TaskCompletionSource startupCompletionSource = new(); @@ -84,17 +85,21 @@ public bool OpenGate(string name) logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); // If this was the last gate, link the deferred buffer and mark hub as started - if (gates.IsEmpty) + // Use lock to ensure atomicity with ScheduleNotify checking gates.IsEmpty + lock (gateStateLock) { - if (hub.RunLevel < MessageHubRunLevel.Started) + if (gates.IsEmpty) { - startupTimer.Dispose(); - hub.Start(); - // Link the deferred buffer to process all deferred messages - deferredBuffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = false }); - // Complete the deferred buffer so no new messages go there - deferredBuffer.Complete(); - logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); + if (hub.RunLevel < MessageHubRunLevel.Started) + { + startupTimer.Dispose(); + hub.Start(); + // Complete the deferred buffer first to prevent new messages from entering + deferredBuffer.Complete(); + // Then link it to process all buffered messages + deferredBuffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = false }); + logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); + } } } @@ -117,7 +122,7 @@ private IMessageDelivery ReportFailure(IMessageDelivery delivery) { try { - var message = delivery.Properties.TryGetValue("Error", out var error) ? error?.ToString() : $"Message delivery failed in address {Address}d}}"; + var message = delivery.Properties.TryGetValue("Error", out var error) ? error.ToString() : $"Message delivery failed in address {Address}d}}"; Post(new DeliveryFailure(delivery, message), new PostOptions(Address).ResponseFor(delivery)); } catch (Exception ex) @@ -154,29 +159,11 @@ private IMessageDelivery ScheduleNotify(IMessageDelivery delivery, CancellationT delivery.Message.GetType().Name, Address, delivery.Id); // Determine which buffer to post to based on gate predicates + // Use lock to ensure atomicity with OpenGate checking and modifying gate state var shouldDefer = !gates.IsEmpty; if (shouldDefer) { - // Check all gate predicates - foreach (var (name, allowDuringInit) in gates) - { - if (allowDuringInit(delivery)) - { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, name, Address); - shouldDefer = false; - break; - } - } - } - - // Post to appropriate buffer - if (shouldDefer) - { - logger.LogDebug("Deferring message {MessageType} (ID: {MessageId}) in {Address}", - delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); + DeferMessage(delivery, cancellationToken); } else { @@ -187,6 +174,43 @@ private IMessageDelivery ScheduleNotify(IMessageDelivery delivery, CancellationT delivery.Message.GetType().Name, Address, delivery.Id); return delivery.Forwarded(); } + + private void DeferMessage(IMessageDelivery delivery, CancellationToken cancellationToken) + { + lock (gateStateLock) + { + var shouldDefer = !gates.IsEmpty; + if (shouldDefer) + { + // Check all gate predicates + foreach (var (name, allowDuringInit) in gates) + { + if (allowDuringInit(delivery)) + { + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, name, Address); + shouldDefer = false; + break; + } + } + } + + // Post to appropriate buffer while still holding the lock + // This ensures no race condition between checking gates.IsEmpty and posting + if (shouldDefer) + { + logger.LogDebug("Deferring message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); + } + else + { + buffer.Post(() => NotifyAsync(delivery, cancellationToken)); + } + } + } + private async Task NotifyAsync(IMessageDelivery delivery, CancellationToken cancellationToken) { var name = GetMessageType(delivery); From 089049dce529a1e8e45525f59c22237e32d3a439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 18:34:36 +0100 Subject: [PATCH 35/57] introducing a test for pricing catalog. --- .../InsuranceTestBase.cs | 16 +-- .../PricingCatalogTests.cs | 115 ++++++++++++++++++ .../MessageHubConfiguration.cs | 2 +- templates/MeshWeaverApp1.Portal/Program.cs | 2 +- 4 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs diff --git a/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs b/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs index aa6d15db0..e60de47a4 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs @@ -1,9 +1,10 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using FluentAssertions.Extensions; using MeshWeaver.Data; using MeshWeaver.Hosting.Monolith.TestBase; using MeshWeaver.Insurance.Domain; using MeshWeaver.Mesh; -using System.Text.Json; -using System.Text.Json.Nodes; using Xunit; namespace MeshWeaver.Insurance.Test; @@ -15,8 +16,8 @@ protected override MeshBuilder ConfigureMesh(MeshBuilder builder) return base.ConfigureMesh(builder) .ConfigureHub(c => c .AddData() - .ConfigureInsuranceApplication() - ); + ) + .InstallAssemblies(typeof(InsuranceApplicationAttribute).Assembly.Location); } protected async Task> GetPropertyRisksAsync(PricingAddress address) @@ -41,10 +42,11 @@ protected async Task> GetPricingsAsync() var pricingsResp = await hub.AwaitResponse( new GetDataRequest(new CollectionReference(nameof(Pricing))), o => o.WithTarget(InsuranceApplicationAttribute.Address), - TestContext.Current.CancellationToken); + new CancellationTokenSource(10.Seconds()).Token); - return (pricingsResp?.Message?.Data as IEnumerable)? - .Select(x => x as Pricing ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) + return (pricingsResp.Message.Data as InstanceCollection)? + .Instances.Values + .Select(x => x as Pricing ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) .Where(x => x != null) .Cast() .ToList() diff --git a/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs b/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs new file mode 100644 index 000000000..4b130738e --- /dev/null +++ b/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs @@ -0,0 +1,115 @@ +using FluentAssertions; +using MeshWeaver.Insurance.Domain.Services; +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace MeshWeaver.Insurance.Test; + +public class PricingCatalogTests(ITestOutputHelper output) : InsuranceTestBase(output) +{ + protected override MeshBuilder ConfigureMesh(MeshBuilder builder) + { + return base.ConfigureMesh(builder) + .ConfigureServices(services => services + .AddSingleton() + ); + } + + [Fact] + public async Task GetPricingCatalog_ShouldReturnPricings() + { + // Act - Get the pricing catalog from the Insurance hub + var pricings = await GetPricingsAsync(); + + // Assert - Verify that the catalog contains pricings + pricings.Should().NotBeNull("catalog should not be null"); + pricings.Should().NotBeEmpty("catalog should contain sample pricings"); + + // Verify that pricings have required fields + pricings.All(p => !string.IsNullOrWhiteSpace(p.Id)).Should().BeTrue("all pricings should have an Id"); + pricings.All(p => !string.IsNullOrWhiteSpace(p.InsuredName)).Should().BeTrue("all pricings should have an InsuredName"); + pricings.All(p => !string.IsNullOrWhiteSpace(p.Status)).Should().BeTrue("all pricings should have a Status"); + + // Output summary + Output.WriteLine($"Successfully retrieved {pricings.Count} pricings from catalog"); + foreach (var pricing in pricings) + { + Output.WriteLine($" - {pricing.Id}: {pricing.InsuredName} ({pricing.Status}) - {pricing.LineOfBusiness}/{pricing.Country}"); + } + } + + [Fact] + public async Task GetPricingCatalog_ShouldHaveValidDimensions() + { + // Act + var pricings = await GetPricingsAsync(); + + // Assert - Verify dimension fields are populated + pricings.Should().NotBeEmpty(); + + pricings.All(p => !string.IsNullOrWhiteSpace(p.LineOfBusiness)).Should().BeTrue("all pricings should have a LineOfBusiness"); + pricings.All(p => !string.IsNullOrWhiteSpace(p.Country)).Should().BeTrue("all pricings should have a Country"); + pricings.All(p => !string.IsNullOrWhiteSpace(p.LegalEntity)).Should().BeTrue("all pricings should have a LegalEntity"); + pricings.All(p => !string.IsNullOrWhiteSpace(p.Currency)).Should().BeTrue("all pricings should have a Currency"); + + // Output dimension information + Output.WriteLine("Pricing dimensions:"); + Output.WriteLine($" Lines of Business: {string.Join(", ", pricings.Select(p => p.LineOfBusiness).Distinct())}"); + Output.WriteLine($" Countries: {string.Join(", ", pricings.Select(p => p.Country).Distinct())}"); + Output.WriteLine($" Legal Entities: {string.Join(", ", pricings.Select(p => p.LegalEntity).Distinct())}"); + Output.WriteLine($" Currencies: {string.Join(", ", pricings.Select(p => p.Currency).Distinct())}"); + } + + [Fact] + public async Task GetPricingCatalog_ShouldHaveValidDates() + { + // Act + var pricings = await GetPricingsAsync(); + + // Assert + pricings.Should().NotBeEmpty(); + + foreach (var pricing in pricings) + { + pricing.InceptionDate.Should().NotBeNull( + $"pricing {pricing.Id} should have an inception date"); + pricing.ExpirationDate.Should().NotBeNull( + $"pricing {pricing.Id} should have an expiration date"); + + if (pricing.InceptionDate.HasValue && pricing.ExpirationDate.HasValue) + { + pricing.ExpirationDate.Value.Should().BeAfter(pricing.InceptionDate.Value, + $"pricing {pricing.Id} expiration date should be after inception date"); + } + + pricing.UnderwritingYear.Should().NotBeNull( + $"pricing {pricing.Id} should have an underwriting year"); + pricing.UnderwritingYear.Should().BeGreaterThan(2000, + $"pricing {pricing.Id} should have a valid underwriting year"); + } + + Output.WriteLine($"All {pricings.Count} pricings have valid dates"); + } + + [Fact] + public async Task PricingHub_ShouldStartSuccessfully() + { + // This test verifies that the pricing hub initializes correctly + // by successfully retrieving the catalog without errors + + // Act + var pricings = await GetPricingsAsync(); + + // Assert - Hub started if we can get data + pricings.Should().NotBeNull("hub should start and return catalog"); + + // Verify the hub is accessible + Mesh.Should().NotBeNull("mesh should be initialized"); + Mesh.Address.Should().NotBeNull("mesh should have an address"); + + Output.WriteLine($"Pricing hub started successfully"); + Output.WriteLine($"Hub Address: {Mesh.Address}"); + Output.WriteLine($"Retrieved {pricings.Count} pricings from catalog"); + } +} diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 4db364cb7..9fe427d42 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -203,7 +203,7 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli }); } internal ImmutableList> DeliveryPipeline { get; set; } - internal TimeSpan StartupTimeout { get; init; } = new(0, 0, 10); // Default 10 seconds + internal TimeSpan StartupTimeout { get; init; } = new(0, 0, 30); // Default 10 seconds internal TimeSpan RequestTimeout { get; init; } = new(0, 0, 30); /// diff --git a/templates/MeshWeaverApp1.Portal/Program.cs b/templates/MeshWeaverApp1.Portal/Program.cs index da4d79c3a..713616e20 100644 --- a/templates/MeshWeaverApp1.Portal/Program.cs +++ b/templates/MeshWeaverApp1.Portal/Program.cs @@ -15,7 +15,7 @@ .ConfigureWebPortal() .ConfigurePortalMesh() .UseMonolithMesh() - .ConfigureServices(services => services.AddContentCollections()) + .ConfigureHub(hub => hub.AddContentCollections()) ); var app = builder.Build(); From aafc7b85e311e979f7fa8c2483e674c5a613f07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 22:10:18 +0100 Subject: [PATCH 36/57] Correcting deferrals in MessageService --- .../PricingCatalogTests.cs | 34 ++++- .../Serialization/SynchronizationStream.cs | 2 +- .../MessageHubConfiguration.cs | 2 +- .../MessageService.cs | 118 +++++++++--------- 4 files changed, 95 insertions(+), 61 deletions(-) diff --git a/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs b/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs index 4b130738e..cfb8c39fa 100644 --- a/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs +++ b/modules/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs @@ -1,5 +1,11 @@ -using FluentAssertions; +using System.Reactive.Linq; +using System.Text.Json; +using FluentAssertions; +using FluentAssertions.Extensions; +using MeshWeaver.Data; +using MeshWeaver.Insurance.Domain; using MeshWeaver.Insurance.Domain.Services; +using MeshWeaver.Layout; using MeshWeaver.Mesh; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -112,4 +118,30 @@ public async Task PricingHub_ShouldStartSuccessfully() Output.WriteLine($"Hub Address: {Mesh.Address}"); Output.WriteLine($"Retrieved {pricings.Count} pricings from catalog"); } + + [Fact] + public async Task GetPricingCatalog_UsingLayoutAreaReference_ShouldReturnPricingsControl() + { + // Arrange + var reference = new LayoutAreaReference("Pricings"); + var workspace = Mesh.ServiceProvider.GetRequiredService(); + + // Act - Get the remote stream using LayoutAreaReference + var stream = workspace.GetRemoteStream( + InsuranceApplicationAttribute.Address, + reference + ); + + // Get the control from the stream + var control = await stream.GetControlStream(reference.Area) + .Timeout(10.Seconds()) + .FirstAsync(x => x != null); + + // Assert + control.Should().NotBeNull("layout area should return a control"); + + // Output control information + Output.WriteLine($"Received control type: {control.GetType().Name}"); + Output.WriteLine($"Successfully retrieved Pricings layout area using GetRemoteStream"); + } } diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 74fe3cbe6..6be338468 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -256,7 +256,7 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat return request.Processed(); }) .WithInitialization(InitializeAsync) - .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full }); + .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full } || d.Message is UpdateStreamRequest); } diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 9fe427d42..18aeb5b34 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -203,7 +203,7 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli }); } internal ImmutableList> DeliveryPipeline { get; set; } - internal TimeSpan StartupTimeout { get; init; } = new(0, 0, 30); // Default 10 seconds + internal TimeSpan? StartupTimeout { get; init; } //= new(0, 0, 30); // Default 10 seconds internal TimeSpan RequestTimeout { get; init; } = new(0, 0, 30); /// diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index c18085066..710b1b6ba 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -54,11 +54,12 @@ public MessageService( (p, c) => c.Invoke(p)).AsyncDelivery; // Store gate names from configuration for tracking which gates are still open gates = new(hub.Configuration.InitializationGates); - startupTimer = new(NotifyStartupFailure, null, hub.Configuration.StartupTimeout, Timeout.InfiniteTimeSpan); + if (hub.Configuration.StartupTimeout is not null) + startupTimer = new(NotifyStartupFailure, null, hub.Configuration.StartupTimeout.Value, Timeout.InfiniteTimeSpan); } - private readonly Timer startupTimer; + private readonly Timer? startupTimer; void IMessageService.Start() @@ -82,7 +83,7 @@ public bool OpenGate(string name) { if (gates.TryRemove(name, out _)) { - logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}", name, Address); + logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}. Closed gates {Gates}", name, Address, gates.Keys); // If this was the last gate, link the deferred buffer and mark hub as started // Use lock to ensure atomicity with ScheduleNotify checking gates.IsEmpty @@ -92,7 +93,7 @@ public bool OpenGate(string name) { if (hub.RunLevel < MessageHubRunLevel.Started) { - startupTimer.Dispose(); + startupTimer?.Dispose(); hub.Start(); // Complete the deferred buffer first to prevent new messages from entering deferredBuffer.Complete(); @@ -158,63 +159,20 @@ private IMessageDelivery ScheduleNotify(IMessageDelivery delivery, CancellationT logger.LogTrace("MESSAGE_FLOW: POSTING_TO_DELIVERY_PIPELINE | {MessageType} | Hub: {Address} | MessageId: {MessageId}", delivery.Message.GetType().Name, Address, delivery.Id); - // Determine which buffer to post to based on gate predicates - // Use lock to ensure atomicity with OpenGate checking and modifying gate state - var shouldDefer = !gates.IsEmpty; - if (shouldDefer) - { - DeferMessage(delivery, cancellationToken); - } - else - { - buffer.Post(() => NotifyAsync(delivery, cancellationToken)); - } + // Always buffer to the main buffer - deferral logic will be handled in NotifyAsync + // based on whether the message is actually targeted at this hub + buffer.Post(() => NotifyAsync(delivery, cancellationToken)); logger.LogTrace("MESSAGE_FLOW: SCHEDULE_NOTIFY_END | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: Forwarded", delivery.Message.GetType().Name, Address, delivery.Id); return delivery.Forwarded(); } - private void DeferMessage(IMessageDelivery delivery, CancellationToken cancellationToken) - { - lock (gateStateLock) - { - var shouldDefer = !gates.IsEmpty; - if (shouldDefer) - { - // Check all gate predicates - foreach (var (name, allowDuringInit) in gates) - { - if (allowDuringInit(delivery)) - { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, name, Address); - shouldDefer = false; - break; - } - } - } - - // Post to appropriate buffer while still holding the lock - // This ensures no race condition between checking gates.IsEmpty and posting - if (shouldDefer) - { - logger.LogDebug("Deferring message {MessageType} (ID: {MessageId}) in {Address}", - delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); - } - else - { - buffer.Post(() => NotifyAsync(delivery, cancellationToken)); - } - } - } - private async Task NotifyAsync(IMessageDelivery delivery, CancellationToken cancellationToken) { var name = GetMessageType(delivery); - logger.LogDebug("MESSAGE_FLOW: NOTIFY_START | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", + logger.LogDebug( + "MESSAGE_FLOW: NOTIFY_START | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", name, Address, delivery.Id, delivery.Target); if (delivery.State != MessageDeliveryState.Submitted) @@ -224,7 +182,8 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc // For all other messages, wait for parent to be ready before routing if (ParentHub is not null) { - if (delivery.Target is HostedAddress ha && hub.Address.Equals(ha.Address) && ha.Host.Equals(ParentHub.Address)) + if (delivery.Target is HostedAddress ha && hub.Address.Equals(ha.Address) && + ha.Host.Equals(ParentHub.Address)) delivery = delivery.WithTarget(ha.Address); } @@ -233,8 +192,12 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc delivery = delivery.AddToRoutingPath(hub.Address); var isOnTarget = delivery.Target is null || delivery.Target.Equals(hub.Address); + + // Only defer messages that are targeted at this hub + // Messages being routed through should not be deferred if (isOnTarget) { + delivery = UnpackIfNecessary(delivery); logger.LogTrace("MESSAGE_FLOW: Unpacking message | {MessageType} | Hub: {Address} | MessageId: {MessageId}", name, Address, delivery.Id); @@ -245,17 +208,56 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc - logger.LogTrace("MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", + logger.LogTrace( + "MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", name, Address, delivery.Id, delivery.Target); delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); - logger.LogTrace("MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", + logger.LogTrace( + "MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", name, Address, delivery.Id, delivery.State); if (isOnTarget) { - logger.LogTrace("MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", - name, Address, delivery.Id); - delivery = await deliveryPipeline.Invoke(delivery, cancellationToken); + // Check if we need to defer this message + var shouldDefer = !gates.IsEmpty; + if (shouldDefer) + { + lock (gateStateLock) + { + shouldDefer = !gates.IsEmpty; + if (shouldDefer) + { + // Check all gate predicates + foreach (var (gateName, allowDuringInit) in gates) + { + if (allowDuringInit(delivery)) + { + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, gateName, Address); + shouldDefer = false; + break; + } + } + } + + // If we still need to defer, post to deferred buffer and return + if (shouldDefer) + { + logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); + return delivery.Forwarded(); + } + } + + logger.LogTrace( + "MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", + name, Address, delivery.Id); + } + + return await deliveryPipeline.Invoke(delivery, cancellationToken); + } return delivery; From 2b095692a9f17eb231cc8ea97814fb35ffb2dff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 22:26:52 +0100 Subject: [PATCH 37/57] corercting locking in MessageService --- .../CollectionLayoutArea.cs | 4 +- .../MessageService.cs | 53 +++++++++---------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/MeshWeaver.ContentCollections/CollectionLayoutArea.cs b/src/MeshWeaver.ContentCollections/CollectionLayoutArea.cs index 50184bf23..702565516 100644 --- a/src/MeshWeaver.ContentCollections/CollectionLayoutArea.cs +++ b/src/MeshWeaver.ContentCollections/CollectionLayoutArea.cs @@ -1,4 +1,5 @@ -using MeshWeaver.Layout; +using System.ComponentModel; +using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; namespace MeshWeaver.ContentCollections; @@ -13,6 +14,7 @@ public static class CollectionLayoutArea /// Renders a file browser for the specified collection at the given path. /// The collection and path are parsed from the host reference ID in format: {collection}/{path} /// + [Browsable(false)] public static UiControl Collection(LayoutAreaHost host, RenderingContext _) { var split = host.Reference.Id?.ToString()?.Split("/", StringSplitOptions.RemoveEmptyEntries); diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 710b1b6ba..9e2857072 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -218,46 +218,41 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc if (isOnTarget) { - // Check if we need to defer this message - var shouldDefer = !gates.IsEmpty; - if (shouldDefer) + // Check if we need to defer this message - must check inside lock to avoid race with OpenGate + bool shouldDefer; + lock (gateStateLock) { - lock (gateStateLock) + shouldDefer = !gates.IsEmpty; + if (shouldDefer) { - shouldDefer = !gates.IsEmpty; - if (shouldDefer) + // Check all gate predicates + foreach (var (gateName, allowDuringInit) in gates) { - // Check all gate predicates - foreach (var (gateName, allowDuringInit) in gates) + if (allowDuringInit(delivery)) { - if (allowDuringInit(delivery)) - { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, gateName, Address); - shouldDefer = false; - break; - } + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, gateName, Address); + shouldDefer = false; + break; } } - - // If we still need to defer, post to deferred buffer and return - if (shouldDefer) - { - logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", - delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); - return delivery.Forwarded(); - } } - logger.LogTrace( - "MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", - name, Address, delivery.Id); + // If we still need to defer, post to deferred buffer and return + if (shouldDefer) + { + logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); + return delivery.Forwarded(); + } } + logger.LogTrace( + "MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", + name, Address, delivery.Id); return await deliveryPipeline.Invoke(delivery, cancellationToken); - } return delivery; From 696f684f551ae15de21b05905089d680ad5a0d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 22:36:39 +0100 Subject: [PATCH 38/57] avoid completing the deferral buffer. --- src/MeshWeaver.Messaging.Hub/MessageService.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 9e2857072..216eed123 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -95,9 +95,8 @@ public bool OpenGate(string name) { startupTimer?.Dispose(); hub.Start(); - // Complete the deferred buffer first to prevent new messages from entering - deferredBuffer.Complete(); - // Then link it to process all buffered messages + // Link the deferred buffer to process all buffered messages + // DO NOT complete it - messages can still be posted during the race window deferredBuffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = false }); logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); } From c72cb39196fda4bbd5bbc2af3f76d57999dc601f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 22:47:50 +0100 Subject: [PATCH 39/57] improving thread safety of MessageService --- .../MessageService.cs | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 216eed123..f7b1774ba 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -194,30 +194,10 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc // Only defer messages that are targeted at this hub // Messages being routed through should not be deferred - if (isOnTarget) - { - - delivery = UnpackIfNecessary(delivery); - logger.LogTrace("MESSAGE_FLOW: Unpacking message | {MessageType} | Hub: {Address} | MessageId: {MessageId}", - name, Address, delivery.Id); - - if (delivery.State == MessageDeliveryState.Failed) - return ReportFailure(delivery); - } - - - - logger.LogTrace( - "MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", - name, Address, delivery.Id, delivery.Target); - delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); - logger.LogTrace( - "MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", - name, Address, delivery.Id, delivery.State); - if (isOnTarget) { // Check if we need to defer this message - must check inside lock to avoid race with OpenGate + // IMPORTANT: Check deferral BEFORE unpacking to avoid wasted work bool shouldDefer; lock (gateStateLock) { @@ -243,11 +223,30 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); + deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); return delivery.Forwarded(); } } + // Only unpack after we've determined we're not deferring + delivery = UnpackIfNecessary(delivery); + logger.LogTrace("MESSAGE_FLOW: Unpacking message | {MessageType} | Hub: {Address} | MessageId: {MessageId}", + name, Address, delivery.Id); + + if (delivery.State == MessageDeliveryState.Failed) + return ReportFailure(delivery); + } + + logger.LogTrace( + "MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", + name, Address, delivery.Id, delivery.Target); + delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); + logger.LogTrace( + "MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", + name, Address, delivery.Id, delivery.State); + + if (isOnTarget) + { logger.LogTrace( "MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", name, Address, delivery.Id); From b8c81f5dd39dba83a2c162cdde5a9086556551a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 23:18:20 +0100 Subject: [PATCH 40/57] trying not to duplicate routing path. --- src/MeshWeaver.Messaging.Hub/MessageService.cs | 5 +++-- test/appsettings.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index f7b1774ba..779d7dc57 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -187,8 +187,9 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc } - // Add current address to routing path - delivery = delivery.AddToRoutingPath(hub.Address); + // Add current address to routing path (only if not already present to handle deferred messages) + if (!delivery.RoutingPath.Contains(hub.Address)) + delivery = delivery.AddToRoutingPath(hub.Address); var isOnTarget = delivery.Target is null || delivery.Target.Equals(hub.Address); diff --git a/test/appsettings.json b/test/appsettings.json index 7077e231d..128fe5ba4 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -4,7 +4,7 @@ "Default": "Information", "MeshWeaver.Import": "Warning", "MeshWeaver.Data": "Warning", - "MeshWeaver.Messaging": "Warning", + "MeshWeaver.Messaging": "Debug", "Microsoft": "Warning", "System": "Warning" }, From cf91a5aa3a963ecdd41cb7c79c391d35a95281dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sat, 1 Nov 2025 23:55:06 +0100 Subject: [PATCH 41/57] fixing message construction for callbacks --- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 7e685c1a3..dcfa3abec 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -6,6 +6,7 @@ using MeshWeaver.Domain; using MeshWeaver.Reflection; using MeshWeaver.ServiceProvider; +using MeshWeaver.ShortGuid; using MeshWeaver.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -361,13 +362,33 @@ private IMessageDelivery FinishDelivery(IMessageDelivery delivery) public Task AwaitResponse(object r, Func options, Func selector, CancellationToken cancellationToken = default) { - var request = r as IMessageDelivery ?? Post(r, options)!; - var response = RegisterCallback( - request.Id, - d => d, - cancellationToken - ); - var task = response + // Check if r is already a delivery (in which case it's already posted) + if (r is IMessageDelivery existingDelivery) + { + var response = RegisterCallback( + existingDelivery.Id, + d => d, + cancellationToken + ); + return response.ContinueWith(t => + { + var ret = t.Result; + return InnerCallback(existingDelivery.Id, ret, selector); + }, cancellationToken); + } + + // For new messages, we need to generate the ID first, register callback, THEN post + // to avoid race condition where response arrives before callback is registered + var messageId = Guid.NewGuid().AsString(); + var response2 = RegisterCallback(messageId, d => d, cancellationToken); + + // Now post the message with the pre-generated ID + var request = Post(r, opts => { + var configured = options(opts); + return configured.WithMessageId(messageId); + })!; + + var task = response2 .ContinueWith(t => { var ret = t.Result; From bf532b8a4ec81bf0a6ecd863f1a47444a592fa91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 00:34:55 +0100 Subject: [PATCH 42/57] better treatment of deferred messages. --- .../MessageService.cs | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 779d7dc57..aeb354941 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -224,7 +224,9 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => NotifyAsync(delivery, cancellationToken)); + // Post directly to processing, bypassing deferral check on reprocessing + // This prevents infinite deferral loops + deferredBuffer.Post(() => ProcessDeferredMessage(delivery, cancellationToken)); return delivery.Forwarded(); } } @@ -273,6 +275,39 @@ private static string ExtractJsonType(string rawJsonContent) return "Unknown"; } + /// + /// Process a deferred message, bypassing the deferral check to prevent infinite loops + /// + private async Task ProcessDeferredMessage(IMessageDelivery delivery, CancellationToken cancellationToken) + { + logger.LogDebug("Processing deferred message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + + // Add to routing path if not already present + if (!delivery.RoutingPath.Contains(hub.Address)) + delivery = delivery.AddToRoutingPath(hub.Address); + + var isOnTarget = delivery.Target is null || delivery.Target.Equals(hub.Address); + + // Skip deferral check - we're reprocessing after gates opened + if (isOnTarget) + { + delivery = UnpackIfNecessary(delivery); + + if (delivery.State == MessageDeliveryState.Failed) + return ReportFailure(delivery); + } + + delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); + + if (isOnTarget) + { + return await deliveryPipeline.Invoke(delivery, cancellationToken); + } + + return delivery; + } + private readonly CancellationTokenSource cancellationTokenSource = new(); private IMessageDelivery ScheduleExecution(IMessageDelivery delivery) { From 4b015d7547c02efc71601239ebbdc870951f9d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 12:22:24 +0100 Subject: [PATCH 43/57] placing locks outside removal of gate --- .../MessageService.cs | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index aeb354941..052ddc6ae 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -81,14 +81,15 @@ private void NotifyStartupFailure(object? _) public bool OpenGate(string name) { - if (gates.TryRemove(name, out _)) + lock (gateStateLock) { - logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}. Closed gates {Gates}", name, Address, gates.Keys); - - // If this was the last gate, link the deferred buffer and mark hub as started - // Use lock to ensure atomicity with ScheduleNotify checking gates.IsEmpty - lock (gateStateLock) + if (gates.TryRemove(name, out _)) { + logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}. Closed gates {Gates}", name, + Address, gates.Keys); + + // If this was the last gate, link the deferred buffer and mark hub as started + // Use lock to ensure atomicity with ScheduleNotify checking gates.IsEmpty if (gates.IsEmpty) { if (hub.RunLevel < MessageHubRunLevel.Started) @@ -101,9 +102,9 @@ public bool OpenGate(string name) logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); } } - } - return true; + return true; + } } logger.LogDebug("Initialization gate '{Name}' not found in hub {Address} (may have already been opened)", name, @@ -199,36 +200,40 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { // Check if we need to defer this message - must check inside lock to avoid race with OpenGate // IMPORTANT: Check deferral BEFORE unpacking to avoid wasted work - bool shouldDefer; - lock (gateStateLock) + bool shouldDefer = !gates.IsEmpty; + if (shouldDefer) { - shouldDefer = !gates.IsEmpty; - if (shouldDefer) + lock (gateStateLock) { - // Check all gate predicates - foreach (var (gateName, allowDuringInit) in gates) + shouldDefer = !gates.IsEmpty; + if (shouldDefer) { - if (allowDuringInit(delivery)) + // Check all gate predicates + foreach (var (gateName, allowDuringInit) in gates) { - logger.LogDebug( - "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", - delivery.Message.GetType().Name, delivery.Id, gateName, Address); - shouldDefer = false; - break; + if (allowDuringInit(delivery)) + { + logger.LogDebug( + "Allowing message {MessageType} (ID: {MessageId}) through gate '{GateName}' for hub {Address}", + delivery.Message.GetType().Name, delivery.Id, gateName, Address); + shouldDefer = false; + break; + } } } - } - // If we still need to defer, post to deferred buffer and return - if (shouldDefer) - { - logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", - delivery.Message.GetType().Name, delivery.Id, Address); - // Post directly to processing, bypassing deferral check on reprocessing - // This prevents infinite deferral loops - deferredBuffer.Post(() => ProcessDeferredMessage(delivery, cancellationToken)); - return delivery.Forwarded(); + // If we still need to defer, post to deferred buffer and return + if (shouldDefer) + { + logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", + delivery.Message.GetType().Name, delivery.Id, Address); + // Post directly to processing, bypassing deferral check on reprocessing + // This prevents infinite deferral loops + deferredBuffer.Post(() => ProcessDeferredMessage(delivery, cancellationToken)); + return delivery.Forwarded(); + } } + } // Only unpack after we've determined we're not deferring From 3701c3775f7c1180469c9cf192366826f3e644a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 21:02:12 +0100 Subject: [PATCH 44/57] Revert "improving thread safety of MessageService" This reverts commit c72cb39196fda4bbd5bbc2af3f76d57999dc601f. # Conflicts: # src/MeshWeaver.Messaging.Hub/MessageService.cs --- .../MessageService.cs | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 052ddc6ae..28e2ab6b5 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -196,10 +196,30 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc // Only defer messages that are targeted at this hub // Messages being routed through should not be deferred + if (isOnTarget) + { + + delivery = UnpackIfNecessary(delivery); + logger.LogTrace("MESSAGE_FLOW: Unpacking message | {MessageType} | Hub: {Address} | MessageId: {MessageId}", + name, Address, delivery.Id); + + if (delivery.State == MessageDeliveryState.Failed) + return ReportFailure(delivery); + } + + + + logger.LogTrace( + "MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", + name, Address, delivery.Id, delivery.Target); + delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); + logger.LogTrace( + "MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", + name, Address, delivery.Id, delivery.State); + if (isOnTarget) { // Check if we need to defer this message - must check inside lock to avoid race with OpenGate - // IMPORTANT: Check deferral BEFORE unpacking to avoid wasted work bool shouldDefer = !gates.IsEmpty; if (shouldDefer) { @@ -227,7 +247,7 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", delivery.Message.GetType().Name, delivery.Id, Address); - // Post directly to processing, bypassing deferral check on reprocessing + deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); // This prevents infinite deferral loops deferredBuffer.Post(() => ProcessDeferredMessage(delivery, cancellationToken)); return delivery.Forwarded(); @@ -236,25 +256,6 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc } - // Only unpack after we've determined we're not deferring - delivery = UnpackIfNecessary(delivery); - logger.LogTrace("MESSAGE_FLOW: Unpacking message | {MessageType} | Hub: {Address} | MessageId: {MessageId}", - name, Address, delivery.Id); - - if (delivery.State == MessageDeliveryState.Failed) - return ReportFailure(delivery); - } - - logger.LogTrace( - "MESSAGE_FLOW: ROUTING_TO_HIERARCHICAL | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Target: {Target}", - name, Address, delivery.Id, delivery.Target); - delivery = await hierarchicalRouting.RouteMessageAsync(delivery, cancellationToken); - logger.LogTrace( - "MESSAGE_FLOW: HIERARCHICAL_ROUTING_RESULT | {MessageType} | Hub: {Address} | MessageId: {MessageId} | Result: {State}", - name, Address, delivery.Id, delivery.State); - - if (isOnTarget) - { logger.LogTrace( "MESSAGE_FLOW: ROUTING_TO_LOCAL_EXECUTION | {MessageType} | Hub: {Address} | MessageId: {MessageId}", name, Address, delivery.Id); From 32161cf2eb62cd57ceda93982ed08f6edfb63a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 21:57:11 +0100 Subject: [PATCH 45/57] avoid duplicate messages --- src/MeshWeaver.Messaging.Hub/MessageService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index 28e2ab6b5..d2d755b18 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -247,8 +247,6 @@ private async Task NotifyAsync(IMessageDelivery delivery, Canc { logger.LogDebug("Deferring on-target message {MessageType} (ID: {MessageId}) in {Address}", delivery.Message.GetType().Name, delivery.Id, Address); - deferredBuffer.Post(() => deliveryPipeline.Invoke(delivery, cancellationToken)); - // This prevents infinite deferral loops deferredBuffer.Post(() => ProcessDeferredMessage(delivery, cancellationToken)); return delivery.Forwarded(); } From c35171e37550f0eaab34c2d1b199ed9d56443420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 22:15:27 +0100 Subject: [PATCH 46/57] linking deferredBuffer to buffer rather than action block --- src/MeshWeaver.Messaging.Hub/MessageService.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/MeshWeaver.Messaging.Hub/MessageService.cs b/src/MeshWeaver.Messaging.Hub/MessageService.cs index d2d755b18..ada54bbe0 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageService.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageService.cs @@ -88,7 +88,7 @@ public bool OpenGate(string name) logger.LogDebug("Opening initialization gate '{Name}' for hub {Address}. Closed gates {Gates}", name, Address, gates.Keys); - // If this was the last gate, link the deferred buffer and mark hub as started + // If this was the last gate, link deferred buffer to main buffer and mark hub as started // Use lock to ensure atomicity with ScheduleNotify checking gates.IsEmpty if (gates.IsEmpty) { @@ -96,9 +96,14 @@ public bool OpenGate(string name) { startupTimer?.Dispose(); hub.Start(); - // Link the deferred buffer to process all buffered messages - // DO NOT complete it - messages can still be posted during the race window - deferredBuffer.LinkTo(deliveryAction, new DataflowLinkOptions { PropagateCompletion = false }); + + // Link deferred buffer to main buffer to preserve FIFO order + // This creates a chain: deferredBuffer → buffer → deliveryAction + // All deferred messages will flow through the main buffer, ensuring they are + // processed before any new messages that arrive after the gate opens + logger.LogDebug("Linking deferred buffer to main buffer for hub {Address}", Address); + deferredBuffer.LinkTo(buffer, new DataflowLinkOptions { PropagateCompletion = false }); + logger.LogInformation("Message hub {address} fully initialized (all gates opened)", Address); } } From 30df1a79da1c484f3518b2953936522fa23196f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 22:50:14 +0100 Subject: [PATCH 47/57] removing UpdateStreamRequest from list of non-deferred messages --- src/MeshWeaver.Data/Serialization/SynchronizationStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 6be338468..74fe3cbe6 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -256,7 +256,7 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat return request.Processed(); }) .WithInitialization(InitializeAsync) - .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full } || d.Message is UpdateStreamRequest); + .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full }); } From cd393dad7ac086463e2b93677f920d464347e258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Sun, 2 Nov 2025 23:19:46 +0100 Subject: [PATCH 48/57] eliminating delayedStart logic from layout areas --- src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs index 819330862..d6fa9b320 100644 --- a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs +++ b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs @@ -48,7 +48,6 @@ public LayoutAreaHost(IWorkspace workspace, var context = new RenderingContext(reference.Area) { Layout = reference.Layout }; LayoutDefinition = uiControlService.LayoutDefinition; configuration ??= c => c; - var delayedStart = new TaskCompletionSource(); Stream = new SynchronizationStream( new(workspace.Hub.Address, reference), workspace.Hub, @@ -57,7 +56,6 @@ public LayoutAreaHost(IWorkspace workspace, c => configuration.Invoke(c) .WithInitialization(async (_, _) => { - await delayedStart.Task; return ( await LayoutDefinition .RenderAsync(this, context, new EntityStore() @@ -87,7 +85,6 @@ await LayoutDefinition ); logger = Stream.Hub.ServiceProvider.GetRequiredService>(); - delayedStart.SetResult(); } From 1611dc8cf1209e13894ce1f935fbe90d94c53407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 10:43:54 +0100 Subject: [PATCH 49/57] getting rid of blocking await in LayoutAreaHost --- .../Serialization/SynchronizationStream.cs | 26 +++++++- .../Composition/LayoutAreaHost.cs | 8 ++- src/MeshWeaver.Messaging.Hub/MessageHub.cs | 3 +- .../MessageHubConfiguration.cs | 16 +++++ .../DataChangeStreamUpdateTest.cs | 65 +++++++++---------- 5 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 74fe3cbe6..1e948fc55 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -196,7 +196,7 @@ public SynchronizationStream( private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfiguration config) { - return config + config = config .WithTypes( typeof(EntityStore), typeof(JsonElement), @@ -256,8 +256,13 @@ private MessageHubConfiguration ConfigureSynchronizationHub(MessageHubConfigurat return request.Processed(); }) .WithInitialization(InitializeAsync) - .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent { ChangeType: ChangeType.Full }); + .WithInitializationGate(SynchronizationGate, d => d.Message is SetCurrentRequest || d.Message is DataChangedEvent); + // Apply deferred initialization if configured + if (Configuration.DeferredInitialization) + config = config.WithDeferredInitialization(); + + return config; } private async Task InitializeAsync(IMessageHub hub, CancellationToken ct) @@ -384,6 +389,13 @@ public StreamConfiguration ReturnNullWhenNotPresent() internal Func ExceptionCallback { get; init; } = _ => Task.CompletedTask; + /// + /// When true, the stream's hosted hub will not automatically post InitializeHubRequest during construction. + /// Manual initialization is required by posting InitializeHubRequest to the stream's hub. + /// This is useful when the stream initialization depends on properties that are set after stream construction. + /// + internal bool DeferredInitialization { get; init; } + public StreamConfiguration WithInitialization(Func, CancellationToken, Task> init) => this with { Initialization = init }; @@ -392,4 +404,14 @@ public StreamConfiguration WithExceptionCallback(Func public StreamConfiguration WithExceptionCallback(Action exceptionCallback) => this with { ExceptionCallback = ex => { exceptionCallback(ex); return Task.CompletedTask; } }; + + /// + /// Enables deferred initialization for the stream's hosted hub. When enabled, the hub will not automatically + /// post InitializeHubRequest during construction. Manual initialization is required by posting InitializeHubRequest + /// to the stream's hub after the stream is fully constructed. + /// + /// Whether to defer initialization (default: true) + /// Updated configuration + public StreamConfiguration WithDeferredInitialization(bool deferred = true) + => this with { DeferredInitialization = deferred }; } diff --git a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs index d6fa9b320..e4b02d869 100644 --- a/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs +++ b/src/MeshWeaver.Layout/Composition/LayoutAreaHost.cs @@ -48,12 +48,14 @@ public LayoutAreaHost(IWorkspace workspace, var context = new RenderingContext(reference.Area) { Layout = reference.Layout }; LayoutDefinition = uiControlService.LayoutDefinition; configuration ??= c => c; + // Create stream with deferred initialization to avoid circular dependency + // where initialization lambda uses 'this' before Stream property is assigned Stream = new SynchronizationStream( new(workspace.Hub.Address, reference), workspace.Hub, reference, workspace.ReduceManager.ReduceTo(), - c => configuration.Invoke(c) + c => configuration.Invoke(c.WithDeferredInitialization()) .WithInitialization(async (_, _) => { return ( @@ -70,6 +72,10 @@ await LayoutDefinition return Task.CompletedTask; })); Reference = reference; + + // Manually trigger initialization now that Stream property is assigned + // This resolves the circular dependency where initialization lambda uses 'this' + Stream.Hub.Post(new InitializeHubRequest()); Stream.RegisterForDisposal(this); Stream.RegisterForDisposal( Stream.Hub.Register( diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index dcfa3abec..ba0592e19 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -91,7 +91,8 @@ public MessageHub( Register(ExecuteRequest); Register(HandleCallbacks); messageService.Start(); - Post(new InitializeHubRequest()); + if (!configuration.DeferredInitialization) + Post(new InitializeHubRequest()); } diff --git a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs index 18aeb5b34..99313adbc 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHubConfiguration.cs @@ -206,6 +206,12 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli internal TimeSpan? StartupTimeout { get; init; } //= new(0, 0, 30); // Default 10 seconds internal TimeSpan RequestTimeout { get; init; } = new(0, 0, 30); + /// + /// When true, the hub will not automatically post InitializeHubRequest during construction. + /// Manual initialization is required by posting InitializeHubRequest to the hub. + /// + internal bool DeferredInitialization { get; init; } + /// /// Sets the timeout allowed for startup /// @@ -219,6 +225,16 @@ private SyncPipelineConfig UserServicePostPipeline(SyncPipelineConfig syncPipeli /// /// public MessageHubConfiguration WithRequestTimeout(TimeSpan timeout) => this with { RequestTimeout = timeout }; + + /// + /// Enables deferred initialization. When enabled, the hub will not automatically post InitializeHubRequest + /// during construction. Manual initialization is required by posting InitializeHubRequest to the hub. + /// This is useful when the hub needs to be fully constructed before initialization can proceed. + /// + /// Whether to defer initialization (default: true) + /// Updated configuration + public MessageHubConfiguration WithDeferredInitialization(bool deferred = true) => this with { DeferredInitialization = deferred }; + public MessageHubConfiguration AddDeliveryPipeline(Func pipeline) => this with { DeliveryPipeline = DeliveryPipeline.Add(pipeline) }; private AsyncPipelineConfig UserServiceDeliveryPipeline(AsyncPipelineConfig asyncPipeline) { diff --git a/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs b/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs index 262798425..75b2639b9 100644 --- a/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs +++ b/test/MeshWeaver.Layout.Test/DataChangeStreamUpdateTest.cs @@ -10,7 +10,6 @@ using MeshWeaver.Fixture; using MeshWeaver.Layout.Composition; using MeshWeaver.Messaging; -using Xunit; namespace MeshWeaver.Layout.Test; @@ -28,7 +27,7 @@ DateTime UpdatedAt /// /// Initial test data for seeding /// - public static readonly TestTaskItem[] InitialData = + public static readonly TestTaskItem[] InitialData = [ new("task-1", "First Task", "Pending", DateTime.UtcNow.AddDays(-1), DateTime.UtcNow.AddDays(-1)), new("task-2", "Second Task", "InProgress", DateTime.UtcNow.AddHours(-2), DateTime.UtcNow.AddHours(-2)), @@ -46,8 +45,6 @@ DateTime UpdatedAt /// public class DataChangeStreamUpdateTest(ITestOutputHelper output) : HubTestBase(output) { - private const string TaskListArea = nameof(TaskListArea); - private const string TaskCountArea = nameof(TaskCountArea); /// /// Step 1: Configure host with TestTaskItem entity type and initial data @@ -68,12 +65,12 @@ protected override MessageHubConfiguration ConfigureHost(MessageHubConfiguration .AddLayout(layout => layout // Step 2: Create layout area that subscribes to stream and shows property we'll change - .WithView(TaskListArea, TaskListView) - .WithView(TaskCountArea, TaskCountView) + .WithView(nameof(TaskListView), TaskListView) + .WithView(nameof(TaskCountView), TaskCountView) ); } - protected override MessageHubConfiguration ConfigureClient(MessageHubConfiguration configuration) + protected override MessageHubConfiguration ConfigureClient(MessageHubConfiguration configuration) => base.ConfigureClient(configuration).AddLayoutClient(d => d); /// @@ -109,7 +106,7 @@ private static IObservable TaskCountView(LayoutAreaHost host, Renderi private static UiControl CreateTaskListMarkdown(IReadOnlyCollection taskItems) { var markdown = "# Task List\n\n"; - + if (!taskItems.Any()) { markdown += "*No tasks found.*"; @@ -121,7 +118,7 @@ private static UiControl CreateTaskListMarkdown(IReadOnlyCollection "⏳", - "InProgress" => "🔄", + "InProgress" => "🔄", "Completed" => "✅", _ => "❓" }; @@ -141,7 +138,7 @@ private static UiControl CreateTaskListMarkdown(IReadOnlyCollection taskItems) { var markdown = "# Task Count\n\n"; - + if (!taskItems.Any()) { markdown += "*No tasks found.*"; @@ -154,7 +151,7 @@ private static UiControl CreateTaskCountMarkdown(IReadOnlyCollection( new HostAddress(), - new LayoutAreaReference(TaskListArea) + new LayoutAreaReference(nameof(TaskListView)) ); // Verify initial data is loaded in layout area var initialControl = await stream - .GetControlStream(TaskListArea) + .GetControlStream(nameof(TaskListView)) .Timeout(10.Seconds()) .FirstAsync(x => x != null && x.ToString().Contains("First Task")); @@ -213,18 +210,18 @@ public async Task DataChangeRequest_ShouldUpdateLayoutAreaViews() Output.WriteLine($"🎯 Target task found: '{taskToUpdate.Title}' with status '{taskToUpdate.Status}'"); // Step 4: Emit DataChangeRequest to change the status - var updatedTask = taskToUpdate with - { - Status = "InProgress", - UpdatedAt = DateTime.UtcNow + var updatedTask = taskToUpdate with + { + Status = "InProgress", + UpdatedAt = DateTime.UtcNow }; var changeRequest = new DataChangeRequest().WithUpdates(updatedTask); - + Output.WriteLine($"📤 Sending DataChangeRequest to change status: {taskToUpdate.Status} → {updatedTask.Status}"); var updatedControlTask = stream - .GetControlStream(TaskListArea) + .GetControlStream(nameof(TaskListView)) .Skip(1) .Where(x => x != null && x.ToString().Contains("Status:** InProgress")) .Timeout(10.Seconds()) @@ -245,11 +242,11 @@ public async Task DataChangeRequest_ShouldUpdateLayoutAreaViews() // Additional verification: Check that task count view also updates var countStream = workspace.GetRemoteStream( new HostAddress(), - new LayoutAreaReference(TaskCountArea) + new LayoutAreaReference(nameof(TaskCountView)) ); var updatedCountControl = await countStream - .GetControlStream(TaskCountArea) + .GetControlStream(nameof(TaskCountView)) .Where(x => x != null && x.ToString().Contains("🔄 **InProgress:** 2")) // Should now have 2 InProgress tasks .Timeout(10.Seconds()) .FirstAsync(); @@ -274,12 +271,12 @@ public async Task MultipleDataChanges_ShouldUpdateLayoutAreaViews() var stream = workspace.GetRemoteStream( new HostAddress(), - new LayoutAreaReference(TaskCountArea) + new LayoutAreaReference(nameof(TaskCountView)) ); // Wait for initial data await stream - .GetControlStream(TaskCountArea) + .GetControlStream(nameof(TaskCountView)) .Timeout(5.Seconds()) .FirstAsync(x => x != null && x.ToString().Contains("Total Tasks")); @@ -302,7 +299,7 @@ await stream // Set up the completion watch BEFORE posting the change to avoid race condition var allCompletedTask = stream - .GetControlStream(TaskCountArea) + .GetControlStream(nameof(TaskCountView)) .Where(x => x != null && x.ToString().Contains("✅ **Completed:** 3")) .Timeout(10.Seconds()) .FirstAsync(); @@ -332,12 +329,12 @@ public async Task CreateNewTask_ShouldUpdateLayoutAreaViews() var stream = workspace.GetRemoteStream( new HostAddress(), - new LayoutAreaReference(TaskCountArea) + new LayoutAreaReference(nameof(TaskCountView)) ); // Wait for initial data (should show 3 tasks) await stream - .GetControlStream(TaskCountArea) + .GetControlStream(nameof(TaskCountView)) .Timeout(5.Seconds()) .FirstAsync(x => x != null && x.ToString().Contains("Total Tasks:** 3")); @@ -351,13 +348,13 @@ await stream ); var createRequest = new DataChangeRequest().WithCreations(newTask); - + Output.WriteLine($"📤 Creating new task: '{newTask.Title}'"); client.Post(createRequest, o => o.WithTarget(new HostAddress())); // Verify task count increased var updatedControl = await stream - .GetControlStream(TaskCountArea) + .GetControlStream(nameof(TaskCountView)) .Where(x => x != null && x.ToString().Contains("Total Tasks:** 4")) .Timeout(10.Seconds()) .FirstAsync(); @@ -365,7 +362,7 @@ await stream updatedControl.Should().NotBeNull(); var content = updatedControl.ToString(); content.Should().Contain("Total Tasks:** 4"); - + Output.WriteLine("✅ New task creation updated layout area correctly"); } @@ -380,12 +377,12 @@ public async Task DeleteTask_ShouldUpdateLayoutAreaViews() var stream = workspace.GetRemoteStream( new HostAddress(), - new LayoutAreaReference(TaskListArea) + new LayoutAreaReference(nameof(TaskListView)) ); // Wait for initial data await stream - .GetControlStream(TaskListArea) + .GetControlStream(nameof(TaskListView)) .Timeout(5.Seconds()) .FirstAsync(x => x != null && x.ToString().Contains("First Task")); @@ -397,13 +394,13 @@ await stream var taskToDelete = tasksData.First(t => t.Id == "task-1"); var deleteRequest = new DataChangeRequest().WithDeletions(taskToDelete); - + Output.WriteLine($"📤 Deleting task: '{taskToDelete.Title}'"); client.Post(deleteRequest, o => o.WithTarget(new HostAddress())); // Verify task is no longer in the list var updatedControl = await stream - .GetControlStream(TaskListArea) + .GetControlStream(nameof(TaskListView)) .Where(x => x != null && !x.ToString().Contains("First Task")) .Timeout(10.Seconds()) .FirstAsync(); @@ -412,7 +409,7 @@ await stream var content = updatedControl.ToString(); content.Should().NotContain("First Task"); content.Should().Contain("Second Task"); // Other tasks should still be there - + Output.WriteLine("✅ Task deletion updated layout area correctly"); } } From 18033cc55a58140866cdf0d89af5028c30d2e28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 11:28:33 +0100 Subject: [PATCH 50/57] improve logging --- .../Serialization/SynchronizationStream.cs | 41 +++++++++++++++---- src/MeshWeaver.Data/WorkspaceExtensions.cs | 30 ++++++++++---- test/appsettings.json | 2 +- 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index 1e948fc55..ff2f01d62 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -77,10 +77,13 @@ public virtual IDisposable Subscribe(IObserver> observer) { try { - return Store.Synchronize().Subscribe(observer); + var subscription = Store.Synchronize().Subscribe(observer); + logger.LogDebug("[SYNC_STREAM] Subscribe for {StreamId}, subscription created", StreamId); + return subscription; } - catch (ObjectDisposedException) + catch (ObjectDisposedException e) { + logger.LogDebug("[SYNC_STREAM] Subscribe failed for {StreamId} - Store is disposed: {Exception}", StreamId, e.Message); return new AnonymousDisposable(() => { }); } } @@ -113,21 +116,33 @@ private void SetCurrent(IMessageHub hub, ChangeItem? value) { if (isDisposed || value == null) { - logger.LogWarning("Not setting {StreamId} to {Value} because the stream is disposed or value is null.", StreamId, value); + logger.LogWarning("[SYNC_STREAM] Not setting {StreamId} to {Value} because the stream is disposed or value is null. IsDisposed={IsDisposed}", StreamId, value, isDisposed); return; } - if (current is not null && Equals(current.Value, value.Value)) + + var currentVersion = current?.Version; + var newVersion = value.Version; + var valuesEqual = current is not null && Equals(current.Value, value.Value); + + + if (current is not null && current.Version == value.Version && valuesEqual) + { + logger.LogDebug("[SYNC_STREAM] Skipping SetCurrent for {StreamId} - same version and equal values", StreamId); return; + } + current = value; try { - logger.LogDebug("Setting value for {StreamId} to {Value}", StreamId, JsonSerializer.Serialize(value, Host.JsonSerializerOptions)); + logger.LogDebug("[SYNC_STREAM] Emitting OnNext for {StreamId}, Version={Version}, Store.IsDisposed={IsDisposed}, Store.HasObservers={HasObservers}", + StreamId, value.Version, Store.IsDisposed, Store.HasObservers); Store.OnNext(value); + logger.LogDebug("[SYNC_STREAM] OnNext completed for {StreamId}, opening gate", StreamId); hub.OpenGate(SynchronizationGate); } catch (Exception e) { - logger.LogWarning("Exception setting current value for {Address}: {Exception}", Hub.Address, e); + logger.LogWarning(e, "[SYNC_STREAM] Exception setting current value for {Address}", Hub.Address); } } @@ -189,7 +204,7 @@ public SynchronizationStream( this.Reference = Reference; logger = Host.ServiceProvider.GetRequiredService>>(); - logger.LogInformation("Creating Synchronization Stream {StreamId} for Host {Host} and {StreamIdentity} and {Reference}", StreamId, Host.Address, StreamIdentity, Reference); + logger.LogDebug("Creating Synchronization Stream {StreamId} for Host {Host} and {StreamIdentity} and {Reference}", StreamId, Host.Address, StreamIdentity, Reference); Hub = Host.GetHostedHub(new SynchronizationAddress(ClientId), ConfigureSynchronizationHub); } @@ -281,11 +296,19 @@ private async Task InitializeAsync(IMessageHub hub, CancellationToken ct) private void UpdateStream(IMessageDelivery delivery, IMessageHub hub) where TChange : JsonChange { + logger.LogDebug("[SYNC_STREAM] UpdateStream called for {StreamId}, ChangeType={ChangeType}, Version={Version}, MessageId={MessageId}", + StreamId, delivery.Message.ChangeType, delivery.Message.Version, delivery.Id); + if (Hub.Disposal is not null) + { + logger.LogWarning("[SYNC_STREAM] UpdateStream skipped for {StreamId} - hub is disposing", StreamId); return; + } + var currentJson = Get(); if (delivery.Message.ChangeType == ChangeType.Full) { + logger.LogDebug("[SYNC_STREAM] Processing Full change for {StreamId}", StreamId); currentJson = JsonSerializer.Deserialize(delivery.Message.Change.Content); try { @@ -297,12 +320,14 @@ private void UpdateStream(IMessageDelivery delivery, IMessageH } catch (Exception ex) { + logger.LogWarning(ex, "[SYNC_STREAM] Failed to process Full change for {StreamId}", StreamId); SyncFailed(delivery, ex); } } else { + logger.LogDebug("[SYNC_STREAM] Processing Patch change for {StreamId}", StreamId); (currentJson, var patch) = delivery.Message.UpdateJsonElement(currentJson, hub.JsonSerializerOptions); try { @@ -314,11 +339,13 @@ private void UpdateStream(IMessageDelivery delivery, IMessageH } catch (Exception ex) { + logger.LogError(ex, "[SYNC_STREAM] Failed to process Patch change for {StreamId}", StreamId); SyncFailed(delivery, ex); } } Set(currentJson); + logger.LogDebug("[SYNC_STREAM] UpdateStream completed for {StreamId}", StreamId); } private void SyncFailed(IMessageDelivery delivery, Exception exception) diff --git a/src/MeshWeaver.Data/WorkspaceExtensions.cs b/src/MeshWeaver.Data/WorkspaceExtensions.cs index 3d2735e37..0eb74951e 100644 --- a/src/MeshWeaver.Data/WorkspaceExtensions.cs +++ b/src/MeshWeaver.Data/WorkspaceExtensions.cs @@ -1,7 +1,7 @@ using System.Data; using System.Reactive.Linq; -using Microsoft.Extensions.DependencyInjection; using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace MeshWeaver.Data; @@ -27,16 +27,30 @@ public static IObservable> GetObservable(this IWorkspa { var logger = workspace.Hub.ServiceProvider.GetRequiredService>(); var stream = workspace.GetStream(typeof(T)); - logger.LogDebug("Retrieved stream {StreamId} for type {Type}: {Identity}, {Reference}", stream.StreamId, typeof(T).Name, stream.StreamIdentity, stream.Reference); + logger.LogDebug("[WORKSPACE] GetObservable called for type {Type}, StreamId={StreamId}, Identity={Identity}, Reference={Reference}", + typeof(T).Name, stream.StreamId, stream.StreamIdentity, stream.Reference); return stream .Synchronize() - .Select(ws => ws.Value?.GetData().ToArray()) - .Where(x => x != null) + .Do(_ => logger.LogDebug("[WORKSPACE] Subscription created for {Type}, StreamId={StreamId}", typeof(T).Name, stream.StreamId)) + .Select(ws => + { + logger.LogDebug("[WORKSPACE] Received change item for {Type}, StreamId={StreamId}, HasValue={HasValue}", + typeof(T).Name, stream.StreamId, ws.Value != null); + return ws.Value?.GetData().ToArray(); + }) + .Where(x => + { + var hasData = x != null; + logger.LogDebug("[WORKSPACE] Filter check for {Type}, StreamId={StreamId}, HasData={HasData}", + typeof(T).Name, stream.StreamId, hasData); + return hasData; + }) .Select(x => { var ret = (IReadOnlyCollection)x!; - logger.LogDebug("Stream {StreamId}: Observable Value for {Type}: {val}", stream.StreamId,typeof(T).Name, string.Join(", ", ret.Select(y => y!.ToString()))); + logger.LogDebug("[WORKSPACE] Emitting collection for {Type}, StreamId={StreamId}, Count={Count}, Items={Items}", + stream.StreamId, typeof(T).Name, ret.Count, string.Join(", ", ret.Select(y => y!.ToString()))); return ret; }); } @@ -50,7 +64,7 @@ public static ChangeItem ApplyChanges( this ISynchronizationStream? stream, EntityStoreAndUpdates storeAndUpdates) => new(storeAndUpdates.Store, - storeAndUpdates.ChangedBy ?? stream!.StreamId, + storeAndUpdates.ChangedBy ?? stream!.StreamId, stream!.StreamId, ChangeType.Patch, stream!.Hub.Version, @@ -66,8 +80,8 @@ public static EntityStore AddInstances(this IWorkspace workspace, EntityStore st if (typeSource == null) throw new DataException($"Type {g.Key.Name} is not mapped to the workspace."); var collection = s.Collections.GetValueOrDefault(typeSource.CollectionName); - - collection = collection == null ? new InstanceCollection(g.ToDictionary(typeSource.TypeDefinition.GetKey)) : collection with{Instances = collection.Instances.SetItems(g.ToDictionary(typeSource.TypeDefinition.GetKey)) }; + + collection = collection == null ? new InstanceCollection(g.ToDictionary(typeSource.TypeDefinition.GetKey)) : collection with { Instances = collection.Instances.SetItems(g.ToDictionary(typeSource.TypeDefinition.GetKey)) }; return s with { Collections = s.Collections.SetItem(typeSource.CollectionName, collection) diff --git a/test/appsettings.json b/test/appsettings.json index 128fe5ba4..5bd67cf3b 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -3,8 +3,8 @@ "LogLevel": { "Default": "Information", "MeshWeaver.Import": "Warning", - "MeshWeaver.Data": "Warning", "MeshWeaver.Messaging": "Debug", + "MeshWeaver.Data": "Debug", "Microsoft": "Warning", "System": "Warning" }, From eacd1d8b1c7d8ee394bb45a83e5fd6bb19a0d9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 11:52:35 +0100 Subject: [PATCH 51/57] changing log level --- test/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/appsettings.json b/test/appsettings.json index 5bd67cf3b..069627311 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -3,7 +3,7 @@ "LogLevel": { "Default": "Information", "MeshWeaver.Import": "Warning", - "MeshWeaver.Messaging": "Debug", + "MeshWeaver.Messaging": "Warning", "MeshWeaver.Data": "Debug", "Microsoft": "Warning", "System": "Warning" From 0df795a2249dcd56ee46ad5e9b8b92e66807ec4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 14:05:19 +0100 Subject: [PATCH 52/57] missing .Synchronize in WorkspaceStreams --- src/MeshWeaver.Data/Serialization/SynchronizationStream.cs | 4 +--- src/MeshWeaver.Data/WorkspaceStreams.cs | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs index ff2f01d62..84ef83051 100644 --- a/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/SynchronizationStream.cs @@ -120,12 +120,10 @@ private void SetCurrent(IMessageHub hub, ChangeItem? value) return; } - var currentVersion = current?.Version; - var newVersion = value.Version; var valuesEqual = current is not null && Equals(current.Value, value.Value); - if (current is not null && current.Version == value.Version && valuesEqual) + if (current is not null && valuesEqual) { logger.LogDebug("[SYNC_STREAM] Skipping SetCurrent for {StreamId} - same version and equal values", StreamId); return; diff --git a/src/MeshWeaver.Data/WorkspaceStreams.cs b/src/MeshWeaver.Data/WorkspaceStreams.cs index 1633af6fd..a1b1fd55e 100644 --- a/src/MeshWeaver.Data/WorkspaceStreams.cs +++ b/src/MeshWeaver.Data/WorkspaceStreams.cs @@ -188,6 +188,7 @@ internal static ISynchronizationStream CreateReducedStream reducer.Invoke(change, (TReference)reducedStream.Reference, i++ == 0)); if (!reducedStream.Configuration.NullReturn) From 322fbcead6bfb9e6d9dcaa66a7f6ff04154d7da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 17:05:24 +0100 Subject: [PATCH 53/57] more fixes --- .../GenericUnpartitionedDataSource.cs | 24 +++++++------ .../JsonSynchronizationStream.cs | 35 +++++++++++-------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs index 06cca06c0..667fddaba 100644 --- a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs @@ -287,8 +287,9 @@ protected override ISynchronizationStream SetupDataSourceStream(Str var isFirst = true; stream.RegisterForDisposal( - stream.Where(x => isFirst || (x.ChangedBy is not null && !x.ChangedBy.Equals(Id))) + stream .Synchronize() + .Where(x => isFirst || (x.ChangedBy is not null && !x.ChangedBy.Equals(Id))) .Subscribe(change => { if (isFirst) @@ -370,17 +371,18 @@ protected override ISynchronizationStream SetupDataSourceStream(Str var isFirst = true; stream.RegisterForDisposal( - stream.Where(x => isFirst || (x.ChangedBy is not null && !x.ChangedBy.Equals(Id))) - .Synchronize() - .Subscribe(change => - { - if (isFirst) + stream + .Synchronize() + .Where(x => isFirst || (x.ChangedBy is not null && !x.ChangedBy.Equals(Id))) + .Subscribe(change => { - isFirst = false; - return; // Skip processing on first emission (initialization) - } - Synchronize(change); - }) + if (isFirst) + { + isFirst = false; + return; // Skip processing on first emission (initialization) + } + Synchronize(change); + }) ); return stream; } diff --git a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs index be3b2d5b3..5334e8d36 100644 --- a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs @@ -41,9 +41,9 @@ TReference reference if (typeof(TReduced) == typeof(JsonElement)) reduced.RegisterForDisposal( reduced + .Synchronize() .ToDataChanged(c => reduced.ClientId.Equals(c.ChangedBy)) .Where(x => x is not null) - .Synchronize() .Subscribe(e => { logger.LogDebug("Stream {streamId} sending change notification to owner {owner}", @@ -55,9 +55,9 @@ TReference reference else reduced.RegisterForDisposal( reduced + .Synchronize() .ToDataChangeRequest(c => reduced.ClientId.Equals(c.StreamId)) .Where(x => x.Creations.Any() || x.Deletions.Any() || x.Updates.Any()) - .Synchronize() .Subscribe(e => { logger.LogDebug("Stream {streamId} sending change notification to owner {owner}", @@ -127,10 +127,10 @@ fromWorkspace as ISynchronizationStream var isFirst = true; reduced.RegisterForDisposal( reduced + .Synchronize() .ToDataChanged(c => isFirst || !reduced.ClientId.Equals(c.ChangedBy)) .Where(x => x is not null) .Select(x => x!) - .Synchronize() .Subscribe(e => { if (isFirst) @@ -146,17 +146,24 @@ fromWorkspace as ISynchronizationStream }) ); - // outgoing data changed - reduced.RegisterForDisposal( - reduced - .ToDataChangeRequest(c => reduced.ClientId.Equals(c.ChangedBy)) - .Synchronize() - .Subscribe(e => - { - logger.LogDebug("Issuing change request from stream {subscriber} to owner {owner}", reduced.StreamId, reduced.Owner); - reduced.Host.GetWorkspace().RequestChange(e, null, null); - }) - ); + // NOTE: The following subscription was causing an infinite feedback loop. + // When a client sends a DataChangeRequest, the workspace processes it and updates the stream. + // The stream emits with ChangedBy = ClientId, matching the predicate below, which calls + // RequestChange() again, creating an infinite loop. + // All changes should flow through DataChangeRequest messages, not through stream subscriptions. + // Removed to fix the feedback loop bug. + + // // outgoing data changed + // reduced.RegisterForDisposal( + // reduced + // .ToDataChangeRequest(c => reduced.ClientId.Equals(c.ChangedBy)) + // .Synchronize() + // .Subscribe(e => + // { + // logger.LogDebug("Issuing change request from stream {subscriber} to owner {owner}", reduced.StreamId, reduced.Owner); + // reduced.Host.GetWorkspace().RequestChange(e, null, null); + // }) + // ); return reduced; } From 6d68462aa15d2a011d64da662b888f872ba54362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 17:19:25 +0100 Subject: [PATCH 54/57] fixing compilation --- .../Serialization/JsonSynchronizationStream.cs | 5 ++--- .../MeshWeaver.Data.Test/SerializationAndSchemaTest.cs | 10 ++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs index 5334e8d36..70829d968 100644 --- a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs @@ -41,7 +41,6 @@ TReference reference if (typeof(TReduced) == typeof(JsonElement)) reduced.RegisterForDisposal( reduced - .Synchronize() .ToDataChanged(c => reduced.ClientId.Equals(c.ChangedBy)) .Where(x => x is not null) .Subscribe(e => @@ -55,7 +54,6 @@ TReference reference else reduced.RegisterForDisposal( reduced - .Synchronize() .ToDataChangeRequest(c => reduced.ClientId.Equals(c.StreamId)) .Where(x => x.Creations.Any() || x.Deletions.Any() || x.Updates.Any()) .Subscribe(e => @@ -127,7 +125,6 @@ fromWorkspace as ISynchronizationStream var isFirst = true; reduced.RegisterForDisposal( reduced - .Synchronize() .ToDataChanged(c => isFirst || !reduced.ClientId.Equals(c.ChangedBy)) .Where(x => x is not null) .Select(x => x!) @@ -170,6 +167,7 @@ fromWorkspace as ISynchronizationStream private static IObservable ToDataChanged( this ISynchronizationStream stream, Func, bool> predicate) where TChange : JsonChange => stream + .Synchronize() .Where(predicate) .Select(x => { @@ -319,6 +317,7 @@ internal static (InstanceCollection, JsonPatch) UpdateJsonElement(this DataChang internal static IObservable ToDataChangeRequest( this ISynchronizationStream stream, Func, bool> predicate) => stream + .Synchronize() .Where(predicate) .Select(x => x.Updates.ToDataChangeRequest(stream.ClientId)); diff --git a/test/MeshWeaver.Data.Test/SerializationAndSchemaTest.cs b/test/MeshWeaver.Data.Test/SerializationAndSchemaTest.cs index 0b961d75f..22de55c78 100644 --- a/test/MeshWeaver.Data.Test/SerializationAndSchemaTest.cs +++ b/test/MeshWeaver.Data.Test/SerializationAndSchemaTest.cs @@ -245,8 +245,8 @@ public async Task GetSchemaRequest_ShouldHandleEnumTypes() ); // assert var schemaResponse = response.Message.Should().BeOfType().Which; var schemaJson = JsonDocument.Parse(schemaResponse.Schema); - var properties = FindPropertiesInSchema(schemaJson); - + var properties = FindPropertiesInSchema(schemaJson); + var statusProperty = properties.GetProperty("status"); // Check if the status property has enum values (the key enum feature) @@ -371,7 +371,9 @@ public async Task GetDomainTypesRequest_ShouldIncludeAllRegisteredTypes() public async Task DataSerialization_ShouldPreserveComplexObjects() { // arrange - var client = GetClient(); var testData = new SerializationTestData( + var client = GetClient(); + + var testData = new SerializationTestData( name: "Serialization Test", nullableNumber: null, createdAt: DateTime.UtcNow, @@ -382,7 +384,7 @@ public async Task DataSerialization_ShouldPreserveComplexObjects() // act var response = await client.AwaitResponse( - DataChangeRequest.Update(new object[] { testData }), + DataChangeRequest.Update([testData]), o => o.WithTarget(new ClientAddress()), new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token ); From 35f8adb0757d65d58f2da0855a43fb6d26b9c546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 21:55:53 +0100 Subject: [PATCH 55/57] more .Synchronize() --- src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs | 3 +++ src/MeshWeaver.Data/WorkspaceStreams.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs index 70829d968..346af4a57 100644 --- a/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs +++ b/src/MeshWeaver.Data/Serialization/JsonSynchronizationStream.cs @@ -42,6 +42,7 @@ TReference reference reduced.RegisterForDisposal( reduced .ToDataChanged(c => reduced.ClientId.Equals(c.ChangedBy)) + .Synchronize() .Where(x => x is not null) .Subscribe(e => { @@ -55,6 +56,7 @@ TReference reference reduced.RegisterForDisposal( reduced .ToDataChangeRequest(c => reduced.ClientId.Equals(c.StreamId)) + .Synchronize() .Where(x => x.Creations.Any() || x.Deletions.Any() || x.Updates.Any()) .Subscribe(e => { @@ -126,6 +128,7 @@ fromWorkspace as ISynchronizationStream reduced.RegisterForDisposal( reduced .ToDataChanged(c => isFirst || !reduced.ClientId.Equals(c.ChangedBy)) + .Synchronize() .Where(x => x is not null) .Select(x => x!) .Subscribe(e => diff --git a/src/MeshWeaver.Data/WorkspaceStreams.cs b/src/MeshWeaver.Data/WorkspaceStreams.cs index a1b1fd55e..16883772c 100644 --- a/src/MeshWeaver.Data/WorkspaceStreams.cs +++ b/src/MeshWeaver.Data/WorkspaceStreams.cs @@ -194,7 +194,8 @@ internal static ISynchronizationStream CreateReducedStream x is { Value: not null }); + .Where(x => x is { Value: not null }) + .Synchronize(); } From 2f67dede826b5aefdfb1ea17334d474b0933b1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 22:11:38 +0100 Subject: [PATCH 56/57] setting logging to Warning --- test/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/appsettings.json b/test/appsettings.json index 069627311..98fad356c 100644 --- a/test/appsettings.json +++ b/test/appsettings.json @@ -4,7 +4,7 @@ "Default": "Information", "MeshWeaver.Import": "Warning", "MeshWeaver.Messaging": "Warning", - "MeshWeaver.Data": "Debug", + "MeshWeaver.Data": "Warning", "Microsoft": "Warning", "System": "Warning" }, From 48e724af5d7b24852f14c4003cc62aa006d5d93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Mon, 3 Nov 2025 22:41:36 +0100 Subject: [PATCH 57/57] improving race conditions for closing activitieis --- src/MeshWeaver.Data/DataExtensions.cs | 8 +++++--- src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs | 8 +++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/MeshWeaver.Data/DataExtensions.cs b/src/MeshWeaver.Data/DataExtensions.cs index 8a6a70d68..0772ebae7 100644 --- a/src/MeshWeaver.Data/DataExtensions.cs +++ b/src/MeshWeaver.Data/DataExtensions.cs @@ -199,10 +199,10 @@ private static IMessageDelivery HandleDataChangeRequest(IMessageHub hub, IMessageDelivery request) { var activity = hub.Address is ActivityAddress ? null : new Activity(ActivityCategory.DataUpdate, hub); - hub.GetWorkspace().RequestChange(request.Message with { ChangedBy = request.Message.ChangedBy }, activity, - request); if (activity is not null) { + // Register completion action BEFORE starting work to avoid race condition + // where sub-activities complete and auto-dispose before the completion action is registered activity.Complete(log => { hub.Post(new DataChangeResponse(hub.Version, log), @@ -210,7 +210,9 @@ private static IMessageDelivery HandleDataChangeRequest(IMessageHub hub, }); } - else + hub.GetWorkspace().RequestChange(request.Message with { ChangedBy = request.Message.ChangedBy }, activity, + request); + if (activity is null) hub.Post(new DataChangeResponse(hub.Version, new(ActivityCategory.DataUpdate) { Status = ActivityStatus.Succeeded }), o => o.ResponseFor(request)); return request.Processed(); diff --git a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs index 667fddaba..259fa66ec 100644 --- a/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs +++ b/src/MeshWeaver.Data/GenericUnpartitionedDataSource.cs @@ -129,7 +129,13 @@ protected virtual CollectionsReference GetReference() => public virtual void Dispose() { - foreach (var stream in Streams.Values) + ISynchronizationStream[] streamsToDispose; + lock (Streams) + { + streamsToDispose = Streams.Values.ToArray(); + } + + foreach (var stream in streamsToDispose) stream.Dispose(); if (changesSubscriptions != null)