From 7baa30f27ddc9829ec863f42f33323bd62c4815b Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Tue, 8 Jul 2025 13:01:21 -0400 Subject: [PATCH 01/12] add overlays option --- .../Configuration/GenerationConfiguration.cs | 3 + src/Kiota.Builder/Kiota.Builder.csproj | 5 +- .../OpenApiDocumentDownloadService.cs | 49 +- src/kiota/Handlers/Client/AddHandler.cs | 13 + src/kiota/Handlers/Client/EditHandler.cs | 15 + .../Handlers/KiotaGenerateCommandHandler.cs | 13 + src/kiota/Handlers/Plugin/AddHandler.cs | 16 + src/kiota/Handlers/Plugin/EditHandler.cs | 15 + src/kiota/KiotaClientCommands.cs | 6 + src/kiota/KiotaHost.cs | 10 + src/kiota/KiotaPluginCommands.cs | 17 +- src/kiota/Rpc/IServer.cs | 4 +- src/kiota/Rpc/Server.cs | 14 +- .../npm-package/lib/generateClient.ts | 116 ++-- .../npm-package/lib/generatePlugin.ts | 113 ++-- vscode/packages/npm-package/types.ts | 523 +++++++++--------- 16 files changed, 551 insertions(+), 381 deletions(-) diff --git a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs index eada1ab6f3..5f0f4e7f20 100644 --- a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs +++ b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs @@ -151,6 +151,8 @@ public bool NoWorkspace get; set; } + public HashSet Overlays { get; set; } = new(0, StringComparer.OrdinalIgnoreCase); + public int MaxDegreeOfParallelism { get; set; } = -1; public object Clone() { @@ -185,6 +187,7 @@ public object Clone() DisableSSLValidation = DisableSSLValidation, ExportPublicApi = ExportPublicApi, PluginAuthInformation = PluginAuthInformation, + Overlays = Overlays }; } private static readonly StringIEnumerableDeepComparer comparer = new(); diff --git a/src/Kiota.Builder/Kiota.Builder.csproj b/src/Kiota.Builder/Kiota.Builder.csproj index feb7054ccf..1cceaa4b2c 100644 --- a/src/Kiota.Builder/Kiota.Builder.csproj +++ b/src/Kiota.Builder/Kiota.Builder.csproj @@ -36,12 +36,13 @@ + - + - + diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs index 44e19e1a4d..16f75cafb8 100644 --- a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -1,11 +1,13 @@ using System; using System.Diagnostics; using System.IO; +using System.Linq; using System.Net.Http; using System.Security; using System.Threading; using System.Threading.Tasks; using AsyncKeyedLock; +using BinkyLabs.OpenApi.Overlays; using Kiota.Builder.Caching; using Kiota.Builder.Configuration; using Kiota.Builder.Extensions; @@ -96,7 +98,7 @@ ex is SecurityException || return (input, isDescriptionFromWorkspaceCopy); } - internal async Task GetDocumentWithResultFromStreamAsync(Stream input, GenerationConfiguration config, bool generating = false, CancellationToken cancellationToken = default) + internal async Task GetDocumentWithResultFromStreamAsync(Stream input, GenerationConfiguration config, bool generating = false, CancellationToken cancellationToken = default) { var stopwatch = new Stopwatch(); stopwatch.Start(); @@ -137,7 +139,52 @@ ex is SecurityException || { // couldn't parse the URL, it's probably a local file } + var readResult = await OpenApiDocument.LoadAsync(input, settings: settings, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (config.Overlays.Count != 0) + { + // TODO : handle multiple Overlays + var overlay = config.Overlays.First(); + var overlaysSettings = new OverlayReaderSettings + { + OpenApiSettings = settings + }; + + var readOverlayResult = await OverlayDocument.LoadFromUrlAsync(overlay, settings: overlaysSettings, token: cancellationToken).ConfigureAwait(false); + + + if (readOverlayResult?.Document is not null && settings.BaseUrl is not null) + { + var readOverlayAppliedResult = await readOverlayResult.Document.ApplyToDocumentStreamAsync(input, settings.BaseUrl, null, overlaysSettings, cancellationToken: cancellationToken) + .ConfigureAwait(false); + + if (readOverlayResult is not null) + { + if (readOverlayResult.Diagnostic is not null) + { + var diagnostics = new OpenApiDiagnostic() + { + Errors = readOverlayResult.Diagnostic.Errors, + Warnings = readOverlayResult.Diagnostic.Warnings, + }; + + if (readResult.Diagnostic is not null) + { + diagnostics.AppendDiagnostic(readResult.Diagnostic); + } + else + { + readResult.Diagnostic = diagnostics; + } + } + + readResult.Document = readOverlayAppliedResult.Item1; + } + } + } + + stopwatch.Stop(); if (generatingMode && readResult.Diagnostic?.Warnings is { Count: > 0 }) foreach (var warning in readResult.Diagnostic.Warnings) diff --git a/src/kiota/Handlers/Client/AddHandler.cs b/src/kiota/Handlers/Client/AddHandler.cs index e790296586..58366d2580 100644 --- a/src/kiota/Handlers/Client/AddHandler.cs +++ b/src/kiota/Handlers/Client/AddHandler.cs @@ -80,6 +80,10 @@ public required Option SkipGenerationOption { get; init; } + public required Option> OverlaysOption + { + get; init; + } public override async Task InvokeAsync(InvocationContext context) { @@ -101,6 +105,7 @@ public override async Task InvokeAsync(InvocationContext context) List? excludePatterns0 = context.ParseResult.GetValueForOption(ExcludePatternsOption); List? disabledValidationRules0 = context.ParseResult.GetValueForOption(DisabledValidationRulesOption); List? structuredMimeTypes0 = context.ParseResult.GetValueForOption(StructuredMimeTypesOption); + List? overlays0 = context.ParseResult.GetValueForOption(OverlaysOption); var logLevel = context.ParseResult.FindResultFor(LogLevelOption)?.GetValueOrDefault() as LogLevel?; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; @@ -125,6 +130,7 @@ public override async Task InvokeAsync(InvocationContext context) List excludePatterns = excludePatterns0.OrEmpty(); List disabledValidationRules = disabledValidationRules0.OrEmpty(); List structuredMimeTypes = structuredMimeTypes0.OrEmpty(); + List overlays = overlays0.OrEmpty(); AssignIfNotNullOrEmpty(output, (c, s) => c.OutputPath = s); AssignIfNotNullOrEmpty(openapi, (c, s) => c.OpenAPIFilePath = s); AssignIfNotNullOrEmpty(className, (c, s) => c.ClientClassName = s); @@ -132,6 +138,7 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.UsesBackingStore = backingStore; Configuration.Generation.ExcludeBackwardCompatible = excludeBackwardCompatible; Configuration.Generation.IncludeAdditionalData = includeAdditionalData; + Configuration.Generation.Language = language; WarnUsingPreviewLanguage(language); Configuration.Generation.TypeAccessModifier = typeAccessModifier; @@ -150,6 +157,12 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.StructuredMimeTypes = new(structuredMimeTypes.SelectMany(static x => x.Split(' ', StringSplitOptions.RemoveEmptyEntries)) .Select(static x => x.TrimQuotes())); + if (overlays.Count != 0) + Configuration.Generation.Overlays = overlays + .Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + Configuration.Generation.OpenAPIFilePath = GetAbsolutePath(Configuration.Generation.OpenAPIFilePath); Configuration.Generation.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.OutputPath)); Configuration.Generation.ApiManifestPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.ApiManifestPath)); diff --git a/src/kiota/Handlers/Client/EditHandler.cs b/src/kiota/Handlers/Client/EditHandler.cs index ddd2cef292..4ed7a24b28 100644 --- a/src/kiota/Handlers/Client/EditHandler.cs +++ b/src/kiota/Handlers/Client/EditHandler.cs @@ -80,6 +80,10 @@ public required Option SkipGenerationOption { get; init; } + public required Option> OverlaysOption + { + get; init; + } public override async Task InvokeAsync(InvocationContext context) { @@ -101,6 +105,8 @@ public override async Task InvokeAsync(InvocationContext context) List? excludePatterns = context.ParseResult.GetValueForOption(ExcludePatternsOption); List? disabledValidationRules = context.ParseResult.GetValueForOption(DisabledValidationRulesOption); List? structuredMimeTypes = context.ParseResult.GetValueForOption(StructuredMimeTypesOption); + List? overlays = context.ParseResult.GetValueForOption(OverlaysOption); + var logLevel = context.ParseResult.FindResultFor(LogLevelOption)?.GetValueOrDefault() as LogLevel?; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; @@ -159,6 +165,7 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.ExcludeBackwardCompatible = excludeBackwardCompatible.Value; if (includeAdditionalData.HasValue) Configuration.Generation.IncludeAdditionalData = includeAdditionalData.Value; + AssignIfNotNullOrEmpty(output, (c, s) => c.OutputPath = s); AssignIfNotNullOrEmpty(openapi, (c, s) => c.OpenAPIFilePath = s); AssignIfNotNullOrEmpty(className, (c, s) => c.ClientClassName = s); @@ -176,6 +183,14 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.StructuredMimeTypes = new(structuredMimeTypes.SelectMany(static x => x.Split(' ', StringSplitOptions.RemoveEmptyEntries)) .Select(static x => x.TrimQuotes())); + if (overlays is { Count: > 0 }) + Configuration.Generation.Overlays = overlays.Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + + + DefaultSerializersAndDeserializers(Configuration.Generation); var builder = new KiotaBuilder(logger, Configuration.Generation, httpClient, true); var result = await builder.GenerateClientAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/kiota/Handlers/KiotaGenerateCommandHandler.cs b/src/kiota/Handlers/KiotaGenerateCommandHandler.cs index 57cbf15686..00595cc752 100644 --- a/src/kiota/Handlers/KiotaGenerateCommandHandler.cs +++ b/src/kiota/Handlers/KiotaGenerateCommandHandler.cs @@ -73,6 +73,11 @@ public required Option> StructuredMimeTypesOption { get; init; } + + public required Option> OverlaysOption + { + get; init; + } public override async Task InvokeAsync(InvocationContext context) { // Span start time @@ -97,6 +102,7 @@ public override async Task InvokeAsync(InvocationContext context) List? includePatterns0 = context.ParseResult.GetValueForOption(IncludePatternsOption); List? excludePatterns0 = context.ParseResult.GetValueForOption(ExcludePatternsOption); List? disabledValidationRules0 = context.ParseResult.GetValueForOption(DisabledValidationRulesOption); + List? overlays0 = context.ParseResult.GetValueForOption(OverlaysOption); bool cleanOutput = context.ParseResult.GetValueForOption(CleanOutputOption); List? structuredMimeTypes0 = context.ParseResult.GetValueForOption(StructuredMimeTypesOption); var logLevel = context.ParseResult.FindResultFor(LogLevelOption)?.GetValueOrDefault() as LogLevel?; @@ -123,6 +129,7 @@ public override async Task InvokeAsync(InvocationContext context) List excludePatterns = excludePatterns0.OrEmpty(); List disabledValidationRules = disabledValidationRules0.OrEmpty(); List structuredMimeTypes = structuredMimeTypes0.OrEmpty(); + List overlays = overlays0.OrEmpty(); AssignIfNotNullOrEmpty(output, (c, s) => c.OutputPath = s); AssignIfNotNullOrEmpty(openapi, (c, s) => c.OpenAPIFilePath = s); AssignIfNotNullOrEmpty(manifest, (c, s) => c.ApiManifestPath = s); @@ -151,6 +158,12 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.StructuredMimeTypes = new(structuredMimeTypes.SelectMany(static x => x.Split(' ', StringSplitOptions.RemoveEmptyEntries)) .Select(static x => x.TrimQuotes())); + if (overlays.Count != 0) + Configuration.Generation.Overlays = overlays + .Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + Configuration.Generation.OpenAPIFilePath = GetAbsolutePath(Configuration.Generation.OpenAPIFilePath); Configuration.Generation.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.OutputPath)); Configuration.Generation.ApiManifestPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.ApiManifestPath)); diff --git a/src/kiota/Handlers/Plugin/AddHandler.cs b/src/kiota/Handlers/Plugin/AddHandler.cs index 0914da3c6a..287b9e79b6 100644 --- a/src/kiota/Handlers/Plugin/AddHandler.cs +++ b/src/kiota/Handlers/Plugin/AddHandler.cs @@ -66,6 +66,11 @@ public required Option NoWorkspaceOption { get; init; } + public required Option> OverlaysOption + { + get; init; + } + public override async Task InvokeAsync(InvocationContext context) { // Span start time @@ -82,6 +87,8 @@ public override async Task InvokeAsync(InvocationContext context) string? className = context.ParseResult.GetValueForOption(ClassOption); List? includePatterns0 = context.ParseResult.GetValueForOption(IncludePatternsOption); List? excludePatterns0 = context.ParseResult.GetValueForOption(ExcludePatternsOption); + List? overlays0 = context.ParseResult.GetValueForOption(OverlaysOption); + var logLevel = context.ParseResult.FindResultFor(LogLevelOption)?.GetValueOrDefault() as LogLevel?; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; @@ -108,6 +115,7 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.SkipGeneration = skipGeneration; Configuration.Generation.NoWorkspace = noWorkspace; Configuration.Generation.Operation = ConsumerOperation.Add; + if (pluginTypes is { Count: > 0 }) Configuration.Generation.PluginTypes = pluginTypes.ToHashSet(); if (pluginAuthType.HasValue && !string.IsNullOrWhiteSpace(pluginAuthRefId)) @@ -116,6 +124,14 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.IncludePatterns = includePatterns0.Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); if (excludePatterns0 is { Count: > 0 }) Configuration.Generation.ExcludePatterns = excludePatterns0.Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (overlays0 is { Count: > 0 }) + Configuration.Generation.Overlays = overlays0 + .Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + Configuration.Generation.OpenAPIFilePath = GetAbsolutePath(Configuration.Generation.OpenAPIFilePath); Configuration.Generation.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.OutputPath)); var (loggerFactory, logger) = GetLoggerAndFactory(context, Configuration.Generation.OutputPath); diff --git a/src/kiota/Handlers/Plugin/EditHandler.cs b/src/kiota/Handlers/Plugin/EditHandler.cs index 6cdc82cb80..ad38fad8cb 100644 --- a/src/kiota/Handlers/Plugin/EditHandler.cs +++ b/src/kiota/Handlers/Plugin/EditHandler.cs @@ -61,6 +61,10 @@ public required Option PluginAuthRefIdOption { get; init; } + public required Option> OverlaysOption + { + get; init; + } public override async Task InvokeAsync(InvocationContext context) { @@ -77,6 +81,9 @@ public override async Task InvokeAsync(InvocationContext context) string? className0 = context.ParseResult.GetValueForOption(ClassOption); List? includePatterns = context.ParseResult.GetValueForOption(IncludePatternsOption); List? excludePatterns = context.ParseResult.GetValueForOption(ExcludePatternsOption); + List? overlays = context.ParseResult.GetValueForOption(OverlaysOption); + + var logLevel = context.ParseResult.FindResultFor(LogLevelOption)?.GetValueOrDefault() as LogLevel?; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; @@ -135,8 +142,16 @@ public override async Task InvokeAsync(InvocationContext context) Configuration.Generation.ExcludePatterns = excludePatterns.Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); if (pluginTypes is { Count: > 0 }) Configuration.Generation.PluginTypes = pluginTypes.ToHashSet(); + if (overlays is { Count: > 0 }) + Configuration.Generation.Overlays = overlays + .Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + Configuration.Generation.OpenAPIFilePath = GetAbsolutePath(Configuration.Generation.OpenAPIFilePath); Configuration.Generation.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(Configuration.Generation.OutputPath)); + DefaultSerializersAndDeserializers(Configuration.Generation); var builder = new KiotaBuilder(logger, Configuration.Generation, httpClient, true); var result = await builder.GeneratePluginAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/kiota/KiotaClientCommands.cs b/src/kiota/KiotaClientCommands.cs index cd912c3b90..58e8c1ca70 100644 --- a/src/kiota/KiotaClientCommands.cs +++ b/src/kiota/KiotaClientCommands.cs @@ -46,6 +46,7 @@ public static Command GetAddCommand() var dvrOption = KiotaHost.GetDisableValidationRulesOption(); var skipGenerationOption = GetSkipGenerationOption(); var clientNameOption = GetClientNameOption(); + var overlaysOption = KiotaHost.GetOverlaysOption(); var command = new Command("add", "Adds a new client to the Kiota configuration"){ descriptionOption, @@ -63,6 +64,7 @@ public static Command GetAddCommand() excludePatterns, dvrOption, skipGenerationOption, + overlaysOption }; command.Handler = new AddHandler { @@ -81,6 +83,7 @@ public static Command GetAddCommand() ExcludePatternsOption = excludePatterns, DisabledValidationRulesOption = dvrOption, SkipGenerationOption = skipGenerationOption, + OverlaysOption = overlaysOption }; return command; @@ -120,6 +123,7 @@ public static Command GetEditCommand() var dvrOption = KiotaHost.GetDisableValidationRulesOption(); var skipGenerationOption = GetSkipGenerationOption(); var clientNameOption = GetClientNameOption(); + var overlaysOption = KiotaHost.GetOverlaysOption(); var command = new Command("edit", "Edits a client from the Kiota configuration") { descriptionOption, @@ -137,6 +141,7 @@ public static Command GetEditCommand() excludePatterns, dvrOption, skipGenerationOption, + overlaysOption }; command.Handler = new EditHandler { @@ -155,6 +160,7 @@ public static Command GetEditCommand() ExcludePatternsOption = excludePatterns, DisabledValidationRulesOption = dvrOption, SkipGenerationOption = skipGenerationOption, + OverlaysOption = overlaysOption }; return command; } diff --git a/src/kiota/KiotaHost.cs b/src/kiota/KiotaHost.cs index 9f50f7502f..c671440f05 100644 --- a/src/kiota/KiotaHost.cs +++ b/src/kiota/KiotaHost.cs @@ -433,6 +433,12 @@ internal static Option> GetStructuredMimeTypesOption(List d structuredMimeTypesOption.AddAlias("-m"); return structuredMimeTypesOption; } + internal static Option> GetOverlaysOption() + { + var overlaysOption = new Option>("--overlays", "Enable overlays for models"); + overlaysOption.AddAlias("--ov"); + return overlaysOption; + } private static Command GetGenerateCommand() { var defaultConfiguration = new GenerationConfiguration(); @@ -484,6 +490,8 @@ private static Command GetGenerateCommand() var clearCacheOption = GetClearCacheOption(defaultConfiguration.ClearCache); + var overlaysOption = GetOverlaysOption(); + var disableSSLValidationOption = GetDisableSSLValidationOption(defaultConfiguration.DisableSSLValidation); var command = new Command("generate", "Generates a REST HTTP API client from an OpenAPI description file.") { @@ -507,6 +515,7 @@ private static Command GetGenerateCommand() dvrOption, clearCacheOption, disableSSLValidationOption, + overlaysOption }; command.Handler = new KiotaGenerateCommandHandler { @@ -530,6 +539,7 @@ private static Command GetGenerateCommand() DisabledValidationRulesOption = dvrOption, ClearCacheOption = clearCacheOption, DisableSSLValidationOption = disableSSLValidationOption, + OverlaysOption = overlaysOption, }; return command; } diff --git a/src/kiota/KiotaPluginCommands.cs b/src/kiota/KiotaPluginCommands.cs index 679e050dee..9d71f0aa6d 100644 --- a/src/kiota/KiotaPluginCommands.cs +++ b/src/kiota/KiotaPluginCommands.cs @@ -70,6 +70,15 @@ internal static Option GetNoWorkspaceOption() noWorkspaceOption.AddAlias("--nw"); return noWorkspaceOption; } + internal static Option> GetOverlaysOption() + { + var overlaysOption = new Option>("--overlays", "Enable Overlays for the plugin.") + { + IsRequired = false, + }; + overlaysOption.AddAlias("--ov"); + return overlaysOption; + } public static Command GetAddCommand() { var defaultConfiguration = new GenerationConfiguration(); @@ -83,6 +92,7 @@ public static Command GetAddCommand() var pluginType = GetPluginTypeOption(); var pluginAuthTypeOption = GetPluginAuthenticationTypeOption(); var pluginAuthRefIdOption = GetPluginAuthenticationReferenceIdOption(); + var overlaysOption = KiotaHost.GetOverlaysOption(); var command = new Command("add", "Adds a new plugin to the Kiota configuration"){ descriptionOption, includePatterns, @@ -95,7 +105,7 @@ public static Command GetAddCommand() pluginAuthTypeOption, pluginAuthRefIdOption, noWorkspaceOption, - //TODO overlay when we have support for it in OAI.net + overlaysOption }; command.AddValidator(commandResult => { @@ -114,6 +124,7 @@ public static Command GetAddCommand() SkipGenerationOption = skipGenerationOption, LogLevelOption = logLevelOption, NoWorkspaceOption = noWorkspaceOption, + OverlaysOption = overlaysOption, }; return command; } @@ -128,6 +139,7 @@ public static Command GetEditCommand() var pluginTypes = GetPluginTypeOption(false); var pluginAuthTypeOption = GetPluginAuthenticationTypeOption(); var pluginAuthRefIdOption = GetPluginAuthenticationReferenceIdOption(); + var overlaysOption = KiotaHost.GetOverlaysOption(); var command = new Command("edit", "Edits a plugin configuration and updates the Kiota configuration"){ descriptionOption, includePatterns, @@ -139,7 +151,7 @@ public static Command GetEditCommand() pluginTypes, pluginAuthTypeOption, pluginAuthRefIdOption, - //TODO overlay when we have support for it in OAI.net + overlaysOption }; command.AddValidator(commandResult => { @@ -156,6 +168,7 @@ public static Command GetEditCommand() IncludePatternsOption = includePatterns, ExcludePatternsOption = excludePatterns, SkipGenerationOption = skipGenerationOption, + OverlaysOption = overlaysOption, LogLevelOption = logLevelOption, }; return command; diff --git a/src/kiota/Rpc/IServer.cs b/src/kiota/Rpc/IServer.cs index 7c0110e593..7f014aecf6 100644 --- a/src/kiota/Rpc/IServer.cs +++ b/src/kiota/Rpc/IServer.cs @@ -10,9 +10,9 @@ internal interface IServer Task SearchAsync(string searchTerm, bool clearCache, CancellationToken cancellationToken); Task ShowAsync(string descriptionPath, string[] includeFilters, string[] excludeFilters, bool clearCache, bool includeKiotaValidationRules, CancellationToken cancellationToken); Task GetManifestDetailsAsync(string manifestPath, string apiIdentifier, bool clearCache, CancellationToken cancellationToken); - Task> GenerateAsync(string openAPIFilePath, string outputPath, GenerationLanguage language, string[] includePatterns, string[] excludePatterns, string clientClassName, string clientNamespaceName, bool usesBackingStore, bool cleanOutput, bool clearCache, bool excludeBackwardCompatible, string[] disabledValidationRules, string[] serializers, string[] deserializers, string[] structuredMimeTypes, bool includeAdditionalData, ConsumerOperation operation, CancellationToken cancellationToken); + Task> GenerateAsync(string openAPIFilePath, string outputPath, GenerationLanguage language, string[] includePatterns, string[] excludePatterns, string clientClassName, string clientNamespaceName, bool usesBackingStore, bool cleanOutput, bool clearCache, bool excludeBackwardCompatible, string[] disabledValidationRules, string[] serializers, string[] deserializers, string[] structuredMimeTypes, bool includeAdditionalData, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken); Task InfoForDescriptionAsync(string descriptionPath, bool clearCache, CancellationToken cancellationToken); - Task> GeneratePluginAsync(string openAPIFilePath, string outputPath, PluginType[] pluginTypes, string[] includePatterns, string[] excludePatterns, string clientClassName, bool cleanOutput, bool clearCache, string[] disabledValidationRules, bool? noWorkspace, PluginAuthType? pluginAuthType, string pluginAuthRefid, ConsumerOperation operation, CancellationToken cancellationToken); + Task> GeneratePluginAsync(string openAPIFilePath, string outputPath, PluginType[] pluginTypes, string[] includePatterns, string[] excludePatterns, string clientClassName, bool cleanOutput, bool clearCache, string[] disabledValidationRules, bool? noWorkspace, PluginAuthType? pluginAuthType, string pluginAuthRefid, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken); Task> MigrateFromLockFileAsync(string lockDirectoryPath, CancellationToken cancellationToken); Task> RemoveClientAsync(string clientName, bool cleanOutput, CancellationToken cancellationToken); Task> RemovePluginAsync(string pluginName, bool cleanOutput, CancellationToken cancellationToken); diff --git a/src/kiota/Rpc/Server.cs b/src/kiota/Rpc/Server.cs index a420f0ab7f..0cde941c6f 100644 --- a/src/kiota/Rpc/Server.cs +++ b/src/kiota/Rpc/Server.cs @@ -152,7 +152,7 @@ private static string NormalizeOperationNodePath(OpenApiUrlTreeNode node, HttpMe return indexingNormalizationRegex().Replace(name, "{}"); return name; } - public async Task> GenerateAsync(string openAPIFilePath, string outputPath, GenerationLanguage language, string[] includePatterns, string[] excludePatterns, string clientClassName, string clientNamespaceName, bool usesBackingStore, bool cleanOutput, bool clearCache, bool excludeBackwardCompatible, string[] disabledValidationRules, string[] serializers, string[] deserializers, string[] structuredMimeTypes, bool includeAdditionalData, ConsumerOperation operation, CancellationToken cancellationToken) + public async Task> GenerateAsync(string openAPIFilePath, string outputPath, GenerationLanguage language, string[] includePatterns, string[] excludePatterns, string clientClassName, string clientNamespaceName, bool usesBackingStore, bool cleanOutput, bool clearCache, bool excludeBackwardCompatible, string[] disabledValidationRules, string[] serializers, string[] deserializers, string[] structuredMimeTypes, bool includeAdditionalData, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken) { var logger = new ForwardedLogger(); var configuration = Configuration.Generation; @@ -167,6 +167,7 @@ public async Task> GenerateAsync(string openAPIFilePath, string o configuration.ExcludeBackwardCompatible = excludeBackwardCompatible; configuration.IncludeAdditionalData = includeAdditionalData; configuration.Operation = operation; + if (disabledValidationRules is { Length: > 0 }) configuration.DisabledValidationRules = disabledValidationRules.ToHashSet(StringComparer.OrdinalIgnoreCase); if (serializers is { Length: > 0 }) @@ -175,6 +176,8 @@ public async Task> GenerateAsync(string openAPIFilePath, string o configuration.Deserializers = deserializers.ToHashSet(StringComparer.OrdinalIgnoreCase); if (structuredMimeTypes is { Length: > 0 }) configuration.StructuredMimeTypes = new(structuredMimeTypes); + if (overlays is { Length: > 0 }) + configuration.Overlays = new(overlays); if (IsConfigPreviewEnabled.Value) { configuration.Serializers.Clear(); @@ -200,7 +203,7 @@ public async Task> GenerateAsync(string openAPIFilePath, string o } public async Task> GeneratePluginAsync(string openAPIFilePath, string outputPath, PluginType[] pluginTypes, string[] includePatterns, string[] excludePatterns, string clientClassName, bool cleanOutput, bool clearCache, string[] disabledValidationRules, - bool? noWorkspace, PluginAuthType? pluginAuthType, string? pluginAuthRefid, ConsumerOperation operation, CancellationToken cancellationToken) + bool? noWorkspace, PluginAuthType? pluginAuthType, string? pluginAuthRefid, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken) { var globalLogger = new ForwardedLogger(); var configuration = Configuration.Generation; @@ -221,6 +224,13 @@ public async Task> GeneratePluginAsync(string openAPIFilePath, st configuration.IncludePatterns = includePatterns.Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); if (excludePatterns is { Length: > 0 }) configuration.ExcludePatterns = excludePatterns.Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (overlays is { Length: > 0 }) + Configuration.Generation.Overlays = overlays + .Select(static x => x.TrimQuotes()) + .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + configuration.OpenAPIFilePath = GetAbsolutePath(configuration.OpenAPIFilePath); configuration.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(configuration.OutputPath)); if (!string.IsNullOrEmpty(pluginAuthRefid) && pluginAuthType != null) diff --git a/vscode/packages/npm-package/lib/generateClient.ts b/vscode/packages/npm-package/lib/generateClient.ts index 3bf65071f4..049f86f75d 100644 --- a/vscode/packages/npm-package/lib/generateClient.ts +++ b/vscode/packages/npm-package/lib/generateClient.ts @@ -5,25 +5,27 @@ import connectToKiota from "../connect"; import { KiotaGenerationLanguage, KiotaResult } from "../types"; export interface ClientGenerationOptions { - openAPIFilePath: string; - clientClassName: string; - clientNamespaceName: string; - language: KiotaGenerationLanguage; - outputPath: string; - operation: ConsumerOperation; - workingDirectory: string; + openAPIFilePath: string; + clientClassName: string; + clientNamespaceName: string; + language: KiotaGenerationLanguage; + outputPath: string; + operation: ConsumerOperation; + workingDirectory: string; + + deserializers?: string[]; + disabledValidationRules?: string[]; + excludeBackwardCompatible?: boolean; + excludePatterns?: string[]; + includeAdditionalData?: boolean; + includePatterns?: string[]; + clearCache?: boolean; + cleanOutput?: boolean; + serializers?: string[]; + structuredMimeTypes?: string[]; + usesBackingStore?: boolean; + overlays?: string[]; - deserializers?: string[]; - disabledValidationRules?: string[]; - excludeBackwardCompatible?: boolean; - excludePatterns?: string[]; - includeAdditionalData?: boolean; - includePatterns?: string[]; - clearCache?: boolean; - cleanOutput?: boolean; - serializers?: string[]; - structuredMimeTypes?: string[]; - usesBackingStore?: boolean; } /** @@ -50,52 +52,54 @@ export interface ClientGenerationOptions { * @param {string[]} [clientGenerationOptions.serializers] - The list of serializers to use. * @param {string[]} [clientGenerationOptions.structuredMimeTypes] - The list of structured MIME types to support. * @param {boolean} [clientGenerationOptions.usesBackingStore] - Whether the generated client uses a backing store. - + * @param {string[]} [clientGenerationOptions.overlays] - Whether the generated client uses overlays. + * * @returns {Promise} A promise that resolves to a KiotaResult if successful, or undefined if not. * @throws {Error} If an error occurs during the client generation process. */ export async function generateClient(clientGenerationOptions: ClientGenerationOptions): Promise { - const result = await connectToKiota(async (connection) => { - const request = new rpc.RequestType1( - "Generate" - ); + const result = await connectToKiota(async (connection) => { + const request = new rpc.RequestType1( + "Generate" + ); - return await connection.sendRequest( - request, - { - openAPIFilePath: clientGenerationOptions.openAPIFilePath, - clientClassName: clientGenerationOptions.clientClassName, - clientNamespaceName: clientGenerationOptions.clientNamespaceName, - language: clientGenerationOptions.language, - outputPath: clientGenerationOptions.outputPath, - operation: clientGenerationOptions.operation, + return await connection.sendRequest( + request, + { + openAPIFilePath: clientGenerationOptions.openAPIFilePath, + clientClassName: clientGenerationOptions.clientClassName, + clientNamespaceName: clientGenerationOptions.clientNamespaceName, + language: clientGenerationOptions.language, + outputPath: clientGenerationOptions.outputPath, + operation: clientGenerationOptions.operation, - deserializers: clientGenerationOptions.deserializers ?? [], - disabledValidationRules: clientGenerationOptions.disabledValidationRules ?? [], - excludeBackwardCompatible: clientGenerationOptions.excludeBackwardCompatible ?? false, - excludePatterns: clientGenerationOptions.excludePatterns ?? [], - includeAdditionalData: clientGenerationOptions.includeAdditionalData ?? false, - cleanOutput: clientGenerationOptions.cleanOutput ?? false, - clearCache: clientGenerationOptions.clearCache ?? false, - includePatterns: clientGenerationOptions.includePatterns ?? [], - serializers: clientGenerationOptions.serializers ?? [], - structuredMimeTypes: clientGenerationOptions.structuredMimeTypes ?? [], - usesBackingStore: clientGenerationOptions.usesBackingStore ?? false, - } as GenerationConfiguration, - ); - }, clientGenerationOptions.workingDirectory); + deserializers: clientGenerationOptions.deserializers ?? [], + disabledValidationRules: clientGenerationOptions.disabledValidationRules ?? [], + excludeBackwardCompatible: clientGenerationOptions.excludeBackwardCompatible ?? false, + excludePatterns: clientGenerationOptions.excludePatterns ?? [], + includeAdditionalData: clientGenerationOptions.includeAdditionalData ?? false, + cleanOutput: clientGenerationOptions.cleanOutput ?? false, + clearCache: clientGenerationOptions.clearCache ?? false, + includePatterns: clientGenerationOptions.includePatterns ?? [], + serializers: clientGenerationOptions.serializers ?? [], + structuredMimeTypes: clientGenerationOptions.structuredMimeTypes ?? [], + usesBackingStore: clientGenerationOptions.usesBackingStore ?? false, + overlays: clientGenerationOptions.overlays ?? [], + } as GenerationConfiguration, + ); + }, clientGenerationOptions.workingDirectory); - if (result instanceof Error) { - throw result; - } + if (result instanceof Error) { + throw result; + } - if (result) { - return { - isSuccess: checkForSuccess(result as KiotaLogEntry[]), - logs: result - }; - } + if (result) { + return { + isSuccess: checkForSuccess(result as KiotaLogEntry[]), + logs: result + }; + } - return undefined; + return undefined; }; diff --git a/vscode/packages/npm-package/lib/generatePlugin.ts b/vscode/packages/npm-package/lib/generatePlugin.ts index fcb44b505f..fc4382da21 100644 --- a/vscode/packages/npm-package/lib/generatePlugin.ts +++ b/vscode/packages/npm-package/lib/generatePlugin.ts @@ -6,21 +6,22 @@ import { KiotaPluginType, GeneratePluginResult } from "../types"; import * as path from "path"; export interface PluginGenerationOptions { - descriptionPath: string; - outputPath: string; - pluginName: string; - operation: ConsumerOperation; - workingDirectory: string; + descriptionPath: string; + outputPath: string; + pluginName: string; + operation: ConsumerOperation; + workingDirectory: string; - pluginType?: KiotaPluginType; - includePatterns?: string[]; - excludePatterns?: string[]; - clearCache?: boolean; - cleanOutput?: boolean; - disabledValidationRules?: string[]; - noWorkspace?: boolean; - pluginAuthType?: PluginAuthType | null; - pluginAuthRefid?: string; + pluginType?: KiotaPluginType; + includePatterns?: string[]; + excludePatterns?: string[]; + clearCache?: boolean; + cleanOutput?: boolean; + disabledValidationRules?: string[]; + noWorkspace?: boolean; + pluginAuthType?: PluginAuthType | null; + pluginAuthRefid?: string; + overlays?: string[]; } /** @@ -41,6 +42,7 @@ export interface PluginGenerationOptions { * @param {boolean} [pluginGenerationOptions.noWorkspace] - Whether to generate without a workspace. * @param {PluginAuthType | null} [pluginGenerationOptions.pluginAuthType] - The authentication type for the plugin, if any. * @param {string} [pluginGenerationOptions.pluginAuthRefid] - The reference ID for the plugin authentication, if any. + * @param {string[]} [pluginGenerationOptions.overlays] - List of overlays to use in the generation process. * @returns {Promise} A promise that resolves to a KiotaResult if successful, or undefined if not. * @throws {Error} If an error occurs during the generation process. * @@ -49,50 +51,51 @@ export interface PluginGenerationOptions { */ export async function generatePlugin(pluginGenerationOptions: PluginGenerationOptions ): Promise { - const pluginType = pluginGenerationOptions.pluginType ?? KiotaPluginType.ApiPlugin; - const result = await connectToKiota(async (connection) => { - const request = new rpc.RequestType1( - "GeneratePlugin" - ); - return await connection.sendRequest( - request, - { - openAPIFilePath: pluginGenerationOptions.descriptionPath, - outputPath: pluginGenerationOptions.outputPath, - operation: pluginGenerationOptions.operation, - clientClassName: pluginGenerationOptions.pluginName, + const pluginType = pluginGenerationOptions.pluginType ?? KiotaPluginType.ApiPlugin; + const result = await connectToKiota(async (connection) => { + const request = new rpc.RequestType1( + "GeneratePlugin" + ); + return await connection.sendRequest( + request, + { + openAPIFilePath: pluginGenerationOptions.descriptionPath, + outputPath: pluginGenerationOptions.outputPath, + operation: pluginGenerationOptions.operation, + clientClassName: pluginGenerationOptions.pluginName, - pluginTypes: [pluginType], - cleanOutput: pluginGenerationOptions.cleanOutput ?? false, - clearCache: pluginGenerationOptions.clearCache ?? false, - disabledValidationRules: pluginGenerationOptions.disabledValidationRules ?? [], - excludePatterns: pluginGenerationOptions.excludePatterns ?? [], - includePatterns: pluginGenerationOptions.includePatterns ?? [], - noWorkspace: pluginGenerationOptions.noWorkspace ?? null, - pluginAuthType: pluginGenerationOptions.pluginAuthType ?? null, - pluginAuthRefid: pluginGenerationOptions.pluginAuthRefid ?? '', - } as GenerationConfiguration, - ); - }, pluginGenerationOptions.workingDirectory); + pluginTypes: [pluginType], + cleanOutput: pluginGenerationOptions.cleanOutput ?? false, + clearCache: pluginGenerationOptions.clearCache ?? false, + disabledValidationRules: pluginGenerationOptions.disabledValidationRules ?? [], + excludePatterns: pluginGenerationOptions.excludePatterns ?? [], + includePatterns: pluginGenerationOptions.includePatterns ?? [], + noWorkspace: pluginGenerationOptions.noWorkspace ?? null, + pluginAuthType: pluginGenerationOptions.pluginAuthType ?? null, + pluginAuthRefid: pluginGenerationOptions.pluginAuthRefid ?? '', + overlays: pluginGenerationOptions.overlays ?? [], + } as GenerationConfiguration, + ); + }, pluginGenerationOptions.workingDirectory); - if (result instanceof Error) { - throw result; - } + if (result instanceof Error) { + throw result; + } - if (result) { - const outputPath = pluginGenerationOptions.outputPath; - const pluginName = pluginGenerationOptions.pluginName; - const pathOfSpec = path.join(outputPath, `${pluginName.toLowerCase()}-openapi.yml`); - const plugingTypeName = KiotaPluginType[pluginType]; - const pathPluginManifest = path.join(outputPath, `${pluginName.toLowerCase()}-${plugingTypeName.toLowerCase()}.json`); - return { - aiPlugin: pathPluginManifest, - openAPISpec: pathOfSpec, - isSuccess: checkForSuccess(result as KiotaLogEntry[]), - logs: result - }; - } + if (result) { + const outputPath = pluginGenerationOptions.outputPath; + const pluginName = pluginGenerationOptions.pluginName; + const pathOfSpec = path.join(outputPath, `${pluginName.toLowerCase()}-openapi.yml`); + const plugingTypeName = KiotaPluginType[pluginType]; + const pathPluginManifest = path.join(outputPath, `${pluginName.toLowerCase()}-${plugingTypeName.toLowerCase()}.json`); + return { + aiPlugin: pathPluginManifest, + openAPISpec: pathOfSpec, + isSuccess: checkForSuccess(result as KiotaLogEntry[]), + logs: result + }; + } - return undefined; + return undefined; }; diff --git a/vscode/packages/npm-package/types.ts b/vscode/packages/npm-package/types.ts index 4c042cca61..40ed43b035 100644 --- a/vscode/packages/npm-package/types.ts +++ b/vscode/packages/npm-package/types.ts @@ -1,189 +1,189 @@ export enum KiotaGenerationLanguage { - // eslint-disable-next-line @typescript-eslint/naming-convention - CSharp = 0, - // eslint-disable-next-line @typescript-eslint/naming-convention - Java = 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - TypeScript = 2, - // eslint-disable-next-line @typescript-eslint/naming-convention - PHP = 3, - // eslint-disable-next-line @typescript-eslint/naming-convention - Python = 4, - // eslint-disable-next-line @typescript-eslint/naming-convention - Go = 5, - // eslint-disable-next-line @typescript-eslint/naming-convention - Swift = 6, - // eslint-disable-next-line @typescript-eslint/naming-convention - Ruby = 7, - // eslint-disable-next-line @typescript-eslint/naming-convention - CLI = 8, - // eslint-disable-next-line @typescript-eslint/naming-convention - Dart = 9, - // eslint-disable-next-line @typescript-eslint/naming-convention - HTTP = 10, + // eslint-disable-next-line @typescript-eslint/naming-convention + CSharp = 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + Java = 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + TypeScript = 2, + // eslint-disable-next-line @typescript-eslint/naming-convention + PHP = 3, + // eslint-disable-next-line @typescript-eslint/naming-convention + Python = 4, + // eslint-disable-next-line @typescript-eslint/naming-convention + Go = 5, + // eslint-disable-next-line @typescript-eslint/naming-convention + Swift = 6, + // eslint-disable-next-line @typescript-eslint/naming-convention + Ruby = 7, + // eslint-disable-next-line @typescript-eslint/naming-convention + CLI = 8, + // eslint-disable-next-line @typescript-eslint/naming-convention + Dart = 9, + // eslint-disable-next-line @typescript-eslint/naming-convention + HTTP = 10, }; export enum KiotaPluginType { - // eslint-disable-next-line @typescript-eslint/naming-convention - OpenAI = 0, - // eslint-disable-next-line @typescript-eslint/naming-convention - ApiManifest = 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - ApiPlugin = 2, + // eslint-disable-next-line @typescript-eslint/naming-convention + OpenAI = 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + ApiManifest = 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + ApiPlugin = 2, }; export interface KiotaLogEntry { - level: LogLevel; - message: string; + level: LogLevel; + message: string; } export enum OpenApiAuthType { - None = 0, - ApiKey = 1, - Http = 2, - OAuth2 = 3, - OpenIdConnect = 4, + None = 0, + ApiKey = 1, + Http = 2, + OAuth2 = 3, + OpenIdConnect = 4, } // key is the security scheme name, value is array of scopes export interface SecurityRequirementObject { - [name: string]: string[]; + [name: string]: string[]; } export interface KiotaOpenApiNode { - segment: string, - path: string, - children: KiotaOpenApiNode[], - operationId?: string, - summary?: string, - description?: string, - selected?: boolean, - isOperation?: boolean; - documentationUrl?: string; - clientNameOrPluginName?: string; - authType?: OpenApiAuthType; - logs?: KiotaLogEntry[]; - servers?: string[]; - security?: SecurityRequirementObject[]; - adaptiveCard?: AdaptiveCardInfo; + segment: string, + path: string, + children: KiotaOpenApiNode[], + operationId?: string, + summary?: string, + description?: string, + selected?: boolean, + isOperation?: boolean; + documentationUrl?: string; + clientNameOrPluginName?: string; + authType?: OpenApiAuthType; + logs?: KiotaLogEntry[]; + servers?: string[]; + security?: SecurityRequirementObject[]; + adaptiveCard?: AdaptiveCardInfo; } export interface AdaptiveCardInfo { - dataPath: string; - file: string; + dataPath: string; + file: string; } export interface CacheClearableConfiguration { - clearCache: boolean; + clearCache: boolean; } export interface KiotaShowConfiguration extends CacheClearableConfiguration { - includeFilters: string[]; - excludeFilters: string[]; - descriptionPath: string; - includeKiotaValidationRules: boolean; + includeFilters: string[]; + excludeFilters: string[]; + descriptionPath: string; + includeKiotaValidationRules: boolean; } export interface KiotaGetManifestDetailsConfiguration extends CacheClearableConfiguration { - manifestPath: string; - apiIdentifier: string; + manifestPath: string; + apiIdentifier: string; } export interface KiotaLoggedResult { - logs: KiotaLogEntry[]; + logs: KiotaLogEntry[]; } export enum OpenApiSpecVersion { - // eslint-disable-next-line @typescript-eslint/naming-convention - V2_0 = 0, - // eslint-disable-next-line @typescript-eslint/naming-convention - V3_0 = 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - V3_1 = 2, + // eslint-disable-next-line @typescript-eslint/naming-convention + V2_0 = 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + V3_0 = 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + V3_1 = 2, } export interface KiotaTreeResult extends KiotaLoggedResult { - specVersion: OpenApiSpecVersion; - rootNode?: KiotaOpenApiNode; - apiTitle?: string; - servers?: string[]; - security?: SecurityRequirementObject[]; - securitySchemes?: { [key: string]: SecuritySchemeObject }; + specVersion: OpenApiSpecVersion; + rootNode?: KiotaOpenApiNode; + apiTitle?: string; + servers?: string[]; + security?: SecurityRequirementObject[]; + securitySchemes?: { [key: string]: SecuritySchemeObject }; } export interface KiotaManifestResult extends KiotaLoggedResult { - apiDescriptionPath?: string; - selectedPaths?: string[]; + apiDescriptionPath?: string; + selectedPaths?: string[]; } export interface KiotaSearchResult extends KiotaLoggedResult { - results: Record; + results: Record; } export interface KiotaSearchResultItem { - // eslint-disable-next-line @typescript-eslint/naming-convention - Title: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - Description: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - ServiceUrl?: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - DescriptionUrl?: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - VersionLabels?: string[]; + // eslint-disable-next-line @typescript-eslint/naming-convention + Title: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + Description: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + ServiceUrl?: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + DescriptionUrl?: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + VersionLabels?: string[]; } export enum ConsumerOperation { - // eslint-disable-next-line @typescript-eslint/naming-convention - Add, - // eslint-disable-next-line @typescript-eslint/naming-convention - Edit, - // eslint-disable-next-line @typescript-eslint/naming-convention - Remove, - // eslint-disable-next-line @typescript-eslint/naming-convention - Generate + // eslint-disable-next-line @typescript-eslint/naming-convention + Add, + // eslint-disable-next-line @typescript-eslint/naming-convention + Edit, + // eslint-disable-next-line @typescript-eslint/naming-convention + Remove, + // eslint-disable-next-line @typescript-eslint/naming-convention + Generate } export function generationLanguageToString(language: KiotaGenerationLanguage): string { - switch (language) { - case KiotaGenerationLanguage.CSharp: - return "CSharp"; - case KiotaGenerationLanguage.Java: - return "Java"; - case KiotaGenerationLanguage.TypeScript: - return "TypeScript"; - case KiotaGenerationLanguage.PHP: - return "PHP"; - case KiotaGenerationLanguage.Python: - return "Python"; - case KiotaGenerationLanguage.Go: - return "Go"; - case KiotaGenerationLanguage.Swift: - return "Swift"; - case KiotaGenerationLanguage.Ruby: - return "Ruby"; - case KiotaGenerationLanguage.CLI: - return "CLI"; - case KiotaGenerationLanguage.Dart: - return "Dart"; - case KiotaGenerationLanguage.HTTP: - return "HTTP"; - default: - throw new Error("unknown language"); - } + switch (language) { + case KiotaGenerationLanguage.CSharp: + return "CSharp"; + case KiotaGenerationLanguage.Java: + return "Java"; + case KiotaGenerationLanguage.TypeScript: + return "TypeScript"; + case KiotaGenerationLanguage.PHP: + return "PHP"; + case KiotaGenerationLanguage.Python: + return "Python"; + case KiotaGenerationLanguage.Go: + return "Go"; + case KiotaGenerationLanguage.Swift: + return "Swift"; + case KiotaGenerationLanguage.Ruby: + return "Ruby"; + case KiotaGenerationLanguage.CLI: + return "CLI"; + case KiotaGenerationLanguage.Dart: + return "Dart"; + case KiotaGenerationLanguage.HTTP: + return "HTTP"; + default: + throw new Error("unknown language"); + } } export const allGenerationLanguages = [ - KiotaGenerationLanguage.CSharp, - KiotaGenerationLanguage.Go, - KiotaGenerationLanguage.Java, - KiotaGenerationLanguage.PHP, - KiotaGenerationLanguage.Python, - KiotaGenerationLanguage.Ruby, - KiotaGenerationLanguage.CLI, - KiotaGenerationLanguage.Swift, - KiotaGenerationLanguage.TypeScript, - KiotaGenerationLanguage.Dart, - KiotaGenerationLanguage.HTTP, + KiotaGenerationLanguage.CSharp, + KiotaGenerationLanguage.Go, + KiotaGenerationLanguage.Java, + KiotaGenerationLanguage.PHP, + KiotaGenerationLanguage.Python, + KiotaGenerationLanguage.Ruby, + KiotaGenerationLanguage.CLI, + KiotaGenerationLanguage.Swift, + KiotaGenerationLanguage.TypeScript, + KiotaGenerationLanguage.Dart, + KiotaGenerationLanguage.HTTP, ]; /** @@ -191,206 +191,207 @@ export const allGenerationLanguages = [ * @see https://learn.microsoft.com/en-us/dotnet/api/microsoft.extensions.logging.loglevel?view=dotnet-plat-ext-7.0 */ export enum LogLevel { - trace = 0, - debug = 1, - information = 2, - warning = 3, - error = 4, - critical = 5, - none = 6, + trace = 0, + debug = 1, + information = 2, + warning = 3, + error = 4, + critical = 5, + none = 6, } export interface LanguageInformation { - // eslint-disable-next-line @typescript-eslint/naming-convention - MaturityLevel: MaturityLevel; - // eslint-disable-next-line @typescript-eslint/naming-convention - Dependencies: LanguageDependency[]; - // eslint-disable-next-line @typescript-eslint/naming-convention - DependencyInstallCommand: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - ClientNamespaceName: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - ClientClassName: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - StructuredMimeTypes: string[]; + // eslint-disable-next-line @typescript-eslint/naming-convention + MaturityLevel: MaturityLevel; + // eslint-disable-next-line @typescript-eslint/naming-convention + Dependencies: LanguageDependency[]; + // eslint-disable-next-line @typescript-eslint/naming-convention + DependencyInstallCommand: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + ClientNamespaceName: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + ClientClassName: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + StructuredMimeTypes: string[]; } export interface LanguageDependency { - // eslint-disable-next-line @typescript-eslint/naming-convention - Name: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - Version: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - DependencyType: DependencyType; + // eslint-disable-next-line @typescript-eslint/naming-convention + Name: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + Version: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + DependencyType: DependencyType; } export enum MaturityLevel { - experimental = 0, - preview = 1, - stable = 2, + experimental = 0, + preview = 1, + stable = 2, } export enum DependencyType { - abstractions, - serialization, - authentication, - http, - bundle, - additional, + abstractions, + serialization, + authentication, + http, + bundle, + additional, } export interface ConfigurationFile { - version: string; - clients: Record; - plugins: Record; + version: string; + clients: Record; + plugins: Record; } export interface GenerationConfiguration { - cleanOutput: boolean; - clearCache: boolean; - clientClassName: string; - clientNamespaceName: string; - deserializers: string[]; - disabledValidationRules: string[]; - excludeBackwardCompatible: boolean; - excludePatterns: string[]; - includeAdditionalData: boolean; - includePatterns: string[]; - language: KiotaGenerationLanguage; - openAPIFilePath: string; - outputPath: string; - serializers: string[]; - structuredMimeTypes: string[]; - usesBackingStore: boolean; - pluginTypes: KiotaPluginType[]; - operation: ConsumerOperation; - noWorkspace?: boolean, - pluginAuthRefid?: string; - pluginAuthType?: PluginAuthType | null; + cleanOutput: boolean; + clearCache: boolean; + clientClassName: string; + clientNamespaceName: string; + deserializers: string[]; + disabledValidationRules: string[]; + excludeBackwardCompatible: boolean; + excludePatterns: string[]; + includeAdditionalData: boolean; + includePatterns: string[]; + language: KiotaGenerationLanguage; + openAPIFilePath: string; + outputPath: string; + serializers: string[]; + structuredMimeTypes: string[]; + usesBackingStore: boolean; + pluginTypes: KiotaPluginType[]; + operation: ConsumerOperation; + noWorkspace?: boolean, + pluginAuthRefid?: string; + pluginAuthType?: PluginAuthType | null; + overlays?: string[]; } export enum PluginAuthType { - oAuthPluginVault = "OAuthPluginVault", - apiKeyPluginVault = "ApiKeyPluginVault" + oAuthPluginVault = "OAuthPluginVault", + apiKeyPluginVault = "ApiKeyPluginVault" } export interface WorkspaceObjectProperties { - descriptionLocation: string; - includePatterns: string[]; - excludePatterns: string[]; - outputPath: string; + descriptionLocation: string; + includePatterns: string[]; + excludePatterns: string[]; + outputPath: string; } export interface ClientObjectProperties extends WorkspaceObjectProperties { - language: string; - structuredMimeTypes: string[]; - clientNamespaceName: string; - usesBackingStore: boolean; - includeAdditionalData: boolean; - excludeBackwardCompatible: boolean; - disabledValidationRules: string[]; + language: string; + structuredMimeTypes: string[]; + clientNamespaceName: string; + usesBackingStore: boolean; + includeAdditionalData: boolean; + excludeBackwardCompatible: boolean; + disabledValidationRules: string[]; } export interface PluginObjectProperties extends WorkspaceObjectProperties { - types: string[]; - authType?: PluginAuthType, - authReferenceId?: string; + types: string[]; + authType?: PluginAuthType, + authReferenceId?: string; } export type ClientOrPluginProperties = ClientObjectProperties | PluginObjectProperties; export interface LanguagesInformation { - [key: string]: LanguageInformation; + [key: string]: LanguageInformation; } export interface KiotaResult extends KiotaLoggedResult { - isSuccess: boolean; + isSuccess: boolean; } export interface ValidateOpenApiResult extends KiotaLoggedResult { } export interface GeneratePluginResult extends KiotaResult { - aiPlugin: string; - openAPISpec: string; + aiPlugin: string; + openAPISpec: string; } export interface PluginManifestResult extends KiotaResult { - isValid: boolean; - schema_version: string; - name_for_human: string; - functions: PluginFunction[]; - runtime: PluginRuntime[]; + isValid: boolean; + schema_version: string; + name_for_human: string; + functions: PluginFunction[]; + runtime: PluginRuntime[]; } export interface PluginFunction { - name: string; - description: string; + name: string; + description: string; } export interface PluginAuth { - type: string; // None, OAuthPluginVault, ApiKeyPluginVault - reference_id?: string; + type: string; // None, OAuthPluginVault, ApiKeyPluginVault + reference_id?: string; } export interface PluginRuntime { - type: string; - auth: PluginAuth; - run_for_functions: string[]; + type: string; + auth: PluginAuth; + run_for_functions: string[]; } export type SecuritySchemeObject = - | HttpSecurityScheme - | ApiKeySecurityScheme - | OAuth2SecurityScheme - | OpenIdSecurityScheme; + | HttpSecurityScheme + | ApiKeySecurityScheme + | OAuth2SecurityScheme + | OpenIdSecurityScheme; export interface AuthReferenceId { - referenceId: string; + referenceId: string; } export interface HttpSecurityScheme extends AuthReferenceId { - type: 'http'; - description?: string; - scheme: string; - bearerFormat?: string; + type: 'http'; + description?: string; + scheme: string; + bearerFormat?: string; } export interface ApiKeySecurityScheme extends AuthReferenceId { - type: 'apiKey'; - description?: string; - name: string; - in: string; + type: 'apiKey'; + description?: string; + name: string; + in: string; } export interface OAuth2SecurityScheme extends AuthReferenceId { - type: 'oauth2'; - description?: string; - flows: { - implicit?: { - authorizationUrl: string; - refreshUrl?: string; - scopes: { [scope: string]: string }; + type: 'oauth2'; + description?: string; + flows: { + implicit?: { + authorizationUrl: string; + refreshUrl?: string; + scopes: { [scope: string]: string }; + }; + password?: { + tokenUrl: string; + refreshUrl?: string; + scopes: { [scope: string]: string }; + }; + clientCredentials?: { + tokenUrl: string; + refreshUrl?: string; + scopes: { [scope: string]: string }; + }; + authorizationCode?: { + authorizationUrl: string; + tokenUrl: string; + refreshUrl?: string; + scopes: { [scope: string]: string }; + }; }; - password?: { - tokenUrl: string; - refreshUrl?: string; - scopes: { [scope: string]: string }; - }; - clientCredentials?: { - tokenUrl: string; - refreshUrl?: string; - scopes: { [scope: string]: string }; - }; - authorizationCode?: { - authorizationUrl: string; - tokenUrl: string; - refreshUrl?: string; - scopes: { [scope: string]: string }; - }; - }; } export interface OpenIdSecurityScheme extends AuthReferenceId { - type: 'openIdConnect'; - description?: string; - openIdConnectUrl: string; + type: 'openIdConnect'; + description?: string; + openIdConnectUrl: string; } \ No newline at end of file From a1f6e9d21e6a8307cc335c1b29e1ba03da453c3d Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Fri, 11 Jul 2025 12:02:18 -0400 Subject: [PATCH 02/12] caching overlay seams ok --- .../Caching/OverlayCachingProvider.cs | 115 +++++++++ src/Kiota.Builder/Lock/KiotaLock.cs | 7 + .../OpenApiDocumentDownloadService.cs | 92 ++++--- .../OpenApiDocumentDownloadServiceTests.cs | 225 ++++++++++++++++++ 4 files changed, 410 insertions(+), 29 deletions(-) create mode 100644 src/Kiota.Builder/Caching/OverlayCachingProvider.cs diff --git a/src/Kiota.Builder/Caching/OverlayCachingProvider.cs b/src/Kiota.Builder/Caching/OverlayCachingProvider.cs new file mode 100644 index 0000000000..7ba1bf5577 --- /dev/null +++ b/src/Kiota.Builder/Caching/OverlayCachingProvider.cs @@ -0,0 +1,115 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using AsyncKeyedLock; +using Microsoft.Extensions.Logging; + +namespace Kiota.Builder.Caching; + +public class OverlayCachingProvider +{ + private static readonly ThreadLocal HashAlgorithm = new(SHA256.Create); + public bool ClearCache + { + get; set; + } + private readonly HttpClient HttpClient; + private readonly ILogger Logger; + public TimeSpan Duration { get; set; } = TimeSpan.FromHours(1); + public OverlayCachingProvider(HttpClient client, ILogger logger) + { + ArgumentNullException.ThrowIfNull(client); + ArgumentNullException.ThrowIfNull(logger); + HttpClient = client; + Logger = logger; + } + public Task GetOverlayAsync(Uri OverlayUri, string intermediateFolderName, string fileName, string? accept = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(OverlayUri); + ArgumentException.ThrowIfNullOrEmpty(intermediateFolderName); + ArgumentException.ThrowIfNullOrEmpty(fileName); + return GetOverlayInternalAsync(OverlayUri, intermediateFolderName, fileName, false, accept, cancellationToken); + } + private async Task GetOverlayInternalAsync(Uri OverlayUri, string intermediateFolderName, string fileName, bool couldNotDelete, string? accept, CancellationToken token) + { + var hashedUrl = Convert.ToHexString((HashAlgorithm.Value ?? throw new InvalidOperationException("unable to get hash algorithm")).ComputeHash(Encoding.UTF8.GetBytes(OverlayUri.ToString()))).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase); + var target = Path.Combine(Path.GetTempPath(), Constants.TempDirectoryName, "cache", intermediateFolderName, hashedUrl); + using (await _locks.LockAsync(target, token).ConfigureAwait(false)) + {// if multiple clients are being updated for the same description, we'll have concurrent download of the file without the lock + if (!File.Exists(target) || couldNotDelete) + return await DownloadOverlayFromSourceAsync(OverlayUri, target, accept, token).ConfigureAwait(false); + + var lastModificationDate = File.GetLastWriteTime(target); + //var lastModificationDate = DateTime.Now.AddDays(-1); + if (lastModificationDate.Add(Duration) > DateTime.Now && !ClearCache) + { + Logger.LogDebug("cache file {CacheFile} is up to date and clearCache is {ClearCache}, using it", target, ClearCache); + return File.OpenRead(target); + } + else + { + Logger.LogDebug("cache file {CacheFile} is out of date, downloading from {Url}", target, OverlayUri); + try + { + File.Delete(target); + } + catch (IOException ex) + { + couldNotDelete = true; + Logger.LogWarning("could not delete cache file {CacheFile}, reason: {Reason}", target, ex.Message); + } + } + } + return await GetOverlayInternalAsync(OverlayUri, intermediateFolderName, fileName, couldNotDelete, accept, token).ConfigureAwait(false); + } + private static readonly AsyncKeyedLocker _locks = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); + private async Task DownloadOverlayFromSourceAsync(Uri OverlayUri, string target, string? accept, CancellationToken token) + { + Logger.LogDebug("cache file {CacheFile} not found, downloading from {Url}", target, OverlayUri); + var directory = Path.GetDirectoryName(target); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + Directory.CreateDirectory(directory); + Stream content = Stream.Null; + try + { + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, OverlayUri); + if (!string.IsNullOrEmpty(accept)) + requestMessage.Headers.Add("Accept", accept); + using var responseMessage = await HttpClient.SendAsync(requestMessage, token).ConfigureAwait(false); + responseMessage.EnsureSuccessStatusCode(); + content = new MemoryStream(); + await responseMessage.Content.CopyToAsync(content, token).ConfigureAwait(false); + if (OverlayUri.IsLoopback && false) + Logger.LogInformation("skipping cache write for URI {Uri} as it is a loopback address", OverlayUri); + else + { +#pragma warning disable CA2007 + await using var fileStream = File.Create(target); +#pragma warning restore CA2007 + content.Position = 0; + await content.CopyToAsync(fileStream, token).ConfigureAwait(false); + await fileStream.FlushAsync(token).ConfigureAwait(false); + } + content.Position = 0; + return content; + } + catch (HttpRequestException ex) + { + throw new InvalidOperationException($"Could not download the file at {OverlayUri}, reason: {ex.Message}", ex); + } + catch (IOException ex) + { + Logger.LogWarning("could not write to cache file {CacheFile}, reason: {Reason}", target, ex.Message); + content.Position = 0; + return content; + } + } +} diff --git a/src/Kiota.Builder/Lock/KiotaLock.cs b/src/Kiota.Builder/Lock/KiotaLock.cs index 671def487b..7943be9d83 100644 --- a/src/Kiota.Builder/Lock/KiotaLock.cs +++ b/src/Kiota.Builder/Lock/KiotaLock.cs @@ -98,6 +98,11 @@ public bool DisableSSLValidation /// The OpenAPI validation rules to disable during the generation. /// public HashSet DisabledValidationRules { get; set; } = new(StringComparer.OrdinalIgnoreCase); + /// + /// The overlays used for this client. + /// + public HashSet Overlays { get; set; } = new(StringComparer.OrdinalIgnoreCase); + #pragma warning restore CA2227 /// /// Updates the passed configuration with the values from the lock file. @@ -123,6 +128,7 @@ public void UpdateGenerationConfigurationFromLock(GenerationConfiguration config config.OpenAPIFilePath = DescriptionLocation; config.DisabledValidationRules = DisabledValidationRules.ToHashSet(StringComparer.OrdinalIgnoreCase); config.DisableSSLValidation = DisableSSLValidation; + config.Overlays = Overlays; } /// /// Initializes a new instance of the class. @@ -152,5 +158,6 @@ public KiotaLock(GenerationConfiguration config) DescriptionLocation = config.OpenAPIFilePath; DisabledValidationRules = config.DisabledValidationRules.ToHashSet(StringComparer.OrdinalIgnoreCase); DisableSSLValidation = config.DisableSSLValidation; + Overlays = config.Overlays; } } diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs index 16f75cafb8..929281046d 100644 --- a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -3,11 +3,13 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Reflection.Metadata; using System.Security; using System.Threading; using System.Threading.Tasks; using AsyncKeyedLock; using BinkyLabs.OpenApi.Overlays; +using BinkyLabs.OpenApi.Overlays.Reader; using Kiota.Builder.Caching; using Kiota.Builder.Configuration; using Kiota.Builder.Extensions; @@ -124,9 +126,9 @@ ex is SecurityException || if (addPluginsExtensions) settings.AddPluginsExtensions();// Add all extensions for plugins + var rawUri = config.OpenAPIFilePath.TrimEnd(KiotaBuilder.ForwardSlash); try { - var rawUri = config.OpenAPIFilePath.TrimEnd(KiotaBuilder.ForwardSlash); var lastSlashIndex = rawUri.LastIndexOf(KiotaBuilder.ForwardSlash); if (lastSlashIndex < 0) lastSlashIndex = rawUri.Length - 1; @@ -140,8 +142,7 @@ ex is SecurityException || // couldn't parse the URL, it's probably a local file } - var readResult = await OpenApiDocument.LoadAsync(input, settings: settings, cancellationToken: cancellationToken).ConfigureAwait(false); - + Microsoft.OpenApi.Reader.ReadResult readResult = new Microsoft.OpenApi.Reader.ReadResult() { }; if (config.Overlays.Count != 0) { // TODO : handle multiple Overlays @@ -151,37 +152,70 @@ ex is SecurityException || OpenApiSettings = settings }; - var readOverlayResult = await OverlayDocument.LoadFromUrlAsync(overlay, settings: overlaysSettings, token: cancellationToken).ConfigureAwait(false); + var cachingProvider = new OverlayCachingProvider(HttpClient, Logger) + { + ClearCache = config.ClearCache, + }; + Uri? overlayUri = null; + if (Uri.TryCreate(overlay, UriKind.Absolute, out var absoluteUri)) + { + overlayUri = absoluteUri; + } + else if (Uri.TryCreate(overlay, UriKind.Relative, out var relativeUri)) + { + // Optionally resolve relative URIs against a base URI if needed + // overlayUri = new Uri(baseUri, relativeUri); + overlayUri = relativeUri; + } - if (readOverlayResult?.Document is not null && settings.BaseUrl is not null) + if (overlayUri is null) { - var readOverlayAppliedResult = await readOverlayResult.Document.ApplyToDocumentStreamAsync(input, settings.BaseUrl, null, overlaysSettings, cancellationToken: cancellationToken) - .ConfigureAwait(false); + throw new InvalidOperationException($"The overlay '{overlay}' is not a valid URI."); + } - if (readOverlayResult is not null) - { - if (readOverlayResult.Diagnostic is not null) - { - var diagnostics = new OpenApiDiagnostic() - { - Errors = readOverlayResult.Diagnostic.Errors, - Warnings = readOverlayResult.Diagnostic.Warnings, - }; - - if (readResult.Diagnostic is not null) - { - diagnostics.AppendDiagnostic(readResult.Diagnostic); - } - else - { - readResult.Diagnostic = diagnostics; - } - } - - readResult.Document = readOverlayAppliedResult.Item1; - } + BinkyLabs.OpenApi.Overlays.ReadResult? readOverlayResult = null; + + if (overlayUri.IsAbsoluteUri && overlayUri.Scheme is "http" or "https") + { + var fileName = overlay is string name && !string.IsNullOrEmpty(name) ? name : "overlay.yml"; + var inputO = await cachingProvider.GetOverlayAsync(overlayUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + + readOverlayResult = await OverlayDocument.LoadFromStreamAsync(inputO, null, overlaysSettings, cancellationToken).ConfigureAwait(false); + } + else + { + readOverlayResult = await OverlayDocument.LoadFromUrlAsync(overlay, overlaysSettings, cancellationToken).ConfigureAwait(false); + + } + + if (readOverlayResult is null) + { + throw new InvalidOperationException($"Could not read the overlay document at {overlayUri}. Please ensure the URI is valid and accessible."); } + + readResult.Diagnostic = new OpenApiDiagnostic() + { + Errors = readOverlayResult.Diagnostic?.Errors ?? [], + Warnings = readOverlayResult.Diagnostic?.Warnings ?? [], + }; + + if (readOverlayResult.Document is not null) + { + var (document, overlaysDiagnostics, documentDiagnostics) = await readOverlayResult.Document.ApplyToDocumentStreamAsync(input, settings.BaseUrl ?? new Uri("file://" + rawUri), null, overlaysSettings, cancellationToken: cancellationToken) + .ConfigureAwait(false); + + readResult.Diagnostic.Errors.AddRange(documentDiagnostics?.Errors ?? []); + readResult.Diagnostic.Warnings.AddRange(documentDiagnostics?.Warnings ?? []); + readResult.Diagnostic.Errors.AddRange(overlaysDiagnostics?.Errors ?? []); + readResult.Diagnostic.Warnings.AddRange(overlaysDiagnostics?.Warnings ?? []); + + readResult.Document = document; + } + } + else + { + readResult = await OpenApiDocument.LoadAsync(input, settings: settings, cancellationToken: cancellationToken).ConfigureAwait(false); } diff --git a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs index aeb9044576..1265140487 100644 --- a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs +++ b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -35,6 +36,229 @@ public void Dispose() _httpClient.Dispose(); } + [Fact] + public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRelativePath() + { + // Assert + var yaml = """ + overlay: "1.0.0" + info: + title: "Test Overlay" + version: "2.0.0" + actions: + - target: "$.info" + update: + title: "Updated Title" + description: "Updated Description" + """; + + + var fakeLogger = new FakeLogger(); + + + var overlaysPath = Path.GetRandomFileName() + "overlays.yaml"; + await File.WriteAllTextAsync(overlaysPath, yaml); + + var generationConfig = new GenerationConfiguration + { + Overlays = new HashSet() { + overlaysPath + } + }; + + //Act + using var inputDocumentStream = CreateMemoryStreamFromString(DocumentContentWithNoServer); + var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); + var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); + + + // Assert + Assert.NotNull(document); + Assert.Equal("Updated Title", document.Info.Title); + Assert.Equal("Updated Description", document.Info.Description); + + // Clean up + if (Directory.Exists(overlaysPath)) + Directory.Delete(overlaysPath, true); + } + + [Fact] + public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigAbsolutePath() + { + // Assert + var yaml = """ + overlay: "1.0.0" + info: + title: "Test Overlay" + version: "2.0.0" + actions: + - target: "$.info" + update: + title: "Updated Title" + description: "Updated Description" + """; + + + var fakeLogger = new FakeLogger(); + + + var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + await File.WriteAllTextAsync(overlaysPath, yaml); + + var generationConfig = new GenerationConfiguration + { + Overlays = new HashSet() { + overlaysPath + } + }; + + //Act + using var inputDocumentStream = CreateMemoryStreamFromString(DocumentContentWithNoServer); + var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); + var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); + + + // Assert + Assert.NotNull(document); + Assert.Equal("Updated Title", document.Info.Title); + Assert.Equal("Updated Description", document.Info.Description); + + // Clean up + if (Directory.Exists(workingDirectory)) + Directory.Delete(workingDirectory, true); + } + + [Fact] + public async Task GetDocumentFromStreamAsyncTest_WithInvalidUpdatePropertyInOverlays() + { + // Assert + var json = """ + overlay: "1.0.0" + info: + title: "Test Overlay" + version: "2.0.0" + actions: + - target: "$.info" + update: + randomProperty: "Updated RandomProperty" + description: "Updated Description" + """; + + + var fakeLogger = new FakeLogger(); + + + var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + await File.WriteAllTextAsync(overlaysPath, json); + + var generationConfig = new GenerationConfiguration + { + Overlays = new HashSet() { + overlaysPath + } + }; + + //Act + using var inputDocumentStream = CreateMemoryStreamFromString(DocumentContentWithNoServer); + var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); + var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); + + + // Assert + Assert.NotNull(document); + Assert.Equal("Updated Description", document.Info.Description); + var diagError = fakeLogger.LogEntries + .Where(l => l.message.StartsWith("OpenAPI error:")); + Assert.Single(diagError); + + // Clean up + if (Directory.Exists(workingDirectory)) + Directory.Delete(workingDirectory, true); + } + + [Fact] + public async Task GetDocumentFromStreamAsyncTest_WithOverlaysJsonInConfig() + { + var json = + """ + { + "openapi": "3.1.0", + "info": { + "title": "Test Overlay", + "version": "2.0.0", + "description": "Description API" + }, + "paths": { + "/test": { + "get": { + "summary": "Test endpoint", + "responses": { + "200": { + "description": "OK" + } + } + } + } + } + } + """; + + // Assert + var jsonOverlays = """ + { + "overlay": "1.0.0", + "info": { + "title": "Test Overlay", + "version": "2.0.0" + }, + "extends": "x-extends", + "actions": [ + { + "target": "$.info", + "update": { + "title": "Updated Title YES" + } + }, + { + "target": "$.info.description", + "remove": true + } + ], + + } + """; + + var fakeLogger = new FakeLogger(); + + var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + await File.WriteAllTextAsync(overlaysPath, jsonOverlays); + + var generationConfig = new GenerationConfiguration + { + Overlays = new HashSet() { + overlaysPath + } + }; + + //Act + using var inputDocumentStream = CreateMemoryStreamFromString(json); + var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); + var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); + + + // Assert + Assert.NotNull(document); + Assert.Equal("Updated Title YES", document.Info.Title); + Assert.Null(document.Info.Description); + + // Clean up + if (Directory.Exists(workingDirectory)) + Directory.Delete(workingDirectory, true); + } + [Fact] public async Task GetDocumentFromStreamAsyncTest_IncludeKiotaValidationRulesInConfig() { @@ -56,6 +280,7 @@ public async Task GetDocumentFromStreamAsyncTest_IncludeKiotaValidationRulesInCo Assert.Single(logEntryForNoServerRule); } + [Fact] public async Task GetDocumentFromStreamAsyncTest_No_IncludeKiotaValidationRulesInConfig() { From 7878482e073fae93b0835f8912df0d8ac57f1d0b Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Fri, 11 Jul 2025 12:20:18 -0400 Subject: [PATCH 03/12] cleanup --- .../Caching/OverlayCachingProvider.cs | 115 ------------------ .../OpenApiDocumentDownloadService.cs | 8 +- 2 files changed, 3 insertions(+), 120 deletions(-) delete mode 100644 src/Kiota.Builder/Caching/OverlayCachingProvider.cs diff --git a/src/Kiota.Builder/Caching/OverlayCachingProvider.cs b/src/Kiota.Builder/Caching/OverlayCachingProvider.cs deleted file mode 100644 index 7ba1bf5577..0000000000 --- a/src/Kiota.Builder/Caching/OverlayCachingProvider.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.IO; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using AsyncKeyedLock; -using Microsoft.Extensions.Logging; - -namespace Kiota.Builder.Caching; - -public class OverlayCachingProvider -{ - private static readonly ThreadLocal HashAlgorithm = new(SHA256.Create); - public bool ClearCache - { - get; set; - } - private readonly HttpClient HttpClient; - private readonly ILogger Logger; - public TimeSpan Duration { get; set; } = TimeSpan.FromHours(1); - public OverlayCachingProvider(HttpClient client, ILogger logger) - { - ArgumentNullException.ThrowIfNull(client); - ArgumentNullException.ThrowIfNull(logger); - HttpClient = client; - Logger = logger; - } - public Task GetOverlayAsync(Uri OverlayUri, string intermediateFolderName, string fileName, string? accept = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(OverlayUri); - ArgumentException.ThrowIfNullOrEmpty(intermediateFolderName); - ArgumentException.ThrowIfNullOrEmpty(fileName); - return GetOverlayInternalAsync(OverlayUri, intermediateFolderName, fileName, false, accept, cancellationToken); - } - private async Task GetOverlayInternalAsync(Uri OverlayUri, string intermediateFolderName, string fileName, bool couldNotDelete, string? accept, CancellationToken token) - { - var hashedUrl = Convert.ToHexString((HashAlgorithm.Value ?? throw new InvalidOperationException("unable to get hash algorithm")).ComputeHash(Encoding.UTF8.GetBytes(OverlayUri.ToString()))).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase); - var target = Path.Combine(Path.GetTempPath(), Constants.TempDirectoryName, "cache", intermediateFolderName, hashedUrl); - using (await _locks.LockAsync(target, token).ConfigureAwait(false)) - {// if multiple clients are being updated for the same description, we'll have concurrent download of the file without the lock - if (!File.Exists(target) || couldNotDelete) - return await DownloadOverlayFromSourceAsync(OverlayUri, target, accept, token).ConfigureAwait(false); - - var lastModificationDate = File.GetLastWriteTime(target); - //var lastModificationDate = DateTime.Now.AddDays(-1); - if (lastModificationDate.Add(Duration) > DateTime.Now && !ClearCache) - { - Logger.LogDebug("cache file {CacheFile} is up to date and clearCache is {ClearCache}, using it", target, ClearCache); - return File.OpenRead(target); - } - else - { - Logger.LogDebug("cache file {CacheFile} is out of date, downloading from {Url}", target, OverlayUri); - try - { - File.Delete(target); - } - catch (IOException ex) - { - couldNotDelete = true; - Logger.LogWarning("could not delete cache file {CacheFile}, reason: {Reason}", target, ex.Message); - } - } - } - return await GetOverlayInternalAsync(OverlayUri, intermediateFolderName, fileName, couldNotDelete, accept, token).ConfigureAwait(false); - } - private static readonly AsyncKeyedLocker _locks = new(o => - { - o.PoolSize = 20; - o.PoolInitialFill = 1; - }); - private async Task DownloadOverlayFromSourceAsync(Uri OverlayUri, string target, string? accept, CancellationToken token) - { - Logger.LogDebug("cache file {CacheFile} not found, downloading from {Url}", target, OverlayUri); - var directory = Path.GetDirectoryName(target); - if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) - Directory.CreateDirectory(directory); - Stream content = Stream.Null; - try - { - using var requestMessage = new HttpRequestMessage(HttpMethod.Get, OverlayUri); - if (!string.IsNullOrEmpty(accept)) - requestMessage.Headers.Add("Accept", accept); - using var responseMessage = await HttpClient.SendAsync(requestMessage, token).ConfigureAwait(false); - responseMessage.EnsureSuccessStatusCode(); - content = new MemoryStream(); - await responseMessage.Content.CopyToAsync(content, token).ConfigureAwait(false); - if (OverlayUri.IsLoopback && false) - Logger.LogInformation("skipping cache write for URI {Uri} as it is a loopback address", OverlayUri); - else - { -#pragma warning disable CA2007 - await using var fileStream = File.Create(target); -#pragma warning restore CA2007 - content.Position = 0; - await content.CopyToAsync(fileStream, token).ConfigureAwait(false); - await fileStream.FlushAsync(token).ConfigureAwait(false); - } - content.Position = 0; - return content; - } - catch (HttpRequestException ex) - { - throw new InvalidOperationException($"Could not download the file at {OverlayUri}, reason: {ex.Message}", ex); - } - catch (IOException ex) - { - Logger.LogWarning("could not write to cache file {CacheFile}, reason: {Reason}", target, ex.Message); - content.Position = 0; - return content; - } - } -} diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs index 929281046d..ee64380676 100644 --- a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -152,7 +152,7 @@ ex is SecurityException || OpenApiSettings = settings }; - var cachingProvider = new OverlayCachingProvider(HttpClient, Logger) + var cachingProvider = new DocumentCachingProvider(HttpClient, Logger) { ClearCache = config.ClearCache, }; @@ -164,8 +164,6 @@ ex is SecurityException || } else if (Uri.TryCreate(overlay, UriKind.Relative, out var relativeUri)) { - // Optionally resolve relative URIs against a base URI if needed - // overlayUri = new Uri(baseUri, relativeUri); overlayUri = relativeUri; } @@ -179,9 +177,9 @@ ex is SecurityException || if (overlayUri.IsAbsoluteUri && overlayUri.Scheme is "http" or "https") { var fileName = overlay is string name && !string.IsNullOrEmpty(name) ? name : "overlay.yml"; - var inputO = await cachingProvider.GetOverlayAsync(overlayUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + var inputOverlay = await cachingProvider.GetDocumentAsync(overlayUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); - readOverlayResult = await OverlayDocument.LoadFromStreamAsync(inputO, null, overlaysSettings, cancellationToken).ConfigureAwait(false); + readOverlayResult = await OverlayDocument.LoadFromStreamAsync(inputOverlay, null, overlaysSettings, cancellationToken).ConfigureAwait(false); } else { From 5cf86826c7de517f2010bc83e57e11ffe5ac7b83 Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Fri, 11 Jul 2025 12:28:02 -0400 Subject: [PATCH 04/12] ad d BinkyLabs.OpenApi.Overlays --- src/Kiota.Builder/Kiota.Builder.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Kiota.Builder/Kiota.Builder.csproj b/src/Kiota.Builder/Kiota.Builder.csproj index 15fb42bf1c..5fc4b2e0d9 100644 --- a/src/Kiota.Builder/Kiota.Builder.csproj +++ b/src/Kiota.Builder/Kiota.Builder.csproj @@ -36,6 +36,7 @@ + From 5df7c2e4d03ecdd999d8db4bfefa001769f0544e Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Sat, 12 Jul 2025 11:55:59 -0400 Subject: [PATCH 05/12] chore: fixes nullable information Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/kiota/Rpc/IServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kiota/Rpc/IServer.cs b/src/kiota/Rpc/IServer.cs index 7f014aecf6..58f9d7cec6 100644 --- a/src/kiota/Rpc/IServer.cs +++ b/src/kiota/Rpc/IServer.cs @@ -12,7 +12,7 @@ internal interface IServer Task GetManifestDetailsAsync(string manifestPath, string apiIdentifier, bool clearCache, CancellationToken cancellationToken); Task> GenerateAsync(string openAPIFilePath, string outputPath, GenerationLanguage language, string[] includePatterns, string[] excludePatterns, string clientClassName, string clientNamespaceName, bool usesBackingStore, bool cleanOutput, bool clearCache, bool excludeBackwardCompatible, string[] disabledValidationRules, string[] serializers, string[] deserializers, string[] structuredMimeTypes, bool includeAdditionalData, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken); Task InfoForDescriptionAsync(string descriptionPath, bool clearCache, CancellationToken cancellationToken); - Task> GeneratePluginAsync(string openAPIFilePath, string outputPath, PluginType[] pluginTypes, string[] includePatterns, string[] excludePatterns, string clientClassName, bool cleanOutput, bool clearCache, string[] disabledValidationRules, bool? noWorkspace, PluginAuthType? pluginAuthType, string pluginAuthRefid, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken); + Task> GeneratePluginAsync(string openAPIFilePath, string outputPath, PluginType[] pluginTypes, string[] includePatterns, string[] excludePatterns, string clientClassName, bool cleanOutput, bool clearCache, string[] disabledValidationRules, bool? noWorkspace, PluginAuthType? pluginAuthType, string? pluginAuthRefid, string[] overlays, ConsumerOperation operation, CancellationToken cancellationToken); Task> MigrateFromLockFileAsync(string lockDirectoryPath, CancellationToken cancellationToken); Task> RemoveClientAsync(string clientName, bool cleanOutput, CancellationToken cancellationToken); Task> RemovePluginAsync(string pluginName, bool cleanOutput, CancellationToken cancellationToken); From 7a3c003463e5af478ab2c61e9fe907fcf57ab593 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Sat, 12 Jul 2025 11:57:52 -0400 Subject: [PATCH 06/12] chore: makes a copy of the overlays hashset when cloning --- src/Kiota.Builder/Configuration/GenerationConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs index b7f5cb2b92..eaca578caf 100644 --- a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs +++ b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs @@ -186,7 +186,7 @@ public object Clone() DisableSSLValidation = DisableSSLValidation, ExportPublicApi = ExportPublicApi, PluginAuthInformation = PluginAuthInformation, - Overlays = Overlays + Overlays = new (Overlays, StringComparer.OrdinalIgnoreCase), }; } private static readonly StringIEnumerableDeepComparer comparer = new(); From 695e5d039c8091606d39ae734f78b4212b5ecd35 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Sat, 12 Jul 2025 11:58:32 -0400 Subject: [PATCH 07/12] chore: removes extraneous import Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Kiota.Builder/OpenApiDocumentDownloadService.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs index ee64380676..60d99baf08 100644 --- a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using System.Net.Http; -using System.Reflection.Metadata; using System.Security; using System.Threading; using System.Threading.Tasks; From d475cebe05709dfa000449a3f87719cbd183e427 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Sat, 12 Jul 2025 12:00:10 -0400 Subject: [PATCH 08/12] chore: removes extraneous method --- src/kiota/KiotaPluginCommands.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/kiota/KiotaPluginCommands.cs b/src/kiota/KiotaPluginCommands.cs index 9d71f0aa6d..02ff2da835 100644 --- a/src/kiota/KiotaPluginCommands.cs +++ b/src/kiota/KiotaPluginCommands.cs @@ -70,15 +70,6 @@ internal static Option GetNoWorkspaceOption() noWorkspaceOption.AddAlias("--nw"); return noWorkspaceOption; } - internal static Option> GetOverlaysOption() - { - var overlaysOption = new Option>("--overlays", "Enable Overlays for the plugin.") - { - IsRequired = false, - }; - overlaysOption.AddAlias("--ov"); - return overlaysOption; - } public static Command GetAddCommand() { var defaultConfiguration = new GenerationConfiguration(); From 090624702d043e30e209ca9017322049c54a14d2 Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Tue, 5 Aug 2025 10:04:34 -0400 Subject: [PATCH 09/12] uniformisation --- .../Configuration/GenerationConfiguration.cs | 2 +- src/kiota/Rpc/Server.cs | 4 +--- .../OpenApiDocumentDownloadServiceTests.cs | 14 +++++++------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs index eaca578caf..4a00402113 100644 --- a/src/Kiota.Builder/Configuration/GenerationConfiguration.cs +++ b/src/Kiota.Builder/Configuration/GenerationConfiguration.cs @@ -186,7 +186,7 @@ public object Clone() DisableSSLValidation = DisableSSLValidation, ExportPublicApi = ExportPublicApi, PluginAuthInformation = PluginAuthInformation, - Overlays = new (Overlays, StringComparer.OrdinalIgnoreCase), + Overlays = new(Overlays, StringComparer.OrdinalIgnoreCase), }; } private static readonly StringIEnumerableDeepComparer comparer = new(); diff --git a/src/kiota/Rpc/Server.cs b/src/kiota/Rpc/Server.cs index 0cde941c6f..f2abe70bcc 100644 --- a/src/kiota/Rpc/Server.cs +++ b/src/kiota/Rpc/Server.cs @@ -227,9 +227,7 @@ public async Task> GeneratePluginAsync(string openAPIFilePath, st if (overlays is { Length: > 0 }) Configuration.Generation.Overlays = overlays - .Select(static x => x.TrimQuotes()) - .SelectMany(static x => x.Split(',', StringSplitOptions.RemoveEmptyEntries)) - .ToHashSet(StringComparer.OrdinalIgnoreCase); + .Select(static x => x.TrimQuotes()).ToHashSet(StringComparer.OrdinalIgnoreCase); configuration.OpenAPIFilePath = GetAbsolutePath(configuration.OpenAPIFilePath); configuration.OutputPath = NormalizeSlashesInPath(GetAbsolutePath(configuration.OutputPath)); diff --git a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs index 1265140487..cd606d4a08 100644 --- a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs +++ b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs @@ -56,7 +56,7 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel var fakeLogger = new FakeLogger(); - var overlaysPath = Path.GetRandomFileName() + "overlays.yaml"; + var overlaysPath = Path.Combine(Path.GetRandomFileName(), "overlays.yaml"); await File.WriteAllTextAsync(overlaysPath, yaml); var generationConfig = new GenerationConfiguration @@ -78,8 +78,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel Assert.Equal("Updated Description", document.Info.Description); // Clean up - if (Directory.Exists(overlaysPath)) - Directory.Delete(overlaysPath, true); + if (File.Exists(overlaysPath)) + File.Delete(overlaysPath); } [Fact] @@ -125,8 +125,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigAbsolut Assert.Equal("Updated Description", document.Info.Description); // Clean up - if (Directory.Exists(workingDirectory)) - Directory.Delete(workingDirectory, true); + if (File.Exists(overlaysPath)) + File.Delete(overlaysPath); } [Fact] @@ -174,8 +174,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithInvalidUpdatePropertyInOver Assert.Single(diagError); // Clean up - if (Directory.Exists(workingDirectory)) - Directory.Delete(workingDirectory, true); + if (File.Exists(overlaysPath)) + File.Delete(overlaysPath); } [Fact] From 42d38127b8d1dc639eb1152e25edbec4e302c597 Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Fri, 15 Aug 2025 16:12:54 -0400 Subject: [PATCH 10/12] updqte binkyLabs nugget --- src/Kiota.Builder/Kiota.Builder.csproj | 2 +- .../OpenApiDocumentDownloadService.cs | 87 +++++++++++-------- .../OpenApiDocumentDownloadServiceTests.cs | 40 +++------ 3 files changed, 65 insertions(+), 64 deletions(-) diff --git a/src/Kiota.Builder/Kiota.Builder.csproj b/src/Kiota.Builder/Kiota.Builder.csproj index 1605f0b9c0..c8e52234e4 100644 --- a/src/Kiota.Builder/Kiota.Builder.csproj +++ b/src/Kiota.Builder/Kiota.Builder.csproj @@ -36,8 +36,8 @@ - + diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs index 60d99baf08..9f678faab1 100644 --- a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -141,11 +141,18 @@ ex is SecurityException || // couldn't parse the URL, it's probably a local file } - Microsoft.OpenApi.Reader.ReadResult readResult = new Microsoft.OpenApi.Reader.ReadResult() { }; + Microsoft.OpenApi.Reader.ReadResult readResult = new Microsoft.OpenApi.Reader.ReadResult() + { + Diagnostic = new OpenApiDiagnostic() + { + Errors = [], + Warnings = [], + } + }; + if (config.Overlays.Count != 0) { - // TODO : handle multiple Overlays - var overlay = config.Overlays.First(); + var overlaysSettings = new OverlayReaderSettings { OpenApiSettings = settings @@ -156,50 +163,56 @@ ex is SecurityException || ClearCache = config.ClearCache, }; - Uri? overlayUri = null; - if (Uri.TryCreate(overlay, UriKind.Absolute, out var absoluteUri)) - { - overlayUri = absoluteUri; - } - else if (Uri.TryCreate(overlay, UriKind.Relative, out var relativeUri)) + OverlayDocument? overlayCombined = null; + foreach (var overlay in config.Overlays) { - overlayUri = relativeUri; - } + Uri? overlayUri = null; + if (Uri.TryCreate(overlay, UriKind.Absolute, out var absoluteUri)) + { + overlayUri = absoluteUri; + } + else if (Uri.TryCreate(overlay, UriKind.Relative, out var relativeUri)) + { + overlayUri = relativeUri; + } - if (overlayUri is null) - { - throw new InvalidOperationException($"The overlay '{overlay}' is not a valid URI."); - } + if (overlayUri is null) + { + throw new InvalidOperationException($"The overlay '{overlay}' is not a valid URI."); + } - BinkyLabs.OpenApi.Overlays.ReadResult? readOverlayResult = null; + BinkyLabs.OpenApi.Overlays.ReadResult? overlayToCombineResult = null; + if (overlayUri.IsAbsoluteUri && overlayUri.Scheme is "http" or "https") + { + var fileName = overlay is string name && !string.IsNullOrEmpty(name) ? name : "overlay.yml"; + var inputOverlay = await cachingProvider.GetDocumentAsync(overlayUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); - if (overlayUri.IsAbsoluteUri && overlayUri.Scheme is "http" or "https") - { - var fileName = overlay is string name && !string.IsNullOrEmpty(name) ? name : "overlay.yml"; - var inputOverlay = await cachingProvider.GetDocumentAsync(overlayUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + overlayToCombineResult = await OverlayDocument.LoadFromStreamAsync(inputOverlay, null, overlaysSettings, cancellationToken).ConfigureAwait(false); + } + else + { + overlayToCombineResult = await OverlayDocument.LoadFromUrlAsync(overlay, overlaysSettings, cancellationToken).ConfigureAwait(false); + } - readOverlayResult = await OverlayDocument.LoadFromStreamAsync(inputOverlay, null, overlaysSettings, cancellationToken).ConfigureAwait(false); - } - else - { - readOverlayResult = await OverlayDocument.LoadFromUrlAsync(overlay, overlaysSettings, cancellationToken).ConfigureAwait(false); + if (overlayToCombineResult is null || overlayToCombineResult.Document is null) + { + throw new InvalidOperationException($"Could not read the overlay document at {overlayUri}. Please ensure the overlay is valid and accessible."); + } - } + if (overlayToCombineResult.Diagnostic is not null) + { + readResult.Diagnostic.Errors.AddRange(overlayToCombineResult.Diagnostic.Errors ?? []); + readResult.Diagnostic.Warnings.AddRange(overlayToCombineResult.Diagnostic.Warnings ?? []); + } - if (readOverlayResult is null) - { - throw new InvalidOperationException($"Could not read the overlay document at {overlayUri}. Please ensure the URI is valid and accessible."); + overlayCombined = overlayCombined is null + ? overlayToCombineResult.Document + : overlayCombined.CombineWith(overlayToCombineResult.Document); } - readResult.Diagnostic = new OpenApiDiagnostic() - { - Errors = readOverlayResult.Diagnostic?.Errors ?? [], - Warnings = readOverlayResult.Diagnostic?.Warnings ?? [], - }; - - if (readOverlayResult.Document is not null) + if (overlayCombined is not null) { - var (document, overlaysDiagnostics, documentDiagnostics) = await readOverlayResult.Document.ApplyToDocumentStreamAsync(input, settings.BaseUrl ?? new Uri("file://" + rawUri), null, overlaysSettings, cancellationToken: cancellationToken) + var (document, overlaysDiagnostics, documentDiagnostics) = await overlayCombined.ApplyToDocumentStreamAsync(input, settings.BaseUrl ?? new Uri("file://" + rawUri), null, overlaysSettings, cancellationToken: cancellationToken) .ConfigureAwait(false); readResult.Diagnostic.Errors.AddRange(documentDiagnostics?.Errors ?? []); diff --git a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs index cd606d4a08..2716a8f28e 100644 --- a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs +++ b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs @@ -12,6 +12,7 @@ namespace Kiota.Builder.Tests.OpenApiExtensions; public sealed class OpenApiDocumentDownloadServiceTests : IDisposable { private readonly HttpClient _httpClient = new(); + private readonly string TempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); private const string DocumentContentWithNoServer = @"openapi: 3.0.0 info: title: Graph Users @@ -34,6 +35,10 @@ public sealed class OpenApiDocumentDownloadServiceTests : IDisposable public void Dispose() { _httpClient.Dispose(); + if (Directory.Exists(TempDirectory)) + { + Directory.Delete(TempDirectory, true); + } } [Fact] @@ -55,9 +60,9 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel var fakeLogger = new FakeLogger(); - - var overlaysPath = Path.Combine(Path.GetRandomFileName(), "overlays.yaml"); - await File.WriteAllTextAsync(overlaysPath, yaml); + Directory.CreateDirectory(TempDirectory); + var overlaysPath = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); + await File.WriteAllTextAsync(overlaysPath, yaml).ConfigureAwait(false); var generationConfig = new GenerationConfiguration { @@ -76,10 +81,6 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel Assert.NotNull(document); Assert.Equal("Updated Title", document.Info.Title); Assert.Equal("Updated Description", document.Info.Description); - - // Clean up - if (File.Exists(overlaysPath)) - File.Delete(overlaysPath); } [Fact] @@ -102,8 +103,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigAbsolut var fakeLogger = new FakeLogger(); - var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + Directory.CreateDirectory(TempDirectory); + var overlaysPath = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); await File.WriteAllTextAsync(overlaysPath, yaml); var generationConfig = new GenerationConfiguration @@ -123,10 +124,6 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigAbsolut Assert.NotNull(document); Assert.Equal("Updated Title", document.Info.Title); Assert.Equal("Updated Description", document.Info.Description); - - // Clean up - if (File.Exists(overlaysPath)) - File.Delete(overlaysPath); } [Fact] @@ -149,8 +146,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithInvalidUpdatePropertyInOver var fakeLogger = new FakeLogger(); - var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + Directory.CreateDirectory(TempDirectory); + var overlaysPath = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); await File.WriteAllTextAsync(overlaysPath, json); var generationConfig = new GenerationConfiguration @@ -172,10 +169,6 @@ public async Task GetDocumentFromStreamAsyncTest_WithInvalidUpdatePropertyInOver var diagError = fakeLogger.LogEntries .Where(l => l.message.StartsWith("OpenAPI error:")); Assert.Single(diagError); - - // Clean up - if (File.Exists(overlaysPath)) - File.Delete(overlaysPath); } [Fact] @@ -232,8 +225,8 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysJsonInConfig() var fakeLogger = new FakeLogger(); - var workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - var overlaysPath = Path.Combine(workingDirectory) + "overlays.yaml"; + Directory.CreateDirectory(TempDirectory); + var overlaysPath = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); await File.WriteAllTextAsync(overlaysPath, jsonOverlays); var generationConfig = new GenerationConfiguration @@ -248,15 +241,10 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysJsonInConfig() var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); - // Assert Assert.NotNull(document); Assert.Equal("Updated Title YES", document.Info.Title); Assert.Null(document.Info.Description); - - // Clean up - if (Directory.Exists(workingDirectory)) - Directory.Delete(workingDirectory, true); } [Fact] From e9342a9d1fc9243b2b5d58b9dabc8b3c5109a657 Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Fri, 15 Aug 2025 17:20:39 -0400 Subject: [PATCH 11/12] fix test --- tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs index 2716a8f28e..1afc8278ad 100644 --- a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs +++ b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs @@ -76,7 +76,6 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); - // Assert Assert.NotNull(document); Assert.Equal("Updated Title", document.Info.Title); From 93623bbe2a8a0b7aabf984b1da8dcac730a47715 Mon Sep 17 00:00:00 2001 From: costabello matthieu Date: Mon, 18 Aug 2025 11:33:37 -0400 Subject: [PATCH 12/12] test mutliple overlays --- src/Kiota.Builder/Kiota.Builder.csproj | 2 +- .../OpenApiDocumentDownloadServiceTests.cs | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/Kiota.Builder/Kiota.Builder.csproj b/src/Kiota.Builder/Kiota.Builder.csproj index c8e52234e4..003cf3c5ca 100644 --- a/src/Kiota.Builder/Kiota.Builder.csproj +++ b/src/Kiota.Builder/Kiota.Builder.csproj @@ -37,7 +37,7 @@ - + diff --git a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs index 1afc8278ad..050f8f9649 100644 --- a/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs +++ b/tests/Kiota.Builder.Tests/OpenApiDocumentDownloadServiceTests.cs @@ -82,6 +82,60 @@ public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigWithRel Assert.Equal("Updated Description", document.Info.Description); } + [Fact] + public async Task GetDocumentFromStreamAsyncTest_With2OverlaysYamlInConfigWithRelativePath() + { + // Assert + var yaml = """ + overlay: "1.0.0" + info: + title: "Test Overlay" + version: "2.0.0" + actions: + - target: "$.info" + update: + title: "Updated Title" + """; + + var yaml2 = """ + overlay: "1.0.0" + info: + title: "Test Overlay" + version: "2.0.0" + actions: + - target: "$.info" + update: + description: "Updated Description" + """; + + + var fakeLogger = new FakeLogger(); + + Directory.CreateDirectory(TempDirectory); + var overlaysPath = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); + var overlaysPath2 = Path.Combine(TempDirectory, Path.GetRandomFileName() + "overlays.yaml"); + await File.WriteAllTextAsync(overlaysPath, yaml); + await File.WriteAllTextAsync(overlaysPath2, yaml2); + + var generationConfig = new GenerationConfiguration + { + Overlays = new HashSet() { + overlaysPath, + overlaysPath2 + } + }; + + //Act + using var inputDocumentStream = CreateMemoryStreamFromString(DocumentContentWithNoServer); + var documentDownloadService = new OpenApiDocumentDownloadService(_httpClient, fakeLogger); + var document = await documentDownloadService.GetDocumentFromStreamAsync(inputDocumentStream, generationConfig); + + // Assert + Assert.NotNull(document); + Assert.Equal("Updated Title", document.Info.Title); + Assert.Equal("Updated Description", document.Info.Description); + } + [Fact] public async Task GetDocumentFromStreamAsyncTest_WithOverlaysYamlInConfigAbsolutePath() {