diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 53f5f4bcf..5b0b1332b 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -101,6 +101,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost", "src\In EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost.Tests", "test\InProcessTestHost.Tests\InProcessTestHost.Tests.csproj", "{B894780C-338F-475E-8E84-56AFA8197A06}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExportHistory", "src\ExportHistory\ExportHistory.csproj", "{354CE69B-78DB-9B29-C67E-0DBB862C7A65}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExportHistoryWebApp", "samples\ExportHistoryWebApp\ExportHistoryWebApp.csproj", "{FE1E17DD-595A-123A-EA4C-AA313BBFB685}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -255,14 +259,6 @@ Global {D2779F32-A548-44F8-B60A-6AC018966C79}.Debug|Any CPU.Build.0 = Debug|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.ActiveCfg = Release|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.Build.0 = Release|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.Build.0 = Release|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.Build.0 = Release|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.Build.0 = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -271,6 +267,22 @@ Global {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Debug|Any CPU.Build.0 = Debug|Any CPU {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.ActiveCfg = Release|Any CPU {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.Build.0 = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.Build.0 = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.Build.0 = Release|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Debug|Any CPU.Build.0 = Debug|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Release|Any CPU.ActiveCfg = Release|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Release|Any CPU.Build.0 = Release|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -317,10 +329,12 @@ Global {A89B766C-987F-4C9F-8937-D0AB9FE640C8} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {100348B5-4D97-4A3F-B777-AB14F276F8FE} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {D2779F32-A548-44F8-B60A-6AC018966C79} = {E5637F81-2FB9-4CD7-900D-455363B142A7} - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} - {B894780C-338F-475E-8E84-56AFA8197A06} = {E5637F81-2FB9-4CD7-900D-455363B142A7} {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {B894780C-338F-475E-8E84-56AFA8197A06} = {E5637F81-2FB9-4CD7-900D-455363B142A7} + {354CE69B-78DB-9B29-C67E-0DBB862C7A65} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {FE1E17DD-595A-123A-EA4C-AA313BBFB685} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71} diff --git a/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj b/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj new file mode 100644 index 000000000..884da7e8b --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj @@ -0,0 +1,23 @@ + + + + net6.0 + enable + enable + true + $(BaseIntermediateOutputPath)Generated + + + + + + + + + + + + + + + diff --git a/samples/ExportHistoryWebApp/ExportHistoryWebApp.http b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http new file mode 100644 index 000000000..aeea8e8c0 --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http @@ -0,0 +1,89 @@ +### Variables +@baseUrl = http://localhost:5010 +@jobId = export-job-12345 + +### Create a new batch export job +# @name createBatchExportJob +POST {{baseUrl}}/export-jobs +Content-Type: application/json + +{ + "jobId": "{{jobId}}", + "mode": "Batch", + "completedTimeFrom": "2025-10-01T00:00:00Z", + "completedTimeTo": "2025-11-06T23:59:59Z", + "container": "export-history", + # "prefix": "exports/", + "maxInstancesPerBatch": 1, + "runtimeStatus": [] +} + +### Create a new continuous export job +# @name createContinuousExportJob +POST {{baseUrl}}/export-jobs +Content-Type: application/json + +{ + "jobId": "export-job-continuous-123", + "mode": "Continuous", + "container": "export-history", + # "prefix": "continuous-exports/", + "maxInstancesPerBatch": 1000 + # "runtimeStatus": ["asdasd"] +} + +### Create an export job with default storage (no container specified) +# @name createExportJobWithDefaultStorage +POST {{baseUrl}}/export-jobs +Content-Type: application/json +{ + "jobId": "export-job-default-storage", + "mode": "Batch", + "completedTimeFrom": "2024-01-01T00:00:00Z", + "completedTimeTo": "2024-12-31T23:59:59Z", + "maxInstancesPerBatch": 100 +} + +### Get a specific export job by ID +# Note: This endpoint can be used to verify the export job was created and check its status +# The ID in the URL should match the jobId used in create request +GET {{baseUrl}}/export-jobs/{{jobId}} + +### List all export jobs +GET {{baseUrl}}/export-jobs/list + +### List export jobs with filters +### Filter by status +GET {{baseUrl}}/export-jobs/list?status=Active + +### Filter by job ID prefix +GET {{baseUrl}}/export-jobs/list?jobIdPrefix=export-job- + +### Filter by creation time range +GET {{baseUrl}}/export-jobs/list?createdFrom=2024-01-01T00:00:00Z&createdTo=2024-12-31T23:59:59Z + +### Combined filters +GET {{baseUrl}}/export-jobs/list?status=Completed&jobIdPrefix=export-job-&pageSize=50 + +### Delete an export job +# DELETE {{baseUrl}}/export-jobs/{{jobId}} + +# Delete a continuous export job +DELETE {{baseUrl}}/export-jobs/export-job-continuous-123jk + +### Tips: +# - Replace the baseUrl variable if your application runs on a different port +# - The jobId variable can be changed to test different export job instances +# - Export modes: +# - "Batch": Exports all instances within a time range (requires completedTimeTo) +# - "Continuous": Continuously exports instances from a start time (completedTimeTo must be null) +# - Runtime status filters (valid values): +# - "Completed": Exports only completed orchestrations +# - "Failed": Exports only failed orchestrations +# - "Terminated": Exports only terminated orchestrations +# - "ContinuedAsNew": Exports only continued-as-new orchestrations +# - Dates are in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ) +# - You can use the REST Client extension in VS Code to execute these requests +# - The @name directive allows referencing the response in subsequent requests +# - Export jobs run asynchronously; use GET to check the status after creation + diff --git a/samples/ExportHistoryWebApp/ExportJobController.cs b/samples/ExportHistoryWebApp/ExportJobController.cs new file mode 100644 index 000000000..859950429 --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportJobController.cs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.Mvc; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.ExportHistory; +using ExportHistoryWebApp.Models; + +namespace ExportHistoryWebApp.Controllers; + +/// +/// Controller for managing export history jobs through a REST API. +/// Provides endpoints for creating, reading, listing, and deleting export jobs. +/// +[ApiController] +[Route("export-jobs")] +public class ExportJobController : ControllerBase +{ + readonly ExportHistoryClient exportHistoryClient; + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// Client for managing export history jobs. + /// Logger for recording controller operations. + public ExportJobController( + ExportHistoryClient exportHistoryClient, + ILogger logger) + { + this.exportHistoryClient = exportHistoryClient ?? throw new ArgumentNullException(nameof(exportHistoryClient)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Creates a new export job based on the provided configuration. + /// + /// The export job creation request. + /// The created export job description. + [HttpPost] + public async Task> CreateExportJob([FromBody] CreateExportJobRequest request) + { + if (request == null) + { + return this.BadRequest("createExportJobRequest cannot be null"); + } + + try + { + ExportDestination? destination = null; + if (!string.IsNullOrEmpty(request.Container)) + { + destination = new ExportDestination(request.Container) + { + Prefix = request.Prefix, + }; + } + + ExportJobCreationOptions creationOptions = new ExportJobCreationOptions( + mode: request.Mode, + completedTimeFrom: request.CompletedTimeFrom, + completedTimeTo: request.CompletedTimeTo, + destination: destination, + jobId: request.JobId, + format: request.Format, + runtimeStatus: request.RuntimeStatus, + maxInstancesPerBatch: request.MaxInstancesPerBatch); + + ExportHistoryJobClient jobClient = await this.exportHistoryClient.CreateJobAsync(creationOptions); + ExportJobDescription description = await jobClient.DescribeAsync(); + + this.logger.LogInformation("Created new export job with ID: {JobId}", description.JobId); + + return this.CreatedAtAction(nameof(GetExportJob), new { id = description.JobId }, description); + } + catch (ArgumentException ex) + { + this.logger.LogError(ex, "Validation failed while creating export job {JobId}", request.JobId); + return this.BadRequest(ex.Message); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error creating export job {JobId}", request.JobId); + return this.StatusCode(500, "An error occurred while creating the export job"); + } + } + + /// + /// Retrieves a specific export job by its ID. + /// + /// The ID of the export job to retrieve. + /// The export job description if found. + [HttpGet("{id}")] + public async Task> GetExportJob(string id) + { + try + { + ExportJobDescription? job = await this.exportHistoryClient.GetJobAsync(id); + return this.Ok(job); + } + catch (ExportJobNotFoundException) + { + return this.NotFound(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error retrieving export job {JobId}", id); + return this.StatusCode(500, "An error occurred while retrieving the export job"); + } + } + + /// + /// Lists all export jobs, optionally filtered by query parameters. + /// + /// Optional filter by job status. + /// Optional filter by job ID prefix. + /// Optional filter for jobs created after this time. + /// Optional filter for jobs created before this time. + /// Optional page size for pagination. + /// Optional continuation token for pagination. + /// A collection of export job descriptions. + [HttpGet("list")] + public async Task>> ListExportJobs( + [FromQuery] ExportJobStatus? status = null, + [FromQuery] string? jobIdPrefix = null, + [FromQuery] DateTimeOffset? createdFrom = null, + [FromQuery] DateTimeOffset? createdTo = null, + [FromQuery] int? pageSize = null, + [FromQuery] string? continuationToken = null) + { + this.logger.LogInformation("GET list endpoint called with method: {Method}", this.HttpContext.Request.Method); + try + { + ExportJobQuery? query = null; + if ( + status.HasValue || + !string.IsNullOrEmpty(jobIdPrefix) || + createdFrom.HasValue || + createdTo.HasValue || + pageSize.HasValue || + !string.IsNullOrEmpty(continuationToken) + ) + { + query = new ExportJobQuery + { + Status = status, + JobIdPrefix = jobIdPrefix, + CreatedFrom = createdFrom, + CreatedTo = createdTo, + PageSize = pageSize, + ContinuationToken = continuationToken, + }; + } + + AsyncPageable jobs = this.exportHistoryClient.ListJobsAsync(query); + + // Collect all jobs from the async pageable + List jobList = new List(); + await foreach (ExportJobDescription job in jobs) + { + jobList.Add(job); + } + + return this.Ok(jobList); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error retrieving export jobs"); + return this.StatusCode(500, "An error occurred while retrieving export jobs"); + } + } + + /// + /// Deletes an export job by its ID. + /// + /// The ID of the export job to delete. + /// No content if successful. + [HttpDelete("{id}")] + public async Task DeleteExportJob(string id) + { + this.logger.LogInformation("DELETE endpoint called for job ID: {JobId}", id); + try + { + ExportHistoryJobClient jobClient = this.exportHistoryClient.GetJobClient(id); + await jobClient.DeleteAsync(); + this.logger.LogInformation("Successfully deleted export job {JobId}", id); + return this.NoContent(); + } + catch (ExportJobNotFoundException) + { + this.logger.LogWarning("Export job {JobId} not found for deletion", id); + return this.NotFound(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error deleting export job {JobId}", id); + return this.StatusCode(500, "An error occurred while deleting the export job"); + } + } +} + diff --git a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs new file mode 100644 index 000000000..1e09ddf51 --- /dev/null +++ b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.ExportHistory; + +namespace ExportHistoryWebApp.Models; + +/// +/// Represents a request to create a new export job. +/// +public class CreateExportJobRequest +{ + /// + /// Gets or sets the unique identifier for the export job. If not provided, a GUID will be generated. + /// + public string? JobId { get; set; } + + /// + /// Gets or sets the export mode (Batch or Continuous). + /// + public ExportMode Mode { get; set; } + + /// + /// Gets or sets the start time for the export based on completion time (inclusive). Required. + /// + public DateTimeOffset CompletedTimeFrom { get; set; } + + /// + /// Gets or sets the end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. + /// + public DateTimeOffset? CompletedTimeTo { get; set; } + + /// + /// Gets or sets the blob container name where exported data will be stored. Optional if default storage is configured. + /// + public string? Container { get; set; } + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } + + /// + /// Gets or sets the export format settings. Optional, defaults to jsonl-gzip. + /// + public ExportFormat? Format { get; set; } + + /// + /// Gets or sets the orchestration runtime statuses to filter by. Optional. + /// Valid statuses are: Completed, Failed, Terminated. + /// + public List? RuntimeStatus { get; set; } + + /// + /// Gets or sets the maximum number of instances to fetch per batch. Optional, defaults to 100. + /// + public int? MaxInstancesPerBatch { get; set; } +} + diff --git a/samples/ExportHistoryWebApp/Program.cs b/samples/ExportHistoryWebApp/Program.cs new file mode 100644 index 000000000..46ee02f41 --- /dev/null +++ b/samples/ExportHistoryWebApp/Program.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.AzureManaged; +using Microsoft.DurableTask.ExportHistory; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.AzureManaged; + +WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + +string connectionString = builder.Configuration.GetValue("DURABLE_TASK_CONNECTION_STRING") + ?? throw new InvalidOperationException("Missing required configuration 'DURABLE_TASK_CONNECTION_STRING'"); + +string storageConnectionString = builder.Configuration.GetValue("EXPORT_HISTORY_STORAGE_CONNECTION_STRING") + ?? throw new InvalidOperationException("Missing required configuration 'EXPORT_HISTORY_STORAGE_CONNECTION_STRING'"); + +string containerName = builder.Configuration.GetValue("EXPORT_HISTORY_CONTAINER_NAME") + ?? throw new InvalidOperationException("Missing required configuration 'EXPORT_HISTORY_CONTAINER_NAME'"); + +builder.Services.AddSingleton(sp => sp.GetRequiredService().CreateLogger()); +builder.Services.AddLogging(); + +// Add Durable Task worker with export history support +builder.Services.AddDurableTaskWorker(builder => +{ + builder.UseDurableTaskScheduler(connectionString); + builder.UseExportHistory(); +}); + +// Register the client with export history support +builder.Services.AddDurableTaskClient(clientBuilder => +{ + clientBuilder.UseDurableTaskScheduler(connectionString); + clientBuilder.UseExportHistory(options => + { + options.ConnectionString = storageConnectionString; + options.ContainerName = containerName; + options.Prefix = builder.Configuration.GetValue("EXPORT_HISTORY_PREFIX"); + }); +}); + +// Configure the HTTP request pipeline +builder.Services.AddControllers().AddJsonOptions(options => +{ + options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); + options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; +}); + +// The actual listen URL can be configured in environment variables named "ASPNETCORE_URLS" or "ASPNETCORE_URLS_HTTPS" +WebApplication app = builder.Build(); +app.MapControllers(); +app.Run(); + diff --git a/samples/ExportHistoryWebApp/Properties/launchSettings.json b/samples/ExportHistoryWebApp/Properties/launchSettings.json new file mode 100644 index 000000000..4375d42ec --- /dev/null +++ b/samples/ExportHistoryWebApp/Properties/launchSettings.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:47698", + "sslPort": 44372 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "applicationUrl": "http://localhost:5009", + "dotnetRunMessages": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DURABLE_TASK_CONNECTION_STRING": "", + "EXPORT_HISTORY_STORAGE_CONNECTION_STRING": "", + "EXPORT_HISTORY_CONTAINER_NAME": "export-history", + "EXPORT_HISTORY_PREFIX": "" + } + } + } +} + diff --git a/samples/ExportHistoryWebApp/appsettings.Development.json b/samples/ExportHistoryWebApp/appsettings.Development.json new file mode 100644 index 000000000..21b22dbb4 --- /dev/null +++ b/samples/ExportHistoryWebApp/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} + diff --git a/samples/ExportHistoryWebApp/appsettings.json b/samples/ExportHistoryWebApp/appsettings.json new file mode 100644 index 000000000..fb8785032 --- /dev/null +++ b/samples/ExportHistoryWebApp/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} + diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index f2d658d8f..1ea0b2554 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -1,441 +1,442 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.ComponentModel; -using Microsoft.DurableTask.Client.Entities; -using Microsoft.DurableTask.Internal; - -namespace Microsoft.DurableTask.Client; - -/// -/// Base class that defines client operations for managing durable task instances. -/// -/// -/// -/// Instances of can be used to start, query, raise events to, and terminate -/// orchestration instances. In most cases, methods on this class accept an instance ID as a parameter, which identifies -/// the orchestration instance. -/// -/// At the time of writing, the most common implementation of this class is the gRPC client, which works by making gRPC -/// calls to a remote service (e.g. a sidecar) that implements the operation behavior. To ensure any owned network -/// resources are properly released, instances of should be disposed when they are no -/// longer needed. -/// -/// Instances of this class are expected to be safe for multithreaded apps. You can therefore safely cache instances -/// of this class and reuse them across multiple contexts. Caching these objects is useful to improve overall -/// performance. -/// -/// -public abstract class DurableTaskClient : IOrchestrationSubmitter, IAsyncDisposable -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the client. - protected DurableTaskClient(string name) - { - this.Name = name; - } - - /// - /// Gets the name of the client. - /// - public string Name { get; } - - /// - /// Gets the for interacting with durable entities. - /// - /// - /// Not all clients support durable entities. Refer to a specific client implementation for verifying support. - /// - public virtual DurableEntityClient Entities => - throw new NotSupportedException($"{this.GetType()} does not support durable entities."); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, CancellationToken cancellation) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, null, cancellation); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, object? input, CancellationToken cancellation) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input, null, cancellation); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, StartOrchestrationOptions options, CancellationToken cancellation = default) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, options, cancellation); - - /// - /// Schedules a new orchestration instance for execution. - /// - /// - /// All orchestrations must have a unique instance ID. You can provide an instance ID using the - /// parameter or you can omit this and a random instance ID will be - /// generated for you automatically. If an orchestration with the specified instance ID already exists and is in a - /// non-terminal state (Pending, Running, etc.), then this operation may fail silently. However, if an orchestration - /// instance with this ID already exists in a terminal state (Completed, Terminated, Failed, etc.) then the instance - /// may be recreated automatically, depending on the configuration of the backend instance store. - /// - /// Orchestration instances started with this method will be created in the - /// state and will transition to the - /// after successfully awaiting its first task. The exact time it - /// takes before a scheduled orchestration starts running depends on several factors, including the configuration - /// and health of the backend task hub, and whether a start time was provided via . - /// - /// The task associated with this method completes after the orchestration instance was successfully scheduled. You - /// can use the to query the status of the - /// scheduled instance, the method to wait - /// for the instance to transition out of the status, or the - /// method to wait for the instance to - /// reach a terminal state (Completed, Terminated, Failed, etc.). - /// - /// - /// The name of the orchestrator to schedule. - /// - /// The optional input to pass to the scheduled orchestration instance. This must be a serializable value. - /// - /// The options to start the new orchestration with. - /// - /// The cancellation token. This only cancels enqueueing the new orchestration to the backend. Does not cancel the - /// orchestration once enqueued. - /// - /// - /// A task that completes when the orchestration instance is successfully scheduled. The value of this task is - /// the instance ID of the scheduled orchestration instance. If a non-null instance ID was provided via - /// , the same value will be returned by the completed task. - /// - /// Thrown if is empty. - public abstract Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, - object? input = null, - StartOrchestrationOptions? options = null, - CancellationToken cancellation = default); - - /// - public virtual Task RaiseEventAsync( - string instanceId, string eventName, CancellationToken cancellation) - => this.RaiseEventAsync(instanceId, eventName, null, cancellation); - - /// - /// Sends an event notification message to a waiting orchestration instance. - /// - /// - /// - /// In order to handle the event, the target orchestration instance must be waiting for an - /// event named using the - /// API. - /// If the target orchestration instance is not yet waiting for an event named , - /// then the event will be saved in the orchestration instance state and dispatched immediately when the - /// orchestrator calls . - /// This event saving occurs even if the orchestrator has canceled its wait operation before the event was received. - /// - /// Orchestrators can wait for the same event name multiple times, so sending multiple events with the same name is - /// allowed. Each external event received by an orchestrator will complete just one task returned by the - /// method. - /// - /// Raised events for a completed or non-existent orchestration instance will be silently discarded. - /// - /// - /// The ID of the orchestration instance that will handle the event. - /// The name of the event. Event names are case-insensitive. - /// The serializable data payload to include with the event. - /// - /// The cancellation token. This only cancels enqueueing the event to the backend. Does not abort sending the event - /// once enqueued. - /// - /// A task that completes when the event notification message has been enqueued. - /// - /// Thrown if or is null or empty. - /// - public abstract Task RaiseEventAsync( - string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default); - - /// - public virtual Task WaitForInstanceStartAsync( - string instanceId, CancellationToken cancellation) - => this.WaitForInstanceStartAsync(instanceId, false, cancellation); - - /// - /// Waits for an orchestration to start running and returns a - /// object that contains metadata about the started instance. - /// - /// - /// - /// A "started" orchestration instance is any instance not in the - /// state. - /// - /// If an orchestration instance is already running when this method is called, the method will return immediately. - /// - /// - /// The unique ID of the orchestration instance to wait for. - /// - /// Specify true to fetch the orchestration instance's inputs, outputs, and custom status, or false to - /// omit them. The default value is false to minimize the network bandwidth, serialization, and memory costs - /// associated with fetching the instance metadata. - /// - /// A that can be used to cancel the wait operation. - /// - /// Returns a record that describes the orchestration instance and its execution - /// status or null if no instance with ID is found. - /// - public abstract Task WaitForInstanceStartAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - public virtual Task WaitForInstanceCompletionAsync( - string instanceId, CancellationToken cancellation) - => this.WaitForInstanceCompletionAsync(instanceId, false, cancellation); - - /// - /// Waits for an orchestration to complete and returns a - /// object that contains metadata about the started instance. - /// - /// - /// - /// A "completed" orchestration instance is any instance in one of the terminal states. For example, the - /// , , or - /// states. - /// - /// Orchestrations are long-running and could take hours, days, or months before completing. - /// Orchestrations can also be eternal, in which case they'll never complete unless terminated. - /// In such cases, this call may block indefinitely, so care must be taken to ensure appropriate timeouts are - /// enforced using the parameter. - /// - /// If an orchestration instance is already complete when this method is called, the method will return immediately. - /// - /// - /// - public abstract Task WaitForInstanceCompletionAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - public virtual Task TerminateInstanceAsync(string instanceId, CancellationToken cancellation) - => this.TerminateInstanceAsync(instanceId, null, cancellation); - - /// - public virtual Task TerminateInstanceAsync(string instanceId, object? output, CancellationToken cancellation = default) - { - TerminateInstanceOptions? options = output is null ? null : new() { Output = output }; - return this.TerminateInstanceAsync(instanceId, options, cancellation); - } - - /// - /// Terminates an orchestration instance and updates its runtime status to - /// . - /// - /// - /// - /// This method internally enqueues a "terminate" message in the task hub. When the task hub worker processes - /// this message, it will update the runtime status of the target instance to - /// . You can use the - /// to wait for the instance to reach - /// the terminated state. - /// - /// - /// Terminating an orchestration by default will not terminate any of the child sub-orchestrations that were started by - /// the orchetration instance. If you want to terminate sub-orchestration instances as well, you can set - /// flag to true which will enable termination of child sub-orchestration instances. It is set to false by default. - /// Terminating an orchestration instance has no effect on any in-flight activity function executions - /// that were started by the terminated instance. Those actions will continue to run - /// without interruption. However, their results will be discarded. - /// - /// At the time of writing, there is no way to terminate an in-flight activity execution. - /// - /// Attempting to terminate a completed or non-existent orchestration instance will fail silently. - /// - /// - /// The ID of the orchestration instance to terminate. - /// The optional options for terminating the orchestration. - /// - /// The cancellation token. This only cancels enqueueing the termination request to the backend. Does not abort - /// termination of the orchestration once enqueued. - /// - /// A task that completes when the terminate message is enqueued. - public virtual Task TerminateInstanceAsync(string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support orchestration termination."); - - /// - public virtual Task SuspendInstanceAsync(string instanceId, CancellationToken cancellation) - => this.SuspendInstanceAsync(instanceId, null, cancellation); - - /// - /// Suspends an orchestration instance, halting processing of it until - /// is used to resume the orchestration. - /// - /// The instance ID of the orchestration to suspend. - /// The optional suspension reason. - /// - /// A that can be used to cancel the suspend operation. Note, cancelling this token - /// does not resume the orchestration if suspend was successful. - /// - /// A task that completes when the suspend has been committed to the backend. - public abstract Task SuspendInstanceAsync( - string instanceId, string? reason = null, CancellationToken cancellation = default); - - /// - public virtual Task ResumeInstanceAsync(string instanceId, CancellationToken cancellation) - => this.ResumeInstanceAsync(instanceId, null, cancellation); - - /// - /// Resumes an orchestration instance that was suspended via . - /// - /// The instance ID of the orchestration to resume. - /// The optional resume reason. - /// - /// A that can be used to cancel the resume operation. Note, cancelling this token - /// does not re-suspend the orchestration if resume was successful. - /// - /// A task that completes when the resume has been committed to the backend. - public abstract Task ResumeInstanceAsync( - string instanceId, string? reason = null, CancellationToken cancellation = default); - - /// - public virtual Task GetInstanceAsync( - string instanceId, CancellationToken cancellation) - => this.GetInstanceAsync(instanceId, false, cancellation); - - /// - /// Fetches orchestration instance metadata from the configured durable store. - /// - /// - /// You can use the parameter to determine whether to fetch input and - /// output data for the target orchestration instance. If your code doesn't require access to this data, it's - /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and - /// memory costs associated with fetching the instance metadata. - /// - /// - public virtual Task GetInstanceAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) - => this.GetInstancesAsync(instanceId, getInputsAndOutputs, cancellation); - - /// - [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync - public virtual Task GetInstancesAsync( - string instanceId, CancellationToken cancellation) - => this.GetInstancesAsync(instanceId, false, cancellation); - - /// - /// Fetches orchestration instance metadata from the configured durable store. - /// - /// - /// You can use the parameter to determine whether to fetch input and - /// output data for the target orchestration instance. If your code doesn't require access to this data, it's - /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and - /// memory costs associated with fetching the instance metadata. - /// - /// - [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync - public abstract Task GetInstancesAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - /// Queries orchestration instances. - /// - /// Filters down the instances included in the query. - /// An async pageable of the query results. - public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); - - /// - public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) - => this.PurgeInstanceAsync(instanceId, null, cancellation); - - /// - /// Purges orchestration instance metadata from the durable store. - /// - /// - /// - /// This method can be used to permanently delete orchestration metadata from the underlying storage provider, - /// including any stored inputs, outputs, and orchestration history records. This is often useful for implementing - /// data retention policies and for keeping storage costs minimal. Only orchestration instances in the - /// , , or - /// state can be purged. - /// - /// Purging an orchestration will by default not purge any of the child sub-orchestrations that were started by the - /// orchetration instance. Currently, purging of sub-orchestrations is not supported. - /// If is not found in the data store, or if the instance is found but not in a - /// terminal state, then the returned object will have a - /// value of 0. Otherwise, the existing data will be purged and - /// will be the count of purged instances. - /// - /// - /// The unique ID of the orchestration instance to purge. - /// The optional options for purging the orchestration. - /// - /// A that can be used to cancel the purge operation. - /// - /// - /// This method returns a object after the operation has completed with a - /// indicating the number of orchestration instances that were purged, - /// including the count of sub-orchestrations purged if any. - /// - public virtual Task PurgeInstanceAsync( - string instanceId, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) - { - throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); - } - - /// - public virtual Task PurgeAllInstancesAsync(PurgeInstancesFilter filter, CancellationToken cancellation) - => this.PurgeAllInstancesAsync(filter, null, cancellation); - - /// - /// Purges orchestration instances metadata from the durable store. - /// - /// The filter for which orchestrations to purge. - /// The optional options for purging the orchestration. - /// - /// A that can be used to cancel the purge operation. - /// - /// - /// This method returns a object after the operation has completed with a - /// indicating the number of orchestration instances that were purged. - /// - public virtual Task PurgeAllInstancesAsync( - PurgeInstancesFilter filter, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) - { - throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); - } - - /// - /// Restarts an orchestration instance with the same or a new instance ID. - /// - /// - /// - /// This method restarts an existing orchestration instance. If is true, - /// a new instance ID will be generated for the restarted orchestration. If false, the original instance ID will be reused. - /// - /// The restarted orchestration will use the same input data as the original instance. If the original orchestration - /// instance is not found, an will be thrown. - /// - /// Note that this operation is backend-specific and may not be supported by all durable task backends. - /// If the backend does not support restart operations, a will be thrown. - /// - /// - /// The ID of the orchestration instance to restart. - /// - /// If true, a new instance ID will be generated for the restarted orchestration. - /// If false, the original instance ID will be reused. - /// - /// - /// The cancellation token. This only cancels enqueueing the restart request to the backend. - /// Does not abort restarting the orchestration once enqueued. - /// - /// - /// A task that completes when the orchestration instance is successfully restarted. - /// The value of this task is the instance ID of the restarted orchestration instance. - /// - /// - /// Thrown if an orchestration with the specified was not found. - /// - /// Thrown when attempting to restart an instance using the same instance Id - /// while the instance has not yet reached a completed or terminal state. - /// - /// Thrown if the backend does not support restart operations. - public virtual Task RestartAsync( - string instanceId, - bool restartWithNewInstanceId = false, - CancellationToken cancellation = default) +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.ComponentModel; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Internal; +using DurableTask.Core.History; + +namespace Microsoft.DurableTask.Client; + +/// +/// Base class that defines client operations for managing durable task instances. +/// +/// +/// +/// Instances of can be used to start, query, raise events to, and terminate +/// orchestration instances. In most cases, methods on this class accept an instance ID as a parameter, which identifies +/// the orchestration instance. +/// +/// At the time of writing, the most common implementation of this class is the gRPC client, which works by making gRPC +/// calls to a remote service (e.g. a sidecar) that implements the operation behavior. To ensure any owned network +/// resources are properly released, instances of should be disposed when they are no +/// longer needed. +/// +/// Instances of this class are expected to be safe for multithreaded apps. You can therefore safely cache instances +/// of this class and reuse them across multiple contexts. Caching these objects is useful to improve overall +/// performance. +/// +/// +public abstract class DurableTaskClient : IOrchestrationSubmitter, IAsyncDisposable +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the client. + protected DurableTaskClient(string name) + { + this.Name = name; + } + + /// + /// Gets the name of the client. + /// + public string Name { get; } + + /// + /// Gets the for interacting with durable entities. + /// + /// + /// Not all clients support durable entities. Refer to a specific client implementation for verifying support. + /// + public virtual DurableEntityClient Entities => + throw new NotSupportedException($"{this.GetType()} does not support durable entities."); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, CancellationToken cancellation) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, null, cancellation); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, object? input, CancellationToken cancellation) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input, null, cancellation); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, StartOrchestrationOptions options, CancellationToken cancellation = default) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, options, cancellation); + + /// + /// Schedules a new orchestration instance for execution. + /// + /// + /// All orchestrations must have a unique instance ID. You can provide an instance ID using the + /// parameter or you can omit this and a random instance ID will be + /// generated for you automatically. If an orchestration with the specified instance ID already exists and is in a + /// non-terminal state (Pending, Running, etc.), then this operation may fail silently. However, if an orchestration + /// instance with this ID already exists in a terminal state (Completed, Terminated, Failed, etc.) then the instance + /// may be recreated automatically, depending on the configuration of the backend instance store. + /// + /// Orchestration instances started with this method will be created in the + /// state and will transition to the + /// after successfully awaiting its first task. The exact time it + /// takes before a scheduled orchestration starts running depends on several factors, including the configuration + /// and health of the backend task hub, and whether a start time was provided via . + /// + /// The task associated with this method completes after the orchestration instance was successfully scheduled. You + /// can use the to query the status of the + /// scheduled instance, the method to wait + /// for the instance to transition out of the status, or the + /// method to wait for the instance to + /// reach a terminal state (Completed, Terminated, Failed, etc.). + /// + /// + /// The name of the orchestrator to schedule. + /// + /// The optional input to pass to the scheduled orchestration instance. This must be a serializable value. + /// + /// The options to start the new orchestration with. + /// + /// The cancellation token. This only cancels enqueueing the new orchestration to the backend. Does not cancel the + /// orchestration once enqueued. + /// + /// + /// A task that completes when the orchestration instance is successfully scheduled. The value of this task is + /// the instance ID of the scheduled orchestration instance. If a non-null instance ID was provided via + /// , the same value will be returned by the completed task. + /// + /// Thrown if is empty. + public abstract Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, + object? input = null, + StartOrchestrationOptions? options = null, + CancellationToken cancellation = default); + + /// + public virtual Task RaiseEventAsync( + string instanceId, string eventName, CancellationToken cancellation) + => this.RaiseEventAsync(instanceId, eventName, null, cancellation); + + /// + /// Sends an event notification message to a waiting orchestration instance. + /// + /// + /// + /// In order to handle the event, the target orchestration instance must be waiting for an + /// event named using the + /// API. + /// If the target orchestration instance is not yet waiting for an event named , + /// then the event will be saved in the orchestration instance state and dispatched immediately when the + /// orchestrator calls . + /// This event saving occurs even if the orchestrator has canceled its wait operation before the event was received. + /// + /// Orchestrators can wait for the same event name multiple times, so sending multiple events with the same name is + /// allowed. Each external event received by an orchestrator will complete just one task returned by the + /// method. + /// + /// Raised events for a completed or non-existent orchestration instance will be silently discarded. + /// + /// + /// The ID of the orchestration instance that will handle the event. + /// The name of the event. Event names are case-insensitive. + /// The serializable data payload to include with the event. + /// + /// The cancellation token. This only cancels enqueueing the event to the backend. Does not abort sending the event + /// once enqueued. + /// + /// A task that completes when the event notification message has been enqueued. + /// + /// Thrown if or is null or empty. + /// + public abstract Task RaiseEventAsync( + string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default); + + /// + public virtual Task WaitForInstanceStartAsync( + string instanceId, CancellationToken cancellation) + => this.WaitForInstanceStartAsync(instanceId, false, cancellation); + + /// + /// Waits for an orchestration to start running and returns a + /// object that contains metadata about the started instance. + /// + /// + /// + /// A "started" orchestration instance is any instance not in the + /// state. + /// + /// If an orchestration instance is already running when this method is called, the method will return immediately. + /// + /// + /// The unique ID of the orchestration instance to wait for. + /// + /// Specify true to fetch the orchestration instance's inputs, outputs, and custom status, or false to + /// omit them. The default value is false to minimize the network bandwidth, serialization, and memory costs + /// associated with fetching the instance metadata. + /// + /// A that can be used to cancel the wait operation. + /// + /// Returns a record that describes the orchestration instance and its execution + /// status or null if no instance with ID is found. + /// + public abstract Task WaitForInstanceStartAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + public virtual Task WaitForInstanceCompletionAsync( + string instanceId, CancellationToken cancellation) + => this.WaitForInstanceCompletionAsync(instanceId, false, cancellation); + + /// + /// Waits for an orchestration to complete and returns a + /// object that contains metadata about the started instance. + /// + /// + /// + /// A "completed" orchestration instance is any instance in one of the terminal states. For example, the + /// , , or + /// states. + /// + /// Orchestrations are long-running and could take hours, days, or months before completing. + /// Orchestrations can also be eternal, in which case they'll never complete unless terminated. + /// In such cases, this call may block indefinitely, so care must be taken to ensure appropriate timeouts are + /// enforced using the parameter. + /// + /// If an orchestration instance is already complete when this method is called, the method will return immediately. + /// + /// + /// + public abstract Task WaitForInstanceCompletionAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + public virtual Task TerminateInstanceAsync(string instanceId, CancellationToken cancellation) + => this.TerminateInstanceAsync(instanceId, null, cancellation); + + /// + public virtual Task TerminateInstanceAsync(string instanceId, object? output, CancellationToken cancellation = default) + { + TerminateInstanceOptions? options = output is null ? null : new() { Output = output }; + return this.TerminateInstanceAsync(instanceId, options, cancellation); + } + + /// + /// Terminates an orchestration instance and updates its runtime status to + /// . + /// + /// + /// + /// This method internally enqueues a "terminate" message in the task hub. When the task hub worker processes + /// this message, it will update the runtime status of the target instance to + /// . You can use the + /// to wait for the instance to reach + /// the terminated state. + /// + /// + /// Terminating an orchestration by default will not terminate any of the child sub-orchestrations that were started by + /// the orchetration instance. If you want to terminate sub-orchestration instances as well, you can set + /// flag to true which will enable termination of child sub-orchestration instances. It is set to false by default. + /// Terminating an orchestration instance has no effect on any in-flight activity function executions + /// that were started by the terminated instance. Those actions will continue to run + /// without interruption. However, their results will be discarded. + /// + /// At the time of writing, there is no way to terminate an in-flight activity execution. + /// + /// Attempting to terminate a completed or non-existent orchestration instance will fail silently. + /// + /// + /// The ID of the orchestration instance to terminate. + /// The optional options for terminating the orchestration. + /// + /// The cancellation token. This only cancels enqueueing the termination request to the backend. Does not abort + /// termination of the orchestration once enqueued. + /// + /// A task that completes when the terminate message is enqueued. + public virtual Task TerminateInstanceAsync(string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support orchestration termination."); + + /// + public virtual Task SuspendInstanceAsync(string instanceId, CancellationToken cancellation) + => this.SuspendInstanceAsync(instanceId, null, cancellation); + + /// + /// Suspends an orchestration instance, halting processing of it until + /// is used to resume the orchestration. + /// + /// The instance ID of the orchestration to suspend. + /// The optional suspension reason. + /// + /// A that can be used to cancel the suspend operation. Note, cancelling this token + /// does not resume the orchestration if suspend was successful. + /// + /// A task that completes when the suspend has been committed to the backend. + public abstract Task SuspendInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default); + + /// + public virtual Task ResumeInstanceAsync(string instanceId, CancellationToken cancellation) + => this.ResumeInstanceAsync(instanceId, null, cancellation); + + /// + /// Resumes an orchestration instance that was suspended via . + /// + /// The instance ID of the orchestration to resume. + /// The optional resume reason. + /// + /// A that can be used to cancel the resume operation. Note, cancelling this token + /// does not re-suspend the orchestration if resume was successful. + /// + /// A task that completes when the resume has been committed to the backend. + public abstract Task ResumeInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default); + + /// + public virtual Task GetInstanceAsync( + string instanceId, CancellationToken cancellation) + => this.GetInstanceAsync(instanceId, false, cancellation); + + /// + /// Fetches orchestration instance metadata from the configured durable store. + /// + /// + /// You can use the parameter to determine whether to fetch input and + /// output data for the target orchestration instance. If your code doesn't require access to this data, it's + /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and + /// memory costs associated with fetching the instance metadata. + /// + /// + public virtual Task GetInstanceAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) + => this.GetInstancesAsync(instanceId, getInputsAndOutputs, cancellation); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync + public virtual Task GetInstancesAsync( + string instanceId, CancellationToken cancellation) + => this.GetInstancesAsync(instanceId, false, cancellation); + + /// + /// Fetches orchestration instance metadata from the configured durable store. + /// + /// + /// You can use the parameter to determine whether to fetch input and + /// output data for the target orchestration instance. If your code doesn't require access to this data, it's + /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and + /// memory costs associated with fetching the instance metadata. + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync + public abstract Task GetInstancesAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + /// Queries orchestration instances. + /// + /// Filters down the instances included in the query. + /// An async pageable of the query results. + public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); + + /// + public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) + => this.PurgeInstanceAsync(instanceId, null, cancellation); + + /// + /// Purges orchestration instance metadata from the durable store. + /// + /// + /// + /// This method can be used to permanently delete orchestration metadata from the underlying storage provider, + /// including any stored inputs, outputs, and orchestration history records. This is often useful for implementing + /// data retention policies and for keeping storage costs minimal. Only orchestration instances in the + /// , , or + /// state can be purged. + /// + /// Purging an orchestration will by default not purge any of the child sub-orchestrations that were started by the + /// orchetration instance. Currently, purging of sub-orchestrations is not supported. + /// If is not found in the data store, or if the instance is found but not in a + /// terminal state, then the returned object will have a + /// value of 0. Otherwise, the existing data will be purged and + /// will be the count of purged instances. + /// + /// + /// The unique ID of the orchestration instance to purge. + /// The optional options for purging the orchestration. + /// + /// A that can be used to cancel the purge operation. + /// + /// + /// This method returns a object after the operation has completed with a + /// indicating the number of orchestration instances that were purged, + /// including the count of sub-orchestrations purged if any. + /// + public virtual Task PurgeInstanceAsync( + string instanceId, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); + } + + /// + public virtual Task PurgeAllInstancesAsync(PurgeInstancesFilter filter, CancellationToken cancellation) + => this.PurgeAllInstancesAsync(filter, null, cancellation); + + /// + /// Purges orchestration instances metadata from the durable store. + /// + /// The filter for which orchestrations to purge. + /// The optional options for purging the orchestration. + /// + /// A that can be used to cancel the purge operation. + /// + /// + /// This method returns a object after the operation has completed with a + /// indicating the number of orchestration instances that were purged. + /// + public virtual Task PurgeAllInstancesAsync( + PurgeInstancesFilter filter, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); + } + + /// + /// Restarts an orchestration instance with the same or a new instance ID. + /// + /// + /// + /// This method restarts an existing orchestration instance. If is true, + /// a new instance ID will be generated for the restarted orchestration. If false, the original instance ID will be reused. + /// + /// The restarted orchestration will use the same input data as the original instance. If the original orchestration + /// instance is not found, an will be thrown. + /// + /// Note that this operation is backend-specific and may not be supported by all durable task backends. + /// If the backend does not support restart operations, a will be thrown. + /// + /// + /// The ID of the orchestration instance to restart. + /// + /// If true, a new instance ID will be generated for the restarted orchestration. + /// If false, the original instance ID will be reused. + /// + /// + /// The cancellation token. This only cancels enqueueing the restart request to the backend. + /// Does not abort restarting the orchestration once enqueued. + /// + /// + /// A task that completes when the orchestration instance is successfully restarted. + /// The value of this task is the instance ID of the restarted orchestration instance. + /// + /// + /// Thrown if an orchestration with the specified was not found. + /// + /// Thrown when attempting to restart an instance using the same instance Id + /// while the instance has not yet reached a completed or terminal state. + /// + /// Thrown if the backend does not support restart operations. + public virtual Task RestartAsync( + string instanceId, + bool restartWithNewInstanceId = false, + CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support orchestration restart."); /// @@ -450,7 +451,7 @@ public virtual Task RestartAsync( /// /// The instance ID of the orchestration to rewind. /// The reason for the rewind. - /// The cancellation token. This only cancels enqueueing the rewind request to the backend. + /// The cancellation token. This only cancels enqueueing the rewind request to the backend. /// It does not abort rewinding the orchestration once the request has been enqueued. /// A task that represents the enqueueing of the rewind operation. /// Thrown if this implementation of does not @@ -464,15 +465,61 @@ public virtual Task RewindInstanceAsync( string instanceId, string reason, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support orchestration rewind."); - - // TODO: Create task hub - - // TODO: Delete task hub - - /// - /// Disposes any unmanaged resources associated with this . - /// - /// A that completes when the disposal completes. - public abstract ValueTask DisposeAsync(); -} + => throw new NotSupportedException($"{this.GetType()} does not support orchestration rewind."); + + /// + /// Lists orchestration instance IDs filtered by completed time. + /// + /// The runtime statuses to filter by. + /// The start time for completed time filter (inclusive). + /// The end time for completed time filter (inclusive). + /// The page size for pagination. + /// The last fetched instance key. + /// The cancellation token. + /// A page of instance IDs with continuation token. + public virtual Task> ListInstanceIdsAsync( + IEnumerable? runtimeStatus = null, + DateTimeOffset? completedTimeFrom = null, + DateTimeOffset? completedTimeTo = null, + int pageSize = OrchestrationQuery.DefaultPageSize, + string? lastInstanceKey = null, + CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support listing orchestration instance IDs by completed time."); + } + + /// + /// Streams the execution history events for an orchestration instance. + /// + /// + /// This method returns an async enumerable that yields history events as they are retrieved from the backend. + /// The history events are returned in chronological order and include all events that occurred during the + /// orchestration instance's execution. + /// + /// The instance ID of the orchestration to stream history for. + /// The optional execution ID. If null, the latest execution will be used. + /// + /// A that can be used to cancel the streaming operation. + /// + /// An async enumerable of objects. + /// Thrown if an orchestration with the specified does not exist. + /// Thrown if the operation is canceled via the token. + public virtual IAsyncEnumerable StreamInstanceHistoryAsync( + string instanceId, + string? executionId = null, + CancellationToken cancellation = default) + + { + throw new NotSupportedException($"{this.GetType()} does not support streaming instance history."); + } + + // TODO: Create task hub + + // TODO: Delete task hub + + /// + /// Disposes any unmanaged resources associated with this . + /// + /// A that completes when the disposal completes. + public abstract ValueTask DisposeAsync(); +} \ No newline at end of file diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index efc6d765c..b55fe2378 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -3,7 +3,9 @@ using System.Diagnostics; using System.Text; +using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; +using Grpc.Core; using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Tracing; using Microsoft.Extensions.DependencyInjection; @@ -323,6 +325,53 @@ public override async Task WaitForInstanceStartAsync( } } + /// + public override async Task> ListInstanceIdsAsync( + IEnumerable? runtimeStatus = null, + DateTimeOffset? completedTimeFrom = null, + DateTimeOffset? completedTimeTo = null, + int pageSize = OrchestrationQuery.DefaultPageSize, + string? lastInstanceKey = null, + CancellationToken cancellation = default) + { + Check.NotEntity(this.options.EnableEntitySupport, null); + + P.ListInstanceIdsRequest request = new() + { + PageSize = pageSize, + LastInstanceKey = lastInstanceKey ?? string.Empty, + }; + + if (runtimeStatus != null) + { + request.RuntimeStatus.AddRange(runtimeStatus.Select(x => x.ToGrpcStatus())); + } + + if (completedTimeFrom.HasValue) + { + request.CompletedTimeFrom = completedTimeFrom.Value.ToTimestamp(); + } + + if (completedTimeTo.HasValue) + { + request.CompletedTimeTo = completedTimeTo.Value.ToTimestamp(); + } + + try + { + P.ListInstanceIdsResponse response = await this.sidecarClient.ListInstanceIdsAsync( + request, + cancellationToken: cancellation); + + return new Page(response.InstanceIds.ToList(), response.LastInstanceKey); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.ListInstanceIdsAsync)} operation was canceled.", e, cancellation); + } + } + /// public override async Task WaitForInstanceCompletionAsync( string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) @@ -468,6 +517,64 @@ public override async Task RewindInstanceAsync( } } + /// + public override async IAsyncEnumerable StreamInstanceHistoryAsync( + string instanceId, + string? executionId = null, + CancellationToken cancellation = default) + { + Check.NotNullOrEmpty(instanceId); + Check.NotEntity(this.options.EnableEntitySupport, instanceId); + + P.StreamInstanceHistoryRequest request = new() + { + InstanceId = instanceId, + ExecutionId = executionId, + ForWorkItemProcessing = false, + }; + + using AsyncServerStreamingCall streamResponse = + this.sidecarClient.StreamInstanceHistory(request, cancellationToken: cancellation); + + IAsyncStreamReader responseStream = streamResponse.ResponseStream; + + // Create conversion state to maintain orchestration instance across events + // This is required for entity-related events (EntityOperationCalled, EntityLockRequested, etc.) + // which need the parent orchestration instance information from ExecutionStartedEvent + Microsoft.DurableTask.ProtoUtils.EntityConversionState conversionState = new(insertMissingEntityUnlocks: false); + + bool hasMore = true; + while (hasMore) + { + bool moveNextResult; + try + { + moveNextResult = await responseStream.MoveNext(cancellation).ConfigureAwait(false); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.StreamInstanceHistoryAsync)} operation was canceled.", e, cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.NotFound) + { + throw new ArgumentException($"An orchestration with the instanceId {instanceId} was not found.", e); + } + + hasMore = moveNextResult; + if (hasMore) + { + P.HistoryChunk chunk = responseStream.Current; + foreach (P.HistoryEvent protoEvent in chunk.Events) + { + // Use the conversion state's converter to maintain state across events + // This ensures entity events can access the orchestration instance from ExecutionStartedEvent + yield return conversionState.ConvertFromProto(protoEvent); + } + } + } + } + static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, out CallInvoker callInvoker) { if (options.Channel is GrpcChannel c) diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs new file mode 100644 index 000000000..bc32dd40f --- /dev/null +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using DurableTask.Core.History; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Activity that exports one orchestration instance history to the configured blob destination. +/// +/// +/// Initializes a new instance of the class. +/// +[DurableTask] +public class ExportInstanceHistoryActivity( + DurableTaskClient client, + ILogger logger, + IOptions storageOptions) : TaskActivity +{ + readonly DurableTaskClient client = Check.NotNull(client, nameof(client)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions?.Value, nameof(storageOptions)); + + /// + public override async Task RunAsync(TaskActivityContext context, ExportRequest input) + { + Check.NotNull(input, nameof(input)); + Check.NotNullOrEmpty(input.InstanceId, nameof(input.InstanceId)); + Check.NotNull(input.Destination, nameof(input.Destination)); + Check.NotNull(input.Format, nameof(input.Format)); + + string instanceId = input.InstanceId; + + try + { + this.logger.LogInformation("Starting export for instance {InstanceId}", instanceId); + + // Get instance metadata with inputs and outputs + OrchestrationMetadata? metadata = await this.client.GetInstanceAsync( + instanceId, + getInputsAndOutputs: true, + cancellation: CancellationToken.None); + + if (metadata == null) + { + string error = $"Instance {instanceId} not found"; + this.logger.LogWarning(error); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = error, + }; + } + + // Get completed timestamp (LastUpdatedAt for terminal states) + DateTimeOffset completedTimestamp = metadata.LastUpdatedAt; + if (!metadata.IsCompleted) + { + string error = $"Instance {instanceId} is not in a completed state"; + this.logger.LogWarning(error); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = error, + }; + } + + // Stream all history events + this.logger.LogInformation("Streaming history events for instance {InstanceId}", instanceId); + List historyEvents = new(); + await foreach (HistoryEvent historyEvent in this.client.StreamInstanceHistoryAsync( + instanceId, + executionId: null, // Use latest execution + cancellation: CancellationToken.None)) + { + historyEvents.Add(historyEvent); + } + + this.logger.LogInformation( + "Retrieved {EventCount} history events for instance {InstanceId}", + historyEvents.Count, + instanceId); + + // Create blob filename from hash of completed timestamp and instance ID + string blobFileName = GenerateBlobFileName(completedTimestamp, instanceId, input.Format); + + // Build blob path with prefix if provided + string blobPath = string.IsNullOrEmpty(input.Destination.Prefix) + ? blobFileName + : $"{input.Destination.Prefix.TrimEnd('/')}/{blobFileName}"; + + // Serialize history events to JSON + string jsonContent = SerializeInstanceData(historyEvents, input.Format); + + // Upload to blob storage + await this.UploadToBlobStorageAsync( + input.Destination.Container, + blobPath, + jsonContent, + input.Format, + instanceId, + CancellationToken.None); + + this.logger.LogInformation( + "Successfully exported instance {InstanceId} to blob {BlobPath}", + instanceId, + blobPath); + + return new ExportResult + { + InstanceId = instanceId, + Success = true, + }; + } + catch (Exception ex) + { + this.logger.LogError(ex, "Failed to export instance {InstanceId}", instanceId); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = ex.Message, + }; + } + } + + static string GenerateBlobFileName(DateTimeOffset completedTimestamp, string instanceId, ExportFormat format) + { + // Create hash from completed timestamp and instance ID + string hashInput = $"{completedTimestamp:O}|{instanceId}"; + byte[] hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(hashInput)); + string hash = Convert.ToHexString(hashBytes).ToLowerInvariant(); + + // Build filename with format extension + string extension = GetFileExtension(format); + + return $"{hash}.{extension}"; + } + + /// + /// Gets the file extension for a given export format. + /// + /// The export format. + /// The file extension (e.g., "json", "jsonl.gz"). + static string GetFileExtension(ExportFormat format) + { + return format.Kind switch + { + ExportFormatKind.Jsonl => "jsonl.gz", // JSONL format is compressed + ExportFormatKind.Json => "json", // JSON format is uncompressed + _ => "jsonl.gz", // Default to JSONL compressed + }; + } + + static string SerializeInstanceData( + List historyEvents, + ExportFormat format) + { + JsonSerializerOptions serializerOptions = new JsonSerializerOptions + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + IncludeFields = true, // Include fields, not just properties (matches JsonDataConverter pattern) + Converters = { new JsonStringEnumConverter() }, // Serialize enums as strings + }; + + if (format.Kind == ExportFormatKind.Jsonl) + { + // JSONL format: one history event per line + // Serialize as object to preserve runtime type (polymorphic serialization) + StringBuilder jsonlBuilder = new(); + + foreach (HistoryEvent historyEvent in historyEvents) + { + // Serialize as object to preserve the actual derived type + string json = JsonSerializer.Serialize((object)historyEvent, historyEvent.GetType(), serializerOptions); + jsonlBuilder.AppendLine(json); + } + + return jsonlBuilder.ToString(); + } + else + { + // JSON format: array of history events + // Convert to object array to preserve runtime types + object[] eventsAsObjects = historyEvents.Cast().ToArray(); + return JsonSerializer.Serialize(eventsAsObjects, serializerOptions); + } + } + + async Task UploadToBlobStorageAsync( + string containerName, + string blobPath, + string content, + ExportFormat format, + string instanceId, + CancellationToken cancellationToken) + { + // Create blob service client from connection string + // Note: Azure.Storage.Blobs clients (BlobServiceClient, BlobContainerClient, BlobClient) are lightweight + // wrappers that don't implement IDisposable/IAsyncDisposable. They use HttpClient internally which is + // managed by the framework and doesn't require explicit disposal. The clients are designed to be + // stateless and safe for reuse or short-lived usage without disposal. + BlobServiceClient serviceClient = new(this.storageOptions.ConnectionString); + BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(containerName); + + // Ensure container exists + await containerClient.CreateIfNotExistsAsync( + PublicAccessType.None, + cancellationToken: cancellationToken); + + // Get blob client + BlobClient blobClient = containerClient.GetBlobClient(blobPath); + + // Upload content + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + + if (format.Kind == ExportFormatKind.Jsonl) + { + // Compress with gzip + using MemoryStream compressedStream = new(); + using (GZipStream gzipStream = new(compressedStream, CompressionLevel.Optimal, leaveOpen: true)) + { + await gzipStream.WriteAsync(contentBytes, cancellationToken); + await gzipStream.FlushAsync(cancellationToken); + } + + compressedStream.Position = 0; + + BlobUploadOptions uploadOptions = new() + { + HttpHeaders = new BlobHttpHeaders + { + ContentType = "application/jsonl+gzip", + ContentEncoding = "gzip", + }, + Metadata = new Dictionary + { + { "instanceId", instanceId }, + }, + }; + + await blobClient.UploadAsync(compressedStream, uploadOptions, cancellationToken); + } + else + { + // Upload uncompressed + BlobUploadOptions uploadOptions = new() + { + HttpHeaders = new BlobHttpHeaders + { + ContentType = "application/json", + }, + Metadata = new Dictionary + { + { "instanceId", instanceId }, + }, + }; + + await blobClient.UploadAsync( + new BinaryData(contentBytes), + uploadOptions, + cancellationToken); + } + } +} + +/// +/// Export request for one orchestration instance. +/// +public sealed class ExportRequest +{ + /// + /// Gets or sets the instance ID to export. + /// + public string InstanceId { get; set; } = string.Empty; + + /// + /// Gets or sets the export destination configuration. + /// + public ExportDestination Destination { get; set; } = null!; + + /// + /// Gets or sets the export format configuration. + /// + public ExportFormat Format { get; set; } = null!; +} + +/// +/// Export result. +/// +public sealed class ExportResult +{ + /// + /// Gets or sets the instance ID that was exported. + /// + public string InstanceId { get; set; } = string.Empty; + + /// + /// Gets or sets a value indicating whether gets or sets whether the export was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the error message if the export failed. + /// + public string? Error { get; set; } +} diff --git a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs new file mode 100644 index 000000000..379945710 --- /dev/null +++ b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Input for listing terminal instances activity. +/// +public sealed record ListTerminalInstancesRequest( + DateTimeOffset CompletedTimeFrom, + DateTimeOffset? CompletedTimeTo, + IEnumerable? RuntimeStatus, + string? LastInstanceKey, + int MaxInstancesPerBatch = 100); + +/// +/// Activity that lists terminal orchestration instances using the configured filters and checkpoint. +/// +/// +/// Initializes a new instance of the class. +/// +[DurableTask] +public class ListTerminalInstancesActivity( + DurableTaskClient client, + ILogger logger) : TaskActivity +{ + readonly DurableTaskClient client = Check.NotNull(client, nameof(client)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + + /// + public override async Task RunAsync(TaskActivityContext context, ListTerminalInstancesRequest input) + { + Check.NotNull(input, nameof(input)); + + try + { + // Try to use ListInstanceIds endpoint first (available in gRPC client) + Page page = await this.client.ListInstanceIdsAsync( + runtimeStatus: input.RuntimeStatus, + completedTimeFrom: input.CompletedTimeFrom, + completedTimeTo: input.CompletedTimeTo, + pageSize: input.MaxInstancesPerBatch, + lastInstanceKey: input.LastInstanceKey, + cancellation: CancellationToken.None); + + this.logger.LogInformation( + "ListTerminalInstancesActivity returned {Count} instance IDs using ListInstanceIds", + page.Values.Count); + + return new InstancePage(page.Values.ToList(), new ExportCheckpoint(page.ContinuationToken)); + } + catch (Exception ex) + { + this.logger.LogError(ex, "ListTerminalInstancesActivity failed"); + throw; + } + } +} + +/// +/// A page of instances for export. +/// +public sealed class InstancePage +{ + /// + /// Initializes a new instance of the class. + /// + /// The list of instance IDs. + /// The next checkpoint for pagination. + public InstancePage(List instanceIds, ExportCheckpoint nextCheckpoint) + { + this.InstanceIds = instanceIds; + this.NextCheckpoint = nextCheckpoint; + } + + /// + /// Gets or sets the list of instance IDs. + /// + public List InstanceIds { get; set; } = []; + + /// + /// Gets or sets the next checkpoint for pagination. + /// + public ExportCheckpoint NextCheckpoint { get; set; } +} diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs new file mode 100644 index 000000000..d48326b7c --- /dev/null +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public sealed class DefaultExportHistoryClient( + DurableTaskClient durableTaskClient, + ILogger logger, + ExportHistoryStorageOptions storageOptions) : ExportHistoryClient +{ + readonly DurableTaskClient durableTaskClient = Check.NotNull(durableTaskClient, nameof(durableTaskClient)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions, nameof(storageOptions)); + + /// + public override async Task CreateJobAsync( + ExportJobCreationOptions options, + CancellationToken cancellation = default) + { + Check.NotNull(options, nameof(options)); + this.logger.ClientCreatingExportJob(options); + + try + { + // Create export job client instance + ExportHistoryJobClient exportHistoryJobClient = this.GetJobClient(options.JobId); + + // Create the export job using the client (validation already done in constructor) + await exportHistoryJobClient.CreateAsync(options, cancellation); + + // Return the job client + return exportHistoryJobClient; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.CreateJobAsync), options.JobId, ex); + + throw; + } + } + + /// + public override async Task GetJobAsync(string jobId, CancellationToken cancellation = default) + { + Check.NotNullOrEmpty(jobId, nameof(jobId)); + + try + { + // Get export history job client first + ExportHistoryJobClient exportHistoryJobClient = this.GetJobClient(jobId); + + // Call DescribeAsync which handles all the entity state mapping + return await exportHistoryJobClient.DescribeAsync(cancellation); + } + catch (ExportJobNotFoundException) + { + // Re-throw as the job not being found is an error condition + throw; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.GetJobAsync), jobId, ex); + + throw; + } + } + + /// + public override AsyncPageable ListJobsAsync(ExportJobQuery? filter = null) + { + // TODO: revisit the fields + // Create an async pageable using the Pageable.Create helper + return Pageable.Create(async (continuationToken, pageSize, cancellation) => + { + try + { + EntityQuery query = new EntityQuery + { + InstanceIdStartsWith = $"@{nameof(ExportJob)}@{filter?.JobIdPrefix ?? string.Empty}", + IncludeState = true, + PageSize = filter?.PageSize ?? ExportJobQuery.DefaultPageSize, + ContinuationToken = continuationToken, + }; + + // Get one page of entities + IAsyncEnumerable>> entityPages = + this.durableTaskClient.Entities.GetAllEntitiesAsync(query).AsPages(); + + await foreach (Page> entityPage in entityPages) + { + List exportJobs = new(); + + foreach (EntityMetadata metadata in entityPage.Values) + { + if (filter != null && !MatchesFilter(metadata.State, filter)) + { + continue; + } + + ExportJobState state = metadata.State; + ExportJobConfiguration? config = state.Config; + + exportJobs.Add(new ExportJobDescription + { + JobId = metadata.Id.Key, + Status = state.Status, + CreatedAt = state.CreatedAt, + LastModifiedAt = state.LastModifiedAt, + Config = config, + OrchestratorInstanceId = state.OrchestratorInstanceId, + ScannedInstances = state.ScannedInstances, + ExportedInstances = state.ExportedInstances, + LastError = state.LastError, + Checkpoint = state.Checkpoint, + LastCheckpointTime = state.LastCheckpointTime, + }); + } + + return new Page(exportJobs, entityPage.ContinuationToken); + } + + // Return empty page if no results + return new Page(new List(), null); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.ListJobsAsync), string.Empty, ex); + + throw; + } + }); + } + + /// + /// Gets a job client for the specified job ID. + /// + /// The job ID. + /// The export history job client. + public override ExportHistoryJobClient GetJobClient(string jobId) + { + return new DefaultExportHistoryJobClient( + this.durableTaskClient, + jobId, + this.logger, + this.storageOptions); + } + + /// + /// Checks if an export job state matches the provided filter criteria. + /// + /// The export job state to check. + /// The filter criteria. + /// True if the state matches the filter; otherwise, false. + static bool MatchesFilter(ExportJobState state, ExportJobQuery filter) + { + bool statusMatches = !filter.Status.HasValue || state.Status == filter.Status.Value; + bool createdFromMatches = !filter.CreatedFrom.HasValue || + (state.CreatedAt.HasValue && state.CreatedAt.Value > filter.CreatedFrom.Value); + bool createdToMatches = !filter.CreatedTo.HasValue || + (state.CreatedAt.HasValue && state.CreatedAt.Value < filter.CreatedTo.Value); + + return statusMatches && createdFromMatches && createdToMatches; + } +} diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs new file mode 100644 index 000000000..acd22b0e5 --- /dev/null +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public sealed class DefaultExportHistoryJobClient( + DurableTaskClient durableTaskClient, + string jobId, + ILogger logger, + ExportHistoryStorageOptions storageOptions) : ExportHistoryJobClient(jobId) +{ + readonly DurableTaskClient durableTaskClient = Check.NotNull(durableTaskClient, nameof(durableTaskClient)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions, nameof(storageOptions)); + readonly EntityInstanceId entityId = new(nameof(ExportJob), jobId); + + /// + public override async Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default) + { + try + { + Check.NotNull(options, nameof(options)); + + // Determine default prefix based on mode if not already set + string? defaultPrefix = $"{options.Mode.ToString().ToLower(System.Globalization.CultureInfo.CurrentCulture)}-{this.JobId}/"; + + // If destination is not provided, construct it from storage options + string prefix = options.Destination?.Prefix ?? this.storageOptions.Prefix ?? defaultPrefix; + string container = options.Destination?.Container ?? this.storageOptions.ContainerName; + + ExportDestination destination = new ExportDestination(container) + { + Prefix = prefix, + }; + + ExportJobCreationOptions optionsWithDestination = options with { Destination = destination }; + + ExportJobOperationRequest request = + new ExportJobOperationRequest( + this.entityId, + nameof(ExportJob.Create), + optionsWithDestination); + + string instanceId = await this.durableTaskClient + .ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteExportJobOperationOrchestrator)), + request, + cancellation); + + // Wait for the orchestration to complete + OrchestrationMetadata state = await this.durableTaskClient + .WaitForInstanceCompletionAsync( + instanceId, + true, + cancellation); + + if (state.RuntimeStatus != OrchestrationRuntimeStatus.Completed) + { + throw new InvalidOperationException( + $"Failed to create export job '{this.JobId}': " + + $"{state.FailureDetails?.ErrorMessage ?? string.Empty}"); + } + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.CreateAsync), this.JobId, ex); + + throw; + } + } + + // TODO: there is no atomicity guarantee of deleting entity and purging the orchestrator + // Add sweeping process to clean up orphaned orchestrations failed to be purged + + /// + public override async Task DeleteAsync(CancellationToken cancellation = default) + { + try + { + this.logger.ClientDeletingExportJob(this.JobId); + + string orchestrationInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.JobId); + + // First, delete the entity + ExportJobOperationRequest request = new ExportJobOperationRequest(this.entityId, nameof(ExportJob.Delete)); + await this.durableTaskClient.ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteExportJobOperationOrchestrator)), + request, + cancellation); + + // Then terminate the linked export orchestration if it exists + await this.TerminateAndPurgeOrchestrationAsync(orchestrationInstanceId, cancellation); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.DeleteAsync), this.JobId, ex); + + throw; + } + } + + /// + public override async Task DescribeAsync(CancellationToken cancellation = default) + { + try + { + Check.NotNullOrEmpty(this.JobId, nameof(this.JobId)); + + EntityMetadata? metadata = + await this.durableTaskClient.Entities.GetEntityAsync(this.entityId, cancellation: cancellation); + if (metadata == null) + { + throw new ExportJobNotFoundException(this.JobId); + } + + ExportJobState state = metadata.State; + + ExportJobConfiguration? config = state.Config; + + return new ExportJobDescription + { + JobId = this.JobId, + Status = state.Status, + CreatedAt = state.CreatedAt, + LastModifiedAt = state.LastModifiedAt, + Config = config, + OrchestratorInstanceId = state.OrchestratorInstanceId, + ScannedInstances = state.ScannedInstances, + ExportedInstances = state.ExportedInstances, + LastError = state.LastError, + Checkpoint = state.Checkpoint, + LastCheckpointTime = state.LastCheckpointTime, + }; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.DescribeAsync), this.JobId, ex); + + throw; + } + } + + /// + /// Terminates and purges the export orchestration instance. + /// + /// The orchestration instance ID to terminate and purge. + /// The cancellation token. + async Task TerminateAndPurgeOrchestrationAsync(string orchestrationInstanceId, CancellationToken cancellation) + { + try + { + // Terminate the orchestration (will fail silently if it doesn't exist or already terminated) + await this.durableTaskClient.TerminateInstanceAsync( + orchestrationInstanceId, + new TerminateInstanceOptions { Output = "Export job deleted" }, + cancellation); + + // Wait for the orchestration to be terminated before purging + await this.durableTaskClient.WaitForInstanceCompletionAsync(orchestrationInstanceId, cancellation); + + // Purge the orchestration instance after it's terminated + await this.durableTaskClient.PurgeInstanceAsync(orchestrationInstanceId, cancellation: cancellation); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + { + // Orchestration instance doesn't exist - this is expected if it was already deleted or never existed + this.logger.LogInformation( + "Orchestration instance '{OrchestrationInstanceId}' is already purged or never existed", + orchestrationInstanceId); + } + catch (Exception ex) + { + this.logger.ClientError( + $"Failed to terminate or purge linked orchestration '{orchestrationInstanceId}': {ex.Message}", + this.JobId, + ex); + } + } +} diff --git a/src/ExportHistory/Client/ExportHistoryClient.cs b/src/ExportHistory/Client/ExportHistoryClient.cs new file mode 100644 index 000000000..c53b6f3dd --- /dev/null +++ b/src/ExportHistory/Client/ExportHistoryClient.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public abstract class ExportHistoryClient +{ + /// + /// Creates a new export job. + /// + /// The options for the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. + public abstract Task CreateJobAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + + /// + /// Gets an export job. + /// + /// The ID of the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. + public abstract Task GetJobAsync(string jobId, CancellationToken cancellation = default); + + /// + /// Lists export jobs. + /// + /// The filter for the export jobs. + /// A task representing the asynchronous operation. + public abstract AsyncPageable ListJobsAsync(ExportJobQuery? filter = null); + + /// + /// Gets an export job client. + /// + /// The ID of the export job. + /// The export job client. + public abstract ExportHistoryJobClient GetJobClient(string jobId); +} diff --git a/src/ExportHistory/Client/ExportHistoryJobClient.cs b/src/ExportHistory/Client/ExportHistoryJobClient.cs new file mode 100644 index 000000000..acd4aca45 --- /dev/null +++ b/src/ExportHistory/Client/ExportHistoryJobClient.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +/// +/// Initializes a new instance of the class. +/// +public abstract class ExportHistoryJobClient(string jobId) +{ + /// + /// The ID of the export job. + /// + protected readonly string JobId = Check.NotNullOrEmpty(jobId, nameof(jobId)); + + /// + /// Creates a new export job. + /// + /// The options for the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. + public abstract Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + + /// + /// Describes the export job. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public abstract Task DescribeAsync(CancellationToken cancellation = default); + + /// + /// Deletes the export job. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public abstract Task DeleteAsync(CancellationToken cancellation = default); +} diff --git a/src/ExportHistory/Constants/ExportHistoryConstants.cs b/src/ExportHistory/Constants/ExportHistoryConstants.cs new file mode 100644 index 000000000..964082db4 --- /dev/null +++ b/src/ExportHistory/Constants/ExportHistoryConstants.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Constants used throughout the export history functionality. +/// +static class ExportHistoryConstants +{ + /// + /// The prefix pattern used for generating export job orchestrator instance IDs. + /// Format: "ExportJob-{jobId}". + /// + public const string OrchestratorInstanceIdPrefix = "ExportJob-"; + + /// + /// Generates an orchestrator instance ID for a given export job ID. + /// + /// The export job ID. + /// The orchestrator instance ID. + public static string GetOrchestratorInstanceId(string jobId) => $"{OrchestratorInstanceIdPrefix}{jobId}"; +} diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs new file mode 100644 index 000000000..7ccce1ad1 --- /dev/null +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Durable entity that manages a history export job: lifecycle, configuration, and progress. +/// +/// The logger instance. +class ExportJob(ILogger logger) : TaskEntity +{ + /// + /// Creates a new export job from creation options. + /// + /// The entity context. + /// The export job creation options. + /// Thrown when creationOptions is null. + /// Thrown when invalid state transition is attempted or export job already exists. + public void Create(TaskEntityContext context, ExportJobCreationOptions creationOptions) + { + try + { + Check.NotNull(creationOptions, nameof(creationOptions)); + + if (!this.CanTransitionTo(nameof(this.Create), ExportJobStatus.Active)) + { + throw new ExportJobInvalidTransitionException( + creationOptions.JobId, + this.State.Status, + ExportJobStatus.Active, + nameof(this.Create)); + } + + // Convert ExportJobCreationOptions to ExportJobConfiguration + // Note: RuntimeStatus validation already done in ExportJobCreationOptions constructor + // Note: Destination should be populated by the client before reaching here + Verify.NotNull(creationOptions.Destination, nameof(creationOptions.Destination)); + + ExportJobConfiguration config = new ExportJobConfiguration( + Mode: creationOptions.Mode, + Filter: new ExportFilter( + CompletedTimeFrom: creationOptions.CompletedTimeFrom, + CompletedTimeTo: creationOptions.CompletedTimeTo, + RuntimeStatus: creationOptions.RuntimeStatus), + Destination: creationOptions.Destination, + Format: creationOptions.Format, + MaxInstancesPerBatch: creationOptions.MaxInstancesPerBatch); + + this.State.Config = config; + this.State.Status = ExportJobStatus.Active; + this.State.CreatedAt = this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + // Reset progress counters and checkpoint for clean restart + this.State.ScannedInstances = 0; + this.State.ExportedInstances = 0; + this.State.Checkpoint = null; + this.State.LastCheckpointTime = null; + + logger.CreatedExportJob(creationOptions.JobId); + + // Signal the Run method to start the export + context.SignalEntity( + context.Id, + nameof(this.Run)); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + creationOptions?.JobId ?? string.Empty, + nameof(this.Create), + "Failed to create export job", + ex); + throw; + } + } + + /// + /// Gets the current state of the export job. + /// + /// The entity context. + /// The current export job state. + public ExportJobState Get(TaskEntityContext context) + { + return this.State; + } + + /// + /// Runs the export job by starting the export orchestrator. + /// + /// The entity context. + /// Thrown when export job is not in Active status. + public void Run(TaskEntityContext context) + { + try + { + Verify.NotNull(this.State.Config, nameof(this.State.Config)); + + if (this.State.Status != ExportJobStatus.Active) + { + string errorMessage = "Export job must be in Active status to run."; + logger.ExportJobOperationError(context.Id.Key, nameof(this.Run), errorMessage, new InvalidOperationException(errorMessage)); + throw new InvalidOperationException(errorMessage); + } + + this.StartExportOrchestration(context); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.Run), + "Failed to run export job", + ex); + throw; + } + } + + /// + /// Commits a checkpoint snapshot with progress updates and optional failures. + /// + /// The entity context. + /// The checkpoint commit request containing progress, checkpoint, and failures. + public void CommitCheckpoint(TaskEntityContext context, CommitCheckpointRequest request) + { + Verify.NotNull(request, nameof(request)); + + // Update progress counts + this.State.ScannedInstances += request.ScannedInstances; + this.State.ExportedInstances += request.ExportedInstances; + + // Update checkpoint if provided (successful batch moves cursor forward) + // If null (failed batch), keep current checkpoint to not move cursor forward + if (request.Checkpoint is not null) + { + this.State.Checkpoint = request.Checkpoint; + } + + // Update checkpoint time and last modified time + this.State.LastCheckpointTime = this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + // If there are failures and checkpoint is null (batch failed), mark job as failed + if (request.Checkpoint is null && request.Failures != null && request.Failures.Count > 0) + { + this.State.Status = ExportJobStatus.Failed; + string failureSummary = string.Join("; ", request.Failures.Select(f => $"{f.InstanceId}: {f.Reason}")); + this.State.LastError = $"Batch export failed after retries. Failures: {failureSummary}"; + } + } + + /// + /// Marks the export job as completed. + /// + /// The entity context. + /// Thrown when invalid state transition is attempted. + public void MarkAsCompleted(TaskEntityContext context) + { + try + { + if (!this.CanTransitionTo(nameof(this.MarkAsCompleted), ExportJobStatus.Completed)) + { + throw new ExportJobInvalidTransitionException( + context.Id.Key, + this.State.Status, + ExportJobStatus.Completed, + nameof(this.MarkAsCompleted)); + } + + this.State.Status = ExportJobStatus.Completed; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.MarkAsCompleted), + "Export job marked as completed"); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.MarkAsCompleted), + "Failed to mark export job as completed", + ex); + throw; + } + } + + /// + /// Marks the export job as failed. + /// + /// The entity context. + /// The error message describing why the job failed. + /// Thrown when invalid state transition is attempted. + public void MarkAsFailed(TaskEntityContext context, string? errorMessage = null) + { + try + { + if (!this.CanTransitionTo(nameof(this.MarkAsFailed), ExportJobStatus.Failed)) + { + throw new ExportJobInvalidTransitionException( + context.Id.Key, + this.State.Status, + ExportJobStatus.Failed, + nameof(this.MarkAsFailed)); + } + + this.State.Status = ExportJobStatus.Failed; + this.State.LastError = errorMessage; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.MarkAsFailed), + $"Export job marked as failed: {errorMessage ?? "Unknown error"}"); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.MarkAsFailed), + "Failed to mark export job as failed", + ex); + throw; + } + } + + /// + /// Deletes the export job entity. + /// + /// The entity context. + public void Delete(TaskEntityContext context) + { + try + { + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.Delete), + "Deleting export job entity"); + + // Delete the entity by setting state to null + // This is the standard way to delete a durable entity + this.State = null!; + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.Delete), + "Failed to delete export job entity", + ex); + throw; + } + } + + void StartExportOrchestration(TaskEntityContext context) + { + try + { + // Use a fixed instance ID based on job ID to ensure only one orchestrator runs per job + // This prevents concurrent orchestrators if Run is called multiple times + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new StartOrchestrationOptions(instanceId); + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.StartExportOrchestration), + $"Starting new orchestration named '{nameof(ExportJobOrchestrator)}' with instance ID: {instanceId}"); + + context.ScheduleNewOrchestration( + new TaskName(nameof(ExportJobOrchestrator)), + new ExportJobRunRequest(context.Id), + startOrchestrationOptions); + + this.State.OrchestratorInstanceId = instanceId; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + catch (Exception ex) + { + // Mark job as failed and record the exception + this.State.Status = ExportJobStatus.Failed; + this.State.LastError = ex.Message; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.StartExportOrchestration), + "Failed to start export orchestration", + ex); + } + } + + bool CanTransitionTo(string operationName, ExportJobStatus targetStatus) + { + return ExportJobTransitions.IsValidTransition(operationName, this.State.Status, targetStatus); + } +} diff --git a/src/ExportHistory/Exception/ExportJobClientValidationException.cs b/src/ExportHistory/Exception/ExportJobClientValidationException.cs new file mode 100644 index 000000000..ec18f9bf6 --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobClientValidationException.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when client-side validation fails for export job operations. +/// +public class ExportJobClientValidationException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job that failed validation. + /// The validation error message. + /// The exception that is the cause of the current exception. + public ExportJobClientValidationException(string jobId, string message, Exception? innerException = null) + : base($"Validation failed for export job '{jobId}': {message}", innerException!) + { + this.JobId = jobId; + } + + /// + /// Gets the ID of the export job that failed validation. + /// + public string JobId { get; } +} diff --git a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs new file mode 100644 index 000000000..74eca6c2d --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when an invalid state transition is attempted on an export job. +/// +public class ExportJobInvalidTransitionException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job on which the invalid transition was attempted. + /// The current status of the export job. + /// The target status that was invalid. + /// The name of the operation that was attempted. + /// The exception that is the cause of the current exception. + public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName, Exception? innerException = null) + : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.", innerException!) + { + this.JobId = jobId; + this.FromStatus = fromStatus; + this.ToStatus = toStatus; + this.OperationName = operationName; + } + + /// + /// Gets the ID of the export job that encountered the invalid transition. + /// + public string JobId { get; } + + /// + /// Gets the status the export job was transitioning from. + /// + public ExportJobStatus FromStatus { get; } + + /// + /// Gets the invalid target status that was attempted. + /// + public ExportJobStatus ToStatus { get; } + + /// + /// Gets the name of the operation that was attempted. + /// + public string OperationName { get; } +} diff --git a/src/ExportHistory/Exception/ExportJobNotFoundException.cs b/src/ExportHistory/Exception/ExportJobNotFoundException.cs new file mode 100644 index 000000000..91e9b0a4d --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobNotFoundException.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when attempting to access a schedule that does not exist. +/// +public class ExportJobNotFoundException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export history job that was not found. + /// The exception that is the cause of the current exception. + public ExportJobNotFoundException(string jobId, Exception? innerException = null) + : base($"Export history job with ID '{jobId}' was not found.", innerException!) + { + this.JobId = jobId; + } + + /// + /// Gets the ID of the export history job that was not found. + /// + public string JobId { get; } +} diff --git a/src/ExportHistory/ExportHistory.csproj b/src/ExportHistory/ExportHistory.csproj new file mode 100644 index 000000000..f80fe4877 --- /dev/null +++ b/src/ExportHistory/ExportHistory.csproj @@ -0,0 +1,26 @@ + + + + net6.0 + Durable Task Export History + true + preview.1 + + + + + + + + + + + + + + + + + + + diff --git a/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs new file mode 100644 index 000000000..ed8285e66 --- /dev/null +++ b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Extension methods for configuring Durable Task clients to use export history. +/// +public static class DurableTaskClientBuilderExtensions +{ + /// + /// Enables export history support for the client builder with Azure Storage configuration. + /// + /// The client builder to add export history support to. + /// Callback to configure Azure Storage options. Must not be null. + /// The original builder, for call chaining. + public static IDurableTaskClientBuilder UseExportHistory( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder, nameof(builder)); + Check.NotNull(configure, nameof(configure)); + + IServiceCollection services = builder.Services; + + // Register and validate options + services.AddOptions() + .Configure(configure) + .Validate( + o => + !string.IsNullOrEmpty(o.ConnectionString) && + !string.IsNullOrEmpty(o.ContainerName), + $"{nameof(ExportHistoryStorageOptions)} must specify both {nameof(ExportHistoryStorageOptions.ConnectionString)} and {nameof(ExportHistoryStorageOptions.ContainerName)}."); + + // Register ExportHistoryClient using validated options + services.AddSingleton(sp => + { + DurableTaskClient durableTaskClient = sp.GetRequiredService(); + ILogger logger = sp.GetRequiredService>(); + ExportHistoryStorageOptions options = sp.GetRequiredService>().Value; + + return new DefaultExportHistoryClient(durableTaskClient, logger, options); + }); + + return builder; + } +} diff --git a/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs b/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs new file mode 100644 index 000000000..3e8a28967 --- /dev/null +++ b/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using Microsoft.DurableTask.Worker; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Extension methods for configuring Durable Task workers to use the Azure Durable Task Scheduler service. +/// +public static class DurableTaskWorkerBuilderExtensions +{ + /// + /// Adds export history support to the worker builder. + /// + /// The worker builder to add export history support to. + public static void UseExportHistory(this IDurableTaskWorkerBuilder builder) + { + builder.AddTasks(r => + { + r.AddEntity(); + r.AddOrchestrator(); + r.AddOrchestrator(); + r.AddActivity(); + r.AddActivity(); + }); + } +} diff --git a/src/ExportHistory/Logging/Logs.Client.cs b/src/ExportHistory/Logging/Logs.Client.cs new file mode 100644 index 000000000..f76c30264 --- /dev/null +++ b/src/ExportHistory/Logging/Logs.Client.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Log messages. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 80, Level = LogLevel.Information, Message = "Creating export job with options: {exportJobCreationOptions}")] + public static partial void ClientCreatingExportJob(this ILogger logger, ExportJobCreationOptions exportJobCreationOptions); + + [LoggerMessage(EventId = 84, Level = LogLevel.Information, Message = "Deleting export job '{jobId}'")] + public static partial void ClientDeletingExportJob(this ILogger logger, string jobId); + + [LoggerMessage(EventId = 87, Level = LogLevel.Error, Message = "{message} (JobId: {jobId})")] + public static partial void ClientError(this ILogger logger, string message, string jobId, Exception? exception = null); +} diff --git a/src/ExportHistory/Logging/Logs.Entity.cs b/src/ExportHistory/Logging/Logs.Entity.cs new file mode 100644 index 000000000..1dd0d8257 --- /dev/null +++ b/src/ExportHistory/Logging/Logs.Entity.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Log messages. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 101, Level = LogLevel.Information, Message = "Export job '{jobId}' is created")] + public static partial void CreatedExportJob(this ILogger logger, string jobId); + + [LoggerMessage(EventId = 113, Level = LogLevel.Information, Message = "Export job '{jobId}' operation '{operationName}' info: {infoMessage}")] + public static partial void ExportJobOperationInfo(this ILogger logger, string jobId, string operationName, string infoMessage); + + [LoggerMessage(EventId = 114, Level = LogLevel.Warning, Message = "Export job '{jobId}' operation '{operationName}' warning: {warningMessage}")] + public static partial void ExportJobOperationWarning(this ILogger logger, string jobId, string operationName, string warningMessage); + + [LoggerMessage(EventId = 115, Level = LogLevel.Error, Message = "Operation '{operationName}' failed for export job '{jobId}': {errorMessage}")] + public static partial void ExportJobOperationError(this ILogger logger, string jobId, string operationName, string errorMessage, Exception? exception = null); +} diff --git a/src/ExportHistory/Models/CommitCheckpointRequest.cs b/src/ExportHistory/Models/CommitCheckpointRequest.cs new file mode 100644 index 000000000..a66152411 --- /dev/null +++ b/src/ExportHistory/Models/CommitCheckpointRequest.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Request to commit a checkpoint with progress updates and failures. +/// +public sealed class CommitCheckpointRequest +{ + /// + /// Gets or sets the number of instances scanned in this batch. + /// + public long ScannedInstances { get; set; } + + /// + /// Gets or sets the number of instances successfully exported in this batch. + /// + public long ExportedInstances { get; set; } + + /// + /// Gets or sets the checkpoint to commit. If not null, the checkpoint is updated (cursor moves forward). + /// If null, the current checkpoint is kept (cursor does not move forward), allowing retry of the same batch. + /// + public ExportCheckpoint? Checkpoint { get; set; } + + /// + /// Gets or sets the list of failed instance exports, if any. + /// + public List? Failures { get; set; } +} diff --git a/src/ExportHistory/Models/ExportCheckpoint.cs b/src/ExportHistory/Models/ExportCheckpoint.cs new file mode 100644 index 000000000..561dad6ea --- /dev/null +++ b/src/ExportHistory/Models/ExportCheckpoint.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Checkpoint information used to resume export. +/// +public sealed record ExportCheckpoint(string? LastInstanceKey = null); diff --git a/src/ExportHistory/Models/ExportDestination.cs b/src/ExportHistory/Models/ExportDestination.cs new file mode 100644 index 000000000..bf6a62438 --- /dev/null +++ b/src/ExportHistory/Models/ExportDestination.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export destination settings for Azure Blob Storage. +/// +public sealed class ExportDestination +{ + /// + /// Initializes a new instance of the class. + /// + public ExportDestination() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The blob container name. + /// Thrown when container is null or empty. + public ExportDestination(string container) + { + Check.NotNullOrEmpty(container, nameof(container)); + this.Container = container; + } + + /// + /// Gets or sets the blob container name. + /// + public string Container { get; set; } + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } +} diff --git a/src/ExportHistory/Models/ExportFailure.cs b/src/ExportHistory/Models/ExportFailure.cs new file mode 100644 index 000000000..422ded650 --- /dev/null +++ b/src/ExportHistory/Models/ExportFailure.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Failure of a specific instance export. +/// +/// The instance ID that failed to export. +/// The reason for the failure. +/// The number of attempts made. +/// The timestamp of the last attempt. +public sealed record ExportFailure(string InstanceId, string Reason, int AttemptCount, DateTimeOffset LastAttempt); diff --git a/src/ExportHistory/Models/ExportFilter.cs b/src/ExportHistory/Models/ExportFilter.cs new file mode 100644 index 000000000..2e572edaa --- /dev/null +++ b/src/ExportHistory/Models/ExportFilter.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Filter criteria for selecting orchestration instances to export. +/// +/// The start time for the export based on completion time (inclusive). +/// The end time for the export based on completion time (inclusive). Optional. +/// The orchestration runtime statuses to filter by. Optional. +public record ExportFilter( + DateTimeOffset CompletedTimeFrom, + DateTimeOffset? CompletedTimeTo = null, + IEnumerable? RuntimeStatus = null); diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs new file mode 100644 index 000000000..56efea4bc --- /dev/null +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// The kind of export format. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ExportFormatKind +{ + /// + /// JSONL format (one history event per line, compressed with gzip). + /// + Jsonl, + + /// + /// JSON format (array of history events, uncompressed). + /// + Json, +} + +/// +/// Export format settings. +/// +/// The kind of export format. +/// The schema version. +public record ExportFormat( + ExportFormatKind Kind = ExportFormatKind.Jsonl, + string SchemaVersion = "1.0") +{ + /// + /// Gets the default export format (jsonl with schema version 1.0). + /// + public static ExportFormat Default => new(); +} diff --git a/src/ExportHistory/Models/ExportJobConfiguration.cs b/src/ExportHistory/Models/ExportJobConfiguration.cs new file mode 100644 index 000000000..1de6e51a5 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobConfiguration.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Configuration for an export job. +/// +/// The export mode (Batch or Continuous). +/// The filter criteria for selecting orchestration instances to export. +/// The export destination where exported data will be stored. +/// The export format settings. +/// The maximum number of parallel export operations. Defaults to 32. +/// The maximum number of instances to fetch per batch. Defaults to 100. +public record ExportJobConfiguration( + ExportMode Mode, + ExportFilter Filter, + ExportDestination Destination, + ExportFormat Format, + int MaxParallelExports = 32, + int MaxInstancesPerBatch = 100); diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs new file mode 100644 index 000000000..cc586c9e1 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Configuration for an export job. +/// +public record ExportJobCreationOptions +{ + /// + /// Initializes a new instance of the class. + /// + public ExportJobCreationOptions() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The export mode (Batch or Continuous). + /// The start time for the export based on completion time (inclusive). Required for Batch mode. For Continuous mode, this will be set to UtcNow if not provided. + /// The end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. + /// The export destination where exported data will be stored. Required unless default storage is configured. + /// The unique identifier for the export job. If not provided, a GUID will be generated. + /// The export format settings. Optional, defaults to jsonl-gzip. + /// The orchestration runtime statuses to filter by. Optional. + /// The maximum number of instances to fetch per batch. Optional, defaults to 100. + /// Thrown when validation fails. + public ExportJobCreationOptions( + ExportMode mode, + DateTimeOffset? completedTimeFrom, + DateTimeOffset? completedTimeTo, + ExportDestination? destination, + string? jobId = null, + ExportFormat? format = null, + List? runtimeStatus = null, + int? maxInstancesPerBatch = null) + { + // Generate GUID if jobId not provided + this.JobId = string.IsNullOrEmpty(jobId) ? Guid.NewGuid().ToString("N") : jobId; + + if (mode == ExportMode.Batch) + { + if (!completedTimeFrom.HasValue) + { + throw new ArgumentException( + "CompletedTimeFrom is required for Batch export mode.", + nameof(completedTimeFrom)); + } + + if (!completedTimeTo.HasValue) + { + throw new ArgumentException( + "CompletedTimeTo is required for Batch export mode.", + nameof(completedTimeTo)); + } + + if (completedTimeTo.HasValue && completedTimeTo.Value <= completedTimeFrom) + { + throw new ArgumentException( + $"CompletedTimeTo({completedTimeTo.Value}) must be greater than CompletedTimeFrom({completedTimeFrom}) for Batch export mode.", + nameof(completedTimeTo)); + } + + if (completedTimeTo.HasValue && completedTimeTo.Value > DateTimeOffset.UtcNow) + { + throw new ArgumentException( + $"CompletedTimeTo({completedTimeTo.Value}) cannot be in the future. It must be less than or equal to the current time ({DateTimeOffset.UtcNow}).", + nameof(completedTimeTo)); + } + } + else if (mode == ExportMode.Continuous) + { + if (completedTimeTo.HasValue) + { + throw new ArgumentException( + "CompletedTimeTo is not allowed for Continuous export mode.", + nameof(completedTimeTo)); + } + } + else + { + throw new ArgumentException( + "Invalid export mode.", + nameof(mode)); + } + + // Validate maxInstancesPerBatch range if provided (must be 1..999) + if (maxInstancesPerBatch.HasValue && (maxInstancesPerBatch.Value <= 0 || maxInstancesPerBatch.Value >= 1001)) + { + throw new ArgumentOutOfRangeException( + nameof(maxInstancesPerBatch), + maxInstancesPerBatch, + "MaxInstancesPerBatch must be between 1 and 1000."); + } + + // Validate terminal status-only filter here if provided + if (runtimeStatus is { Count: > 0 } + && runtimeStatus.Any( + s => s is not (OrchestrationRuntimeStatus.Completed + or OrchestrationRuntimeStatus.Failed + or OrchestrationRuntimeStatus.Terminated))) + { + throw new ArgumentException( + "Export supports terminal orchestration statuses only. Valid statuses are: Completed, Failed, and Terminated.", + nameof(runtimeStatus)); + } + + this.Mode = mode; + this.CompletedTimeFrom = completedTimeFrom ?? DateTimeOffset.UtcNow; + this.CompletedTimeTo = completedTimeTo; + this.Destination = destination; + this.Format = format ?? ExportFormat.Default; + this.RuntimeStatus = (runtimeStatus is { Count: > 0 }) + ? runtimeStatus + : new List + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated, + }; + this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; + } + + /// + /// Gets the unique identifier for the export job. + /// + public string JobId { get; init; } + + /// + /// Gets the export mode (Batch or Continuous). + /// + public ExportMode Mode { get; init; } + + /// + /// Gets the start time for the export based on completion time (inclusive). + /// Required for Batch mode. For Continuous mode, this is automatically set to UtcNow when creating the job. + /// + public DateTimeOffset CompletedTimeFrom { get; init; } + + /// + /// Gets the end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. + /// + public DateTimeOffset? CompletedTimeTo { get; init; } + + /// + /// Gets the export destination where exported data will be stored. Optional. + /// + public ExportDestination? Destination { get; init; } + + /// + /// Gets the export format settings. + /// + public ExportFormat Format { get; init; } + + /// + /// Gets the orchestration runtime statuses to filter by. + /// If not specified, all terminal statuses are exported. + /// + public List? RuntimeStatus { get; init; } + + /// + /// Gets the maximum number of instances to fetch per batch. + /// Defaults to 100. + /// + public int MaxInstancesPerBatch { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobDescription.cs b/src/ExportHistory/Models/ExportJobDescription.cs new file mode 100644 index 000000000..189b54f6f --- /dev/null +++ b/src/ExportHistory/Models/ExportJobDescription.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents the comprehensive details of an export job. +/// +public record ExportJobDescription +{ + /// + /// Gets the job identifier. + /// + public string JobId { get; init; } = string.Empty; + + /// + /// Gets the export job status. + /// + public ExportJobStatus Status { get; init; } + + /// + /// Gets the time when this export job was created. + /// + public DateTimeOffset? CreatedAt { get; init; } + + /// + /// Gets the time when this export job was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; init; } + + /// + /// Gets the export job configuration. + /// + public ExportJobConfiguration? Config { get; init; } + + /// + /// Gets the instance ID of the running export orchestrator, if any. + /// + public string? OrchestratorInstanceId { get; init; } + + /// + /// Gets the total number of instances scanned. + /// + public long ScannedInstances { get; init; } + + /// + /// Gets the total number of instances exported. + /// + public long ExportedInstances { get; init; } + + /// + /// Gets the last error message, if any. + /// + public string? LastError { get; init; } + + /// + /// Gets the checkpoint for resuming the export. + /// + public ExportCheckpoint? Checkpoint { get; init; } + + /// + /// Gets the time of the last checkpoint. + /// + public DateTimeOffset? LastCheckpointTime { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobQuery.cs b/src/ExportHistory/Models/ExportJobQuery.cs new file mode 100644 index 000000000..8472ce3f1 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobQuery.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents query parameters for filtering export history jobs. +/// +public record ExportJobQuery +{ + /// + /// The default page size when not supplied. + /// + public const int DefaultPageSize = 100; + + /// + /// Gets the filter for the export history job status. + /// + public ExportJobStatus? Status { get; init; } + + /// + /// Gets the prefix to filter export history job IDs. + /// + public string? JobIdPrefix { get; init; } + + /// + /// Gets the filter for export history jobs created after this time. + /// + public DateTimeOffset? CreatedFrom { get; init; } + + /// + /// Gets the filter for export history jobs created before this time. + /// + public DateTimeOffset? CreatedTo { get; init; } + + /// + /// Gets the maximum number of export history jobs to return per page. + /// + public int? PageSize { get; init; } + + /// + /// Gets the continuation token for retrieving the next page of results. + /// + public string? ContinuationToken { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobState.cs b/src/ExportHistory/Models/ExportJobState.cs new file mode 100644 index 000000000..77a289478 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobState.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export job state stored in the entity. +/// +public sealed class ExportJobState +{ + /// + /// Gets or sets the current status of the export job. + /// + public ExportJobStatus Status { get; set; } + + /// + /// Gets or sets the export job configuration. + /// + public ExportJobConfiguration? Config { get; set; } + + /// + /// Gets or sets the checkpoint for resuming the export. + /// + public ExportCheckpoint? Checkpoint { get; set; } + + /// + /// Gets or sets the time when the export job was created. + /// + public DateTimeOffset? CreatedAt { get; set; } + + /// + /// Gets or sets the time when the export job was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; set; } + + /// + /// Gets or sets the time of the last checkpoint. + /// + public DateTimeOffset? LastCheckpointTime { get; set; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; set; } + + /// + /// Gets or sets the total number of instances scanned. + /// + public long ScannedInstances { get; set; } + + /// + /// Gets or sets the total number of instances exported. + /// + public long ExportedInstances { get; set; } + + /// + /// Gets or sets the instance ID of the orchestrator running this export job, if any. + /// + public string? OrchestratorInstanceId { get; set; } +} diff --git a/src/ExportHistory/Models/ExportJobStatus.cs b/src/ExportHistory/Models/ExportJobStatus.cs new file mode 100644 index 000000000..dbd5b7c27 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobStatus.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents the current status of an export history job. +/// +public enum ExportJobStatus +{ + /// + /// Export history job has not been created. + /// + Uninitialized, + + /// + /// Export history job is active and running. + /// + Active, + + /// + /// Export history job failed. + /// + Failed, + + /// + /// Export history job completed. + /// + Completed, +} diff --git a/src/ExportHistory/Models/ExportJobTransitions.cs b/src/ExportHistory/Models/ExportJobTransitions.cs new file mode 100644 index 000000000..3fbc43020 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobTransitions.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Manages valid state transitions for export jobs. +/// +static class ExportJobTransitions +{ + /// + /// Checks if a transition to the target state is valid for a given export job state and operation. + /// + /// The name of the operation being performed. + /// The current export job state. + /// The target state to transition to. + /// True if the transition is valid; otherwise, false. + public static bool IsValidTransition(string operationName, ExportJobStatus from, ExportJobStatus targetState) + { + return operationName switch + { + nameof(ExportJob.Create) => from switch + { + ExportJobStatus.Uninitialized when targetState == ExportJobStatus.Active => true, + ExportJobStatus.Failed when targetState == ExportJobStatus.Active => true, + ExportJobStatus.Completed when targetState == ExportJobStatus.Active => true, + _ => false, + }, + nameof(ExportJob.MarkAsCompleted) => from switch + { + ExportJobStatus.Active when targetState == ExportJobStatus.Completed => true, + _ => false, + }, + nameof(ExportJob.MarkAsFailed) => from switch + { + ExportJobStatus.Active when targetState == ExportJobStatus.Failed => true, + _ => false, + }, + _ => false, + }; + } +} diff --git a/src/ExportHistory/Models/ExportMode.cs b/src/ExportHistory/Models/ExportMode.cs new file mode 100644 index 000000000..07b7cfca9 --- /dev/null +++ b/src/ExportHistory/Models/ExportMode.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export job modes. +/// +public enum ExportMode +{ + /// Exports a fixed window and completes. + Batch = 1, + + /// Tails terminal instances continuously. + Continuous = 2, +} diff --git a/src/ExportHistory/Options/ExportHistoryStorageOptions.cs b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs new file mode 100644 index 000000000..ad9a590c8 --- /dev/null +++ b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Options for Azure Storage configuration for export history jobs. +/// Supports connection string-based authentication. +/// +public sealed class ExportHistoryStorageOptions +{ + /// + /// Gets or sets the Azure Storage connection string to the customer's storage account. + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Gets or sets the blob container name where export data will be stored. + /// + public string ContainerName { get; set; } = string.Empty; + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } +} diff --git a/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs new file mode 100644 index 000000000..747169a7a --- /dev/null +++ b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Orchestrator that executes operations on export job entities. +/// Calls the specified operation on the target entity and returns the result. +/// +[DurableTask] +public class ExecuteExportJobOperationOrchestrator : TaskOrchestrator +{ + /// + public override async Task RunAsync(TaskOrchestrationContext context, ExportJobOperationRequest input) + { + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + } +} + +/// +/// Request for executing an export job operation. +/// +/// The ID of the entity to execute the operation on. +/// The name of the operation to execute. +/// Optional input for the operation. +public record ExportJobOperationRequest(EntityInstanceId EntityId, string OperationName, object? Input = null); diff --git a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs new file mode 100644 index 000000000..7112c79eb --- /dev/null +++ b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs @@ -0,0 +1,376 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Orchestrator input to start a runner for a given job. +/// +public sealed record ExportJobRunRequest(EntityInstanceId JobEntityId, int ProcessedCycles = 0); + +/// +/// Orchestrator that performs the actual export work by querying orchestration instances +/// and exporting their history to blob storage. +/// +[DurableTask] +public class ExportJobOrchestrator : TaskOrchestrator +{ + const int MaxRetryAttempts = 3; + const int MinBackoffSeconds = 60; // 1 minute + const int MaxBackoffSeconds = 300; // 5 minutes + const int ContinueAsNewFrequency = 5; + static readonly TimeSpan ContinuousExportIdleDelay = TimeSpan.FromMinutes(1); + + // Retry policy for individual export activities: 3 attempts with exponential backoff + // First retry after 15s, second retry after 30s (capped at 60s) + static readonly RetryPolicy ExportActivityRetryPolicy = new( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromSeconds(15), + backoffCoefficient: 2.0, + maxRetryInterval: TimeSpan.FromSeconds(60)); + + /// + public override async Task RunAsync(TaskOrchestrationContext context, ExportJobRunRequest input) + { + ILogger logger = context.CreateReplaySafeLogger(); + string jobId = input.JobEntityId.Key; + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator started"); + + try + { + // Get the export job state and configuration from the entity + ExportJobState? jobState = await context.Entities.CallEntityAsync( + input.JobEntityId, + nameof(ExportJob.Get), + null); + + if (jobState == null || jobState.Config == null) + { + throw new InvalidOperationException($"Export job '{jobId}' not found or has no configuration."); + } + + // Check if job is still active + if (jobState.Status != ExportJobStatus.Active) + { + logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), $"Job status is {jobState.Status}, not Active - orchestrator cancelled"); + return null; + } + + ExportJobConfiguration config = jobState.Config; + + int processedCycles = input.ProcessedCycles; + + while (true) + { + processedCycles++; + if (processedCycles > ContinueAsNewFrequency) + { + context.ContinueAsNew(new ExportJobRunRequest(input.JobEntityId, ProcessedCycles: 0)); + return null!; + } + + // Check if job is still active (entity might have been deleted or failed) + ExportJobState? currentState = await context.Entities.CallEntityAsync( + input.JobEntityId, + nameof(ExportJob.Get), + null); + + if (currentState == null || + currentState.Config == null || + currentState.Status != ExportJobStatus.Active) + { + logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), "Job is no longer active"); + return null; + } + + // Call activity to list terminal instances with only necessary information + ListTerminalInstancesRequest listRequest = new ListTerminalInstancesRequest( + CompletedTimeFrom: currentState.Config.Filter.CompletedTimeFrom, + CompletedTimeTo: currentState.Config.Filter.CompletedTimeTo, + RuntimeStatus: currentState.Config.Filter.RuntimeStatus, + LastInstanceKey: currentState.Checkpoint?.LastInstanceKey, + MaxInstancesPerBatch: currentState.Config.MaxInstancesPerBatch); + + InstancePage pageResult = await context.CallActivityAsync( + nameof(ListTerminalInstancesActivity), + listRequest); + + List instancesToExport = pageResult.InstanceIds; + long scannedCount = instancesToExport.Count; + + if (scannedCount == 0) + { + if (config.Mode == ExportMode.Continuous) + { + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export - export complete."); + await context.CreateTimer(ContinuousExportIdleDelay, default); + continue; + } + else if (config.Mode == ExportMode.Batch) + { + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export currently - will retry later."); + break; + } + else + { + throw new InvalidOperationException("Invalid export mode."); + } + } + + // Process batch with retry logic + BatchExportResult batchResult = await this.ProcessBatchWithRetryAsync( + context, + input.JobEntityId, + instancesToExport, + config); + + // Commit checkpoint based on batch result + if (batchResult.AllSucceeded) + { + // All exports succeeded - commit with checkpoint to move cursor forward + await this.CommitCheckpointAsync( + context, + input.JobEntityId, + scannedInstances: scannedCount, + exportedInstances: batchResult.ExportedCount, + checkpoint: pageResult.NextCheckpoint, + failures: null); + } + else + { + // Batch failed after all retries - commit without checkpoint (don't move cursor), record failures + await this.CommitCheckpointAsync( + context, + input.JobEntityId, + scannedInstances: 0, + exportedInstances: 0, + checkpoint: null, + failures: batchResult.Failures); + + // Throw detailed exception with failure information + string failureDetails; + if (batchResult.Failures != null && batchResult.Failures.Count > 0) + { + failureDetails = string.Join( + "; ", + batchResult.Failures + .Take(10) + .Select(f => + $"InstanceId: {f.InstanceId}, Reason: {f.Reason}")); + } + else + { + failureDetails = "No failure details available"; + } + + if (batchResult.Failures != null && batchResult.Failures.Count > 10) + { + failureDetails += $" ... and {batchResult.Failures.Count - 10} more failures"; + } + + throw new InvalidOperationException( + $"Export job '{jobId}' batch export failed after {MaxRetryAttempts} retry attempts. " + + $"Failure details: {failureDetails}"); + } + } + + await this.MarkAsCompletedAsync(context, input.JobEntityId); + + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator completed"); + return null!; + } + catch (Exception ex) + { + logger.ExportJobOperationError(jobId, nameof(ExportJobOrchestrator), "Export orchestrator failed", ex); + + await this.MarkAsFailedAsync(context, input.JobEntityId, ex.Message); + + throw; + } + } + + async Task ProcessBatchWithRetryAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + List instanceIds, + ExportJobConfiguration config) + { + ILogger logger = context.CreateReplaySafeLogger(); + string jobId = jobEntityId.Key; + + for (int attempt = 1; attempt <= MaxRetryAttempts; attempt++) + { + logger.ExportJobOperationInfo( + jobId, + nameof(this.ProcessBatchWithRetryAsync), + $"Processing batch of {instanceIds.Count} instances (attempt {attempt}/{MaxRetryAttempts})"); + + // Export all instances in the batch + List results = await this.ExportBatchAsync(context, instanceIds, config); + + // Check if all exports succeeded + List failedResults = results.Where(r => !r.Success).ToList(); + + if (failedResults.Count == 0) + { + // All exports succeeded + int exportedCount = results.Count; + logger.ExportJobOperationInfo( + jobId, + nameof(this.ProcessBatchWithRetryAsync), + $"Batch export succeeded on attempt {attempt} - exported {exportedCount} instances"); + + return new BatchExportResult + { + AllSucceeded = true, + ExportedCount = exportedCount, + Failures = null, + }; + } + + // Some exports failed + logger.ExportJobOperationWarning( + jobId, + nameof(this.ProcessBatchWithRetryAsync), + $"Batch export failed on attempt {attempt} - {failedResults.Count} failures out of {instanceIds.Count} instances"); + + // If this is the last attempt, return failures + if (attempt == MaxRetryAttempts) + { + List failures = failedResults.Select(r => new ExportFailure( + InstanceId: r.InstanceId, + Reason: r.Error ?? "Unknown error", + AttemptCount: attempt, + LastAttempt: DateTimeOffset.UtcNow)).ToList(); + + int exportedCount = results.Count(r => r.Success); + + return new BatchExportResult + { + AllSucceeded = false, + ExportedCount = exportedCount, + Failures = failures, + }; + } + + // Calculate exponential backoff: 1min, 2min, 4min (capped at 5min) + int backoffSeconds = Math.Min(MinBackoffSeconds * (int)Math.Pow(2, attempt - 1), MaxBackoffSeconds); + TimeSpan backoffDelay = TimeSpan.FromSeconds(backoffSeconds); + + logger.ExportJobOperationInfo( + jobId, + nameof(this.ProcessBatchWithRetryAsync), + $"Retrying batch export after {backoffDelay.TotalMinutes:F1} minutes (attempt {attempt + 1}/{MaxRetryAttempts})"); + + // Wait before retrying + await context.CreateTimer(backoffDelay, default); + } + + // Should not reach here, but return empty result if we do + return new BatchExportResult + { + AllSucceeded = true, + ExportedCount = 0, + Failures = new List(), + }; + } + + async Task> ExportBatchAsync( + TaskOrchestrationContext context, + List instanceIds, + ExportJobConfiguration config) + { + List results = new(); + List> exportTasks = new(); + + foreach (string instanceId in instanceIds) + { + // Create export request with destination and format + ExportRequest exportRequest = new ExportRequest + { + InstanceId = instanceId, + Destination = config.Destination, + Format = config.Format, + }; + + // Use retry policy for individual export activities (up to 3 attempts) + exportTasks.Add( + context.CallActivityAsync( + nameof(ExportInstanceHistoryActivity), + exportRequest, + new TaskOptions(ExportActivityRetryPolicy))); + + // Limit parallel export activities + if (exportTasks.Count >= config.MaxParallelExports) + { + ExportResult[] batchResults = await Task.WhenAll(exportTasks); + results.AddRange(batchResults); + exportTasks.Clear(); + } + } + + // Wait for remaining export activities + if (exportTasks.Count > 0) + { + ExportResult[] batchResults = await Task.WhenAll(exportTasks); + results.AddRange(batchResults); + } + + return results; + } + + async Task CommitCheckpointAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + long scannedInstances, + long exportedInstances, + ExportCheckpoint? checkpoint, + List? failures) + { + CommitCheckpointRequest request = new CommitCheckpointRequest + { + ScannedInstances = scannedInstances, + ExportedInstances = exportedInstances, + Checkpoint = checkpoint, + Failures = failures, + }; + + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.CommitCheckpoint), + request); + } + + async Task MarkAsCompletedAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId) + { + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.MarkAsCompleted), + null); + } + + async Task MarkAsFailedAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + string? errorMessage) + { + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.MarkAsFailed), + errorMessage); + } + + sealed class BatchExportResult + { + public bool AllSucceeded { get; set; } + + public int ExportedCount { get; set; } + + public List? Failures { get; set; } + } +} diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 1a86c0a27..9608e5180 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -475,6 +475,21 @@ message QueryInstancesResponse { google.protobuf.StringValue continuationToken = 2; } +message ListInstanceIdsRequest { + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp completedTimeFrom = 2; + google.protobuf.Timestamp completedTimeTo = 3; + int32 pageSize = 4; + google.protobuf.StringValue lastInstanceKey = 5; +} + +message ListInstanceIdsResponse { + repeated string instanceIds = 1; + google.protobuf.StringValue lastInstanceKey = 2; +} + +// Removed ListTerminalInstances in favor of using QueryInstances + message PurgeInstancesRequest { oneof request { string instanceId = 1; @@ -739,6 +754,7 @@ service TaskHubSidecarService { // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); + rpc ListInstanceIds(ListInstanceIdsRequest) returns (ListInstanceIdsResponse); rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); diff --git a/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs b/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs new file mode 100644 index 000000000..9fb27fa3a --- /dev/null +++ b/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Client; + +public class DefaultExportHistoryClientTests +{ + readonly Mock durableTaskClient; + readonly Mock entityClient; + readonly ILogger logger; + readonly ExportHistoryStorageOptions storageOptions; + readonly DefaultExportHistoryClient client; + + public DefaultExportHistoryClientTests() + { + this.durableTaskClient = new Mock("test"); + this.entityClient = new Mock("test"); + this.logger = new TestLogger(); + this.storageOptions = new ExportHistoryStorageOptions + { + ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net", + ContainerName = "test-container", + }; + this.durableTaskClient.Setup(x => x.Entities).Returns(this.entityClient.Object); + this.client = new DefaultExportHistoryClient( + this.durableTaskClient.Object, + this.logger, + this.storageOptions); + } + + [Fact] + public void Constructor_WithNullClient_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + null!, + this.logger, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("durableTaskClient"); + } + + [Fact] + public void Constructor_WithNullLogger_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + this.durableTaskClient.Object, + null!, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("logger"); + } + + [Fact] + public void Constructor_WithNullStorageOptions_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + this.durableTaskClient.Object, + this.logger, + null!); + + act.Should().Throw() + .WithParameterName("storageOptions"); + } + + [Fact] + public void GetJobClient_ReturnsValidClient() + { + // Arrange + string jobId = "test-job"; + + // Act + var jobClient = this.client.GetJobClient(jobId); + + // Assert + jobClient.Should().NotBeNull(); + jobClient.Should().BeOfType(); + } + + [Fact] + public async Task CreateJobAsync_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + // Act + var jobClient = await this.client.CreateJobAsync(options); + + // Assert + jobClient.Should().NotBeNull(); + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreateJobAsync_WithNullOptions_ThrowsArgumentNullException() + { + // Act & Assert + Func act = async () => await this.client.CreateJobAsync(null!); + await act.Should().ThrowAsync() + .WithParameterName("options"); + } + + [Fact] + public async Task GetJobAsync_WhenExists_ReturnsDescription() + { + // Arrange + string jobId = "test-job"; + var state = new ExportJobState + { + Status = ExportJobStatus.Active, + Config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default), + }; + + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync(new EntityMetadata(entityInstanceId, state)); + + // Act + var description = await this.client.GetJobAsync(jobId); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().Be(jobId); + description.Status.Should().Be(state.Status); + } + + [Fact] + public async Task GetJobAsync_WhenNotExists_ThrowsExportJobNotFoundException() + { + // Arrange + string jobId = "test-job"; + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync((EntityMetadata?)null); + + // Act & Assert + Func act = async () => await this.client.GetJobAsync(jobId); + await act.Should().ThrowAsync() + .Where(ex => ex.JobId == jobId); + } + + [Fact] + public async Task ListJobsAsync_WithNoFilter_ReturnsAllJobs() + { + // Arrange + var state1 = new ExportJobState + { + Status = ExportJobStatus.Active, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-1), + }; + + var state2 = new ExportJobState + { + Status = ExportJobStatus.Completed, + CreatedAt = DateTimeOffset.UtcNow, + }; + + var entity1 = new EntityInstanceId(nameof(ExportJob), "job-1"); + var entity2 = new EntityInstanceId(nameof(ExportJob), "job-2"); + + var metadata1 = new EntityMetadata(entity1, state1); + var metadata2 = new EntityMetadata(entity2, state2); + + var page = new Page>( + new List> { metadata1, metadata2 }, + null); + + this.entityClient + .Setup(c => c.GetAllEntitiesAsync( + It.IsAny())) + .Returns(Pageable.Create((string? continuation, int? pageSize, CancellationToken cancellation) => + Task.FromResult(page))); + + // Act + var jobs = await this.client.ListJobsAsync().ToListAsync(); + + // Assert + jobs.Should().HaveCount(2); + jobs.Should().Contain(j => j.JobId == "job-1"); + jobs.Should().Contain(j => j.JobId == "job-2"); + } + + [Fact] + public async Task ListJobsAsync_WithStatusFilter_FiltersCorrectly() + { + // Arrange + var state1 = new ExportJobState + { + Status = ExportJobStatus.Active, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-1), + }; + + var state2 = new ExportJobState + { + Status = ExportJobStatus.Completed, + CreatedAt = DateTimeOffset.UtcNow, + }; + + var entity1 = new EntityInstanceId(nameof(ExportJob), "job-1"); + var entity2 = new EntityInstanceId(nameof(ExportJob), "job-2"); + + var metadata1 = new EntityMetadata(entity1, state1); + var metadata2 = new EntityMetadata(entity2, state2); + + var page = new Page>( + new List> { metadata1, metadata2 }, + null); + + this.entityClient + .Setup(c => c.GetAllEntitiesAsync( + It.IsAny())) + .Returns(Pageable.Create((string? continuation, int? pageSize, CancellationToken cancellation) => + Task.FromResult(page))); + + var filter = new ExportJobQuery + { + Status = ExportJobStatus.Active, + }; + + // Act + var jobs = await this.client.ListJobsAsync(filter).ToListAsync(); + + // Assert + jobs.Should().HaveCount(1); + jobs.Should().Contain(j => j.JobId == "job-1" && j.Status == ExportJobStatus.Active); + } +} + diff --git a/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs new file mode 100644 index 000000000..fc4989677 --- /dev/null +++ b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Client; + +public class DefaultExportHistoryJobClientTests +{ + readonly Mock durableTaskClient; + readonly Mock entityClient; + readonly ILogger logger; + readonly ExportHistoryStorageOptions storageOptions; + readonly string jobId = "test-job-123"; + readonly DefaultExportHistoryJobClient client; + + public DefaultExportHistoryJobClientTests() + { + this.durableTaskClient = new Mock("test"); + this.entityClient = new Mock("test"); + this.logger = new TestLogger(); + this.storageOptions = new ExportHistoryStorageOptions + { + ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net", + ContainerName = "test-container", + }; + this.durableTaskClient.Setup(x => x.Entities).Returns(this.entityClient.Object); + this.client = new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + this.logger, + this.storageOptions); + } + + [Fact] + public void Constructor_WithNullClient_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + null!, + this.jobId, + this.logger, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("durableTaskClient"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void Constructor_WithInvalidJobId_ThrowsArgumentException(string? invalidJobId) + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + invalidJobId!, + this.logger, + this.storageOptions); + + act.Should().Throw(); + } + + [Fact] + public void Constructor_WithWhitespaceJobId_DoesNotThrow() + { + // Arrange + // Check.NotNullOrEmpty only checks for null, empty, or strings starting with '\0' + // It does NOT check for whitespace-only strings, so " " is valid + string testJobId = " "; + + // Act + var client = new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + testJobId, + this.logger, + this.storageOptions); + + // Assert + client.Should().NotBeNull(); + } + + [Fact] + public void Constructor_WithNullLogger_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + null!, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("logger"); + } + + [Fact] + public void Constructor_WithNullStorageOptions_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + this.logger, + null!); + + act.Should().Throw() + .WithParameterName("storageOptions"); + } + + [Fact] + public async Task DescribeAsync_WhenExists_ReturnsDescription() + { + // Arrange + var state = new ExportJobState + { + Status = ExportJobStatus.Active, + Config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default), + }; + + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), this.jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync(new EntityMetadata(entityInstanceId, state)); + + // Act + var description = await this.client.DescribeAsync(); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().Be(this.jobId); + description.Status.Should().Be(state.Status); + description.Config.Should().Be(state.Config); + } + + [Fact] + public async Task DescribeAsync_WhenNotExists_ThrowsExportJobNotFoundException() + { + // Arrange + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), this.jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync((EntityMetadata?)null); + + // Act & Assert + Func act = async () => await this.client.DescribeAsync(); + await act.Should().ThrowAsync() + .Where(ex => ex.JobId == this.jobId); + } + + [Fact] + public async Task CreateAsync_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + // Act + await this.client.CreateAsync(options); + + // Assert + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + + this.durableTaskClient.Verify( + c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreateAsync_WithNullOptions_ThrowsArgumentNullException() + { + // Act & Assert + Func act = async () => await this.client.CreateAsync(null!); + await act.Should().ThrowAsync() + .WithParameterName("options"); + } + + [Fact] + public async Task CreateAsync_WhenOrchestrationFails_ThrowsInvalidOperationException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + string errorMessage = "Test error message"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Failed, + FailureDetails = new TaskFailureDetails("TestError", errorMessage, null, null, null), + }); + + // Act & Assert + Func act = async () => await this.client.CreateAsync(options); + await act.Should().ThrowAsync() + .WithMessage($"*Failed to create export job '{this.jobId}'*"); + } + + [Fact] + public async Task DeleteAsync_ExecutesDeleteOperation() + { + // Arrange + string instanceId = "test-instance"; + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.jobId); + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + this.durableTaskClient + .Setup(c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(orchestratorInstanceId, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExportJobOrchestrator), orchestratorInstanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Terminated, + }); + + this.durableTaskClient + .Setup(c => c.PurgeInstanceAsync(orchestratorInstanceId, It.IsAny())) + .ReturnsAsync(new PurgeResult(1)); + + // Act + await this.client.DeleteAsync(); + + // Assert + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_WhenOrchestrationNotFound_HandlesGracefully() + { + // Arrange + string instanceId = "test-instance"; + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.jobId); + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + this.durableTaskClient + .Setup(c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny())) + .ThrowsAsync(new RpcException(new Status(StatusCode.NotFound, "Not found"))); + + // Act + await this.client.DeleteAsync(); + + // Assert - Should not throw + this.durableTaskClient.Verify( + c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny()), + Times.Once); + } +} + diff --git a/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs b/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs new file mode 100644 index 000000000..5630589ce --- /dev/null +++ b/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Constants; + +public class ExportHistoryConstantsTests +{ + [Fact] + public void OrchestratorInstanceIdPrefix_IsCorrect() + { + // Assert + ExportHistoryConstants.OrchestratorInstanceIdPrefix.Should().Be("ExportJob-"); + } + + [Fact] + public void GetOrchestratorInstanceId_WithJobId_ReturnsCorrectFormat() + { + // Arrange + string jobId = "test-job-123"; + + // Act + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(jobId); + + // Assert + instanceId.Should().Be("ExportJob-test-job-123"); + instanceId.Should().StartWith(ExportHistoryConstants.OrchestratorInstanceIdPrefix); + } + + [Theory] + [InlineData("job-1")] + [InlineData("very-long-job-id-with-special-characters")] + [InlineData("")] + public void GetOrchestratorInstanceId_WithVariousJobIds_ReturnsCorrectFormat(string jobId) + { + // Act + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(jobId); + + // Assert + instanceId.Should().Be($"{ExportHistoryConstants.OrchestratorInstanceIdPrefix}{jobId}"); + } +} + diff --git a/test/ExportHistory.Tests/Entity/ExportJobTests.cs b/test/ExportHistory.Tests/Entity/ExportJobTests.cs new file mode 100644 index 000000000..10a780443 --- /dev/null +++ b/test/ExportHistory.Tests/Entity/ExportJobTests.cs @@ -0,0 +1,370 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities.Tests; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Entity; + +public class ExportJobTests +{ + readonly ExportJob exportJob; + readonly TestLogger logger; + + public ExportJobTests() + { + this.logger = new TestLogger(); + this.exportJob = new ExportJob(this.logger); + } + + [Fact] + public async Task Create_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var operation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + + // Act + await this.exportJob.RunAsync(operation); + + // Assert + var state = operation.State.GetState(typeof(ExportJobState)); + state.Should().NotBeNull(); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Active); + jobState.Config.Should().NotBeNull(); + jobState.Config!.Mode.Should().Be(ExportMode.Batch); + jobState.CreatedAt.Should().NotBeNull(); + jobState.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Create_WithNullOptions_ThrowsInvalidOperationException() + { + // Arrange + var operation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + null); + + // Act & Assert + // When no input is provided to an entity operation, it throws InvalidOperationException + // because the entity operation system can't bind the parameter + var exception = await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(operation).AsTask()); + + exception.Message.Should().Contain("expected an input value"); + } + + [Fact] + public async Task Get_ReturnsCurrentState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var getOperation = new TestEntityOperation( + nameof(ExportJob.Get), + createOperation.State, + null); + + // Act + var result = await this.exportJob.RunAsync(getOperation); + + // Assert + result.Should().NotBeNull(); + var state = Assert.IsType(result); + state.Status.Should().Be(ExportJobStatus.Active); + } + + [Fact] + public async Task MarkAsCompleted_WhenActive_TransitionsToCompleted() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var completeOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsCompleted), + createOperation.State, + null); + + // Act + await this.exportJob.RunAsync(completeOperation); + + // Assert + var state = completeOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Completed); + jobState.LastError.Should().BeNull(); + } + + [Fact] + public async Task MarkAsCompleted_WhenNotActive_ThrowsInvalidTransitionException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + // Mark as failed first + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation.State, + "test error"); + await this.exportJob.RunAsync(failOperation); + + var completeOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsCompleted), + failOperation.State, + null); + + // Act & Assert + await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(completeOperation).AsTask()); + } + + [Fact] + public async Task MarkAsFailed_WhenActive_TransitionsToFailed() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + string errorMessage = "Test error"; + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation.State, + errorMessage); + + // Act + await this.exportJob.RunAsync(failOperation); + + // Assert + var state = failOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Failed); + jobState.LastError.Should().Be(errorMessage); + } + + [Fact] + public async Task CommitCheckpoint_WithValidRequest_UpdatesState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var checkpointRequest = new CommitCheckpointRequest + { + ScannedInstances = 100, + ExportedInstances = 95, + Checkpoint = new ExportCheckpoint("last-key"), + }; + + var checkpointOperation = new TestEntityOperation( + nameof(ExportJob.CommitCheckpoint), + createOperation.State, + checkpointRequest); + + // Act + await this.exportJob.RunAsync(checkpointOperation); + + // Assert + var state = checkpointOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.ScannedInstances.Should().Be(100); + jobState.ExportedInstances.Should().Be(95); + jobState.Checkpoint.Should().NotBeNull(); + jobState.Checkpoint!.LastInstanceKey.Should().Be("last-key"); + } + + [Fact] + public async Task CommitCheckpoint_WithFailures_MarksJobAsFailed() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var failures = new List + { + new("instance-1", "error1", 1, DateTimeOffset.UtcNow), + new("instance-2", "error2", 2, DateTimeOffset.UtcNow), + }; + + var checkpointRequest = new CommitCheckpointRequest + { + ScannedInstances = 0, + ExportedInstances = 0, + Checkpoint = null, // No checkpoint means batch failed + Failures = failures, + }; + + var checkpointOperation = new TestEntityOperation( + nameof(ExportJob.CommitCheckpoint), + createOperation.State, + checkpointRequest); + + // Act + await this.exportJob.RunAsync(checkpointOperation); + + // Assert + var state = checkpointOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Failed); + jobState.LastError.Should().NotBeNullOrEmpty(); + jobState.LastError.Should().Contain("Batch export failed"); + } + + [Fact] + public async Task Delete_ClearsState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var deleteOperation = new TestEntityOperation( + nameof(ExportJob.Delete), + createOperation.State, + null); + + // Act + await this.exportJob.RunAsync(deleteOperation); + + // Assert + var state = deleteOperation.State.GetState(typeof(ExportJobState)); + state.Should().BeNull(); + } + + [Fact] + public async Task Create_WhenAlreadyExists_ThrowsInvalidTransitionException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation1 = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation1); + + var createOperation2 = new TestEntityOperation( + nameof(ExportJob.Create), + createOperation1.State, + options); + + // Act & Assert + await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(createOperation2).AsTask()); + } + + [Fact] + public async Task Create_WhenFailed_CanRecreate() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation1 = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation1); + + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation1.State, + "test error"); + await this.exportJob.RunAsync(failOperation); + + var createOperation2 = new TestEntityOperation( + nameof(ExportJob.Create), + failOperation.State, + options); + + // Act + await this.exportJob.RunAsync(createOperation2); + + // Assert + var state = createOperation2.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Active); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs new file mode 100644 index 000000000..f71711a64 --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobClientValidationExceptionTests +{ + [Fact] + public void Constructor_WithJobIdAndMessage_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = "Validation failed: invalid parameters"; + + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(message); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithJobIdMessageAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = "Validation failed: invalid parameters"; + System.Exception innerException = new ArgumentException("Inner error"); + + // Act + var exception = new ExportJobClientValidationException(jobId, message, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(message); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData("", "message")] + [InlineData("job-123", "Validation failed")] + public void Constructor_WithVariousParameters_CreatesInstance(string jobId, string message) + { + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.JobId.Should().Be(jobId); + if (!string.IsNullOrEmpty(message)) + { + exception.Message.Should().Contain(message); + } + } + + [Fact] + public void Constructor_WithEmptyMessage_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = string.Empty; + + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.JobId.Should().Be(jobId); + exception.Message.Should().NotBeNullOrEmpty(); + exception.Message.Should().Contain(jobId); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs new file mode 100644 index 000000000..00e98ae1e --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobInvalidTransitionExceptionTests +{ + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + ExportJobStatus fromStatus = ExportJobStatus.Active; + ExportJobStatus toStatus = ExportJobStatus.Completed; + string operationName = "MarkAsCompleted"; + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + exception.OperationName.Should().Be(operationName); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(fromStatus.ToString()); + exception.Message.Should().Contain(toStatus.ToString()); + exception.Message.Should().Contain(operationName); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithAllParametersAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + ExportJobStatus fromStatus = ExportJobStatus.Failed; + ExportJobStatus toStatus = ExportJobStatus.Active; + string operationName = "Create"; + System.Exception innerException = new InvalidOperationException("Inner error"); + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + exception.OperationName.Should().Be(operationName); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Active)] + public void Constructor_WithVariousStatusTransitions_CreatesInstance( + ExportJobStatus fromStatus, + ExportJobStatus toStatus) + { + // Arrange + string jobId = "job-123"; + string operationName = "TestOperation"; + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName); + + // Assert + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs new file mode 100644 index 000000000..977aa1dc1 --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobNotFoundExceptionTests +{ + [Fact] + public void Constructor_WithJobId_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + + // Act + var exception = new ExportJobNotFoundException(jobId); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain("was not found"); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithJobIdAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + System.Exception innerException = new InvalidOperationException("Inner error"); + + // Act + var exception = new ExportJobNotFoundException(jobId, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain("was not found"); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData("")] + [InlineData("job-123")] + [InlineData("very-long-job-id-with-special-characters-12345")] + public void Constructor_WithVariousJobIds_CreatesInstance(string jobId) + { + // Act + var exception = new ExportJobNotFoundException(jobId); + + // Assert + exception.JobId.Should().Be(jobId); + } +} + diff --git a/test/ExportHistory.Tests/ExportHistory.Tests.csproj b/test/ExportHistory.Tests/ExportHistory.Tests.csproj new file mode 100644 index 000000000..ce8c7d6fa --- /dev/null +++ b/test/ExportHistory.Tests/ExportHistory.Tests.csproj @@ -0,0 +1,19 @@ + + + + + net6.0 + enable + enable + false + true + true + + + + + + + + + diff --git a/test/ExportHistory.Tests/ExportJobTransitionsTests.cs b/test/ExportHistory.Tests/ExportJobTransitionsTests.cs new file mode 100644 index 000000000..e080bad50 --- /dev/null +++ b/test/ExportHistory.Tests/ExportJobTransitionsTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests; + +public class ExportJobTransitionsTests +{ + [Theory] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Active, false)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed, false)] + public void IsValidTransition_Create_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("Create", from, to); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed, true)] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Completed, false)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Completed, false)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Completed, false)] + public void IsValidTransition_MarkAsCompleted_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("MarkAsCompleted", from, to); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed, true)] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Failed, false)] + public void IsValidTransition_MarkAsFailed_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("MarkAsFailed", from, to); + + // Assert + result.Should().Be(expected); + } + + [Fact] + public void IsValidTransition_UnknownOperation_ReturnsFalse() + { + // Arrange + ExportJobStatus from = ExportJobStatus.Active; + ExportJobStatus to = ExportJobStatus.Completed; + + // Act + bool result = ExportJobTransitions.IsValidTransition("UnknownOperation", from, to); + + // Assert + result.Should().BeFalse(); + } + + [Theory] + [InlineData("Create")] + [InlineData("MarkAsCompleted")] + [InlineData("MarkAsFailed")] + public void IsValidTransition_AllValidTransitions_ReturnsTrue(string operationName) + { + // Arrange + ExportJobStatus from = operationName switch + { + "Create" => ExportJobStatus.Uninitialized, + "MarkAsCompleted" => ExportJobStatus.Active, + "MarkAsFailed" => ExportJobStatus.Active, + _ => ExportJobStatus.Uninitialized, + }; + + ExportJobStatus to = operationName switch + { + "Create" => ExportJobStatus.Active, + "MarkAsCompleted" => ExportJobStatus.Completed, + "MarkAsFailed" => ExportJobStatus.Failed, + _ => ExportJobStatus.Active, + }; + + // Act + bool result = ExportJobTransitions.IsValidTransition(operationName, from, to); + + // Assert + result.Should().BeTrue(); + } +} + diff --git a/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs b/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs new file mode 100644 index 000000000..5f39ce393 --- /dev/null +++ b/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class CommitCheckpointRequestTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var request = new CommitCheckpointRequest(); + + // Assert + request.Should().NotBeNull(); + request.ScannedInstances.Should().Be(0); + request.ExportedInstances.Should().Be(0); + request.Checkpoint.Should().BeNull(); + request.Failures.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var request = new CommitCheckpointRequest(); + var checkpoint = new ExportCheckpoint("last-key"); + var failures = new List + { + new("instance-1", "error1", 1, DateTimeOffset.UtcNow), + new("instance-2", "error2", 2, DateTimeOffset.UtcNow), + }; + + // Act + request.ScannedInstances = 100; + request.ExportedInstances = 95; + request.Checkpoint = checkpoint; + request.Failures = failures; + + // Assert + request.ScannedInstances.Should().Be(100); + request.ExportedInstances.Should().Be(95); + request.Checkpoint.Should().Be(checkpoint); + request.Failures.Should().BeEquivalentTo(failures); + } + + [Fact] + public void Properties_CanBeSetToNull() + { + // Arrange + var request = new CommitCheckpointRequest + { + Checkpoint = new ExportCheckpoint("key"), + Failures = new List(), + }; + + // Act + request.Checkpoint = null; + request.Failures = null; + + // Assert + request.Checkpoint.Should().BeNull(); + request.Failures.Should().BeNull(); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs b/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs new file mode 100644 index 000000000..2fb12f566 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportCheckpointTests +{ + [Fact] + public void Constructor_WithNullLastInstanceKey_CreatesInstance() + { + // Act + var checkpoint = new ExportCheckpoint(null); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().BeNull(); + } + + [Fact] + public void Constructor_WithLastInstanceKey_CreatesInstance() + { + // Arrange + string lastInstanceKey = "instance-key-123"; + + // Act + var checkpoint = new ExportCheckpoint(lastInstanceKey); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().Be(lastInstanceKey); + } + + [Fact] + public void Constructor_WithDefaultParameter_CreatesInstance() + { + // Act + var checkpoint = new ExportCheckpoint(); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().BeNull(); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + string key = "test-key"; + var checkpoint1 = new ExportCheckpoint(key); + var checkpoint2 = new ExportCheckpoint(key); + + // Assert + checkpoint1.Should().Be(checkpoint2); + checkpoint1.GetHashCode().Should().Be(checkpoint2.GetHashCode()); + } + + [Fact] + public void Record_Inequality_Works() + { + // Arrange + var checkpoint1 = new ExportCheckpoint("key1"); + var checkpoint2 = new ExportCheckpoint("key2"); + + // Assert + checkpoint1.Should().NotBe(checkpoint2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportDestinationTests.cs b/test/ExportHistory.Tests/Models/ExportDestinationTests.cs new file mode 100644 index 000000000..9a66a0572 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportDestinationTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportDestinationTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var destination = new ExportDestination(); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().BeNull(); + destination.Prefix.Should().BeNull(); + } + + [Fact] + public void Constructor_WithContainer_CreatesInstance() + { + // Arrange + string container = "test-container"; + + // Act + var destination = new ExportDestination(container); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().Be(container); + destination.Prefix.Should().BeNull(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void Constructor_WithNullOrEmptyContainer_ThrowsArgumentException(string? container) + { + // Act + Action act = () => new ExportDestination(container!); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void Constructor_WithWhitespaceContainer_DoesNotThrow() + { + // Arrange + // Check.NotNullOrEmpty only checks for null, empty, or strings starting with '\0' + // It does NOT check for whitespace-only strings, so " " is valid + string container = " "; + + // Act + var destination = new ExportDestination(container); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().Be(container); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var destination = new ExportDestination("test-container"); + + // Act + destination.Prefix = "test-prefix/"; + + // Assert + destination.Container.Should().Be("test-container"); + destination.Prefix.Should().Be("test-prefix/"); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFailureTests.cs b/test/ExportHistory.Tests/Models/ExportFailureTests.cs new file mode 100644 index 000000000..a5403d367 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFailureTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFailureTests +{ + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + string instanceId = "instance-123"; + string reason = "Export failed"; + int attemptCount = 3; + DateTimeOffset lastAttempt = DateTimeOffset.UtcNow; + + // Act + var failure = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + + // Assert + failure.Should().NotBeNull(); + failure.InstanceId.Should().Be(instanceId); + failure.Reason.Should().Be(reason); + failure.AttemptCount.Should().Be(attemptCount); + failure.LastAttempt.Should().Be(lastAttempt); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + string instanceId = "instance-123"; + string reason = "Export failed"; + int attemptCount = 3; + DateTimeOffset lastAttempt = DateTimeOffset.UtcNow; + + var failure1 = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + var failure2 = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + + // Assert + failure1.Should().Be(failure2); + failure1.GetHashCode().Should().Be(failure2.GetHashCode()); + } + + [Fact] + public void Record_Inequality_Works() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + var failure1 = new ExportFailure("instance-1", "reason1", 1, now); + var failure2 = new ExportFailure("instance-2", "reason2", 2, now); + + // Assert + failure1.Should().NotBe(failure2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFilterTests.cs b/test/ExportHistory.Tests/Models/ExportFilterTests.cs new file mode 100644 index 000000000..74155c234 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFilterTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFilterTests +{ + [Fact] + public void Constructor_WithRequiredParameters_CreatesInstance() + { + // Arrange + DateTimeOffset completedTimeFrom = DateTimeOffset.UtcNow.AddDays(-1); + + // Act + var filter = new ExportFilter(completedTimeFrom); + + // Assert + filter.Should().NotBeNull(); + filter.CompletedTimeFrom.Should().Be(completedTimeFrom); + filter.CompletedTimeTo.Should().BeNull(); + filter.RuntimeStatus.Should().BeNull(); + } + + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + DateTimeOffset completedTimeFrom = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset completedTimeTo = DateTimeOffset.UtcNow; + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + }; + + // Act + var filter = new ExportFilter(completedTimeFrom, completedTimeTo, runtimeStatus); + + // Assert + filter.Should().NotBeNull(); + filter.CompletedTimeFrom.Should().Be(completedTimeFrom); + filter.CompletedTimeTo.Should().Be(completedTimeTo); + filter.RuntimeStatus.Should().BeEquivalentTo(runtimeStatus); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + var statuses = new List { OrchestrationRuntimeStatus.Completed }; + + var filter1 = new ExportFilter(from, to, statuses); + var filter2 = new ExportFilter(from, to, statuses); + + // Assert + filter1.Should().Be(filter2); + filter1.GetHashCode().Should().Be(filter2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFormatTests.cs b/test/ExportHistory.Tests/Models/ExportFormatTests.cs new file mode 100644 index 000000000..144bc3b7f --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFormatTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFormatTests +{ + [Fact] + public void Constructor_WithDefaultValues_CreatesInstance() + { + // Act + var format = new ExportFormat(); + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be(ExportFormatKind.Jsonl); + format.SchemaVersion.Should().Be("1.0"); + } + + [Fact] + public void Constructor_WithCustomValues_CreatesInstance() + { + // Arrange + ExportFormatKind kind = ExportFormatKind.Json; + string schemaVersion = "2.0"; + + // Act + var format = new ExportFormat(kind, schemaVersion); + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be(kind); + format.SchemaVersion.Should().Be(schemaVersion); + } + + [Fact] + public void Default_ReturnsDefaultInstance() + { + // Act + var format = ExportFormat.Default; + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be(ExportFormatKind.Jsonl); + format.SchemaVersion.Should().Be("1.0"); + } + + [Fact] + public void Default_IsImmutable() + { + // Arrange + var format1 = ExportFormat.Default; + var format2 = ExportFormat.Default; + + // Act & Assert + format1.Should().Be(format2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs b/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs new file mode 100644 index 000000000..2f8adafab --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobConfigurationTests +{ + [Fact] + public void Constructor_WithRequiredParameters_CreatesInstance() + { + // Arrange + ExportMode mode = ExportMode.Batch; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + + // Act + var config = new ExportJobConfiguration(mode, filter, destination, format); + + // Assert + config.Should().NotBeNull(); + config.Mode.Should().Be(mode); + config.Filter.Should().Be(filter); + config.Destination.Should().Be(destination); + config.Format.Should().Be(format); + config.MaxParallelExports.Should().Be(32); + config.MaxInstancesPerBatch.Should().Be(100); + } + + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + ExportMode mode = ExportMode.Continuous; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1)); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + int maxParallelExports = 64; + int maxInstancesPerBatch = 200; + + // Act + var config = new ExportJobConfiguration(mode, filter, destination, format, maxParallelExports, maxInstancesPerBatch); + + // Assert + config.Should().NotBeNull(); + config.Mode.Should().Be(mode); + config.Filter.Should().Be(filter); + config.Destination.Should().Be(destination); + config.Format.Should().Be(format); + config.MaxParallelExports.Should().Be(maxParallelExports); + config.MaxInstancesPerBatch.Should().Be(maxInstancesPerBatch); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + ExportMode mode = ExportMode.Batch; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + + var config1 = new ExportJobConfiguration(mode, filter, destination, format); + var config2 = new ExportJobConfiguration(mode, filter, destination, format); + + // Assert + config1.Should().Be(config2); + config1.GetHashCode().Should().Be(config2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs b/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs new file mode 100644 index 000000000..89955aa09 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobCreationOptionsTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var options = new ExportJobCreationOptions(); + + // Assert + options.Should().NotBeNull(); + } + + [Fact] + public void Constructor_BatchMode_WithValidParameters_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination); + + // Assert + options.Should().NotBeNull(); + options.Mode.Should().Be(ExportMode.Batch); + options.CompletedTimeFrom.Should().Be(from); + options.CompletedTimeTo.Should().Be(to); + options.Destination.Should().Be(destination); + options.JobId.Should().NotBeNullOrEmpty(); + options.Format.Should().Be(ExportFormat.Default); + options.RuntimeStatus.Should().NotBeNull(); + options.MaxInstancesPerBatch.Should().Be(100); + } + + [Fact] + public void Constructor_BatchMode_WithCustomJobId_CreatesInstance() + { + // Arrange + string jobId = "custom-job-id"; + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + jobId); + + // Assert + options.JobId.Should().Be(jobId); + } + + [Fact] + public void Constructor_BatchMode_WithDefaultJobId_GeneratesGuid() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null); + + // Assert + options.JobId.Should().NotBeNullOrEmpty(); + Guid.TryParse(options.JobId, out _).Should().BeTrue(); + } + + [Fact] + public void Constructor_BatchMode_WithEmptyJobId_GeneratesGuid() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + string.Empty); + + // Assert + options.JobId.Should().NotBeNullOrEmpty(); + Guid.TryParse(options.JobId, out _).Should().BeTrue(); + } + + [Fact] + public void Constructor_BatchMode_WithDefaultCompletedTimeFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = default; + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeFrom is required for Batch export mode*"); + } + + [Fact] + public void Constructor_BatchMode_WithNullCompletedTimeTo_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset? to = null; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo is required for Batch export mode*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToLessThanFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow; + DateTimeOffset to = DateTimeOffset.UtcNow.AddDays(-1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* must be greater than CompletedTimeFrom*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToEqualToFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = from; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* must be greater than CompletedTimeFrom*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToInFuture_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow.AddDays(1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* cannot be in the future*"); + } + + [Fact] + public void Constructor_ContinuousMode_WithValidParameters_CreatesInstance() + { + // Arrange + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Continuous, + default, + null, + destination); + + // Assert + options.Should().NotBeNull(); + options.Mode.Should().Be(ExportMode.Continuous); + options.CompletedTimeFrom.Should().Be(default); + options.CompletedTimeTo.Should().BeNull(); + options.Destination.Should().Be(destination); + } + + [Fact] + public void Constructor_ContinuousMode_WithCompletedTimeFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Continuous, from, null, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeFrom is not allowed for Continuous export mode*"); + } + + [Fact] + public void Constructor_ContinuousMode_WithCompletedTimeTo_ThrowsArgumentException() + { + // Arrange + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Continuous, default, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo is not allowed for Continuous export mode*"); + } + + [Fact] + public void Constructor_WithInvalidMode_ThrowsArgumentException() + { + // Arrange + ExportMode invalidMode = (ExportMode)999; + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(invalidMode, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*Invalid export mode*"); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1001)] + [InlineData(2000)] + public void Constructor_WithInvalidMaxInstancesPerBatch_ThrowsArgumentOutOfRangeException(int maxInstances) + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null, + maxInstances); + + // Assert + act.Should().Throw() + .WithMessage("*MaxInstancesPerBatch must be between 1 and 1000*"); + } + + [Fact] + public void Constructor_WithValidMaxInstancesPerBatch_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + int maxInstances = 500; + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null, + maxInstances); + + // Assert + options.MaxInstancesPerBatch.Should().Be(maxInstances); + } + + [Fact] + public void Constructor_WithNonTerminalRuntimeStatus_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Running, // Not terminal + }; + + // Act + Action act = () => new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + act.Should().Throw() + .WithMessage("*Export supports terminal orchestration statuses only*"); + } + + [Fact] + public void Constructor_WithTerminalRuntimeStatus_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated, + OrchestrationRuntimeStatus.ContinuedAsNew, + }; + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + options.RuntimeStatus.Should().BeEquivalentTo(runtimeStatus); + } + + [Fact] + public void Constructor_WithNullRuntimeStatus_UsesDefaultTerminalStatuses() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null); + + // Assert + options.RuntimeStatus.Should().NotBeNull(); + options.RuntimeStatus.Should().HaveCount(4); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Completed); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Failed); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Terminated); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.ContinuedAsNew); + } + + [Fact] + public void Constructor_WithEmptyRuntimeStatus_UsesDefaultTerminalStatuses() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new(); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + options.RuntimeStatus.Should().NotBeNull(); + options.RuntimeStatus.Should().HaveCount(4); + } + + [Fact] + public void Constructor_WithCustomFormat_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + ExportFormat format = new("json", "2.0"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + format); + + // Assert + options.Format.Should().Be(format); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs b/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs new file mode 100644 index 000000000..cbb1c9752 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobDescriptionTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var description = new ExportJobDescription(); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().BeEmpty(); + description.Status.Should().Be(ExportJobStatus.Uninitialized); + description.CreatedAt.Should().BeNull(); + description.LastModifiedAt.Should().BeNull(); + description.Config.Should().BeNull(); + description.OrchestratorInstanceId.Should().BeNull(); + description.ScannedInstances.Should().Be(0); + description.ExportedInstances.Should().Be(0); + description.LastError.Should().BeNull(); + description.Checkpoint.Should().BeNull(); + description.LastCheckpointTime.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var description = new ExportJobDescription(); + var config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default); + var checkpoint = new ExportCheckpoint("last-key"); + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + description = description with + { + JobId = "job-123", + Status = ExportJobStatus.Active, + CreatedAt = now, + LastModifiedAt = now, + Config = config, + OrchestratorInstanceId = "orchestrator-123", + ScannedInstances = 100, + ExportedInstances = 95, + LastError = "test error", + Checkpoint = checkpoint, + LastCheckpointTime = now, + }; + + // Assert + description.JobId.Should().Be("job-123"); + description.Status.Should().Be(ExportJobStatus.Active); + description.CreatedAt.Should().Be(now); + description.LastModifiedAt.Should().Be(now); + description.Config.Should().Be(config); + description.OrchestratorInstanceId.Should().Be("orchestrator-123"); + description.ScannedInstances.Should().Be(100); + description.ExportedInstances.Should().Be(95); + description.LastError.Should().Be("test error"); + description.Checkpoint.Should().Be(checkpoint); + description.LastCheckpointTime.Should().Be(now); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs b/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs new file mode 100644 index 000000000..adca278a2 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobQueryTests +{ + [Fact] + public void DefaultPageSize_IsCorrect() + { + // Assert + ExportJobQuery.DefaultPageSize.Should().Be(100); + } + + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var query = new ExportJobQuery(); + + // Assert + query.Should().NotBeNull(); + query.Status.Should().BeNull(); + query.JobIdPrefix.Should().BeNull(); + query.CreatedFrom.Should().BeNull(); + query.CreatedTo.Should().BeNull(); + query.PageSize.Should().BeNull(); + query.ContinuationToken.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + var query = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + ContinuationToken = "token-123", + }; + + // Assert + query.Status.Should().Be(ExportJobStatus.Active); + query.JobIdPrefix.Should().Be("test-"); + query.CreatedFrom.Should().Be(now.AddDays(-1)); + query.CreatedTo.Should().Be(now); + query.PageSize.Should().Be(50); + query.ContinuationToken.Should().Be("token-123"); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + var query1 = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + }; + + var query2 = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + }; + + // Assert + query1.Should().Be(query2); + query1.GetHashCode().Should().Be(query2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobStateTests.cs b/test/ExportHistory.Tests/Models/ExportJobStateTests.cs new file mode 100644 index 000000000..2048288d8 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobStateTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobStateTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var state = new ExportJobState(); + + // Assert + state.Should().NotBeNull(); + state.Status.Should().Be(ExportJobStatus.Uninitialized); + state.Config.Should().BeNull(); + state.Checkpoint.Should().BeNull(); + state.CreatedAt.Should().BeNull(); + state.LastModifiedAt.Should().BeNull(); + state.LastCheckpointTime.Should().BeNull(); + state.LastError.Should().BeNull(); + state.ScannedInstances.Should().Be(0); + state.ExportedInstances.Should().Be(0); + state.OrchestratorInstanceId.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var state = new ExportJobState(); + var config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default); + var checkpoint = new ExportCheckpoint("last-key"); + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + state.Status = ExportJobStatus.Active; + state.Config = config; + state.Checkpoint = checkpoint; + state.CreatedAt = now; + state.LastModifiedAt = now; + state.LastCheckpointTime = now; + state.LastError = "test error"; + state.ScannedInstances = 100; + state.ExportedInstances = 95; + state.OrchestratorInstanceId = "orchestrator-123"; + + // Assert + state.Status.Should().Be(ExportJobStatus.Active); + state.Config.Should().Be(config); + state.Checkpoint.Should().Be(checkpoint); + state.CreatedAt.Should().Be(now); + state.LastModifiedAt.Should().Be(now); + state.LastCheckpointTime.Should().Be(now); + state.LastError.Should().Be("test error"); + state.ScannedInstances.Should().Be(100); + state.ExportedInstances.Should().Be(95); + state.OrchestratorInstanceId.Should().Be("orchestrator-123"); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs b/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs new file mode 100644 index 000000000..7710b6a37 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobStatusTests +{ + [Fact] + public void ExportJobStatus_Values_AreDefined() + { + // Assert + Enum.GetValues().Should().HaveCount(4); + Enum.GetValues().Should().Contain(ExportJobStatus.Uninitialized); + Enum.GetValues().Should().Contain(ExportJobStatus.Active); + Enum.GetValues().Should().Contain(ExportJobStatus.Failed); + Enum.GetValues().Should().Contain(ExportJobStatus.Completed); + } + + [Theory] + [InlineData(ExportJobStatus.Uninitialized)] + [InlineData(ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Failed)] + [InlineData(ExportJobStatus.Completed)] + public void ExportJobStatus_CanBeAssigned(ExportJobStatus status) + { + // Arrange + ExportJobStatus assignedStatus = status; + + // Assert + assignedStatus.Should().Be(status); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportModeTests.cs b/test/ExportHistory.Tests/Models/ExportModeTests.cs new file mode 100644 index 000000000..1b83a678e --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportModeTests.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportModeTests +{ + [Fact] + public void ExportMode_Values_AreCorrect() + { + // Assert + ExportMode.Batch.Should().Be((ExportMode)1); + ExportMode.Continuous.Should().Be((ExportMode)2); + } + + [Theory] + [InlineData(ExportMode.Batch, 1)] + [InlineData(ExportMode.Continuous, 2)] + public void ExportMode_EnumValue_MatchesExpected(ExportMode mode, int expectedValue) + { + // Assert + ((int)mode).Should().Be(expectedValue); + } +} + diff --git a/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs b/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs new file mode 100644 index 000000000..ab9bb3bb6 --- /dev/null +++ b/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Options; + +public class ExportHistoryStorageOptionsTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var options = new ExportHistoryStorageOptions(); + + // Assert + options.Should().NotBeNull(); + options.ConnectionString.Should().BeEmpty(); + options.ContainerName.Should().BeEmpty(); + options.Prefix.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var options = new ExportHistoryStorageOptions(); + + // Act + options.ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net"; + options.ContainerName = "test-container"; + options.Prefix = "exports/"; + + // Assert + options.ConnectionString.Should().Be("DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net"); + options.ContainerName.Should().Be("test-container"); + options.Prefix.Should().Be("exports/"); + } + + [Fact] + public void Prefix_CanBeSetToNull() + { + // Arrange + var options = new ExportHistoryStorageOptions + { + Prefix = "test-prefix", + }; + + // Act + options.Prefix = null; + + // Assert + options.Prefix.Should().BeNull(); + } +} + diff --git a/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs new file mode 100644 index 000000000..61d8b836d --- /dev/null +++ b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Orchestrations; + +public class ExecuteExportJobOperationOrchestratorTests +{ + readonly Mock mockContext; + readonly Mock mockEntityClient; + readonly ExecuteExportJobOperationOrchestrator orchestrator; + + public ExecuteExportJobOperationOrchestratorTests() + { + this.mockContext = new Mock(MockBehavior.Strict); + this.mockEntityClient = new Mock(MockBehavior.Loose); + this.mockContext.Setup(c => c.Entities).Returns(this.mockEntityClient.Object); + this.orchestrator = new ExecuteExportJobOperationOrchestrator(); + } + + [Fact] + public async Task RunAsync_ValidRequest_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = "Create"; + var input = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + var expectedResult = new ExportJobState { Status = ExportJobStatus.Active }; + var request = new ExportJobOperationRequest(entityId, operationName, input); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, input, default)) + .ReturnsAsync(expectedResult); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().Be(expectedResult); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, input, default), + Times.Once); + } + + [Fact] + public async Task RunAsync_WithNullInput_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = "Get"; + var request = new ExportJobOperationRequest(entityId, operationName, null); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, null, default)) + .ReturnsAsync(new ExportJobState()); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().NotBeNull(); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, null, default), + Times.Once); + } + + [Fact] + public async Task RunAsync_WithDeleteOperation_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = nameof(ExportJob.Delete); + var request = new ExportJobOperationRequest(entityId, operationName, null); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, null, default)) + .ReturnsAsync(null!); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().BeNull(); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, null, default), + Times.Once); + } +} + diff --git a/test/ExportHistory.Tests/Usings.cs b/test/ExportHistory.Tests/Usings.cs new file mode 100644 index 000000000..e33163f94 --- /dev/null +++ b/test/ExportHistory.Tests/Usings.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +global using FluentAssertions; +global using Xunit; + diff --git a/test/TestHelpers/Logging/TestLogProvider.cs b/test/TestHelpers/Logging/TestLogProvider.cs index a4d1b46fd..426ad6fb1 100644 --- a/test/TestHelpers/Logging/TestLogProvider.cs +++ b/test/TestHelpers/Logging/TestLogProvider.cs @@ -18,7 +18,7 @@ public bool TryGetLogs(string category, out IReadOnlyCollection logs) // (e.g. "Microsoft.DurableTask.Worker" will return all logs for "Microsoft.DurableTask.Worker.*") logs = this.loggers .Where(kvp => kvp.Key.StartsWith(category, StringComparison.OrdinalIgnoreCase)) - .SelectMany(kvp => kvp.Value.GetLogs()) + .SelectMany(kvp => kvp.Value.GetLogs().ToList()) // Create a snapshot to avoid concurrent modification .OrderBy(log => log.Timestamp) .ToList(); return logs.Count > 0;