From 871eefeb73fe2f7720fc3fe81e6c783e2f36fbcf Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 29 Oct 2025 12:53:31 -0700 Subject: [PATCH 1/7] export job --- .../ExportHistoryWebApp.csproj | 23 ++ .../ExportHistoryWebApp.http | 88 +++++ .../ExportJobController.cs | 190 ++++++++++ .../Models/CreateExportJobRequest.cs | 60 ++++ samples/ExportHistoryWebApp/Program.cs | 55 +++ .../Properties/launchSettings.json | 26 ++ .../appsettings.Development.json | 9 + samples/ExportHistoryWebApp/appsettings.json | 10 + src/Client/Grpc/GrpcDurableTaskClient.cs | 50 +++ .../ExportInstanceHistoryActivity.cs | 291 +++++++++++++++ .../ListTerminalInstancesActivity.cs | 106 ++++++ .../Client/DefaultExportHistoryClient.cs | 192 ++++++++++ .../Client/DefaultExportHistoryJobClient.cs | 299 ++++++++++++++++ .../Client/ExportHistoryClient.cs | 19 + .../Client/ExportHistoryJobClient.cs | 29 ++ .../Constants/ExportHistoryConstants.cs | 33 ++ src/ExportHistory/Entity/ExportJob.cs | 264 ++++++++++++++ .../Entity/ExportJobOperations.cs | 21 ++ .../ExportJobClientValidationException.cs | 39 ++ .../ExportJobInvalidTransitionException.cs | 64 ++++ .../Exception/ExportJobNotFoundException.cs | 36 ++ src/ExportHistory/ExportHistory.csproj | 26 ++ .../DurableTaskClientBuilderExtensions.cs | 51 +++ .../DurableTaskWorkerBuilderExtensions.cs | 27 ++ src/ExportHistory/Logging/Logs.Client.cs | 22 ++ src/ExportHistory/Logging/Logs.Entity.cs | 24 ++ .../Models/CommitCheckpointRequest.cs | 32 ++ src/ExportHistory/Models/ExportCheckpoint.cs | 26 ++ src/ExportHistory/Models/ExportDestination.cs | 32 ++ src/ExportHistory/Models/ExportFailure.cs | 14 + src/ExportHistory/Models/ExportFilter.cs | 12 + src/ExportHistory/Models/ExportFormat.cs | 8 + .../Models/ExportJobConfiguration.cs | 12 + .../Models/ExportJobCreationOptions.cs | 112 ++++++ .../Models/ExportJobDescription.cs | 42 +++ src/ExportHistory/Models/ExportJobQuery.cs | 45 +++ src/ExportHistory/Models/ExportJobState.cs | 61 ++++ src/ExportHistory/Models/ExportJobStatus.cs | 30 ++ .../Models/ExportJobTransitions.cs | 43 +++ src/ExportHistory/Models/ExportMode.cs | 17 + .../Options/ExportHistoryStorageOptions.cs | 27 ++ .../ExecuteExportJobOperationOrchestrator.cs | 28 ++ .../Orchestrations/ExportJobOrchestrator.cs | 334 ++++++++++++++++++ src/Grpc/orchestrator_service.proto | 24 +- .../Sidecar/Grpc/TaskHubGrpcServer.cs | 89 ++++- 45 files changed, 3038 insertions(+), 4 deletions(-) create mode 100644 samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj create mode 100644 samples/ExportHistoryWebApp/ExportHistoryWebApp.http create mode 100644 samples/ExportHistoryWebApp/ExportJobController.cs create mode 100644 samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs create mode 100644 samples/ExportHistoryWebApp/Program.cs create mode 100644 samples/ExportHistoryWebApp/Properties/launchSettings.json create mode 100644 samples/ExportHistoryWebApp/appsettings.Development.json create mode 100644 samples/ExportHistoryWebApp/appsettings.json create mode 100644 src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs create mode 100644 src/ExportHistory/Activities/ListTerminalInstancesActivity.cs create mode 100644 src/ExportHistory/Client/DefaultExportHistoryClient.cs create mode 100644 src/ExportHistory/Client/DefaultExportHistoryJobClient.cs create mode 100644 src/ExportHistory/Client/ExportHistoryClient.cs create mode 100644 src/ExportHistory/Client/ExportHistoryJobClient.cs create mode 100644 src/ExportHistory/Constants/ExportHistoryConstants.cs create mode 100644 src/ExportHistory/Entity/ExportJob.cs create mode 100644 src/ExportHistory/Entity/ExportJobOperations.cs create mode 100644 src/ExportHistory/Exception/ExportJobClientValidationException.cs create mode 100644 src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs create mode 100644 src/ExportHistory/Exception/ExportJobNotFoundException.cs create mode 100644 src/ExportHistory/ExportHistory.csproj create mode 100644 src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs create mode 100644 src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs create mode 100644 src/ExportHistory/Logging/Logs.Client.cs create mode 100644 src/ExportHistory/Logging/Logs.Entity.cs create mode 100644 src/ExportHistory/Models/CommitCheckpointRequest.cs create mode 100644 src/ExportHistory/Models/ExportCheckpoint.cs create mode 100644 src/ExportHistory/Models/ExportDestination.cs create mode 100644 src/ExportHistory/Models/ExportFailure.cs create mode 100644 src/ExportHistory/Models/ExportFilter.cs create mode 100644 src/ExportHistory/Models/ExportFormat.cs create mode 100644 src/ExportHistory/Models/ExportJobConfiguration.cs create mode 100644 src/ExportHistory/Models/ExportJobCreationOptions.cs create mode 100644 src/ExportHistory/Models/ExportJobDescription.cs create mode 100644 src/ExportHistory/Models/ExportJobQuery.cs create mode 100644 src/ExportHistory/Models/ExportJobState.cs create mode 100644 src/ExportHistory/Models/ExportJobStatus.cs create mode 100644 src/ExportHistory/Models/ExportJobTransitions.cs create mode 100644 src/ExportHistory/Models/ExportMode.cs create mode 100644 src/ExportHistory/Options/ExportHistoryStorageOptions.cs create mode 100644 src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs create mode 100644 src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs diff --git a/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj b/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj new file mode 100644 index 000000000..884da7e8b --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportHistoryWebApp.csproj @@ -0,0 +1,23 @@ + + + + net6.0 + enable + enable + true + $(BaseIntermediateOutputPath)Generated + + + + + + + + + + + + + + + diff --git a/samples/ExportHistoryWebApp/ExportHistoryWebApp.http b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http new file mode 100644 index 000000000..de24a7a52 --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http @@ -0,0 +1,88 @@ +### Variables +@baseUrl = http://localhost:5009 +@jobId = export-job-123 + +### Create a new batch export job +# @name createBatchExportJob +POST {{baseUrl}}/export-jobs +Content-Type: application/json + +{ + "jobId": "{{jobId}}", + "mode": "Batch", + "createdTimeFrom": "2024-01-01T00:00:00Z", + "createdTimeTo": "2024-12-31T23:59:59Z", + "containerName": "export-history", + "prefix": "exports/", + "maxInstancesPerBatch": 100, + "runtimeStatus": ["Completed", "Failed"] +} + +### Create a new continuous export job +# @name createContinuousExportJob +POST {{baseUrl}}/export-jobs +Content-Type: application/json + +{ + "jobId": "export-job-continuous-123", + "mode": "Continuous", + "createdTimeFrom": "2024-01-01T00:00:00Z", + "createdTimeTo": null, + "containerName": "export-history", + "prefix": "continuous-exports/", + "maxInstancesPerBatch": 50 +} + +### Create an export job with default storage (no container specified) +# @name createExportJobWithDefaultStorage +POST {{baseUrl}}/export-jobs +Content-Type: application/json + +{ + "jobId": "export-job-default-storage", + "mode": "Batch", + "createdTimeFrom": "2024-01-01T00:00:00Z", + "createdTimeTo": "2024-12-31T23:59:59Z", + "maxInstancesPerBatch": 100 +} + +### Get a specific export job by ID +# Note: This endpoint can be used to verify the export job was created and check its status +# The ID in the URL should match the jobId used in create request +GET {{baseUrl}}/export-jobs/{{jobId}} + +### List all export jobs +GET {{baseUrl}}/export-jobs/list + +### List export jobs with filters +# Filter by status +GET {{baseUrl}}/export-jobs/list?status=Active + +# Filter by job ID prefix +GET {{baseUrl}}/export-jobs/list?jobIdPrefix=export-job- + +# Filter by creation time range +GET {{baseUrl}}/export-jobs/list?createdFrom=2024-01-01T00:00:00Z&createdTo=2024-12-31T23:59:59Z + +# Combined filters +GET {{baseUrl}}/export-jobs/list?status=Completed&jobIdPrefix=export-job-&pageSize=50 + +### Delete an export job +DELETE {{baseUrl}}/export-jobs/{{jobId}} + +### Tips: +# - Replace the baseUrl variable if your application runs on a different port +# - The jobId variable can be changed to test different export job instances +# - Export modes: +# - "Batch": Exports all instances within a time range (requires createdTimeTo) +# - "Continuous": Continuously exports instances from a start time (createdTimeTo must be null) +# - Runtime status filters (valid values): +# - "Completed": Exports only completed orchestrations +# - "Failed": Exports only failed orchestrations +# - "Terminated": Exports only terminated orchestrations +# - "ContinuedAsNew": Exports only continued-as-new orchestrations +# - Dates are in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ) +# - You can use the REST Client extension in VS Code to execute these requests +# - The @name directive allows referencing the response in subsequent requests +# - Export jobs run asynchronously; use GET to check the status after creation + diff --git a/samples/ExportHistoryWebApp/ExportJobController.cs b/samples/ExportHistoryWebApp/ExportJobController.cs new file mode 100644 index 000000000..34c510b4d --- /dev/null +++ b/samples/ExportHistoryWebApp/ExportJobController.cs @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.Mvc; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.ExportHistory; +using ExportHistoryWebApp.Models; + +namespace ExportHistoryWebApp.Controllers; + +/// +/// Controller for managing export history jobs through a REST API. +/// Provides endpoints for creating, reading, listing, and deleting export jobs. +/// +[ApiController] +[Route("export-jobs")] +public class ExportJobController : ControllerBase +{ + readonly ExportHistoryClient exportHistoryClient; + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// Client for managing export history jobs. + /// Logger for recording controller operations. + public ExportJobController( + ExportHistoryClient exportHistoryClient, + ILogger logger) + { + this.exportHistoryClient = exportHistoryClient ?? throw new ArgumentNullException(nameof(exportHistoryClient)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Creates a new export job based on the provided configuration. + /// + /// The export job creation request. + /// The created export job description. + [HttpPost] + public async Task> CreateExportJob([FromBody] CreateExportJobRequest request) + { + if (request == null) + { + return this.BadRequest("createExportJobRequest cannot be null"); + } + + try + { + ExportDestination? destination = null; + if (!string.IsNullOrEmpty(request.ContainerName)) + { + destination = new ExportDestination(request.ContainerName) + { + Prefix = request.Prefix, + }; + } + + ExportJobCreationOptions creationOptions = new ExportJobCreationOptions( + mode: request.Mode, + createdTimeFrom: request.CreatedTimeFrom, + createdTimeTo: request.CreatedTimeTo, + destination: destination, + jobId: request.JobId, + format: request.Format, + runtimeStatus: request.RuntimeStatus, + maxInstancesPerBatch: request.MaxInstancesPerBatch); + + ExportHistoryJobClient jobClient = await this.exportHistoryClient.CreateJobAsync(creationOptions); + ExportJobDescription description = await jobClient.DescribeAsync(); + + this.logger.LogInformation("Created new export job with ID: {JobId}", description.JobId); + + return this.CreatedAtAction(nameof(GetExportJob), new { id = description.JobId }, description); + } + catch (ArgumentException ex) + { + this.logger.LogError(ex, "Validation failed while creating export job {JobId}", request.JobId); + return this.BadRequest(ex.Message); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error creating export job {JobId}", request.JobId); + return this.StatusCode(500, "An error occurred while creating the export job"); + } + } + + /// + /// Retrieves a specific export job by its ID. + /// + /// The ID of the export job to retrieve. + /// The export job description if found. + [HttpGet("{id}")] + public async Task> GetExportJob(string id) + { + try + { + ExportJobDescription? job = await this.exportHistoryClient.GetJobAsync(id); + return this.Ok(job); + } + catch (ExportJobNotFoundException) + { + return this.NotFound(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error retrieving export job {JobId}", id); + return this.StatusCode(500, "An error occurred while retrieving the export job"); + } + } + + /// + /// Lists all export jobs, optionally filtered by query parameters. + /// + /// Optional filter by job status. + /// Optional filter by job ID prefix. + /// Optional filter for jobs created after this time. + /// Optional filter for jobs created before this time. + /// Optional page size for pagination. + /// Optional continuation token for pagination. + /// A collection of export job descriptions. + [HttpGet("list")] + public async Task>> ListExportJobs( + [FromQuery] ExportJobStatus? status = null, + [FromQuery] string? jobIdPrefix = null, + [FromQuery] DateTimeOffset? createdFrom = null, + [FromQuery] DateTimeOffset? createdTo = null, + [FromQuery] int? pageSize = null, + [FromQuery] string? continuationToken = null) + { + try + { + ExportJobQuery? query = null; + if (status.HasValue || !string.IsNullOrEmpty(jobIdPrefix) || createdFrom.HasValue || createdTo.HasValue || pageSize.HasValue || !string.IsNullOrEmpty(continuationToken)) + { + query = new ExportJobQuery + { + Status = status, + JobIdPrefix = jobIdPrefix, + CreatedFrom = createdFrom, + CreatedTo = createdTo, + PageSize = pageSize, + ContinuationToken = continuationToken, + }; + } + + AsyncPageable jobs = this.exportHistoryClient.ListJobsAsync(query); + + // Collect all jobs from the async pageable + List jobList = new List(); + await foreach (ExportJobDescription job in jobs) + { + jobList.Add(job); + } + + return this.Ok(jobList); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error retrieving export jobs"); + return this.StatusCode(500, "An error occurred while retrieving export jobs"); + } + } + + /// + /// Deletes an export job by its ID. + /// + /// The ID of the export job to delete. + /// No content if successful. + [HttpDelete("{id}")] + public async Task DeleteExportJob(string id) + { + try + { + ExportHistoryJobClient jobClient = this.exportHistoryClient.GetJobClient(id); + await jobClient.DeleteAsync(); + return this.NoContent(); + } + catch (ExportJobNotFoundException) + { + return this.NotFound(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error deleting export job {JobId}", id); + return this.StatusCode(500, "An error occurred while deleting the export job"); + } + } +} + diff --git a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs new file mode 100644 index 000000000..7863281c7 --- /dev/null +++ b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.ExportHistory; + +namespace ExportHistoryWebApp.Models; + +/// +/// Represents a request to create a new export job. +/// +public class CreateExportJobRequest +{ + /// + /// Gets or sets the unique identifier for the export job. If not provided, a GUID will be generated. + /// + public string? JobId { get; set; } + + /// + /// Gets or sets the export mode (Batch or Continuous). + /// + public ExportMode Mode { get; set; } + + /// + /// Gets or sets the start time for the export (inclusive). Required. + /// + public DateTimeOffset CreatedTimeFrom { get; set; } + + /// + /// Gets or sets the end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// + public DateTimeOffset? CreatedTimeTo { get; set; } + + /// + /// Gets or sets the blob container name where exported data will be stored. Optional if default storage is configured. + /// + public string? ContainerName { get; set; } + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } + + /// + /// Gets or sets the export format settings. Optional, defaults to jsonl-gzip. + /// + public ExportFormat? Format { get; set; } + + /// + /// Gets or sets the orchestration runtime statuses to filter by. Optional. + /// Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew. + /// + public List? RuntimeStatus { get; set; } + + /// + /// Gets or sets the maximum number of instances to fetch per batch. Optional, defaults to 100. + /// + public int? MaxInstancesPerBatch { get; set; } +} + diff --git a/samples/ExportHistoryWebApp/Program.cs b/samples/ExportHistoryWebApp/Program.cs new file mode 100644 index 000000000..46ee02f41 --- /dev/null +++ b/samples/ExportHistoryWebApp/Program.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.AzureManaged; +using Microsoft.DurableTask.ExportHistory; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.AzureManaged; + +WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + +string connectionString = builder.Configuration.GetValue("DURABLE_TASK_CONNECTION_STRING") + ?? throw new InvalidOperationException("Missing required configuration 'DURABLE_TASK_CONNECTION_STRING'"); + +string storageConnectionString = builder.Configuration.GetValue("EXPORT_HISTORY_STORAGE_CONNECTION_STRING") + ?? throw new InvalidOperationException("Missing required configuration 'EXPORT_HISTORY_STORAGE_CONNECTION_STRING'"); + +string containerName = builder.Configuration.GetValue("EXPORT_HISTORY_CONTAINER_NAME") + ?? throw new InvalidOperationException("Missing required configuration 'EXPORT_HISTORY_CONTAINER_NAME'"); + +builder.Services.AddSingleton(sp => sp.GetRequiredService().CreateLogger()); +builder.Services.AddLogging(); + +// Add Durable Task worker with export history support +builder.Services.AddDurableTaskWorker(builder => +{ + builder.UseDurableTaskScheduler(connectionString); + builder.UseExportHistory(); +}); + +// Register the client with export history support +builder.Services.AddDurableTaskClient(clientBuilder => +{ + clientBuilder.UseDurableTaskScheduler(connectionString); + clientBuilder.UseExportHistory(options => + { + options.ConnectionString = storageConnectionString; + options.ContainerName = containerName; + options.Prefix = builder.Configuration.GetValue("EXPORT_HISTORY_PREFIX"); + }); +}); + +// Configure the HTTP request pipeline +builder.Services.AddControllers().AddJsonOptions(options => +{ + options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); + options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; +}); + +// The actual listen URL can be configured in environment variables named "ASPNETCORE_URLS" or "ASPNETCORE_URLS_HTTPS" +WebApplication app = builder.Build(); +app.MapControllers(); +app.Run(); + diff --git a/samples/ExportHistoryWebApp/Properties/launchSettings.json b/samples/ExportHistoryWebApp/Properties/launchSettings.json new file mode 100644 index 000000000..4375d42ec --- /dev/null +++ b/samples/ExportHistoryWebApp/Properties/launchSettings.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:47698", + "sslPort": 44372 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "applicationUrl": "http://localhost:5009", + "dotnetRunMessages": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DURABLE_TASK_CONNECTION_STRING": "", + "EXPORT_HISTORY_STORAGE_CONNECTION_STRING": "", + "EXPORT_HISTORY_CONTAINER_NAME": "export-history", + "EXPORT_HISTORY_PREFIX": "" + } + } + } +} + diff --git a/samples/ExportHistoryWebApp/appsettings.Development.json b/samples/ExportHistoryWebApp/appsettings.Development.json new file mode 100644 index 000000000..21b22dbb4 --- /dev/null +++ b/samples/ExportHistoryWebApp/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} + diff --git a/samples/ExportHistoryWebApp/appsettings.json b/samples/ExportHistoryWebApp/appsettings.json new file mode 100644 index 000000000..fb8785032 --- /dev/null +++ b/samples/ExportHistoryWebApp/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} + diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index c57b22f96..101901074 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -323,6 +323,56 @@ public override async Task WaitForInstanceStartAsync( } } + /// + /// Lists terminal orchestration instances sorted by completed timestamp, returning only instance IDs. + /// + /// Creation date of instances to query from. + /// Creation date of instances to query to. + /// Runtime statuses of instances to query (should be terminal statuses). + /// Maximum number of instance IDs to return per page. + /// The continuation token (instanceId) to continue from. + /// The cancellation token. + /// A tuple containing the list of instance IDs and the continuation token for the next page. + public async Task<(IReadOnlyList InstanceIds, string? ContinuationToken)> ListTerminalInstancesAsync( + DateTimeOffset? createdFrom = null, + DateTimeOffset? createdTo = null, + IEnumerable? statuses = null, + int pageSize = 100, + string? continuationToken = null, + CancellationToken cancellation = default) + { + Check.NotEntity(this.options.EnableEntitySupport, continuationToken); + + P.ListTerminalInstancesRequest request = new() + { + Query = new P.TerminalInstanceQuery + { + CreatedTimeFrom = createdFrom?.ToTimestamp(), + CreatedTimeTo = createdTo?.ToTimestamp(), + MaxInstanceCount = pageSize, + ContinuationToken = continuationToken, + }, + }; + + if (statuses is not null) + { + request.Query.RuntimeStatus.AddRange(statuses.Select(x => x.ToGrpcStatus())); + } + + try + { + P.ListTerminalInstancesResponse response = await this.sidecarClient.ListTerminalInstancesAsync( + request, cancellationToken: cancellation); + + return (response.InstanceIds.ToList(), response.ContinuationToken); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.ListTerminalInstancesAsync)} operation was canceled.", e, cancellation); + } + } + /// public override async Task WaitForInstanceCompletionAsync( string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs new file mode 100644 index 000000000..2a34c0508 --- /dev/null +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Activity that exports one orchestration instance history to the configured blob destination. +/// +[DurableTask] +public class ExportInstanceHistoryActivity : TaskActivity +{ + readonly IDurableTaskClientProvider clientProvider; + readonly ILogger logger; + readonly ExportHistoryStorageOptions storageOptions; + + /// + /// Initializes a new instance of the class. + /// + public ExportInstanceHistoryActivity( + IDurableTaskClientProvider clientProvider, + ILogger logger, + IOptions storageOptions) + { + this.clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + this.logger = Check.NotNull(logger, nameof(logger)); + this.storageOptions = Check.NotNull(storageOptions?.Value, nameof(storageOptions)); + } + + /// + public override async Task RunAsync(TaskActivityContext context, ExportRequest input) + { + Check.NotNull(input, nameof(input)); + Check.NotNullOrEmpty(input.InstanceId, nameof(input.InstanceId)); + Check.NotNull(input.Destination, nameof(input.Destination)); + Check.NotNull(input.Format, nameof(input.Format)); + + string instanceId = input.InstanceId; + + try + { + this.logger.LogInformation("Starting export for instance {InstanceId}", instanceId); + + // Get the client and instance metadata with inputs and outputs + DurableTaskClient client = this.clientProvider.GetClient(); + OrchestrationMetadata? metadata = await client.GetInstanceAsync( + instanceId, + getInputsAndOutputs: true, + cancellation: CancellationToken.None); + + if (metadata == null) + { + string error = $"Instance {instanceId} not found"; + this.logger.LogWarning(error); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = error, + }; + } + + // Get completed timestamp (LastUpdatedAt for terminal states) + DateTimeOffset completedTimestamp = metadata.LastUpdatedAt; + if (!metadata.IsCompleted) + { + string error = $"Instance {instanceId} is not in a completed state"; + this.logger.LogWarning(error); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = error, + }; + } + + // Create blob filename from hash of completed timestamp and instance ID + string blobFileName = GenerateBlobFileName(completedTimestamp, instanceId, input.Format); + + // Build blob path with prefix if provided + string blobPath = string.IsNullOrEmpty(input.Destination.Prefix) + ? blobFileName + : $"{input.Destination.Prefix.TrimEnd('/')}/{blobFileName}"; + + // Serialize instance metadata to JSON + string jsonContent = SerializeInstanceMetadata(metadata); + + // Upload to blob storage + await UploadToBlobStorageAsync( + input.Destination.Container, + blobPath, + jsonContent, + input.Format, + CancellationToken.None); + + this.logger.LogInformation( + "Successfully exported instance {InstanceId} to blob {BlobPath}", + instanceId, + blobPath); + + return new ExportResult + { + InstanceId = instanceId, + Success = true, + }; + } + catch (Exception ex) + { + this.logger.LogError(ex, "Failed to export instance {InstanceId}", instanceId); + return new ExportResult + { + InstanceId = instanceId, + Success = false, + Error = ex.Message, + }; + } + } + + static string GenerateBlobFileName(DateTimeOffset completedTimestamp, string instanceId, ExportFormat format) + { + // Create hash from completed timestamp and instance ID + string hashInput = $"{completedTimestamp:O}|{instanceId}"; + byte[] hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(hashInput)); + string hash = Convert.ToHexString(hashBytes).ToLowerInvariant(); + + // Build filename with format extension + string extension = GetFileExtension(format); + + return $"{hash}.{extension}"; + } + + /// + /// Gets the file extension for a given export format. + /// + /// The export format. + /// The file extension (e.g., "json", "jsonl.gz"). + static string GetFileExtension(ExportFormat format) + { + string formatKind = format.Kind.ToLowerInvariant(); + + return formatKind switch + { + "jsonl" => "jsonl.gz", // JSONL format is compressed + "json" => "json", // JSON format is uncompressed + _ => "jsonl.gz", // Default to JSONL compressed + }; + } + + static string SerializeInstanceMetadata(OrchestrationMetadata metadata) + { + var exportData = new + { + instanceId = metadata.InstanceId, + name = metadata.Name, + runtimeStatus = metadata.RuntimeStatus.ToString(), + createdAt = metadata.CreatedAt, + lastUpdatedAt = metadata.LastUpdatedAt, + input = metadata.SerializedInput, + output = metadata.SerializedOutput, + customStatus = metadata.SerializedCustomStatus, + tags = metadata.Tags, + failureDetails = metadata.FailureDetails != null ? new + { + errorType = metadata.FailureDetails.ErrorType, + errorMessage = metadata.FailureDetails.ErrorMessage, + stackTrace = metadata.FailureDetails.StackTrace, + } : null, + }; + + return JsonSerializer.Serialize(exportData, new JsonSerializerOptions + { + WriteIndented = false, + }); + } + + async Task UploadToBlobStorageAsync( + string containerName, + string blobPath, + string content, + ExportFormat format, + CancellationToken cancellationToken) + { + // Create blob service client from connection string + BlobServiceClient serviceClient = new(this.storageOptions.ConnectionString); + BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(containerName); + + // Ensure container exists + await containerClient.CreateIfNotExistsAsync( + PublicAccessType.None, + cancellationToken: cancellationToken); + + // Get blob client + BlobClient blobClient = containerClient.GetBlobClient(blobPath); + + // Upload content + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + + if (format.Kind.ToLowerInvariant() == "jsonl") + { + // Compress with gzip + using MemoryStream compressedStream = new(); + using (GZipStream gzipStream = new(compressedStream, CompressionLevel.Optimal, leaveOpen: true)) + { + await gzipStream.WriteAsync(contentBytes, cancellationToken); + await gzipStream.FlushAsync(cancellationToken); + } + + compressedStream.Position = 0; + + BlobUploadOptions uploadOptions = new() + { + HttpHeaders = new BlobHttpHeaders + { + ContentType = "application/jsonl+gzip", + ContentEncoding = "gzip", + }, + }; + + await blobClient.UploadAsync(compressedStream, uploadOptions, cancellationToken); + } + else + { + // Upload uncompressed + BlobUploadOptions uploadOptions = new() + { + HttpHeaders = new BlobHttpHeaders + { + ContentType = "application/json", + }, + }; + + await blobClient.UploadAsync( + new BinaryData(contentBytes), + uploadOptions, + cancellationToken); + } + } +} + +/// +/// Export request for one orchestration instance. +/// +public sealed class ExportRequest +{ + /// + /// Gets or sets the instance ID to export. + /// + public string InstanceId { get; set; } = string.Empty; + + /// + /// Gets or sets the export destination configuration. + /// + public ExportDestination Destination { get; set; } = null!; + + /// + /// Gets or sets the export format configuration. + /// + public ExportFormat Format { get; set; } = null!; +} + +/// +/// Export result. +/// +public sealed class ExportResult +{ + /// + /// Gets or sets the instance ID that was exported. + /// + public string InstanceId { get; set; } = string.Empty; + + /// + /// Gets or sets whether the export was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the error message if the export failed. + /// + public string? Error { get; set; } +} + + diff --git a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs new file mode 100644 index 000000000..facc804b1 --- /dev/null +++ b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Grpc; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Input for listing terminal instances activity. +/// +public sealed record ListTerminalInstancesRequest( + DateTimeOffset CreatedTimeFrom, + DateTimeOffset? CreatedTimeTo, + IEnumerable? RuntimeStatus, + string? ContinuationToken, + int MaxInstancesPerBatch = 100); + +/// +/// Activity that lists terminal orchestration instances using the configured filters and checkpoint. +/// +[DurableTask] +public class ListTerminalInstancesActivity : TaskActivity +{ + readonly IDurableTaskClientProvider clientProvider; + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + public ListTerminalInstancesActivity( + IDurableTaskClientProvider clientProvider, + ILogger logger) + { + this.clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + this.logger = Check.NotNull(logger, nameof(logger)); + } + + /// + public override async Task RunAsync(TaskActivityContext context, ListTerminalInstancesRequest request) + { + Check.NotNull(request, nameof(request)); + + try + { + DurableTaskClient client = this.clientProvider.GetClient(); + + // Check if the client is a gRPC client that supports ListTerminalInstances + if (client is not GrpcDurableTaskClient grpcClient) + { + throw new NotSupportedException( + $"ListTerminalInstancesActivity requires a GrpcDurableTaskClient, but got {client.GetType().Name}"); + } + + // Call the gRPC endpoint to list terminal instances + (IReadOnlyList instanceIds, string? nextContinuationToken) = await grpcClient.ListTerminalInstancesAsync( + createdFrom: request.CreatedTimeFrom, + createdTo: request.CreatedTimeTo, + statuses: request.RuntimeStatus, + pageSize: request.MaxInstancesPerBatch, + continuationToken: request.ContinuationToken, + cancellation: CancellationToken.None); + + this.logger.LogInformation( + "ListTerminalInstancesActivity returned {Count} instance IDs", + instanceIds.Count); + + // Create next checkpoint if we have a continuation token + ExportCheckpoint? nextCheckpoint = null; + if (!string.IsNullOrEmpty(nextContinuationToken) && instanceIds.Count > 0) + { + // Use the continuation token from the response (which is the last instanceId) + string lastInstanceId = nextContinuationToken; + nextCheckpoint = new ExportCheckpoint + { + ContinuationToken = lastInstanceId, + LastInstanceIdProcessed = lastInstanceId, + // Note: LastTerminalTimeProcessed would require querying the instance, which we skip for performance + }; + } + + return new InstancePage + { + InstanceIds = instanceIds.ToList(), + NextCheckpoint = nextCheckpoint, + }; + } + catch (Exception ex) + { + this.logger.LogError(ex, "ListTerminalInstancesActivity failed"); + throw; + } + } +} + +/// +/// A page of instances for export. +/// +public sealed class InstancePage +{ + public List InstanceIds { get; set; } = new(); + public ExportCheckpoint? NextCheckpoint { get; set; } +} + + diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs new file mode 100644 index 000000000..9dcd145d4 --- /dev/null +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public sealed class DefaultExportHistoryClient( + DurableTaskClient durableTaskClient, + ILogger logger, + ExportHistoryStorageOptions storageOptions) : ExportHistoryClient +{ + readonly DurableTaskClient durableTaskClient = Check.NotNull(durableTaskClient, nameof(durableTaskClient)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions, nameof(storageOptions)); + + /// + public override async Task CreateJobAsync( + ExportJobCreationOptions options, + CancellationToken cancellation = default) + { + Check.NotNull(options, nameof(options)); + this.logger.ClientCreatingExportJob(options); + + try + { + // Create export job client instance + ExportHistoryJobClient exportHistoryJobClient = new DefaultExportHistoryJobClient( + this.durableTaskClient, + options.JobId, + this.logger, + this.storageOptions + ); + + // Create the export job using the client (validation already done in constructor) + await exportHistoryJobClient.CreateAsync(options, cancellation); + + // Return the job client + return exportHistoryJobClient; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.CreateJobAsync), options.JobId, ex); + + throw; + } + } + + /// + public override async Task GetJobAsync(string jobId, CancellationToken cancellation = default) + { + Check.NotNullOrEmpty(jobId, nameof(jobId)); + + try + { + // Get export history job client first + ExportHistoryJobClient exportHistoryJobClient = this.GetJobClient(jobId); + + // Call DescribeAsync which handles all the entity state mapping + return await exportHistoryJobClient.DescribeAsync(cancellation); + } + catch (ExportJobNotFoundException) + { + // Re-throw as the job not being found is an error condition + throw; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.GetJobAsync), jobId, ex); + + throw; + } + } + + /// + public override AsyncPageable ListJobsAsync(ExportJobQuery? filter = null) + { + // TODO: revisit the fields + // Create an async pageable using the Pageable.Create helper + return Pageable.Create(async (continuationToken, pageSize, cancellation) => + { + try + { + EntityQuery query = new EntityQuery + { + InstanceIdStartsWith = $"@{nameof(ExportJob)}@{filter?.JobIdPrefix ?? string.Empty}", + IncludeState = true, + PageSize = filter?.PageSize ?? ExportJobQuery.DefaultPageSize, + ContinuationToken = continuationToken, + }; + + // Get one page of entities + IAsyncEnumerable>> entityPages = + this.durableTaskClient.Entities.GetAllEntitiesAsync(query).AsPages(); + + await foreach (Page> entityPage in entityPages) + { + List exportJobs = new(); + + foreach (EntityMetadata metadata in entityPage.Values) + { + if (filter != null && !MatchesFilter(metadata.State, filter)) + { + continue; + } + + ExportJobState state = metadata.State; + ExportJobConfiguration? config = state.Config; + + // Determine orchestrator presence for this job + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(metadata.Id.Key); + OrchestrationMetadata? orchestratorState = await this.durableTaskClient.GetInstanceAsync(orchestratorInstanceId, cancellation: cancellation); + string? presentOrchestratorId = orchestratorState != null ? orchestratorInstanceId : null; + + exportJobs.Add(new ExportJobDescription + { + JobId = metadata.Id.Key, + Status = state.Status, + CreatedAt = state.CreatedAt, + LastModifiedAt = state.LastModifiedAt, + Config = config, + OrchestratorInstanceId = presentOrchestratorId, + }); + } + + return new Page(exportJobs, entityPage.ContinuationToken); + } + + // Return empty page if no results + return new Page(new List(), null); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.ListJobsAsync), string.Empty, ex); + + throw; + } + }); + } + + /// + /// Gets a job client for the specified job ID. + /// + /// The job ID. + /// The export history job client. + public override ExportHistoryJobClient GetJobClient(string jobId) + { + return new DefaultExportHistoryJobClient( + this.durableTaskClient, + jobId, + this.logger, + this.storageOptions); + } + + /// + /// Checks if an export job state matches the provided filter criteria. + /// + /// The export job state to check. + /// The filter criteria. + /// True if the state matches the filter; otherwise, false. + static bool MatchesFilter(ExportJobState state, ExportJobQuery filter) + { + bool statusMatches = !filter.Status.HasValue || state.Status == filter.Status.Value; + bool createdFromMatches = !filter.CreatedFrom.HasValue || + (state.CreatedAt.HasValue && state.CreatedAt.Value > filter.CreatedFrom.Value); + bool createdToMatches = !filter.CreatedTo.HasValue || + (state.CreatedAt.HasValue && state.CreatedAt.Value < filter.CreatedTo.Value); + + return statusMatches && createdFromMatches && createdToMatches; + } +} \ No newline at end of file diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs new file mode 100644 index 000000000..7f7764e56 --- /dev/null +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -0,0 +1,299 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public sealed class DefaultExportHistoryJobClient( + DurableTaskClient durableTaskClient, + string jobId, + ILogger logger, + ExportHistoryStorageOptions storageOptions +) : ExportHistoryJobClient(jobId) +{ + readonly DurableTaskClient durableTaskClient = Check.NotNull(durableTaskClient, nameof(durableTaskClient)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions, nameof(storageOptions)); + readonly EntityInstanceId entityId = new(nameof(ExportJob), jobId); + + public override async Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default) + { + try + { + Check.NotNull(options, nameof(options)); + + // If destination is not provided, construct it from storage options + ExportJobCreationOptions optionsWithDestination = options; + if (options.Destination == null) + { + ExportDestination destination = new ExportDestination(this.storageOptions.ContainerName) + { + Prefix = this.storageOptions.Prefix, + }; + + optionsWithDestination = options with { Destination = destination }; + } + + ExportJobOperationRequest request = + new ExportJobOperationRequest( + this.entityId, + nameof(ExportJob.Create), + optionsWithDestination); + + string instanceId = await this.durableTaskClient + .ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteExportJobOperationOrchestrator)), + request, + cancellation); + + // Wait for the orchestration to complete + OrchestrationMetadata state = await this.durableTaskClient + .WaitForInstanceCompletionAsync( + instanceId, + true, + cancellation); + + if (state.RuntimeStatus != OrchestrationRuntimeStatus.Completed) + { + throw new InvalidOperationException( + $"Failed to create export job '{this.JobId}': " + + $"{state.FailureDetails?.ErrorMessage ?? string.Empty}"); + } + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.CreateAsync), this.JobId, ex); + + throw; + } + } + + // TODO: there is no atomicity guarantee of deleting entity and purging the orchestrator + // Add sweeping process to clean up orphaned orchestrations failed to be purged + public override async Task DeleteAsync(CancellationToken cancellation = default) + { + try + { + this.logger.ClientDeletingExportJob(this.JobId); + + string orchestrationInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.JobId); + + // First, delete the entity + ExportJobOperationRequest request = new ExportJobOperationRequest(this.entityId, ExportJobOperations.Delete); + string instanceId = await this.durableTaskClient.ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteExportJobOperationOrchestrator)), + request, + cancellation); + + // Wait for the orchestration to complete + OrchestrationMetadata state = await this.durableTaskClient.WaitForInstanceCompletionAsync(instanceId, true, cancellation); + + if (state.RuntimeStatus != OrchestrationRuntimeStatus.Completed) + { + throw new InvalidOperationException($"Failed to delete export job '{this.JobId}': {state.FailureDetails?.ErrorMessage ?? string.Empty}"); + } + + // Then terminate the linked export orchestration if it exists + await this.TerminateAndPurgeOrchestrationAsync(orchestrationInstanceId, cancellation); + + // Verify both entity and orchestration are gone + await this.VerifyDeletionAsync(orchestrationInstanceId, cancellation); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.DeleteAsync), this.JobId, ex); + + throw; + } + } + + public override async Task DescribeAsync(CancellationToken cancellation = default) + { + try + { + Check.NotNullOrEmpty(this.JobId, nameof(this.JobId)); + + EntityMetadata? metadata = + await this.durableTaskClient.Entities.GetEntityAsync(this.entityId, cancellation: cancellation); + if (metadata == null) + { + throw new ExportJobNotFoundException(this.JobId); + } + + ExportJobState state = metadata.State; + + ExportJobConfiguration? config = state.Config; + + // Determine if the export orchestrator instance exists and capture its instance ID if so + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.JobId); + OrchestrationMetadata? orchestratorState = await this.durableTaskClient.GetInstanceAsync(orchestratorInstanceId, cancellation: cancellation); + string? presentOrchestratorId = orchestratorState != null ? orchestratorInstanceId : null; + + return new ExportJobDescription + { + JobId = this.JobId, + Status = state.Status, + CreatedAt = state.CreatedAt, + LastModifiedAt = state.LastModifiedAt, + Config = config, + OrchestratorInstanceId = presentOrchestratorId, + }; + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + // the operation was cancelled as requested. No need to log this. + throw; + } + catch (Exception ex) + { + this.logger.ClientError(nameof(this.DescribeAsync), this.JobId, ex); + + throw; + } + } + + /// + /// Terminates and purges the export orchestration instance. + /// + /// The orchestration instance ID to terminate and purge. + /// The cancellation token. + async Task TerminateAndPurgeOrchestrationAsync(string orchestrationInstanceId, CancellationToken cancellation) + { + try + { + // Terminate the orchestration (will fail silently if it doesn't exist or already terminated) + await this.durableTaskClient.TerminateInstanceAsync( + orchestrationInstanceId, + new TerminateInstanceOptions { Output = "Export job deleted" }, + cancellation); + + // Wait for the orchestration to be terminated before purging + OrchestrationMetadata? orchestrationState = await this.WaitForOrchestrationTerminationAsync( + orchestrationInstanceId, + cancellation); + + // Purge the orchestration instance after it's terminated + if (orchestrationState != null && DefaultExportHistoryJobClient.IsTerminalStatus(orchestrationState.RuntimeStatus)) + { + await this.durableTaskClient.PurgeInstanceAsync( + orchestrationInstanceId, + cancellation: cancellation); + } + else if (orchestrationState != null) + { + throw new InvalidOperationException( + $"Failed to delete export job '{this.JobId}': Cannot purge orchestration '{orchestrationInstanceId}' because it is still in '{orchestrationState.RuntimeStatus}' status."); + } + } + catch (Exception ex) when (!(ex is InvalidOperationException)) + { + // Log but don't fail if termination fails (orchestration may not exist or already be terminated) + this.logger.ClientError( + $"Failed to terminate or purge linked orchestration '{orchestrationInstanceId}': {ex.Message}", + this.JobId, + ex); + // Continue to verification - if orchestration doesn't exist, verification will pass + } + } + + /// + /// Waits for an orchestration to reach a terminal state. + /// + /// The orchestration instance ID. + /// The cancellation token. + /// The orchestration metadata, or null if the orchestration doesn't exist. + async Task WaitForOrchestrationTerminationAsync( + string orchestrationInstanceId, + CancellationToken cancellation) + { + OrchestrationMetadata? orchestrationState = null; + int waitAttempt = 0; + + while (waitAttempt < ExportHistoryConstants.MaxTerminationWaitAttempts) + { + orchestrationState = await this.durableTaskClient.GetInstanceAsync( + orchestrationInstanceId, + cancellation: cancellation); + + if (orchestrationState == null || DefaultExportHistoryJobClient.IsTerminalStatus(orchestrationState.RuntimeStatus)) + { + break; + } + + // Wait a bit before checking again + await Task.Delay( + TimeSpan.FromMilliseconds(ExportHistoryConstants.TerminationWaitDelayMs), + cancellation); + waitAttempt++; + } + + return orchestrationState; + } + + /// + /// Checks if an orchestration runtime status is a terminal state. + /// + /// The runtime status to check. + /// True if the status is terminal; otherwise, false. + static bool IsTerminalStatus(OrchestrationRuntimeStatus runtimeStatus) + { + return runtimeStatus == OrchestrationRuntimeStatus.Terminated || + runtimeStatus == OrchestrationRuntimeStatus.Completed || + runtimeStatus == OrchestrationRuntimeStatus.Failed; + } + + /// + /// Verifies that both the entity and orchestration have been deleted. + /// + /// The orchestration instance ID to verify. + /// The cancellation token. + async Task VerifyDeletionAsync(string orchestrationInstanceId, CancellationToken cancellation) + { + List stillExist = new(); + + // Check if entity still exists + EntityMetadata? entityMetadata = await this.durableTaskClient.Entities.GetEntityAsync( + this.entityId, + cancellation: cancellation); + if (entityMetadata != null) + { + stillExist.Add($"entity '{this.entityId}'"); + } + + // Check if orchestration still exists + OrchestrationMetadata? orchestrationMetadata = await this.durableTaskClient.GetInstanceAsync( + orchestrationInstanceId, + cancellation: cancellation); + if (orchestrationMetadata != null) + { + stillExist.Add($"orchestration '{orchestrationInstanceId}'"); + } + + // Throw exception if either still exists + if (stillExist.Count > 0) + { + string items = string.Join(" and ", stillExist); + throw new InvalidOperationException( + $"Failed to delete export job '{this.JobId}': The following resources still exist: {items}."); + } + } +} + + diff --git a/src/ExportHistory/Client/ExportHistoryClient.cs b/src/ExportHistory/Client/ExportHistoryClient.cs new file mode 100644 index 000000000..18c06feeb --- /dev/null +++ b/src/ExportHistory/Client/ExportHistoryClient.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public abstract class ExportHistoryClient +{ + public abstract Task CreateJobAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + public abstract Task GetJobAsync(string jobId, CancellationToken cancellation = default); + + public abstract AsyncPageable ListJobsAsync(ExportJobQuery? filter = null); + public abstract ExportHistoryJobClient GetJobClient(string jobId); +} \ No newline at end of file diff --git a/src/ExportHistory/Client/ExportHistoryJobClient.cs b/src/ExportHistory/Client/ExportHistoryJobClient.cs new file mode 100644 index 000000000..8d9994e9e --- /dev/null +++ b/src/ExportHistory/Client/ExportHistoryJobClient.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Convenience client for managing export jobs via entity signals and reads. +/// +public abstract class ExportHistoryJobClient +{ + public readonly string JobId; + + /// + /// Initializes a new instance of the class. + /// + protected ExportHistoryJobClient(string jobId) + { + this.JobId = Check.NotNullOrEmpty(jobId, nameof(jobId)); + } + + public abstract Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + public abstract Task DescribeAsync(CancellationToken cancellation = default); + public abstract Task DeleteAsync(CancellationToken cancellation = default); +} + + diff --git a/src/ExportHistory/Constants/ExportHistoryConstants.cs b/src/ExportHistory/Constants/ExportHistoryConstants.cs new file mode 100644 index 000000000..85cf8f0d0 --- /dev/null +++ b/src/ExportHistory/Constants/ExportHistoryConstants.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Constants used throughout the export history functionality. +/// +static class ExportHistoryConstants +{ + /// + /// The prefix pattern used for generating export job orchestrator instance IDs. + /// Format: "ExportJob-{jobId}" + /// + public const string OrchestratorInstanceIdPrefix = "ExportJob-"; + + /// + /// Maximum number of attempts to wait for orchestration termination during deletion. + /// + public const int MaxTerminationWaitAttempts = 10; + + /// + /// Delay between termination wait attempts in milliseconds. + /// + public const int TerminationWaitDelayMs = 100; + + /// + /// Generates an orchestrator instance ID for a given export job ID. + /// + /// The export job ID. + /// The orchestrator instance ID. + public static string GetOrchestratorInstanceId(string jobId) => $"{OrchestratorInstanceIdPrefix}{jobId}"; +} diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs new file mode 100644 index 000000000..e9f72241d --- /dev/null +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Durable entity that manages a history export job: lifecycle, configuration, and progress. +/// +/// The logger instance. +class ExportJob(ILogger logger) : TaskEntity +{ + /// + /// Creates a new export job from creation options. + /// + /// The entity context. + /// The export job creation options. + /// Thrown when creationOptions is null. + /// Thrown when invalid state transition is attempted or export job already exists. + public void Create(TaskEntityContext context, ExportJobCreationOptions creationOptions) + { + try + { + Check.NotNull(creationOptions, nameof(creationOptions)); + + if (!this.CanTransitionTo(nameof(this.Create), ExportJobStatus.Active)) + { + throw new ExportJobInvalidTransitionException( + creationOptions.JobId, + this.State.Status, + ExportJobStatus.Active, + nameof(this.Create)); + } + + // Convert ExportJobCreationOptions to ExportJobConfiguration + // Note: RuntimeStatus validation already done in ExportJobCreationOptions constructor + // Note: Destination should be populated by the client before reaching here + Verify.NotNull(creationOptions.Destination, nameof(creationOptions.Destination)); + + ExportJobConfiguration config = new ExportJobConfiguration( + Mode: creationOptions.Mode, + Filter: new ExportFilter( + CreatedTimeFrom: creationOptions.CreatedTimeFrom, + CreatedTimeTo: creationOptions.CreatedTimeTo, + RuntimeStatus: creationOptions.RuntimeStatus), + Destination: creationOptions.Destination, + Format: creationOptions.Format, + MaxInstancesPerBatch: creationOptions.MaxInstancesPerBatch); + + this.State.Config = config; + this.State.Status = ExportJobStatus.Active; + this.State.CreatedAt = this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.CreatedExportJob(creationOptions.JobId); + + // Signal the Run method to start the export + context.SignalEntity( + context.Id, + nameof(this.Run)); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + creationOptions?.JobId ?? string.Empty, + nameof(this.Create), + "Failed to create export job", + ex); + throw; + } + } + + /// + /// Runs the export job by starting the export orchestrator. + /// + /// The entity context. + /// Thrown when export job is not in Active status. + public void Run(TaskEntityContext context) + { + try + { + Verify.NotNull(this.State.Config, nameof(this.State.Config)); + + if (this.State.Status != ExportJobStatus.Active) + { + string errorMessage = "Export job must be in Active status to run."; + logger.ExportJobOperationError(context.Id.Key, nameof(this.Run), errorMessage, new InvalidOperationException(errorMessage)); + throw new InvalidOperationException(errorMessage); + } + + this.StartExportOrchestration(context); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.Run), + "Failed to run export job", + ex); + throw; + } + } + + void StartExportOrchestration(TaskEntityContext context) + { + try + { + // Use a fixed instance ID based on job ID to ensure only one orchestrator runs per job + // This prevents concurrent orchestrators if Run is called multiple times + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new StartOrchestrationOptions(instanceId); + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.StartExportOrchestration), + $"Starting new orchestration named '{nameof(ExportJobOrchestrator)}' with instance ID: {instanceId}"); + + context.ScheduleNewOrchestration( + new TaskName(nameof(ExportJobOrchestrator)), + new ExportJobRunRequest(context.Id), + startOrchestrationOptions); + } + catch (Exception ex) + { + // Mark job as failed and record the exception + this.State.Status = ExportJobStatus.Failed; + this.State.LastError = ex.Message; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.StartExportOrchestration), + "Failed to start export orchestration", + ex); + } + } + + bool CanTransitionTo(string operationName, ExportJobStatus targetStatus) + { + return ExportJobTransitions.IsValidTransition(operationName, this.State.Status, targetStatus); + } + + /// + /// Commits a checkpoint snapshot with progress updates and optional failures. + /// + /// The entity context. + /// The checkpoint commit request containing progress, checkpoint, and failures. + public void CommitCheckpoint(TaskEntityContext context, CommitCheckpointRequest request) + { + Verify.NotNull(request, nameof(request)); + + // Update progress counts + this.State.ScannedInstances += request.ScannedInstances; + this.State.ExportedInstances += request.ExportedInstances; + + // Update checkpoint if provided (successful batch moves cursor forward) + // If null (failed batch), keep current checkpoint to not move cursor forward + if (request.Checkpoint is not null) + { + this.State.Checkpoint = request.Checkpoint; + } + + // Record failures if any + if (request.Failures != null && request.Failures.Count > 0) + { + foreach (ExportFailure failure in request.Failures) + { + this.State.FailedInstances[failure.InstanceId] = failure; + } + } + + // Update checkpoint time and last modified time + this.State.LastCheckpointTime = DateTimeOffset.UtcNow; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + // If there are failures and checkpoint is null (batch failed), mark job as failed + if (request.Checkpoint is null && request.Failures != null && request.Failures.Count > 0) + { + this.State.Status = ExportJobStatus.Failed; + string failureSummary = string.Join("; ", request.Failures.Select(f => $"{f.InstanceId}: {f.Reason}")); + this.State.LastError = $"Batch export failed after retries. Failures: {failureSummary}"; + } + } + + /// + /// Marks the export job as completed. + /// + /// The entity context. + /// Thrown when invalid state transition is attempted. + public void MarkAsCompleted(TaskEntityContext context) + { + try + { + if (!this.CanTransitionTo(nameof(this.MarkAsCompleted), ExportJobStatus.Completed)) + { + throw new ExportJobInvalidTransitionException( + context.Id.Key, + this.State.Status, + ExportJobStatus.Completed, + nameof(this.MarkAsCompleted)); + } + + this.State.Status = ExportJobStatus.Completed; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.MarkAsCompleted), + "Export job marked as completed"); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.MarkAsCompleted), + "Failed to mark export job as completed", + ex); + throw; + } + } + + /// + /// Marks the export job as failed. + /// + /// The entity context. + /// The error message describing why the job failed. + /// Thrown when invalid state transition is attempted. + public void MarkAsFailed(TaskEntityContext context, string? errorMessage = null) + { + try + { + if (!this.CanTransitionTo(nameof(this.MarkAsFailed), ExportJobStatus.Failed)) + { + throw new ExportJobInvalidTransitionException( + context.Id.Key, + this.State.Status, + ExportJobStatus.Failed, + nameof(this.MarkAsFailed)); + } + + this.State.Status = ExportJobStatus.Failed; + this.State.LastError = errorMessage; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.MarkAsFailed), + $"Export job marked as failed: {errorMessage ?? "Unknown error"}"); + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.MarkAsFailed), + "Failed to mark export job as failed", + ex); + throw; + } + } +} diff --git a/src/ExportHistory/Entity/ExportJobOperations.cs b/src/ExportHistory/Entity/ExportJobOperations.cs new file mode 100644 index 000000000..c128b96fb --- /dev/null +++ b/src/ExportHistory/Entity/ExportJobOperations.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Constants for export job entity operation names. +/// +static class ExportJobOperations +{ + /// + /// Operation name for getting entity state. + /// + public const string Get = "get"; + + /// + /// Operation name for deleting the entity. + /// + public const string Delete = "delete"; +} + diff --git a/src/ExportHistory/Exception/ExportJobClientValidationException.cs b/src/ExportHistory/Exception/ExportJobClientValidationException.cs new file mode 100644 index 000000000..c2d34a174 --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobClientValidationException.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when client-side validation fails for export job operations. +/// +public class ExportJobClientValidationException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job that failed validation. + /// The validation error message. + public ExportJobClientValidationException(string jobId, string message) + : base($"Validation failed for export job '{jobId}': {message}") + { + this.JobId = jobId; + } + + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job that failed validation. + /// The validation error message. + /// The exception that is the cause of the current exception. + public ExportJobClientValidationException(string jobId, string message, Exception innerException) + : base($"Validation failed for export job '{jobId}': {message}", innerException) + { + this.JobId = jobId; + } + + /// + /// Gets the ID of the export job that failed validation. + /// + public string JobId { get; } +} + diff --git a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs new file mode 100644 index 000000000..13e4174c8 --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when an invalid state transition is attempted on an export job. +/// +public class ExportJobInvalidTransitionException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job on which the invalid transition was attempted. + /// The current status of the export job. + /// The target status that was invalid. + /// The name of the operation that was attempted. + public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName) + : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.") + { + this.JobId = jobId; + this.FromStatus = fromStatus; + this.ToStatus = toStatus; + this.OperationName = operationName; + } + + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export job on which the invalid transition was attempted. + /// The current status of the export job. + /// The target status that was invalid. + /// The name of the operation that was attempted. + /// The exception that is the cause of the current exception. + public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName, Exception innerException) + : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.", innerException) + { + this.JobId = jobId; + this.FromStatus = fromStatus; + this.ToStatus = toStatus; + this.OperationName = operationName; + } + + /// + /// Gets the ID of the export job that encountered the invalid transition. + /// + public string JobId { get; } + + /// + /// Gets the status the export job was transitioning from. + /// + public ExportJobStatus FromStatus { get; } + + /// + /// Gets the invalid target status that was attempted. + /// + public ExportJobStatus ToStatus { get; } + + /// + /// Gets the name of the operation that was attempted. + /// + public string OperationName { get; } +} + diff --git a/src/ExportHistory/Exception/ExportJobNotFoundException.cs b/src/ExportHistory/Exception/ExportJobNotFoundException.cs new file mode 100644 index 000000000..d1c2f9f19 --- /dev/null +++ b/src/ExportHistory/Exception/ExportJobNotFoundException.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Exception thrown when attempting to access a schedule that does not exist. +/// +public class ExportJobNotFoundException : InvalidOperationException +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export history job that was not found. + public ExportJobNotFoundException(string jobId) + : base($"Export history job with ID '{jobId}' was not found.") + { + this.JobId = jobId; + } + + /// + /// Initializes a new instance of the class. + /// + /// The ID of the export history job that was not found. + /// The exception that is the cause of the current exception. + public ExportJobNotFoundException(string jobId, Exception innerException) + : base($"Export history job with ID '{jobId}' was not found.", innerException) + { + this.JobId = jobId; + } + + /// + /// Gets the ID of the export history job that was not found. + /// + public string JobId { get; } +} diff --git a/src/ExportHistory/ExportHistory.csproj b/src/ExportHistory/ExportHistory.csproj new file mode 100644 index 000000000..f80fe4877 --- /dev/null +++ b/src/ExportHistory/ExportHistory.csproj @@ -0,0 +1,26 @@ + + + + net6.0 + Durable Task Export History + true + preview.1 + + + + + + + + + + + + + + + + + + + diff --git a/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs new file mode 100644 index 000000000..99b95624a --- /dev/null +++ b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Extension methods for configuring Durable Task clients to use export history. +/// +public static class DurableTaskClientBuilderExtensions +{ + /// + /// Enables export history support for the client builder with Azure Storage configuration. + /// + /// The client builder to add export history support to. + /// Callback to configure Azure Storage options. Must not be null. + /// The original builder, for call chaining. + public static IDurableTaskClientBuilder UseExportHistory( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder, nameof(builder)); + Check.NotNull(configure, nameof(configure)); + + IServiceCollection services = builder.Services; + + // Register and validate options + services.AddOptions() + .Configure(configure) + .Validate(o => + !string.IsNullOrEmpty(o.ConnectionString) && + !string.IsNullOrEmpty(o.ContainerName), + $"{nameof(ExportHistoryStorageOptions)} must specify both {nameof(ExportHistoryStorageOptions.ConnectionString)} and {nameof(ExportHistoryStorageOptions.ContainerName)}."); + + // Register ExportHistoryClient using validated options + services.AddSingleton(sp => + { + DurableTaskClient durableTaskClient = sp.GetRequiredService(); + ILogger logger = sp.GetRequiredService>(); + ExportHistoryStorageOptions options = sp.GetRequiredService>().Value; + + return new DefaultExportHistoryClient(durableTaskClient, logger, options); + }); + + return builder; + } +} diff --git a/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs b/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs new file mode 100644 index 000000000..3e8a28967 --- /dev/null +++ b/src/ExportHistory/Extension/DurableTaskWorkerBuilderExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using Microsoft.DurableTask.Worker; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Extension methods for configuring Durable Task workers to use the Azure Durable Task Scheduler service. +/// +public static class DurableTaskWorkerBuilderExtensions +{ + /// + /// Adds export history support to the worker builder. + /// + /// The worker builder to add export history support to. + public static void UseExportHistory(this IDurableTaskWorkerBuilder builder) + { + builder.AddTasks(r => + { + r.AddEntity(); + r.AddOrchestrator(); + r.AddOrchestrator(); + r.AddActivity(); + r.AddActivity(); + }); + } +} diff --git a/src/ExportHistory/Logging/Logs.Client.cs b/src/ExportHistory/Logging/Logs.Client.cs new file mode 100644 index 000000000..87f98bf1b --- /dev/null +++ b/src/ExportHistory/Logging/Logs.Client.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Log messages. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 80, Level = LogLevel.Information, Message = "Creating export job with options: {exportJobCreationOptions}")] + public static partial void ClientCreatingExportJob(this ILogger logger, ExportJobCreationOptions exportJobCreationOptions); + + + [LoggerMessage(EventId = 84, Level = LogLevel.Information, Message = "Deleting export job '{jobId}'")] + public static partial void ClientDeletingExportJob(this ILogger logger, string jobId); + + [LoggerMessage(EventId = 87, Level = LogLevel.Error, Message = "{message} (JobId: {jobId})")] + public static partial void ClientError(this ILogger logger, string message, string jobId, Exception? exception = null); +} diff --git a/src/ExportHistory/Logging/Logs.Entity.cs b/src/ExportHistory/Logging/Logs.Entity.cs new file mode 100644 index 000000000..1dd0d8257 --- /dev/null +++ b/src/ExportHistory/Logging/Logs.Entity.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Log messages. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 101, Level = LogLevel.Information, Message = "Export job '{jobId}' is created")] + public static partial void CreatedExportJob(this ILogger logger, string jobId); + + [LoggerMessage(EventId = 113, Level = LogLevel.Information, Message = "Export job '{jobId}' operation '{operationName}' info: {infoMessage}")] + public static partial void ExportJobOperationInfo(this ILogger logger, string jobId, string operationName, string infoMessage); + + [LoggerMessage(EventId = 114, Level = LogLevel.Warning, Message = "Export job '{jobId}' operation '{operationName}' warning: {warningMessage}")] + public static partial void ExportJobOperationWarning(this ILogger logger, string jobId, string operationName, string warningMessage); + + [LoggerMessage(EventId = 115, Level = LogLevel.Error, Message = "Operation '{operationName}' failed for export job '{jobId}': {errorMessage}")] + public static partial void ExportJobOperationError(this ILogger logger, string jobId, string operationName, string errorMessage, Exception? exception = null); +} diff --git a/src/ExportHistory/Models/CommitCheckpointRequest.cs b/src/ExportHistory/Models/CommitCheckpointRequest.cs new file mode 100644 index 000000000..81745f5d6 --- /dev/null +++ b/src/ExportHistory/Models/CommitCheckpointRequest.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Request to commit a checkpoint with progress updates and failures. +/// +public sealed class CommitCheckpointRequest +{ + /// + /// Gets or sets the number of instances scanned in this batch. + /// + public long ScannedInstances { get; set; } + + /// + /// Gets or sets the number of instances successfully exported in this batch. + /// + public long ExportedInstances { get; set; } + + /// + /// Gets or sets the checkpoint to commit. If not null, the checkpoint is updated (cursor moves forward). + /// If null, the current checkpoint is kept (cursor does not move forward), allowing retry of the same batch. + /// + public ExportCheckpoint? Checkpoint { get; set; } + + /// + /// Gets or sets the list of failed instance exports, if any. + /// + public List? Failures { get; set; } +} + diff --git a/src/ExportHistory/Models/ExportCheckpoint.cs b/src/ExportHistory/Models/ExportCheckpoint.cs new file mode 100644 index 000000000..2e404f2ca --- /dev/null +++ b/src/ExportHistory/Models/ExportCheckpoint.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Checkpoint information used to resume export. +/// +public sealed class ExportCheckpoint +{ + /// + /// Gets or sets the last terminal time processed. + /// + public DateTimeOffset? LastTerminalTimeProcessed { get; set; } + + /// + /// Gets or sets the last instance ID processed. + /// + public string? LastInstanceIdProcessed { get; set; } + + /// + /// Gets or sets the continuation token for pagination. + /// + public string? ContinuationToken { get; set; } +} + diff --git a/src/ExportHistory/Models/ExportDestination.cs b/src/ExportHistory/Models/ExportDestination.cs new file mode 100644 index 000000000..fbab33b5b --- /dev/null +++ b/src/ExportHistory/Models/ExportDestination.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export destination settings for Azure Blob Storage. +/// +public sealed class ExportDestination +{ + /// + /// Initializes a new instance of the class. + /// + /// The blob container name. + /// Thrown when container is null or empty. + public ExportDestination(string container) + { + Check.NotNullOrEmpty(container, nameof(container)); + this.Container = container; + } + + /// + /// Gets the blob container name. + /// + public string Container { get; } + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } +} + diff --git a/src/ExportHistory/Models/ExportFailure.cs b/src/ExportHistory/Models/ExportFailure.cs new file mode 100644 index 000000000..bd753a0dd --- /dev/null +++ b/src/ExportHistory/Models/ExportFailure.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Failure of a specific instance export. +/// +/// The instance ID that failed to export. +/// The reason for the failure. +/// The number of attempts made. +/// The timestamp of the last attempt. +public sealed record ExportFailure(string InstanceId, string Reason, int AttemptCount, DateTimeOffset LastAttempt); + diff --git a/src/ExportHistory/Models/ExportFilter.cs b/src/ExportHistory/Models/ExportFilter.cs new file mode 100644 index 000000000..c06dd540c --- /dev/null +++ b/src/ExportHistory/Models/ExportFilter.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +public record ExportFilter( + DateTimeOffset CreatedTimeFrom, + DateTimeOffset? CreatedTimeTo = null, + IEnumerable? RuntimeStatus = null); \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs new file mode 100644 index 000000000..778049ca1 --- /dev/null +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +public record ExportFormat( + string Kind = "jsonl", + string SchemaVersion = "1.0"); diff --git a/src/ExportHistory/Models/ExportJobConfiguration.cs b/src/ExportHistory/Models/ExportJobConfiguration.cs new file mode 100644 index 000000000..42f7ac406 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobConfiguration.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +public record ExportJobConfiguration( + ExportMode Mode, + ExportFilter Filter, + ExportDestination Destination, + ExportFormat Format, + int MaxParallelExports = 32, + int MaxInstancesPerBatch = 100); \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs new file mode 100644 index 000000000..dd465d857 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Configuration for a export job. +/// +public record ExportJobCreationOptions +{ + /// + /// Initializes a new instance of the class. + /// + /// The export mode (Batch or Continuous). + /// The start time for the export (inclusive). Required. + /// The end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// The export destination where exported data will be stored. Required unless default storage is configured. + /// The unique identifier for the export job. If not provided, a GUID will be generated. + /// The export format settings. Optional, defaults to jsonl-gzip. + /// The orchestration runtime statuses to filter by. Optional. + /// The maximum number of instances to fetch per batch. Optional, defaults to 100. + /// Thrown when validation fails. + public ExportJobCreationOptions( + ExportMode mode, + DateTimeOffset createdTimeFrom, + DateTimeOffset? createdTimeTo, + ExportDestination? destination, + string? jobId = null, + ExportFormat? format = null, + List? runtimeStatus = null, + int? maxInstancesPerBatch = null) + { + // Generate GUID if jobId not provided + this.JobId = string.IsNullOrEmpty(jobId) ? Guid.NewGuid().ToString("N") : jobId; + + if (mode == ExportMode.Batch && !createdTimeTo.HasValue) + { + throw new ArgumentException( + "CreatedTimeTo is required for Batch export mode. For Continuous mode, CreatedTimeTo must be null.", + nameof(createdTimeTo)); + } + + if (mode == ExportMode.Continuous && createdTimeTo.HasValue) + { + throw new ArgumentException( + "CreatedTimeTo must be null for Continuous export mode. For Batch mode, CreatedTimeTo is required.", + nameof(createdTimeTo)); + } + + // Validate terminal status-only filter here if provided + if (runtimeStatus?.Any() == true && + runtimeStatus.Any(s => s is not (OrchestrationRuntimeStatus.Completed or OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated or OrchestrationRuntimeStatus.ContinuedAsNew))) + { + throw new ArgumentException( + "Export supports terminal orchestration statuses only. Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew.", + nameof(runtimeStatus)); + } + + this.Mode = mode; + this.CreatedTimeFrom = createdTimeFrom; + this.CreatedTimeTo = createdTimeTo; + this.Destination = destination; + this.Format = format ?? new ExportFormat(); + this.RuntimeStatus = runtimeStatus; + this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; + } + + /// + /// Gets the unique identifier for the export job. + /// + public string JobId { get; init; } + + /// + /// Gets the export mode (Batch or Continuous). + /// + public ExportMode Mode { get; init; } + + /// + /// Gets the start time for the export (inclusive). Required. + /// + public DateTimeOffset CreatedTimeFrom { get; init; } + + /// + /// Gets the end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// + public DateTimeOffset? CreatedTimeTo { get; init; } + + /// + /// Gets the export destination where exported data will be stored. Optional. + /// + public ExportDestination? Destination { get; init; } + + /// + /// Gets the export format settings. + /// + public ExportFormat Format { get; init; } + + /// + /// Gets the orchestration runtime statuses to filter by. + /// If not specified, all terminal statuses are exported. + /// + public List? RuntimeStatus { get; init; } + + /// + /// Gets the maximum number of instances to fetch per batch. + /// Defaults to 100. + /// + public int MaxInstancesPerBatch { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobDescription.cs b/src/ExportHistory/Models/ExportJobDescription.cs new file mode 100644 index 000000000..bd1e2d4c1 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobDescription.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents the comprehensive details of an export job. +/// +public record ExportJobDescription +{ + /// + /// Gets or sets the job identifier. + /// + public string JobId { get; init; } = string.Empty; + + /// + /// Gets or sets the export job status. + /// + public ExportJobStatus Status { get; init; } + + /// + /// Gets or sets the time when this export job was created. + /// + public DateTimeOffset? CreatedAt { get; init; } + + /// + /// Gets or sets the time when this export job was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; init; } + + /// + /// Gets or sets the export job configuration. + /// + public ExportJobConfiguration? Config { get; init; } + + /// + /// Gets or sets the instance ID of the running export orchestrator, if any. + /// + public string? OrchestratorInstanceId { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobQuery.cs b/src/ExportHistory/Models/ExportJobQuery.cs new file mode 100644 index 000000000..8472ce3f1 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobQuery.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents query parameters for filtering export history jobs. +/// +public record ExportJobQuery +{ + /// + /// The default page size when not supplied. + /// + public const int DefaultPageSize = 100; + + /// + /// Gets the filter for the export history job status. + /// + public ExportJobStatus? Status { get; init; } + + /// + /// Gets the prefix to filter export history job IDs. + /// + public string? JobIdPrefix { get; init; } + + /// + /// Gets the filter for export history jobs created after this time. + /// + public DateTimeOffset? CreatedFrom { get; init; } + + /// + /// Gets the filter for export history jobs created before this time. + /// + public DateTimeOffset? CreatedTo { get; init; } + + /// + /// Gets the maximum number of export history jobs to return per page. + /// + public int? PageSize { get; init; } + + /// + /// Gets the continuation token for retrieving the next page of results. + /// + public string? ContinuationToken { get; init; } +} diff --git a/src/ExportHistory/Models/ExportJobState.cs b/src/ExportHistory/Models/ExportJobState.cs new file mode 100644 index 000000000..f1ea294a3 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobState.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export job state stored in the entity. +/// +public sealed class ExportJobState +{ + /// + /// Gets or sets the current status of the export job. + /// + public ExportJobStatus Status { get; set; } + + /// + /// Gets or sets the export job configuration. + /// + public ExportJobConfiguration? Config { get; set; } + + /// + /// Gets or sets the checkpoint for resuming the export. + /// + public ExportCheckpoint? Checkpoint { get; set; } + + /// + /// Gets or sets the time when the export job was created. + /// + public DateTimeOffset? CreatedAt { get; set; } + + /// + /// Gets or sets the time when the export job was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; set; } + + /// + /// Gets or sets the time of the last checkpoint. + /// + public DateTimeOffset? LastCheckpointTime { get; set; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; set; } + + /// + /// Gets or sets the total number of instances scanned. + /// + public long ScannedInstances { get; set; } + + /// + /// Gets or sets the total number of instances exported. + /// + public long ExportedInstances { get; set; } + + /// + /// Gets or sets the dictionary of failed instance exports. + /// + public Dictionary FailedInstances { get; set; } = new(); +} + diff --git a/src/ExportHistory/Models/ExportJobStatus.cs b/src/ExportHistory/Models/ExportJobStatus.cs new file mode 100644 index 000000000..2ff48a68f --- /dev/null +++ b/src/ExportHistory/Models/ExportJobStatus.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Represents the current status of an export history job. +/// +public enum ExportJobStatus +{ + /// + /// Export history job has not been created. + /// + Uninitialized, + + /// + /// Export history job is active and running. + /// + Active, + + /// + /// Export history job failed. + /// + Failed, + + /// + /// Export history job completed. + /// + Completed +} diff --git a/src/ExportHistory/Models/ExportJobTransitions.cs b/src/ExportHistory/Models/ExportJobTransitions.cs new file mode 100644 index 000000000..8e1a24f36 --- /dev/null +++ b/src/ExportHistory/Models/ExportJobTransitions.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Manages valid state transitions for export jobs. +/// +static class ExportJobTransitions +{ + /// + /// Checks if a transition to the target state is valid for a given export job state and operation. + /// + /// The name of the operation being performed. + /// The current export job state. + /// The target state to transition to. + /// True if the transition is valid; otherwise, false. + public static bool IsValidTransition(string operationName, ExportJobStatus from, ExportJobStatus targetState) + { + return operationName switch + { + nameof(ExportJob.Create) => from switch + { + ExportJobStatus.Uninitialized when targetState == ExportJobStatus.Active => true, + ExportJobStatus.Failed when targetState == ExportJobStatus.Active => true, + ExportJobStatus.Completed when targetState == ExportJobStatus.Active => true, + _ => false, + }, + nameof(ExportJob.MarkAsCompleted) => from switch + { + ExportJobStatus.Active when targetState == ExportJobStatus.Completed => true, + _ => false, + }, + nameof(ExportJob.MarkAsFailed) => from switch + { + ExportJobStatus.Active when targetState == ExportJobStatus.Failed => true, + _ => false, + }, + _ => false, + }; + } +} + diff --git a/src/ExportHistory/Models/ExportMode.cs b/src/ExportHistory/Models/ExportMode.cs new file mode 100644 index 000000000..2e3ee0141 --- /dev/null +++ b/src/ExportHistory/Models/ExportMode.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Export job modes. +/// +public enum ExportMode +{ + /// Exports a fixed window and completes. + Batch = 1, + /// Tails terminal instances continuously. + Continuous = 2, +} \ No newline at end of file diff --git a/src/ExportHistory/Options/ExportHistoryStorageOptions.cs b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs new file mode 100644 index 000000000..21d838187 --- /dev/null +++ b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Options for Azure Storage configuration for export history jobs. +/// Supports connection string-based authentication. +/// +public sealed class ExportHistoryStorageOptions +{ + /// + /// Gets or sets the Azure Storage connection string to the customer's storage account. + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Gets or sets the blob container name where export data will be stored. + /// + public string ContainerName { get; set; } = string.Empty; + + /// + /// Gets or sets an optional prefix for blob paths. + /// + public string? Prefix { get; set; } +} + diff --git a/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs new file mode 100644 index 000000000..a860aa516 --- /dev/null +++ b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Orchestrator that executes operations on export job entities. +/// Calls the specified operation on the target entity and returns the result. +/// +[DurableTask] +public class ExecuteExportJobOperationOrchestrator : TaskOrchestrator +{ + /// + public override async Task RunAsync(TaskOrchestrationContext context, ExportJobOperationRequest input) + { + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + } +} + +/// +/// Request for executing a export job operation. +/// +/// The ID of the entity to execute the operation on. +/// The name of the operation to execute. +/// Optional input for the operation. +public record ExportJobOperationRequest(EntityInstanceId EntityId, string OperationName, object? Input = null); diff --git a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs new file mode 100644 index 000000000..356fbacab --- /dev/null +++ b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs @@ -0,0 +1,334 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.ExportHistory; + +/// +/// Orchestrator input to start a runner for a given job. +/// +public sealed record ExportJobRunRequest(EntityInstanceId JobEntityId); + +/// +/// Orchestrator that performs the actual export work by querying orchestration instances +/// and exporting their history to blob storage. +/// +[DurableTask] +public class ExportJobOrchestrator : TaskOrchestrator +{ + readonly ILogger logger; + const int MaxRetryAttempts = 3; + const int MinBackoffSeconds = 60; // 1 minute + const int MaxBackoffSeconds = 300; // 5 minutes + + /// + /// Initializes a new instance of the class. + /// + public ExportJobOrchestrator(ILogger logger) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public override async Task RunAsync(TaskOrchestrationContext context, ExportJobRunRequest input) + { + string jobId = input.JobEntityId.Key; + this.logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator started"); + + try + { + // Get the export job state and configuration from the entity + ExportJobState? jobState = await context.Entities.CallEntityAsync( + input.JobEntityId, + ExportJobOperations.Get, + null); + + if (jobState == null || jobState.Config == null) + { + throw new InvalidOperationException($"Export job '{jobId}' not found or has no configuration."); + } + + // Check if job is still active + if (jobState.Status != ExportJobStatus.Active) + { + this.logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), $"Job status is {jobState.Status}, not Active - orchestrator cancelled"); + return null; + } + + ExportJobConfiguration config = jobState.Config; + + // Process instances in batches using explicit loop state + bool hasMore = true; + while (hasMore) + { + // Check if job is still active (entity might have been deleted or failed) + ExportJobState? currentState = await context.Entities.CallEntityAsync( + input.JobEntityId, + ExportJobOperations.Get, + null); + + if (currentState == null || + currentState.Status != ExportJobStatus.Active) + { + this.logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), "Job is no longer active - orchestrator cancelled"); + hasMore = false; + continue; + } + + // Call activity to list terminal instances with only necessary information + ListTerminalInstancesRequest listRequest = new ListTerminalInstancesRequest( + CreatedTimeFrom: currentState.Config.Filter.CreatedTimeFrom, + CreatedTimeTo: currentState.Config.Filter.CreatedTimeTo, + RuntimeStatus: currentState.Config.Filter.RuntimeStatus, + ContinuationToken: currentState.Checkpoint?.ContinuationToken, + MaxInstancesPerBatch: currentState.Config.MaxInstancesPerBatch); + + InstancePage pageResult = await context.CallActivityAsync( + nameof(ListTerminalInstancesActivity), + listRequest); + + // Handle empty page result - no instances found, treat as end of data + if (pageResult == null || pageResult.InstanceIds.Count == 0) + { + if (config.Mode == ExportMode.Batch) + { + hasMore = false; + continue; + } + await context.CreateTimer(TimeSpan.FromMinutes(5), default); + continue; + } + + List instancesToExport = pageResult.InstanceIds; + long scannedCount = instancesToExport.Count; + + // Process batch with retry logic + BatchExportResult batchResult = await this.ProcessBatchWithRetryAsync( + context, + input.JobEntityId, + instancesToExport, + config); + + // Commit checkpoint based on batch result + if (batchResult.AllSucceeded) + { + // All exports succeeded - commit with checkpoint to move cursor forward + await this.CommitCheckpointAsync( + context, + input.JobEntityId, + scannedInstances: scannedCount, + exportedInstances: batchResult.ExportedCount, + checkpoint: pageResult.NextCheckpoint, + failures: null); + } + else + { + // Batch failed after all retries - commit without checkpoint (don't move cursor), record failures + await this.CommitCheckpointAsync( + context, + input.JobEntityId, + scannedInstances: scannedCount, + exportedInstances: batchResult.ExportedCount, + checkpoint: null, + failures: batchResult.Failures); + + // Job is now marked as failed in the entity, stop processing + hasMore = false; + continue; + } + } + + await this.MarkAsCompletedAsync(context, input.JobEntityId); + + this.logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator completed"); + return null!; + } + catch (Exception ex) + { + this.logger.ExportJobOperationError(jobId, nameof(ExportJobOrchestrator), "Export orchestrator failed", ex); + + await this.MarkAsFailedAsync(context, input.JobEntityId, ex.Message); + + throw; + } + } + + async Task ProcessBatchWithRetryAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + List instanceIds, + ExportJobConfiguration config) + { + string jobId = jobEntityId.Key; + + for (int attempt = 1; attempt <= MaxRetryAttempts; attempt++) + { + this.logger.ExportJobOperationInfo( + jobId, + nameof(ProcessBatchWithRetryAsync), + $"Processing batch of {instanceIds.Count} instances (attempt {attempt}/{MaxRetryAttempts})"); + + // Export all instances in the batch + List results = await this.ExportBatchAsync(context, instanceIds, config); + + // Check if all exports succeeded + List failedResults = results.Where(r => !r.Success).ToList(); + + if (failedResults.Count == 0) + { + // All exports succeeded + int exportedCount = results.Count; + this.logger.ExportJobOperationInfo( + jobId, + nameof(ProcessBatchWithRetryAsync), + $"Batch export succeeded on attempt {attempt} - exported {exportedCount} instances"); + + return new BatchExportResult + { + AllSucceeded = true, + ExportedCount = exportedCount, + Failures = null, + }; + } + + // Some exports failed + this.logger.ExportJobOperationWarning( + jobId, + nameof(ProcessBatchWithRetryAsync), + $"Batch export failed on attempt {attempt} - {failedResults.Count} failures out of {instanceIds.Count} instances"); + + // If this is the last attempt, return failures + if (attempt == MaxRetryAttempts) + { + List failures = failedResults.Select(r => new ExportFailure( + InstanceId: r.InstanceId, + Reason: r.Error ?? "Unknown error", + AttemptCount: attempt, + LastAttempt: DateTimeOffset.UtcNow)).ToList(); + + int exportedCount = results.Count(r => r.Success); + + return new BatchExportResult + { + AllSucceeded = false, + ExportedCount = exportedCount, + Failures = failures, + }; + } + + // Calculate exponential backoff: 1min, 2min, 4min (capped at 5min) + int backoffSeconds = Math.Min(MinBackoffSeconds * (int)Math.Pow(2, attempt - 1), MaxBackoffSeconds); + TimeSpan backoffDelay = TimeSpan.FromSeconds(backoffSeconds); + + this.logger.ExportJobOperationInfo( + jobId, + nameof(ProcessBatchWithRetryAsync), + $"Retrying batch export after {backoffDelay.TotalMinutes:F1} minutes (attempt {attempt + 1}/{MaxRetryAttempts})"); + + // Wait before retrying + await context.CreateTimer(backoffDelay, default); + } + + // Should not reach here, but return empty result if we do + return new BatchExportResult + { + AllSucceeded = false, + ExportedCount = 0, + Failures = new List(), + }; + } + + async Task> ExportBatchAsync( + TaskOrchestrationContext context, + List instanceIds, + ExportJobConfiguration config) + { + List results = new(); + List> exportTasks = new(); + + foreach (string instanceId in instanceIds) + { + // Create export request with destination and format + ExportRequest exportRequest = new ExportRequest + { + InstanceId = instanceId, + Destination = config.Destination, + Format = config.Format, + }; + + exportTasks.Add( + context.CallActivityAsync( + nameof(ExportInstanceHistoryActivity), + exportRequest)); + + // Limit parallel export activities + if (exportTasks.Count >= config.MaxParallelExports) + { + ExportResult[] batchResults = await Task.WhenAll(exportTasks); + results.AddRange(batchResults); + exportTasks.Clear(); + } + } + + // Wait for remaining export activities + if (exportTasks.Count > 0) + { + ExportResult[] batchResults = await Task.WhenAll(exportTasks); + results.AddRange(batchResults); + } + + return results; + } + + async Task CommitCheckpointAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + long scannedInstances, + long exportedInstances, + ExportCheckpoint? checkpoint, + List? failures) + { + CommitCheckpointRequest request = new CommitCheckpointRequest + { + ScannedInstances = scannedInstances, + ExportedInstances = exportedInstances, + Checkpoint = checkpoint, + Failures = failures, + }; + + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.CommitCheckpoint), + request); + } + + async Task MarkAsCompletedAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId) + { + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.MarkAsCompleted), + null); + } + + async Task MarkAsFailedAsync( + TaskOrchestrationContext context, + EntityInstanceId jobEntityId, + string? errorMessage) + { + await context.Entities.CallEntityAsync( + jobEntityId, + nameof(ExportJob.MarkAsFailed), + errorMessage); + } + + sealed class BatchExportResult + { + public bool AllSucceeded { get; set; } + public int ExportedCount { get; set; } + public List? Failures { get; set; } + } +} diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index b2def0878..e99d94e41 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -466,6 +466,23 @@ message QueryInstancesResponse { google.protobuf.StringValue continuationToken = 2; } +message ListTerminalInstancesRequest { + TerminalInstanceQuery query = 1; +} + +message TerminalInstanceQuery { + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp createdTimeFrom = 2; + google.protobuf.Timestamp createdTimeTo = 3; + int32 maxInstanceCount = 4; + google.protobuf.StringValue continuationToken = 5; // instanceId to continue from +} + +message ListTerminalInstancesResponse { + repeated string instanceIds = 1; + google.protobuf.StringValue continuationToken = 2; // last instanceId for next page +} + message PurgeInstancesRequest { oneof request { string instanceId = 1; @@ -683,14 +700,14 @@ message AbandonEntityTaskResponse { } message SkipGracefulOrchestrationTerminationsRequest { - InstanceBatch instanceBatch = 1; - google.protobuf.StringValue reason = 2; + InstanceBatch instanceBatch = 1; + google.protobuf.StringValue reason = 2; } message SkipGracefulOrchestrationTerminationsResponse { // Those instances which could not be terminated because they had locked entities at the time of this termination call, // are already in a terminal state (completed, failed, terminated, etc.), are not orchestrations, or do not exist (i.e. have been purged) - repeated string unterminatedInstanceIds = 1; + repeated string unterminatedInstanceIds = 1; } service TaskHubSidecarService { @@ -730,6 +747,7 @@ service TaskHubSidecarService { // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); + rpc ListTerminalInstances(ListTerminalInstancesRequest) returns (ListTerminalInstancesResponse); rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); diff --git a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs index b25177ae9..6e3bdde39 100644 --- a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System.Collections.Concurrent; @@ -327,6 +327,93 @@ await this.client.ForceTerminateTaskOrchestrationAsync( } } + /// + /// Lists terminal orchestration instances sorted by completed timestamp, returning only instance IDs. + /// + /// The list terminal instances request. + /// The server call context. + /// A list terminal instances response. + public override async Task ListTerminalInstances(P.ListTerminalInstancesRequest request, ServerCallContext context) + { + if (this.client is IOrchestrationServiceQueryClient queryClient) + { + // Build query for terminal instances + var query = new OrchestrationQuery + { + RuntimeStatus = request.Query.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList(), + CreatedTimeFrom = request.Query.CreatedTimeFrom?.ToDateTime(), + CreatedTimeTo = request.Query.CreatedTimeTo?.ToDateTime(), + PageSize = request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100, + }; + + // Query all matching instances (we'll filter and sort terminal ones) + // Use a larger page size to ensure we have enough instances after filtering + query.PageSize = request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount * 2 : 200; + OrchestrationQueryResult result = await queryClient.GetOrchestrationWithQueryAsync(query, context.CancellationToken); + + // Filter to only terminal instances and sort by completed timestamp + var terminalInstances = result.OrchestrationState + .Where(state => IsTerminalStatus(state.OrchestrationStatus)) + .OrderBy(state => state.CompletedTime != default ? state.CompletedTime : state.LastUpdatedTime) // Sort by completed timestamp first + .ThenBy(state => state.OrchestrationInstance.InstanceId) // Secondary sort by instanceId for stable ordering + .ToList(); + + // Apply continuation token filter after sorting + if (!string.IsNullOrEmpty(request.Query.ContinuationToken)) + { + // Find the position of the continuation token instanceId in the sorted list + // and skip all instances up to and including it + int continuationIndex = terminalInstances.FindIndex( + state => state.OrchestrationInstance.InstanceId == request.Query.ContinuationToken); + + if (continuationIndex >= 0) + { + // Skip the continuation token instance and all instances before it + terminalInstances = terminalInstances.Skip(continuationIndex + 1).ToList(); + } + else + { + // If continuation token not found, skip instances with instanceId <= continuation token + // This handles the case where the continuation token instance might have been deleted + terminalInstances = terminalInstances + .Where(state => string.Compare(state.OrchestrationInstance.InstanceId, request.Query.ContinuationToken, StringComparison.Ordinal) > 0) + .ToList(); + } + } + + // Take the requested page size + terminalInstances = terminalInstances + .Take(request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100) + .ToList(); + + var response = new P.ListTerminalInstancesResponse(); + foreach (var state in terminalInstances) + { + response.InstanceIds.Add(state.OrchestrationInstance.InstanceId); + } + + // Set continuation token to last instanceId if we have results + if (terminalInstances.Count() > 0 && terminalInstances.Count() == (request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100)) + { + response.ContinuationToken = terminalInstances.Last().OrchestrationInstance.InstanceId; + } + + return response; + } + else + { + throw new NotSupportedException($"{this.client.GetType().Name} doesn't support query operations."); + } + } + + static bool IsTerminalStatus(OrchestrationStatus status) + { + return status == OrchestrationStatus.Completed || + status == OrchestrationStatus.Failed || + status == OrchestrationStatus.Terminated || + status == OrchestrationStatus.Canceled; + } + /// /// Purges orchestration instances. /// From 13c0dcbf0835923a974c1ad55600019224496ee5 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:34:38 -0800 Subject: [PATCH 2/7] get history events --- .../ExportJobController.cs | 9 +- src/Client/Core/DurableTaskClient.cs | 928 +++++++++--------- src/Client/Grpc/GrpcDurableTaskClient.cs | 103 +- .../ExportInstanceHistoryActivity.cs | 76 +- .../ListTerminalInstancesActivity.cs | 56 +- .../Client/DefaultExportHistoryClient.cs | 7 +- .../Client/DefaultExportHistoryJobClient.cs | 120 +-- .../Constants/ExportHistoryConstants.cs | 10 - src/ExportHistory/Entity/ExportJob.cs | 15 +- src/ExportHistory/Models/ExportCheckpoint.cs | 21 +- src/ExportHistory/Models/ExportFormat.cs | 12 + .../Models/ExportJobCreationOptions.cs | 41 +- src/ExportHistory/Models/ExportJobState.cs | 4 +- .../Orchestrations/ExportJobOrchestrator.cs | 45 +- src/Grpc/orchestrator_service.proto | 18 +- .../Sidecar/Grpc/TaskHubGrpcServer.cs | 87 +- 16 files changed, 709 insertions(+), 843 deletions(-) diff --git a/samples/ExportHistoryWebApp/ExportJobController.cs b/samples/ExportHistoryWebApp/ExportJobController.cs index 34c510b4d..1bc69a2ce 100644 --- a/samples/ExportHistoryWebApp/ExportJobController.cs +++ b/samples/ExportHistoryWebApp/ExportJobController.cs @@ -131,7 +131,14 @@ public async Task>> ListExportJob try { ExportJobQuery? query = null; - if (status.HasValue || !string.IsNullOrEmpty(jobIdPrefix) || createdFrom.HasValue || createdTo.HasValue || pageSize.HasValue || !string.IsNullOrEmpty(continuationToken)) + if ( + status.HasValue || + !string.IsNullOrEmpty(jobIdPrefix) || + createdFrom.HasValue || + createdTo.HasValue || + pageSize.HasValue || + !string.IsNullOrEmpty(continuationToken) + ) { query = new ExportJobQuery { diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index f2d658d8f..d2cf94c84 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -1,441 +1,443 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.ComponentModel; -using Microsoft.DurableTask.Client.Entities; -using Microsoft.DurableTask.Internal; - -namespace Microsoft.DurableTask.Client; - -/// -/// Base class that defines client operations for managing durable task instances. -/// -/// -/// -/// Instances of can be used to start, query, raise events to, and terminate -/// orchestration instances. In most cases, methods on this class accept an instance ID as a parameter, which identifies -/// the orchestration instance. -/// -/// At the time of writing, the most common implementation of this class is the gRPC client, which works by making gRPC -/// calls to a remote service (e.g. a sidecar) that implements the operation behavior. To ensure any owned network -/// resources are properly released, instances of should be disposed when they are no -/// longer needed. -/// -/// Instances of this class are expected to be safe for multithreaded apps. You can therefore safely cache instances -/// of this class and reuse them across multiple contexts. Caching these objects is useful to improve overall -/// performance. -/// -/// -public abstract class DurableTaskClient : IOrchestrationSubmitter, IAsyncDisposable -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the client. - protected DurableTaskClient(string name) - { - this.Name = name; - } - - /// - /// Gets the name of the client. - /// - public string Name { get; } - - /// - /// Gets the for interacting with durable entities. - /// - /// - /// Not all clients support durable entities. Refer to a specific client implementation for verifying support. - /// - public virtual DurableEntityClient Entities => - throw new NotSupportedException($"{this.GetType()} does not support durable entities."); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, CancellationToken cancellation) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, null, cancellation); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, object? input, CancellationToken cancellation) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input, null, cancellation); - - /// - public virtual Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, StartOrchestrationOptions options, CancellationToken cancellation = default) - => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, options, cancellation); - - /// - /// Schedules a new orchestration instance for execution. - /// - /// - /// All orchestrations must have a unique instance ID. You can provide an instance ID using the - /// parameter or you can omit this and a random instance ID will be - /// generated for you automatically. If an orchestration with the specified instance ID already exists and is in a - /// non-terminal state (Pending, Running, etc.), then this operation may fail silently. However, if an orchestration - /// instance with this ID already exists in a terminal state (Completed, Terminated, Failed, etc.) then the instance - /// may be recreated automatically, depending on the configuration of the backend instance store. - /// - /// Orchestration instances started with this method will be created in the - /// state and will transition to the - /// after successfully awaiting its first task. The exact time it - /// takes before a scheduled orchestration starts running depends on several factors, including the configuration - /// and health of the backend task hub, and whether a start time was provided via . - /// - /// The task associated with this method completes after the orchestration instance was successfully scheduled. You - /// can use the to query the status of the - /// scheduled instance, the method to wait - /// for the instance to transition out of the status, or the - /// method to wait for the instance to - /// reach a terminal state (Completed, Terminated, Failed, etc.). - /// - /// - /// The name of the orchestrator to schedule. - /// - /// The optional input to pass to the scheduled orchestration instance. This must be a serializable value. - /// - /// The options to start the new orchestration with. - /// - /// The cancellation token. This only cancels enqueueing the new orchestration to the backend. Does not cancel the - /// orchestration once enqueued. - /// - /// - /// A task that completes when the orchestration instance is successfully scheduled. The value of this task is - /// the instance ID of the scheduled orchestration instance. If a non-null instance ID was provided via - /// , the same value will be returned by the completed task. - /// - /// Thrown if is empty. - public abstract Task ScheduleNewOrchestrationInstanceAsync( - TaskName orchestratorName, - object? input = null, - StartOrchestrationOptions? options = null, - CancellationToken cancellation = default); - - /// - public virtual Task RaiseEventAsync( - string instanceId, string eventName, CancellationToken cancellation) - => this.RaiseEventAsync(instanceId, eventName, null, cancellation); - - /// - /// Sends an event notification message to a waiting orchestration instance. - /// - /// - /// - /// In order to handle the event, the target orchestration instance must be waiting for an - /// event named using the - /// API. - /// If the target orchestration instance is not yet waiting for an event named , - /// then the event will be saved in the orchestration instance state and dispatched immediately when the - /// orchestrator calls . - /// This event saving occurs even if the orchestrator has canceled its wait operation before the event was received. - /// - /// Orchestrators can wait for the same event name multiple times, so sending multiple events with the same name is - /// allowed. Each external event received by an orchestrator will complete just one task returned by the - /// method. - /// - /// Raised events for a completed or non-existent orchestration instance will be silently discarded. - /// - /// - /// The ID of the orchestration instance that will handle the event. - /// The name of the event. Event names are case-insensitive. - /// The serializable data payload to include with the event. - /// - /// The cancellation token. This only cancels enqueueing the event to the backend. Does not abort sending the event - /// once enqueued. - /// - /// A task that completes when the event notification message has been enqueued. - /// - /// Thrown if or is null or empty. - /// - public abstract Task RaiseEventAsync( - string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default); - - /// - public virtual Task WaitForInstanceStartAsync( - string instanceId, CancellationToken cancellation) - => this.WaitForInstanceStartAsync(instanceId, false, cancellation); - - /// - /// Waits for an orchestration to start running and returns a - /// object that contains metadata about the started instance. - /// - /// - /// - /// A "started" orchestration instance is any instance not in the - /// state. - /// - /// If an orchestration instance is already running when this method is called, the method will return immediately. - /// - /// - /// The unique ID of the orchestration instance to wait for. - /// - /// Specify true to fetch the orchestration instance's inputs, outputs, and custom status, or false to - /// omit them. The default value is false to minimize the network bandwidth, serialization, and memory costs - /// associated with fetching the instance metadata. - /// - /// A that can be used to cancel the wait operation. - /// - /// Returns a record that describes the orchestration instance and its execution - /// status or null if no instance with ID is found. - /// - public abstract Task WaitForInstanceStartAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - public virtual Task WaitForInstanceCompletionAsync( - string instanceId, CancellationToken cancellation) - => this.WaitForInstanceCompletionAsync(instanceId, false, cancellation); - - /// - /// Waits for an orchestration to complete and returns a - /// object that contains metadata about the started instance. - /// - /// - /// - /// A "completed" orchestration instance is any instance in one of the terminal states. For example, the - /// , , or - /// states. - /// - /// Orchestrations are long-running and could take hours, days, or months before completing. - /// Orchestrations can also be eternal, in which case they'll never complete unless terminated. - /// In such cases, this call may block indefinitely, so care must be taken to ensure appropriate timeouts are - /// enforced using the parameter. - /// - /// If an orchestration instance is already complete when this method is called, the method will return immediately. - /// - /// - /// - public abstract Task WaitForInstanceCompletionAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - public virtual Task TerminateInstanceAsync(string instanceId, CancellationToken cancellation) - => this.TerminateInstanceAsync(instanceId, null, cancellation); - - /// - public virtual Task TerminateInstanceAsync(string instanceId, object? output, CancellationToken cancellation = default) - { - TerminateInstanceOptions? options = output is null ? null : new() { Output = output }; - return this.TerminateInstanceAsync(instanceId, options, cancellation); - } - - /// - /// Terminates an orchestration instance and updates its runtime status to - /// . - /// - /// - /// - /// This method internally enqueues a "terminate" message in the task hub. When the task hub worker processes - /// this message, it will update the runtime status of the target instance to - /// . You can use the - /// to wait for the instance to reach - /// the terminated state. - /// - /// - /// Terminating an orchestration by default will not terminate any of the child sub-orchestrations that were started by - /// the orchetration instance. If you want to terminate sub-orchestration instances as well, you can set - /// flag to true which will enable termination of child sub-orchestration instances. It is set to false by default. - /// Terminating an orchestration instance has no effect on any in-flight activity function executions - /// that were started by the terminated instance. Those actions will continue to run - /// without interruption. However, their results will be discarded. - /// - /// At the time of writing, there is no way to terminate an in-flight activity execution. - /// - /// Attempting to terminate a completed or non-existent orchestration instance will fail silently. - /// - /// - /// The ID of the orchestration instance to terminate. - /// The optional options for terminating the orchestration. - /// - /// The cancellation token. This only cancels enqueueing the termination request to the backend. Does not abort - /// termination of the orchestration once enqueued. - /// - /// A task that completes when the terminate message is enqueued. - public virtual Task TerminateInstanceAsync(string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support orchestration termination."); - - /// - public virtual Task SuspendInstanceAsync(string instanceId, CancellationToken cancellation) - => this.SuspendInstanceAsync(instanceId, null, cancellation); - - /// - /// Suspends an orchestration instance, halting processing of it until - /// is used to resume the orchestration. - /// - /// The instance ID of the orchestration to suspend. - /// The optional suspension reason. - /// - /// A that can be used to cancel the suspend operation. Note, cancelling this token - /// does not resume the orchestration if suspend was successful. - /// - /// A task that completes when the suspend has been committed to the backend. - public abstract Task SuspendInstanceAsync( - string instanceId, string? reason = null, CancellationToken cancellation = default); - - /// - public virtual Task ResumeInstanceAsync(string instanceId, CancellationToken cancellation) - => this.ResumeInstanceAsync(instanceId, null, cancellation); - - /// - /// Resumes an orchestration instance that was suspended via . - /// - /// The instance ID of the orchestration to resume. - /// The optional resume reason. - /// - /// A that can be used to cancel the resume operation. Note, cancelling this token - /// does not re-suspend the orchestration if resume was successful. - /// - /// A task that completes when the resume has been committed to the backend. - public abstract Task ResumeInstanceAsync( - string instanceId, string? reason = null, CancellationToken cancellation = default); - - /// - public virtual Task GetInstanceAsync( - string instanceId, CancellationToken cancellation) - => this.GetInstanceAsync(instanceId, false, cancellation); - - /// - /// Fetches orchestration instance metadata from the configured durable store. - /// - /// - /// You can use the parameter to determine whether to fetch input and - /// output data for the target orchestration instance. If your code doesn't require access to this data, it's - /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and - /// memory costs associated with fetching the instance metadata. - /// - /// - public virtual Task GetInstanceAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) - => this.GetInstancesAsync(instanceId, getInputsAndOutputs, cancellation); - - /// - [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync - public virtual Task GetInstancesAsync( - string instanceId, CancellationToken cancellation) - => this.GetInstancesAsync(instanceId, false, cancellation); - - /// - /// Fetches orchestration instance metadata from the configured durable store. - /// - /// - /// You can use the parameter to determine whether to fetch input and - /// output data for the target orchestration instance. If your code doesn't require access to this data, it's - /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and - /// memory costs associated with fetching the instance metadata. - /// - /// - [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync - public abstract Task GetInstancesAsync( - string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); - - /// - /// Queries orchestration instances. - /// - /// Filters down the instances included in the query. - /// An async pageable of the query results. - public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); - - /// - public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) - => this.PurgeInstanceAsync(instanceId, null, cancellation); - - /// - /// Purges orchestration instance metadata from the durable store. - /// - /// - /// - /// This method can be used to permanently delete orchestration metadata from the underlying storage provider, - /// including any stored inputs, outputs, and orchestration history records. This is often useful for implementing - /// data retention policies and for keeping storage costs minimal. Only orchestration instances in the - /// , , or - /// state can be purged. - /// - /// Purging an orchestration will by default not purge any of the child sub-orchestrations that were started by the - /// orchetration instance. Currently, purging of sub-orchestrations is not supported. - /// If is not found in the data store, or if the instance is found but not in a - /// terminal state, then the returned object will have a - /// value of 0. Otherwise, the existing data will be purged and - /// will be the count of purged instances. - /// - /// - /// The unique ID of the orchestration instance to purge. - /// The optional options for purging the orchestration. - /// - /// A that can be used to cancel the purge operation. - /// - /// - /// This method returns a object after the operation has completed with a - /// indicating the number of orchestration instances that were purged, - /// including the count of sub-orchestrations purged if any. - /// - public virtual Task PurgeInstanceAsync( - string instanceId, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) - { - throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); - } - - /// - public virtual Task PurgeAllInstancesAsync(PurgeInstancesFilter filter, CancellationToken cancellation) - => this.PurgeAllInstancesAsync(filter, null, cancellation); - - /// - /// Purges orchestration instances metadata from the durable store. - /// - /// The filter for which orchestrations to purge. - /// The optional options for purging the orchestration. - /// - /// A that can be used to cancel the purge operation. - /// - /// - /// This method returns a object after the operation has completed with a - /// indicating the number of orchestration instances that were purged. - /// - public virtual Task PurgeAllInstancesAsync( - PurgeInstancesFilter filter, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) - { - throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); - } - - /// - /// Restarts an orchestration instance with the same or a new instance ID. - /// - /// - /// - /// This method restarts an existing orchestration instance. If is true, - /// a new instance ID will be generated for the restarted orchestration. If false, the original instance ID will be reused. - /// - /// The restarted orchestration will use the same input data as the original instance. If the original orchestration - /// instance is not found, an will be thrown. - /// - /// Note that this operation is backend-specific and may not be supported by all durable task backends. - /// If the backend does not support restart operations, a will be thrown. - /// - /// - /// The ID of the orchestration instance to restart. - /// - /// If true, a new instance ID will be generated for the restarted orchestration. - /// If false, the original instance ID will be reused. - /// - /// - /// The cancellation token. This only cancels enqueueing the restart request to the backend. - /// Does not abort restarting the orchestration once enqueued. - /// - /// - /// A task that completes when the orchestration instance is successfully restarted. - /// The value of this task is the instance ID of the restarted orchestration instance. - /// - /// - /// Thrown if an orchestration with the specified was not found. - /// - /// Thrown when attempting to restart an instance using the same instance Id - /// while the instance has not yet reached a completed or terminal state. - /// - /// Thrown if the backend does not support restart operations. - public virtual Task RestartAsync( - string instanceId, - bool restartWithNewInstanceId = false, - CancellationToken cancellation = default) +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.ComponentModel; +using DurableTask.Core.History; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Internal; + +namespace Microsoft.DurableTask.Client; + +/// +/// Base class that defines client operations for managing durable task instances. +/// +/// +/// +/// Instances of can be used to start, query, raise events to, and terminate +/// orchestration instances. In most cases, methods on this class accept an instance ID as a parameter, which identifies +/// the orchestration instance. +/// +/// At the time of writing, the most common implementation of this class is the gRPC client, which works by making gRPC +/// calls to a remote service (e.g. a sidecar) that implements the operation behavior. To ensure any owned network +/// resources are properly released, instances of should be disposed when they are no +/// longer needed. +/// +/// Instances of this class are expected to be safe for multithreaded apps. You can therefore safely cache instances +/// of this class and reuse them across multiple contexts. Caching these objects is useful to improve overall +/// performance. +/// +/// +public abstract class DurableTaskClient : IOrchestrationSubmitter, IAsyncDisposable +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the client. + protected DurableTaskClient(string name) + { + this.Name = name; + } + + /// + /// Gets the name of the client. + /// + public string Name { get; } + + /// + /// Gets the for interacting with durable entities. + /// + /// + /// Not all clients support durable entities. Refer to a specific client implementation for verifying support. + /// + public virtual DurableEntityClient Entities => + throw new NotSupportedException($"{this.GetType()} does not support durable entities."); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, CancellationToken cancellation) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, null, cancellation); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, object? input, CancellationToken cancellation) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input, null, cancellation); + + /// + public virtual Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, StartOrchestrationOptions options, CancellationToken cancellation = default) + => this.ScheduleNewOrchestrationInstanceAsync(orchestratorName, null, options, cancellation); + + /// + /// Schedules a new orchestration instance for execution. + /// + /// + /// All orchestrations must have a unique instance ID. You can provide an instance ID using the + /// parameter or you can omit this and a random instance ID will be + /// generated for you automatically. If an orchestration with the specified instance ID already exists and is in a + /// non-terminal state (Pending, Running, etc.), then this operation may fail silently. However, if an orchestration + /// instance with this ID already exists in a terminal state (Completed, Terminated, Failed, etc.) then the instance + /// may be recreated automatically, depending on the configuration of the backend instance store. + /// + /// Orchestration instances started with this method will be created in the + /// state and will transition to the + /// after successfully awaiting its first task. The exact time it + /// takes before a scheduled orchestration starts running depends on several factors, including the configuration + /// and health of the backend task hub, and whether a start time was provided via . + /// + /// The task associated with this method completes after the orchestration instance was successfully scheduled. You + /// can use the to query the status of the + /// scheduled instance, the method to wait + /// for the instance to transition out of the status, or the + /// method to wait for the instance to + /// reach a terminal state (Completed, Terminated, Failed, etc.). + /// + /// + /// The name of the orchestrator to schedule. + /// + /// The optional input to pass to the scheduled orchestration instance. This must be a serializable value. + /// + /// The options to start the new orchestration with. + /// + /// The cancellation token. This only cancels enqueueing the new orchestration to the backend. Does not cancel the + /// orchestration once enqueued. + /// + /// + /// A task that completes when the orchestration instance is successfully scheduled. The value of this task is + /// the instance ID of the scheduled orchestration instance. If a non-null instance ID was provided via + /// , the same value will be returned by the completed task. + /// + /// Thrown if is empty. + public abstract Task ScheduleNewOrchestrationInstanceAsync( + TaskName orchestratorName, + object? input = null, + StartOrchestrationOptions? options = null, + CancellationToken cancellation = default); + + /// + public virtual Task RaiseEventAsync( + string instanceId, string eventName, CancellationToken cancellation) + => this.RaiseEventAsync(instanceId, eventName, null, cancellation); + + /// + /// Sends an event notification message to a waiting orchestration instance. + /// + /// + /// + /// In order to handle the event, the target orchestration instance must be waiting for an + /// event named using the + /// API. + /// If the target orchestration instance is not yet waiting for an event named , + /// then the event will be saved in the orchestration instance state and dispatched immediately when the + /// orchestrator calls . + /// This event saving occurs even if the orchestrator has canceled its wait operation before the event was received. + /// + /// Orchestrators can wait for the same event name multiple times, so sending multiple events with the same name is + /// allowed. Each external event received by an orchestrator will complete just one task returned by the + /// method. + /// + /// Raised events for a completed or non-existent orchestration instance will be silently discarded. + /// + /// + /// The ID of the orchestration instance that will handle the event. + /// The name of the event. Event names are case-insensitive. + /// The serializable data payload to include with the event. + /// + /// The cancellation token. This only cancels enqueueing the event to the backend. Does not abort sending the event + /// once enqueued. + /// + /// A task that completes when the event notification message has been enqueued. + /// + /// Thrown if or is null or empty. + /// + public abstract Task RaiseEventAsync( + string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default); + + /// + public virtual Task WaitForInstanceStartAsync( + string instanceId, CancellationToken cancellation) + => this.WaitForInstanceStartAsync(instanceId, false, cancellation); + + /// + /// Waits for an orchestration to start running and returns a + /// object that contains metadata about the started instance. + /// + /// + /// + /// A "started" orchestration instance is any instance not in the + /// state. + /// + /// If an orchestration instance is already running when this method is called, the method will return immediately. + /// + /// + /// The unique ID of the orchestration instance to wait for. + /// + /// Specify true to fetch the orchestration instance's inputs, outputs, and custom status, or false to + /// omit them. The default value is false to minimize the network bandwidth, serialization, and memory costs + /// associated with fetching the instance metadata. + /// + /// A that can be used to cancel the wait operation. + /// + /// Returns a record that describes the orchestration instance and its execution + /// status or null if no instance with ID is found. + /// + public abstract Task WaitForInstanceStartAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + public virtual Task WaitForInstanceCompletionAsync( + string instanceId, CancellationToken cancellation) + => this.WaitForInstanceCompletionAsync(instanceId, false, cancellation); + + /// + /// Waits for an orchestration to complete and returns a + /// object that contains metadata about the started instance. + /// + /// + /// + /// A "completed" orchestration instance is any instance in one of the terminal states. For example, the + /// , , or + /// states. + /// + /// Orchestrations are long-running and could take hours, days, or months before completing. + /// Orchestrations can also be eternal, in which case they'll never complete unless terminated. + /// In such cases, this call may block indefinitely, so care must be taken to ensure appropriate timeouts are + /// enforced using the parameter. + /// + /// If an orchestration instance is already complete when this method is called, the method will return immediately. + /// + /// + /// + public abstract Task WaitForInstanceCompletionAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + public virtual Task TerminateInstanceAsync(string instanceId, CancellationToken cancellation) + => this.TerminateInstanceAsync(instanceId, null, cancellation); + + /// + public virtual Task TerminateInstanceAsync(string instanceId, object? output, CancellationToken cancellation = default) + { + TerminateInstanceOptions? options = output is null ? null : new() { Output = output }; + return this.TerminateInstanceAsync(instanceId, options, cancellation); + } + + /// + /// Terminates an orchestration instance and updates its runtime status to + /// . + /// + /// + /// + /// This method internally enqueues a "terminate" message in the task hub. When the task hub worker processes + /// this message, it will update the runtime status of the target instance to + /// . You can use the + /// to wait for the instance to reach + /// the terminated state. + /// + /// + /// Terminating an orchestration by default will not terminate any of the child sub-orchestrations that were started by + /// the orchetration instance. If you want to terminate sub-orchestration instances as well, you can set + /// flag to true which will enable termination of child sub-orchestration instances. It is set to false by default. + /// Terminating an orchestration instance has no effect on any in-flight activity function executions + /// that were started by the terminated instance. Those actions will continue to run + /// without interruption. However, their results will be discarded. + /// + /// At the time of writing, there is no way to terminate an in-flight activity execution. + /// + /// Attempting to terminate a completed or non-existent orchestration instance will fail silently. + /// + /// + /// The ID of the orchestration instance to terminate. + /// The optional options for terminating the orchestration. + /// + /// The cancellation token. This only cancels enqueueing the termination request to the backend. Does not abort + /// termination of the orchestration once enqueued. + /// + /// A task that completes when the terminate message is enqueued. + public virtual Task TerminateInstanceAsync(string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support orchestration termination."); + + /// + public virtual Task SuspendInstanceAsync(string instanceId, CancellationToken cancellation) + => this.SuspendInstanceAsync(instanceId, null, cancellation); + + /// + /// Suspends an orchestration instance, halting processing of it until + /// is used to resume the orchestration. + /// + /// The instance ID of the orchestration to suspend. + /// The optional suspension reason. + /// + /// A that can be used to cancel the suspend operation. Note, cancelling this token + /// does not resume the orchestration if suspend was successful. + /// + /// A task that completes when the suspend has been committed to the backend. + public abstract Task SuspendInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default); + + /// + public virtual Task ResumeInstanceAsync(string instanceId, CancellationToken cancellation) + => this.ResumeInstanceAsync(instanceId, null, cancellation); + + /// + /// Resumes an orchestration instance that was suspended via . + /// + /// The instance ID of the orchestration to resume. + /// The optional resume reason. + /// + /// A that can be used to cancel the resume operation. Note, cancelling this token + /// does not re-suspend the orchestration if resume was successful. + /// + /// A task that completes when the resume has been committed to the backend. + public abstract Task ResumeInstanceAsync( + string instanceId, string? reason = null, CancellationToken cancellation = default); + + /// + public virtual Task GetInstanceAsync( + string instanceId, CancellationToken cancellation) + => this.GetInstanceAsync(instanceId, false, cancellation); + + /// + /// Fetches orchestration instance metadata from the configured durable store. + /// + /// + /// You can use the parameter to determine whether to fetch input and + /// output data for the target orchestration instance. If your code doesn't require access to this data, it's + /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and + /// memory costs associated with fetching the instance metadata. + /// + /// + public virtual Task GetInstanceAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) + => this.GetInstancesAsync(instanceId, getInputsAndOutputs, cancellation); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync + public virtual Task GetInstancesAsync( + string instanceId, CancellationToken cancellation) + => this.GetInstancesAsync(instanceId, false, cancellation); + + /// + /// Fetches orchestration instance metadata from the configured durable store. + /// + /// + /// You can use the parameter to determine whether to fetch input and + /// output data for the target orchestration instance. If your code doesn't require access to this data, it's + /// recommended that you set this parameter to false to minimize the network bandwidth, serialization, and + /// memory costs associated with fetching the instance metadata. + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] // use GetInstanceAsync + public abstract Task GetInstancesAsync( + string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default); + + /// + /// Queries orchestration instances. + /// + /// Filters down the instances included in the query. + /// An async pageable of the query results. + public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); + + /// + public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) + => this.PurgeInstanceAsync(instanceId, null, cancellation); + + /// + /// Purges orchestration instance metadata from the durable store. + /// + /// + /// + /// This method can be used to permanently delete orchestration metadata from the underlying storage provider, + /// including any stored inputs, outputs, and orchestration history records. This is often useful for implementing + /// data retention policies and for keeping storage costs minimal. Only orchestration instances in the + /// , , or + /// state can be purged. + /// + /// Purging an orchestration will by default not purge any of the child sub-orchestrations that were started by the + /// orchetration instance. Currently, purging of sub-orchestrations is not supported. + /// If is not found in the data store, or if the instance is found but not in a + /// terminal state, then the returned object will have a + /// value of 0. Otherwise, the existing data will be purged and + /// will be the count of purged instances. + /// + /// + /// The unique ID of the orchestration instance to purge. + /// The optional options for purging the orchestration. + /// + /// A that can be used to cancel the purge operation. + /// + /// + /// This method returns a object after the operation has completed with a + /// indicating the number of orchestration instances that were purged, + /// including the count of sub-orchestrations purged if any. + /// + public virtual Task PurgeInstanceAsync( + string instanceId, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); + } + + /// + public virtual Task PurgeAllInstancesAsync(PurgeInstancesFilter filter, CancellationToken cancellation) + => this.PurgeAllInstancesAsync(filter, null, cancellation); + + /// + /// Purges orchestration instances metadata from the durable store. + /// + /// The filter for which orchestrations to purge. + /// The optional options for purging the orchestration. + /// + /// A that can be used to cancel the purge operation. + /// + /// + /// This method returns a object after the operation has completed with a + /// indicating the number of orchestration instances that were purged. + /// + public virtual Task PurgeAllInstancesAsync( + PurgeInstancesFilter filter, PurgeInstanceOptions? options = null, CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support purging of orchestration instances."); + } + + /// + /// Restarts an orchestration instance with the same or a new instance ID. + /// + /// + /// + /// This method restarts an existing orchestration instance. If is true, + /// a new instance ID will be generated for the restarted orchestration. If false, the original instance ID will be reused. + /// + /// The restarted orchestration will use the same input data as the original instance. If the original orchestration + /// instance is not found, an will be thrown. + /// + /// Note that this operation is backend-specific and may not be supported by all durable task backends. + /// If the backend does not support restart operations, a will be thrown. + /// + /// + /// The ID of the orchestration instance to restart. + /// + /// If true, a new instance ID will be generated for the restarted orchestration. + /// If false, the original instance ID will be reused. + /// + /// + /// The cancellation token. This only cancels enqueueing the restart request to the backend. + /// Does not abort restarting the orchestration once enqueued. + /// + /// + /// A task that completes when the orchestration instance is successfully restarted. + /// The value of this task is the instance ID of the restarted orchestration instance. + /// + /// + /// Thrown if an orchestration with the specified was not found. + /// + /// Thrown when attempting to restart an instance using the same instance Id + /// while the instance has not yet reached a completed or terminal state. + /// + /// Thrown if the backend does not support restart operations. + public virtual Task RestartAsync( + string instanceId, + bool restartWithNewInstanceId = false, + CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support orchestration restart."); /// @@ -450,7 +452,7 @@ public virtual Task RestartAsync( /// /// The instance ID of the orchestration to rewind. /// The reason for the rewind. - /// The cancellation token. This only cancels enqueueing the rewind request to the backend. + /// The cancellation token. This only cancels enqueueing the rewind request to the backend. /// It does not abort rewinding the orchestration once the request has been enqueued. /// A task that represents the enqueueing of the rewind operation. /// Thrown if this implementation of does not @@ -464,15 +466,39 @@ public virtual Task RewindInstanceAsync( string instanceId, string reason, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support orchestration rewind."); - - // TODO: Create task hub - - // TODO: Delete task hub - - /// - /// Disposes any unmanaged resources associated with this . - /// - /// A that completes when the disposal completes. - public abstract ValueTask DisposeAsync(); -} + => throw new NotSupportedException($"{this.GetType()} does not support orchestration rewind."); + + /// + /// Streams the execution history events for an orchestration instance. + /// + /// + /// This method returns an async enumerable that yields history events as they are retrieved from the backend. + /// The history events are returned in chronological order and include all events that occurred during the + /// orchestration instance's execution. + /// + /// The instance ID of the orchestration to stream history for. + /// The optional execution ID. If null, the latest execution will be used. + /// + /// A that can be used to cancel the streaming operation. + /// + /// An async enumerable of objects. + /// Thrown if an orchestration with the specified does not exist. + /// Thrown if the operation is canceled via the token. + public virtual IAsyncEnumerable StreamInstanceHistoryAsync( + string instanceId, + string? executionId = null, + CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support streaming instance history."); + } + + // TODO: Create task hub + + // TODO: Delete task hub + + /// + /// Disposes any unmanaged resources associated with this . + /// + /// A that completes when the disposal completes. + public abstract ValueTask DisposeAsync(); +} diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index f42cd2f20..ff106a556 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -3,7 +3,9 @@ using System.Diagnostics; using System.Text; +using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; +using Grpc.Core; using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Tracing; using Microsoft.Extensions.DependencyInjection; @@ -323,55 +325,7 @@ public override async Task WaitForInstanceStartAsync( } } - /// - /// Lists terminal orchestration instances sorted by completed timestamp, returning only instance IDs. - /// - /// Creation date of instances to query from. - /// Creation date of instances to query to. - /// Runtime statuses of instances to query (should be terminal statuses). - /// Maximum number of instance IDs to return per page. - /// The continuation token (instanceId) to continue from. - /// The cancellation token. - /// A tuple containing the list of instance IDs and the continuation token for the next page. - public async Task<(IReadOnlyList InstanceIds, string? ContinuationToken)> ListTerminalInstancesAsync( - DateTimeOffset? createdFrom = null, - DateTimeOffset? createdTo = null, - IEnumerable? statuses = null, - int pageSize = 100, - string? continuationToken = null, - CancellationToken cancellation = default) - { - Check.NotEntity(this.options.EnableEntitySupport, continuationToken); - - P.ListTerminalInstancesRequest request = new() - { - Query = new P.TerminalInstanceQuery - { - CreatedTimeFrom = createdFrom?.ToTimestamp(), - CreatedTimeTo = createdTo?.ToTimestamp(), - MaxInstanceCount = pageSize, - ContinuationToken = continuationToken, - }, - }; - - if (statuses is not null) - { - request.Query.RuntimeStatus.AddRange(statuses.Select(x => x.ToGrpcStatus())); - } - - try - { - P.ListTerminalInstancesResponse response = await this.sidecarClient.ListTerminalInstancesAsync( - request, cancellationToken: cancellation); - - return (response.InstanceIds.ToList(), response.ContinuationToken); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.ListTerminalInstancesAsync)} operation was canceled.", e, cancellation); - } - } + // Removed ListTerminalInstances; use GetAllInstancesAsync with OrchestrationQuery instead /// public override async Task WaitForInstanceCompletionAsync( @@ -518,6 +472,57 @@ public override async Task RewindInstanceAsync( } } + /// + public override async IAsyncEnumerable StreamInstanceHistoryAsync( + string instanceId, + string? executionId = null, + CancellationToken cancellation = default) + { + Check.NotNullOrEmpty(instanceId); + Check.NotEntity(this.options.EnableEntitySupport, instanceId); + + P.StreamInstanceHistoryRequest request = new() + { + InstanceId = instanceId, + ExecutionId = executionId, + ForWorkItemProcessing = false, + }; + + using AsyncServerStreamingCall streamResponse = + this.sidecarClient.StreamInstanceHistory(request, cancellationToken: cancellation); + + IAsyncStreamReader responseStream = streamResponse.ResponseStream; + + bool hasMore = true; + while (hasMore) + { + bool moveNextResult; + try + { + moveNextResult = await responseStream.MoveNext(cancellation).ConfigureAwait(false); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.StreamInstanceHistoryAsync)} operation was canceled.", e, cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.NotFound) + { + throw new ArgumentException($"An orchestration with the instanceId {instanceId} was not found.", e); + } + + hasMore = moveNextResult; + if (hasMore) + { + P.HistoryChunk chunk = responseStream.Current; + foreach (P.HistoryEvent protoEvent in chunk.Events) + { + yield return Microsoft.DurableTask.ProtoUtils.ConvertHistoryEvent(protoEvent); + } + } + } + } + static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, out CallInvoker callInvoker) { if (options.Channel is GrpcChannel c) diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs index 2a34c0508..2056a509d 100644 --- a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -5,8 +5,10 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using DurableTask.Core.History; using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -52,6 +54,7 @@ public override async Task RunAsync(TaskActivityContext context, E // Get the client and instance metadata with inputs and outputs DurableTaskClient client = this.clientProvider.GetClient(); + OrchestrationMetadata? metadata = await client.GetInstanceAsync( instanceId, getInputsAndOutputs: true, @@ -83,6 +86,22 @@ public override async Task RunAsync(TaskActivityContext context, E }; } + // Stream all history events + this.logger.LogInformation("Streaming history events for instance {InstanceId}", instanceId); + List historyEvents = new(); + await foreach (HistoryEvent historyEvent in client.StreamInstanceHistoryAsync( + instanceId, + executionId: null, // Use latest execution + cancellation: CancellationToken.None)) + { + historyEvents.Add(historyEvent); + } + + this.logger.LogInformation( + "Retrieved {EventCount} history events for instance {InstanceId}", + historyEvents.Count, + instanceId); + // Create blob filename from hash of completed timestamp and instance ID string blobFileName = GenerateBlobFileName(completedTimestamp, instanceId, input.Format); @@ -91,11 +110,11 @@ public override async Task RunAsync(TaskActivityContext context, E ? blobFileName : $"{input.Destination.Prefix.TrimEnd('/')}/{blobFileName}"; - // Serialize instance metadata to JSON - string jsonContent = SerializeInstanceMetadata(metadata); + // Serialize history events to JSON + string jsonContent = SerializeInstanceData(historyEvents, input.Format); // Upload to blob storage - await UploadToBlobStorageAsync( + await this.UploadToBlobStorageAsync( input.Destination.Container, blobPath, jsonContent, @@ -146,7 +165,7 @@ static string GenerateBlobFileName(DateTimeOffset completedTimestamp, string ins static string GetFileExtension(ExportFormat format) { string formatKind = format.Kind.ToLowerInvariant(); - + return formatKind switch { "jsonl" => "jsonl.gz", // JSONL format is compressed @@ -155,31 +174,36 @@ static string GetFileExtension(ExportFormat format) }; } - static string SerializeInstanceMetadata(OrchestrationMetadata metadata) + static string SerializeInstanceData( + List historyEvents, + ExportFormat format) { - var exportData = new + string formatKind = format.Kind.ToLowerInvariant(); + var serializerOptions = new JsonSerializerOptions { - instanceId = metadata.InstanceId, - name = metadata.Name, - runtimeStatus = metadata.RuntimeStatus.ToString(), - createdAt = metadata.CreatedAt, - lastUpdatedAt = metadata.LastUpdatedAt, - input = metadata.SerializedInput, - output = metadata.SerializedOutput, - customStatus = metadata.SerializedCustomStatus, - tags = metadata.Tags, - failureDetails = metadata.FailureDetails != null ? new - { - errorType = metadata.FailureDetails.ErrorType, - errorMessage = metadata.FailureDetails.ErrorMessage, - stackTrace = metadata.FailureDetails.StackTrace, - } : null, + WriteIndented = false, + ReferenceHandler = ReferenceHandler.IgnoreCycles, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; - return JsonSerializer.Serialize(exportData, new JsonSerializerOptions + if (formatKind == "jsonl") { - WriteIndented = false, - }); + // JSONL format: one history event per line + StringBuilder jsonlBuilder = new(); + + foreach (HistoryEvent historyEvent in historyEvents) + { + jsonlBuilder.AppendLine(JsonSerializer.Serialize(historyEvent, serializerOptions)); + } + + return jsonlBuilder.ToString(); + } + else + { + // JSON format: array of history events + return JsonSerializer.Serialize(historyEvents, serializerOptions); + } } async Task UploadToBlobStorageAsync( @@ -278,7 +302,7 @@ public sealed class ExportResult public string InstanceId { get; set; } = string.Empty; /// - /// Gets or sets whether the export was successful. + /// Gets or sets a value indicating whether gets or sets whether the export was successful. /// public bool Success { get; set; } @@ -287,5 +311,3 @@ public sealed class ExportResult /// public string? Error { get; set; } } - - diff --git a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs index facc804b1..91b75c980 100644 --- a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs +++ b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs @@ -2,7 +2,7 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Client.Grpc; +using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.ExportHistory; @@ -46,45 +46,33 @@ public override async Task RunAsync(TaskActivityContext context, L { DurableTaskClient client = this.clientProvider.GetClient(); - // Check if the client is a gRPC client that supports ListTerminalInstances - if (client is not GrpcDurableTaskClient grpcClient) + // Use QueryInstances to fetch terminal instances and project to IDs + OrchestrationQuery query = new( + CreatedFrom: request.CreatedTimeFrom, + CreatedTo: request.CreatedTimeTo, + Statuses: request.RuntimeStatus, + PageSize: request.MaxInstancesPerBatch, + FetchInputsAndOutputs: false, + ContinuationToken: request.ContinuationToken); + + List instanceIds = new(); + string? nextContinuationToken = null; + + await foreach (Page page in client + .GetAllInstancesAsync(query) + .AsPages() + .WithCancellation(CancellationToken.None)) { - throw new NotSupportedException( - $"ListTerminalInstancesActivity requires a GrpcDurableTaskClient, but got {client.GetType().Name}"); + instanceIds.AddRange(page.Values.Select(v => v.InstanceId)); + nextContinuationToken = page.ContinuationToken; + break; } - // Call the gRPC endpoint to list terminal instances - (IReadOnlyList instanceIds, string? nextContinuationToken) = await grpcClient.ListTerminalInstancesAsync( - createdFrom: request.CreatedTimeFrom, - createdTo: request.CreatedTimeTo, - statuses: request.RuntimeStatus, - pageSize: request.MaxInstancesPerBatch, - continuationToken: request.ContinuationToken, - cancellation: CancellationToken.None); - this.logger.LogInformation( "ListTerminalInstancesActivity returned {Count} instance IDs", instanceIds.Count); - // Create next checkpoint if we have a continuation token - ExportCheckpoint? nextCheckpoint = null; - if (!string.IsNullOrEmpty(nextContinuationToken) && instanceIds.Count > 0) - { - // Use the continuation token from the response (which is the last instanceId) - string lastInstanceId = nextContinuationToken; - nextCheckpoint = new ExportCheckpoint - { - ContinuationToken = lastInstanceId, - LastInstanceIdProcessed = lastInstanceId, - // Note: LastTerminalTimeProcessed would require querying the instance, which we skip for performance - }; - } - - return new InstancePage - { - InstanceIds = instanceIds.ToList(), - NextCheckpoint = nextCheckpoint, - }; + return new InstancePage(instanceIds, new ExportCheckpoint(nextContinuationToken)); } catch (Exception ex) { @@ -100,7 +88,7 @@ public override async Task RunAsync(TaskActivityContext context, L public sealed class InstancePage { public List InstanceIds { get; set; } = new(); - public ExportCheckpoint? NextCheckpoint { get; set; } + public ExportCheckpoint NextCheckpoint { get; set; } } diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs index 9dcd145d4..c55f641c0 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -123,11 +123,6 @@ public override AsyncPageable ListJobsAsync(ExportJobQuery ExportJobState state = metadata.State; ExportJobConfiguration? config = state.Config; - // Determine orchestrator presence for this job - string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(metadata.Id.Key); - OrchestrationMetadata? orchestratorState = await this.durableTaskClient.GetInstanceAsync(orchestratorInstanceId, cancellation: cancellation); - string? presentOrchestratorId = orchestratorState != null ? orchestratorInstanceId : null; - exportJobs.Add(new ExportJobDescription { JobId = metadata.Id.Key, @@ -135,7 +130,7 @@ public override AsyncPageable ListJobsAsync(ExportJobQuery CreatedAt = state.CreatedAt, LastModifiedAt = state.LastModifiedAt, Config = config, - OrchestratorInstanceId = presentOrchestratorId, + OrchestratorInstanceId = state.OrchestratorInstanceId, }); } diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs index 7f7764e56..22d8dd3fc 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -97,19 +97,8 @@ public override async Task DeleteAsync(CancellationToken cancellation = default) request, cancellation); - // Wait for the orchestration to complete - OrchestrationMetadata state = await this.durableTaskClient.WaitForInstanceCompletionAsync(instanceId, true, cancellation); - - if (state.RuntimeStatus != OrchestrationRuntimeStatus.Completed) - { - throw new InvalidOperationException($"Failed to delete export job '{this.JobId}': {state.FailureDetails?.ErrorMessage ?? string.Empty}"); - } - // Then terminate the linked export orchestration if it exists await this.TerminateAndPurgeOrchestrationAsync(orchestrationInstanceId, cancellation); - - // Verify both entity and orchestration are gone - await this.VerifyDeletionAsync(orchestrationInstanceId, cancellation); } catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { @@ -141,11 +130,6 @@ public override async Task DescribeAsync(CancellationToken ExportJobConfiguration? config = state.Config; - // Determine if the export orchestrator instance exists and capture its instance ID if so - string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.JobId); - OrchestrationMetadata? orchestratorState = await this.durableTaskClient.GetInstanceAsync(orchestratorInstanceId, cancellation: cancellation); - string? presentOrchestratorId = orchestratorState != null ? orchestratorInstanceId : null; - return new ExportJobDescription { JobId = this.JobId, @@ -153,7 +137,7 @@ public override async Task DescribeAsync(CancellationToken CreatedAt = state.CreatedAt, LastModifiedAt = state.LastModifiedAt, Config = config, - OrchestratorInstanceId = presentOrchestratorId, + OrchestratorInstanceId = state.OrchestratorInstanceId, }; } catch (OperationCanceledException) when (cancellation.IsCancellationRequested) @@ -185,113 +169,17 @@ await this.durableTaskClient.TerminateInstanceAsync( cancellation); // Wait for the orchestration to be terminated before purging - OrchestrationMetadata? orchestrationState = await this.WaitForOrchestrationTerminationAsync( - orchestrationInstanceId, - cancellation); + await this.durableTaskClient.WaitForInstanceCompletionAsync(orchestrationInstanceId, cancellation); // Purge the orchestration instance after it's terminated - if (orchestrationState != null && DefaultExportHistoryJobClient.IsTerminalStatus(orchestrationState.RuntimeStatus)) - { - await this.durableTaskClient.PurgeInstanceAsync( - orchestrationInstanceId, - cancellation: cancellation); - } - else if (orchestrationState != null) - { - throw new InvalidOperationException( - $"Failed to delete export job '{this.JobId}': Cannot purge orchestration '{orchestrationInstanceId}' because it is still in '{orchestrationState.RuntimeStatus}' status."); - } + await this.durableTaskClient.PurgeInstanceAsync(orchestrationInstanceId, cancellation: cancellation); } - catch (Exception ex) when (!(ex is InvalidOperationException)) + catch (Exception ex) { - // Log but don't fail if termination fails (orchestration may not exist or already be terminated) this.logger.ClientError( $"Failed to terminate or purge linked orchestration '{orchestrationInstanceId}': {ex.Message}", this.JobId, ex); - // Continue to verification - if orchestration doesn't exist, verification will pass - } - } - - /// - /// Waits for an orchestration to reach a terminal state. - /// - /// The orchestration instance ID. - /// The cancellation token. - /// The orchestration metadata, or null if the orchestration doesn't exist. - async Task WaitForOrchestrationTerminationAsync( - string orchestrationInstanceId, - CancellationToken cancellation) - { - OrchestrationMetadata? orchestrationState = null; - int waitAttempt = 0; - - while (waitAttempt < ExportHistoryConstants.MaxTerminationWaitAttempts) - { - orchestrationState = await this.durableTaskClient.GetInstanceAsync( - orchestrationInstanceId, - cancellation: cancellation); - - if (orchestrationState == null || DefaultExportHistoryJobClient.IsTerminalStatus(orchestrationState.RuntimeStatus)) - { - break; - } - - // Wait a bit before checking again - await Task.Delay( - TimeSpan.FromMilliseconds(ExportHistoryConstants.TerminationWaitDelayMs), - cancellation); - waitAttempt++; - } - - return orchestrationState; - } - - /// - /// Checks if an orchestration runtime status is a terminal state. - /// - /// The runtime status to check. - /// True if the status is terminal; otherwise, false. - static bool IsTerminalStatus(OrchestrationRuntimeStatus runtimeStatus) - { - return runtimeStatus == OrchestrationRuntimeStatus.Terminated || - runtimeStatus == OrchestrationRuntimeStatus.Completed || - runtimeStatus == OrchestrationRuntimeStatus.Failed; - } - - /// - /// Verifies that both the entity and orchestration have been deleted. - /// - /// The orchestration instance ID to verify. - /// The cancellation token. - async Task VerifyDeletionAsync(string orchestrationInstanceId, CancellationToken cancellation) - { - List stillExist = new(); - - // Check if entity still exists - EntityMetadata? entityMetadata = await this.durableTaskClient.Entities.GetEntityAsync( - this.entityId, - cancellation: cancellation); - if (entityMetadata != null) - { - stillExist.Add($"entity '{this.entityId}'"); - } - - // Check if orchestration still exists - OrchestrationMetadata? orchestrationMetadata = await this.durableTaskClient.GetInstanceAsync( - orchestrationInstanceId, - cancellation: cancellation); - if (orchestrationMetadata != null) - { - stillExist.Add($"orchestration '{orchestrationInstanceId}'"); - } - - // Throw exception if either still exists - if (stillExist.Count > 0) - { - string items = string.Join(" and ", stillExist); - throw new InvalidOperationException( - $"Failed to delete export job '{this.JobId}': The following resources still exist: {items}."); } } } diff --git a/src/ExportHistory/Constants/ExportHistoryConstants.cs b/src/ExportHistory/Constants/ExportHistoryConstants.cs index 85cf8f0d0..567d0f3bc 100644 --- a/src/ExportHistory/Constants/ExportHistoryConstants.cs +++ b/src/ExportHistory/Constants/ExportHistoryConstants.cs @@ -14,16 +14,6 @@ static class ExportHistoryConstants /// public const string OrchestratorInstanceIdPrefix = "ExportJob-"; - /// - /// Maximum number of attempts to wait for orchestration termination during deletion. - /// - public const int MaxTerminationWaitAttempts = 10; - - /// - /// Delay between termination wait attempts in milliseconds. - /// - public const int TerminationWaitDelayMs = 100; - /// /// Generates an orchestrator instance ID for a given export job ID. /// diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs index e9f72241d..afa04feb2 100644 --- a/src/ExportHistory/Entity/ExportJob.cs +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -122,6 +122,9 @@ void StartExportOrchestration(TaskEntityContext context) new TaskName(nameof(ExportJobOrchestrator)), new ExportJobRunRequest(context.Id), startOrchestrationOptions); + + this.State.OrchestratorInstanceId = instanceId; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; } catch (Exception ex) { @@ -163,18 +166,8 @@ public void CommitCheckpoint(TaskEntityContext context, CommitCheckpointRequest this.State.Checkpoint = request.Checkpoint; } - // Record failures if any - if (request.Failures != null && request.Failures.Count > 0) - { - foreach (ExportFailure failure in request.Failures) - { - this.State.FailedInstances[failure.InstanceId] = failure; - } - } - // Update checkpoint time and last modified time - this.State.LastCheckpointTime = DateTimeOffset.UtcNow; - this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastCheckpointTime = this.State.LastModifiedAt = DateTimeOffset.UtcNow; // If there are failures and checkpoint is null (batch failed), mark job as failed if (request.Checkpoint is null && request.Failures != null && request.Failures.Count > 0) diff --git a/src/ExportHistory/Models/ExportCheckpoint.cs b/src/ExportHistory/Models/ExportCheckpoint.cs index 2e404f2ca..1a5a5690d 100644 --- a/src/ExportHistory/Models/ExportCheckpoint.cs +++ b/src/ExportHistory/Models/ExportCheckpoint.cs @@ -1,26 +1,9 @@ -// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// Copyright (c) Microsoft Corporation. namespace Microsoft.DurableTask.ExportHistory; /// /// Checkpoint information used to resume export. /// -public sealed class ExportCheckpoint -{ - /// - /// Gets or sets the last terminal time processed. - /// - public DateTimeOffset? LastTerminalTimeProcessed { get; set; } - - /// - /// Gets or sets the last instance ID processed. - /// - public string? LastInstanceIdProcessed { get; set; } - - /// - /// Gets or sets the continuation token for pagination. - /// - public string? ContinuationToken { get; set; } -} - +public sealed record ExportCheckpoint(string? ContinuationToken = null); \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs index 778049ca1..290372fd0 100644 --- a/src/ExportHistory/Models/ExportFormat.cs +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -6,3 +6,15 @@ namespace Microsoft.DurableTask.ExportHistory; public record ExportFormat( string Kind = "jsonl", string SchemaVersion = "1.0"); + +public static class ExportFormatDefaults +{ + // Backing type to expose a singleton default instance while keeping record immutable defaults + public static readonly ExportFormat Default = new(); +} + +// Maintain existing usage via type to keep call-sites clean +partial class ExportFormat +{ + public static ExportFormat Default => ExportFormatDefaults.Default; +} \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs index dd465d857..9544285af 100644 --- a/src/ExportHistory/Models/ExportJobCreationOptions.cs +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -39,20 +39,40 @@ public ExportJobCreationOptions( if (mode == ExportMode.Batch && !createdTimeTo.HasValue) { throw new ArgumentException( - "CreatedTimeTo is required for Batch export mode. For Continuous mode, CreatedTimeTo must be null.", + "CreatedTimeTo is required for Batch export mode.", + nameof(createdTimeTo)); + } + + if (mode == ExportMode.Batch && createdTimeTo.HasValue && createdTimeTo.Value <= createdTimeFrom) + { + throw new ArgumentException( + $"CreatedTimeTo({createdTimeTo.Value}) must be greater than CreatedTimeFrom({createdTimeFrom}) for Batch export mode.", nameof(createdTimeTo)); } if (mode == ExportMode.Continuous && createdTimeTo.HasValue) { throw new ArgumentException( - "CreatedTimeTo must be null for Continuous export mode. For Batch mode, CreatedTimeTo is required.", + "CreatedTimeTo must be null for Continuous export mode.", nameof(createdTimeTo)); } + // Validate maxInstancesPerBatch range if provided (must be 1..999) + if (maxInstancesPerBatch.HasValue && (maxInstancesPerBatch.Value <= 0 || maxInstancesPerBatch.Value >= 1001)) + { + throw new ArgumentOutOfRangeException( + nameof(maxInstancesPerBatch), + maxInstancesPerBatch, + "MaxInstancesPerBatch must be between 1 and 1000."); + } + // Validate terminal status-only filter here if provided - if (runtimeStatus?.Any() == true && - runtimeStatus.Any(s => s is not (OrchestrationRuntimeStatus.Completed or OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated or OrchestrationRuntimeStatus.ContinuedAsNew))) + if (runtimeStatus?.Any() == true + && runtimeStatus.Any( + s => s is not (OrchestrationRuntimeStatus.Completed + or OrchestrationRuntimeStatus.Failed + or OrchestrationRuntimeStatus.Terminated + or OrchestrationRuntimeStatus.ContinuedAsNew))) { throw new ArgumentException( "Export supports terminal orchestration statuses only. Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew.", @@ -63,10 +83,17 @@ public ExportJobCreationOptions( this.CreatedTimeFrom = createdTimeFrom; this.CreatedTimeTo = createdTimeTo; this.Destination = destination; - this.Format = format ?? new ExportFormat(); - this.RuntimeStatus = runtimeStatus; + this.Format = format ?? ExportFormat.Default; + this.RuntimeStatus = (runtimeStatus is { Count: > 0 }) + ? runtimeStatus + : new List + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated, + OrchestrationRuntimeStatus.ContinuedAsNew + }; this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; - } /// /// Gets the unique identifier for the export job. diff --git a/src/ExportHistory/Models/ExportJobState.cs b/src/ExportHistory/Models/ExportJobState.cs index f1ea294a3..2b09b8587 100644 --- a/src/ExportHistory/Models/ExportJobState.cs +++ b/src/ExportHistory/Models/ExportJobState.cs @@ -54,8 +54,8 @@ public sealed class ExportJobState public long ExportedInstances { get; set; } /// - /// Gets or sets the dictionary of failed instance exports. + /// Gets or sets the instance ID of the orchestrator running this export job, if any. /// - public Dictionary FailedInstances { get; set; } = new(); + public string? OrchestratorInstanceId { get; set; } } diff --git a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs index 356fbacab..21fd4710d 100644 --- a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs +++ b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs @@ -22,6 +22,7 @@ public class ExportJobOrchestrator : TaskOrchestrator /// Initializes a new instance of the class. @@ -60,7 +61,10 @@ public ExportJobOrchestrator(ILogger logger) ExportJobConfiguration config = jobState.Config; // Process instances in batches using explicit loop state + bool hasMore = true; + int batchesProcessed = 0; + while (hasMore) { // Check if job is still active (entity might have been deleted or failed) @@ -73,8 +77,7 @@ public ExportJobOrchestrator(ILogger logger) currentState.Status != ExportJobStatus.Active) { this.logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), "Job is no longer active - orchestrator cancelled"); - hasMore = false; - continue; + return null; } // Call activity to list terminal instances with only necessary information @@ -114,6 +117,7 @@ public ExportJobOrchestrator(ILogger logger) // Commit checkpoint based on batch result if (batchResult.AllSucceeded) { + batchesProcessed++; // All exports succeeded - commit with checkpoint to move cursor forward await this.CommitCheckpointAsync( context, @@ -122,6 +126,12 @@ await this.CommitCheckpointAsync( exportedInstances: batchResult.ExportedCount, checkpoint: pageResult.NextCheckpoint, failures: null); + + if (batchesProcessed >= ContinueAsNewFrequency) + { + context.ContinueAsNew(input); + return null; + } } else { @@ -129,14 +139,35 @@ await this.CommitCheckpointAsync( await this.CommitCheckpointAsync( context, input.JobEntityId, - scannedInstances: scannedCount, - exportedInstances: batchResult.ExportedCount, + scannedInstances: 0, + exportedInstances: 0, checkpoint: null, failures: batchResult.Failures); - // Job is now marked as failed in the entity, stop processing - hasMore = false; - continue; + // Throw detailed exception with failure information + int failureCount = batchResult.Failures?.Count ?? 0; + string failureDetails; + if (batchResult.Failures != null && batchResult.Failures.Count > 0) + { + failureDetails = string.Join("; ", + batchResult.Failures + .Take(10) + .Select(f => + $"InstanceId: {f.InstanceId}, Reason: {f.Reason}")); + } + else + { + failureDetails = "No failure details available"; + } + + if (batchResult.Failures != null && batchResult.Failures.Count > 10) + { + failureDetails += $" ... and {batchResult.Failures.Count - 10} more failures"; + } + + throw new InvalidOperationException( + $"Export job '{jobId}' batch export failed after {MaxRetryAttempts} retry attempts. " + + $"Failure details: {failureDetails}"); } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index a04bfda06..32cb6add2 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -475,22 +475,7 @@ message QueryInstancesResponse { google.protobuf.StringValue continuationToken = 2; } -message ListTerminalInstancesRequest { - TerminalInstanceQuery query = 1; -} - -message TerminalInstanceQuery { - repeated OrchestrationStatus runtimeStatus = 1; - google.protobuf.Timestamp createdTimeFrom = 2; - google.protobuf.Timestamp createdTimeTo = 3; - int32 maxInstanceCount = 4; - google.protobuf.StringValue continuationToken = 5; // instanceId to continue from -} - -message ListTerminalInstancesResponse { - repeated string instanceIds = 1; - google.protobuf.StringValue continuationToken = 2; // last instanceId for next page -} +// Removed ListTerminalInstances in favor of using QueryInstances message PurgeInstancesRequest { oneof request { @@ -756,7 +741,6 @@ service TaskHubSidecarService { // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); - rpc ListTerminalInstances(ListTerminalInstancesRequest) returns (ListTerminalInstancesResponse); rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); diff --git a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs index 6e3bdde39..800cb22c3 100644 --- a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs @@ -327,92 +327,7 @@ await this.client.ForceTerminateTaskOrchestrationAsync( } } - /// - /// Lists terminal orchestration instances sorted by completed timestamp, returning only instance IDs. - /// - /// The list terminal instances request. - /// The server call context. - /// A list terminal instances response. - public override async Task ListTerminalInstances(P.ListTerminalInstancesRequest request, ServerCallContext context) - { - if (this.client is IOrchestrationServiceQueryClient queryClient) - { - // Build query for terminal instances - var query = new OrchestrationQuery - { - RuntimeStatus = request.Query.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList(), - CreatedTimeFrom = request.Query.CreatedTimeFrom?.ToDateTime(), - CreatedTimeTo = request.Query.CreatedTimeTo?.ToDateTime(), - PageSize = request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100, - }; - - // Query all matching instances (we'll filter and sort terminal ones) - // Use a larger page size to ensure we have enough instances after filtering - query.PageSize = request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount * 2 : 200; - OrchestrationQueryResult result = await queryClient.GetOrchestrationWithQueryAsync(query, context.CancellationToken); - - // Filter to only terminal instances and sort by completed timestamp - var terminalInstances = result.OrchestrationState - .Where(state => IsTerminalStatus(state.OrchestrationStatus)) - .OrderBy(state => state.CompletedTime != default ? state.CompletedTime : state.LastUpdatedTime) // Sort by completed timestamp first - .ThenBy(state => state.OrchestrationInstance.InstanceId) // Secondary sort by instanceId for stable ordering - .ToList(); - - // Apply continuation token filter after sorting - if (!string.IsNullOrEmpty(request.Query.ContinuationToken)) - { - // Find the position of the continuation token instanceId in the sorted list - // and skip all instances up to and including it - int continuationIndex = terminalInstances.FindIndex( - state => state.OrchestrationInstance.InstanceId == request.Query.ContinuationToken); - - if (continuationIndex >= 0) - { - // Skip the continuation token instance and all instances before it - terminalInstances = terminalInstances.Skip(continuationIndex + 1).ToList(); - } - else - { - // If continuation token not found, skip instances with instanceId <= continuation token - // This handles the case where the continuation token instance might have been deleted - terminalInstances = terminalInstances - .Where(state => string.Compare(state.OrchestrationInstance.InstanceId, request.Query.ContinuationToken, StringComparison.Ordinal) > 0) - .ToList(); - } - } - - // Take the requested page size - terminalInstances = terminalInstances - .Take(request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100) - .ToList(); - - var response = new P.ListTerminalInstancesResponse(); - foreach (var state in terminalInstances) - { - response.InstanceIds.Add(state.OrchestrationInstance.InstanceId); - } - - // Set continuation token to last instanceId if we have results - if (terminalInstances.Count() > 0 && terminalInstances.Count() == (request.Query.MaxInstanceCount > 0 ? request.Query.MaxInstanceCount : 100)) - { - response.ContinuationToken = terminalInstances.Last().OrchestrationInstance.InstanceId; - } - - return response; - } - else - { - throw new NotSupportedException($"{this.client.GetType().Name} doesn't support query operations."); - } - } - - static bool IsTerminalStatus(OrchestrationStatus status) - { - return status == OrchestrationStatus.Completed || - status == OrchestrationStatus.Failed || - status == OrchestrationStatus.Terminated || - status == OrchestrationStatus.Canceled; - } + // Removed ListTerminalInstances; use QueryInstances instead on client side /// /// Purges orchestration instances. From 52bf21c5719385a8b55edb89654910b36364ca92 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 5 Nov 2025 13:41:19 -0800 Subject: [PATCH 3/7] v1 --- Microsoft.DurableTask.sln | 34 ++-- .../ExportHistoryWebApp.http | 51 +++--- .../ExportJobController.cs | 13 +- .../Models/CreateExportJobRequest.cs | 10 +- src/Client/Core/DurableTaskClient.cs | 21 +++ src/Client/Grpc/GrpcDurableTaskClient.cs | 56 ++++++- .../ExportInstanceHistoryActivity.cs | 26 ++- .../ListTerminalInstancesActivity.cs | 90 +++++------ .../Client/DefaultExportHistoryClient.cs | 5 + .../Client/DefaultExportHistoryJobClient.cs | 30 +++- src/ExportHistory/Entity/ExportJob.cs | 42 ++++- .../Entity/ExportJobOperations.cs | 11 +- src/ExportHistory/Models/ExportCheckpoint.cs | 2 +- src/ExportHistory/Models/ExportDestination.cs | 3 +- src/ExportHistory/Models/ExportFilter.cs | 4 +- src/ExportHistory/Models/ExportFormat.cs | 18 +-- .../Models/ExportJobCreationOptions.cs | 86 +++++++--- .../Models/ExportJobDescription.cs | 25 +++ .../Orchestrations/ExportJobOrchestrator.cs | 152 +++++++++++------- src/Grpc/orchestrator_service.proto | 14 ++ .../ExecuteScheduleOperationOrchestrator.cs | 20 ++- 21 files changed, 503 insertions(+), 210 deletions(-) diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 53f5f4bcf..5b0b1332b 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -101,6 +101,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost", "src\In EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost.Tests", "test\InProcessTestHost.Tests\InProcessTestHost.Tests.csproj", "{B894780C-338F-475E-8E84-56AFA8197A06}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExportHistory", "src\ExportHistory\ExportHistory.csproj", "{354CE69B-78DB-9B29-C67E-0DBB862C7A65}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExportHistoryWebApp", "samples\ExportHistoryWebApp\ExportHistoryWebApp.csproj", "{FE1E17DD-595A-123A-EA4C-AA313BBFB685}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -255,14 +259,6 @@ Global {D2779F32-A548-44F8-B60A-6AC018966C79}.Debug|Any CPU.Build.0 = Debug|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.ActiveCfg = Release|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.Build.0 = Release|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.Build.0 = Release|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.Build.0 = Release|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.Build.0 = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -271,6 +267,22 @@ Global {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Debug|Any CPU.Build.0 = Debug|Any CPU {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.ActiveCfg = Release|Any CPU {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.Build.0 = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.Build.0 = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.Build.0 = Release|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Debug|Any CPU.Build.0 = Debug|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Release|Any CPU.ActiveCfg = Release|Any CPU + {354CE69B-78DB-9B29-C67E-0DBB862C7A65}.Release|Any CPU.Build.0 = Release|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE1E17DD-595A-123A-EA4C-AA313BBFB685}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -317,10 +329,12 @@ Global {A89B766C-987F-4C9F-8937-D0AB9FE640C8} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {100348B5-4D97-4A3F-B777-AB14F276F8FE} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {D2779F32-A548-44F8-B60A-6AC018966C79} = {E5637F81-2FB9-4CD7-900D-455363B142A7} - {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} - {B894780C-338F-475E-8E84-56AFA8197A06} = {E5637F81-2FB9-4CD7-900D-455363B142A7} {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {B894780C-338F-475E-8E84-56AFA8197A06} = {E5637F81-2FB9-4CD7-900D-455363B142A7} + {354CE69B-78DB-9B29-C67E-0DBB862C7A65} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {FE1E17DD-595A-123A-EA4C-AA313BBFB685} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71} diff --git a/samples/ExportHistoryWebApp/ExportHistoryWebApp.http b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http index de24a7a52..aeea8e8c0 100644 --- a/samples/ExportHistoryWebApp/ExportHistoryWebApp.http +++ b/samples/ExportHistoryWebApp/ExportHistoryWebApp.http @@ -1,6 +1,6 @@ ### Variables -@baseUrl = http://localhost:5009 -@jobId = export-job-123 +@baseUrl = http://localhost:5010 +@jobId = export-job-12345 ### Create a new batch export job # @name createBatchExportJob @@ -8,14 +8,14 @@ POST {{baseUrl}}/export-jobs Content-Type: application/json { - "jobId": "{{jobId}}", - "mode": "Batch", - "createdTimeFrom": "2024-01-01T00:00:00Z", - "createdTimeTo": "2024-12-31T23:59:59Z", - "containerName": "export-history", - "prefix": "exports/", - "maxInstancesPerBatch": 100, - "runtimeStatus": ["Completed", "Failed"] + "jobId": "{{jobId}}", + "mode": "Batch", + "completedTimeFrom": "2025-10-01T00:00:00Z", + "completedTimeTo": "2025-11-06T23:59:59Z", + "container": "export-history", + # "prefix": "exports/", + "maxInstancesPerBatch": 1, + "runtimeStatus": [] } ### Create a new continuous export job @@ -26,23 +26,21 @@ Content-Type: application/json { "jobId": "export-job-continuous-123", "mode": "Continuous", - "createdTimeFrom": "2024-01-01T00:00:00Z", - "createdTimeTo": null, - "containerName": "export-history", - "prefix": "continuous-exports/", - "maxInstancesPerBatch": 50 + "container": "export-history", + # "prefix": "continuous-exports/", + "maxInstancesPerBatch": 1000 + # "runtimeStatus": ["asdasd"] } ### Create an export job with default storage (no container specified) # @name createExportJobWithDefaultStorage POST {{baseUrl}}/export-jobs Content-Type: application/json - { "jobId": "export-job-default-storage", "mode": "Batch", - "createdTimeFrom": "2024-01-01T00:00:00Z", - "createdTimeTo": "2024-12-31T23:59:59Z", + "completedTimeFrom": "2024-01-01T00:00:00Z", + "completedTimeTo": "2024-12-31T23:59:59Z", "maxInstancesPerBatch": 100 } @@ -55,27 +53,30 @@ GET {{baseUrl}}/export-jobs/{{jobId}} GET {{baseUrl}}/export-jobs/list ### List export jobs with filters -# Filter by status +### Filter by status GET {{baseUrl}}/export-jobs/list?status=Active -# Filter by job ID prefix +### Filter by job ID prefix GET {{baseUrl}}/export-jobs/list?jobIdPrefix=export-job- -# Filter by creation time range +### Filter by creation time range GET {{baseUrl}}/export-jobs/list?createdFrom=2024-01-01T00:00:00Z&createdTo=2024-12-31T23:59:59Z -# Combined filters +### Combined filters GET {{baseUrl}}/export-jobs/list?status=Completed&jobIdPrefix=export-job-&pageSize=50 ### Delete an export job -DELETE {{baseUrl}}/export-jobs/{{jobId}} +# DELETE {{baseUrl}}/export-jobs/{{jobId}} + +# Delete a continuous export job +DELETE {{baseUrl}}/export-jobs/export-job-continuous-123jk ### Tips: # - Replace the baseUrl variable if your application runs on a different port # - The jobId variable can be changed to test different export job instances # - Export modes: -# - "Batch": Exports all instances within a time range (requires createdTimeTo) -# - "Continuous": Continuously exports instances from a start time (createdTimeTo must be null) +# - "Batch": Exports all instances within a time range (requires completedTimeTo) +# - "Continuous": Continuously exports instances from a start time (completedTimeTo must be null) # - Runtime status filters (valid values): # - "Completed": Exports only completed orchestrations # - "Failed": Exports only failed orchestrations diff --git a/samples/ExportHistoryWebApp/ExportJobController.cs b/samples/ExportHistoryWebApp/ExportJobController.cs index 1bc69a2ce..859950429 100644 --- a/samples/ExportHistoryWebApp/ExportJobController.cs +++ b/samples/ExportHistoryWebApp/ExportJobController.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using Microsoft.AspNetCore.Mvc; +using Microsoft.DurableTask; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.ExportHistory; using ExportHistoryWebApp.Models; @@ -48,9 +49,9 @@ public async Task> CreateExportJob([FromBody] try { ExportDestination? destination = null; - if (!string.IsNullOrEmpty(request.ContainerName)) + if (!string.IsNullOrEmpty(request.Container)) { - destination = new ExportDestination(request.ContainerName) + destination = new ExportDestination(request.Container) { Prefix = request.Prefix, }; @@ -58,8 +59,8 @@ public async Task> CreateExportJob([FromBody] ExportJobCreationOptions creationOptions = new ExportJobCreationOptions( mode: request.Mode, - createdTimeFrom: request.CreatedTimeFrom, - createdTimeTo: request.CreatedTimeTo, + completedTimeFrom: request.CompletedTimeFrom, + completedTimeTo: request.CompletedTimeTo, destination: destination, jobId: request.JobId, format: request.Format, @@ -128,6 +129,7 @@ public async Task>> ListExportJob [FromQuery] int? pageSize = null, [FromQuery] string? continuationToken = null) { + this.logger.LogInformation("GET list endpoint called with method: {Method}", this.HttpContext.Request.Method); try { ExportJobQuery? query = null; @@ -177,14 +179,17 @@ public async Task>> ListExportJob [HttpDelete("{id}")] public async Task DeleteExportJob(string id) { + this.logger.LogInformation("DELETE endpoint called for job ID: {JobId}", id); try { ExportHistoryJobClient jobClient = this.exportHistoryClient.GetJobClient(id); await jobClient.DeleteAsync(); + this.logger.LogInformation("Successfully deleted export job {JobId}", id); return this.NoContent(); } catch (ExportJobNotFoundException) { + this.logger.LogWarning("Export job {JobId} not found for deletion", id); return this.NotFound(); } catch (Exception ex) diff --git a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs index 7863281c7..4bb843816 100644 --- a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs +++ b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs @@ -22,19 +22,19 @@ public class CreateExportJobRequest public ExportMode Mode { get; set; } /// - /// Gets or sets the start time for the export (inclusive). Required. + /// Gets or sets the start time for the export based on completion time (inclusive). Required. /// - public DateTimeOffset CreatedTimeFrom { get; set; } + public DateTimeOffset CompletedTimeFrom { get; set; } /// - /// Gets or sets the end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// Gets or sets the end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. /// - public DateTimeOffset? CreatedTimeTo { get; set; } + public DateTimeOffset? CompletedTimeTo { get; set; } /// /// Gets or sets the blob container name where exported data will be stored. Optional if default storage is configured. /// - public string? ContainerName { get; set; } + public string? Container { get; set; } /// /// Gets or sets an optional prefix for blob paths. diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index d2cf94c84..5786d036f 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -338,6 +338,27 @@ public abstract Task ResumeInstanceAsync( /// An async pageable of the query results. public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); + /// + /// Lists orchestration instance IDs filtered by completed time. + /// + /// The runtime statuses to filter by. + /// The start time for completed time filter (inclusive). + /// The end time for completed time filter (inclusive). + /// The page size for pagination. + /// The last fetched instance key. + /// The cancellation token. + /// A page of instance IDs with continuation token. + public virtual Task> ListInstanceIdsAsync( + IEnumerable? runtimeStatus = null, + DateTimeOffset? completedTimeFrom = null, + DateTimeOffset? completedTimeTo = null, + int pageSize = OrchestrationQuery.DefaultPageSize, + string? lastInstanceKey = null, + CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support listing orchestration instance IDs by completed time."); + } + /// public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) => this.PurgeInstanceAsync(instanceId, null, cancellation); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index ff106a556..81f4966c7 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -327,6 +327,53 @@ public override async Task WaitForInstanceStartAsync( // Removed ListTerminalInstances; use GetAllInstancesAsync with OrchestrationQuery instead + /// + public override async Task> ListInstanceIdsAsync( + IEnumerable? runtimeStatus = null, + DateTimeOffset? completedTimeFrom = null, + DateTimeOffset? completedTimeTo = null, + int pageSize = OrchestrationQuery.DefaultPageSize, + string? lastInstanceKey = null, + CancellationToken cancellation = default) + { + Check.NotEntity(this.options.EnableEntitySupport, null); + + P.ListInstanceIdsRequest request = new() + { + PageSize = pageSize, + LastInstanceKey = lastInstanceKey ?? string.Empty, + }; + + if (runtimeStatus != null) + { + request.RuntimeStatus.AddRange(runtimeStatus.Select(x => x.ToGrpcStatus())); + } + + if (completedTimeFrom.HasValue) + { + request.CompletedTimeFrom = completedTimeFrom.Value.ToTimestamp(); + } + + if (completedTimeTo.HasValue) + { + request.CompletedTimeTo = completedTimeTo.Value.ToTimestamp(); + } + + try + { + P.ListInstanceIdsResponse response = await this.sidecarClient.ListInstanceIdsAsync( + request, + cancellationToken: cancellation); + + return new Page(response.InstanceIds.ToList(), response.LastInstanceKey); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.ListInstanceIdsAsync)} operation was canceled.", e, cancellation); + } + } + /// public override async Task WaitForInstanceCompletionAsync( string instanceId, bool getInputsAndOutputs = false, CancellationToken cancellation = default) @@ -493,6 +540,11 @@ public override async IAsyncEnumerable StreamInstanceHistoryAsync( IAsyncStreamReader responseStream = streamResponse.ResponseStream; + // Create conversion state to maintain orchestration instance across events + // This is required for entity-related events (EntityOperationCalled, EntityLockRequested, etc.) + // which need the parent orchestration instance information from ExecutionStartedEvent + Microsoft.DurableTask.ProtoUtils.EntityConversionState conversionState = new(insertMissingEntityUnlocks: false); + bool hasMore = true; while (hasMore) { @@ -517,7 +569,9 @@ public override async IAsyncEnumerable StreamInstanceHistoryAsync( P.HistoryChunk chunk = responseStream.Current; foreach (P.HistoryEvent protoEvent in chunk.Events) { - yield return Microsoft.DurableTask.ProtoUtils.ConvertHistoryEvent(protoEvent); + // Use the conversion state's converter to maintain state across events + // This ensures entity events can access the orchestration instance from ExecutionStartedEvent + yield return conversionState.ConvertFromProto(protoEvent); } } } diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs index 2056a509d..741ac6e81 100644 --- a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -119,6 +119,7 @@ await this.UploadToBlobStorageAsync( blobPath, jsonContent, input.Format, + instanceId, CancellationToken.None); this.logger.LogInformation( @@ -179,22 +180,26 @@ static string SerializeInstanceData( ExportFormat format) { string formatKind = format.Kind.ToLowerInvariant(); - var serializerOptions = new JsonSerializerOptions + JsonSerializerOptions serializerOptions = new JsonSerializerOptions { WriteIndented = false, - ReferenceHandler = ReferenceHandler.IgnoreCycles, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + IncludeFields = true, // Include fields, not just properties (matches JsonDataConverter pattern) + Converters = { new JsonStringEnumConverter() }, // Serialize enums as strings }; if (formatKind == "jsonl") { // JSONL format: one history event per line + // Serialize as object to preserve runtime type (polymorphic serialization) StringBuilder jsonlBuilder = new(); foreach (HistoryEvent historyEvent in historyEvents) { - jsonlBuilder.AppendLine(JsonSerializer.Serialize(historyEvent, serializerOptions)); + // Serialize as object to preserve the actual derived type + string json = JsonSerializer.Serialize((object)historyEvent, historyEvent.GetType(), serializerOptions); + jsonlBuilder.AppendLine(json); } return jsonlBuilder.ToString(); @@ -202,7 +207,9 @@ static string SerializeInstanceData( else { // JSON format: array of history events - return JsonSerializer.Serialize(historyEvents, serializerOptions); + // Convert to object array to preserve runtime types + object[] eventsAsObjects = historyEvents.Cast().ToArray(); + return JsonSerializer.Serialize(eventsAsObjects, serializerOptions); } } @@ -211,6 +218,7 @@ async Task UploadToBlobStorageAsync( string blobPath, string content, ExportFormat format, + string instanceId, CancellationToken cancellationToken) { // Create blob service client from connection string @@ -228,7 +236,7 @@ await containerClient.CreateIfNotExistsAsync( // Upload content byte[] contentBytes = Encoding.UTF8.GetBytes(content); - if (format.Kind.ToLowerInvariant() == "jsonl") + if (format.Kind.Equals("jsonl", StringComparison.InvariantCultureIgnoreCase)) { // Compress with gzip using MemoryStream compressedStream = new(); @@ -247,6 +255,10 @@ await containerClient.CreateIfNotExistsAsync( ContentType = "application/jsonl+gzip", ContentEncoding = "gzip", }, + Metadata = new Dictionary + { + { "instanceId", instanceId }, + }, }; await blobClient.UploadAsync(compressedStream, uploadOptions, cancellationToken); @@ -260,6 +272,10 @@ await containerClient.CreateIfNotExistsAsync( { ContentType = "application/json", }, + Metadata = new Dictionary + { + { "instanceId", instanceId }, + }, }; await blobClient.UploadAsync( diff --git a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs index 91b75c980..b01d2c5e5 100644 --- a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs +++ b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; @@ -11,68 +10,49 @@ namespace Microsoft.DurableTask.ExportHistory; /// Input for listing terminal instances activity. /// public sealed record ListTerminalInstancesRequest( - DateTimeOffset CreatedTimeFrom, - DateTimeOffset? CreatedTimeTo, + DateTimeOffset CompletedTimeFrom, + DateTimeOffset? CompletedTimeTo, IEnumerable? RuntimeStatus, - string? ContinuationToken, + string? LastInstanceKey, int MaxInstancesPerBatch = 100); /// /// Activity that lists terminal orchestration instances using the configured filters and checkpoint. /// +/// +/// Initializes a new instance of the class. +/// [DurableTask] -public class ListTerminalInstancesActivity : TaskActivity +public class ListTerminalInstancesActivity( + IDurableTaskClientProvider clientProvider, + ILogger logger) : TaskActivity { - readonly IDurableTaskClientProvider clientProvider; - readonly ILogger logger; - - /// - /// Initializes a new instance of the class. - /// - public ListTerminalInstancesActivity( - IDurableTaskClientProvider clientProvider, - ILogger logger) - { - this.clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); - this.logger = Check.NotNull(logger, nameof(logger)); - } + readonly IDurableTaskClientProvider clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); /// - public override async Task RunAsync(TaskActivityContext context, ListTerminalInstancesRequest request) + public override async Task RunAsync(TaskActivityContext context, ListTerminalInstancesRequest input) { - Check.NotNull(request, nameof(request)); + Check.NotNull(input, nameof(input)); try { DurableTaskClient client = this.clientProvider.GetClient(); - // Use QueryInstances to fetch terminal instances and project to IDs - OrchestrationQuery query = new( - CreatedFrom: request.CreatedTimeFrom, - CreatedTo: request.CreatedTimeTo, - Statuses: request.RuntimeStatus, - PageSize: request.MaxInstancesPerBatch, - FetchInputsAndOutputs: false, - ContinuationToken: request.ContinuationToken); - - List instanceIds = new(); - string? nextContinuationToken = null; - - await foreach (Page page in client - .GetAllInstancesAsync(query) - .AsPages() - .WithCancellation(CancellationToken.None)) - { - instanceIds.AddRange(page.Values.Select(v => v.InstanceId)); - nextContinuationToken = page.ContinuationToken; - break; - } + // Try to use ListInstanceIds endpoint first (available in gRPC client) + Page page = await client.ListInstanceIdsAsync( + runtimeStatus: input.RuntimeStatus, + completedTimeFrom: input.CompletedTimeFrom, + completedTimeTo: input.CompletedTimeTo, + pageSize: input.MaxInstancesPerBatch, + lastInstanceKey: input.LastInstanceKey, + cancellation: CancellationToken.None); this.logger.LogInformation( - "ListTerminalInstancesActivity returned {Count} instance IDs", - instanceIds.Count); + "ListTerminalInstancesActivity returned {Count} instance IDs using ListInstanceIds", + page.Values.Count); - return new InstancePage(instanceIds, new ExportCheckpoint(nextContinuationToken)); + return new InstancePage(page.Values.ToList(), new ExportCheckpoint(page.ContinuationToken)); } catch (Exception ex) { @@ -87,8 +67,24 @@ public override async Task RunAsync(TaskActivityContext context, L /// public sealed class InstancePage { - public List InstanceIds { get; set; } = new(); - public ExportCheckpoint NextCheckpoint { get; set; } -} + /// + /// Initializes a new instance of the class. + /// + /// The list of instance IDs. + /// The next checkpoint for pagination. + public InstancePage(List instanceIds, ExportCheckpoint nextCheckpoint) + { + this.InstanceIds = instanceIds; + this.NextCheckpoint = nextCheckpoint; + } + /// + /// Gets or sets the list of instance IDs. + /// + public List InstanceIds { get; set; } = []; + /// + /// Gets or sets the next checkpoint for pagination. + /// + public ExportCheckpoint NextCheckpoint { get; set; } +} diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs index c55f641c0..484e7e612 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -131,6 +131,11 @@ public override AsyncPageable ListJobsAsync(ExportJobQuery LastModifiedAt = state.LastModifiedAt, Config = config, OrchestratorInstanceId = state.OrchestratorInstanceId, + ScannedInstances = state.ScannedInstances, + ExportedInstances = state.ExportedInstances, + LastError = state.LastError, + Checkpoint = state.Checkpoint, + LastCheckpointTime = state.LastCheckpointTime, }); } diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs index 22d8dd3fc..52418723a 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; @@ -29,17 +30,32 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell { Check.NotNull(options, nameof(options)); + // Determine default prefix based on mode if not already set + string? defaultPrefix = $"{options.Mode.ToString().ToLower()}/"; + // If destination is not provided, construct it from storage options ExportJobCreationOptions optionsWithDestination = options; if (options.Destination == null) { + // Use storage options prefix if provided, otherwise use mode-based default + string? prefix = this.storageOptions.Prefix ?? defaultPrefix; + ExportDestination destination = new ExportDestination(this.storageOptions.ContainerName) { - Prefix = this.storageOptions.Prefix, + Prefix = prefix, }; optionsWithDestination = options with { Destination = destination }; } + else if (string.IsNullOrEmpty(options.Destination.Prefix)) + { + // Destination provided but no prefix set - use mode-based default + ExportDestination destinationWithPrefix = new ExportDestination(options.Destination.Container) + { + Prefix = defaultPrefix, + }; + optionsWithDestination = options with { Destination = destinationWithPrefix }; + } ExportJobOperationRequest request = new ExportJobOperationRequest( @@ -138,6 +154,11 @@ public override async Task DescribeAsync(CancellationToken LastModifiedAt = state.LastModifiedAt, Config = config, OrchestratorInstanceId = state.OrchestratorInstanceId, + ScannedInstances = state.ScannedInstances, + ExportedInstances = state.ExportedInstances, + LastError = state.LastError, + Checkpoint = state.Checkpoint, + LastCheckpointTime = state.LastCheckpointTime, }; } catch (OperationCanceledException) when (cancellation.IsCancellationRequested) @@ -174,6 +195,13 @@ await this.durableTaskClient.TerminateInstanceAsync( // Purge the orchestration instance after it's terminated await this.durableTaskClient.PurgeInstanceAsync(orchestrationInstanceId, cancellation: cancellation); } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + { + // Orchestration instance doesn't exist - this is expected if it was already deleted or never existed + this.logger.LogInformation( + "Orchestration instance '{OrchestrationInstanceId}' is already purged", + orchestrationInstanceId); + } catch (Exception ex) { this.logger.ClientError( diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs index afa04feb2..af23deec0 100644 --- a/src/ExportHistory/Entity/ExportJob.cs +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -43,8 +43,8 @@ public void Create(TaskEntityContext context, ExportJobCreationOptions creationO ExportJobConfiguration config = new ExportJobConfiguration( Mode: creationOptions.Mode, Filter: new ExportFilter( - CreatedTimeFrom: creationOptions.CreatedTimeFrom, - CreatedTimeTo: creationOptions.CreatedTimeTo, + CompletedTimeFrom: creationOptions.CompletedTimeFrom, + CompletedTimeTo: creationOptions.CompletedTimeTo, RuntimeStatus: creationOptions.RuntimeStatus), Destination: creationOptions.Destination, Format: creationOptions.Format, @@ -73,6 +73,16 @@ public void Create(TaskEntityContext context, ExportJobCreationOptions creationO } } + /// + /// Gets the current state of the export job. + /// + /// The entity context. + /// The current export job state. + public ExportJobState Get(TaskEntityContext context) + { + return this.State; + } + /// /// Runs the export job by starting the export orchestrator. /// @@ -254,4 +264,32 @@ public void MarkAsFailed(TaskEntityContext context, string? errorMessage = null) throw; } } + + /// + /// Deletes the export job entity. + /// + /// The entity context. + public void Delete(TaskEntityContext context) + { + try + { + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.Delete), + "Deleting export job entity"); + + // Delete the entity by setting state to null + // This is the standard way to delete a durable entity + this.State = null!; + } + catch (Exception ex) + { + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.Delete), + "Failed to delete export job entity", + ex); + throw; + } + } } diff --git a/src/ExportHistory/Entity/ExportJobOperations.cs b/src/ExportHistory/Entity/ExportJobOperations.cs index c128b96fb..339abf4bb 100644 --- a/src/ExportHistory/Entity/ExportJobOperations.cs +++ b/src/ExportHistory/Entity/ExportJobOperations.cs @@ -6,16 +6,15 @@ namespace Microsoft.DurableTask.ExportHistory; /// /// Constants for export job entity operation names. /// +/// +/// Operation names are case-insensitive when matching to entity methods. +/// These constants match the method names on for consistency. +/// static class ExportJobOperations { - /// - /// Operation name for getting entity state. - /// - public const string Get = "get"; - /// /// Operation name for deleting the entity. /// - public const string Delete = "delete"; + public const string Delete = "Delete"; } diff --git a/src/ExportHistory/Models/ExportCheckpoint.cs b/src/ExportHistory/Models/ExportCheckpoint.cs index 1a5a5690d..efe0765c9 100644 --- a/src/ExportHistory/Models/ExportCheckpoint.cs +++ b/src/ExportHistory/Models/ExportCheckpoint.cs @@ -6,4 +6,4 @@ namespace Microsoft.DurableTask.ExportHistory; /// /// Checkpoint information used to resume export. /// -public sealed record ExportCheckpoint(string? ContinuationToken = null); \ No newline at end of file +public sealed record ExportCheckpoint(string? LastInstanceKey = null); diff --git a/src/ExportHistory/Models/ExportDestination.cs b/src/ExportHistory/Models/ExportDestination.cs index fbab33b5b..d4c11b04c 100644 --- a/src/ExportHistory/Models/ExportDestination.cs +++ b/src/ExportHistory/Models/ExportDestination.cs @@ -8,6 +8,7 @@ namespace Microsoft.DurableTask.ExportHistory; /// public sealed class ExportDestination { + public ExportDestination() { } /// /// Initializes a new instance of the class. /// @@ -22,7 +23,7 @@ public ExportDestination(string container) /// /// Gets the blob container name. /// - public string Container { get; } + public string Container { get; set; } /// /// Gets or sets an optional prefix for blob paths. diff --git a/src/ExportHistory/Models/ExportFilter.cs b/src/ExportHistory/Models/ExportFilter.cs index c06dd540c..ee6f4ac52 100644 --- a/src/ExportHistory/Models/ExportFilter.cs +++ b/src/ExportHistory/Models/ExportFilter.cs @@ -7,6 +7,6 @@ namespace Microsoft.DurableTask.ExportHistory; public record ExportFilter( - DateTimeOffset CreatedTimeFrom, - DateTimeOffset? CreatedTimeTo = null, + DateTimeOffset CompletedTimeFrom, + DateTimeOffset? CompletedTimeTo = null, IEnumerable? RuntimeStatus = null); \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs index 290372fd0..e441c09f5 100644 --- a/src/ExportHistory/Models/ExportFormat.cs +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -3,18 +3,12 @@ namespace Microsoft.DurableTask.ExportHistory; -public record ExportFormat( +public partial record ExportFormat( string Kind = "jsonl", - string SchemaVersion = "1.0"); - -public static class ExportFormatDefaults + string SchemaVersion = "1.0") { - // Backing type to expose a singleton default instance while keeping record immutable defaults - public static readonly ExportFormat Default = new(); + /// + /// Gets the default export format (jsonl with schema version 1.0). + /// + public static ExportFormat Default => new(); } - -// Maintain existing usage via type to keep call-sites clean -partial class ExportFormat -{ - public static ExportFormat Default => ExportFormatDefaults.Default; -} \ No newline at end of file diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs index 9544285af..91e081eb3 100644 --- a/src/ExportHistory/Models/ExportJobCreationOptions.cs +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. - +using System.Text.Json.Serialization; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Entities; @@ -11,12 +11,16 @@ namespace Microsoft.DurableTask.ExportHistory; /// public record ExportJobCreationOptions { + public ExportJobCreationOptions() + { + } + /// /// Initializes a new instance of the class. /// /// The export mode (Batch or Continuous). - /// The start time for the export (inclusive). Required. - /// The end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// The start time for the export based on completion time (inclusive). Required for Batch mode. For Continuous mode, this will be set to UtcNow if not provided. + /// The end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. /// The export destination where exported data will be stored. Required unless default storage is configured. /// The unique identifier for the export job. If not provided, a GUID will be generated. /// The export format settings. Optional, defaults to jsonl-gzip. @@ -25,8 +29,8 @@ public record ExportJobCreationOptions /// Thrown when validation fails. public ExportJobCreationOptions( ExportMode mode, - DateTimeOffset createdTimeFrom, - DateTimeOffset? createdTimeTo, + DateTimeOffset completedTimeFrom, + DateTimeOffset? completedTimeTo, ExportDestination? destination, string? jobId = null, ExportFormat? format = null, @@ -36,25 +40,53 @@ public ExportJobCreationOptions( // Generate GUID if jobId not provided this.JobId = string.IsNullOrEmpty(jobId) ? Guid.NewGuid().ToString("N") : jobId; - if (mode == ExportMode.Batch && !createdTimeTo.HasValue) + if (mode == ExportMode.Batch) { - throw new ArgumentException( - "CreatedTimeTo is required for Batch export mode.", - nameof(createdTimeTo)); - } + if (completedTimeFrom == default) + { + throw new ArgumentException( + "CompletedTimeFrom is required for Batch export mode.", + nameof(completedTimeFrom)); + } - if (mode == ExportMode.Batch && createdTimeTo.HasValue && createdTimeTo.Value <= createdTimeFrom) - { - throw new ArgumentException( - $"CreatedTimeTo({createdTimeTo.Value}) must be greater than CreatedTimeFrom({createdTimeFrom}) for Batch export mode.", - nameof(createdTimeTo)); - } + if (!completedTimeTo.HasValue) + { + throw new ArgumentException( + "CompletedTimeTo is required for Batch export mode.", + nameof(completedTimeTo)); + } - if (mode == ExportMode.Continuous && createdTimeTo.HasValue) - { + if (completedTimeTo.HasValue && completedTimeTo.Value <= completedTimeFrom) + { + throw new ArgumentException( + $"CompletedTimeTo({completedTimeTo.Value}) must be greater than CompletedTimeFrom({completedTimeFrom}) for Batch export mode.", + nameof(completedTimeTo)); + } + + if (completedTimeTo.HasValue && completedTimeTo.Value > DateTimeOffset.UtcNow) + { + throw new ArgumentException( + $"CompletedTimeTo({completedTimeTo.Value}) cannot be in the future. It must be less than or equal to the current time ({DateTimeOffset.UtcNow}).", + nameof(completedTimeTo)); + } + } else if (mode == ExportMode.Continuous) { + if (completedTimeFrom != default) + { + throw new ArgumentException( + "CompletedTimeFrom is not allowed for Continuous export mode.", + nameof(completedTimeFrom)); + } + + if (completedTimeTo.HasValue) + { + throw new ArgumentException( + "CompletedTimeTo is not allowed for Continuous export mode.", + nameof(completedTimeTo)); + } + } else { throw new ArgumentException( - "CreatedTimeTo must be null for Continuous export mode.", - nameof(createdTimeTo)); + "Invalid export mode.", + nameof(mode)); } // Validate maxInstancesPerBatch range if provided (must be 1..999) @@ -80,8 +112,8 @@ or OrchestrationRuntimeStatus.Terminated } this.Mode = mode; - this.CreatedTimeFrom = createdTimeFrom; - this.CreatedTimeTo = createdTimeTo; + this.CompletedTimeFrom = completedTimeFrom; + this.CompletedTimeTo = completedTimeTo; this.Destination = destination; this.Format = format ?? ExportFormat.Default; this.RuntimeStatus = (runtimeStatus is { Count: > 0 }) @@ -94,6 +126,7 @@ or OrchestrationRuntimeStatus.Terminated OrchestrationRuntimeStatus.ContinuedAsNew }; this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; + } /// /// Gets the unique identifier for the export job. @@ -106,14 +139,15 @@ or OrchestrationRuntimeStatus.Terminated public ExportMode Mode { get; init; } /// - /// Gets the start time for the export (inclusive). Required. + /// Gets the start time for the export based on completion time (inclusive). + /// Required for Batch mode. For Continuous mode, this is automatically set to UtcNow when creating the job. /// - public DateTimeOffset CreatedTimeFrom { get; init; } + public DateTimeOffset CompletedTimeFrom { get; init; } /// - /// Gets the end time for the export (inclusive). Required for Batch mode, null for Continuous mode. + /// Gets the end time for the export based on completion time (inclusive). Required for Batch mode, null for Continuous mode. /// - public DateTimeOffset? CreatedTimeTo { get; init; } + public DateTimeOffset? CompletedTimeTo { get; init; } /// /// Gets the export destination where exported data will be stored. Optional. diff --git a/src/ExportHistory/Models/ExportJobDescription.cs b/src/ExportHistory/Models/ExportJobDescription.cs index bd1e2d4c1..986618c68 100644 --- a/src/ExportHistory/Models/ExportJobDescription.cs +++ b/src/ExportHistory/Models/ExportJobDescription.cs @@ -39,4 +39,29 @@ public record ExportJobDescription /// Gets or sets the instance ID of the running export orchestrator, if any. /// public string? OrchestratorInstanceId { get; init; } + + /// + /// Gets or sets the total number of instances scanned. + /// + public long ScannedInstances { get; init; } + + /// + /// Gets or sets the total number of instances exported. + /// + public long ExportedInstances { get; init; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; init; } + + /// + /// Gets or sets the checkpoint for resuming the export. + /// + public ExportCheckpoint? Checkpoint { get; init; } + + /// + /// Gets or sets the time of the last checkpoint. + /// + public DateTimeOffset? LastCheckpointTime { get; init; } } diff --git a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs index 21fd4710d..c8eab1cf0 100644 --- a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs +++ b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs @@ -9,7 +9,7 @@ namespace Microsoft.DurableTask.ExportHistory; /// /// Orchestrator input to start a runner for a given job. /// -public sealed record ExportJobRunRequest(EntityInstanceId JobEntityId); +public sealed record ExportJobRunRequest(EntityInstanceId JobEntityId, int ProcessedCycles = 0); /// /// Orchestrator that performs the actual export work by querying orchestration instances @@ -18,32 +18,33 @@ public sealed record ExportJobRunRequest(EntityInstanceId JobEntityId); [DurableTask] public class ExportJobOrchestrator : TaskOrchestrator { - readonly ILogger logger; const int MaxRetryAttempts = 3; const int MinBackoffSeconds = 60; // 1 minute const int MaxBackoffSeconds = 300; // 5 minutes - const int ContinueAsNewFrequency = 50; + const int ContinueAsNewFrequency = 5; + static readonly TimeSpan ContinuousExportIdleDelay = TimeSpan.FromMinutes(1); - /// - /// Initializes a new instance of the class. - /// - public ExportJobOrchestrator(ILogger logger) - { - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + // Retry policy for individual export activities: 3 attempts with exponential backoff + // First retry after 15s, second retry after 30s (capped at 60s) + static readonly RetryPolicy ExportActivityRetryPolicy = new( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromSeconds(15), + backoffCoefficient: 2.0, + maxRetryInterval: TimeSpan.FromSeconds(60)); /// public override async Task RunAsync(TaskOrchestrationContext context, ExportJobRunRequest input) { + ILogger logger = context.CreateReplaySafeLogger(); string jobId = input.JobEntityId.Key; - this.logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator started"); + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator started"); try { // Get the export job state and configuration from the entity ExportJobState? jobState = await context.Entities.CallEntityAsync( input.JobEntityId, - ExportJobOperations.Get, + nameof(ExportJob.Get), null); if (jobState == null || jobState.Config == null) @@ -54,59 +55,91 @@ public ExportJobOrchestrator(ILogger logger) // Check if job is still active if (jobState.Status != ExportJobStatus.Active) { - this.logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), $"Job status is {jobState.Status}, not Active - orchestrator cancelled"); + logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), $"Job status is {jobState.Status}, not Active - orchestrator cancelled"); return null; } ExportJobConfiguration config = jobState.Config; - // Process instances in batches using explicit loop state - - bool hasMore = true; - int batchesProcessed = 0; + int processedCycles = input.ProcessedCycles; - while (hasMore) + while (true) { + processedCycles++; + if (processedCycles > ContinueAsNewFrequency) + { + context.ContinueAsNew(new ExportJobRunRequest(input.JobEntityId, ProcessedCycles: 0)); + return null!; + } + // Check if job is still active (entity might have been deleted or failed) ExportJobState? currentState = await context.Entities.CallEntityAsync( input.JobEntityId, - ExportJobOperations.Get, + nameof(ExportJob.Get), null); - if (currentState == null || + if (currentState == null || + currentState.Config == null || currentState.Status != ExportJobStatus.Active) { - this.logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), "Job is no longer active - orchestrator cancelled"); + logger.ExportJobOperationWarning(jobId, nameof(ExportJobOrchestrator), "Job is no longer active"); return null; } + // if (currentState.Checkpoint is not null && + // string.IsNullOrEmpty(currentState.Checkpoint.LastInstanceKey)) + // { + // if (config.Mode == ExportMode.Batch) + // { + // logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export - export complete."); + // break; + // } + // else if (config.Mode == ExportMode.Continuous) + // { + // logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export currently - will check again later."); + // await context.CreateTimer(ContinuousExportIdleDelay, default); + // continue; + // } + // else + // { + // throw new InvalidOperationException("Invalid export mode."); + // } + // } + // Call activity to list terminal instances with only necessary information ListTerminalInstancesRequest listRequest = new ListTerminalInstancesRequest( - CreatedTimeFrom: currentState.Config.Filter.CreatedTimeFrom, - CreatedTimeTo: currentState.Config.Filter.CreatedTimeTo, + CompletedTimeFrom: currentState.Config.Filter.CompletedTimeFrom, + CompletedTimeTo: currentState.Config.Filter.CompletedTimeTo, RuntimeStatus: currentState.Config.Filter.RuntimeStatus, - ContinuationToken: currentState.Checkpoint?.ContinuationToken, + LastInstanceKey: currentState.Checkpoint?.LastInstanceKey, MaxInstancesPerBatch: currentState.Config.MaxInstancesPerBatch); InstancePage pageResult = await context.CallActivityAsync( nameof(ListTerminalInstancesActivity), listRequest); - // Handle empty page result - no instances found, treat as end of data - if (pageResult == null || pageResult.InstanceIds.Count == 0) + List instancesToExport = pageResult.InstanceIds; + long scannedCount = instancesToExport.Count; + + if (scannedCount == 0) { - if (config.Mode == ExportMode.Batch) + if (config.Mode == ExportMode.Continuous) { - hasMore = false; + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export - export complete."); + await context.CreateTimer(ContinuousExportIdleDelay, default); continue; } - await context.CreateTimer(TimeSpan.FromMinutes(5), default); - continue; + else if (config.Mode == ExportMode.Batch) + { + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "No more instances to export currently - will retry later."); + break; + } + else + { + throw new InvalidOperationException("Invalid export mode."); + } } - List instancesToExport = pageResult.InstanceIds; - long scannedCount = instancesToExport.Count; - // Process batch with retry logic BatchExportResult batchResult = await this.ProcessBatchWithRetryAsync( context, @@ -117,7 +150,6 @@ public ExportJobOrchestrator(ILogger logger) // Commit checkpoint based on batch result if (batchResult.AllSucceeded) { - batchesProcessed++; // All exports succeeded - commit with checkpoint to move cursor forward await this.CommitCheckpointAsync( context, @@ -126,12 +158,6 @@ await this.CommitCheckpointAsync( exportedInstances: batchResult.ExportedCount, checkpoint: pageResult.NextCheckpoint, failures: null); - - if (batchesProcessed >= ContinueAsNewFrequency) - { - context.ContinueAsNew(input); - return null; - } } else { @@ -149,7 +175,8 @@ await this.CommitCheckpointAsync( string failureDetails; if (batchResult.Failures != null && batchResult.Failures.Count > 0) { - failureDetails = string.Join("; ", + failureDetails = string.Join( + "; ", batchResult.Failures .Take(10) .Select(f => @@ -159,7 +186,7 @@ await this.CommitCheckpointAsync( { failureDetails = "No failure details available"; } - + if (batchResult.Failures != null && batchResult.Failures.Count > 10) { failureDetails += $" ... and {batchResult.Failures.Count - 10} more failures"; @@ -173,15 +200,15 @@ await this.CommitCheckpointAsync( await this.MarkAsCompletedAsync(context, input.JobEntityId); - this.logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator completed"); + logger.ExportJobOperationInfo(jobId, nameof(ExportJobOrchestrator), "Export orchestrator completed"); return null!; } catch (Exception ex) { - this.logger.ExportJobOperationError(jobId, nameof(ExportJobOrchestrator), "Export orchestrator failed", ex); - + logger.ExportJobOperationError(jobId, nameof(ExportJobOrchestrator), "Export orchestrator failed", ex); + await this.MarkAsFailedAsync(context, input.JobEntityId, ex.Message); - + throw; } } @@ -192,13 +219,14 @@ async Task ProcessBatchWithRetryAsync( List instanceIds, ExportJobConfiguration config) { + ILogger logger = context.CreateReplaySafeLogger(); string jobId = jobEntityId.Key; - + for (int attempt = 1; attempt <= MaxRetryAttempts; attempt++) { - this.logger.ExportJobOperationInfo( + logger.ExportJobOperationInfo( jobId, - nameof(ProcessBatchWithRetryAsync), + nameof(this.ProcessBatchWithRetryAsync), $"Processing batch of {instanceIds.Count} instances (attempt {attempt}/{MaxRetryAttempts})"); // Export all instances in the batch @@ -206,16 +234,16 @@ async Task ProcessBatchWithRetryAsync( // Check if all exports succeeded List failedResults = results.Where(r => !r.Success).ToList(); - + if (failedResults.Count == 0) { // All exports succeeded int exportedCount = results.Count; - this.logger.ExportJobOperationInfo( + logger.ExportJobOperationInfo( jobId, - nameof(ProcessBatchWithRetryAsync), + nameof(this.ProcessBatchWithRetryAsync), $"Batch export succeeded on attempt {attempt} - exported {exportedCount} instances"); - + return new BatchExportResult { AllSucceeded = true, @@ -225,9 +253,9 @@ async Task ProcessBatchWithRetryAsync( } // Some exports failed - this.logger.ExportJobOperationWarning( + logger.ExportJobOperationWarning( jobId, - nameof(ProcessBatchWithRetryAsync), + nameof(this.ProcessBatchWithRetryAsync), $"Batch export failed on attempt {attempt} - {failedResults.Count} failures out of {instanceIds.Count} instances"); // If this is the last attempt, return failures @@ -240,7 +268,7 @@ async Task ProcessBatchWithRetryAsync( LastAttempt: DateTimeOffset.UtcNow)).ToList(); int exportedCount = results.Count(r => r.Success); - + return new BatchExportResult { AllSucceeded = false, @@ -253,9 +281,9 @@ async Task ProcessBatchWithRetryAsync( int backoffSeconds = Math.Min(MinBackoffSeconds * (int)Math.Pow(2, attempt - 1), MaxBackoffSeconds); TimeSpan backoffDelay = TimeSpan.FromSeconds(backoffSeconds); - this.logger.ExportJobOperationInfo( + logger.ExportJobOperationInfo( jobId, - nameof(ProcessBatchWithRetryAsync), + nameof(this.ProcessBatchWithRetryAsync), $"Retrying batch export after {backoffDelay.TotalMinutes:F1} minutes (attempt {attempt + 1}/{MaxRetryAttempts})"); // Wait before retrying @@ -265,7 +293,7 @@ async Task ProcessBatchWithRetryAsync( // Should not reach here, but return empty result if we do return new BatchExportResult { - AllSucceeded = false, + AllSucceeded = true, ExportedCount = 0, Failures = new List(), }; @@ -289,10 +317,12 @@ async Task> ExportBatchAsync( Format = config.Format, }; + // Use retry policy for individual export activities (up to 3 attempts) exportTasks.Add( context.CallActivityAsync( nameof(ExportInstanceHistoryActivity), - exportRequest)); + exportRequest, + new TaskOptions(ExportActivityRetryPolicy))); // Limit parallel export activities if (exportTasks.Count >= config.MaxParallelExports) @@ -359,7 +389,9 @@ await context.Entities.CallEntityAsync( sealed class BatchExportResult { public bool AllSucceeded { get; set; } + public int ExportedCount { get; set; } + public List? Failures { get; set; } } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 32cb6add2..8596242c7 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -475,6 +475,19 @@ message QueryInstancesResponse { google.protobuf.StringValue continuationToken = 2; } +message ListInstanceIdsRequest { + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp completedTimeFrom = 2; + google.protobuf.Timestamp completedTimeTo = 3; + int32 pageSize = 4; + google.protobuf.StringValue lastInstanceKey = 5; +} + +message ListInstanceIdsResponse { + repeated string instanceIds = 1; + google.protobuf.StringValue lastInstanceKey = 2; +} + // Removed ListTerminalInstances in favor of using QueryInstances message PurgeInstancesRequest { @@ -741,6 +754,7 @@ service TaskHubSidecarService { // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); + rpc ListInstanceIds(ListInstanceIdsRequest) returns (ListInstanceIdsResponse); rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); diff --git a/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs b/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs index 0f2baa0b2..7eedcfe71 100644 --- a/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs +++ b/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs @@ -2,10 +2,10 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.ScheduledTasks; -// TODO: logging // TODO: May need separate orchs, result is obj now /// @@ -18,7 +18,23 @@ public class ExecuteScheduleOperationOrchestrator : TaskOrchestrator public override async Task RunAsync(TaskOrchestrationContext context, ScheduleOperationRequest input) { - return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + ILogger logger = context.CreateReplaySafeLogger(); + + try + { + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + } + catch (Exception ex) + { + logger.LogError( + ex, + "Failed to execute schedule operation '{OperationName}' on entity '{EntityId}': {ErrorMessage}", + input.OperationName, + input.EntityId, + ex.Message); + + return null!; + } } } From ced2c4b0890789c48be3d96652f157e2b245199f Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Thu, 6 Nov 2025 19:33:05 -0800 Subject: [PATCH 4/7] revert --- .../Sidecar/Grpc/TaskHubGrpcServer.cs | 4 +--- .../ExecuteScheduleOperationOrchestrator.cs | 20 ++----------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs index 800cb22c3..b25177ae9 100644 --- a/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System.Collections.Concurrent; @@ -327,8 +327,6 @@ await this.client.ForceTerminateTaskOrchestrationAsync( } } - // Removed ListTerminalInstances; use QueryInstances instead on client side - /// /// Purges orchestration instances. /// diff --git a/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs b/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs index 7eedcfe71..0f2baa0b2 100644 --- a/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs +++ b/src/ScheduledTasks/Orchestrations/ExecuteScheduleOperationOrchestrator.cs @@ -2,10 +2,10 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Entities; -using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.ScheduledTasks; +// TODO: logging // TODO: May need separate orchs, result is obj now /// @@ -18,23 +18,7 @@ public class ExecuteScheduleOperationOrchestrator : TaskOrchestrator public override async Task RunAsync(TaskOrchestrationContext context, ScheduleOperationRequest input) { - ILogger logger = context.CreateReplaySafeLogger(); - - try - { - return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); - } - catch (Exception ex) - { - logger.LogError( - ex, - "Failed to execute schedule operation '{OperationName}' on entity '{EntityId}': {ErrorMessage}", - input.OperationName, - input.EntityId, - ex.Message); - - return null!; - } + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); } } From 46b01e472f9948ff2d4f24bea55b45bec7ce264e Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Thu, 6 Nov 2025 19:41:16 -0800 Subject: [PATCH 5/7] warning fix --- .../ExportInstanceHistoryActivity.cs | 29 +++---- .../Client/DefaultExportHistoryClient.cs | 6 +- .../Client/DefaultExportHistoryJobClient.cs | 21 ++--- .../Client/ExportHistoryClient.cs | 29 ++++++- .../Client/ExportHistoryJobClient.cs | 37 +++++--- .../Constants/ExportHistoryConstants.cs | 2 +- src/ExportHistory/Entity/ExportJob.cs | 87 +++++++++---------- .../Entity/ExportJobOperations.cs | 1 - .../ExportJobClientValidationException.cs | 1 - .../ExportJobInvalidTransitionException.cs | 1 - .../Exception/ExportJobNotFoundException.cs | 2 +- .../DurableTaskClientBuilderExtensions.cs | 3 +- src/ExportHistory/Logging/Logs.Client.cs | 1 - .../Models/CommitCheckpointRequest.cs | 1 - src/ExportHistory/Models/ExportCheckpoint.cs | 2 +- src/ExportHistory/Models/ExportDestination.cs | 11 ++- src/ExportHistory/Models/ExportFailure.cs | 1 - src/ExportHistory/Models/ExportFilter.cs | 9 +- src/ExportHistory/Models/ExportFormat.cs | 5 ++ .../Models/ExportJobConfiguration.cs | 11 ++- .../Models/ExportJobCreationOptions.cs | 17 ++-- .../Models/ExportJobDescription.cs | 24 +++-- src/ExportHistory/Models/ExportJobState.cs | 1 - src/ExportHistory/Models/ExportJobStatus.cs | 2 +- .../Models/ExportJobTransitions.cs | 1 - src/ExportHistory/Models/ExportMode.cs | 5 +- .../Options/ExportHistoryStorageOptions.cs | 1 - 27 files changed, 176 insertions(+), 135 deletions(-) diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs index 741ac6e81..7a0b19537 100644 --- a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -18,25 +18,18 @@ namespace Microsoft.DurableTask.ExportHistory; /// /// Activity that exports one orchestration instance history to the configured blob destination. /// +/// +/// Initializes a new instance of the class. +/// [DurableTask] -public class ExportInstanceHistoryActivity : TaskActivity +public class ExportInstanceHistoryActivity( + IDurableTaskClientProvider clientProvider, + ILogger logger, + IOptions storageOptions) : TaskActivity { - readonly IDurableTaskClientProvider clientProvider; - readonly ILogger logger; - readonly ExportHistoryStorageOptions storageOptions; - - /// - /// Initializes a new instance of the class. - /// - public ExportInstanceHistoryActivity( - IDurableTaskClientProvider clientProvider, - ILogger logger, - IOptions storageOptions) - { - this.clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); - this.logger = Check.NotNull(logger, nameof(logger)); - this.storageOptions = Check.NotNull(storageOptions?.Value, nameof(storageOptions)); - } + readonly IDurableTaskClientProvider clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + readonly ILogger logger = Check.NotNull(logger, nameof(logger)); + readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions?.Value, nameof(storageOptions)); /// public override async Task RunAsync(TaskActivityContext context, ExportRequest input) @@ -236,7 +229,7 @@ await containerClient.CreateIfNotExistsAsync( // Upload content byte[] contentBytes = Encoding.UTF8.GetBytes(content); - if (format.Kind.Equals("jsonl", StringComparison.InvariantCultureIgnoreCase)) + if (format.Kind.Equals("jsonl", StringComparison.OrdinalIgnoreCase)) { // Compress with gzip using MemoryStream compressedStream = new(); diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs index 484e7e612..b76bea6b6 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -3,7 +3,6 @@ using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.Entities; -using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.ExportHistory; @@ -35,8 +34,7 @@ public override async Task CreateJobAsync( this.durableTaskClient, options.JobId, this.logger, - this.storageOptions - ); + this.storageOptions); // Create the export job using the client (validation already done in constructor) await exportHistoryJobClient.CreateAsync(options, cancellation); @@ -189,4 +187,4 @@ static bool MatchesFilter(ExportJobState state, ExportJobQuery filter) return statusMatches && createdFromMatches && createdToMatches; } -} \ No newline at end of file +} diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs index 52418723a..b2a53c460 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -16,14 +16,14 @@ public sealed class DefaultExportHistoryJobClient( DurableTaskClient durableTaskClient, string jobId, ILogger logger, - ExportHistoryStorageOptions storageOptions -) : ExportHistoryJobClient(jobId) + ExportHistoryStorageOptions storageOptions) : ExportHistoryJobClient(jobId) { readonly DurableTaskClient durableTaskClient = Check.NotNull(durableTaskClient, nameof(durableTaskClient)); readonly ILogger logger = Check.NotNull(logger, nameof(logger)); readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions, nameof(storageOptions)); readonly EntityInstanceId entityId = new(nameof(ExportJob), jobId); + /// public override async Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default) { try @@ -31,7 +31,7 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell Check.NotNull(options, nameof(options)); // Determine default prefix based on mode if not already set - string? defaultPrefix = $"{options.Mode.ToString().ToLower()}/"; + string? defaultPrefix = $"{options.Mode.ToString().ToLower(System.Globalization.CultureInfo.CurrentCulture)}/"; // If destination is not provided, construct it from storage options ExportJobCreationOptions optionsWithDestination = options; @@ -57,10 +57,10 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell optionsWithDestination = options with { Destination = destinationWithPrefix }; } - ExportJobOperationRequest request = + ExportJobOperationRequest request = new ExportJobOperationRequest( - this.entityId, - nameof(ExportJob.Create), + this.entityId, + nameof(ExportJob.Create), optionsWithDestination); string instanceId = await this.durableTaskClient @@ -72,8 +72,8 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell // Wait for the orchestration to complete OrchestrationMetadata state = await this.durableTaskClient .WaitForInstanceCompletionAsync( - instanceId, - true, + instanceId, + true, cancellation); if (state.RuntimeStatus != OrchestrationRuntimeStatus.Completed) @@ -98,6 +98,8 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell // TODO: there is no atomicity guarantee of deleting entity and purging the orchestrator // Add sweeping process to clean up orphaned orchestrations failed to be purged + + /// public override async Task DeleteAsync(CancellationToken cancellation = default) { try @@ -129,6 +131,7 @@ public override async Task DeleteAsync(CancellationToken cancellation = default) } } + /// public override async Task DescribeAsync(CancellationToken cancellation = default) { try @@ -211,5 +214,3 @@ await this.durableTaskClient.TerminateInstanceAsync( } } } - - diff --git a/src/ExportHistory/Client/ExportHistoryClient.cs b/src/ExportHistory/Client/ExportHistoryClient.cs index 18c06feeb..c53b6f3dd 100644 --- a/src/ExportHistory/Client/ExportHistoryClient.cs +++ b/src/ExportHistory/Client/ExportHistoryClient.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Entities; - namespace Microsoft.DurableTask.ExportHistory; /// @@ -11,9 +8,33 @@ namespace Microsoft.DurableTask.ExportHistory; /// public abstract class ExportHistoryClient { + /// + /// Creates a new export job. + /// + /// The options for the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. public abstract Task CreateJobAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + + /// + /// Gets an export job. + /// + /// The ID of the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. public abstract Task GetJobAsync(string jobId, CancellationToken cancellation = default); + /// + /// Lists export jobs. + /// + /// The filter for the export jobs. + /// A task representing the asynchronous operation. public abstract AsyncPageable ListJobsAsync(ExportJobQuery? filter = null); + + /// + /// Gets an export job client. + /// + /// The ID of the export job. + /// The export job client. public abstract ExportHistoryJobClient GetJobClient(string jobId); -} \ No newline at end of file +} diff --git a/src/ExportHistory/Client/ExportHistoryJobClient.cs b/src/ExportHistory/Client/ExportHistoryJobClient.cs index 8d9994e9e..acd4aca45 100644 --- a/src/ExportHistory/Client/ExportHistoryJobClient.cs +++ b/src/ExportHistory/Client/ExportHistoryJobClient.cs @@ -1,29 +1,40 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Entities; - namespace Microsoft.DurableTask.ExportHistory; /// /// Convenience client for managing export jobs via entity signals and reads. /// -public abstract class ExportHistoryJobClient +/// +/// Initializes a new instance of the class. +/// +public abstract class ExportHistoryJobClient(string jobId) { - public readonly string JobId; - /// - /// Initializes a new instance of the class. + /// The ID of the export job. /// - protected ExportHistoryJobClient(string jobId) - { - this.JobId = Check.NotNullOrEmpty(jobId, nameof(jobId)); - } + protected readonly string JobId = Check.NotNullOrEmpty(jobId, nameof(jobId)); + /// + /// Creates a new export job. + /// + /// The options for the export job. + /// The cancellation token. + /// A task representing the asynchronous operation. public abstract Task CreateAsync(ExportJobCreationOptions options, CancellationToken cancellation = default); + + /// + /// Describes the export job. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. public abstract Task DescribeAsync(CancellationToken cancellation = default); + + /// + /// Deletes the export job. + /// + /// The cancellation token. + /// A task representing the asynchronous operation. public abstract Task DeleteAsync(CancellationToken cancellation = default); } - - diff --git a/src/ExportHistory/Constants/ExportHistoryConstants.cs b/src/ExportHistory/Constants/ExportHistoryConstants.cs index 567d0f3bc..964082db4 100644 --- a/src/ExportHistory/Constants/ExportHistoryConstants.cs +++ b/src/ExportHistory/Constants/ExportHistoryConstants.cs @@ -10,7 +10,7 @@ static class ExportHistoryConstants { /// /// The prefix pattern used for generating export job orchestrator instance IDs. - /// Format: "ExportJob-{jobId}" + /// Format: "ExportJob-{jobId}". /// public const string OrchestratorInstanceIdPrefix = "ExportJob-"; diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs index af23deec0..9d33b6cd4 100644 --- a/src/ExportHistory/Entity/ExportJob.cs +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Logging; @@ -39,7 +38,7 @@ public void Create(TaskEntityContext context, ExportJobCreationOptions creationO // Note: RuntimeStatus validation already done in ExportJobCreationOptions constructor // Note: Destination should be populated by the client before reaching here Verify.NotNull(creationOptions.Destination, nameof(creationOptions.Destination)); - + ExportJobConfiguration config = new ExportJobConfiguration( Mode: creationOptions.Mode, Filter: new ExportFilter( @@ -114,48 +113,6 @@ public void Run(TaskEntityContext context) } } - void StartExportOrchestration(TaskEntityContext context) - { - try - { - // Use a fixed instance ID based on job ID to ensure only one orchestrator runs per job - // This prevents concurrent orchestrators if Run is called multiple times - string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(context.Id.Key); - StartOrchestrationOptions startOrchestrationOptions = new StartOrchestrationOptions(instanceId); - - logger.ExportJobOperationInfo( - context.Id.Key, - nameof(this.StartExportOrchestration), - $"Starting new orchestration named '{nameof(ExportJobOrchestrator)}' with instance ID: {instanceId}"); - - context.ScheduleNewOrchestration( - new TaskName(nameof(ExportJobOrchestrator)), - new ExportJobRunRequest(context.Id), - startOrchestrationOptions); - - this.State.OrchestratorInstanceId = instanceId; - this.State.LastModifiedAt = DateTimeOffset.UtcNow; - } - catch (Exception ex) - { - // Mark job as failed and record the exception - this.State.Status = ExportJobStatus.Failed; - this.State.LastError = ex.Message; - this.State.LastModifiedAt = DateTimeOffset.UtcNow; - - logger.ExportJobOperationError( - context.Id.Key, - nameof(this.StartExportOrchestration), - "Failed to start export orchestration", - ex); - } - } - - bool CanTransitionTo(string operationName, ExportJobStatus targetStatus) - { - return ExportJobTransitions.IsValidTransition(operationName, this.State.Status, targetStatus); - } - /// /// Commits a checkpoint snapshot with progress updates and optional failures. /// @@ -292,4 +249,46 @@ public void Delete(TaskEntityContext context) throw; } } + + void StartExportOrchestration(TaskEntityContext context) + { + try + { + // Use a fixed instance ID based on job ID to ensure only one orchestrator runs per job + // This prevents concurrent orchestrators if Run is called multiple times + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new StartOrchestrationOptions(instanceId); + + logger.ExportJobOperationInfo( + context.Id.Key, + nameof(this.StartExportOrchestration), + $"Starting new orchestration named '{nameof(ExportJobOrchestrator)}' with instance ID: {instanceId}"); + + context.ScheduleNewOrchestration( + new TaskName(nameof(ExportJobOrchestrator)), + new ExportJobRunRequest(context.Id), + startOrchestrationOptions); + + this.State.OrchestratorInstanceId = instanceId; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + catch (Exception ex) + { + // Mark job as failed and record the exception + this.State.Status = ExportJobStatus.Failed; + this.State.LastError = ex.Message; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.ExportJobOperationError( + context.Id.Key, + nameof(this.StartExportOrchestration), + "Failed to start export orchestration", + ex); + } + } + + bool CanTransitionTo(string operationName, ExportJobStatus targetStatus) + { + return ExportJobTransitions.IsValidTransition(operationName, this.State.Status, targetStatus); + } } diff --git a/src/ExportHistory/Entity/ExportJobOperations.cs b/src/ExportHistory/Entity/ExportJobOperations.cs index 339abf4bb..3ec695168 100644 --- a/src/ExportHistory/Entity/ExportJobOperations.cs +++ b/src/ExportHistory/Entity/ExportJobOperations.cs @@ -17,4 +17,3 @@ static class ExportJobOperations /// public const string Delete = "Delete"; } - diff --git a/src/ExportHistory/Exception/ExportJobClientValidationException.cs b/src/ExportHistory/Exception/ExportJobClientValidationException.cs index c2d34a174..25118f087 100644 --- a/src/ExportHistory/Exception/ExportJobClientValidationException.cs +++ b/src/ExportHistory/Exception/ExportJobClientValidationException.cs @@ -36,4 +36,3 @@ public ExportJobClientValidationException(string jobId, string message, Exceptio /// public string JobId { get; } } - diff --git a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs index 13e4174c8..05119c30d 100644 --- a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs +++ b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs @@ -61,4 +61,3 @@ public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromSta /// public string OperationName { get; } } - diff --git a/src/ExportHistory/Exception/ExportJobNotFoundException.cs b/src/ExportHistory/Exception/ExportJobNotFoundException.cs index d1c2f9f19..0889c7731 100644 --- a/src/ExportHistory/Exception/ExportJobNotFoundException.cs +++ b/src/ExportHistory/Exception/ExportJobNotFoundException.cs @@ -19,7 +19,7 @@ public ExportJobNotFoundException(string jobId) } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The ID of the export history job that was not found. /// The exception that is the cause of the current exception. diff --git a/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs index 99b95624a..ed8285e66 100644 --- a/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs +++ b/src/ExportHistory/Extension/DurableTaskClientBuilderExtensions.cs @@ -31,7 +31,8 @@ public static IDurableTaskClientBuilder UseExportHistory( // Register and validate options services.AddOptions() .Configure(configure) - .Validate(o => + .Validate( + o => !string.IsNullOrEmpty(o.ConnectionString) && !string.IsNullOrEmpty(o.ContainerName), $"{nameof(ExportHistoryStorageOptions)} must specify both {nameof(ExportHistoryStorageOptions.ConnectionString)} and {nameof(ExportHistoryStorageOptions.ContainerName)}."); diff --git a/src/ExportHistory/Logging/Logs.Client.cs b/src/ExportHistory/Logging/Logs.Client.cs index 87f98bf1b..f76c30264 100644 --- a/src/ExportHistory/Logging/Logs.Client.cs +++ b/src/ExportHistory/Logging/Logs.Client.cs @@ -13,7 +13,6 @@ static partial class Logs [LoggerMessage(EventId = 80, Level = LogLevel.Information, Message = "Creating export job with options: {exportJobCreationOptions}")] public static partial void ClientCreatingExportJob(this ILogger logger, ExportJobCreationOptions exportJobCreationOptions); - [LoggerMessage(EventId = 84, Level = LogLevel.Information, Message = "Deleting export job '{jobId}'")] public static partial void ClientDeletingExportJob(this ILogger logger, string jobId); diff --git a/src/ExportHistory/Models/CommitCheckpointRequest.cs b/src/ExportHistory/Models/CommitCheckpointRequest.cs index 81745f5d6..a66152411 100644 --- a/src/ExportHistory/Models/CommitCheckpointRequest.cs +++ b/src/ExportHistory/Models/CommitCheckpointRequest.cs @@ -29,4 +29,3 @@ public sealed class CommitCheckpointRequest /// public List? Failures { get; set; } } - diff --git a/src/ExportHistory/Models/ExportCheckpoint.cs b/src/ExportHistory/Models/ExportCheckpoint.cs index efe0765c9..561dad6ea 100644 --- a/src/ExportHistory/Models/ExportCheckpoint.cs +++ b/src/ExportHistory/Models/ExportCheckpoint.cs @@ -1,5 +1,5 @@ -// Licensed under the MIT License. // Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. namespace Microsoft.DurableTask.ExportHistory; diff --git a/src/ExportHistory/Models/ExportDestination.cs b/src/ExportHistory/Models/ExportDestination.cs index d4c11b04c..bf6a62438 100644 --- a/src/ExportHistory/Models/ExportDestination.cs +++ b/src/ExportHistory/Models/ExportDestination.cs @@ -8,7 +8,13 @@ namespace Microsoft.DurableTask.ExportHistory; /// public sealed class ExportDestination { - public ExportDestination() { } + /// + /// Initializes a new instance of the class. + /// + public ExportDestination() + { + } + /// /// Initializes a new instance of the class. /// @@ -21,7 +27,7 @@ public ExportDestination(string container) } /// - /// Gets the blob container name. + /// Gets or sets the blob container name. /// public string Container { get; set; } @@ -30,4 +36,3 @@ public ExportDestination(string container) /// public string? Prefix { get; set; } } - diff --git a/src/ExportHistory/Models/ExportFailure.cs b/src/ExportHistory/Models/ExportFailure.cs index bd753a0dd..422ded650 100644 --- a/src/ExportHistory/Models/ExportFailure.cs +++ b/src/ExportHistory/Models/ExportFailure.cs @@ -11,4 +11,3 @@ namespace Microsoft.DurableTask.ExportHistory; /// The number of attempts made. /// The timestamp of the last attempt. public sealed record ExportFailure(string InstanceId, string Reason, int AttemptCount, DateTimeOffset LastAttempt); - diff --git a/src/ExportHistory/Models/ExportFilter.cs b/src/ExportHistory/Models/ExportFilter.cs index ee6f4ac52..2e572edaa 100644 --- a/src/ExportHistory/Models/ExportFilter.cs +++ b/src/ExportHistory/Models/ExportFilter.cs @@ -2,11 +2,16 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Entities; namespace Microsoft.DurableTask.ExportHistory; +/// +/// Filter criteria for selecting orchestration instances to export. +/// +/// The start time for the export based on completion time (inclusive). +/// The end time for the export based on completion time (inclusive). Optional. +/// The orchestration runtime statuses to filter by. Optional. public record ExportFilter( DateTimeOffset CompletedTimeFrom, DateTimeOffset? CompletedTimeTo = null, - IEnumerable? RuntimeStatus = null); \ No newline at end of file + IEnumerable? RuntimeStatus = null); diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs index e441c09f5..2047e5edc 100644 --- a/src/ExportHistory/Models/ExportFormat.cs +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -3,6 +3,11 @@ namespace Microsoft.DurableTask.ExportHistory; +/// +/// Export format settings. +/// +/// The kind of export format. +/// The schema version. public partial record ExportFormat( string Kind = "jsonl", string SchemaVersion = "1.0") diff --git a/src/ExportHistory/Models/ExportJobConfiguration.cs b/src/ExportHistory/Models/ExportJobConfiguration.cs index 42f7ac406..1de6e51a5 100644 --- a/src/ExportHistory/Models/ExportJobConfiguration.cs +++ b/src/ExportHistory/Models/ExportJobConfiguration.cs @@ -3,10 +3,19 @@ namespace Microsoft.DurableTask.ExportHistory; +/// +/// Configuration for an export job. +/// +/// The export mode (Batch or Continuous). +/// The filter criteria for selecting orchestration instances to export. +/// The export destination where exported data will be stored. +/// The export format settings. +/// The maximum number of parallel export operations. Defaults to 32. +/// The maximum number of instances to fetch per batch. Defaults to 100. public record ExportJobConfiguration( ExportMode Mode, ExportFilter Filter, ExportDestination Destination, ExportFormat Format, int MaxParallelExports = 32, - int MaxInstancesPerBatch = 100); \ No newline at end of file + int MaxInstancesPerBatch = 100); diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs index 91e081eb3..fddacc0cd 100644 --- a/src/ExportHistory/Models/ExportJobCreationOptions.cs +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Text.Json.Serialization; using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Entities; namespace Microsoft.DurableTask.ExportHistory; @@ -11,6 +9,9 @@ namespace Microsoft.DurableTask.ExportHistory; /// public record ExportJobCreationOptions { + /// + /// Initializes a new instance of the class. + /// public ExportJobCreationOptions() { } @@ -69,7 +70,9 @@ public ExportJobCreationOptions( $"CompletedTimeTo({completedTimeTo.Value}) cannot be in the future. It must be less than or equal to the current time ({DateTimeOffset.UtcNow}).", nameof(completedTimeTo)); } - } else if (mode == ExportMode.Continuous) { + } + else if (mode == ExportMode.Continuous) + { if (completedTimeFrom != default) { throw new ArgumentException( @@ -83,7 +86,9 @@ public ExportJobCreationOptions( "CompletedTimeTo is not allowed for Continuous export mode.", nameof(completedTimeTo)); } - } else { + } + else + { throw new ArgumentException( "Invalid export mode.", nameof(mode)); @@ -99,7 +104,7 @@ public ExportJobCreationOptions( } // Validate terminal status-only filter here if provided - if (runtimeStatus?.Any() == true + if (runtimeStatus is { Count: > 0 } && runtimeStatus.Any( s => s is not (OrchestrationRuntimeStatus.Completed or OrchestrationRuntimeStatus.Failed @@ -123,7 +128,7 @@ or OrchestrationRuntimeStatus.Terminated OrchestrationRuntimeStatus.Completed, OrchestrationRuntimeStatus.Failed, OrchestrationRuntimeStatus.Terminated, - OrchestrationRuntimeStatus.ContinuedAsNew + OrchestrationRuntimeStatus.ContinuedAsNew, }; this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; } diff --git a/src/ExportHistory/Models/ExportJobDescription.cs b/src/ExportHistory/Models/ExportJobDescription.cs index 986618c68..189b54f6f 100644 --- a/src/ExportHistory/Models/ExportJobDescription.cs +++ b/src/ExportHistory/Models/ExportJobDescription.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Text.Json; - namespace Microsoft.DurableTask.ExportHistory; /// @@ -11,57 +9,57 @@ namespace Microsoft.DurableTask.ExportHistory; public record ExportJobDescription { /// - /// Gets or sets the job identifier. + /// Gets the job identifier. /// public string JobId { get; init; } = string.Empty; /// - /// Gets or sets the export job status. + /// Gets the export job status. /// public ExportJobStatus Status { get; init; } /// - /// Gets or sets the time when this export job was created. + /// Gets the time when this export job was created. /// public DateTimeOffset? CreatedAt { get; init; } /// - /// Gets or sets the time when this export job was last modified. + /// Gets the time when this export job was last modified. /// public DateTimeOffset? LastModifiedAt { get; init; } /// - /// Gets or sets the export job configuration. + /// Gets the export job configuration. /// public ExportJobConfiguration? Config { get; init; } /// - /// Gets or sets the instance ID of the running export orchestrator, if any. + /// Gets the instance ID of the running export orchestrator, if any. /// public string? OrchestratorInstanceId { get; init; } /// - /// Gets or sets the total number of instances scanned. + /// Gets the total number of instances scanned. /// public long ScannedInstances { get; init; } /// - /// Gets or sets the total number of instances exported. + /// Gets the total number of instances exported. /// public long ExportedInstances { get; init; } /// - /// Gets or sets the last error message, if any. + /// Gets the last error message, if any. /// public string? LastError { get; init; } /// - /// Gets or sets the checkpoint for resuming the export. + /// Gets the checkpoint for resuming the export. /// public ExportCheckpoint? Checkpoint { get; init; } /// - /// Gets or sets the time of the last checkpoint. + /// Gets the time of the last checkpoint. /// public DateTimeOffset? LastCheckpointTime { get; init; } } diff --git a/src/ExportHistory/Models/ExportJobState.cs b/src/ExportHistory/Models/ExportJobState.cs index 2b09b8587..77a289478 100644 --- a/src/ExportHistory/Models/ExportJobState.cs +++ b/src/ExportHistory/Models/ExportJobState.cs @@ -58,4 +58,3 @@ public sealed class ExportJobState /// public string? OrchestratorInstanceId { get; set; } } - diff --git a/src/ExportHistory/Models/ExportJobStatus.cs b/src/ExportHistory/Models/ExportJobStatus.cs index 2ff48a68f..dbd5b7c27 100644 --- a/src/ExportHistory/Models/ExportJobStatus.cs +++ b/src/ExportHistory/Models/ExportJobStatus.cs @@ -26,5 +26,5 @@ public enum ExportJobStatus /// /// Export history job completed. /// - Completed + Completed, } diff --git a/src/ExportHistory/Models/ExportJobTransitions.cs b/src/ExportHistory/Models/ExportJobTransitions.cs index 8e1a24f36..3fbc43020 100644 --- a/src/ExportHistory/Models/ExportJobTransitions.cs +++ b/src/ExportHistory/Models/ExportJobTransitions.cs @@ -40,4 +40,3 @@ public static bool IsValidTransition(string operationName, ExportJobStatus from, }; } } - diff --git a/src/ExportHistory/Models/ExportMode.cs b/src/ExportHistory/Models/ExportMode.cs index 2e3ee0141..07b7cfca9 100644 --- a/src/ExportHistory/Models/ExportMode.cs +++ b/src/ExportHistory/Models/ExportMode.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Entities; - namespace Microsoft.DurableTask.ExportHistory; /// @@ -12,6 +10,7 @@ public enum ExportMode { /// Exports a fixed window and completes. Batch = 1, + /// Tails terminal instances continuously. Continuous = 2, -} \ No newline at end of file +} diff --git a/src/ExportHistory/Options/ExportHistoryStorageOptions.cs b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs index 21d838187..ad9a590c8 100644 --- a/src/ExportHistory/Options/ExportHistoryStorageOptions.cs +++ b/src/ExportHistory/Options/ExportHistoryStorageOptions.cs @@ -24,4 +24,3 @@ public sealed class ExportHistoryStorageOptions /// public string? Prefix { get; set; } } - From 2df6b5390cc4e5c97bf1a9b28c9689f2300c896b Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:10:03 -0800 Subject: [PATCH 6/7] unit tests --- .../Client/DefaultExportHistoryClientTests.cs | 275 +++++++++++ .../DefaultExportHistoryJobClientTests.cs | 333 +++++++++++++ .../Constants/ExportHistoryConstantsTests.cs | 45 ++ .../Entity/ExportJobTests.cs | 370 +++++++++++++++ ...ExportJobClientValidationExceptionTests.cs | 80 ++++ ...xportJobInvalidTransitionExceptionTests.cs | 80 ++++ .../ExportJobNotFoundExceptionTests.cs | 59 +++ .../ExportHistory.Tests.csproj | 19 + .../ExportJobTransitionsTests.cs | 108 +++++ .../Models/CommitCheckpointRequestTests.cs | 69 +++ .../Models/ExportCheckpointTests.cs | 71 +++ .../Models/ExportDestinationTests.cs | 80 ++++ .../Models/ExportFailureTests.cs | 60 +++ .../Models/ExportFilterTests.cs | 66 +++ .../Models/ExportFormatTests.cs | 62 +++ .../Models/ExportJobConfigurationTests.cs | 74 +++ .../Models/ExportJobCreationOptionsTests.cs | 441 ++++++++++++++++++ .../Models/ExportJobDescriptionTests.cs | 75 +++ .../Models/ExportJobQueryTests.cs | 88 ++++ .../Models/ExportJobStateTests.cs | 69 +++ .../Models/ExportJobStatusTests.cs | 36 ++ .../Models/ExportModeTests.cs | 28 ++ .../ExportHistoryStorageOptionsTests.cs | 57 +++ ...cuteExportJobOperationOrchestratorTests.cs | 99 ++++ test/ExportHistory.Tests/Usings.cs | 6 + 25 files changed, 2750 insertions(+) create mode 100644 test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs create mode 100644 test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs create mode 100644 test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs create mode 100644 test/ExportHistory.Tests/Entity/ExportJobTests.cs create mode 100644 test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs create mode 100644 test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs create mode 100644 test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs create mode 100644 test/ExportHistory.Tests/ExportHistory.Tests.csproj create mode 100644 test/ExportHistory.Tests/ExportJobTransitionsTests.cs create mode 100644 test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportCheckpointTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportDestinationTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportFailureTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportFilterTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportFormatTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobQueryTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobStateTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportJobStatusTests.cs create mode 100644 test/ExportHistory.Tests/Models/ExportModeTests.cs create mode 100644 test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs create mode 100644 test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs create mode 100644 test/ExportHistory.Tests/Usings.cs diff --git a/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs b/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs new file mode 100644 index 000000000..9fb27fa3a --- /dev/null +++ b/test/ExportHistory.Tests/Client/DefaultExportHistoryClientTests.cs @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Client; + +public class DefaultExportHistoryClientTests +{ + readonly Mock durableTaskClient; + readonly Mock entityClient; + readonly ILogger logger; + readonly ExportHistoryStorageOptions storageOptions; + readonly DefaultExportHistoryClient client; + + public DefaultExportHistoryClientTests() + { + this.durableTaskClient = new Mock("test"); + this.entityClient = new Mock("test"); + this.logger = new TestLogger(); + this.storageOptions = new ExportHistoryStorageOptions + { + ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net", + ContainerName = "test-container", + }; + this.durableTaskClient.Setup(x => x.Entities).Returns(this.entityClient.Object); + this.client = new DefaultExportHistoryClient( + this.durableTaskClient.Object, + this.logger, + this.storageOptions); + } + + [Fact] + public void Constructor_WithNullClient_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + null!, + this.logger, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("durableTaskClient"); + } + + [Fact] + public void Constructor_WithNullLogger_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + this.durableTaskClient.Object, + null!, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("logger"); + } + + [Fact] + public void Constructor_WithNullStorageOptions_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryClient( + this.durableTaskClient.Object, + this.logger, + null!); + + act.Should().Throw() + .WithParameterName("storageOptions"); + } + + [Fact] + public void GetJobClient_ReturnsValidClient() + { + // Arrange + string jobId = "test-job"; + + // Act + var jobClient = this.client.GetJobClient(jobId); + + // Assert + jobClient.Should().NotBeNull(); + jobClient.Should().BeOfType(); + } + + [Fact] + public async Task CreateJobAsync_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + // Act + var jobClient = await this.client.CreateJobAsync(options); + + // Assert + jobClient.Should().NotBeNull(); + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreateJobAsync_WithNullOptions_ThrowsArgumentNullException() + { + // Act & Assert + Func act = async () => await this.client.CreateJobAsync(null!); + await act.Should().ThrowAsync() + .WithParameterName("options"); + } + + [Fact] + public async Task GetJobAsync_WhenExists_ReturnsDescription() + { + // Arrange + string jobId = "test-job"; + var state = new ExportJobState + { + Status = ExportJobStatus.Active, + Config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default), + }; + + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync(new EntityMetadata(entityInstanceId, state)); + + // Act + var description = await this.client.GetJobAsync(jobId); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().Be(jobId); + description.Status.Should().Be(state.Status); + } + + [Fact] + public async Task GetJobAsync_WhenNotExists_ThrowsExportJobNotFoundException() + { + // Arrange + string jobId = "test-job"; + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync((EntityMetadata?)null); + + // Act & Assert + Func act = async () => await this.client.GetJobAsync(jobId); + await act.Should().ThrowAsync() + .Where(ex => ex.JobId == jobId); + } + + [Fact] + public async Task ListJobsAsync_WithNoFilter_ReturnsAllJobs() + { + // Arrange + var state1 = new ExportJobState + { + Status = ExportJobStatus.Active, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-1), + }; + + var state2 = new ExportJobState + { + Status = ExportJobStatus.Completed, + CreatedAt = DateTimeOffset.UtcNow, + }; + + var entity1 = new EntityInstanceId(nameof(ExportJob), "job-1"); + var entity2 = new EntityInstanceId(nameof(ExportJob), "job-2"); + + var metadata1 = new EntityMetadata(entity1, state1); + var metadata2 = new EntityMetadata(entity2, state2); + + var page = new Page>( + new List> { metadata1, metadata2 }, + null); + + this.entityClient + .Setup(c => c.GetAllEntitiesAsync( + It.IsAny())) + .Returns(Pageable.Create((string? continuation, int? pageSize, CancellationToken cancellation) => + Task.FromResult(page))); + + // Act + var jobs = await this.client.ListJobsAsync().ToListAsync(); + + // Assert + jobs.Should().HaveCount(2); + jobs.Should().Contain(j => j.JobId == "job-1"); + jobs.Should().Contain(j => j.JobId == "job-2"); + } + + [Fact] + public async Task ListJobsAsync_WithStatusFilter_FiltersCorrectly() + { + // Arrange + var state1 = new ExportJobState + { + Status = ExportJobStatus.Active, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-1), + }; + + var state2 = new ExportJobState + { + Status = ExportJobStatus.Completed, + CreatedAt = DateTimeOffset.UtcNow, + }; + + var entity1 = new EntityInstanceId(nameof(ExportJob), "job-1"); + var entity2 = new EntityInstanceId(nameof(ExportJob), "job-2"); + + var metadata1 = new EntityMetadata(entity1, state1); + var metadata2 = new EntityMetadata(entity2, state2); + + var page = new Page>( + new List> { metadata1, metadata2 }, + null); + + this.entityClient + .Setup(c => c.GetAllEntitiesAsync( + It.IsAny())) + .Returns(Pageable.Create((string? continuation, int? pageSize, CancellationToken cancellation) => + Task.FromResult(page))); + + var filter = new ExportJobQuery + { + Status = ExportJobStatus.Active, + }; + + // Act + var jobs = await this.client.ListJobsAsync(filter).ToListAsync(); + + // Assert + jobs.Should().HaveCount(1); + jobs.Should().Contain(j => j.JobId == "job-1" && j.Status == ExportJobStatus.Active); + } +} + diff --git a/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs new file mode 100644 index 000000000..e1d91b0ae --- /dev/null +++ b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Client; + +public class DefaultExportHistoryJobClientTests +{ + readonly Mock durableTaskClient; + readonly Mock entityClient; + readonly ILogger logger; + readonly ExportHistoryStorageOptions storageOptions; + readonly string jobId = "test-job-123"; + readonly DefaultExportHistoryJobClient client; + + public DefaultExportHistoryJobClientTests() + { + this.durableTaskClient = new Mock("test"); + this.entityClient = new Mock("test"); + this.logger = new TestLogger(); + this.storageOptions = new ExportHistoryStorageOptions + { + ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net", + ContainerName = "test-container", + }; + this.durableTaskClient.Setup(x => x.Entities).Returns(this.entityClient.Object); + this.client = new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + this.logger, + this.storageOptions); + } + + [Fact] + public void Constructor_WithNullClient_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + null!, + this.jobId, + this.logger, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("durableTaskClient"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void Constructor_WithInvalidJobId_ThrowsArgumentException(string? invalidJobId) + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + invalidJobId!, + this.logger, + this.storageOptions); + + act.Should().Throw(); + } + + [Fact] + public void Constructor_WithWhitespaceJobId_DoesNotThrow() + { + // Arrange + // Check.NotNullOrEmpty only checks for null, empty, or strings starting with '\0' + // It does NOT check for whitespace-only strings, so " " is valid + string jobId = " "; + + // Act + var client = new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + jobId, + this.logger, + this.storageOptions); + + // Assert + client.Should().NotBeNull(); + } + + [Fact] + public void Constructor_WithNullLogger_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + null!, + this.storageOptions); + + act.Should().Throw() + .WithParameterName("logger"); + } + + [Fact] + public void Constructor_WithNullStorageOptions_ThrowsArgumentNullException() + { + // Act & Assert + Action act = () => new DefaultExportHistoryJobClient( + this.durableTaskClient.Object, + this.jobId, + this.logger, + null!); + + act.Should().Throw() + .WithParameterName("storageOptions"); + } + + [Fact] + public async Task DescribeAsync_WhenExists_ReturnsDescription() + { + // Arrange + var state = new ExportJobState + { + Status = ExportJobStatus.Active, + Config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default), + }; + + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), this.jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync(new EntityMetadata(entityInstanceId, state)); + + // Act + var description = await this.client.DescribeAsync(); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().Be(this.jobId); + description.Status.Should().Be(state.Status); + description.Config.Should().Be(state.Config); + } + + [Fact] + public async Task DescribeAsync_WhenNotExists_ThrowsExportJobNotFoundException() + { + // Arrange + var entityInstanceId = new EntityInstanceId(nameof(ExportJob), this.jobId); + + this.entityClient + .Setup(c => c.GetEntityAsync( + It.Is(id => id.Name == entityInstanceId.Name && id.Key == entityInstanceId.Key), + It.IsAny())) + .ReturnsAsync((EntityMetadata?)null); + + // Act & Assert + Func act = async () => await this.client.DescribeAsync(); + await act.Should().ThrowAsync() + .Where(ex => ex.JobId == this.jobId); + } + + [Fact] + public async Task CreateAsync_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + // Act + await this.client.CreateAsync(options); + + // Assert + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + + this.durableTaskClient.Verify( + c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreateAsync_WithNullOptions_ThrowsArgumentNullException() + { + // Act & Assert + Func act = async () => await this.client.CreateAsync(null!); + await act.Should().ThrowAsync() + .WithParameterName("options"); + } + + [Fact] + public async Task CreateAsync_WhenOrchestrationFails_ThrowsInvalidOperationException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + string instanceId = "test-instance"; + string errorMessage = "Test error message"; + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Failed, + FailureDetails = new TaskFailureDetails("TestError", errorMessage, null, null, null), + }); + + // Act & Assert + Func act = async () => await this.client.CreateAsync(options); + await act.Should().ThrowAsync() + .WithMessage($"*Failed to create export job '{this.jobId}'*"); + } + + [Fact] + public async Task DeleteAsync_ExecutesDeleteOperation() + { + // Arrange + string instanceId = "test-instance"; + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.jobId); + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + this.durableTaskClient + .Setup(c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(orchestratorInstanceId, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExportJobOrchestrator), orchestratorInstanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Terminated, + }); + + this.durableTaskClient + .Setup(c => c.PurgeInstanceAsync(orchestratorInstanceId, It.IsAny())) + .ReturnsAsync(new PurgeResult(1)); + + // Act + await this.client.DeleteAsync(); + + // Assert + this.durableTaskClient.Verify( + c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_WhenOrchestrationNotFound_HandlesGracefully() + { + // Arrange + string instanceId = "test-instance"; + string orchestratorInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.jobId); + + this.durableTaskClient + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.Is(n => n.Name == nameof(ExecuteExportJobOperationOrchestrator)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(instanceId); + + this.durableTaskClient + .Setup(c => c.WaitForInstanceCompletionAsync(instanceId, true, It.IsAny())) + .ReturnsAsync(new OrchestrationMetadata(nameof(ExecuteExportJobOperationOrchestrator), instanceId) + { + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + }); + + this.durableTaskClient + .Setup(c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny())) + .ThrowsAsync(new RpcException(new Status(StatusCode.NotFound, "Not found"))); + + // Act + await this.client.DeleteAsync(); + + // Assert - Should not throw + this.durableTaskClient.Verify( + c => c.TerminateInstanceAsync(orchestratorInstanceId, It.IsAny(), It.IsAny()), + Times.Once); + } +} + diff --git a/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs b/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs new file mode 100644 index 000000000..5630589ce --- /dev/null +++ b/test/ExportHistory.Tests/Constants/ExportHistoryConstantsTests.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Constants; + +public class ExportHistoryConstantsTests +{ + [Fact] + public void OrchestratorInstanceIdPrefix_IsCorrect() + { + // Assert + ExportHistoryConstants.OrchestratorInstanceIdPrefix.Should().Be("ExportJob-"); + } + + [Fact] + public void GetOrchestratorInstanceId_WithJobId_ReturnsCorrectFormat() + { + // Arrange + string jobId = "test-job-123"; + + // Act + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(jobId); + + // Assert + instanceId.Should().Be("ExportJob-test-job-123"); + instanceId.Should().StartWith(ExportHistoryConstants.OrchestratorInstanceIdPrefix); + } + + [Theory] + [InlineData("job-1")] + [InlineData("very-long-job-id-with-special-characters")] + [InlineData("")] + public void GetOrchestratorInstanceId_WithVariousJobIds_ReturnsCorrectFormat(string jobId) + { + // Act + string instanceId = ExportHistoryConstants.GetOrchestratorInstanceId(jobId); + + // Assert + instanceId.Should().Be($"{ExportHistoryConstants.OrchestratorInstanceIdPrefix}{jobId}"); + } +} + diff --git a/test/ExportHistory.Tests/Entity/ExportJobTests.cs b/test/ExportHistory.Tests/Entity/ExportJobTests.cs new file mode 100644 index 000000000..10a780443 --- /dev/null +++ b/test/ExportHistory.Tests/Entity/ExportJobTests.cs @@ -0,0 +1,370 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities.Tests; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Entity; + +public class ExportJobTests +{ + readonly ExportJob exportJob; + readonly TestLogger logger; + + public ExportJobTests() + { + this.logger = new TestLogger(); + this.exportJob = new ExportJob(this.logger); + } + + [Fact] + public async Task Create_WithValidOptions_CreatesJob() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var operation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + + // Act + await this.exportJob.RunAsync(operation); + + // Assert + var state = operation.State.GetState(typeof(ExportJobState)); + state.Should().NotBeNull(); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Active); + jobState.Config.Should().NotBeNull(); + jobState.Config!.Mode.Should().Be(ExportMode.Batch); + jobState.CreatedAt.Should().NotBeNull(); + jobState.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Create_WithNullOptions_ThrowsInvalidOperationException() + { + // Arrange + var operation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + null); + + // Act & Assert + // When no input is provided to an entity operation, it throws InvalidOperationException + // because the entity operation system can't bind the parameter + var exception = await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(operation).AsTask()); + + exception.Message.Should().Contain("expected an input value"); + } + + [Fact] + public async Task Get_ReturnsCurrentState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var getOperation = new TestEntityOperation( + nameof(ExportJob.Get), + createOperation.State, + null); + + // Act + var result = await this.exportJob.RunAsync(getOperation); + + // Assert + result.Should().NotBeNull(); + var state = Assert.IsType(result); + state.Status.Should().Be(ExportJobStatus.Active); + } + + [Fact] + public async Task MarkAsCompleted_WhenActive_TransitionsToCompleted() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var completeOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsCompleted), + createOperation.State, + null); + + // Act + await this.exportJob.RunAsync(completeOperation); + + // Assert + var state = completeOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Completed); + jobState.LastError.Should().BeNull(); + } + + [Fact] + public async Task MarkAsCompleted_WhenNotActive_ThrowsInvalidTransitionException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + // Mark as failed first + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation.State, + "test error"); + await this.exportJob.RunAsync(failOperation); + + var completeOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsCompleted), + failOperation.State, + null); + + // Act & Assert + await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(completeOperation).AsTask()); + } + + [Fact] + public async Task MarkAsFailed_WhenActive_TransitionsToFailed() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + string errorMessage = "Test error"; + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation.State, + errorMessage); + + // Act + await this.exportJob.RunAsync(failOperation); + + // Assert + var state = failOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Failed); + jobState.LastError.Should().Be(errorMessage); + } + + [Fact] + public async Task CommitCheckpoint_WithValidRequest_UpdatesState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var checkpointRequest = new CommitCheckpointRequest + { + ScannedInstances = 100, + ExportedInstances = 95, + Checkpoint = new ExportCheckpoint("last-key"), + }; + + var checkpointOperation = new TestEntityOperation( + nameof(ExportJob.CommitCheckpoint), + createOperation.State, + checkpointRequest); + + // Act + await this.exportJob.RunAsync(checkpointOperation); + + // Assert + var state = checkpointOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.ScannedInstances.Should().Be(100); + jobState.ExportedInstances.Should().Be(95); + jobState.Checkpoint.Should().NotBeNull(); + jobState.Checkpoint!.LastInstanceKey.Should().Be("last-key"); + } + + [Fact] + public async Task CommitCheckpoint_WithFailures_MarksJobAsFailed() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var failures = new List + { + new("instance-1", "error1", 1, DateTimeOffset.UtcNow), + new("instance-2", "error2", 2, DateTimeOffset.UtcNow), + }; + + var checkpointRequest = new CommitCheckpointRequest + { + ScannedInstances = 0, + ExportedInstances = 0, + Checkpoint = null, // No checkpoint means batch failed + Failures = failures, + }; + + var checkpointOperation = new TestEntityOperation( + nameof(ExportJob.CommitCheckpoint), + createOperation.State, + checkpointRequest); + + // Act + await this.exportJob.RunAsync(checkpointOperation); + + // Assert + var state = checkpointOperation.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Failed); + jobState.LastError.Should().NotBeNullOrEmpty(); + jobState.LastError.Should().Contain("Batch export failed"); + } + + [Fact] + public async Task Delete_ClearsState() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation); + + var deleteOperation = new TestEntityOperation( + nameof(ExportJob.Delete), + createOperation.State, + null); + + // Act + await this.exportJob.RunAsync(deleteOperation); + + // Assert + var state = deleteOperation.State.GetState(typeof(ExportJobState)); + state.Should().BeNull(); + } + + [Fact] + public async Task Create_WhenAlreadyExists_ThrowsInvalidTransitionException() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation1 = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation1); + + var createOperation2 = new TestEntityOperation( + nameof(ExportJob.Create), + createOperation1.State, + options); + + // Act & Assert + await Assert.ThrowsAsync(() => + this.exportJob.RunAsync(createOperation2).AsTask()); + } + + [Fact] + public async Task Create_WhenFailed_CanRecreate() + { + // Arrange + var options = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("test-container")); + + var createOperation1 = new TestEntityOperation( + nameof(ExportJob.Create), + new TestEntityState(null), + options); + await this.exportJob.RunAsync(createOperation1); + + var failOperation = new TestEntityOperation( + nameof(ExportJob.MarkAsFailed), + createOperation1.State, + "test error"); + await this.exportJob.RunAsync(failOperation); + + var createOperation2 = new TestEntityOperation( + nameof(ExportJob.Create), + failOperation.State, + options); + + // Act + await this.exportJob.RunAsync(createOperation2); + + // Assert + var state = createOperation2.State.GetState(typeof(ExportJobState)); + var jobState = Assert.IsType(state); + jobState.Status.Should().Be(ExportJobStatus.Active); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs new file mode 100644 index 000000000..f71711a64 --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobClientValidationExceptionTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobClientValidationExceptionTests +{ + [Fact] + public void Constructor_WithJobIdAndMessage_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = "Validation failed: invalid parameters"; + + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(message); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithJobIdMessageAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = "Validation failed: invalid parameters"; + System.Exception innerException = new ArgumentException("Inner error"); + + // Act + var exception = new ExportJobClientValidationException(jobId, message, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(message); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData("", "message")] + [InlineData("job-123", "Validation failed")] + public void Constructor_WithVariousParameters_CreatesInstance(string jobId, string message) + { + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.JobId.Should().Be(jobId); + if (!string.IsNullOrEmpty(message)) + { + exception.Message.Should().Contain(message); + } + } + + [Fact] + public void Constructor_WithEmptyMessage_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + string message = string.Empty; + + // Act + var exception = new ExportJobClientValidationException(jobId, message); + + // Assert + exception.JobId.Should().Be(jobId); + exception.Message.Should().NotBeNullOrEmpty(); + exception.Message.Should().Contain(jobId); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs new file mode 100644 index 000000000..00e98ae1e --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobInvalidTransitionExceptionTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobInvalidTransitionExceptionTests +{ + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + ExportJobStatus fromStatus = ExportJobStatus.Active; + ExportJobStatus toStatus = ExportJobStatus.Completed; + string operationName = "MarkAsCompleted"; + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + exception.OperationName.Should().Be(operationName); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain(fromStatus.ToString()); + exception.Message.Should().Contain(toStatus.ToString()); + exception.Message.Should().Contain(operationName); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithAllParametersAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + ExportJobStatus fromStatus = ExportJobStatus.Failed; + ExportJobStatus toStatus = ExportJobStatus.Active; + string operationName = "Create"; + System.Exception innerException = new InvalidOperationException("Inner error"); + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + exception.OperationName.Should().Be(operationName); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Active)] + public void Constructor_WithVariousStatusTransitions_CreatesInstance( + ExportJobStatus fromStatus, + ExportJobStatus toStatus) + { + // Arrange + string jobId = "job-123"; + string operationName = "TestOperation"; + + // Act + var exception = new ExportJobInvalidTransitionException(jobId, fromStatus, toStatus, operationName); + + // Assert + exception.FromStatus.Should().Be(fromStatus); + exception.ToStatus.Should().Be(toStatus); + } +} + diff --git a/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs b/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs new file mode 100644 index 000000000..977aa1dc1 --- /dev/null +++ b/test/ExportHistory.Tests/Exception/ExportJobNotFoundExceptionTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Exception; + +public class ExportJobNotFoundExceptionTests +{ + [Fact] + public void Constructor_WithJobId_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + + // Act + var exception = new ExportJobNotFoundException(jobId); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain("was not found"); + exception.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithJobIdAndInnerException_CreatesInstance() + { + // Arrange + string jobId = "job-123"; + System.Exception innerException = new InvalidOperationException("Inner error"); + + // Act + var exception = new ExportJobNotFoundException(jobId, innerException); + + // Assert + exception.Should().NotBeNull(); + exception.JobId.Should().Be(jobId); + exception.Message.Should().Contain(jobId); + exception.Message.Should().Contain("was not found"); + exception.InnerException.Should().Be(innerException); + } + + [Theory] + [InlineData("")] + [InlineData("job-123")] + [InlineData("very-long-job-id-with-special-characters-12345")] + public void Constructor_WithVariousJobIds_CreatesInstance(string jobId) + { + // Act + var exception = new ExportJobNotFoundException(jobId); + + // Assert + exception.JobId.Should().Be(jobId); + } +} + diff --git a/test/ExportHistory.Tests/ExportHistory.Tests.csproj b/test/ExportHistory.Tests/ExportHistory.Tests.csproj new file mode 100644 index 000000000..ce8c7d6fa --- /dev/null +++ b/test/ExportHistory.Tests/ExportHistory.Tests.csproj @@ -0,0 +1,19 @@ + + + + + net6.0 + enable + enable + false + true + true + + + + + + + + + diff --git a/test/ExportHistory.Tests/ExportJobTransitionsTests.cs b/test/ExportHistory.Tests/ExportJobTransitionsTests.cs new file mode 100644 index 000000000..e080bad50 --- /dev/null +++ b/test/ExportHistory.Tests/ExportJobTransitionsTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests; + +public class ExportJobTransitionsTests +{ + [Theory] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Active, true)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Active, false)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed, false)] + public void IsValidTransition_Create_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("Create", from, to); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Completed, true)] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Completed, false)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Completed, false)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Completed, false)] + public void IsValidTransition_MarkAsCompleted_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("MarkAsCompleted", from, to); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(ExportJobStatus.Active, ExportJobStatus.Failed, true)] + [InlineData(ExportJobStatus.Uninitialized, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Failed, ExportJobStatus.Failed, false)] + [InlineData(ExportJobStatus.Completed, ExportJobStatus.Failed, false)] + public void IsValidTransition_MarkAsFailed_ValidatesCorrectly( + ExportJobStatus from, + ExportJobStatus to, + bool expected) + { + // Act + bool result = ExportJobTransitions.IsValidTransition("MarkAsFailed", from, to); + + // Assert + result.Should().Be(expected); + } + + [Fact] + public void IsValidTransition_UnknownOperation_ReturnsFalse() + { + // Arrange + ExportJobStatus from = ExportJobStatus.Active; + ExportJobStatus to = ExportJobStatus.Completed; + + // Act + bool result = ExportJobTransitions.IsValidTransition("UnknownOperation", from, to); + + // Assert + result.Should().BeFalse(); + } + + [Theory] + [InlineData("Create")] + [InlineData("MarkAsCompleted")] + [InlineData("MarkAsFailed")] + public void IsValidTransition_AllValidTransitions_ReturnsTrue(string operationName) + { + // Arrange + ExportJobStatus from = operationName switch + { + "Create" => ExportJobStatus.Uninitialized, + "MarkAsCompleted" => ExportJobStatus.Active, + "MarkAsFailed" => ExportJobStatus.Active, + _ => ExportJobStatus.Uninitialized, + }; + + ExportJobStatus to = operationName switch + { + "Create" => ExportJobStatus.Active, + "MarkAsCompleted" => ExportJobStatus.Completed, + "MarkAsFailed" => ExportJobStatus.Failed, + _ => ExportJobStatus.Active, + }; + + // Act + bool result = ExportJobTransitions.IsValidTransition(operationName, from, to); + + // Assert + result.Should().BeTrue(); + } +} + diff --git a/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs b/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs new file mode 100644 index 000000000..5f39ce393 --- /dev/null +++ b/test/ExportHistory.Tests/Models/CommitCheckpointRequestTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class CommitCheckpointRequestTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var request = new CommitCheckpointRequest(); + + // Assert + request.Should().NotBeNull(); + request.ScannedInstances.Should().Be(0); + request.ExportedInstances.Should().Be(0); + request.Checkpoint.Should().BeNull(); + request.Failures.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var request = new CommitCheckpointRequest(); + var checkpoint = new ExportCheckpoint("last-key"); + var failures = new List + { + new("instance-1", "error1", 1, DateTimeOffset.UtcNow), + new("instance-2", "error2", 2, DateTimeOffset.UtcNow), + }; + + // Act + request.ScannedInstances = 100; + request.ExportedInstances = 95; + request.Checkpoint = checkpoint; + request.Failures = failures; + + // Assert + request.ScannedInstances.Should().Be(100); + request.ExportedInstances.Should().Be(95); + request.Checkpoint.Should().Be(checkpoint); + request.Failures.Should().BeEquivalentTo(failures); + } + + [Fact] + public void Properties_CanBeSetToNull() + { + // Arrange + var request = new CommitCheckpointRequest + { + Checkpoint = new ExportCheckpoint("key"), + Failures = new List(), + }; + + // Act + request.Checkpoint = null; + request.Failures = null; + + // Assert + request.Checkpoint.Should().BeNull(); + request.Failures.Should().BeNull(); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs b/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs new file mode 100644 index 000000000..2fb12f566 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportCheckpointTests.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportCheckpointTests +{ + [Fact] + public void Constructor_WithNullLastInstanceKey_CreatesInstance() + { + // Act + var checkpoint = new ExportCheckpoint(null); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().BeNull(); + } + + [Fact] + public void Constructor_WithLastInstanceKey_CreatesInstance() + { + // Arrange + string lastInstanceKey = "instance-key-123"; + + // Act + var checkpoint = new ExportCheckpoint(lastInstanceKey); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().Be(lastInstanceKey); + } + + [Fact] + public void Constructor_WithDefaultParameter_CreatesInstance() + { + // Act + var checkpoint = new ExportCheckpoint(); + + // Assert + checkpoint.Should().NotBeNull(); + checkpoint.LastInstanceKey.Should().BeNull(); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + string key = "test-key"; + var checkpoint1 = new ExportCheckpoint(key); + var checkpoint2 = new ExportCheckpoint(key); + + // Assert + checkpoint1.Should().Be(checkpoint2); + checkpoint1.GetHashCode().Should().Be(checkpoint2.GetHashCode()); + } + + [Fact] + public void Record_Inequality_Works() + { + // Arrange + var checkpoint1 = new ExportCheckpoint("key1"); + var checkpoint2 = new ExportCheckpoint("key2"); + + // Assert + checkpoint1.Should().NotBe(checkpoint2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportDestinationTests.cs b/test/ExportHistory.Tests/Models/ExportDestinationTests.cs new file mode 100644 index 000000000..9a66a0572 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportDestinationTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportDestinationTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var destination = new ExportDestination(); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().BeNull(); + destination.Prefix.Should().BeNull(); + } + + [Fact] + public void Constructor_WithContainer_CreatesInstance() + { + // Arrange + string container = "test-container"; + + // Act + var destination = new ExportDestination(container); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().Be(container); + destination.Prefix.Should().BeNull(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void Constructor_WithNullOrEmptyContainer_ThrowsArgumentException(string? container) + { + // Act + Action act = () => new ExportDestination(container!); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void Constructor_WithWhitespaceContainer_DoesNotThrow() + { + // Arrange + // Check.NotNullOrEmpty only checks for null, empty, or strings starting with '\0' + // It does NOT check for whitespace-only strings, so " " is valid + string container = " "; + + // Act + var destination = new ExportDestination(container); + + // Assert + destination.Should().NotBeNull(); + destination.Container.Should().Be(container); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var destination = new ExportDestination("test-container"); + + // Act + destination.Prefix = "test-prefix/"; + + // Assert + destination.Container.Should().Be("test-container"); + destination.Prefix.Should().Be("test-prefix/"); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFailureTests.cs b/test/ExportHistory.Tests/Models/ExportFailureTests.cs new file mode 100644 index 000000000..a5403d367 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFailureTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFailureTests +{ + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + string instanceId = "instance-123"; + string reason = "Export failed"; + int attemptCount = 3; + DateTimeOffset lastAttempt = DateTimeOffset.UtcNow; + + // Act + var failure = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + + // Assert + failure.Should().NotBeNull(); + failure.InstanceId.Should().Be(instanceId); + failure.Reason.Should().Be(reason); + failure.AttemptCount.Should().Be(attemptCount); + failure.LastAttempt.Should().Be(lastAttempt); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + string instanceId = "instance-123"; + string reason = "Export failed"; + int attemptCount = 3; + DateTimeOffset lastAttempt = DateTimeOffset.UtcNow; + + var failure1 = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + var failure2 = new ExportFailure(instanceId, reason, attemptCount, lastAttempt); + + // Assert + failure1.Should().Be(failure2); + failure1.GetHashCode().Should().Be(failure2.GetHashCode()); + } + + [Fact] + public void Record_Inequality_Works() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + var failure1 = new ExportFailure("instance-1", "reason1", 1, now); + var failure2 = new ExportFailure("instance-2", "reason2", 2, now); + + // Assert + failure1.Should().NotBe(failure2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFilterTests.cs b/test/ExportHistory.Tests/Models/ExportFilterTests.cs new file mode 100644 index 000000000..74155c234 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFilterTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFilterTests +{ + [Fact] + public void Constructor_WithRequiredParameters_CreatesInstance() + { + // Arrange + DateTimeOffset completedTimeFrom = DateTimeOffset.UtcNow.AddDays(-1); + + // Act + var filter = new ExportFilter(completedTimeFrom); + + // Assert + filter.Should().NotBeNull(); + filter.CompletedTimeFrom.Should().Be(completedTimeFrom); + filter.CompletedTimeTo.Should().BeNull(); + filter.RuntimeStatus.Should().BeNull(); + } + + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + DateTimeOffset completedTimeFrom = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset completedTimeTo = DateTimeOffset.UtcNow; + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + }; + + // Act + var filter = new ExportFilter(completedTimeFrom, completedTimeTo, runtimeStatus); + + // Assert + filter.Should().NotBeNull(); + filter.CompletedTimeFrom.Should().Be(completedTimeFrom); + filter.CompletedTimeTo.Should().Be(completedTimeTo); + filter.RuntimeStatus.Should().BeEquivalentTo(runtimeStatus); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + var statuses = new List { OrchestrationRuntimeStatus.Completed }; + + var filter1 = new ExportFilter(from, to, statuses); + var filter2 = new ExportFilter(from, to, statuses); + + // Assert + filter1.Should().Be(filter2); + filter1.GetHashCode().Should().Be(filter2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportFormatTests.cs b/test/ExportHistory.Tests/Models/ExportFormatTests.cs new file mode 100644 index 000000000..a393fb39f --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportFormatTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportFormatTests +{ + [Fact] + public void Constructor_WithDefaultValues_CreatesInstance() + { + // Act + var format = new ExportFormat(); + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be("jsonl"); + format.SchemaVersion.Should().Be("1.0"); + } + + [Fact] + public void Constructor_WithCustomValues_CreatesInstance() + { + // Arrange + string kind = "json"; + string schemaVersion = "2.0"; + + // Act + var format = new ExportFormat(kind, schemaVersion); + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be(kind); + format.SchemaVersion.Should().Be(schemaVersion); + } + + [Fact] + public void Default_ReturnsDefaultInstance() + { + // Act + var format = ExportFormat.Default; + + // Assert + format.Should().NotBeNull(); + format.Kind.Should().Be("jsonl"); + format.SchemaVersion.Should().Be("1.0"); + } + + [Fact] + public void Default_IsImmutable() + { + // Arrange + var format1 = ExportFormat.Default; + var format2 = ExportFormat.Default; + + // Act & Assert + format1.Should().Be(format2); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs b/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs new file mode 100644 index 000000000..2f8adafab --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobConfigurationTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobConfigurationTests +{ + [Fact] + public void Constructor_WithRequiredParameters_CreatesInstance() + { + // Arrange + ExportMode mode = ExportMode.Batch; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + + // Act + var config = new ExportJobConfiguration(mode, filter, destination, format); + + // Assert + config.Should().NotBeNull(); + config.Mode.Should().Be(mode); + config.Filter.Should().Be(filter); + config.Destination.Should().Be(destination); + config.Format.Should().Be(format); + config.MaxParallelExports.Should().Be(32); + config.MaxInstancesPerBatch.Should().Be(100); + } + + [Fact] + public void Constructor_WithAllParameters_CreatesInstance() + { + // Arrange + ExportMode mode = ExportMode.Continuous; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1)); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + int maxParallelExports = 64; + int maxInstancesPerBatch = 200; + + // Act + var config = new ExportJobConfiguration(mode, filter, destination, format, maxParallelExports, maxInstancesPerBatch); + + // Assert + config.Should().NotBeNull(); + config.Mode.Should().Be(mode); + config.Filter.Should().Be(filter); + config.Destination.Should().Be(destination); + config.Format.Should().Be(format); + config.MaxParallelExports.Should().Be(maxParallelExports); + config.MaxInstancesPerBatch.Should().Be(maxInstancesPerBatch); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + ExportMode mode = ExportMode.Batch; + ExportFilter filter = new(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow); + ExportDestination destination = new("test-container"); + ExportFormat format = ExportFormat.Default; + + var config1 = new ExportJobConfiguration(mode, filter, destination, format); + var config2 = new ExportJobConfiguration(mode, filter, destination, format); + + // Assert + config1.Should().Be(config2); + config1.GetHashCode().Should().Be(config2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs b/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs new file mode 100644 index 000000000..89955aa09 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobCreationOptionsTests.cs @@ -0,0 +1,441 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobCreationOptionsTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var options = new ExportJobCreationOptions(); + + // Assert + options.Should().NotBeNull(); + } + + [Fact] + public void Constructor_BatchMode_WithValidParameters_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination); + + // Assert + options.Should().NotBeNull(); + options.Mode.Should().Be(ExportMode.Batch); + options.CompletedTimeFrom.Should().Be(from); + options.CompletedTimeTo.Should().Be(to); + options.Destination.Should().Be(destination); + options.JobId.Should().NotBeNullOrEmpty(); + options.Format.Should().Be(ExportFormat.Default); + options.RuntimeStatus.Should().NotBeNull(); + options.MaxInstancesPerBatch.Should().Be(100); + } + + [Fact] + public void Constructor_BatchMode_WithCustomJobId_CreatesInstance() + { + // Arrange + string jobId = "custom-job-id"; + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + jobId); + + // Assert + options.JobId.Should().Be(jobId); + } + + [Fact] + public void Constructor_BatchMode_WithDefaultJobId_GeneratesGuid() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null); + + // Assert + options.JobId.Should().NotBeNullOrEmpty(); + Guid.TryParse(options.JobId, out _).Should().BeTrue(); + } + + [Fact] + public void Constructor_BatchMode_WithEmptyJobId_GeneratesGuid() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + string.Empty); + + // Assert + options.JobId.Should().NotBeNullOrEmpty(); + Guid.TryParse(options.JobId, out _).Should().BeTrue(); + } + + [Fact] + public void Constructor_BatchMode_WithDefaultCompletedTimeFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = default; + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeFrom is required for Batch export mode*"); + } + + [Fact] + public void Constructor_BatchMode_WithNullCompletedTimeTo_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset? to = null; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo is required for Batch export mode*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToLessThanFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow; + DateTimeOffset to = DateTimeOffset.UtcNow.AddDays(-1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* must be greater than CompletedTimeFrom*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToEqualToFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = from; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* must be greater than CompletedTimeFrom*"); + } + + [Fact] + public void Constructor_BatchMode_WithCompletedTimeToInFuture_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow.AddDays(1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Batch, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo* cannot be in the future*"); + } + + [Fact] + public void Constructor_ContinuousMode_WithValidParameters_CreatesInstance() + { + // Arrange + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Continuous, + default, + null, + destination); + + // Assert + options.Should().NotBeNull(); + options.Mode.Should().Be(ExportMode.Continuous); + options.CompletedTimeFrom.Should().Be(default); + options.CompletedTimeTo.Should().BeNull(); + options.Destination.Should().Be(destination); + } + + [Fact] + public void Constructor_ContinuousMode_WithCompletedTimeFrom_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Continuous, from, null, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeFrom is not allowed for Continuous export mode*"); + } + + [Fact] + public void Constructor_ContinuousMode_WithCompletedTimeTo_ThrowsArgumentException() + { + // Arrange + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(ExportMode.Continuous, default, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*CompletedTimeTo is not allowed for Continuous export mode*"); + } + + [Fact] + public void Constructor_WithInvalidMode_ThrowsArgumentException() + { + // Arrange + ExportMode invalidMode = (ExportMode)999; + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions(invalidMode, from, to, destination); + + // Assert + act.Should().Throw() + .WithMessage("*Invalid export mode*"); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1001)] + [InlineData(2000)] + public void Constructor_WithInvalidMaxInstancesPerBatch_ThrowsArgumentOutOfRangeException(int maxInstances) + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + Action act = () => new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null, + maxInstances); + + // Assert + act.Should().Throw() + .WithMessage("*MaxInstancesPerBatch must be between 1 and 1000*"); + } + + [Fact] + public void Constructor_WithValidMaxInstancesPerBatch_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + int maxInstances = 500; + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null, + maxInstances); + + // Assert + options.MaxInstancesPerBatch.Should().Be(maxInstances); + } + + [Fact] + public void Constructor_WithNonTerminalRuntimeStatus_ThrowsArgumentException() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Running, // Not terminal + }; + + // Act + Action act = () => new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + act.Should().Throw() + .WithMessage("*Export supports terminal orchestration statuses only*"); + } + + [Fact] + public void Constructor_WithTerminalRuntimeStatus_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new() + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated, + OrchestrationRuntimeStatus.ContinuedAsNew, + }; + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + options.RuntimeStatus.Should().BeEquivalentTo(runtimeStatus); + } + + [Fact] + public void Constructor_WithNullRuntimeStatus_UsesDefaultTerminalStatuses() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + null); + + // Assert + options.RuntimeStatus.Should().NotBeNull(); + options.RuntimeStatus.Should().HaveCount(4); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Completed); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Failed); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.Terminated); + options.RuntimeStatus.Should().Contain(OrchestrationRuntimeStatus.ContinuedAsNew); + } + + [Fact] + public void Constructor_WithEmptyRuntimeStatus_UsesDefaultTerminalStatuses() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + List runtimeStatus = new(); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + null, + runtimeStatus); + + // Assert + options.RuntimeStatus.Should().NotBeNull(); + options.RuntimeStatus.Should().HaveCount(4); + } + + [Fact] + public void Constructor_WithCustomFormat_CreatesInstance() + { + // Arrange + DateTimeOffset from = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset to = DateTimeOffset.UtcNow; + ExportDestination destination = new("test-container"); + ExportFormat format = new("json", "2.0"); + + // Act + var options = new ExportJobCreationOptions( + ExportMode.Batch, + from, + to, + destination, + null, + format); + + // Assert + options.Format.Should().Be(format); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs b/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs new file mode 100644 index 000000000..cbb1c9752 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobDescriptionTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobDescriptionTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var description = new ExportJobDescription(); + + // Assert + description.Should().NotBeNull(); + description.JobId.Should().BeEmpty(); + description.Status.Should().Be(ExportJobStatus.Uninitialized); + description.CreatedAt.Should().BeNull(); + description.LastModifiedAt.Should().BeNull(); + description.Config.Should().BeNull(); + description.OrchestratorInstanceId.Should().BeNull(); + description.ScannedInstances.Should().Be(0); + description.ExportedInstances.Should().Be(0); + description.LastError.Should().BeNull(); + description.Checkpoint.Should().BeNull(); + description.LastCheckpointTime.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var description = new ExportJobDescription(); + var config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default); + var checkpoint = new ExportCheckpoint("last-key"); + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + description = description with + { + JobId = "job-123", + Status = ExportJobStatus.Active, + CreatedAt = now, + LastModifiedAt = now, + Config = config, + OrchestratorInstanceId = "orchestrator-123", + ScannedInstances = 100, + ExportedInstances = 95, + LastError = "test error", + Checkpoint = checkpoint, + LastCheckpointTime = now, + }; + + // Assert + description.JobId.Should().Be("job-123"); + description.Status.Should().Be(ExportJobStatus.Active); + description.CreatedAt.Should().Be(now); + description.LastModifiedAt.Should().Be(now); + description.Config.Should().Be(config); + description.OrchestratorInstanceId.Should().Be("orchestrator-123"); + description.ScannedInstances.Should().Be(100); + description.ExportedInstances.Should().Be(95); + description.LastError.Should().Be("test error"); + description.Checkpoint.Should().Be(checkpoint); + description.LastCheckpointTime.Should().Be(now); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs b/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs new file mode 100644 index 000000000..adca278a2 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobQueryTests.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobQueryTests +{ + [Fact] + public void DefaultPageSize_IsCorrect() + { + // Assert + ExportJobQuery.DefaultPageSize.Should().Be(100); + } + + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var query = new ExportJobQuery(); + + // Assert + query.Should().NotBeNull(); + query.Status.Should().BeNull(); + query.JobIdPrefix.Should().BeNull(); + query.CreatedFrom.Should().BeNull(); + query.CreatedTo.Should().BeNull(); + query.PageSize.Should().BeNull(); + query.ContinuationToken.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + var query = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + ContinuationToken = "token-123", + }; + + // Assert + query.Status.Should().Be(ExportJobStatus.Active); + query.JobIdPrefix.Should().Be("test-"); + query.CreatedFrom.Should().Be(now.AddDays(-1)); + query.CreatedTo.Should().Be(now); + query.PageSize.Should().Be(50); + query.ContinuationToken.Should().Be("token-123"); + } + + [Fact] + public void Record_Equality_Works() + { + // Arrange + DateTimeOffset now = DateTimeOffset.UtcNow; + var query1 = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + }; + + var query2 = new ExportJobQuery + { + Status = ExportJobStatus.Active, + JobIdPrefix = "test-", + CreatedFrom = now.AddDays(-1), + CreatedTo = now, + PageSize = 50, + }; + + // Assert + query1.Should().Be(query2); + query1.GetHashCode().Should().Be(query2.GetHashCode()); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobStateTests.cs b/test/ExportHistory.Tests/Models/ExportJobStateTests.cs new file mode 100644 index 000000000..2048288d8 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobStateTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobStateTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var state = new ExportJobState(); + + // Assert + state.Should().NotBeNull(); + state.Status.Should().Be(ExportJobStatus.Uninitialized); + state.Config.Should().BeNull(); + state.Checkpoint.Should().BeNull(); + state.CreatedAt.Should().BeNull(); + state.LastModifiedAt.Should().BeNull(); + state.LastCheckpointTime.Should().BeNull(); + state.LastError.Should().BeNull(); + state.ScannedInstances.Should().Be(0); + state.ExportedInstances.Should().Be(0); + state.OrchestratorInstanceId.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var state = new ExportJobState(); + var config = new ExportJobConfiguration( + ExportMode.Batch, + new ExportFilter(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow), + new ExportDestination("container"), + ExportFormat.Default); + var checkpoint = new ExportCheckpoint("last-key"); + DateTimeOffset now = DateTimeOffset.UtcNow; + + // Act + state.Status = ExportJobStatus.Active; + state.Config = config; + state.Checkpoint = checkpoint; + state.CreatedAt = now; + state.LastModifiedAt = now; + state.LastCheckpointTime = now; + state.LastError = "test error"; + state.ScannedInstances = 100; + state.ExportedInstances = 95; + state.OrchestratorInstanceId = "orchestrator-123"; + + // Assert + state.Status.Should().Be(ExportJobStatus.Active); + state.Config.Should().Be(config); + state.Checkpoint.Should().Be(checkpoint); + state.CreatedAt.Should().Be(now); + state.LastModifiedAt.Should().Be(now); + state.LastCheckpointTime.Should().Be(now); + state.LastError.Should().Be("test error"); + state.ScannedInstances.Should().Be(100); + state.ExportedInstances.Should().Be(95); + state.OrchestratorInstanceId.Should().Be("orchestrator-123"); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs b/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs new file mode 100644 index 000000000..7710b6a37 --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportJobStatusTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportJobStatusTests +{ + [Fact] + public void ExportJobStatus_Values_AreDefined() + { + // Assert + Enum.GetValues().Should().HaveCount(4); + Enum.GetValues().Should().Contain(ExportJobStatus.Uninitialized); + Enum.GetValues().Should().Contain(ExportJobStatus.Active); + Enum.GetValues().Should().Contain(ExportJobStatus.Failed); + Enum.GetValues().Should().Contain(ExportJobStatus.Completed); + } + + [Theory] + [InlineData(ExportJobStatus.Uninitialized)] + [InlineData(ExportJobStatus.Active)] + [InlineData(ExportJobStatus.Failed)] + [InlineData(ExportJobStatus.Completed)] + public void ExportJobStatus_CanBeAssigned(ExportJobStatus status) + { + // Arrange + ExportJobStatus assignedStatus = status; + + // Assert + assignedStatus.Should().Be(status); + } +} + diff --git a/test/ExportHistory.Tests/Models/ExportModeTests.cs b/test/ExportHistory.Tests/Models/ExportModeTests.cs new file mode 100644 index 000000000..1b83a678e --- /dev/null +++ b/test/ExportHistory.Tests/Models/ExportModeTests.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Models; + +public class ExportModeTests +{ + [Fact] + public void ExportMode_Values_AreCorrect() + { + // Assert + ExportMode.Batch.Should().Be((ExportMode)1); + ExportMode.Continuous.Should().Be((ExportMode)2); + } + + [Theory] + [InlineData(ExportMode.Batch, 1)] + [InlineData(ExportMode.Continuous, 2)] + public void ExportMode_EnumValue_MatchesExpected(ExportMode mode, int expectedValue) + { + // Assert + ((int)mode).Should().Be(expectedValue); + } +} + diff --git a/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs b/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs new file mode 100644 index 000000000..ab9bb3bb6 --- /dev/null +++ b/test/ExportHistory.Tests/Options/ExportHistoryStorageOptionsTests.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Options; + +public class ExportHistoryStorageOptionsTests +{ + [Fact] + public void Constructor_Default_CreatesInstance() + { + // Act + var options = new ExportHistoryStorageOptions(); + + // Assert + options.Should().NotBeNull(); + options.ConnectionString.Should().BeEmpty(); + options.ContainerName.Should().BeEmpty(); + options.Prefix.Should().BeNull(); + } + + [Fact] + public void Properties_CanBeSet() + { + // Arrange + var options = new ExportHistoryStorageOptions(); + + // Act + options.ConnectionString = "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net"; + options.ContainerName = "test-container"; + options.Prefix = "exports/"; + + // Assert + options.ConnectionString.Should().Be("DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key;EndpointSuffix=core.windows.net"); + options.ContainerName.Should().Be("test-container"); + options.Prefix.Should().Be("exports/"); + } + + [Fact] + public void Prefix_CanBeSetToNull() + { + // Arrange + var options = new ExportHistoryStorageOptions + { + Prefix = "test-prefix", + }; + + // Act + options.Prefix = null; + + // Assert + options.Prefix.Should().BeNull(); + } +} + diff --git a/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs new file mode 100644 index 000000000..7b36acd1f --- /dev/null +++ b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.ExportHistory.Tests.Orchestrations; + +public class ExecuteExportJobOperationOrchestratorTests +{ + readonly Mock mockContext; + readonly Mock mockEntityClient; + readonly ExecuteExportJobOperationOrchestrator orchestrator; + + public ExecuteExportJobOperationOrchestratorTests() + { + this.mockContext = new Mock(MockBehavior.Strict); + this.mockEntityClient = new Mock(MockBehavior.Loose); + this.mockContext.Setup(c => c.Entities).Returns(this.mockEntityClient.Object); + this.orchestrator = new ExecuteExportJobOperationOrchestrator(); + } + + [Fact] + public async Task RunAsync_ValidRequest_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = "Create"; + var input = new ExportJobCreationOptions( + ExportMode.Batch, + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow, + new ExportDestination("container")); + var expectedResult = new ExportJobState { Status = ExportJobStatus.Active }; + var request = new ExportJobOperationRequest(entityId, operationName, input); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, input, default)) + .ReturnsAsync(expectedResult); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().Be(expectedResult); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, input, default), + Times.Once); + } + + [Fact] + public async Task RunAsync_WithNullInput_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = "Get"; + var request = new ExportJobOperationRequest(entityId, operationName, null); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, null, default)) + .ReturnsAsync(new ExportJobState()); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().NotBeNull(); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, null, default), + Times.Once); + } + + [Fact] + public async Task RunAsync_WithDeleteOperation_CallsEntityOperation() + { + // Arrange + var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); + string operationName = ExportJobOperations.Delete; + var request = new ExportJobOperationRequest(entityId, operationName, null); + + this.mockEntityClient + .Setup(e => e.CallEntityAsync(entityId, operationName, null, default)) + .ReturnsAsync(null!); + + // Act + var result = await this.orchestrator.RunAsync(this.mockContext.Object, request); + + // Assert + result.Should().BeNull(); + this.mockEntityClient.Verify( + e => e.CallEntityAsync(entityId, operationName, null, default), + Times.Once); + } +} + diff --git a/test/ExportHistory.Tests/Usings.cs b/test/ExportHistory.Tests/Usings.cs new file mode 100644 index 000000000..e33163f94 --- /dev/null +++ b/test/ExportHistory.Tests/Usings.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +global using FluentAssertions; +global using Xunit; + From 208aade2258d3d1f3f1f66e0c7b5297db78b8d7d Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:09:39 -0800 Subject: [PATCH 7/7] feedback --- .../Models/CreateExportJobRequest.cs | 2 +- src/Client/Core/DurableTaskClient.cs | 48 +++++++++---------- src/Client/Grpc/GrpcDurableTaskClient.cs | 2 - .../ExportInstanceHistoryActivity.cs | 31 ++++++------ .../ListTerminalInstancesActivity.cs | 8 ++-- .../Client/DefaultExportHistoryClient.cs | 6 +-- .../Client/DefaultExportHistoryJobClient.cs | 35 +++++--------- src/ExportHistory/Entity/ExportJob.cs | 6 +++ .../Entity/ExportJobOperations.cs | 19 -------- .../ExportJobClientValidationException.cs | 15 +----- .../ExportJobInvalidTransitionException.cs | 20 +------- .../Exception/ExportJobNotFoundException.cs | 14 +----- src/ExportHistory/Models/ExportFormat.cs | 23 ++++++++- .../Models/ExportJobCreationOptions.cs | 21 +++----- .../ExecuteExportJobOperationOrchestrator.cs | 2 +- .../Orchestrations/ExportJobOrchestrator.cs | 21 -------- .../DefaultExportHistoryJobClientTests.cs | 4 +- .../Models/ExportFormatTests.cs | 6 +-- ...cuteExportJobOperationOrchestratorTests.cs | 2 +- test/TestHelpers/Logging/TestLogProvider.cs | 2 +- 20 files changed, 102 insertions(+), 185 deletions(-) delete mode 100644 src/ExportHistory/Entity/ExportJobOperations.cs diff --git a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs index 4bb843816..1e09ddf51 100644 --- a/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs +++ b/samples/ExportHistoryWebApp/Models/CreateExportJobRequest.cs @@ -48,7 +48,7 @@ public class CreateExportJobRequest /// /// Gets or sets the orchestration runtime statuses to filter by. Optional. - /// Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew. + /// Valid statuses are: Completed, Failed, Terminated. /// public List? RuntimeStatus { get; set; } diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index 5786d036f..1ea0b2554 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; using System.ComponentModel; -using DurableTask.Core.History; using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Internal; +using DurableTask.Core.History; namespace Microsoft.DurableTask.Client; @@ -338,27 +337,6 @@ public abstract Task ResumeInstanceAsync( /// An async pageable of the query results. public abstract AsyncPageable GetAllInstancesAsync(OrchestrationQuery? filter = null); - /// - /// Lists orchestration instance IDs filtered by completed time. - /// - /// The runtime statuses to filter by. - /// The start time for completed time filter (inclusive). - /// The end time for completed time filter (inclusive). - /// The page size for pagination. - /// The last fetched instance key. - /// The cancellation token. - /// A page of instance IDs with continuation token. - public virtual Task> ListInstanceIdsAsync( - IEnumerable? runtimeStatus = null, - DateTimeOffset? completedTimeFrom = null, - DateTimeOffset? completedTimeTo = null, - int pageSize = OrchestrationQuery.DefaultPageSize, - string? lastInstanceKey = null, - CancellationToken cancellation = default) - { - throw new NotSupportedException($"{this.GetType()} does not support listing orchestration instance IDs by completed time."); - } - /// public virtual Task PurgeInstanceAsync(string instanceId, CancellationToken cancellation) => this.PurgeInstanceAsync(instanceId, null, cancellation); @@ -489,6 +467,27 @@ public virtual Task RewindInstanceAsync( CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support orchestration rewind."); + /// + /// Lists orchestration instance IDs filtered by completed time. + /// + /// The runtime statuses to filter by. + /// The start time for completed time filter (inclusive). + /// The end time for completed time filter (inclusive). + /// The page size for pagination. + /// The last fetched instance key. + /// The cancellation token. + /// A page of instance IDs with continuation token. + public virtual Task> ListInstanceIdsAsync( + IEnumerable? runtimeStatus = null, + DateTimeOffset? completedTimeFrom = null, + DateTimeOffset? completedTimeTo = null, + int pageSize = OrchestrationQuery.DefaultPageSize, + string? lastInstanceKey = null, + CancellationToken cancellation = default) + { + throw new NotSupportedException($"{this.GetType()} does not support listing orchestration instance IDs by completed time."); + } + /// /// Streams the execution history events for an orchestration instance. /// @@ -509,6 +508,7 @@ public virtual IAsyncEnumerable StreamInstanceHistoryAsync( string instanceId, string? executionId = null, CancellationToken cancellation = default) + { throw new NotSupportedException($"{this.GetType()} does not support streaming instance history."); } @@ -522,4 +522,4 @@ public virtual IAsyncEnumerable StreamInstanceHistoryAsync( /// /// A that completes when the disposal completes. public abstract ValueTask DisposeAsync(); -} +} \ No newline at end of file diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 81f4966c7..b55fe2378 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -325,8 +325,6 @@ public override async Task WaitForInstanceStartAsync( } } - // Removed ListTerminalInstances; use GetAllInstancesAsync with OrchestrationQuery instead - /// public override async Task> ListInstanceIdsAsync( IEnumerable? runtimeStatus = null, diff --git a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs index 7a0b19537..bc32dd40f 100644 --- a/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs +++ b/src/ExportHistory/Activities/ExportInstanceHistoryActivity.cs @@ -23,11 +23,11 @@ namespace Microsoft.DurableTask.ExportHistory; /// [DurableTask] public class ExportInstanceHistoryActivity( - IDurableTaskClientProvider clientProvider, + DurableTaskClient client, ILogger logger, IOptions storageOptions) : TaskActivity { - readonly IDurableTaskClientProvider clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + readonly DurableTaskClient client = Check.NotNull(client, nameof(client)); readonly ILogger logger = Check.NotNull(logger, nameof(logger)); readonly ExportHistoryStorageOptions storageOptions = Check.NotNull(storageOptions?.Value, nameof(storageOptions)); @@ -45,10 +45,8 @@ public override async Task RunAsync(TaskActivityContext context, E { this.logger.LogInformation("Starting export for instance {InstanceId}", instanceId); - // Get the client and instance metadata with inputs and outputs - DurableTaskClient client = this.clientProvider.GetClient(); - - OrchestrationMetadata? metadata = await client.GetInstanceAsync( + // Get instance metadata with inputs and outputs + OrchestrationMetadata? metadata = await this.client.GetInstanceAsync( instanceId, getInputsAndOutputs: true, cancellation: CancellationToken.None); @@ -82,7 +80,7 @@ public override async Task RunAsync(TaskActivityContext context, E // Stream all history events this.logger.LogInformation("Streaming history events for instance {InstanceId}", instanceId); List historyEvents = new(); - await foreach (HistoryEvent historyEvent in client.StreamInstanceHistoryAsync( + await foreach (HistoryEvent historyEvent in this.client.StreamInstanceHistoryAsync( instanceId, executionId: null, // Use latest execution cancellation: CancellationToken.None)) @@ -158,13 +156,11 @@ static string GenerateBlobFileName(DateTimeOffset completedTimestamp, string ins /// The file extension (e.g., "json", "jsonl.gz"). static string GetFileExtension(ExportFormat format) { - string formatKind = format.Kind.ToLowerInvariant(); - - return formatKind switch + return format.Kind switch { - "jsonl" => "jsonl.gz", // JSONL format is compressed - "json" => "json", // JSON format is uncompressed - _ => "jsonl.gz", // Default to JSONL compressed + ExportFormatKind.Jsonl => "jsonl.gz", // JSONL format is compressed + ExportFormatKind.Json => "json", // JSON format is uncompressed + _ => "jsonl.gz", // Default to JSONL compressed }; } @@ -172,7 +168,6 @@ static string SerializeInstanceData( List historyEvents, ExportFormat format) { - string formatKind = format.Kind.ToLowerInvariant(); JsonSerializerOptions serializerOptions = new JsonSerializerOptions { WriteIndented = false, @@ -182,7 +177,7 @@ static string SerializeInstanceData( Converters = { new JsonStringEnumConverter() }, // Serialize enums as strings }; - if (formatKind == "jsonl") + if (format.Kind == ExportFormatKind.Jsonl) { // JSONL format: one history event per line // Serialize as object to preserve runtime type (polymorphic serialization) @@ -215,6 +210,10 @@ async Task UploadToBlobStorageAsync( CancellationToken cancellationToken) { // Create blob service client from connection string + // Note: Azure.Storage.Blobs clients (BlobServiceClient, BlobContainerClient, BlobClient) are lightweight + // wrappers that don't implement IDisposable/IAsyncDisposable. They use HttpClient internally which is + // managed by the framework and doesn't require explicit disposal. The clients are designed to be + // stateless and safe for reuse or short-lived usage without disposal. BlobServiceClient serviceClient = new(this.storageOptions.ConnectionString); BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(containerName); @@ -229,7 +228,7 @@ await containerClient.CreateIfNotExistsAsync( // Upload content byte[] contentBytes = Encoding.UTF8.GetBytes(content); - if (format.Kind.Equals("jsonl", StringComparison.OrdinalIgnoreCase)) + if (format.Kind == ExportFormatKind.Jsonl) { // Compress with gzip using MemoryStream compressedStream = new(); diff --git a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs index b01d2c5e5..379945710 100644 --- a/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs +++ b/src/ExportHistory/Activities/ListTerminalInstancesActivity.cs @@ -24,10 +24,10 @@ public sealed record ListTerminalInstancesRequest( /// [DurableTask] public class ListTerminalInstancesActivity( - IDurableTaskClientProvider clientProvider, + DurableTaskClient client, ILogger logger) : TaskActivity { - readonly IDurableTaskClientProvider clientProvider = Check.NotNull(clientProvider, nameof(clientProvider)); + readonly DurableTaskClient client = Check.NotNull(client, nameof(client)); readonly ILogger logger = Check.NotNull(logger, nameof(logger)); /// @@ -37,10 +37,8 @@ public override async Task RunAsync(TaskActivityContext context, L try { - DurableTaskClient client = this.clientProvider.GetClient(); - // Try to use ListInstanceIds endpoint first (available in gRPC client) - Page page = await client.ListInstanceIdsAsync( + Page page = await this.client.ListInstanceIdsAsync( runtimeStatus: input.RuntimeStatus, completedTimeFrom: input.CompletedTimeFrom, completedTimeTo: input.CompletedTimeTo, diff --git a/src/ExportHistory/Client/DefaultExportHistoryClient.cs b/src/ExportHistory/Client/DefaultExportHistoryClient.cs index b76bea6b6..d48326b7c 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryClient.cs @@ -30,11 +30,7 @@ public override async Task CreateJobAsync( try { // Create export job client instance - ExportHistoryJobClient exportHistoryJobClient = new DefaultExportHistoryJobClient( - this.durableTaskClient, - options.JobId, - this.logger, - this.storageOptions); + ExportHistoryJobClient exportHistoryJobClient = this.GetJobClient(options.JobId); // Create the export job using the client (validation already done in constructor) await exportHistoryJobClient.CreateAsync(options, cancellation); diff --git a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs index b2a53c460..acd22b0e5 100644 --- a/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs +++ b/src/ExportHistory/Client/DefaultExportHistoryJobClient.cs @@ -31,31 +31,18 @@ public override async Task CreateAsync(ExportJobCreationOptions options, Cancell Check.NotNull(options, nameof(options)); // Determine default prefix based on mode if not already set - string? defaultPrefix = $"{options.Mode.ToString().ToLower(System.Globalization.CultureInfo.CurrentCulture)}/"; + string? defaultPrefix = $"{options.Mode.ToString().ToLower(System.Globalization.CultureInfo.CurrentCulture)}-{this.JobId}/"; // If destination is not provided, construct it from storage options - ExportJobCreationOptions optionsWithDestination = options; - if (options.Destination == null) - { - // Use storage options prefix if provided, otherwise use mode-based default - string? prefix = this.storageOptions.Prefix ?? defaultPrefix; - - ExportDestination destination = new ExportDestination(this.storageOptions.ContainerName) - { - Prefix = prefix, - }; + string prefix = options.Destination?.Prefix ?? this.storageOptions.Prefix ?? defaultPrefix; + string container = options.Destination?.Container ?? this.storageOptions.ContainerName; - optionsWithDestination = options with { Destination = destination }; - } - else if (string.IsNullOrEmpty(options.Destination.Prefix)) + ExportDestination destination = new ExportDestination(container) { - // Destination provided but no prefix set - use mode-based default - ExportDestination destinationWithPrefix = new ExportDestination(options.Destination.Container) - { - Prefix = defaultPrefix, - }; - optionsWithDestination = options with { Destination = destinationWithPrefix }; - } + Prefix = prefix, + }; + + ExportJobCreationOptions optionsWithDestination = options with { Destination = destination }; ExportJobOperationRequest request = new ExportJobOperationRequest( @@ -109,8 +96,8 @@ public override async Task DeleteAsync(CancellationToken cancellation = default) string orchestrationInstanceId = ExportHistoryConstants.GetOrchestratorInstanceId(this.JobId); // First, delete the entity - ExportJobOperationRequest request = new ExportJobOperationRequest(this.entityId, ExportJobOperations.Delete); - string instanceId = await this.durableTaskClient.ScheduleNewOrchestrationInstanceAsync( + ExportJobOperationRequest request = new ExportJobOperationRequest(this.entityId, nameof(ExportJob.Delete)); + await this.durableTaskClient.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteExportJobOperationOrchestrator)), request, cancellation); @@ -202,7 +189,7 @@ await this.durableTaskClient.TerminateInstanceAsync( { // Orchestration instance doesn't exist - this is expected if it was already deleted or never existed this.logger.LogInformation( - "Orchestration instance '{OrchestrationInstanceId}' is already purged", + "Orchestration instance '{OrchestrationInstanceId}' is already purged or never existed", orchestrationInstanceId); } catch (Exception ex) diff --git a/src/ExportHistory/Entity/ExportJob.cs b/src/ExportHistory/Entity/ExportJob.cs index 9d33b6cd4..7ccce1ad1 100644 --- a/src/ExportHistory/Entity/ExportJob.cs +++ b/src/ExportHistory/Entity/ExportJob.cs @@ -54,6 +54,12 @@ public void Create(TaskEntityContext context, ExportJobCreationOptions creationO this.State.CreatedAt = this.State.LastModifiedAt = DateTimeOffset.UtcNow; this.State.LastError = null; + // Reset progress counters and checkpoint for clean restart + this.State.ScannedInstances = 0; + this.State.ExportedInstances = 0; + this.State.Checkpoint = null; + this.State.LastCheckpointTime = null; + logger.CreatedExportJob(creationOptions.JobId); // Signal the Run method to start the export diff --git a/src/ExportHistory/Entity/ExportJobOperations.cs b/src/ExportHistory/Entity/ExportJobOperations.cs deleted file mode 100644 index 3ec695168..000000000 --- a/src/ExportHistory/Entity/ExportJobOperations.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.ExportHistory; - -/// -/// Constants for export job entity operation names. -/// -/// -/// Operation names are case-insensitive when matching to entity methods. -/// These constants match the method names on for consistency. -/// -static class ExportJobOperations -{ - /// - /// Operation name for deleting the entity. - /// - public const string Delete = "Delete"; -} diff --git a/src/ExportHistory/Exception/ExportJobClientValidationException.cs b/src/ExportHistory/Exception/ExportJobClientValidationException.cs index 25118f087..ec18f9bf6 100644 --- a/src/ExportHistory/Exception/ExportJobClientValidationException.cs +++ b/src/ExportHistory/Exception/ExportJobClientValidationException.cs @@ -8,25 +8,14 @@ namespace Microsoft.DurableTask.ExportHistory; /// public class ExportJobClientValidationException : InvalidOperationException { - /// - /// Initializes a new instance of the class. - /// - /// The ID of the export job that failed validation. - /// The validation error message. - public ExportJobClientValidationException(string jobId, string message) - : base($"Validation failed for export job '{jobId}': {message}") - { - this.JobId = jobId; - } - /// /// Initializes a new instance of the class. /// /// The ID of the export job that failed validation. /// The validation error message. /// The exception that is the cause of the current exception. - public ExportJobClientValidationException(string jobId, string message, Exception innerException) - : base($"Validation failed for export job '{jobId}': {message}", innerException) + public ExportJobClientValidationException(string jobId, string message, Exception? innerException = null) + : base($"Validation failed for export job '{jobId}': {message}", innerException!) { this.JobId = jobId; } diff --git a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs index 05119c30d..74eca6c2d 100644 --- a/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs +++ b/src/ExportHistory/Exception/ExportJobInvalidTransitionException.cs @@ -8,22 +8,6 @@ namespace Microsoft.DurableTask.ExportHistory; /// public class ExportJobInvalidTransitionException : InvalidOperationException { - /// - /// Initializes a new instance of the class. - /// - /// The ID of the export job on which the invalid transition was attempted. - /// The current status of the export job. - /// The target status that was invalid. - /// The name of the operation that was attempted. - public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName) - : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.") - { - this.JobId = jobId; - this.FromStatus = fromStatus; - this.ToStatus = toStatus; - this.OperationName = operationName; - } - /// /// Initializes a new instance of the class. /// @@ -32,8 +16,8 @@ public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromSta /// The target status that was invalid. /// The name of the operation that was attempted. /// The exception that is the cause of the current exception. - public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName, Exception innerException) - : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.", innerException) + public ExportJobInvalidTransitionException(string jobId, ExportJobStatus fromStatus, ExportJobStatus toStatus, string operationName, Exception? innerException = null) + : base($"Invalid state transition attempted for export job '{jobId}': Cannot transition from {fromStatus} to {toStatus} during {operationName} operation.", innerException!) { this.JobId = jobId; this.FromStatus = fromStatus; diff --git a/src/ExportHistory/Exception/ExportJobNotFoundException.cs b/src/ExportHistory/Exception/ExportJobNotFoundException.cs index 0889c7731..91e9b0a4d 100644 --- a/src/ExportHistory/Exception/ExportJobNotFoundException.cs +++ b/src/ExportHistory/Exception/ExportJobNotFoundException.cs @@ -8,23 +8,13 @@ namespace Microsoft.DurableTask.ExportHistory; /// public class ExportJobNotFoundException : InvalidOperationException { - /// - /// Initializes a new instance of the class. - /// - /// The ID of the export history job that was not found. - public ExportJobNotFoundException(string jobId) - : base($"Export history job with ID '{jobId}' was not found.") - { - this.JobId = jobId; - } - /// /// Initializes a new instance of the class. /// /// The ID of the export history job that was not found. /// The exception that is the cause of the current exception. - public ExportJobNotFoundException(string jobId, Exception innerException) - : base($"Export history job with ID '{jobId}' was not found.", innerException) + public ExportJobNotFoundException(string jobId, Exception? innerException = null) + : base($"Export history job with ID '{jobId}' was not found.", innerException!) { this.JobId = jobId; } diff --git a/src/ExportHistory/Models/ExportFormat.cs b/src/ExportHistory/Models/ExportFormat.cs index 2047e5edc..56efea4bc 100644 --- a/src/ExportHistory/Models/ExportFormat.cs +++ b/src/ExportHistory/Models/ExportFormat.cs @@ -1,15 +1,34 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text.Json.Serialization; + namespace Microsoft.DurableTask.ExportHistory; +/// +/// The kind of export format. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ExportFormatKind +{ + /// + /// JSONL format (one history event per line, compressed with gzip). + /// + Jsonl, + + /// + /// JSON format (array of history events, uncompressed). + /// + Json, +} + /// /// Export format settings. /// /// The kind of export format. /// The schema version. -public partial record ExportFormat( - string Kind = "jsonl", +public record ExportFormat( + ExportFormatKind Kind = ExportFormatKind.Jsonl, string SchemaVersion = "1.0") { /// diff --git a/src/ExportHistory/Models/ExportJobCreationOptions.cs b/src/ExportHistory/Models/ExportJobCreationOptions.cs index fddacc0cd..cc586c9e1 100644 --- a/src/ExportHistory/Models/ExportJobCreationOptions.cs +++ b/src/ExportHistory/Models/ExportJobCreationOptions.cs @@ -5,7 +5,7 @@ namespace Microsoft.DurableTask.ExportHistory; /// -/// Configuration for a export job. +/// Configuration for an export job. /// public record ExportJobCreationOptions { @@ -30,7 +30,7 @@ public ExportJobCreationOptions() /// Thrown when validation fails. public ExportJobCreationOptions( ExportMode mode, - DateTimeOffset completedTimeFrom, + DateTimeOffset? completedTimeFrom, DateTimeOffset? completedTimeTo, ExportDestination? destination, string? jobId = null, @@ -43,7 +43,7 @@ public ExportJobCreationOptions( if (mode == ExportMode.Batch) { - if (completedTimeFrom == default) + if (!completedTimeFrom.HasValue) { throw new ArgumentException( "CompletedTimeFrom is required for Batch export mode.", @@ -73,13 +73,6 @@ public ExportJobCreationOptions( } else if (mode == ExportMode.Continuous) { - if (completedTimeFrom != default) - { - throw new ArgumentException( - "CompletedTimeFrom is not allowed for Continuous export mode.", - nameof(completedTimeFrom)); - } - if (completedTimeTo.HasValue) { throw new ArgumentException( @@ -108,16 +101,15 @@ public ExportJobCreationOptions( && runtimeStatus.Any( s => s is not (OrchestrationRuntimeStatus.Completed or OrchestrationRuntimeStatus.Failed - or OrchestrationRuntimeStatus.Terminated - or OrchestrationRuntimeStatus.ContinuedAsNew))) + or OrchestrationRuntimeStatus.Terminated))) { throw new ArgumentException( - "Export supports terminal orchestration statuses only. Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew.", + "Export supports terminal orchestration statuses only. Valid statuses are: Completed, Failed, and Terminated.", nameof(runtimeStatus)); } this.Mode = mode; - this.CompletedTimeFrom = completedTimeFrom; + this.CompletedTimeFrom = completedTimeFrom ?? DateTimeOffset.UtcNow; this.CompletedTimeTo = completedTimeTo; this.Destination = destination; this.Format = format ?? ExportFormat.Default; @@ -128,7 +120,6 @@ or OrchestrationRuntimeStatus.Terminated OrchestrationRuntimeStatus.Completed, OrchestrationRuntimeStatus.Failed, OrchestrationRuntimeStatus.Terminated, - OrchestrationRuntimeStatus.ContinuedAsNew, }; this.MaxInstancesPerBatch = maxInstancesPerBatch ?? 100; } diff --git a/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs index a860aa516..747169a7a 100644 --- a/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs +++ b/src/ExportHistory/Orchestrations/ExecuteExportJobOperationOrchestrator.cs @@ -20,7 +20,7 @@ public override async Task RunAsync(TaskOrchestrationContext context, Ex } /// -/// Request for executing a export job operation. +/// Request for executing an export job operation. /// /// The ID of the entity to execute the operation on. /// The name of the operation to execute. diff --git a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs index c8eab1cf0..7112c79eb 100644 --- a/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs +++ b/src/ExportHistory/Orchestrations/ExportJobOrchestrator.cs @@ -86,26 +86,6 @@ public class ExportJobOrchestrator : TaskOrchestrator 0) { diff --git a/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs index e1d91b0ae..fc4989677 100644 --- a/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs +++ b/test/ExportHistory.Tests/Client/DefaultExportHistoryJobClientTests.cs @@ -74,12 +74,12 @@ public void Constructor_WithWhitespaceJobId_DoesNotThrow() // Arrange // Check.NotNullOrEmpty only checks for null, empty, or strings starting with '\0' // It does NOT check for whitespace-only strings, so " " is valid - string jobId = " "; + string testJobId = " "; // Act var client = new DefaultExportHistoryJobClient( this.durableTaskClient.Object, - jobId, + testJobId, this.logger, this.storageOptions); diff --git a/test/ExportHistory.Tests/Models/ExportFormatTests.cs b/test/ExportHistory.Tests/Models/ExportFormatTests.cs index a393fb39f..144bc3b7f 100644 --- a/test/ExportHistory.Tests/Models/ExportFormatTests.cs +++ b/test/ExportHistory.Tests/Models/ExportFormatTests.cs @@ -16,7 +16,7 @@ public void Constructor_WithDefaultValues_CreatesInstance() // Assert format.Should().NotBeNull(); - format.Kind.Should().Be("jsonl"); + format.Kind.Should().Be(ExportFormatKind.Jsonl); format.SchemaVersion.Should().Be("1.0"); } @@ -24,7 +24,7 @@ public void Constructor_WithDefaultValues_CreatesInstance() public void Constructor_WithCustomValues_CreatesInstance() { // Arrange - string kind = "json"; + ExportFormatKind kind = ExportFormatKind.Json; string schemaVersion = "2.0"; // Act @@ -44,7 +44,7 @@ public void Default_ReturnsDefaultInstance() // Assert format.Should().NotBeNull(); - format.Kind.Should().Be("jsonl"); + format.Kind.Should().Be(ExportFormatKind.Jsonl); format.SchemaVersion.Should().Be("1.0"); } diff --git a/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs index 7b36acd1f..61d8b836d 100644 --- a/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs +++ b/test/ExportHistory.Tests/Orchestrations/ExecuteExportJobOperationOrchestratorTests.cs @@ -79,7 +79,7 @@ public async Task RunAsync_WithDeleteOperation_CallsEntityOperation() { // Arrange var entityId = new EntityInstanceId(nameof(ExportJob), "test-job"); - string operationName = ExportJobOperations.Delete; + string operationName = nameof(ExportJob.Delete); var request = new ExportJobOperationRequest(entityId, operationName, null); this.mockEntityClient diff --git a/test/TestHelpers/Logging/TestLogProvider.cs b/test/TestHelpers/Logging/TestLogProvider.cs index a4d1b46fd..426ad6fb1 100644 --- a/test/TestHelpers/Logging/TestLogProvider.cs +++ b/test/TestHelpers/Logging/TestLogProvider.cs @@ -18,7 +18,7 @@ public bool TryGetLogs(string category, out IReadOnlyCollection logs) // (e.g. "Microsoft.DurableTask.Worker" will return all logs for "Microsoft.DurableTask.Worker.*") logs = this.loggers .Where(kvp => kvp.Key.StartsWith(category, StringComparison.OrdinalIgnoreCase)) - .SelectMany(kvp => kvp.Value.GetLogs()) + .SelectMany(kvp => kvp.Value.GetLogs().ToList()) // Create a snapshot to avoid concurrent modification .OrderBy(log => log.Timestamp) .ToList(); return logs.Count > 0;