diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/KustoBaseApiClient.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/KustoBaseApiClient.kt index a96f3c94..4559d5dc 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/KustoBaseApiClient.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/KustoBaseApiClient.kt @@ -41,7 +41,8 @@ open class KustoBaseApiClient( init { // Validate endpoint is trusted unless security checks are skipped // Note: dmUrl might be empty/null in some test scenarios (e.g., mocked clients) - // The null check is required for Java interop - Java callers can pass null despite Kotlin's non-null type + // The null check is required for Java interop - Java callers can pass null despite Kotlin's + // non-null type if (!skipSecurityChecks && dmUrl != null && dmUrl.isNotBlank()) { KustoTrustedEndpoints.validateTrustedEndpoint(dmUrl) } diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/FastSuffixMatcher.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/FastSuffixMatcher.kt index efeb6786..8decc416 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/FastSuffixMatcher.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/FastSuffixMatcher.kt @@ -4,38 +4,37 @@ package com.microsoft.azure.kusto.ingest.v2.auth.endpoints /** * Represents a matching rule for endpoint validation. + * * @param suffix The suffix or hostname to match - * @param exact If true, the candidate must exactly match the suffix. If false, candidate must end with the suffix. + * @param exact If true, the candidate must exactly match the suffix. If false, + * candidate must end with the suffix. */ -data class MatchRule( - val suffix: String, - val exact: Boolean, -) { +data class MatchRule(val suffix: String, val exact: Boolean) { val suffixLength: Int get() = suffix.length } /** * Result of a match operation. + * * @param isMatch Whether the candidate matched * @param matchedRule The rule that matched, or null if no match */ -data class MatchResult( - val isMatch: Boolean, - val matchedRule: MatchRule?, -) +data class MatchResult(val isMatch: Boolean, val matchedRule: MatchRule?) /** - * A fast suffix matcher that efficiently matches hostnames against a set of rules. - * Uses a map indexed by suffix tail for O(1) lookup. + * A fast suffix matcher that efficiently matches hostnames against a set of + * rules. Uses a map indexed by suffix tail for O(1) lookup. */ -class FastSuffixMatcher private constructor( +class FastSuffixMatcher +private constructor( private val suffixLength: Int, private val rules: Map>, ) { companion object { /** * Creates a new matcher with the provided matching rules. + * * @param rules One or more matching rules to apply when match is called * @return FastSuffixMatcher */ @@ -50,7 +49,9 @@ class FastSuffixMatcher private constructor( val processedRules = mutableMapOf>() for (rule in rules) { val suffix = rule.suffix.takeLast(minRuleLength).lowercase() - processedRules.getOrPut(suffix) { mutableListOf() }.add(rule.copy()) + processedRules + .getOrPut(suffix) { mutableListOf() } + .add(rule.copy()) } return FastSuffixMatcher(minRuleLength, processedRules) @@ -59,6 +60,7 @@ class FastSuffixMatcher private constructor( /** * Creates a new matcher with the provided matching rules, extending an * existing matcher. + * * @param existing An existing matcher whose rules are to be baseline * @param rules One or more matching rules to apply when match is called * @return FastSuffixMatcher @@ -75,14 +77,14 @@ class FastSuffixMatcher private constructor( return existing } - val combinedRules = - rules + existing.rules.values.flatten() + val combinedRules = rules + existing.rules.values.flatten() return create(combinedRules) } } /** * Checks if a candidate string matches any of the rules. + * * @param candidate A string to match to the list of match rules * @return true if at least one of the rules matched */ @@ -90,6 +92,7 @@ class FastSuffixMatcher private constructor( /** * Matches an input string to the list of match rules. + * * @param candidate A string to match * @return MatchResult with match status and the matched rule if any */ @@ -113,4 +116,4 @@ class FastSuffixMatcher private constructor( return MatchResult(false, null) } -} \ No newline at end of file +} diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/KustoTrustedEndpoints.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/KustoTrustedEndpoints.kt index 33efe691..4908092c 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/KustoTrustedEndpoints.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/KustoTrustedEndpoints.kt @@ -8,17 +8,17 @@ import java.net.URI import java.net.URISyntaxException /** - * A helper class to determine which DNS names are "well-known/trusted" - * Kusto endpoints. Untrusted endpoints might require additional configuration - * before they can be used, for security reasons. + * A helper class to determine which DNS names are "well-known/trusted" Kusto + * endpoints. Untrusted endpoints might require additional configuration before + * they can be used, for security reasons. */ object KustoTrustedEndpoints { - private val logger = LoggerFactory.getLogger(KustoTrustedEndpoints::class.java) + private val logger = + LoggerFactory.getLogger(KustoTrustedEndpoints::class.java) /** - * Global flag to enable/disable endpoint validation. - * When false, untrusted endpoints will only log a warning instead of - * throwing an exception. + * Global flag to enable/disable endpoint validation. When false, untrusted + * endpoints will only log a warning instead of throwing an exception. */ @JvmField @Volatile @@ -26,11 +26,9 @@ object KustoTrustedEndpoints { private val matchers: MutableMap = mutableMapOf() - @Volatile - private var additionalMatcher: FastSuffixMatcher? = null + @Volatile private var additionalMatcher: FastSuffixMatcher? = null - @Volatile - private var overrideMatcher: ((String) -> Boolean)? = null + @Volatile private var overrideMatcher: ((String) -> Boolean)? = null // Default login endpoint for public cloud private const val DEFAULT_PUBLIC_LOGIN_ENDPOINT = @@ -44,7 +42,8 @@ object KustoTrustedEndpoints { try { val endpointsData = WellKnownKustoEndpointsData.getInstance() - endpointsData.allowedEndpointsByLogin.forEach { (loginEndpoint, allowedEndpoints) -> + endpointsData.allowedEndpointsByLogin.forEach { + (loginEndpoint, allowedEndpoints) -> val rules = mutableListOf() // Add suffix rules (exact = false) @@ -58,7 +57,8 @@ object KustoTrustedEndpoints { } if (rules.isNotEmpty()) { - matchers[loginEndpoint.lowercase()] = FastSuffixMatcher.create(rules) + matchers[loginEndpoint.lowercase()] = + FastSuffixMatcher.create(rules) } } @@ -74,6 +74,7 @@ object KustoTrustedEndpoints { /** * Sets an override policy for endpoint validation. + * * @param matcher Rules that determine if a hostname is a valid/trusted * Kusto endpoint (replaces existing rules) */ @@ -83,13 +84,11 @@ object KustoTrustedEndpoints { /** * Adds additional trusted hosts to the matcher. + * * @param rules A set of rules * @param replace If true, nullifies the last added rules */ - fun addTrustedHosts( - rules: List?, - replace: Boolean, - ) { + fun addTrustedHosts(rules: List?, replace: Boolean) { if (rules.isNullOrEmpty()) { if (replace) { additionalMatcher = null @@ -98,14 +97,20 @@ object KustoTrustedEndpoints { } additionalMatcher = - FastSuffixMatcher.create(if (replace) null else additionalMatcher, rules) + FastSuffixMatcher.create( + if (replace) null else additionalMatcher, + rules, + ) } /** * Validates that the endpoint is trusted. + * * @param uri Kusto endpoint URI string - * @param loginEndpoint The login endpoint to check against (optional, defaults to public cloud) - * @throws KustoClientInvalidConnectionStringException if endpoint is not trusted + * @param loginEndpoint The login endpoint to check against (optional, + * defaults to public cloud) + * @throws KustoClientInvalidConnectionStringException if endpoint is not + * trusted */ fun validateTrustedEndpoint( uri: String, @@ -114,29 +119,34 @@ object KustoTrustedEndpoints { try { validateTrustedEndpoint(URI(uri), loginEndpoint) } catch (ex: URISyntaxException) { - throw KustoClientInvalidConnectionStringException(uri, ex.message ?: "Invalid URI", ex) + throw KustoClientInvalidConnectionStringException( + uri, + ex.message ?: "Invalid URI", + ex, + ) } } /** * Validates that the endpoint is trusted. + * * @param uri Kusto endpoint URI * @param loginEndpoint The login endpoint to check against - * @throws KustoClientInvalidConnectionStringException if endpoint is not trusted + * @throws KustoClientInvalidConnectionStringException if endpoint is not + * trusted */ - fun validateTrustedEndpoint( - uri: URI, - loginEndpoint: String, - ) { + fun validateTrustedEndpoint(uri: URI, loginEndpoint: String) { val host = uri.host ?: uri.toString() validateHostnameIsTrusted(host, loginEndpoint) } /** * Validates that a hostname is trusted. + * * @param hostname The hostname to validate * @param loginEndpoint The login endpoint to check against - * @throws KustoClientInvalidConnectionStringException if hostname is not trusted + * @throws KustoClientInvalidConnectionStringException if hostname is not + * trusted */ private fun validateHostnameIsTrusted( hostname: String, @@ -184,9 +194,7 @@ object KustoTrustedEndpoints { ) } - /** - * Checks if the hostname is a local/loopback address. - */ + /** Checks if the hostname is a local/loopback address. */ private fun isLocalAddress(hostname: String): Boolean { val lowerHost = hostname.lowercase() return lowerHost == "localhost" || @@ -198,6 +206,7 @@ object KustoTrustedEndpoints { /** * Checks if a hostname is trusted without throwing an exception. + * * @param hostname The hostname to check * @param loginEndpoint The login endpoint to check against * @return true if the hostname is trusted diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/WellKnownKustoEndpointsData.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/WellKnownKustoEndpointsData.kt index 978c7608..21cfec8d 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/WellKnownKustoEndpointsData.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/auth/endpoints/WellKnownKustoEndpointsData.kt @@ -3,13 +3,11 @@ package com.microsoft.azure.kusto.ingest.v2.auth.endpoints import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable as KSerializable import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory +import kotlinx.serialization.Serializable as KSerializable -/** - * Data class representing the structure of WellKnownKustoEndpoints.json - */ +/** Data class representing the structure of WellKnownKustoEndpoints.json */ @KSerializable data class AllowedEndpoints( @SerialName("AllowedKustoSuffixes") @@ -20,39 +18,49 @@ data class AllowedEndpoints( @KSerializable data class WellKnownKustoEndpointsData( - @SerialName("_Comments") - val comments: List = emptyList(), + @SerialName("_Comments") val comments: List = emptyList(), @SerialName("AllowedEndpointsByLogin") val allowedEndpointsByLogin: Map = emptyMap(), ) { companion object { - private val logger = LoggerFactory.getLogger(WellKnownKustoEndpointsData::class.java) - - @Volatile - private var instance: WellKnownKustoEndpointsData? = null - + private val logger = + LoggerFactory.getLogger(WellKnownKustoEndpointsData::class.java) + + @Volatile private var instance: WellKnownKustoEndpointsData? = null + private val json = Json { ignoreUnknownKeys = true isLenient = true } fun getInstance(): WellKnownKustoEndpointsData { - return instance ?: synchronized(this) { - instance ?: readInstance().also { instance = it } - } + return instance + ?: synchronized(this) { + instance ?: readInstance().also { instance = it } + } } private fun readInstance(): WellKnownKustoEndpointsData { return try { - val resourceStream = WellKnownKustoEndpointsData::class.java - .getResourceAsStream("/WellKnownKustoEndpoints.json") - ?: throw RuntimeException("WellKnownKustoEndpoints.json not found in classpath") - - val content = resourceStream.bufferedReader().use { it.readText() } + val resourceStream = + WellKnownKustoEndpointsData::class + .java + .getResourceAsStream( + "/WellKnownKustoEndpoints.json", + ) + ?: throw RuntimeException( + "WellKnownKustoEndpoints.json not found in classpath", + ) + + val content = + resourceStream.bufferedReader().use { it.readText() } json.decodeFromString(content) } catch (ex: Exception) { logger.error("Failed to read WellKnownKustoEndpoints.json", ex) - throw RuntimeException("Failed to read WellKnownKustoEndpoints.json", ex) + throw RuntimeException( + "Failed to read WellKnownKustoEndpoints.json", + ex, + ) } } } diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/common/ConfigurationCache.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/common/ConfigurationCache.kt index d9530f54..04e60041 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/common/ConfigurationCache.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/common/ConfigurationCache.kt @@ -11,6 +11,7 @@ import com.microsoft.azure.kusto.ingest.v2.models.ConfigurationResponse import java.lang.AutoCloseable import java.time.Duration import java.util.concurrent.atomic.AtomicReference +import kotlin.math.min /** * Interface for caching configuration data. @@ -150,25 +151,101 @@ class DefaultConfigurationCache( } /** - * Holds both the configuration and its refresh timestamp atomically. This - * prevents race conditions between checking expiration and updating. + * Holds the configuration, its refresh timestamp, and the effective refresh + * interval atomically. This prevents race conditions between checking + * expiration and updating, and ensures we use the correct refresh interval + * from when the config was fetched. */ private data class CachedData( val configuration: ConfigurationResponse, val timestamp: Long, + val refreshInterval: Long, ) private val cache = AtomicReference(null) + /** + * Parses a .NET TimeSpan format string to a Java Duration. + * + * Supports formats: + * - HH:mm:ss (e.g., "01:00:00" = 1 hour) + * - d.HH:mm:ss (e.g., "1.02:30:00" = 1 day, 2 hours, 30 minutes) + * - HH:mm:ss.fffffff (with fractional seconds) + * + * @param timeSpan The TimeSpan string to parse + * @return The parsed Duration, or null if parsing fails + */ + private fun parseTimeSpanToDuration(timeSpan: String): Duration? { + return try { + // Split by '.' to handle days (format: "d.HH:mm:ss") + val parts = timeSpan.split('.') + val timePart = if (parts.size > 1) parts[1] else parts[0] + val days = if (parts.size > 1) parts[0].toLongOrNull() ?: 0L else 0L + + // Split time part by ':' to get hours, minutes, seconds + val timeParts = timePart.split(':') + if (timeParts.size < 3) return null + + val hours = timeParts[0].toLongOrNull() ?: return null + val minutes = timeParts[1].toLongOrNull() ?: return null + + // Handle fractional seconds (e.g., "30.1234567") + val secondsPart = timeParts[2] + val secondsValue = secondsPart.toDoubleOrNull() ?: return null + + // Build duration + var duration = + Duration.ofDays(days) + .plusHours(hours) + .plusMinutes(minutes) + .plusSeconds(secondsValue.toLong()) + + // Add fractional seconds if present + val fractionalSeconds = (secondsValue - secondsValue.toLong()) + if (fractionalSeconds > 0) { + duration = + duration.plusMillis((fractionalSeconds * 1000).toLong()) + } + + duration + } catch (_: Exception) { + null + } + } + + /** + * Helper function to calculate effective refresh interval from a + * configuration response. If the configuration specifies a refresh + * interval, use the minimum of that and the default. Otherwise, use the + * default refresh interval. + */ + private fun calculateEffectiveRefreshInterval( + config: ConfigurationResponse?, + ): Long { + val configRefreshInterval = config?.containerSettings?.refreshInterval + return if (configRefreshInterval?.isNotEmpty() == true) { + val parsedDuration = parseTimeSpanToDuration(configRefreshInterval) + if (parsedDuration != null) { + min(this.refreshInterval.toMillis(), parsedDuration.toMillis()) + } else { + // If parsing fails, log warning and use default + this.refreshInterval.toMillis() + } + } else { + this.refreshInterval.toMillis() + } + } + override suspend fun getConfiguration(): ConfigurationResponse { val currentTime = System.currentTimeMillis() - val cached = cache.get() + val cachedData = cache.get() - // Check if we need to refresh + // Check if we need to refresh based on the effective refresh interval + // stored with the cached data val needsRefresh = - cached == null || - (currentTime - cached.timestamp) >= - refreshInterval.toMillis() + cachedData == null || + (currentTime - cachedData.timestamp) >= + cachedData.refreshInterval if (needsRefresh) { // Attempt to refresh - only one thread will succeed @@ -176,19 +253,27 @@ class DefaultConfigurationCache( runCatching { provider() } .getOrElse { // If fetch fails, return cached if available, otherwise rethrow - cached?.configuration ?: throw it + cachedData?.configuration ?: throw it } + // Calculate effective refresh interval from the NEW configuration + val newEffectiveRefreshInterval = + calculateEffectiveRefreshInterval(newConfig) + // Atomically update if still needed (prevents thundering herd) cache.updateAndGet { current -> - val currentTimestamp = current?.timestamp ?: 0 - // Only update if current is null or still stale + // Only update if current is null or still stale based on its + // stored effective interval if ( current == null || - (currentTime - currentTimestamp) >= - refreshInterval.toMillis() + (currentTime - current.timestamp) >= + current.refreshInterval ) { - CachedData(newConfig, currentTime) + CachedData( + newConfig, + currentTime, + newEffectiveRefreshInterval, + ) } else { // Another thread already refreshed current diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ContainerUploaderBase.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ContainerUploaderBase.kt index c7536f76..cb6bb4bb 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ContainerUploaderBase.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ContainerUploaderBase.kt @@ -54,7 +54,7 @@ abstract class ContainerUploaderBase( private val retryPolicy: IngestRetryPolicy, private val maxConcurrency: Int, private val maxDataSize: Long, - private val configurationCache: ConfigurationCache, + protected val configurationCache: ConfigurationCache, private val uploadMethod: UploadMethod, private val tokenCredential: TokenCredential?, ) : IUploader { @@ -112,7 +112,7 @@ abstract class ContainerUploaderBase( } // Get containers from configuration - val containers = selectContainers(configurationCache, uploadMethod) + val containers = selectContainers(uploadMethod) if (containers.isEmpty()) { logger.error("No containers available for upload") @@ -278,7 +278,10 @@ abstract class ContainerUploaderBase( // Select container using incrementing counter for round-robin distribution // Note: Math.floorMod handles negative values correctly if overflow occurs var containerIndex = - Math.floorMod(containerIndexCounter.getAndIncrement(), containers.size) + Math.floorMod( + containerIndexCounter.getAndIncrement(), + containers.size, + ) logger.debug( "Starting upload with {} containers, round-robin index: {}", @@ -390,7 +393,8 @@ abstract class ContainerUploaderBase( ) // TODO check and validate failure scenarios // Use semaphore for true streaming parallelism - // This allows up to effectiveMaxConcurrency concurrent uploads, starting new ones as soon as slots + // This allows up to effectiveMaxConcurrency concurrent uploads, starting new ones as soon + // as slots // are available val semaphore = Semaphore(effectiveMaxConcurrency) @@ -755,17 +759,14 @@ abstract class ContainerUploaderBase( } /** - * Selects the appropriate containers for upload based on the provided - * configuration cache and upload method. + * Selects the appropriate containers for upload based on the uploader's + * configuration cache and the specified upload method. * - * @param configurationCache The configuration cache to use for selecting - * containers. * @param uploadMethod The upload method to consider when selecting * containers. * @return A list of selected container information. */ abstract suspend fun selectContainers( - configurationCache: ConfigurationCache, uploadMethod: UploadMethod, ): List } diff --git a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploader.kt b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploader.kt index a2084e20..df38b52e 100644 --- a/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploader.kt +++ b/ingest-v2/src/main/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploader.kt @@ -40,7 +40,6 @@ internal constructor( } override suspend fun selectContainers( - configurationCache: ConfigurationCache, uploadMethod: UploadMethod, ): List { // This method is delegated to and this calls getConfiguration again to ensure fresh data is diff --git a/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/TrustedEndpointValidationTest.kt b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/TrustedEndpointValidationTest.kt index e2b1c489..aa5db6c9 100644 --- a/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/TrustedEndpointValidationTest.kt +++ b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/TrustedEndpointValidationTest.kt @@ -34,19 +34,27 @@ import kotlin.test.assertTrue * - KustoTrustedEndpoints object for validation logic * - KustoClientInvalidConnectionStringException for untrusted endpoints * - * Note: This test class uses SAME_THREAD execution mode to prevent race conditions - * when modifying the global enableWellKnownKustoEndpointsValidation flag. + * Note: This test class uses SAME_THREAD execution mode to prevent race + * conditions when modifying the global enableWellKnownKustoEndpointsValidation + * flag. */ @Execution(ExecutionMode.SAME_THREAD) class TrustedEndpointValidationTest { // Mock token credential for testing - private val mockTokenCredential = TokenCredential { _: TokenRequestContext -> - Mono.just(AccessToken("mock-token", OffsetDateTime.now().plusHours(1))) - } + private val mockTokenCredential = + TokenCredential { _: TokenRequestContext -> + Mono.just( + AccessToken( + "mock-token", + OffsetDateTime.now().plusHours(1), + ), + ) + } // Example of an adhoc/untrusted endpoint - private val untrustedEndpoint = "https://my-random-adhoc-cluster.example.com" + private val untrustedEndpoint = + "https://my-random-adhoc-cluster.example.com" // Example of a trusted Kusto endpoint (public cloud) private val trustedEndpoint = "https://mycluster.kusto.windows.net" @@ -74,7 +82,9 @@ class TrustedEndpointValidationTest { // ============================================================================ @Test - @DisplayName("StreamingIngestClient: Untrusted endpoint throws exception without skipSecurityChecks") + @DisplayName( + "StreamingIngestClient: Untrusted endpoint throws exception without skipSecurityChecks", + ) fun `streaming client - untrusted endpoint throws without skip security checks`() { val exception = assertThrows { @@ -86,13 +96,16 @@ class TrustedEndpointValidationTest { assertTrue( exception.message?.contains("not trusted") == true || - exception.message?.contains("kustotrustedendpoints") == true, + exception.message?.contains("kustotrustedendpoints") == + true, "Exception should indicate endpoint is not trusted. Actual: ${exception.message}", ) } @Test - @DisplayName("QueuedIngestClient: Untrusted endpoint throws exception without skipSecurityChecks") + @DisplayName( + "QueuedIngestClient: Untrusted endpoint throws exception without skipSecurityChecks", + ) fun `queued client - untrusted endpoint throws without skip security checks`() { val exception = assertThrows { @@ -104,7 +117,8 @@ class TrustedEndpointValidationTest { assertTrue( exception.message?.contains("not trusted") == true || - exception.message?.contains("kustotrustedendpoints") == true, + exception.message?.contains("kustotrustedendpoints") == + true, "Exception should indicate endpoint is not trusted. Actual: ${exception.message}", ) } @@ -114,7 +128,9 @@ class TrustedEndpointValidationTest { // ============================================================================ @Test - @DisplayName("StreamingIngestClient: Untrusted endpoint works with skipSecurityChecks") + @DisplayName( + "StreamingIngestClient: Untrusted endpoint works with skipSecurityChecks", + ) fun `streaming client - untrusted endpoint works with skip security checks`() { assertDoesNotThrow { StreamingIngestClientBuilder.create(untrustedEndpoint) @@ -125,7 +141,9 @@ class TrustedEndpointValidationTest { } @Test - @DisplayName("QueuedIngestClient: Untrusted endpoint works with skipSecurityChecks") + @DisplayName( + "QueuedIngestClient: Untrusted endpoint works with skipSecurityChecks", + ) fun `queued client - untrusted endpoint works with skip security checks`() { assertDoesNotThrow { QueuedIngestClientBuilder.create(untrustedEndpoint) @@ -140,7 +158,9 @@ class TrustedEndpointValidationTest { // ============================================================================ @Test - @DisplayName("StreamingIngestClient: Trusted Kusto endpoint works without skipSecurityChecks") + @DisplayName( + "StreamingIngestClient: Trusted Kusto endpoint works without skipSecurityChecks", + ) fun `streaming client - trusted endpoint works without skip security checks`() { assertDoesNotThrow { StreamingIngestClientBuilder.create(trustedEndpoint) @@ -150,7 +170,9 @@ class TrustedEndpointValidationTest { } @Test - @DisplayName("QueuedIngestClient: Trusted Kusto endpoint works without skipSecurityChecks") + @DisplayName( + "QueuedIngestClient: Trusted Kusto endpoint works without skipSecurityChecks", + ) fun `queued client - trusted endpoint works without skip security checks`() { assertDoesNotThrow { QueuedIngestClientBuilder.create(trustedEndpoint) @@ -221,7 +243,9 @@ class TrustedEndpointValidationTest { ) localhostEndpoints.forEach { endpoint -> - assertDoesNotThrow("Localhost endpoint $endpoint should be trusted") { + assertDoesNotThrow( + "Localhost endpoint $endpoint should be trusted", + ) { StreamingIngestClientBuilder.create(endpoint) .withAuthentication(mockTokenCredential) .build() @@ -241,7 +265,9 @@ class TrustedEndpointValidationTest { "Public cloud endpoint should be trusted", ) assertTrue( - KustoTrustedEndpoints.isTrusted("mycluster.kusto.fabric.microsoft.com"), + KustoTrustedEndpoints.isTrusted( + "mycluster.kusto.fabric.microsoft.com", + ), "Fabric endpoint should be trusted", ) assertTrue( @@ -255,18 +281,26 @@ class TrustedEndpointValidationTest { } @Test - @DisplayName("KustoTrustedEndpoints.validateTrustedEndpoint throws for untrusted") + @DisplayName( + "KustoTrustedEndpoints.validateTrustedEndpoint throws for untrusted", + ) fun `validateTrustedEndpoint throws for untrusted endpoints`() { assertThrows { - KustoTrustedEndpoints.validateTrustedEndpoint("https://evil.example.com") + KustoTrustedEndpoints.validateTrustedEndpoint( + "https://evil.example.com", + ) } } @Test - @DisplayName("KustoTrustedEndpoints.validateTrustedEndpoint passes for trusted") + @DisplayName( + "KustoTrustedEndpoints.validateTrustedEndpoint passes for trusted", + ) fun `validateTrustedEndpoint passes for trusted endpoints`() { assertDoesNotThrow { - KustoTrustedEndpoints.validateTrustedEndpoint("https://mycluster.kusto.windows.net") + KustoTrustedEndpoints.validateTrustedEndpoint( + "https://mycluster.kusto.windows.net", + ) } } -} \ No newline at end of file +} diff --git a/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/common/TimeSpanParsingTest.kt b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/common/TimeSpanParsingTest.kt new file mode 100644 index 00000000..5e00d68b --- /dev/null +++ b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/common/TimeSpanParsingTest.kt @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.azure.kusto.ingest.v2.common + +import com.microsoft.azure.kusto.ingest.v2.common.models.ClientDetails +import com.microsoft.azure.kusto.ingest.v2.models.ConfigurationResponse +import com.microsoft.azure.kusto.ingest.v2.models.ContainerSettings +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test +import java.time.Duration +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class TimeSpanParsingTest { + + private fun createClientDetails() = + ClientDetails( + applicationForTracing = "test", + userNameForTracing = "testUser", + clientVersionForTracing = "1.0", + ) + + @Test + fun `parseTimeSpan handles standard HH_mm_ss format`() { + runBlocking { + // Given a configuration with "01:00:00" (1 hour) + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "01:00:00", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofHours(2), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should parse "01:00:00" as 1 hour + assertNotNull(result) + assertEquals("01:00:00", result.containerSettings?.refreshInterval) + } + } + + @Test + fun `parseTimeSpan handles days format d_HH_mm_ss`() { + runBlocking { + // Given a configuration with "1.02:30:00" (1 day, 2 hours, 30 minutes) + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "1.02:30:00", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofDays(2), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should parse correctly + assertNotNull(result) + assertEquals( + "1.02:30:00", + result.containerSettings?.refreshInterval, + ) + } + } + + @Test + fun `parseTimeSpan handles fractional seconds`() { + runBlocking { + // Given a configuration with "00:00:30.5" (30.5 seconds) + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "00:00:30.5", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofMinutes(1), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should parse correctly + assertNotNull(result) + assertEquals( + "00:00:30.5", + result.containerSettings?.refreshInterval, + ) + } + } + + @Test + fun `parseTimeSpan uses default when format is invalid`() { + runBlocking { + // Given a configuration with invalid format + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "invalid", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val defaultRefresh = Duration.ofHours(3) + val cache = + DefaultConfigurationCache( + refreshInterval = defaultRefresh, + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should fall back to default and not throw + assertNotNull(result) + } + } + + @Test + fun `parseTimeSpan uses minimum of config and default`() { + runBlocking { + // Given a configuration with "00:30:00" (30 minutes) + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "00:30:00", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + // And default is 2 hours + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofHours(2), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then the effective refresh should be the minimum (30 minutes from config) + assertNotNull(result) + } + } + + @Test + fun `parseTimeSpan handles empty string`() { + runBlocking { + // Given a configuration with empty refreshInterval + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = "", + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofHours(1), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should use default and not throw + assertNotNull(result) + } + } + + @Test + fun `parseTimeSpan handles null refreshInterval`() { + runBlocking { + // Given a configuration with null refreshInterval + val config = + ConfigurationResponse( + containerSettings = + ContainerSettings( + containers = emptyList(), + lakeFolders = emptyList(), + refreshInterval = null, + preferredUploadMethod = null, + ), + ingestionSettings = null, + ) + + val cache = + DefaultConfigurationCache( + refreshInterval = Duration.ofHours(1), + clientDetails = createClientDetails(), + configurationProvider = { config }, + ) + + // When getting configuration + val result = cache.getConfiguration() + + // Then it should use default and not throw + assertNotNull(result) + } + } +} diff --git a/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploaderTest.kt b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploaderTest.kt new file mode 100644 index 00000000..d201de9e --- /dev/null +++ b/ingest-v2/src/test/kotlin/com/microsoft/azure/kusto/ingest/v2/uploader/ManagedUploaderTest.kt @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.azure.kusto.ingest.v2.uploader + +import com.microsoft.azure.kusto.ingest.v2.common.ConfigurationCache +import com.microsoft.azure.kusto.ingest.v2.common.serialization.OffsetDateTimeSerializer +import com.microsoft.azure.kusto.ingest.v2.models.ConfigurationResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.modules.SerializersModule +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths +import java.time.Duration +import java.time.OffsetDateTime + +class ManagedUploaderTest { + + @ParameterizedTest(name = "PreferredUploadMethod={0}") + @CsvSource("DEFAULT", "STORAGE", "LAKE") + fun selectContainers(preferredUploadMethod: String): Unit = runBlocking { + val uploadMethod = UploadMethod.valueOf(preferredUploadMethod) + val configurationCache = TestConfigurationCache() + val managedUploader = + ManagedUploaderBuilder.create() + .withConfigurationCache(configurationCache) + .build() + val selectedContainers = managedUploader.selectContainers(uploadMethod) + assertNotNull(selectedContainers) + assertTrue(selectedContainers.isNotEmpty()) + selectedContainers.forEach { + assertNotNull(it.containerInfo.path) + // When the server configuration prefers Lake and the user does not specify (DEFAULT), + // ManagedUploader should honor the server preference and use Lake. If the user + // explicitly + // specifies a method (e.g., STORAGE), that explicit choice is respected. + if (uploadMethod != UploadMethod.STORAGE) { + assertTrue( + it.containerInfo.path?.contains("alakefolder") ?: false, + ) + assertFalse( + it.containerInfo.path?.contains("somecontainer") + ?: false, + ) + } else { + // User mentioned storage here, use that + assertFalse( + it.containerInfo.path?.contains("alakefolder") ?: false, + ) + assertTrue( + it.containerInfo.path?.contains("somecontainer") + ?: false, + ) + } + } + } + + private class TestConfigurationCache : ConfigurationCache { + private val json = Json { + ignoreUnknownKeys = true + serializersModule = SerializersModule { + contextual(OffsetDateTime::class, OffsetDateTimeSerializer) + } + } + override val refreshInterval: Duration + get() = Duration.ofHours(1) + + override suspend fun getConfiguration(): ConfigurationResponse { + val resourcesDirectory = "src/test/resources/" + val fileName = "config-response.json" + val configContent = + withContext(Dispatchers.IO) { + Files.readString( + Paths.get(resourcesDirectory + fileName), + StandardCharsets.UTF_8, + ) + } + val configurationResponse = + json.decodeFromString(configContent) + + assertNotNull(configurationResponse) + assertNotNull(configurationResponse.containerSettings) + return configurationResponse + } + + override fun close() { + // No resources to clean up in this test implementation + } + } +} diff --git a/ingest-v2/src/test/resources/config-response.json b/ingest-v2/src/test/resources/config-response.json new file mode 100644 index 00000000..d58a019c --- /dev/null +++ b/ingest-v2/src/test/resources/config-response.json @@ -0,0 +1,21 @@ +{ + "containerSettings": { + "containers": [ + { + "path": "https://somecontainer.z11.blob.storage.azure.net/trdwvweg9nfnngghb1eey-20260108-ingestdata-e5c334ee145d4b4-0?sv=keys" + } + ], + "lakeFolders": [ + { + "path": "https://alakefolder.onelake.fabric.microsoft.com/17a97d10-a17f-4d72-8f38-858aac992978/bb9c26d4-4f99-44b5-9614-3ebb037f3510/Ingestions/20260108-lakedata" + } + ], + "refreshInterval": "01:00:00", + "preferredUploadMethod": "Lake" + }, + "ingestionSettings": { + "maxBlobsPerBatch": 20, + "maxDataSize": 6442450944, + "preferredIngestionMethod": "Rest" + } +} \ No newline at end of file