diff --git a/.gitignore b/.gitignore index 2920730cd3..72771fae96 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,4 @@ venv # to override default properties for local development. # And then use `./gradlew run -Dquarkus.profile=local` to run Polaris with dev profile. application-local.properties +.polaris-work-notes.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ef7108a6..74022d7fc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,6 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti ### Breaking changes -- The (Before/After)CommitViewEvent has been removed. - The (Before/After)CommitTableEvent has been removed. - The `PolarisMetricsReporter.reportMetric()` method signature has been extended to include a `receivedTimestamp` parameter of type `java.time.Instant`. diff --git a/persistence/relational-jdbc/build.gradle.kts b/persistence/relational-jdbc/build.gradle.kts index c3e4253727..3b9c7e2b07 100644 --- a/persistence/relational-jdbc/build.gradle.kts +++ b/persistence/relational-jdbc/build.gradle.kts @@ -29,10 +29,16 @@ dependencies { compileOnly(platform(libs.jackson.bom)) compileOnly("com.fasterxml.jackson.core:jackson-annotations") + compileOnly("com.fasterxml.jackson.core:jackson-databind") compileOnly(libs.jakarta.annotation.api) compileOnly(libs.jakarta.enterprise.cdi.api) compileOnly(libs.jakarta.inject.api) + // Iceberg API for metrics report conversion + compileOnly(platform(libs.iceberg.bom)) + compileOnly("org.apache.iceberg:iceberg-api") + compileOnly("org.apache.iceberg:iceberg-core") + implementation(libs.smallrye.common.annotation) // @Identifier implementation(libs.postgresql) @@ -43,6 +49,10 @@ dependencies { testImplementation(libs.h2) testImplementation(testFixtures(project(":polaris-core"))) + // Iceberg API for SpiModelConverter tests + testImplementation(platform(libs.iceberg.bom)) + testImplementation("org.apache.iceberg:iceberg-api") + testImplementation(platform(libs.testcontainers.bom)) testImplementation("org.testcontainers:testcontainers-junit-jupiter") diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index 1ae1b35a55..b29b5732a9 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -60,4 +60,24 @@ public InputStream openInitScriptResource(int schemaVersion) { ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); return classLoader.getResourceAsStream(resourceName); } + + /** + * Open an InputStream that contains data from the metrics schema init script. This stream should + * be closed by the caller. + * + * @param metricsSchemaVersion the metrics schema version (currently only 1 is supported) + * @return an InputStream for the metrics schema SQL file + */ + public InputStream openMetricsSchemaResource(int metricsSchemaVersion) { + if (metricsSchemaVersion != 1) { + throw new IllegalArgumentException( + "Unknown or invalid metrics schema version " + metricsSchemaVersion); + } + + final String resourceName = + String.format("%s/schema-metrics-v%d.sql", this.getDisplayName(), metricsSchemaVersion); + + ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + return classLoader.getResourceAsStream(resourceName); + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java index 814417d1b8..a6e691cd01 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java @@ -88,4 +88,18 @@ public static int getRequestedSchemaVersion(BootstrapOptions bootstrapOptions) { } return -1; } + + /** + * Determines whether the metrics schema should be included during bootstrap. + * + * @param bootstrapOptions The bootstrap options containing schema information. + * @return true if the metrics schema should be included, false otherwise. + */ + public static boolean shouldIncludeMetrics(BootstrapOptions bootstrapOptions) { + SchemaOptions schemaOptions = bootstrapOptions.schemaOptions(); + if (schemaOptions != null) { + return schemaOptions.includeMetrics(); + } + return false; + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 26f38fc31b..28dbfb6eac 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -52,6 +52,7 @@ import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,6 +79,10 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; @Inject RealmConfig realmConfig; + @Inject + @Identifier("relational-jdbc") + MetricsSchemaBootstrap metricsSchemaBootstrap; + protected JdbcMetaStoreManagerFactory() {} protected PrincipalSecretsGenerator secretsGenerator( @@ -172,6 +177,11 @@ public synchronized Map bootstrapRealms( datasourceOperations .getDatabaseType() .openInitScriptResource(effectiveSchemaVersion)); + + // Run the metrics schema bootstrap if requested + if (JdbcBootstrapUtils.shouldIncludeMetrics(bootstrapOptions)) { + metricsSchemaBootstrap.bootstrap(realm); + } } catch (SQLException e) { throw new RuntimeException( String.format("Error executing sql script: %s", e.getMessage()), e); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistence.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistence.java new file mode 100644 index 0000000000..a4621b5397 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistence.java @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.apache.polaris.persistence.relational.jdbc.QueryGenerator.PreparedQuery; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.MetricsQueryCriteria; +import org.apache.polaris.core.persistence.metrics.ReportIdToken; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.pagination.Token; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReportConverter; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReportConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JDBC implementation of {@link MetricsPersistence}. + * + *

This class provides direct JDBC persistence for metrics reports, converting between SPI record + * types ({@link ScanMetricsRecord}, {@link CommitMetricsRecord}) and JDBC model types ({@link + * ModelScanMetricsReport}, {@link ModelCommitMetricsReport}). + * + *

Metrics tables (scan_metrics_report, commit_metrics_report) were introduced in schema version + * 4. On older schemas, all operations are no-ops. + */ +public class JdbcMetricsPersistence implements MetricsPersistence { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcMetricsPersistence.class); + + // Minimum schema version that includes metrics tables + private static final int METRICS_TABLES_MIN_SCHEMA_VERSION = 4; + + private final DatasourceOperations datasourceOperations; + private final String realmId; + private final int schemaVersion; + + /** + * Creates a new JdbcMetricsPersistence instance. + * + * @param datasourceOperations the datasource operations for JDBC access + * @param realmId the realm ID for multi-tenancy + * @param schemaVersion the current schema version + */ + public JdbcMetricsPersistence( + DatasourceOperations datasourceOperations, String realmId, int schemaVersion) { + this.datasourceOperations = datasourceOperations; + this.realmId = realmId; + this.schemaVersion = schemaVersion; + } + + /** + * Returns true if the current schema version supports metrics persistence tables. + * + * @return true if schema version >= 4, false otherwise + */ + public boolean supportsMetricsPersistence() { + return this.schemaVersion >= METRICS_TABLES_MIN_SCHEMA_VERSION; + } + + @Override + public void writeScanReport(@Nonnull ScanMetricsRecord record) { + if (!supportsMetricsPersistence()) { + LOGGER.debug( + "Schema version {} does not support metrics tables. Skipping scan metrics write.", + schemaVersion); + return; + } + ModelScanMetricsReport model = SpiModelConverter.toModelScanReport(record, realmId); + writeScanMetricsReport(model); + } + + @Override + public void writeCommitReport(@Nonnull CommitMetricsRecord record) { + if (!supportsMetricsPersistence()) { + LOGGER.debug( + "Schema version {} does not support metrics tables. Skipping commit metrics write.", + schemaVersion); + return; + } + ModelCommitMetricsReport model = SpiModelConverter.toModelCommitReport(record, realmId); + writeCommitMetricsReport(model); + } + + @Override + @Nonnull + public Page queryScanReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken) { + if (!supportsMetricsPersistence()) { + return Page.fromItems(List.of()); + } + + // catalogId and tableId are required for queries + if (criteria.catalogId().isEmpty() || criteria.tableId().isEmpty()) { + return Page.fromItems(List.of()); + } + + int limit = pageToken.pageSize().orElse(100); + Long startTimeMs = criteria.startTime().map(t -> t.toEpochMilli()).orElse(null); + Long endTimeMs = criteria.endTime().map(t -> t.toEpochMilli()).orElse(null); + + // Extract cursor from page token if present + String lastReportId = + pageToken.valueAs(ReportIdToken.class).map(ReportIdToken::reportId).orElse(null); + + List models = + queryScanMetricsReports( + criteria.catalogId().getAsLong(), + criteria.tableId().getAsLong(), + startTimeMs, + endTimeMs, + lastReportId, + limit); + + List records = + models.stream().map(SpiModelConverter::toScanMetricsRecord).collect(Collectors.toList()); + + // Build continuation token if we have results (there may be more pages) + Token nextToken = + records.isEmpty() ? null : ReportIdToken.fromReportId(records.getLast().reportId()); + + return Page.page(pageToken, records, nextToken); + } + + @Override + @Nonnull + public Page queryCommitReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken) { + if (!supportsMetricsPersistence()) { + return Page.fromItems(List.of()); + } + + // catalogId and tableId are required for queries + if (criteria.catalogId().isEmpty() || criteria.tableId().isEmpty()) { + return Page.fromItems(List.of()); + } + + int limit = pageToken.pageSize().orElse(100); + Long startTimeMs = criteria.startTime().map(t -> t.toEpochMilli()).orElse(null); + Long endTimeMs = criteria.endTime().map(t -> t.toEpochMilli()).orElse(null); + + // Extract cursor from page token if present + String lastReportId = + pageToken.valueAs(ReportIdToken.class).map(ReportIdToken::reportId).orElse(null); + + List models = + queryCommitMetricsReports( + criteria.catalogId().getAsLong(), + criteria.tableId().getAsLong(), + startTimeMs, + endTimeMs, + lastReportId, + limit); + + List records = + models.stream().map(SpiModelConverter::toCommitMetricsRecord).collect(Collectors.toList()); + + // Build continuation token if we have results (there may be more pages) + Token nextToken = + records.isEmpty() ? null : ReportIdToken.fromReportId(records.getLast().reportId()); + + return Page.page(pageToken, records, nextToken); + } + + // ========== Internal JDBC methods ========== + + /** + * Writes a scan metrics report to the database. + * + * @param report the scan metrics report to persist + */ + void writeScanMetricsReport(@Nonnull ModelScanMetricsReport report) { + try { + PreparedQuery pq = + QueryGenerator.generateInsertQueryWithoutRealmId( + ModelScanMetricsReport.ALL_COLUMNS, + ModelScanMetricsReport.TABLE_NAME, + report.toMap(datasourceOperations.getDatabaseType()).values().stream().toList()); + int updated = datasourceOperations.executeUpdate(pq); + if (updated == 0) { + throw new SQLException("Scan metrics report was not inserted."); + } + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to write scan metrics report due to %s", e.getMessage()), e); + } + } + + /** + * Writes a commit metrics report to the database. + * + * @param report the commit metrics report to persist + */ + void writeCommitMetricsReport(@Nonnull ModelCommitMetricsReport report) { + try { + PreparedQuery pq = + QueryGenerator.generateInsertQueryWithoutRealmId( + ModelCommitMetricsReport.ALL_COLUMNS, + ModelCommitMetricsReport.TABLE_NAME, + report.toMap(datasourceOperations.getDatabaseType()).values().stream().toList()); + int updated = datasourceOperations.executeUpdate(pq); + if (updated == 0) { + throw new SQLException("Commit metrics report was not inserted."); + } + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to write commit metrics report due to %s", e.getMessage()), e); + } + } + + /** + * Retrieves scan metrics reports for a specific table within a time range. + * + * @param catalogId the catalog entity ID + * @param tableId the table entity ID + * @param startTimeMs start of time range (inclusive), or null for no lower bound + * @param endTimeMs end of time range (exclusive), or null for no upper bound + * @param lastReportId cursor for pagination: return results after this report ID, or null for + * first page + * @param limit maximum number of results to return + * @return list of scan metrics reports matching the criteria + */ + @Nonnull + List queryScanMetricsReports( + long catalogId, + long tableId, + @Nullable Long startTimeMs, + @Nullable Long endTimeMs, + @Nullable String lastReportId, + int limit) { + try { + StringBuilder whereClause = new StringBuilder(); + whereClause.append("realm_id = ? AND catalog_id = ? AND table_id = ?"); + List values = new ArrayList<>(List.of(realmId, catalogId, tableId)); + + if (startTimeMs != null) { + whereClause.append(" AND timestamp_ms >= ?"); + values.add(startTimeMs); + } + if (endTimeMs != null) { + whereClause.append(" AND timestamp_ms < ?"); + values.add(endTimeMs); + } + if (lastReportId != null) { + whereClause.append(" AND report_id > ?"); + values.add(lastReportId); + } + + String sql = + "SELECT * FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelScanMetricsReport.TABLE_NAME) + + " WHERE " + + whereClause + + " ORDER BY report_id ASC LIMIT " + + limit; + + PreparedQuery query = new PreparedQuery(sql, values); + var results = + datasourceOperations.executeSelect(query, new ModelScanMetricsReportConverter()); + return results == null ? Collections.emptyList() : results; + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to query scan metrics reports due to %s", e.getMessage()), e); + } + } + + /** + * Retrieves commit metrics reports for a specific table within a time range. + * + * @param catalogId the catalog entity ID + * @param tableId the table entity ID + * @param startTimeMs start of time range (inclusive), or null for no lower bound + * @param endTimeMs end of time range (exclusive), or null for no upper bound + * @param lastReportId cursor for pagination: return results after this report ID, or null for + * first page + * @param limit maximum number of results to return + * @return list of commit metrics reports matching the criteria + */ + @Nonnull + List queryCommitMetricsReports( + long catalogId, + long tableId, + @Nullable Long startTimeMs, + @Nullable Long endTimeMs, + @Nullable String lastReportId, + int limit) { + try { + List values = new ArrayList<>(List.of(realmId, catalogId, tableId)); + + StringBuilder whereClause = new StringBuilder(); + whereClause.append("realm_id = ? AND catalog_id = ? AND table_id = ?"); + + if (startTimeMs != null) { + whereClause.append(" AND timestamp_ms >= ?"); + values.add(startTimeMs); + } + if (endTimeMs != null) { + whereClause.append(" AND timestamp_ms < ?"); + values.add(endTimeMs); + } + if (lastReportId != null) { + whereClause.append(" AND report_id > ?"); + values.add(lastReportId); + } + + String sql = + "SELECT * FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelCommitMetricsReport.TABLE_NAME) + + " WHERE " + + whereClause + + " ORDER BY report_id ASC LIMIT " + + limit; + + PreparedQuery query = new PreparedQuery(sql, values); + var results = + datasourceOperations.executeSelect(query, new ModelCommitMetricsReportConverter()); + return results == null ? Collections.emptyList() : results; + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to query commit metrics reports due to %s", e.getMessage()), e); + } + } + + /** + * Retrieves scan metrics reports by OpenTelemetry trace ID. + * + * @param traceId the OpenTelemetry trace ID + * @return list of scan metrics reports with the given trace ID + */ + @Nonnull + public List queryScanMetricsReportsByTraceId(@Nonnull String traceId) { + if (!supportsMetricsPersistence()) { + return Collections.emptyList(); + } + try { + String sql = + "SELECT * FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelScanMetricsReport.TABLE_NAME) + + " WHERE realm_id = ? AND otel_trace_id = ? ORDER BY timestamp_ms DESC"; + + PreparedQuery query = new PreparedQuery(sql, List.of(realmId, traceId)); + var results = + datasourceOperations.executeSelect(query, new ModelScanMetricsReportConverter()); + return results == null ? Collections.emptyList() : results; + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "Failed to query scan metrics reports by trace ID due to %s", e.getMessage()), + e); + } + } + + /** + * Retrieves commit metrics reports by OpenTelemetry trace ID. + * + * @param traceId the OpenTelemetry trace ID + * @return list of commit metrics reports with the given trace ID + */ + @Nonnull + public List queryCommitMetricsReportsByTraceId( + @Nonnull String traceId) { + if (!supportsMetricsPersistence()) { + return Collections.emptyList(); + } + try { + String sql = + "SELECT * FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelCommitMetricsReport.TABLE_NAME) + + " WHERE realm_id = ? AND otel_trace_id = ? ORDER BY timestamp_ms DESC"; + + PreparedQuery query = new PreparedQuery(sql, List.of(realmId, traceId)); + var results = + datasourceOperations.executeSelect(query, new ModelCommitMetricsReportConverter()); + return results == null ? Collections.emptyList() : results; + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "Failed to query commit metrics reports by trace ID due to %s", e.getMessage()), + e); + } + } + + /** + * Deletes scan metrics reports older than the specified timestamp. + * + * @param olderThanMs timestamp in milliseconds; reports with timestamp_ms less than this will be + * deleted + * @return the number of reports deleted, or 0 if schema version < 4 + */ + public int deleteScanMetricsReportsOlderThan(long olderThanMs) { + if (!supportsMetricsPersistence()) { + return 0; + } + try { + String sql = + "DELETE FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelScanMetricsReport.TABLE_NAME) + + " WHERE realm_id = ? AND timestamp_ms < ?"; + + PreparedQuery query = new PreparedQuery(sql, List.of(realmId, olderThanMs)); + return datasourceOperations.executeUpdate(query); + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to delete old scan metrics reports due to %s", e.getMessage()), e); + } + } + + /** + * Deletes commit metrics reports older than the specified timestamp. + * + * @param olderThanMs timestamp in milliseconds; reports with timestamp_ms less than this will be + * deleted + * @return the number of reports deleted, or 0 if schema version < 4 + */ + public int deleteCommitMetricsReportsOlderThan(long olderThanMs) { + if (!supportsMetricsPersistence()) { + return 0; + } + try { + String sql = + "DELETE FROM " + + QueryGenerator.getFullyQualifiedTableName(ModelCommitMetricsReport.TABLE_NAME) + + " WHERE realm_id = ? AND timestamp_ms < ?"; + + PreparedQuery query = new PreparedQuery(sql, List.of(realmId, olderThanMs)); + return datasourceOperations.executeUpdate(query); + } catch (SQLException e) { + throw new RuntimeException( + String.format("Failed to delete old commit metrics reports due to %s", e.getMessage()), + e); + } + } + + /** + * Deletes all metrics reports (both scan and commit) older than the specified timestamp. + * + * @param olderThanMs timestamp in milliseconds; reports with timestamp_ms less than this will be + * deleted + * @return the total number of reports deleted (scan + commit), or 0 if schema version < 4 + */ + public int deleteAllMetricsReportsOlderThan(long olderThanMs) { + int scanDeleted = deleteScanMetricsReportsOlderThan(olderThanMs); + int commitDeleted = deleteCommitMetricsReportsOlderThan(olderThanMs); + return scanDeleted + commitDeleted; + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistenceProducer.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistenceProducer.java new file mode 100644 index 0000000000..d65a5d7de7 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsPersistenceProducer.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; +import java.sql.SQLException; +import javax.sql.DataSource; +import org.apache.polaris.core.config.BehaviorChangeConfiguration; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CDI producer for {@link MetricsPersistence} in the JDBC persistence backend. + * + *

This producer creates {@link JdbcMetricsPersistence} instances when the JDBC persistence + * backend is in use. When metrics tables are not available (schema version < 4), the produced + * instance will report this via {@link JdbcMetricsPersistence#supportsMetricsPersistence()}. + */ +@ApplicationScoped +@Identifier("relational-jdbc") +public class JdbcMetricsPersistenceProducer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(JdbcMetricsPersistenceProducer.class); + + @Inject Instance dataSource; + + @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; + + /** + * Produces a {@link MetricsPersistence} instance for the current request. + * + *

This method creates a new {@link JdbcMetricsPersistence} configured with the current realm + * and schema version. If the schema version is less than 4 (which includes metrics tables), the + * returned instance will be functional but all operations will be no-ops. + * + * @param realmContext the current realm context (request-scoped) + * @param realmConfig the realm configuration (request-scoped) + * @return a MetricsPersistence implementation for JDBC + */ + @Produces + @RequestScoped + @Identifier("relational-jdbc") + public MetricsPersistence metricsPersistence(RealmContext realmContext, RealmConfig realmConfig) { + try { + DatasourceOperations datasourceOperations = + new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); + + String realmId = realmContext.getRealmIdentifier(); + + int schemaVersion = + JdbcBasePersistenceImpl.loadSchemaVersion( + datasourceOperations, + realmConfig.getConfig(BehaviorChangeConfiguration.SCHEMA_VERSION_FALL_BACK_ON_DNE)); + + JdbcMetricsPersistence persistence = + new JdbcMetricsPersistence(datasourceOperations, realmId, schemaVersion); + + if (!persistence.supportsMetricsPersistence()) { + LOGGER.debug( + "Schema version {} does not support metrics tables. " + + "Metrics persistence operations will be no-ops.", + schemaVersion); + } + + return persistence; + } catch (SQLException e) { + LOGGER.warn( + "Failed to create JdbcMetricsPersistence due to {}. Returning NOOP implementation.", + e.getMessage()); + return MetricsPersistence.NOOP; + } + } + + /** + * Produces a {@link MetricsSchemaBootstrap} instance for the JDBC backend. + * + *

This producer creates a {@link JdbcMetricsSchemaBootstrap} that can bootstrap the metrics + * schema tables independently from the entity schema. + * + * @return a MetricsSchemaBootstrap implementation for JDBC + */ + @Produces + @ApplicationScoped + @Identifier("relational-jdbc") + public MetricsSchemaBootstrap metricsSchemaBootstrap() { + try { + DatasourceOperations datasourceOperations = + new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); + return new JdbcMetricsSchemaBootstrap(datasourceOperations); + } catch (SQLException e) { + LOGGER.warn( + "Failed to create JdbcMetricsSchemaBootstrap due to {}. Returning NOOP implementation.", + e.getMessage()); + return MetricsSchemaBootstrap.NOOP; + } + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsSchemaBootstrap.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsSchemaBootstrap.java new file mode 100644 index 0000000000..0802f1f5aa --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetricsSchemaBootstrap.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import java.sql.SQLException; +import java.util.List; +import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap; +import org.apache.polaris.persistence.relational.jdbc.models.MetricsSchemaVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JDBC implementation of {@link MetricsSchemaBootstrap}. + * + *

This implementation creates the metrics schema tables (scan_metrics_report, + * commit_metrics_report, metrics_version) in the configured JDBC database. + * + *

The metrics schema is separate from the entity schema and can be bootstrapped independently. + * This allows operators to add metrics support to existing Polaris deployments without + * re-bootstrapping the entity schema. + */ +public class JdbcMetricsSchemaBootstrap implements MetricsSchemaBootstrap { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcMetricsSchemaBootstrap.class); + + /** Current metrics schema version. */ + private static final int METRICS_SCHEMA_VERSION = 1; + + private final DatasourceOperations datasourceOperations; + + public JdbcMetricsSchemaBootstrap(DatasourceOperations datasourceOperations) { + this.datasourceOperations = datasourceOperations; + } + + @Override + public void bootstrap(String realmId) { + if (isBootstrapped(realmId)) { + LOGGER.debug("Metrics schema already bootstrapped for realm: {}", realmId); + return; + } + + LOGGER.info("Bootstrapping metrics schema v{} for realm: {}", METRICS_SCHEMA_VERSION, realmId); + + try { + datasourceOperations.executeScript( + datasourceOperations.getDatabaseType().openMetricsSchemaResource(METRICS_SCHEMA_VERSION)); + LOGGER.info( + "Successfully bootstrapped metrics schema v{} for realm: {}", + METRICS_SCHEMA_VERSION, + realmId); + } catch (SQLException e) { + throw new RuntimeException( + String.format( + "Failed to bootstrap metrics schema for realm '%s': %s", realmId, e.getMessage()), + e); + } + } + + @Override + public boolean isBootstrapped(String realmId) { + return loadMetricsSchemaVersion() > 0; + } + + /** + * Loads the current metrics schema version from the database. + * + * @return the metrics schema version, or 0 if not bootstrapped + */ + int loadMetricsSchemaVersion() { + QueryGenerator.PreparedQuery query = QueryGenerator.generateMetricsVersionQuery(); + try { + List versions = + datasourceOperations.executeSelect(query, new MetricsSchemaVersion()); + if (versions == null || versions.isEmpty()) { + return 0; + } + return versions.getFirst().getValue(); + } catch (SQLException e) { + if (datasourceOperations.isRelationDoesNotExist(e)) { + // Table doesn't exist yet - schema not bootstrapped + LOGGER.debug("Metrics schema version table not found: {}", e.getMessage()); + return 0; + } + LOGGER.error("Failed to load metrics schema version due to {}", e.getMessage(), e); + throw new IllegalStateException("Failed to retrieve metrics schema version", e); + } + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java index 485956ed85..8f770b521b 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java @@ -169,6 +169,30 @@ public static PreparedQuery generateInsertQuery( return new PreparedQuery(sql, finalValues); } + /** + * Generates an INSERT query for a given table without appending realm_id. Use this when the + * columns already include realm_id. + * + * @param allColumns Columns to insert values into (should already include realm_id if needed). + * @param tableName Target table name. + * @param values Values for each column (must match order of columns). + * @return INSERT query with value bindings. + */ + public static PreparedQuery generateInsertQueryWithoutRealmId( + @Nonnull List allColumns, @Nonnull String tableName, List values) { + String columns = String.join(", ", allColumns); + String placeholders = allColumns.stream().map(c -> "?").collect(Collectors.joining(", ")); + String sql = + "INSERT INTO " + + getFullyQualifiedTableName(tableName) + + " (" + + columns + + ") VALUES (" + + placeholders + + ")"; + return new PreparedQuery(sql, values); + } + /** * Builds an UPDATE query. * @@ -256,6 +280,13 @@ static PreparedQuery generateVersionQuery() { return new PreparedQuery("SELECT version_value FROM POLARIS_SCHEMA.VERSION", List.of()); } + @VisibleForTesting + static PreparedQuery generateMetricsVersionQuery() { + return new PreparedQuery( + "SELECT version_value FROM POLARIS_SCHEMA.metrics_version WHERE version_key = 'metrics_version'", + List.of()); + } + @VisibleForTesting static PreparedQuery generateEntityTableExistQuery() { return new PreparedQuery( @@ -317,7 +348,7 @@ public static PreparedQuery generateOverlapQuery( return new PreparedQuery(query.sql(), where.parameters()); } - private static String getFullyQualifiedTableName(String tableName) { + static String getFullyQualifiedTableName(String tableName) { // TODO: make schema name configurable. return "POLARIS_SCHEMA." + tableName; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverter.java new file mode 100644 index 0000000000..11999b33e2 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverter.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelScanMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; + +/** + * Converter between SPI metrics records and JDBC model classes. + * + *

This utility class provides methods to convert between the backend-agnostic SPI types ({@link + * ScanMetricsRecord}, {@link CommitMetricsRecord}) and the JDBC-specific model types ({@link + * ModelScanMetricsReport}, {@link ModelCommitMetricsReport}). + * + *

Key conversions handled: + * + *

    + *
  • catalogId: long (SPI) ↔ long (Model) + *
  • timestamp: Instant (SPI) ↔ long milliseconds (Model) + *
  • metadata: Map<String, String> (SPI) ↔ JSON string (Model) + *
  • projectedFieldIds/Names: List (SPI) ↔ comma-separated string (Model) + *
+ */ +public final class SpiModelConverter { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private SpiModelConverter() { + // Utility class + } + + /** + * Converts a ScanMetricsRecord (SPI) to ModelScanMetricsReport (JDBC). + * + * @param record the SPI record + * @param realmId the realm ID for multi-tenancy + * @return the JDBC model + */ + public static ModelScanMetricsReport toModelScanReport(ScanMetricsRecord record, String realmId) { + return ImmutableModelScanMetricsReport.builder() + .reportId(record.reportId()) + .realmId(realmId) + .catalogId(record.catalogId()) + .tableId(record.tableId()) + .timestampMs(record.timestamp().toEpochMilli()) + .snapshotId(record.snapshotId().orElse(null)) + .schemaId(record.schemaId().orElse(null)) + .filterExpression(record.filterExpression().orElse(null)) + .projectedFieldIds(toCommaSeparated(record.projectedFieldIds())) + .projectedFieldNames(toCommaSeparated(record.projectedFieldNames())) + .resultDataFiles(record.resultDataFiles()) + .resultDeleteFiles(record.resultDeleteFiles()) + .totalFileSizeBytes(record.totalFileSizeBytes()) + .totalDataManifests(record.totalDataManifests()) + .totalDeleteManifests(record.totalDeleteManifests()) + .scannedDataManifests(record.scannedDataManifests()) + .scannedDeleteManifests(record.scannedDeleteManifests()) + .skippedDataManifests(record.skippedDataManifests()) + .skippedDeleteManifests(record.skippedDeleteManifests()) + .skippedDataFiles(record.skippedDataFiles()) + .skippedDeleteFiles(record.skippedDeleteFiles()) + .totalPlanningDurationMs(record.totalPlanningDurationMs()) + .equalityDeleteFiles(record.equalityDeleteFiles()) + .positionalDeleteFiles(record.positionalDeleteFiles()) + .indexedDeleteFiles(record.indexedDeleteFiles()) + .totalDeleteFileSizeBytes(record.totalDeleteFileSizeBytes()) + .metadata(toJsonString(record.metadata())) + .build(); + } + + /** + * Converts a CommitMetricsRecord (SPI) to ModelCommitMetricsReport (JDBC). + * + * @param record the SPI record + * @param realmId the realm ID for multi-tenancy + * @return the JDBC model + */ + public static ModelCommitMetricsReport toModelCommitReport( + CommitMetricsRecord record, String realmId) { + return ImmutableModelCommitMetricsReport.builder() + .reportId(record.reportId()) + .realmId(realmId) + .catalogId(record.catalogId()) + .tableId(record.tableId()) + .timestampMs(record.timestamp().toEpochMilli()) + .snapshotId(record.snapshotId()) + .sequenceNumber(record.sequenceNumber().orElse(null)) + .operation(record.operation()) + .addedDataFiles(record.addedDataFiles()) + .removedDataFiles(record.removedDataFiles()) + .totalDataFiles(record.totalDataFiles()) + .addedDeleteFiles(record.addedDeleteFiles()) + .removedDeleteFiles(record.removedDeleteFiles()) + .totalDeleteFiles(record.totalDeleteFiles()) + .addedEqualityDeleteFiles(record.addedEqualityDeleteFiles()) + .removedEqualityDeleteFiles(record.removedEqualityDeleteFiles()) + .addedPositionalDeleteFiles(record.addedPositionalDeleteFiles()) + .removedPositionalDeleteFiles(record.removedPositionalDeleteFiles()) + .addedRecords(record.addedRecords()) + .removedRecords(record.removedRecords()) + .totalRecords(record.totalRecords()) + .addedFileSizeBytes(record.addedFileSizeBytes()) + .removedFileSizeBytes(record.removedFileSizeBytes()) + .totalFileSizeBytes(record.totalFileSizeBytes()) + .totalDurationMs(record.totalDurationMs().orElse(0L)) + .attempts(record.attempts()) + .metadata(toJsonString(record.metadata())) + .build(); + } + + /** + * Converts a ModelScanMetricsReport (JDBC) to ScanMetricsRecord (SPI). + * + * @param model the JDBC model + * @return the SPI record + */ + public static ScanMetricsRecord toScanMetricsRecord(ModelScanMetricsReport model) { + return ScanMetricsRecord.builder() + .reportId(model.getReportId()) + .catalogId(model.getCatalogId()) + .tableId(model.getTableId()) + .timestamp(Instant.ofEpochMilli(model.getTimestampMs())) + .snapshotId(Optional.ofNullable(model.getSnapshotId())) + .schemaId(Optional.ofNullable(model.getSchemaId())) + .filterExpression(Optional.ofNullable(model.getFilterExpression())) + .projectedFieldIds(parseIntList(model.getProjectedFieldIds())) + .projectedFieldNames(parseStringList(model.getProjectedFieldNames())) + .resultDataFiles(model.getResultDataFiles()) + .resultDeleteFiles(model.getResultDeleteFiles()) + .totalFileSizeBytes(model.getTotalFileSizeBytes()) + .totalDataManifests(model.getTotalDataManifests()) + .totalDeleteManifests(model.getTotalDeleteManifests()) + .scannedDataManifests(model.getScannedDataManifests()) + .scannedDeleteManifests(model.getScannedDeleteManifests()) + .skippedDataManifests(model.getSkippedDataManifests()) + .skippedDeleteManifests(model.getSkippedDeleteManifests()) + .skippedDataFiles(model.getSkippedDataFiles()) + .skippedDeleteFiles(model.getSkippedDeleteFiles()) + .totalPlanningDurationMs(model.getTotalPlanningDurationMs()) + .equalityDeleteFiles(model.getEqualityDeleteFiles()) + .positionalDeleteFiles(model.getPositionalDeleteFiles()) + .indexedDeleteFiles(model.getIndexedDeleteFiles()) + .totalDeleteFileSizeBytes(model.getTotalDeleteFileSizeBytes()) + .metadata(parseMetadataJson(model.getMetadata())) + .build(); + } + + /** + * Converts a ModelCommitMetricsReport (JDBC) to CommitMetricsRecord (SPI). + * + * @param model the JDBC model + * @return the SPI record + */ + public static CommitMetricsRecord toCommitMetricsRecord(ModelCommitMetricsReport model) { + return CommitMetricsRecord.builder() + .reportId(model.getReportId()) + .catalogId(model.getCatalogId()) + .tableId(model.getTableId()) + .timestamp(Instant.ofEpochMilli(model.getTimestampMs())) + .snapshotId(model.getSnapshotId()) + .sequenceNumber(Optional.ofNullable(model.getSequenceNumber())) + .operation(model.getOperation()) + .addedDataFiles(model.getAddedDataFiles()) + .removedDataFiles(model.getRemovedDataFiles()) + .totalDataFiles(model.getTotalDataFiles()) + .addedDeleteFiles(model.getAddedDeleteFiles()) + .removedDeleteFiles(model.getRemovedDeleteFiles()) + .totalDeleteFiles(model.getTotalDeleteFiles()) + .addedEqualityDeleteFiles(model.getAddedEqualityDeleteFiles()) + .removedEqualityDeleteFiles(model.getRemovedEqualityDeleteFiles()) + .addedPositionalDeleteFiles(model.getAddedPositionalDeleteFiles()) + .removedPositionalDeleteFiles(model.getRemovedPositionalDeleteFiles()) + .addedRecords(model.getAddedRecords()) + .removedRecords(model.getRemovedRecords()) + .totalRecords(model.getTotalRecords()) + .addedFileSizeBytes(model.getAddedFileSizeBytes()) + .removedFileSizeBytes(model.getRemovedFileSizeBytes()) + .totalFileSizeBytes(model.getTotalFileSizeBytes()) + .totalDurationMs( + model.getTotalDurationMs() > 0 + ? Optional.of(model.getTotalDurationMs()) + : Optional.empty()) + .attempts(model.getAttempts()) + .metadata(parseMetadataJson(model.getMetadata())) + .build(); + } + + // === Helper Methods === + + private static String toCommaSeparated(List list) { + if (list == null || list.isEmpty()) { + return null; + } + return list.stream().map(Object::toString).collect(Collectors.joining(",")); + } + + private static List parseIntList(String commaSeparated) { + if (commaSeparated == null || commaSeparated.isEmpty()) { + return Collections.emptyList(); + } + return java.util.Arrays.stream(commaSeparated.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Integer::parseInt) + .collect(Collectors.toList()); + } + + private static List parseStringList(String commaSeparated) { + if (commaSeparated == null || commaSeparated.isEmpty()) { + return Collections.emptyList(); + } + return java.util.Arrays.stream(commaSeparated.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + private static String toJsonString(Map map) { + if (map == null || map.isEmpty()) { + return "{}"; + } + try { + return OBJECT_MAPPER.writeValueAsString(map); + } catch (JsonProcessingException e) { + return "{}"; + } + } + + private static Map parseMetadataJson(String json) { + if (json == null || json.isEmpty() || "{}".equals(json)) { + return Collections.emptyMap(); + } + try { + return OBJECT_MAPPER.readValue(json, new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Collections.emptyMap(); + } + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsReportConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsReportConverter.java new file mode 100644 index 0000000000..4bd841f0a0 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsReportConverter.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.iceberg.metrics.CommitMetricsResult; +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.CounterResult; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.iceberg.metrics.TimerResult; + +/** + * Converter utility class for transforming Iceberg metrics reports into persistence model classes. + */ +public final class MetricsReportConverter { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private MetricsReportConverter() { + // Utility class + } + + /** + * Converts an Iceberg ScanReport to a ModelScanMetricsReport. + * + * @param scanReport the Iceberg scan report + * @param realmId the realm ID for multi-tenancy + * @param catalogId the catalog ID + * @param tableId the table entity ID + * @param principalName the principal who initiated the scan (optional) + * @param requestId the request ID (optional) + * @param otelTraceId OpenTelemetry trace ID (optional) + * @param otelSpanId OpenTelemetry span ID (optional) + * @return the converted ModelScanMetricsReport + */ + public static ModelScanMetricsReport fromScanReport( + ScanReport scanReport, + String realmId, + long catalogId, + long tableId, + @Nullable String principalName, + @Nullable String requestId, + @Nullable String otelTraceId, + @Nullable String otelSpanId) { + + String reportId = UUID.randomUUID().toString(); + long timestampMs = System.currentTimeMillis(); + + ScanMetricsResult metrics = scanReport.scanMetrics(); + + ImmutableModelScanMetricsReport.Builder builder = + ImmutableModelScanMetricsReport.builder() + .reportId(reportId) + .realmId(realmId) + .catalogId(catalogId) + .tableId(tableId) + .timestampMs(timestampMs) + .principalName(principalName) + .requestId(requestId) + .otelTraceId(otelTraceId) + .otelSpanId(otelSpanId) + .snapshotId(scanReport.snapshotId()) + .schemaId(scanReport.schemaId()) + .filterExpression(scanReport.filter() != null ? scanReport.filter().toString() : null) + .projectedFieldIds(formatIntegerList(scanReport.projectedFieldIds())) + .projectedFieldNames(formatStringList(scanReport.projectedFieldNames())); + + // Extract metrics values + if (metrics != null) { + builder + .resultDataFiles(getCounterValue(metrics.resultDataFiles())) + .resultDeleteFiles(getCounterValue(metrics.resultDeleteFiles())) + .totalFileSizeBytes(getCounterValue(metrics.totalFileSizeInBytes())) + .totalDataManifests(getCounterValue(metrics.totalDataManifests())) + .totalDeleteManifests(getCounterValue(metrics.totalDeleteManifests())) + .scannedDataManifests(getCounterValue(metrics.scannedDataManifests())) + .scannedDeleteManifests(getCounterValue(metrics.scannedDeleteManifests())) + .skippedDataManifests(getCounterValue(metrics.skippedDataManifests())) + .skippedDeleteManifests(getCounterValue(metrics.skippedDeleteManifests())) + .skippedDataFiles(getCounterValue(metrics.skippedDataFiles())) + .skippedDeleteFiles(getCounterValue(metrics.skippedDeleteFiles())) + .totalPlanningDurationMs(getTimerValueMs(metrics.totalPlanningDuration())) + .equalityDeleteFiles(getCounterValue(metrics.equalityDeleteFiles())) + .positionalDeleteFiles(getCounterValue(metrics.positionalDeleteFiles())) + .indexedDeleteFiles(getCounterValue(metrics.indexedDeleteFiles())) + .totalDeleteFileSizeBytes(getCounterValue(metrics.totalDeleteFileSizeInBytes())); + } else { + builder + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L); + } + + // Store additional metadata as JSON + Map metadata = scanReport.metadata(); + if (metadata != null && !metadata.isEmpty()) { + builder.metadata(toJson(metadata)); + } + + return builder.build(); + } + + /** + * Converts an Iceberg CommitReport to a ModelCommitMetricsReport. + * + * @param commitReport the Iceberg commit report + * @param realmId the realm ID for multi-tenancy + * @param catalogId the catalog ID + * @param tableId the table entity ID + * @param principalName the principal who initiated the commit (optional) + * @param requestId the request ID (optional) + * @param otelTraceId OpenTelemetry trace ID (optional) + * @param otelSpanId OpenTelemetry span ID (optional) + * @return the converted ModelCommitMetricsReport + */ + public static ModelCommitMetricsReport fromCommitReport( + CommitReport commitReport, + String realmId, + long catalogId, + long tableId, + @Nullable String principalName, + @Nullable String requestId, + @Nullable String otelTraceId, + @Nullable String otelSpanId) { + + String reportId = UUID.randomUUID().toString(); + long timestampMs = System.currentTimeMillis(); + + CommitMetricsResult metrics = commitReport.commitMetrics(); + + ImmutableModelCommitMetricsReport.Builder builder = + ImmutableModelCommitMetricsReport.builder() + .reportId(reportId) + .realmId(realmId) + .catalogId(catalogId) + .tableId(tableId) + .timestampMs(timestampMs) + .principalName(principalName) + .requestId(requestId) + .otelTraceId(otelTraceId) + .otelSpanId(otelSpanId) + .snapshotId(commitReport.snapshotId()) + .sequenceNumber(commitReport.sequenceNumber()) + .operation(commitReport.operation() != null ? commitReport.operation() : "UNKNOWN"); + + // Extract metrics values + if (metrics != null) { + builder + .addedDataFiles(getCounterValue(metrics.addedDataFiles())) + .removedDataFiles(getCounterValue(metrics.removedDataFiles())) + .totalDataFiles(getCounterValue(metrics.totalDataFiles())) + .addedDeleteFiles(getCounterValue(metrics.addedDeleteFiles())) + .removedDeleteFiles(getCounterValue(metrics.removedDeleteFiles())) + .totalDeleteFiles(getCounterValue(metrics.totalDeleteFiles())) + .addedEqualityDeleteFiles(getCounterValue(metrics.addedEqualityDeleteFiles())) + .removedEqualityDeleteFiles(getCounterValue(metrics.removedEqualityDeleteFiles())) + .addedPositionalDeleteFiles(getCounterValue(metrics.addedPositionalDeleteFiles())) + .removedPositionalDeleteFiles(getCounterValue(metrics.removedPositionalDeleteFiles())) + .addedRecords(getCounterValue(metrics.addedRecords())) + .removedRecords(getCounterValue(metrics.removedRecords())) + .totalRecords(getCounterValue(metrics.totalRecords())) + .addedFileSizeBytes(getCounterValue(metrics.addedFilesSizeInBytes())) + .removedFileSizeBytes(getCounterValue(metrics.removedFilesSizeInBytes())) + .totalFileSizeBytes(getCounterValue(metrics.totalFilesSizeInBytes())) + .totalDurationMs(getTimerValueMs(metrics.totalDuration())) + .attempts(getCounterValueInt(metrics.attempts())); + } else { + builder + .addedDataFiles(0L) + .removedDataFiles(0L) + .totalDataFiles(0L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(0L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(0L) + .removedRecords(0L) + .totalRecords(0L) + .addedFileSizeBytes(0L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(0L) + .totalDurationMs(0L) + .attempts(1); + } + + // Store additional metadata as JSON + Map metadata = commitReport.metadata(); + if (metadata != null && !metadata.isEmpty()) { + builder.metadata(toJson(metadata)); + } + + return builder.build(); + } + + private static long getCounterValue(@Nullable CounterResult counter) { + return counter != null ? counter.value() : 0L; + } + + private static int getCounterValueInt(@Nullable CounterResult counter) { + return counter != null ? (int) counter.value() : 1; + } + + private static long getTimerValueMs(@Nullable TimerResult timer) { + return timer != null && timer.totalDuration() != null ? timer.totalDuration().toMillis() : 0L; + } + + private static String formatIntegerList(@Nullable List list) { + if (list == null || list.isEmpty()) { + return null; + } + return list.stream().map(String::valueOf).collect(Collectors.joining(",")); + } + + private static String formatStringList(@Nullable List list) { + if (list == null || list.isEmpty()) { + return null; + } + return String.join(",", list); + } + + private static String toJson(Map map) { + try { + return OBJECT_MAPPER.writeValueAsString(map); + } catch (JsonProcessingException e) { + return "{}"; + } + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsSchemaVersion.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsSchemaVersion.java new file mode 100644 index 0000000000..b6b73426ab --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/MetricsSchemaVersion.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +/** + * Model class for the metrics schema version table. + * + *

This is separate from {@link SchemaVersion} which tracks the entity schema version. The + * metrics schema can evolve independently from the entity schema. + */ +public class MetricsSchemaVersion implements Converter { + + private final Integer value; + + public MetricsSchemaVersion() { + this.value = null; + } + + private MetricsSchemaVersion(int value) { + this.value = value; + } + + public int getValue() { + if (value == null) { + throw new IllegalStateException( + "Metrics schema version should be constructed via fromResultSet"); + } + return value; + } + + @Override + public MetricsSchemaVersion fromResultSet(ResultSet rs) throws SQLException { + return new MetricsSchemaVersion(rs.getInt("version_value")); + } + + @Override + public Map toMap(DatabaseType databaseType) { + return Map.of("version_value", this.value); + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java new file mode 100644 index 0000000000..3bb6de016d --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import jakarta.annotation.Nullable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +/** Model class for commit_metrics_report table - stores commit metrics as first-class entities. */ +@PolarisImmutable +public interface ModelCommitMetricsReport extends Converter { + String TABLE_NAME = "COMMIT_METRICS_REPORT"; + + // Column names + String REPORT_ID = "report_id"; + String REALM_ID = "realm_id"; + String CATALOG_ID = "catalog_id"; + String TABLE_ID_COL = "table_id"; + String TIMESTAMP_MS = "timestamp_ms"; + String PRINCIPAL_NAME = "principal_name"; + String REQUEST_ID = "request_id"; + String OTEL_TRACE_ID = "otel_trace_id"; + String OTEL_SPAN_ID = "otel_span_id"; + String REPORT_TRACE_ID = "report_trace_id"; + String SNAPSHOT_ID = "snapshot_id"; + String SEQUENCE_NUMBER = "sequence_number"; + String OPERATION = "operation"; + String ADDED_DATA_FILES = "added_data_files"; + String REMOVED_DATA_FILES = "removed_data_files"; + String TOTAL_DATA_FILES = "total_data_files"; + String ADDED_DELETE_FILES = "added_delete_files"; + String REMOVED_DELETE_FILES = "removed_delete_files"; + String TOTAL_DELETE_FILES = "total_delete_files"; + String ADDED_EQUALITY_DELETE_FILES = "added_equality_delete_files"; + String REMOVED_EQUALITY_DELETE_FILES = "removed_equality_delete_files"; + String ADDED_POSITIONAL_DELETE_FILES = "added_positional_delete_files"; + String REMOVED_POSITIONAL_DELETE_FILES = "removed_positional_delete_files"; + String ADDED_RECORDS = "added_records"; + String REMOVED_RECORDS = "removed_records"; + String TOTAL_RECORDS = "total_records"; + String ADDED_FILE_SIZE_BYTES = "added_file_size_bytes"; + String REMOVED_FILE_SIZE_BYTES = "removed_file_size_bytes"; + String TOTAL_FILE_SIZE_BYTES = "total_file_size_bytes"; + String TOTAL_DURATION_MS = "total_duration_ms"; + String ATTEMPTS = "attempts"; + String METADATA = "metadata"; + + List ALL_COLUMNS = + List.of( + REPORT_ID, + REALM_ID, + CATALOG_ID, + TABLE_ID_COL, + TIMESTAMP_MS, + PRINCIPAL_NAME, + REQUEST_ID, + OTEL_TRACE_ID, + OTEL_SPAN_ID, + REPORT_TRACE_ID, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + OPERATION, + ADDED_DATA_FILES, + REMOVED_DATA_FILES, + TOTAL_DATA_FILES, + ADDED_DELETE_FILES, + REMOVED_DELETE_FILES, + TOTAL_DELETE_FILES, + ADDED_EQUALITY_DELETE_FILES, + REMOVED_EQUALITY_DELETE_FILES, + ADDED_POSITIONAL_DELETE_FILES, + REMOVED_POSITIONAL_DELETE_FILES, + ADDED_RECORDS, + REMOVED_RECORDS, + TOTAL_RECORDS, + ADDED_FILE_SIZE_BYTES, + REMOVED_FILE_SIZE_BYTES, + TOTAL_FILE_SIZE_BYTES, + TOTAL_DURATION_MS, + ATTEMPTS, + METADATA); + + // Getters + String getReportId(); + + String getRealmId(); + + long getCatalogId(); + + long getTableId(); + + long getTimestampMs(); + + @Nullable + String getPrincipalName(); + + @Nullable + String getRequestId(); + + @Nullable + String getOtelTraceId(); + + @Nullable + String getOtelSpanId(); + + @Nullable + String getReportTraceId(); + + long getSnapshotId(); + + @Nullable + Long getSequenceNumber(); + + String getOperation(); + + long getAddedDataFiles(); + + long getRemovedDataFiles(); + + long getTotalDataFiles(); + + long getAddedDeleteFiles(); + + long getRemovedDeleteFiles(); + + long getTotalDeleteFiles(); + + long getAddedEqualityDeleteFiles(); + + long getRemovedEqualityDeleteFiles(); + + long getAddedPositionalDeleteFiles(); + + long getRemovedPositionalDeleteFiles(); + + long getAddedRecords(); + + long getRemovedRecords(); + + long getTotalRecords(); + + long getAddedFileSizeBytes(); + + long getRemovedFileSizeBytes(); + + long getTotalFileSizeBytes(); + + long getTotalDurationMs(); + + int getAttempts(); + + @Nullable + String getMetadata(); + + @Override + default ModelCommitMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelCommitMetricsReport.builder() + .reportId(rs.getString(REPORT_ID)) + .realmId(rs.getString(REALM_ID)) + .catalogId(rs.getLong(CATALOG_ID)) + .tableId(rs.getLong(TABLE_ID_COL)) + .timestampMs(rs.getLong(TIMESTAMP_MS)) + .principalName(rs.getString(PRINCIPAL_NAME)) + .requestId(rs.getString(REQUEST_ID)) + .otelTraceId(rs.getString(OTEL_TRACE_ID)) + .otelSpanId(rs.getString(OTEL_SPAN_ID)) + .reportTraceId(rs.getString(REPORT_TRACE_ID)) + .snapshotId(rs.getLong(SNAPSHOT_ID)) + .sequenceNumber(rs.getObject(SEQUENCE_NUMBER, Long.class)) + .operation(rs.getString(OPERATION)) + .addedDataFiles(rs.getLong(ADDED_DATA_FILES)) + .removedDataFiles(rs.getLong(REMOVED_DATA_FILES)) + .totalDataFiles(rs.getLong(TOTAL_DATA_FILES)) + .addedDeleteFiles(rs.getLong(ADDED_DELETE_FILES)) + .removedDeleteFiles(rs.getLong(REMOVED_DELETE_FILES)) + .totalDeleteFiles(rs.getLong(TOTAL_DELETE_FILES)) + .addedEqualityDeleteFiles(rs.getLong(ADDED_EQUALITY_DELETE_FILES)) + .removedEqualityDeleteFiles(rs.getLong(REMOVED_EQUALITY_DELETE_FILES)) + .addedPositionalDeleteFiles(rs.getLong(ADDED_POSITIONAL_DELETE_FILES)) + .removedPositionalDeleteFiles(rs.getLong(REMOVED_POSITIONAL_DELETE_FILES)) + .addedRecords(rs.getLong(ADDED_RECORDS)) + .removedRecords(rs.getLong(REMOVED_RECORDS)) + .totalRecords(rs.getLong(TOTAL_RECORDS)) + .addedFileSizeBytes(rs.getLong(ADDED_FILE_SIZE_BYTES)) + .removedFileSizeBytes(rs.getLong(REMOVED_FILE_SIZE_BYTES)) + .totalFileSizeBytes(rs.getLong(TOTAL_FILE_SIZE_BYTES)) + .totalDurationMs(rs.getLong(TOTAL_DURATION_MS)) + .attempts(rs.getInt(ATTEMPTS)) + .metadata(rs.getString(METADATA)) + .build(); + } + + @Override + default Map toMap(DatabaseType databaseType) { + Map map = new LinkedHashMap<>(); + map.put(REPORT_ID, getReportId()); + map.put(REALM_ID, getRealmId()); + map.put(CATALOG_ID, getCatalogId()); + map.put(TABLE_ID_COL, getTableId()); + map.put(TIMESTAMP_MS, getTimestampMs()); + map.put(PRINCIPAL_NAME, getPrincipalName()); + map.put(REQUEST_ID, getRequestId()); + map.put(OTEL_TRACE_ID, getOtelTraceId()); + map.put(OTEL_SPAN_ID, getOtelSpanId()); + map.put(REPORT_TRACE_ID, getReportTraceId()); + map.put(SNAPSHOT_ID, getSnapshotId()); + map.put(SEQUENCE_NUMBER, getSequenceNumber()); + map.put(OPERATION, getOperation()); + map.put(ADDED_DATA_FILES, getAddedDataFiles()); + map.put(REMOVED_DATA_FILES, getRemovedDataFiles()); + map.put(TOTAL_DATA_FILES, getTotalDataFiles()); + map.put(ADDED_DELETE_FILES, getAddedDeleteFiles()); + map.put(REMOVED_DELETE_FILES, getRemovedDeleteFiles()); + map.put(TOTAL_DELETE_FILES, getTotalDeleteFiles()); + map.put(ADDED_EQUALITY_DELETE_FILES, getAddedEqualityDeleteFiles()); + map.put(REMOVED_EQUALITY_DELETE_FILES, getRemovedEqualityDeleteFiles()); + map.put(ADDED_POSITIONAL_DELETE_FILES, getAddedPositionalDeleteFiles()); + map.put(REMOVED_POSITIONAL_DELETE_FILES, getRemovedPositionalDeleteFiles()); + map.put(ADDED_RECORDS, getAddedRecords()); + map.put(REMOVED_RECORDS, getRemovedRecords()); + map.put(TOTAL_RECORDS, getTotalRecords()); + map.put(ADDED_FILE_SIZE_BYTES, getAddedFileSizeBytes()); + map.put(REMOVED_FILE_SIZE_BYTES, getRemovedFileSizeBytes()); + map.put(TOTAL_FILE_SIZE_BYTES, getTotalFileSizeBytes()); + map.put(TOTAL_DURATION_MS, getTotalDurationMs()); + map.put(ATTEMPTS, getAttempts()); + + if (databaseType.equals(DatabaseType.POSTGRES)) { + map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); + } else { + map.put(METADATA, getMetadata() != null ? getMetadata() : "{}"); + } + return map; + } + + /** Dummy instance to be used as a Converter when calling fromResultSet(). */ + ModelCommitMetricsReport CONVERTER = + ImmutableModelCommitMetricsReport.builder() + .reportId("") + .realmId("") + .catalogId(0L) + .tableId(0L) + .timestampMs(0L) + .snapshotId(0L) + .operation("") + .addedDataFiles(0L) + .removedDataFiles(0L) + .totalDataFiles(0L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(0L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(0L) + .removedRecords(0L) + .totalRecords(0L) + .addedFileSizeBytes(0L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(0L) + .totalDurationMs(0L) + .attempts(1) + .build(); +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportConverter.java new file mode 100644 index 0000000000..9bb8527f8d --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportConverter.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +/** + * Converter for reading ModelCommitMetricsReport from database result sets. This class is needed + * because the Immutables-generated class cannot be instantiated without required fields. + */ +public class ModelCommitMetricsReportConverter implements Converter { + + @Override + public ModelCommitMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelCommitMetricsReport.builder() + .reportId(rs.getString(ModelCommitMetricsReport.REPORT_ID)) + .realmId(rs.getString(ModelCommitMetricsReport.REALM_ID)) + .catalogId(rs.getLong(ModelCommitMetricsReport.CATALOG_ID)) + .tableId(rs.getLong(ModelCommitMetricsReport.TABLE_ID_COL)) + .timestampMs(rs.getLong(ModelCommitMetricsReport.TIMESTAMP_MS)) + .principalName(rs.getString(ModelCommitMetricsReport.PRINCIPAL_NAME)) + .requestId(rs.getString(ModelCommitMetricsReport.REQUEST_ID)) + .otelTraceId(rs.getString(ModelCommitMetricsReport.OTEL_TRACE_ID)) + .otelSpanId(rs.getString(ModelCommitMetricsReport.OTEL_SPAN_ID)) + .reportTraceId(rs.getString(ModelCommitMetricsReport.REPORT_TRACE_ID)) + .snapshotId(rs.getObject(ModelCommitMetricsReport.SNAPSHOT_ID, Long.class)) + .sequenceNumber(rs.getObject(ModelCommitMetricsReport.SEQUENCE_NUMBER, Long.class)) + .operation(rs.getString(ModelCommitMetricsReport.OPERATION)) + .addedDataFiles(rs.getLong(ModelCommitMetricsReport.ADDED_DATA_FILES)) + .removedDataFiles(rs.getLong(ModelCommitMetricsReport.REMOVED_DATA_FILES)) + .totalDataFiles(rs.getLong(ModelCommitMetricsReport.TOTAL_DATA_FILES)) + .addedDeleteFiles(rs.getLong(ModelCommitMetricsReport.ADDED_DELETE_FILES)) + .removedDeleteFiles(rs.getLong(ModelCommitMetricsReport.REMOVED_DELETE_FILES)) + .totalDeleteFiles(rs.getLong(ModelCommitMetricsReport.TOTAL_DELETE_FILES)) + .addedEqualityDeleteFiles(rs.getLong(ModelCommitMetricsReport.ADDED_EQUALITY_DELETE_FILES)) + .removedEqualityDeleteFiles( + rs.getLong(ModelCommitMetricsReport.REMOVED_EQUALITY_DELETE_FILES)) + .addedPositionalDeleteFiles( + rs.getLong(ModelCommitMetricsReport.ADDED_POSITIONAL_DELETE_FILES)) + .removedPositionalDeleteFiles( + rs.getLong(ModelCommitMetricsReport.REMOVED_POSITIONAL_DELETE_FILES)) + .addedRecords(rs.getLong(ModelCommitMetricsReport.ADDED_RECORDS)) + .removedRecords(rs.getLong(ModelCommitMetricsReport.REMOVED_RECORDS)) + .totalRecords(rs.getLong(ModelCommitMetricsReport.TOTAL_RECORDS)) + .addedFileSizeBytes(rs.getLong(ModelCommitMetricsReport.ADDED_FILE_SIZE_BYTES)) + .removedFileSizeBytes(rs.getLong(ModelCommitMetricsReport.REMOVED_FILE_SIZE_BYTES)) + .totalFileSizeBytes(rs.getLong(ModelCommitMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .totalDurationMs(rs.getObject(ModelCommitMetricsReport.TOTAL_DURATION_MS, Long.class)) + .attempts(rs.getObject(ModelCommitMetricsReport.ATTEMPTS, Integer.class)) + .metadata(rs.getString(ModelCommitMetricsReport.METADATA)) + .build(); + } + + @Override + public Map toMap(DatabaseType databaseType) { + throw new UnsupportedOperationException("Converter is read-only"); + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java new file mode 100644 index 0000000000..91dc88905c --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import jakarta.annotation.Nullable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.polaris.immutables.PolarisImmutable; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +/** Model class for scan_metrics_report table - stores scan metrics as first-class entities. */ +@PolarisImmutable +public interface ModelScanMetricsReport extends Converter { + String TABLE_NAME = "SCAN_METRICS_REPORT"; + + // Column names + String REPORT_ID = "report_id"; + String REALM_ID = "realm_id"; + String CATALOG_ID = "catalog_id"; + String TABLE_ID_COL = "table_id"; + String TIMESTAMP_MS = "timestamp_ms"; + String PRINCIPAL_NAME = "principal_name"; + String REQUEST_ID = "request_id"; + String OTEL_TRACE_ID = "otel_trace_id"; + String OTEL_SPAN_ID = "otel_span_id"; + String REPORT_TRACE_ID = "report_trace_id"; + String SNAPSHOT_ID = "snapshot_id"; + String SCHEMA_ID = "schema_id"; + String FILTER_EXPRESSION = "filter_expression"; + String PROJECTED_FIELD_IDS = "projected_field_ids"; + String PROJECTED_FIELD_NAMES = "projected_field_names"; + String RESULT_DATA_FILES = "result_data_files"; + String RESULT_DELETE_FILES = "result_delete_files"; + String TOTAL_FILE_SIZE_BYTES = "total_file_size_bytes"; + String TOTAL_DATA_MANIFESTS = "total_data_manifests"; + String TOTAL_DELETE_MANIFESTS = "total_delete_manifests"; + String SCANNED_DATA_MANIFESTS = "scanned_data_manifests"; + String SCANNED_DELETE_MANIFESTS = "scanned_delete_manifests"; + String SKIPPED_DATA_MANIFESTS = "skipped_data_manifests"; + String SKIPPED_DELETE_MANIFESTS = "skipped_delete_manifests"; + String SKIPPED_DATA_FILES = "skipped_data_files"; + String SKIPPED_DELETE_FILES = "skipped_delete_files"; + String TOTAL_PLANNING_DURATION_MS = "total_planning_duration_ms"; + String EQUALITY_DELETE_FILES = "equality_delete_files"; + String POSITIONAL_DELETE_FILES = "positional_delete_files"; + String INDEXED_DELETE_FILES = "indexed_delete_files"; + String TOTAL_DELETE_FILE_SIZE_BYTES = "total_delete_file_size_bytes"; + String METADATA = "metadata"; + + List ALL_COLUMNS = + List.of( + REPORT_ID, + REALM_ID, + CATALOG_ID, + TABLE_ID_COL, + TIMESTAMP_MS, + PRINCIPAL_NAME, + REQUEST_ID, + OTEL_TRACE_ID, + OTEL_SPAN_ID, + REPORT_TRACE_ID, + SNAPSHOT_ID, + SCHEMA_ID, + FILTER_EXPRESSION, + PROJECTED_FIELD_IDS, + PROJECTED_FIELD_NAMES, + RESULT_DATA_FILES, + RESULT_DELETE_FILES, + TOTAL_FILE_SIZE_BYTES, + TOTAL_DATA_MANIFESTS, + TOTAL_DELETE_MANIFESTS, + SCANNED_DATA_MANIFESTS, + SCANNED_DELETE_MANIFESTS, + SKIPPED_DATA_MANIFESTS, + SKIPPED_DELETE_MANIFESTS, + SKIPPED_DATA_FILES, + SKIPPED_DELETE_FILES, + TOTAL_PLANNING_DURATION_MS, + EQUALITY_DELETE_FILES, + POSITIONAL_DELETE_FILES, + INDEXED_DELETE_FILES, + TOTAL_DELETE_FILE_SIZE_BYTES, + METADATA); + + // Getters + String getReportId(); + + String getRealmId(); + + long getCatalogId(); + + long getTableId(); + + long getTimestampMs(); + + @Nullable + String getPrincipalName(); + + @Nullable + String getRequestId(); + + @Nullable + String getOtelTraceId(); + + @Nullable + String getOtelSpanId(); + + @Nullable + String getReportTraceId(); + + @Nullable + Long getSnapshotId(); + + @Nullable + Integer getSchemaId(); + + @Nullable + String getFilterExpression(); + + @Nullable + String getProjectedFieldIds(); + + @Nullable + String getProjectedFieldNames(); + + long getResultDataFiles(); + + long getResultDeleteFiles(); + + long getTotalFileSizeBytes(); + + long getTotalDataManifests(); + + long getTotalDeleteManifests(); + + long getScannedDataManifests(); + + long getScannedDeleteManifests(); + + long getSkippedDataManifests(); + + long getSkippedDeleteManifests(); + + long getSkippedDataFiles(); + + long getSkippedDeleteFiles(); + + long getTotalPlanningDurationMs(); + + long getEqualityDeleteFiles(); + + long getPositionalDeleteFiles(); + + long getIndexedDeleteFiles(); + + long getTotalDeleteFileSizeBytes(); + + @Nullable + String getMetadata(); + + @Override + default ModelScanMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelScanMetricsReport.builder() + .reportId(rs.getString(REPORT_ID)) + .realmId(rs.getString(REALM_ID)) + .catalogId(rs.getLong(CATALOG_ID)) + .tableId(rs.getLong(TABLE_ID_COL)) + .timestampMs(rs.getLong(TIMESTAMP_MS)) + .principalName(rs.getString(PRINCIPAL_NAME)) + .requestId(rs.getString(REQUEST_ID)) + .otelTraceId(rs.getString(OTEL_TRACE_ID)) + .otelSpanId(rs.getString(OTEL_SPAN_ID)) + .reportTraceId(rs.getString(REPORT_TRACE_ID)) + .snapshotId(rs.getObject(SNAPSHOT_ID, Long.class)) + .schemaId(rs.getObject(SCHEMA_ID, Integer.class)) + .filterExpression(rs.getString(FILTER_EXPRESSION)) + .projectedFieldIds(rs.getString(PROJECTED_FIELD_IDS)) + .projectedFieldNames(rs.getString(PROJECTED_FIELD_NAMES)) + .resultDataFiles(rs.getLong(RESULT_DATA_FILES)) + .resultDeleteFiles(rs.getLong(RESULT_DELETE_FILES)) + .totalFileSizeBytes(rs.getLong(TOTAL_FILE_SIZE_BYTES)) + .totalDataManifests(rs.getLong(TOTAL_DATA_MANIFESTS)) + .totalDeleteManifests(rs.getLong(TOTAL_DELETE_MANIFESTS)) + .scannedDataManifests(rs.getLong(SCANNED_DATA_MANIFESTS)) + .scannedDeleteManifests(rs.getLong(SCANNED_DELETE_MANIFESTS)) + .skippedDataManifests(rs.getLong(SKIPPED_DATA_MANIFESTS)) + .skippedDeleteManifests(rs.getLong(SKIPPED_DELETE_MANIFESTS)) + .skippedDataFiles(rs.getLong(SKIPPED_DATA_FILES)) + .skippedDeleteFiles(rs.getLong(SKIPPED_DELETE_FILES)) + .totalPlanningDurationMs(rs.getLong(TOTAL_PLANNING_DURATION_MS)) + .equalityDeleteFiles(rs.getLong(EQUALITY_DELETE_FILES)) + .positionalDeleteFiles(rs.getLong(POSITIONAL_DELETE_FILES)) + .indexedDeleteFiles(rs.getLong(INDEXED_DELETE_FILES)) + .totalDeleteFileSizeBytes(rs.getLong(TOTAL_DELETE_FILE_SIZE_BYTES)) + .metadata(rs.getString(METADATA)) + .build(); + } + + @Override + default Map toMap(DatabaseType databaseType) { + Map map = new LinkedHashMap<>(); + map.put(REPORT_ID, getReportId()); + map.put(REALM_ID, getRealmId()); + map.put(CATALOG_ID, getCatalogId()); + map.put(TABLE_ID_COL, getTableId()); + map.put(TIMESTAMP_MS, getTimestampMs()); + map.put(PRINCIPAL_NAME, getPrincipalName()); + map.put(REQUEST_ID, getRequestId()); + map.put(OTEL_TRACE_ID, getOtelTraceId()); + map.put(OTEL_SPAN_ID, getOtelSpanId()); + map.put(REPORT_TRACE_ID, getReportTraceId()); + map.put(SNAPSHOT_ID, getSnapshotId()); + map.put(SCHEMA_ID, getSchemaId()); + map.put(FILTER_EXPRESSION, getFilterExpression()); + map.put(PROJECTED_FIELD_IDS, getProjectedFieldIds()); + map.put(PROJECTED_FIELD_NAMES, getProjectedFieldNames()); + map.put(RESULT_DATA_FILES, getResultDataFiles()); + map.put(RESULT_DELETE_FILES, getResultDeleteFiles()); + map.put(TOTAL_FILE_SIZE_BYTES, getTotalFileSizeBytes()); + map.put(TOTAL_DATA_MANIFESTS, getTotalDataManifests()); + map.put(TOTAL_DELETE_MANIFESTS, getTotalDeleteManifests()); + map.put(SCANNED_DATA_MANIFESTS, getScannedDataManifests()); + map.put(SCANNED_DELETE_MANIFESTS, getScannedDeleteManifests()); + map.put(SKIPPED_DATA_MANIFESTS, getSkippedDataManifests()); + map.put(SKIPPED_DELETE_MANIFESTS, getSkippedDeleteManifests()); + map.put(SKIPPED_DATA_FILES, getSkippedDataFiles()); + map.put(SKIPPED_DELETE_FILES, getSkippedDeleteFiles()); + map.put(TOTAL_PLANNING_DURATION_MS, getTotalPlanningDurationMs()); + map.put(EQUALITY_DELETE_FILES, getEqualityDeleteFiles()); + map.put(POSITIONAL_DELETE_FILES, getPositionalDeleteFiles()); + map.put(INDEXED_DELETE_FILES, getIndexedDeleteFiles()); + map.put(TOTAL_DELETE_FILE_SIZE_BYTES, getTotalDeleteFileSizeBytes()); + + if (databaseType.equals(DatabaseType.POSTGRES)) { + map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); + } else { + map.put(METADATA, getMetadata() != null ? getMetadata() : "{}"); + } + return map; + } + + /** Dummy instance to be used as a Converter when calling fromResultSet(). */ + ModelScanMetricsReport CONVERTER = + ImmutableModelScanMetricsReport.builder() + .reportId("") + .realmId("") + .catalogId(0L) + .tableId(0L) + .timestampMs(0L) + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportConverter.java new file mode 100644 index 0000000000..ab7c8f4e60 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportConverter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; + +/** + * Converter for reading ModelScanMetricsReport from database result sets. This class is needed + * because the Immutables-generated class cannot be instantiated without required fields. + */ +public class ModelScanMetricsReportConverter implements Converter { + + @Override + public ModelScanMetricsReport fromResultSet(ResultSet rs) throws SQLException { + return ImmutableModelScanMetricsReport.builder() + .reportId(rs.getString(ModelScanMetricsReport.REPORT_ID)) + .realmId(rs.getString(ModelScanMetricsReport.REALM_ID)) + .catalogId(rs.getLong(ModelScanMetricsReport.CATALOG_ID)) + .tableId(rs.getLong(ModelScanMetricsReport.TABLE_ID_COL)) + .timestampMs(rs.getLong(ModelScanMetricsReport.TIMESTAMP_MS)) + .principalName(rs.getString(ModelScanMetricsReport.PRINCIPAL_NAME)) + .requestId(rs.getString(ModelScanMetricsReport.REQUEST_ID)) + .otelTraceId(rs.getString(ModelScanMetricsReport.OTEL_TRACE_ID)) + .otelSpanId(rs.getString(ModelScanMetricsReport.OTEL_SPAN_ID)) + .reportTraceId(rs.getString(ModelScanMetricsReport.REPORT_TRACE_ID)) + .snapshotId(rs.getObject(ModelScanMetricsReport.SNAPSHOT_ID, Long.class)) + .schemaId(rs.getObject(ModelScanMetricsReport.SCHEMA_ID, Integer.class)) + .filterExpression(rs.getString(ModelScanMetricsReport.FILTER_EXPRESSION)) + .projectedFieldIds(rs.getString(ModelScanMetricsReport.PROJECTED_FIELD_IDS)) + .projectedFieldNames(rs.getString(ModelScanMetricsReport.PROJECTED_FIELD_NAMES)) + .resultDataFiles(rs.getLong(ModelScanMetricsReport.RESULT_DATA_FILES)) + .resultDeleteFiles(rs.getLong(ModelScanMetricsReport.RESULT_DELETE_FILES)) + .totalFileSizeBytes(rs.getLong(ModelScanMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .totalDataManifests(rs.getLong(ModelScanMetricsReport.TOTAL_DATA_MANIFESTS)) + .totalDeleteManifests(rs.getLong(ModelScanMetricsReport.TOTAL_DELETE_MANIFESTS)) + .scannedDataManifests(rs.getLong(ModelScanMetricsReport.SCANNED_DATA_MANIFESTS)) + .scannedDeleteManifests(rs.getLong(ModelScanMetricsReport.SCANNED_DELETE_MANIFESTS)) + .skippedDataManifests(rs.getLong(ModelScanMetricsReport.SKIPPED_DATA_MANIFESTS)) + .skippedDeleteManifests(rs.getLong(ModelScanMetricsReport.SKIPPED_DELETE_MANIFESTS)) + .skippedDataFiles(rs.getLong(ModelScanMetricsReport.SKIPPED_DATA_FILES)) + .skippedDeleteFiles(rs.getLong(ModelScanMetricsReport.SKIPPED_DELETE_FILES)) + .totalPlanningDurationMs(rs.getLong(ModelScanMetricsReport.TOTAL_PLANNING_DURATION_MS)) + .equalityDeleteFiles(rs.getLong(ModelScanMetricsReport.EQUALITY_DELETE_FILES)) + .positionalDeleteFiles(rs.getLong(ModelScanMetricsReport.POSITIONAL_DELETE_FILES)) + .indexedDeleteFiles(rs.getLong(ModelScanMetricsReport.INDEXED_DELETE_FILES)) + .totalDeleteFileSizeBytes(rs.getLong(ModelScanMetricsReport.TOTAL_DELETE_FILE_SIZE_BYTES)) + .metadata(rs.getString(ModelScanMetricsReport.METADATA)) + .build(); + } + + @Override + public Map toMap(DatabaseType databaseType) { + throw new UnsupportedOperationException("Converter is read-only"); + } +} diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql new file mode 100644 index 0000000000..8acf84c398 --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-metrics-v1.sql @@ -0,0 +1,169 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +-- ============================================================================ +-- POLARIS METRICS SCHEMA VERSION 1 (H2) +-- ============================================================================ +-- This schema is SEPARATE from the entity schema and can evolve independently. +-- It contains tables for storing Iceberg metrics reports. +-- +-- Tables: +-- * `metrics_version` - Version tracking for the metrics schema +-- * `scan_metrics_report` - Scan metrics reports +-- * `commit_metrics_report` - Commit metrics reports +-- ============================================================================ + +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET SCHEMA POLARIS_SCHEMA; + +-- Metrics schema version tracking (separate from entity schema version) +CREATE TABLE IF NOT EXISTS metrics_version ( + version_key VARCHAR PRIMARY KEY, + version_value INTEGER NOT NULL +); + +MERGE INTO metrics_version (version_key, version_value) + KEY (version_key) + VALUES ('metrics_version', 1); + +COMMENT ON TABLE metrics_version IS 'the version of the metrics schema in use'; + +-- ============================================================================ +-- SCAN METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS scan_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Scan context + snapshot_id BIGINT, + schema_id INTEGER, + filter_expression TEXT, + projected_field_ids TEXT, + projected_field_names TEXT, + + -- Scan metrics + result_data_files BIGINT DEFAULT 0, + result_delete_files BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + total_data_manifests BIGINT DEFAULT 0, + total_delete_manifests BIGINT DEFAULT 0, + scanned_data_manifests BIGINT DEFAULT 0, + scanned_delete_manifests BIGINT DEFAULT 0, + skipped_data_manifests BIGINT DEFAULT 0, + skipped_delete_manifests BIGINT DEFAULT 0, + skipped_data_files BIGINT DEFAULT 0, + skipped_delete_files BIGINT DEFAULT 0, + total_planning_duration_ms BIGINT DEFAULT 0, + + -- Equality/positional delete metrics + equality_delete_files BIGINT DEFAULT 0, + positional_delete_files BIGINT DEFAULT 0, + indexed_delete_files BIGINT DEFAULT 0, + total_delete_file_size_bytes BIGINT DEFAULT 0, + + -- Additional metadata (for extensibility) + metadata TEXT DEFAULT '{}', + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE scan_metrics_report IS 'Scan metrics reports as first-class entities'; + +-- Index for retention cleanup by timestamp +CREATE INDEX IF NOT EXISTS idx_scan_report_timestamp ON scan_metrics_report(realm_id, timestamp_ms); + +-- ============================================================================ +-- COMMIT METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS commit_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Commit context + snapshot_id BIGINT NOT NULL, + sequence_number BIGINT, + operation TEXT NOT NULL, + + -- File metrics + added_data_files BIGINT DEFAULT 0, + removed_data_files BIGINT DEFAULT 0, + total_data_files BIGINT DEFAULT 0, + added_delete_files BIGINT DEFAULT 0, + removed_delete_files BIGINT DEFAULT 0, + total_delete_files BIGINT DEFAULT 0, + + -- Equality delete files + added_equality_delete_files BIGINT DEFAULT 0, + removed_equality_delete_files BIGINT DEFAULT 0, + + -- Positional delete files + added_positional_delete_files BIGINT DEFAULT 0, + removed_positional_delete_files BIGINT DEFAULT 0, + + -- Record metrics + added_records BIGINT DEFAULT 0, + removed_records BIGINT DEFAULT 0, + total_records BIGINT DEFAULT 0, + + -- Size metrics + added_file_size_bytes BIGINT DEFAULT 0, + removed_file_size_bytes BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + + -- Duration and attempts + total_duration_ms BIGINT DEFAULT 0, + attempts INTEGER DEFAULT 1, + + -- Additional metadata (for extensibility) + metadata TEXT DEFAULT '{}', + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE commit_metrics_report IS 'Commit metrics reports as first-class entities'; + +-- Index for retention cleanup by timestamp +CREATE INDEX IF NOT EXISTS idx_commit_report_timestamp ON commit_metrics_report(realm_id, timestamp_ms); diff --git a/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql b/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql new file mode 100644 index 0000000000..4725974b79 --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/postgres/schema-metrics-v1.sql @@ -0,0 +1,180 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ============================================================================ +-- POLARIS METRICS SCHEMA VERSION 1 (PostgreSQL) +-- ============================================================================ +-- This schema is SEPARATE from the entity schema and can evolve independently. +-- It contains tables for storing Iceberg metrics reports. +-- +-- Tables: +-- * `metrics_version` - Version tracking for the metrics schema +-- * `scan_metrics_report` - Scan metrics reports +-- * `commit_metrics_report` - Commit metrics reports +-- ============================================================================ + +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET search_path TO POLARIS_SCHEMA; + +-- Metrics schema version tracking (separate from entity schema version) +CREATE TABLE IF NOT EXISTS metrics_version ( + version_key TEXT PRIMARY KEY, + version_value INTEGER NOT NULL +); + +INSERT INTO metrics_version (version_key, version_value) +VALUES ('metrics_version', 1) +ON CONFLICT (version_key) DO UPDATE +SET version_value = EXCLUDED.version_value; + +COMMENT ON TABLE metrics_version IS 'the version of the metrics schema in use'; + +-- ============================================================================ +-- SCAN METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS scan_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Scan context + snapshot_id BIGINT, + schema_id INTEGER, + filter_expression TEXT, + projected_field_ids TEXT, + projected_field_names TEXT, + + -- Scan metrics + result_data_files BIGINT DEFAULT 0, + result_delete_files BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + total_data_manifests BIGINT DEFAULT 0, + total_delete_manifests BIGINT DEFAULT 0, + scanned_data_manifests BIGINT DEFAULT 0, + scanned_delete_manifests BIGINT DEFAULT 0, + skipped_data_manifests BIGINT DEFAULT 0, + skipped_delete_manifests BIGINT DEFAULT 0, + skipped_data_files BIGINT DEFAULT 0, + skipped_delete_files BIGINT DEFAULT 0, + total_planning_duration_ms BIGINT DEFAULT 0, + + -- Equality/positional delete metrics + equality_delete_files BIGINT DEFAULT 0, + positional_delete_files BIGINT DEFAULT 0, + indexed_delete_files BIGINT DEFAULT 0, + total_delete_file_size_bytes BIGINT DEFAULT 0, + + -- Additional metadata (for extensibility) + metadata JSONB DEFAULT '{}'::JSONB, + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE scan_metrics_report IS 'Scan metrics reports as first-class entities'; +COMMENT ON COLUMN scan_metrics_report.report_id IS 'Unique identifier for the report'; +COMMENT ON COLUMN scan_metrics_report.realm_id IS 'Realm ID for multi-tenancy'; +COMMENT ON COLUMN scan_metrics_report.catalog_id IS 'Catalog ID'; +COMMENT ON COLUMN scan_metrics_report.otel_trace_id IS 'OpenTelemetry trace ID from HTTP headers'; +COMMENT ON COLUMN scan_metrics_report.report_trace_id IS 'Trace ID from report metadata'; + +-- Index for retention cleanup by timestamp +CREATE INDEX IF NOT EXISTS idx_scan_report_timestamp + ON scan_metrics_report(realm_id, timestamp_ms DESC); + +-- ============================================================================ +-- COMMIT METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS commit_metrics_report ( + report_id TEXT NOT NULL, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + -- Report metadata + timestamp_ms BIGINT NOT NULL, + principal_name TEXT, + request_id TEXT, + + -- Trace correlation + otel_trace_id TEXT, + otel_span_id TEXT, + report_trace_id TEXT, + + -- Commit context + snapshot_id BIGINT NOT NULL, + sequence_number BIGINT, + operation TEXT NOT NULL, + + -- File metrics + added_data_files BIGINT DEFAULT 0, + removed_data_files BIGINT DEFAULT 0, + total_data_files BIGINT DEFAULT 0, + added_delete_files BIGINT DEFAULT 0, + removed_delete_files BIGINT DEFAULT 0, + total_delete_files BIGINT DEFAULT 0, + + -- Equality delete files + added_equality_delete_files BIGINT DEFAULT 0, + removed_equality_delete_files BIGINT DEFAULT 0, + + -- Positional delete files + added_positional_delete_files BIGINT DEFAULT 0, + removed_positional_delete_files BIGINT DEFAULT 0, + + -- Record metrics + added_records BIGINT DEFAULT 0, + removed_records BIGINT DEFAULT 0, + total_records BIGINT DEFAULT 0, + + -- Size metrics + added_file_size_bytes BIGINT DEFAULT 0, + removed_file_size_bytes BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + + -- Duration and attempts + total_duration_ms BIGINT DEFAULT 0, + attempts INTEGER DEFAULT 1, + + -- Additional metadata (for extensibility) + metadata JSONB DEFAULT '{}'::JSONB, + + PRIMARY KEY (realm_id, report_id) +); + +COMMENT ON TABLE commit_metrics_report IS 'Commit metrics reports as first-class entities'; +COMMENT ON COLUMN commit_metrics_report.report_id IS 'Unique identifier for the report'; +COMMENT ON COLUMN commit_metrics_report.realm_id IS 'Realm ID for multi-tenancy'; +COMMENT ON COLUMN commit_metrics_report.operation IS 'Commit operation type: append, overwrite, delete, replace'; +COMMENT ON COLUMN commit_metrics_report.otel_trace_id IS 'OpenTelemetry trace ID from HTTP headers'; + +-- Index for retention cleanup by timestamp +CREATE INDEX IF NOT EXISTS idx_commit_report_timestamp + ON commit_metrics_report(realm_id, timestamp_ms DESC); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java index 6a9eb95524..f36ab68488 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtilsTest.java @@ -150,4 +150,35 @@ void whenSchemaOptionsIsNull_shouldReturnDefault() { assertEquals(-1, result); } } + + @Nested + @ExtendWith(MockitoExtension.class) + class ShouldIncludeMetricsTests { + + @Mock private BootstrapOptions mockBootstrapOptions; + @Mock private SchemaOptions mockSchemaOptions; + + @Test + void whenSchemaOptionsIsNull_shouldReturnFalse() { + when(mockBootstrapOptions.schemaOptions()).thenReturn(null); + boolean result = JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions); + assertEquals(false, result); + } + + @Test + void whenIncludeMetricsIsTrue_shouldReturnTrue() { + when(mockBootstrapOptions.schemaOptions()).thenReturn(mockSchemaOptions); + when(mockSchemaOptions.includeMetrics()).thenReturn(true); + boolean result = JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions); + assertEquals(true, result); + } + + @Test + void whenIncludeMetricsIsFalse_shouldReturnFalse() { + when(mockBootstrapOptions.schemaOptions()).thenReturn(mockSchemaOptions); + when(mockSchemaOptions.includeMetrics()).thenReturn(false); + boolean result = JdbcBootstrapUtils.shouldIncludeMetrics(mockBootstrapOptions); + assertEquals(false, result); + } + } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java new file mode 100644 index 0000000000..d78d44f1cb --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java @@ -0,0 +1,632 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.InputStream; +import java.sql.SQLException; +import java.util.Optional; +import java.util.UUID; +import javax.sql.DataSource; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelScanMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; +import org.h2.jdbcx.JdbcConnectionPool; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for metrics report persistence using JdbcMetricsPersistence. Tests the full + * flow of writing scan and commit metrics reports to the database. + */ +class MetricsReportPersistenceTest { + + private JdbcMetricsPersistence metricsPersistence; + private DatasourceOperations datasourceOperations; + + @BeforeEach + void setUp() throws SQLException { + DataSource dataSource = + JdbcConnectionPool.create( + "jdbc:h2:mem:test_metrics_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1", "sa", ""); + + datasourceOperations = new DatasourceOperations(dataSource, new TestJdbcConfiguration()); + + // Execute schema v4 for entity tables + ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + InputStream schemaStream = classLoader.getResourceAsStream("h2/schema-v4.sql"); + datasourceOperations.executeScript(schemaStream); + + // Execute metrics schema v1 for metrics tables + InputStream metricsSchemaStream = classLoader.getResourceAsStream("h2/schema-metrics-v1.sql"); + datasourceOperations.executeScript(metricsSchemaStream); + + metricsPersistence = new JdbcMetricsPersistence(datasourceOperations, "TEST_REALM", 4); + } + + @Test + void testWriteScanMetricsReport() { + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(67890L) + .timestampMs(System.currentTimeMillis()) + .snapshotId(12345L) + .schemaId(1) + .filterExpression("id > 100") + .resultDataFiles(10L) + .resultDeleteFiles(2L) + .totalFileSizeBytes(1024000L) + .totalDataManifests(5L) + .totalDeleteManifests(1L) + .scannedDataManifests(3L) + .scannedDeleteManifests(1L) + .skippedDataManifests(2L) + .skippedDeleteManifests(0L) + .skippedDataFiles(5L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(150L) + .equalityDeleteFiles(1L) + .positionalDeleteFiles(1L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(10240L) + .principalName("test-user") + .requestId("req-123") + .otelTraceId("trace-abc") + .otelSpanId("span-xyz") + .reportTraceId("report-trace-1") + .build(); + + // Should not throw + metricsPersistence.writeScanMetricsReport(report); + } + + @Test + void testWriteCommitMetricsReport() { + ModelCommitMetricsReport report = + ImmutableModelCommitMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(67890L) + .timestampMs(System.currentTimeMillis()) + .snapshotId(12345L) + .sequenceNumber(1L) + .operation("append") + .addedDataFiles(5L) + .removedDataFiles(0L) + .totalDataFiles(100L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(2L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(1000L) + .removedRecords(0L) + .totalRecords(50000L) + .addedFileSizeBytes(102400L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(5120000L) + .totalDurationMs(250L) + .attempts(1) + .principalName("test-user") + .requestId("req-456") + .otelTraceId("trace-def") + .otelSpanId("span-uvw") + .reportTraceId("report-trace-2") + .build(); + + // Should not throw + metricsPersistence.writeCommitMetricsReport(report); + } + + @Test + void testWriteMultipleScanReports() { + for (int i = 0; i < 10; i++) { + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(100L + i) + .timestampMs(System.currentTimeMillis()) + .resultDataFiles((long) (i * 10)) + .resultDeleteFiles(0L) + .totalFileSizeBytes((long) (i * 1024)) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs((long) (i * 10)) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + + metricsPersistence.writeScanMetricsReport(report); + } + } + + @Test + void testWriteReportWithNullOptionalFields() { + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(99999L) + .timestampMs(System.currentTimeMillis()) + // All optional fields left as null + .resultDataFiles(1L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(100L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + + // Should not throw even with null optional fields + metricsPersistence.writeScanMetricsReport(report); + } + + @Test + void testQueryScanMetricsReportsByTable() { + long baseTime = System.currentTimeMillis(); + + // Write multiple reports for the same table + for (int i = 0; i < 5; i++) { + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(88888L) + .timestampMs(baseTime + i * 1000) + .resultDataFiles((long) i) + .resultDeleteFiles(0L) + .totalFileSizeBytes(100L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + metricsPersistence.writeScanMetricsReport(report); + } + + // Query all reports for the table + var results = metricsPersistence.queryScanMetricsReports(12345L, 88888L, null, null, null, 10); + assertThat(results).hasSize(5); + + // Query with time range + var rangeResults = + metricsPersistence.queryScanMetricsReports( + 12345L, 88888L, baseTime + 1000, baseTime + 4000, null, 10); + assertThat(rangeResults).hasSize(3); + + // Query with limit + var limitedResults = + metricsPersistence.queryScanMetricsReports(12345L, 88888L, null, null, null, 2); + assertThat(limitedResults).hasSize(2); + } + + @Test + void testQueryScanMetricsReportsByTraceId() { + String traceId = "test-trace-" + UUID.randomUUID(); + + // Write a report with trace ID + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(77777L) + .timestampMs(System.currentTimeMillis()) + .otelTraceId(traceId) + .resultDataFiles(1L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(100L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + metricsPersistence.writeScanMetricsReport(report); + + // Query by trace ID + var results = metricsPersistence.queryScanMetricsReportsByTraceId(traceId); + assertThat(results).hasSize(1); + assertThat(results.get(0).getOtelTraceId()).isEqualTo(traceId); + } + + @Test + void testDeleteOldScanMetricsReports() { + // Create reports with different timestamps + long now = System.currentTimeMillis(); + long oneHourAgo = now - 3600_000; + long twoDaysAgo = now - 2 * 24 * 3600_000; + + // Create an old report (2 days ago) + ModelScanMetricsReport oldReport = + ImmutableModelScanMetricsReport.builder() + .reportId("old-report-" + UUID.randomUUID()) + .realmId("TEST_REALM") + .catalogId(11111L) + .tableId(67890L) + .timestampMs(twoDaysAgo) + .resultDataFiles(10L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(1000L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + metricsPersistence.writeScanMetricsReport(oldReport); + + // Create a recent report (1 hour ago) + ModelScanMetricsReport recentReport = + ImmutableModelScanMetricsReport.builder() + .reportId("recent-report-" + UUID.randomUUID()) + .realmId("TEST_REALM") + .catalogId(11111L) + .tableId(67890L) + .timestampMs(oneHourAgo) + .resultDataFiles(10L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(1000L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + metricsPersistence.writeScanMetricsReport(recentReport); + + // Delete reports older than 1 day + long oneDayAgo = now - 24 * 3600_000; + int deleted = metricsPersistence.deleteScanMetricsReportsOlderThan(oneDayAgo); + + // Should have deleted the old report + assertThat(deleted).isEqualTo(1); + + // Query to verify only recent report remains + var results = metricsPersistence.queryScanMetricsReports(11111L, 67890L, null, null, null, 10); + assertThat(results).hasSize(1); + assertThat(results.get(0).getReportId()).isEqualTo(recentReport.getReportId()); + } + + @Test + void testDeleteOldCommitMetricsReports() { + // Create reports with different timestamps + long now = System.currentTimeMillis(); + long oneHourAgo = now - 3600_000; + long twoDaysAgo = now - 2 * 24 * 3600_000; + + // Create an old report (2 days ago) + ModelCommitMetricsReport oldReport = + ImmutableModelCommitMetricsReport.builder() + .reportId("old-commit-" + UUID.randomUUID()) + .realmId("TEST_REALM") + .catalogId(11111L) + .tableId(67890L) + .timestampMs(twoDaysAgo) + .snapshotId(100L) + .sequenceNumber(1L) + .operation("append") + .addedDataFiles(5L) + .removedDataFiles(0L) + .totalDataFiles(5L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(0L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(1000L) + .removedRecords(0L) + .totalRecords(1000L) + .addedFileSizeBytes(10000L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(10000L) + .totalDurationMs(50L) + .attempts(1) + .build(); + metricsPersistence.writeCommitMetricsReport(oldReport); + + // Create a recent report (1 hour ago) + ModelCommitMetricsReport recentReport = + ImmutableModelCommitMetricsReport.builder() + .reportId("recent-commit-" + UUID.randomUUID()) + .realmId("TEST_REALM") + .catalogId(11111L) + .tableId(67890L) + .timestampMs(oneHourAgo) + .snapshotId(101L) + .sequenceNumber(2L) + .operation("append") + .addedDataFiles(3L) + .removedDataFiles(0L) + .totalDataFiles(8L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(0L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(500L) + .removedRecords(0L) + .totalRecords(1500L) + .addedFileSizeBytes(5000L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(15000L) + .totalDurationMs(30L) + .attempts(1) + .build(); + metricsPersistence.writeCommitMetricsReport(recentReport); + + // Delete reports older than 1 day + long oneDayAgo = now - 24 * 3600_000; + int deleted = metricsPersistence.deleteCommitMetricsReportsOlderThan(oneDayAgo); + + // Should have deleted the old report + assertThat(deleted).isEqualTo(1); + + // Query to verify only recent report remains + var results = + metricsPersistence.queryCommitMetricsReports(11111L, 67890L, null, null, null, 10); + assertThat(results).hasSize(1); + assertThat(results.get(0).getReportId()).isEqualTo(recentReport.getReportId()); + } + + // ==================== Schema Version < 4 Tests ==================== + // These tests verify graceful degradation when metrics tables don't exist + + @Test + void testSupportsMetricsPersistence_SchemaV4() { + assertThat(metricsPersistence.supportsMetricsPersistence()).isTrue(); + } + + @Test + void testSupportsMetricsPersistence_SchemaV3() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + assertThat(v3Persistence.supportsMetricsPersistence()).isFalse(); + } + + @Test + void testSupportsMetricsPersistence_SchemaV1() { + JdbcMetricsPersistence v1Persistence = createMetricsPersistenceWithSchemaVersion(1); + assertThat(v1Persistence.supportsMetricsPersistence()).isFalse(); + } + + @Test + void testWriteScanMetricsReport_OlderSchema_IsNoOp() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + ModelScanMetricsReport report = + ImmutableModelScanMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(67890L) + .timestampMs(System.currentTimeMillis()) + .resultDataFiles(1L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(100L) + .totalDataManifests(1L) + .totalDeleteManifests(0L) + .scannedDataManifests(1L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(10L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + + // Should not throw - silently ignored on older schemas + v3Persistence.writeScanMetricsReport(report); + } + + @Test + void testWriteCommitMetricsReport_OlderSchema_IsNoOp() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + ModelCommitMetricsReport report = + ImmutableModelCommitMetricsReport.builder() + .reportId(UUID.randomUUID().toString()) + .realmId("TEST_REALM") + .catalogId(12345L) + .tableId(67890L) + .timestampMs(System.currentTimeMillis()) + .snapshotId(12345L) + .operation("append") + .addedDataFiles(1L) + .removedDataFiles(0L) + .totalDataFiles(1L) + .addedDeleteFiles(0L) + .removedDeleteFiles(0L) + .totalDeleteFiles(0L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(0L) + .removedPositionalDeleteFiles(0L) + .addedRecords(100L) + .removedRecords(0L) + .totalRecords(100L) + .addedFileSizeBytes(1000L) + .removedFileSizeBytes(0L) + .totalFileSizeBytes(1000L) + .totalDurationMs(50L) + .attempts(1) + .build(); + + // Should not throw - silently ignored on older schemas + v3Persistence.writeCommitMetricsReport(report); + } + + @Test + void testQueryScanMetricsReports_OlderSchema_ReturnsEmptyList() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + var results = v3Persistence.queryScanMetricsReports(12345L, 67890L, null, null, null, 10); + + assertThat(results).isEmpty(); + } + + @Test + void testQueryCommitMetricsReports_OlderSchema_ReturnsEmptyList() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + var results = v3Persistence.queryCommitMetricsReports(12345L, 67890L, null, null, null, 10); + + assertThat(results).isEmpty(); + } + + @Test + void testQueryScanMetricsReportsByTraceId_OlderSchema_ReturnsEmptyList() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + var results = v3Persistence.queryScanMetricsReportsByTraceId("trace-123"); + + assertThat(results).isEmpty(); + } + + @Test + void testQueryCommitMetricsReportsByTraceId_OlderSchema_ReturnsEmptyList() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + var results = v3Persistence.queryCommitMetricsReportsByTraceId("trace-123"); + + assertThat(results).isEmpty(); + } + + @Test + void testDeleteScanMetricsReportsOlderThan_OlderSchema_ReturnsZero() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + int deleted = v3Persistence.deleteScanMetricsReportsOlderThan(System.currentTimeMillis()); + + assertThat(deleted).isEqualTo(0); + } + + @Test + void testDeleteCommitMetricsReportsOlderThan_OlderSchema_ReturnsZero() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + int deleted = v3Persistence.deleteCommitMetricsReportsOlderThan(System.currentTimeMillis()); + + assertThat(deleted).isEqualTo(0); + } + + @Test + void testDeleteAllMetricsReportsOlderThan_OlderSchema_ReturnsZero() { + JdbcMetricsPersistence v3Persistence = createMetricsPersistenceWithSchemaVersion(3); + + int deleted = v3Persistence.deleteAllMetricsReportsOlderThan(System.currentTimeMillis()); + + assertThat(deleted).isEqualTo(0); + } + + /** + * Creates a JdbcMetricsPersistence with the specified schema version. This uses the same + * datasource but with a different reported schema version to test graceful degradation. + */ + private JdbcMetricsPersistence createMetricsPersistenceWithSchemaVersion(int schemaVersion) { + return new JdbcMetricsPersistence(datasourceOperations, "TEST_REALM", schemaVersion); + } + + private static class TestJdbcConfiguration implements RelationalJdbcConfiguration { + @Override + public Optional maxRetries() { + return Optional.of(1); + } + + @Override + public Optional maxDurationInMs() { + return Optional.of(100L); + } + + @Override + public Optional initialDelayInMs() { + return Optional.of(10L); + } + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java new file mode 100644 index 0000000000..c283209b72 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SpiModelConverterTest.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ImmutableModelScanMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport; +import org.apache.polaris.persistence.relational.jdbc.models.ModelScanMetricsReport; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link SpiModelConverter}. */ +public class SpiModelConverterTest { + + private static final String TEST_REPORT_ID = "report-123"; + private static final String TEST_REALM_ID = "realm-1"; + private static final long TEST_CATALOG_ID = 12345L; + private static final long TEST_TABLE_ID = 67890L; + private static final Instant TEST_TIMESTAMP = Instant.ofEpochMilli(1704067200000L); + private static final long TEST_TIMESTAMP_MS = 1704067200000L; + + // === Scan Metrics Test === + + @Test + void testToModelScanReport() { + ScanMetricsRecord record = createTestScanRecord(); + + ModelScanMetricsReport model = SpiModelConverter.toModelScanReport(record, TEST_REALM_ID); + + assertThat(model.getReportId()).isEqualTo(TEST_REPORT_ID); + assertThat(model.getRealmId()).isEqualTo(TEST_REALM_ID); + assertThat(model.getCatalogId()).isEqualTo(TEST_CATALOG_ID); + assertThat(model.getTableId()).isEqualTo(TEST_TABLE_ID); + assertThat(model.getTimestampMs()).isEqualTo(TEST_TIMESTAMP_MS); + assertThat(model.getSnapshotId()).isEqualTo(123456789L); + assertThat(model.getSchemaId()).isEqualTo(1); + assertThat(model.getFilterExpression()).isEqualTo("id > 100"); + assertThat(model.getProjectedFieldIds()).isEqualTo("1,2,3"); + assertThat(model.getProjectedFieldNames()).isEqualTo("id,name,value"); + assertThat(model.getResultDataFiles()).isEqualTo(10L); + assertThat(model.getResultDeleteFiles()).isEqualTo(2L); + assertThat(model.getTotalFileSizeBytes()).isEqualTo(1024000L); + assertThat(model.getMetadata()).isEqualTo("{\"custom\":\"value\"}"); + } + + @Test + void testToScanMetricsRecord() { + ModelScanMetricsReport model = createTestModelScanReport(); + + ScanMetricsRecord record = SpiModelConverter.toScanMetricsRecord(model); + + assertThat(record.reportId()).isEqualTo(TEST_REPORT_ID); + assertThat(record.catalogId()).isEqualTo(TEST_CATALOG_ID); + assertThat(record.tableId()).isEqualTo(TEST_TABLE_ID); + assertThat(record.timestamp()).isEqualTo(TEST_TIMESTAMP); + assertThat(record.snapshotId()).isEqualTo(Optional.of(123456789L)); + assertThat(record.schemaId()).isEqualTo(Optional.of(1)); + assertThat(record.filterExpression()).isEqualTo(Optional.of("id > 100")); + assertThat(record.projectedFieldIds()).containsExactly(1, 2, 3); + assertThat(record.projectedFieldNames()).containsExactly("id", "name", "value"); + assertThat(record.resultDataFiles()).isEqualTo(10L); + assertThat(record.metadata()).containsEntry("custom", "value"); + } + + @Test + void testScanRecordRoundTrip() { + ScanMetricsRecord original = createTestScanRecord(); + + ModelScanMetricsReport model = SpiModelConverter.toModelScanReport(original, TEST_REALM_ID); + ScanMetricsRecord roundTripped = SpiModelConverter.toScanMetricsRecord(model); + + assertThat(roundTripped.reportId()).isEqualTo(original.reportId()); + assertThat(roundTripped.catalogId()).isEqualTo(original.catalogId()); + assertThat(roundTripped.tableId()).isEqualTo(original.tableId()); + assertThat(roundTripped.timestamp()).isEqualTo(original.timestamp()); + assertThat(roundTripped.resultDataFiles()).isEqualTo(original.resultDataFiles()); + } + + // === Commit Metrics Test === + + @Test + void testToModelCommitReport() { + CommitMetricsRecord record = createTestCommitRecord(); + + ModelCommitMetricsReport model = SpiModelConverter.toModelCommitReport(record, TEST_REALM_ID); + + assertThat(model.getReportId()).isEqualTo(TEST_REPORT_ID); + assertThat(model.getRealmId()).isEqualTo(TEST_REALM_ID); + assertThat(model.getCatalogId()).isEqualTo(TEST_CATALOG_ID); + assertThat(model.getTableId()).isEqualTo(TEST_TABLE_ID); + assertThat(model.getTimestampMs()).isEqualTo(TEST_TIMESTAMP_MS); + assertThat(model.getSnapshotId()).isEqualTo(987654321L); + assertThat(model.getSequenceNumber()).isEqualTo(5L); + assertThat(model.getOperation()).isEqualTo("append"); + assertThat(model.getAddedDataFiles()).isEqualTo(10L); + assertThat(model.getRemovedDataFiles()).isEqualTo(2L); + assertThat(model.getTotalDataFiles()).isEqualTo(100L); + assertThat(model.getAttempts()).isEqualTo(1); + } + + @Test + void testToCommitMetricsRecord() { + ModelCommitMetricsReport model = createTestModelCommitReport(); + + CommitMetricsRecord record = SpiModelConverter.toCommitMetricsRecord(model); + + assertThat(record.reportId()).isEqualTo(TEST_REPORT_ID); + assertThat(record.catalogId()).isEqualTo(TEST_CATALOG_ID); + assertThat(record.tableId()).isEqualTo(TEST_TABLE_ID); + assertThat(record.timestamp()).isEqualTo(TEST_TIMESTAMP); + assertThat(record.snapshotId()).isEqualTo(987654321L); + assertThat(record.sequenceNumber()).isEqualTo(Optional.of(5L)); + assertThat(record.operation()).isEqualTo("append"); + assertThat(record.addedDataFiles()).isEqualTo(10L); + assertThat(record.attempts()).isEqualTo(1); + } + + @Test + void testCommitRecordRoundTrip() { + CommitMetricsRecord original = createTestCommitRecord(); + + ModelCommitMetricsReport model = SpiModelConverter.toModelCommitReport(original, TEST_REALM_ID); + CommitMetricsRecord roundTripped = SpiModelConverter.toCommitMetricsRecord(model); + + assertThat(roundTripped.reportId()).isEqualTo(original.reportId()); + assertThat(roundTripped.catalogId()).isEqualTo(original.catalogId()); + assertThat(roundTripped.tableId()).isEqualTo(original.tableId()); + assertThat(roundTripped.timestamp()).isEqualTo(original.timestamp()); + assertThat(roundTripped.snapshotId()).isEqualTo(original.snapshotId()); + assertThat(roundTripped.operation()).isEqualTo(original.operation()); + } + + // === Edge Cases === + + @Test + void testNullOptionalFields() { + ScanMetricsRecord record = + ScanMetricsRecord.builder() + .reportId(TEST_REPORT_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestamp(TEST_TIMESTAMP) + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + + ModelScanMetricsReport model = SpiModelConverter.toModelScanReport(record, TEST_REALM_ID); + assertThat(model.getSnapshotId()).isNull(); + assertThat(model.getSchemaId()).isNull(); + assertThat(model.getFilterExpression()).isNull(); + assertThat(model.getProjectedFieldIds()).isNull(); + assertThat(model.getProjectedFieldNames()).isNull(); + } + + @Test + void testEmptyMetadata() { + ScanMetricsRecord record = + ScanMetricsRecord.builder() + .reportId(TEST_REPORT_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestamp(TEST_TIMESTAMP) + .resultDataFiles(0L) + .resultDeleteFiles(0L) + .totalFileSizeBytes(0L) + .totalDataManifests(0L) + .totalDeleteManifests(0L) + .scannedDataManifests(0L) + .scannedDeleteManifests(0L) + .skippedDataManifests(0L) + .skippedDeleteManifests(0L) + .skippedDataFiles(0L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(0L) + .equalityDeleteFiles(0L) + .positionalDeleteFiles(0L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(0L) + .build(); + + ModelScanMetricsReport model = SpiModelConverter.toModelScanReport(record, TEST_REALM_ID); + assertThat(model.getMetadata()).isEqualTo("{}"); + } + + // === Helper Methods === + + private ScanMetricsRecord createTestScanRecord() { + return ScanMetricsRecord.builder() + .reportId(TEST_REPORT_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestamp(TEST_TIMESTAMP) + .snapshotId(123456789L) + .schemaId(1) + .filterExpression("id > 100") + .projectedFieldIds(List.of(1, 2, 3)) + .projectedFieldNames(List.of("id", "name", "value")) + .resultDataFiles(10L) + .resultDeleteFiles(2L) + .totalFileSizeBytes(1024000L) + .totalDataManifests(5L) + .totalDeleteManifests(1L) + .scannedDataManifests(3L) + .scannedDeleteManifests(1L) + .skippedDataManifests(2L) + .skippedDeleteManifests(0L) + .skippedDataFiles(5L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(150L) + .equalityDeleteFiles(1L) + .positionalDeleteFiles(1L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(2048L) + .metadata(Map.of("custom", "value")) + .build(); + } + + private ModelScanMetricsReport createTestModelScanReport() { + return ImmutableModelScanMetricsReport.builder() + .reportId(TEST_REPORT_ID) + .realmId(TEST_REALM_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestampMs(TEST_TIMESTAMP_MS) + .snapshotId(123456789L) + .schemaId(1) + .filterExpression("id > 100") + .projectedFieldIds("1,2,3") + .projectedFieldNames("id,name,value") + .resultDataFiles(10L) + .resultDeleteFiles(2L) + .totalFileSizeBytes(1024000L) + .totalDataManifests(5L) + .totalDeleteManifests(1L) + .scannedDataManifests(3L) + .scannedDeleteManifests(1L) + .skippedDataManifests(2L) + .skippedDeleteManifests(0L) + .skippedDataFiles(5L) + .skippedDeleteFiles(0L) + .totalPlanningDurationMs(150L) + .equalityDeleteFiles(1L) + .positionalDeleteFiles(1L) + .indexedDeleteFiles(0L) + .totalDeleteFileSizeBytes(2048L) + .metadata("{\"custom\":\"value\"}") + .build(); + } + + private CommitMetricsRecord createTestCommitRecord() { + return CommitMetricsRecord.builder() + .reportId(TEST_REPORT_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestamp(TEST_TIMESTAMP) + .snapshotId(987654321L) + .sequenceNumber(5L) + .operation("append") + .addedDataFiles(10L) + .removedDataFiles(2L) + .totalDataFiles(100L) + .addedDeleteFiles(1L) + .removedDeleteFiles(0L) + .totalDeleteFiles(5L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(1L) + .removedPositionalDeleteFiles(0L) + .addedRecords(1000L) + .removedRecords(50L) + .totalRecords(50000L) + .addedFileSizeBytes(102400L) + .removedFileSizeBytes(5120L) + .totalFileSizeBytes(5120000L) + .totalDurationMs(250L) + .attempts(1) + .metadata(Map.of("custom", "value")) + .build(); + } + + private ModelCommitMetricsReport createTestModelCommitReport() { + return ImmutableModelCommitMetricsReport.builder() + .reportId(TEST_REPORT_ID) + .realmId(TEST_REALM_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestampMs(TEST_TIMESTAMP_MS) + .snapshotId(987654321L) + .sequenceNumber(5L) + .operation("append") + .addedDataFiles(10L) + .removedDataFiles(2L) + .totalDataFiles(100L) + .addedDeleteFiles(1L) + .removedDeleteFiles(0L) + .totalDeleteFiles(5L) + .addedEqualityDeleteFiles(0L) + .removedEqualityDeleteFiles(0L) + .addedPositionalDeleteFiles(1L) + .removedPositionalDeleteFiles(0L) + .addedRecords(1000L) + .removedRecords(50L) + .totalRecords(50000L) + .addedFileSizeBytes(102400L) + .removedFileSizeBytes(5120L) + .totalFileSizeBytes(5120000L) + .totalDurationMs(250L) + .attempts(1) + .metadata("{\"custom\":\"value\"}") + .build(); + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java new file mode 100644 index 0000000000..514e28f902 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReportTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; +import org.junit.jupiter.api.Test; +import org.postgresql.util.PGobject; + +public class ModelCommitMetricsReportTest { + + private static final String TEST_REPORT_ID = "commit-report-123"; + private static final String TEST_REALM_ID = "realm-1"; + private static final long TEST_CATALOG_ID = 12345L; + private static final long TEST_TABLE_ID = 67890L; + private static final long TEST_TIMESTAMP_MS = 1704067200000L; + private static final String TEST_PRINCIPAL = "user@example.com"; + private static final String TEST_REQUEST_ID = "req-456"; + private static final String TEST_OTEL_TRACE_ID = "trace-789"; + private static final String TEST_OTEL_SPAN_ID = "span-012"; + private static final String TEST_REPORT_TRACE_ID = "report-trace-345"; + private static final long TEST_SNAPSHOT_ID = 987654321L; + private static final Long TEST_SEQUENCE_NUMBER = 5L; + private static final String TEST_OPERATION = "append"; + private static final long TEST_ADDED_DATA_FILES = 10L; + private static final long TEST_REMOVED_DATA_FILES = 2L; + private static final long TEST_TOTAL_DATA_FILES = 50L; + private static final long TEST_ADDED_DELETE_FILES = 1L; + private static final long TEST_REMOVED_DELETE_FILES = 0L; + private static final long TEST_TOTAL_DELETE_FILES = 3L; + private static final long TEST_ADDED_EQUALITY_DELETE_FILES = 1L; + private static final long TEST_REMOVED_EQUALITY_DELETE_FILES = 0L; + private static final long TEST_ADDED_POSITIONAL_DELETE_FILES = 0L; + private static final long TEST_REMOVED_POSITIONAL_DELETE_FILES = 0L; + private static final long TEST_ADDED_RECORDS = 1000L; + private static final long TEST_REMOVED_RECORDS = 50L; + private static final long TEST_TOTAL_RECORDS = 10000L; + private static final long TEST_ADDED_FILE_SIZE = 1024000L; + private static final long TEST_REMOVED_FILE_SIZE = 51200L; + private static final long TEST_TOTAL_FILE_SIZE = 10240000L; + private static final long TEST_TOTAL_DURATION = 250L; + private static final int TEST_ATTEMPTS = 1; + private static final String TEST_METADATA = "{\"commit\":\"info\"}"; + + @Test + public void testFromResultSet() throws SQLException { + ResultSet mockResultSet = mock(ResultSet.class); + when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.TIMESTAMP_MS)) + .thenReturn(TEST_TIMESTAMP_MS); + when(mockResultSet.getString(ModelCommitMetricsReport.PRINCIPAL_NAME)) + .thenReturn(TEST_PRINCIPAL); + when(mockResultSet.getString(ModelCommitMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_TRACE_ID)) + .thenReturn(TEST_OTEL_TRACE_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_SPAN_ID)) + .thenReturn(TEST_OTEL_SPAN_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_TRACE_ID)) + .thenReturn(TEST_REPORT_TRACE_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.SNAPSHOT_ID)).thenReturn(TEST_SNAPSHOT_ID); + when(mockResultSet.getObject(ModelCommitMetricsReport.SEQUENCE_NUMBER, Long.class)) + .thenReturn(TEST_SEQUENCE_NUMBER); + when(mockResultSet.getString(ModelCommitMetricsReport.OPERATION)).thenReturn(TEST_OPERATION); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DATA_FILES)) + .thenReturn(TEST_ADDED_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DATA_FILES)) + .thenReturn(TEST_REMOVED_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DATA_FILES)) + .thenReturn(TEST_TOTAL_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DELETE_FILES)) + .thenReturn(TEST_ADDED_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DELETE_FILES)) + .thenReturn(TEST_REMOVED_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DELETE_FILES)) + .thenReturn(TEST_TOTAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_EQUALITY_DELETE_FILES)) + .thenReturn(TEST_ADDED_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_EQUALITY_DELETE_FILES)) + .thenReturn(TEST_REMOVED_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_ADDED_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_REMOVED_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_RECORDS)) + .thenReturn(TEST_ADDED_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_RECORDS)) + .thenReturn(TEST_REMOVED_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_RECORDS)) + .thenReturn(TEST_TOTAL_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_FILE_SIZE_BYTES)) + .thenReturn(TEST_ADDED_FILE_SIZE); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_FILE_SIZE_BYTES)) + .thenReturn(TEST_REMOVED_FILE_SIZE); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .thenReturn(TEST_TOTAL_FILE_SIZE); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DURATION_MS)) + .thenReturn(TEST_TOTAL_DURATION); + when(mockResultSet.getInt(ModelCommitMetricsReport.ATTEMPTS)).thenReturn(TEST_ATTEMPTS); + when(mockResultSet.getString(ModelCommitMetricsReport.METADATA)).thenReturn(TEST_METADATA); + + ModelCommitMetricsReport result = + ModelCommitMetricsReport.CONVERTER.fromResultSet(mockResultSet); + + assertEquals(TEST_REPORT_ID, result.getReportId()); + assertEquals(TEST_REALM_ID, result.getRealmId()); + assertEquals(TEST_CATALOG_ID, result.getCatalogId()); + assertEquals(TEST_TABLE_ID, result.getTableId()); + assertEquals(TEST_TIMESTAMP_MS, result.getTimestampMs()); + assertEquals(TEST_SNAPSHOT_ID, result.getSnapshotId()); + assertEquals(TEST_OPERATION, result.getOperation()); + assertEquals(TEST_ADDED_DATA_FILES, result.getAddedDataFiles()); + assertEquals(TEST_ADDED_RECORDS, result.getAddedRecords()); + assertEquals(TEST_TOTAL_DURATION, result.getTotalDurationMs()); + assertEquals(TEST_ATTEMPTS, result.getAttempts()); + assertEquals(TEST_METADATA, result.getMetadata()); + } + + @Test + public void testToMapWithH2DatabaseType() { + ModelCommitMetricsReport report = createTestReport(); + + Map resultMap = report.toMap(DatabaseType.H2); + + assertEquals(TEST_REPORT_ID, resultMap.get(ModelCommitMetricsReport.REPORT_ID)); + assertEquals(TEST_REALM_ID, resultMap.get(ModelCommitMetricsReport.REALM_ID)); + assertEquals(TEST_SNAPSHOT_ID, resultMap.get(ModelCommitMetricsReport.SNAPSHOT_ID)); + assertEquals(TEST_OPERATION, resultMap.get(ModelCommitMetricsReport.OPERATION)); + assertEquals(TEST_ADDED_DATA_FILES, resultMap.get(ModelCommitMetricsReport.ADDED_DATA_FILES)); + assertEquals(TEST_METADATA, resultMap.get(ModelCommitMetricsReport.METADATA)); + } + + @Test + public void testToMapWithPostgresType() { + ModelCommitMetricsReport report = createTestReport(); + + Map resultMap = report.toMap(DatabaseType.POSTGRES); + + assertEquals(TEST_REPORT_ID, resultMap.get(ModelCommitMetricsReport.REPORT_ID)); + PGobject pgObject = (PGobject) resultMap.get(ModelCommitMetricsReport.METADATA); + assertEquals("jsonb", pgObject.getType()); + assertEquals(TEST_METADATA, pgObject.getValue()); + } + + @Test + public void testConverterFromResultSet() throws SQLException { + // Test the separate ModelCommitMetricsReportConverter class (used in query methods) + ModelCommitMetricsReportConverter converter = new ModelCommitMetricsReportConverter(); + + ResultSet mockResultSet = mock(ResultSet.class); + when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); + when(mockResultSet.getLong(ModelCommitMetricsReport.TIMESTAMP_MS)) + .thenReturn(TEST_TIMESTAMP_MS); + when(mockResultSet.getString(ModelCommitMetricsReport.PRINCIPAL_NAME)) + .thenReturn(TEST_PRINCIPAL); + when(mockResultSet.getString(ModelCommitMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_TRACE_ID)) + .thenReturn(TEST_OTEL_TRACE_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.OTEL_SPAN_ID)) + .thenReturn(TEST_OTEL_SPAN_ID); + when(mockResultSet.getString(ModelCommitMetricsReport.REPORT_TRACE_ID)) + .thenReturn(TEST_REPORT_TRACE_ID); + when(mockResultSet.getObject(ModelCommitMetricsReport.SNAPSHOT_ID, Long.class)) + .thenReturn(TEST_SNAPSHOT_ID); + when(mockResultSet.getObject(ModelCommitMetricsReport.SEQUENCE_NUMBER, Long.class)) + .thenReturn(TEST_SEQUENCE_NUMBER); + when(mockResultSet.getString(ModelCommitMetricsReport.OPERATION)).thenReturn(TEST_OPERATION); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DATA_FILES)) + .thenReturn(TEST_ADDED_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DATA_FILES)) + .thenReturn(TEST_REMOVED_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DATA_FILES)) + .thenReturn(TEST_TOTAL_DATA_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_DELETE_FILES)) + .thenReturn(TEST_ADDED_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_DELETE_FILES)) + .thenReturn(TEST_REMOVED_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_DELETE_FILES)) + .thenReturn(TEST_TOTAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_EQUALITY_DELETE_FILES)) + .thenReturn(TEST_ADDED_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_EQUALITY_DELETE_FILES)) + .thenReturn(TEST_REMOVED_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_ADDED_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_REMOVED_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_RECORDS)) + .thenReturn(TEST_ADDED_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_RECORDS)) + .thenReturn(TEST_REMOVED_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_RECORDS)) + .thenReturn(TEST_TOTAL_RECORDS); + when(mockResultSet.getLong(ModelCommitMetricsReport.ADDED_FILE_SIZE_BYTES)) + .thenReturn(TEST_ADDED_FILE_SIZE); + when(mockResultSet.getLong(ModelCommitMetricsReport.REMOVED_FILE_SIZE_BYTES)) + .thenReturn(TEST_REMOVED_FILE_SIZE); + when(mockResultSet.getLong(ModelCommitMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .thenReturn(TEST_TOTAL_FILE_SIZE); + when(mockResultSet.getObject(ModelCommitMetricsReport.TOTAL_DURATION_MS, Long.class)) + .thenReturn(TEST_TOTAL_DURATION); + when(mockResultSet.getObject(ModelCommitMetricsReport.ATTEMPTS, Integer.class)) + .thenReturn(TEST_ATTEMPTS); + when(mockResultSet.getString(ModelCommitMetricsReport.METADATA)).thenReturn(TEST_METADATA); + + ModelCommitMetricsReport result = converter.fromResultSet(mockResultSet); + + assertEquals(TEST_REPORT_ID, result.getReportId()); + assertEquals(TEST_REALM_ID, result.getRealmId()); + assertEquals(TEST_CATALOG_ID, result.getCatalogId()); + assertEquals(TEST_METADATA, result.getMetadata()); + } + + private ModelCommitMetricsReport createTestReport() { + return ImmutableModelCommitMetricsReport.builder() + .reportId(TEST_REPORT_ID) + .realmId(TEST_REALM_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestampMs(TEST_TIMESTAMP_MS) + .principalName(TEST_PRINCIPAL) + .requestId(TEST_REQUEST_ID) + .otelTraceId(TEST_OTEL_TRACE_ID) + .snapshotId(TEST_SNAPSHOT_ID) + .sequenceNumber(TEST_SEQUENCE_NUMBER) + .operation(TEST_OPERATION) + .addedDataFiles(TEST_ADDED_DATA_FILES) + .removedDataFiles(TEST_REMOVED_DATA_FILES) + .totalDataFiles(TEST_TOTAL_DATA_FILES) + .addedDeleteFiles(TEST_ADDED_DELETE_FILES) + .removedDeleteFiles(TEST_REMOVED_DELETE_FILES) + .totalDeleteFiles(TEST_TOTAL_DELETE_FILES) + .addedEqualityDeleteFiles(TEST_ADDED_EQUALITY_DELETE_FILES) + .removedEqualityDeleteFiles(TEST_REMOVED_EQUALITY_DELETE_FILES) + .addedPositionalDeleteFiles(TEST_ADDED_POSITIONAL_DELETE_FILES) + .removedPositionalDeleteFiles(TEST_REMOVED_POSITIONAL_DELETE_FILES) + .addedRecords(TEST_ADDED_RECORDS) + .removedRecords(TEST_REMOVED_RECORDS) + .totalRecords(TEST_TOTAL_RECORDS) + .addedFileSizeBytes(TEST_ADDED_FILE_SIZE) + .removedFileSizeBytes(TEST_REMOVED_FILE_SIZE) + .totalFileSizeBytes(TEST_TOTAL_FILE_SIZE) + .totalDurationMs(TEST_TOTAL_DURATION) + .attempts(TEST_ATTEMPTS) + .metadata(TEST_METADATA) + .build(); + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java new file mode 100644 index 0000000000..fc8e43b270 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReportTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import org.apache.polaris.persistence.relational.jdbc.DatabaseType; +import org.junit.jupiter.api.Test; +import org.postgresql.util.PGobject; + +public class ModelScanMetricsReportTest { + + private static final String TEST_REPORT_ID = "report-123"; + private static final String TEST_REALM_ID = "realm-1"; + private static final long TEST_CATALOG_ID = 12345L; + private static final long TEST_TABLE_ID = 67890L; + private static final long TEST_TIMESTAMP_MS = 1704067200000L; + private static final String TEST_PRINCIPAL = "user@example.com"; + private static final String TEST_REQUEST_ID = "req-456"; + private static final String TEST_OTEL_TRACE_ID = "trace-789"; + private static final String TEST_OTEL_SPAN_ID = "span-012"; + private static final String TEST_REPORT_TRACE_ID = "report-trace-345"; + private static final Long TEST_SNAPSHOT_ID = 123456789L; + private static final Integer TEST_SCHEMA_ID = 1; + private static final String TEST_FILTER = "id > 100"; + private static final String TEST_PROJECTED_IDS = "1,2,3"; + private static final String TEST_PROJECTED_NAMES = "id,name,value"; + private static final long TEST_RESULT_DATA_FILES = 10L; + private static final long TEST_RESULT_DELETE_FILES = 2L; + private static final long TEST_TOTAL_FILE_SIZE = 1024000L; + private static final long TEST_TOTAL_DATA_MANIFESTS = 5L; + private static final long TEST_TOTAL_DELETE_MANIFESTS = 1L; + private static final long TEST_SCANNED_DATA_MANIFESTS = 3L; + private static final long TEST_SCANNED_DELETE_MANIFESTS = 1L; + private static final long TEST_SKIPPED_DATA_MANIFESTS = 2L; + private static final long TEST_SKIPPED_DELETE_MANIFESTS = 0L; + private static final long TEST_SKIPPED_DATA_FILES = 5L; + private static final long TEST_SKIPPED_DELETE_FILES = 0L; + private static final long TEST_PLANNING_DURATION = 150L; + private static final long TEST_EQUALITY_DELETE_FILES = 1L; + private static final long TEST_POSITIONAL_DELETE_FILES = 1L; + private static final long TEST_INDEXED_DELETE_FILES = 0L; + private static final long TEST_DELETE_FILE_SIZE = 2048L; + private static final String TEST_METADATA = "{\"custom\":\"value\"}"; + + @Test + public void testFromResultSet() throws SQLException { + ResultSet mockResultSet = mock(ResultSet.class); + when(mockResultSet.getString(ModelScanMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); + when(mockResultSet.getString(ModelScanMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.TIMESTAMP_MS)).thenReturn(TEST_TIMESTAMP_MS); + when(mockResultSet.getString(ModelScanMetricsReport.PRINCIPAL_NAME)).thenReturn(TEST_PRINCIPAL); + when(mockResultSet.getString(ModelScanMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); + when(mockResultSet.getString(ModelScanMetricsReport.OTEL_TRACE_ID)) + .thenReturn(TEST_OTEL_TRACE_ID); + when(mockResultSet.getString(ModelScanMetricsReport.OTEL_SPAN_ID)) + .thenReturn(TEST_OTEL_SPAN_ID); + when(mockResultSet.getString(ModelScanMetricsReport.REPORT_TRACE_ID)) + .thenReturn(TEST_REPORT_TRACE_ID); + when(mockResultSet.getObject(ModelScanMetricsReport.SNAPSHOT_ID, Long.class)) + .thenReturn(TEST_SNAPSHOT_ID); + when(mockResultSet.getObject(ModelScanMetricsReport.SCHEMA_ID, Integer.class)) + .thenReturn(TEST_SCHEMA_ID); + when(mockResultSet.getString(ModelScanMetricsReport.FILTER_EXPRESSION)).thenReturn(TEST_FILTER); + when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_IDS)) + .thenReturn(TEST_PROJECTED_IDS); + when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_NAMES)) + .thenReturn(TEST_PROJECTED_NAMES); + when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DATA_FILES)) + .thenReturn(TEST_RESULT_DATA_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DELETE_FILES)) + .thenReturn(TEST_RESULT_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .thenReturn(TEST_TOTAL_FILE_SIZE); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DATA_MANIFESTS)) + .thenReturn(TEST_TOTAL_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_MANIFESTS)) + .thenReturn(TEST_TOTAL_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DATA_MANIFESTS)) + .thenReturn(TEST_SCANNED_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DELETE_MANIFESTS)) + .thenReturn(TEST_SCANNED_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_MANIFESTS)) + .thenReturn(TEST_SKIPPED_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_MANIFESTS)) + .thenReturn(TEST_SKIPPED_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_FILES)) + .thenReturn(TEST_SKIPPED_DATA_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_FILES)) + .thenReturn(TEST_SKIPPED_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_PLANNING_DURATION_MS)) + .thenReturn(TEST_PLANNING_DURATION); + when(mockResultSet.getLong(ModelScanMetricsReport.EQUALITY_DELETE_FILES)) + .thenReturn(TEST_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.INDEXED_DELETE_FILES)) + .thenReturn(TEST_INDEXED_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_FILE_SIZE_BYTES)) + .thenReturn(TEST_DELETE_FILE_SIZE); + when(mockResultSet.getString(ModelScanMetricsReport.METADATA)).thenReturn(TEST_METADATA); + + ModelScanMetricsReport result = ModelScanMetricsReport.CONVERTER.fromResultSet(mockResultSet); + + assertEquals(TEST_REPORT_ID, result.getReportId()); + assertEquals(TEST_REALM_ID, result.getRealmId()); + assertEquals(TEST_CATALOG_ID, result.getCatalogId()); + assertEquals(TEST_TABLE_ID, result.getTableId()); + assertEquals(TEST_TIMESTAMP_MS, result.getTimestampMs()); + assertEquals(TEST_PRINCIPAL, result.getPrincipalName()); + assertEquals(TEST_REQUEST_ID, result.getRequestId()); + assertEquals(TEST_OTEL_TRACE_ID, result.getOtelTraceId()); + assertEquals(TEST_SNAPSHOT_ID, result.getSnapshotId()); + assertEquals(TEST_RESULT_DATA_FILES, result.getResultDataFiles()); + assertEquals(TEST_TOTAL_FILE_SIZE, result.getTotalFileSizeBytes()); + assertEquals(TEST_PLANNING_DURATION, result.getTotalPlanningDurationMs()); + assertEquals(TEST_METADATA, result.getMetadata()); + } + + @Test + public void testToMapWithH2DatabaseType() { + ModelScanMetricsReport report = createTestReport(); + + Map resultMap = report.toMap(DatabaseType.H2); + + assertEquals(TEST_REPORT_ID, resultMap.get(ModelScanMetricsReport.REPORT_ID)); + assertEquals(TEST_REALM_ID, resultMap.get(ModelScanMetricsReport.REALM_ID)); + assertEquals(TEST_CATALOG_ID, resultMap.get(ModelScanMetricsReport.CATALOG_ID)); + assertEquals(TEST_TABLE_ID, resultMap.get(ModelScanMetricsReport.TABLE_ID_COL)); + assertEquals(TEST_TIMESTAMP_MS, resultMap.get(ModelScanMetricsReport.TIMESTAMP_MS)); + assertEquals(TEST_RESULT_DATA_FILES, resultMap.get(ModelScanMetricsReport.RESULT_DATA_FILES)); + assertEquals(TEST_METADATA, resultMap.get(ModelScanMetricsReport.METADATA)); + } + + @Test + public void testToMapWithPostgresType() { + ModelScanMetricsReport report = createTestReport(); + + Map resultMap = report.toMap(DatabaseType.POSTGRES); + + assertEquals(TEST_REPORT_ID, resultMap.get(ModelScanMetricsReport.REPORT_ID)); + PGobject pgObject = (PGobject) resultMap.get(ModelScanMetricsReport.METADATA); + assertEquals("jsonb", pgObject.getType()); + assertEquals(TEST_METADATA, pgObject.getValue()); + } + + @Test + public void testConverterFromResultSet() throws SQLException { + // Test the separate ModelScanMetricsReportConverter class (used in query methods) + ModelScanMetricsReportConverter converter = new ModelScanMetricsReportConverter(); + + ResultSet mockResultSet = mock(ResultSet.class); + when(mockResultSet.getString(ModelScanMetricsReport.REPORT_ID)).thenReturn(TEST_REPORT_ID); + when(mockResultSet.getString(ModelScanMetricsReport.REALM_ID)).thenReturn(TEST_REALM_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.CATALOG_ID)).thenReturn(TEST_CATALOG_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.TABLE_ID_COL)).thenReturn(TEST_TABLE_ID); + when(mockResultSet.getLong(ModelScanMetricsReport.TIMESTAMP_MS)).thenReturn(TEST_TIMESTAMP_MS); + when(mockResultSet.getString(ModelScanMetricsReport.PRINCIPAL_NAME)).thenReturn(TEST_PRINCIPAL); + when(mockResultSet.getString(ModelScanMetricsReport.REQUEST_ID)).thenReturn(TEST_REQUEST_ID); + when(mockResultSet.getString(ModelScanMetricsReport.OTEL_TRACE_ID)) + .thenReturn(TEST_OTEL_TRACE_ID); + when(mockResultSet.getString(ModelScanMetricsReport.OTEL_SPAN_ID)) + .thenReturn(TEST_OTEL_SPAN_ID); + when(mockResultSet.getString(ModelScanMetricsReport.REPORT_TRACE_ID)) + .thenReturn(TEST_REPORT_TRACE_ID); + when(mockResultSet.getObject(ModelScanMetricsReport.SNAPSHOT_ID, Long.class)) + .thenReturn(TEST_SNAPSHOT_ID); + when(mockResultSet.getObject(ModelScanMetricsReport.SCHEMA_ID, Integer.class)) + .thenReturn(TEST_SCHEMA_ID); + when(mockResultSet.getString(ModelScanMetricsReport.FILTER_EXPRESSION)).thenReturn(TEST_FILTER); + when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_IDS)) + .thenReturn(TEST_PROJECTED_IDS); + when(mockResultSet.getString(ModelScanMetricsReport.PROJECTED_FIELD_NAMES)) + .thenReturn(TEST_PROJECTED_NAMES); + when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DATA_FILES)) + .thenReturn(TEST_RESULT_DATA_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.RESULT_DELETE_FILES)) + .thenReturn(TEST_RESULT_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_FILE_SIZE_BYTES)) + .thenReturn(TEST_TOTAL_FILE_SIZE); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DATA_MANIFESTS)) + .thenReturn(TEST_TOTAL_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_MANIFESTS)) + .thenReturn(TEST_TOTAL_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DATA_MANIFESTS)) + .thenReturn(TEST_SCANNED_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SCANNED_DELETE_MANIFESTS)) + .thenReturn(TEST_SCANNED_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_MANIFESTS)) + .thenReturn(TEST_SKIPPED_DATA_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_MANIFESTS)) + .thenReturn(TEST_SKIPPED_DELETE_MANIFESTS); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DATA_FILES)) + .thenReturn(TEST_SKIPPED_DATA_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.SKIPPED_DELETE_FILES)) + .thenReturn(TEST_SKIPPED_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_PLANNING_DURATION_MS)) + .thenReturn(TEST_PLANNING_DURATION); + when(mockResultSet.getLong(ModelScanMetricsReport.EQUALITY_DELETE_FILES)) + .thenReturn(TEST_EQUALITY_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.POSITIONAL_DELETE_FILES)) + .thenReturn(TEST_POSITIONAL_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.INDEXED_DELETE_FILES)) + .thenReturn(TEST_INDEXED_DELETE_FILES); + when(mockResultSet.getLong(ModelScanMetricsReport.TOTAL_DELETE_FILE_SIZE_BYTES)) + .thenReturn(TEST_DELETE_FILE_SIZE); + when(mockResultSet.getString(ModelScanMetricsReport.METADATA)).thenReturn(TEST_METADATA); + + ModelScanMetricsReport result = converter.fromResultSet(mockResultSet); + + assertEquals(TEST_REPORT_ID, result.getReportId()); + assertEquals(TEST_REALM_ID, result.getRealmId()); + assertEquals(TEST_CATALOG_ID, result.getCatalogId()); + assertEquals(TEST_METADATA, result.getMetadata()); + } + + private ModelScanMetricsReport createTestReport() { + return ImmutableModelScanMetricsReport.builder() + .reportId(TEST_REPORT_ID) + .realmId(TEST_REALM_ID) + .catalogId(TEST_CATALOG_ID) + .tableId(TEST_TABLE_ID) + .timestampMs(TEST_TIMESTAMP_MS) + .principalName(TEST_PRINCIPAL) + .requestId(TEST_REQUEST_ID) + .otelTraceId(TEST_OTEL_TRACE_ID) + .snapshotId(TEST_SNAPSHOT_ID) + .resultDataFiles(TEST_RESULT_DATA_FILES) + .resultDeleteFiles(TEST_RESULT_DELETE_FILES) + .totalFileSizeBytes(TEST_TOTAL_FILE_SIZE) + .totalDataManifests(TEST_TOTAL_DATA_MANIFESTS) + .totalDeleteManifests(TEST_TOTAL_DELETE_MANIFESTS) + .scannedDataManifests(TEST_SCANNED_DATA_MANIFESTS) + .scannedDeleteManifests(TEST_SCANNED_DELETE_MANIFESTS) + .skippedDataManifests(TEST_SKIPPED_DATA_MANIFESTS) + .skippedDeleteManifests(TEST_SKIPPED_DELETE_MANIFESTS) + .skippedDataFiles(TEST_SKIPPED_DATA_FILES) + .skippedDeleteFiles(TEST_SKIPPED_DELETE_FILES) + .totalPlanningDurationMs(TEST_PLANNING_DURATION) + .equalityDeleteFiles(TEST_EQUALITY_DELETE_FILES) + .positionalDeleteFiles(TEST_POSITIONAL_DELETE_FILES) + .indexedDeleteFiles(TEST_INDEXED_DELETE_FILES) + .totalDeleteFileSizeBytes(TEST_DELETE_FILE_SIZE) + .metadata(TEST_METADATA) + .build(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java b/polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java new file mode 100644 index 0000000000..039011c2d0 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/metrics/iceberg/MetricsRecordConverter.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.metrics.iceberg; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.apache.iceberg.metrics.CommitMetricsResult; +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.CounterResult; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.iceberg.metrics.TimerResult; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; + +/** + * Converts Iceberg metrics reports to SPI record types using a fluent builder API. + * + *

This converter extracts all relevant metrics from Iceberg's {@link ScanReport} and {@link + * CommitReport} and combines them with context information to create persistence-ready records. + * + *

Example usage: + * + * 

{@code
+ * ScanMetricsRecord record = MetricsRecordConverter.forScanReport(scanReport)
+ *     .catalogId(catalog.getId())
+ *     .tableId(tableEntity.getId())
+ *     .namespace(namespace)
+ *     .build();
+ * }
+ */ +public final class MetricsRecordConverter { + + private MetricsRecordConverter() { + // Utility class + } + + /** + * Creates a builder for converting a ScanReport to a ScanMetricsRecord. + * + * @param scanReport the Iceberg scan report + * @return builder for configuring the conversion + */ + public static ScanReportBuilder forScanReport(ScanReport scanReport) { + return new ScanReportBuilder(scanReport); + } + + /** + * Creates a builder for converting a CommitReport to a CommitMetricsRecord. + * + * @param commitReport the Iceberg commit report + * @return builder for configuring the conversion + */ + public static CommitReportBuilder forCommitReport(CommitReport commitReport) { + return new CommitReportBuilder(commitReport); + } + + /** Builder for converting ScanReport to ScanMetricsRecord. */ + public static final class ScanReportBuilder { + private final ScanReport scanReport; + private long catalogId; + private long tableId; + private List namespace = Collections.emptyList(); + private Instant timestamp; + + private ScanReportBuilder(ScanReport scanReport) { + this.scanReport = scanReport; + } + + public ScanReportBuilder catalogId(long catalogId) { + this.catalogId = catalogId; + return this; + } + + /** + * Sets the table entity ID. + * + *

This is the internal Polaris entity ID for the table. + * + * @param tableId the table entity ID + * @return this builder + */ + public ScanReportBuilder tableId(long tableId) { + this.tableId = tableId; + return this; + } + + /** + * Sets the namespace as a list of levels. + * + * @param namespace the namespace levels + * @return this builder + */ + public ScanReportBuilder namespace(List namespace) { + this.namespace = namespace != null ? namespace : Collections.emptyList(); + return this; + } + + /** + * Sets the timestamp for the metrics record. + * + *

This should be the time the metrics report was received by the server, which may differ + * from the time it was recorded by the client. + * + * @param timestamp the timestamp + * @return this builder + */ + public ScanReportBuilder timestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public ScanMetricsRecord build() { + ScanMetricsResult metrics = scanReport.scanMetrics(); + Map reportMetadata = + scanReport.metadata() != null ? scanReport.metadata() : Collections.emptyMap(); + + return ScanMetricsRecord.builder() + .reportId(UUID.randomUUID().toString()) + .catalogId(catalogId) + .namespace(namespace) + .tableId(tableId) + .timestamp(timestamp != null ? timestamp : Instant.now()) + .snapshotId(Optional.of(scanReport.snapshotId())) + .schemaId(Optional.of(scanReport.schemaId())) + .filterExpression( + scanReport.filter() != null + ? Optional.of(scanReport.filter().toString()) + : Optional.empty()) + .projectedFieldIds( + scanReport.projectedFieldIds() != null + ? scanReport.projectedFieldIds() + : Collections.emptyList()) + .projectedFieldNames( + scanReport.projectedFieldNames() != null + ? scanReport.projectedFieldNames() + : Collections.emptyList()) + .resultDataFiles(getCounterValue(metrics.resultDataFiles())) + .resultDeleteFiles(getCounterValue(metrics.resultDeleteFiles())) + .totalFileSizeBytes(getCounterValue(metrics.totalFileSizeInBytes())) + .totalDataManifests(getCounterValue(metrics.totalDataManifests())) + .totalDeleteManifests(getCounterValue(metrics.totalDeleteManifests())) + .scannedDataManifests(getCounterValue(metrics.scannedDataManifests())) + .scannedDeleteManifests(getCounterValue(metrics.scannedDeleteManifests())) + .skippedDataManifests(getCounterValue(metrics.skippedDataManifests())) + .skippedDeleteManifests(getCounterValue(metrics.skippedDeleteManifests())) + .skippedDataFiles(getCounterValue(metrics.skippedDataFiles())) + .skippedDeleteFiles(getCounterValue(metrics.skippedDeleteFiles())) + .totalPlanningDurationMs(getTimerValueMs(metrics.totalPlanningDuration())) + .equalityDeleteFiles(getCounterValue(metrics.equalityDeleteFiles())) + .positionalDeleteFiles(getCounterValue(metrics.positionalDeleteFiles())) + .indexedDeleteFiles(getCounterValue(metrics.indexedDeleteFiles())) + .totalDeleteFileSizeBytes(getCounterValue(metrics.totalDeleteFileSizeInBytes())) + .metadata(reportMetadata) + .build(); + } + } + + /** Builder for converting CommitReport to CommitMetricsRecord. */ + public static final class CommitReportBuilder { + private final CommitReport commitReport; + private long catalogId; + private long tableId; + private List namespace = Collections.emptyList(); + private Instant timestamp; + + private CommitReportBuilder(CommitReport commitReport) { + this.commitReport = commitReport; + } + + public CommitReportBuilder catalogId(long catalogId) { + this.catalogId = catalogId; + return this; + } + + /** + * Sets the table entity ID. + * + *

This is the internal Polaris entity ID for the table. + * + * @param tableId the table entity ID + * @return this builder + */ + public CommitReportBuilder tableId(long tableId) { + this.tableId = tableId; + return this; + } + + /** + * Sets the namespace as a list of levels. + * + * @param namespace the namespace levels + * @return this builder + */ + public CommitReportBuilder namespace(List namespace) { + this.namespace = namespace != null ? namespace : Collections.emptyList(); + return this; + } + + /** + * Sets the timestamp for the metrics record. + * + *

This should be the time the metrics report was received by the server, which may differ + * from the time it was recorded by the client. + * + * @param timestamp the timestamp + * @return this builder + */ + public CommitReportBuilder timestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public CommitMetricsRecord build() { + CommitMetricsResult metrics = commitReport.commitMetrics(); + Map reportMetadata = + commitReport.metadata() != null ? commitReport.metadata() : Collections.emptyMap(); + + return CommitMetricsRecord.builder() + .reportId(UUID.randomUUID().toString()) + .catalogId(catalogId) + .namespace(namespace) + .tableId(tableId) + .timestamp(timestamp != null ? timestamp : Instant.now()) + .snapshotId(commitReport.snapshotId()) + .sequenceNumber(Optional.of(commitReport.sequenceNumber())) + .operation(commitReport.operation()) + .addedDataFiles(getCounterValue(metrics.addedDataFiles())) + .removedDataFiles(getCounterValue(metrics.removedDataFiles())) + .totalDataFiles(getCounterValue(metrics.totalDataFiles())) + .addedDeleteFiles(getCounterValue(metrics.addedDeleteFiles())) + .removedDeleteFiles(getCounterValue(metrics.removedDeleteFiles())) + .totalDeleteFiles(getCounterValue(metrics.totalDeleteFiles())) + .addedEqualityDeleteFiles(getCounterValue(metrics.addedEqualityDeleteFiles())) + .removedEqualityDeleteFiles(getCounterValue(metrics.removedEqualityDeleteFiles())) + .addedPositionalDeleteFiles(getCounterValue(metrics.addedPositionalDeleteFiles())) + .removedPositionalDeleteFiles(getCounterValue(metrics.removedPositionalDeleteFiles())) + .addedRecords(getCounterValue(metrics.addedRecords())) + .removedRecords(getCounterValue(metrics.removedRecords())) + .totalRecords(getCounterValue(metrics.totalRecords())) + .addedFileSizeBytes(getCounterValue(metrics.addedFilesSizeInBytes())) + .removedFileSizeBytes(getCounterValue(metrics.removedFilesSizeInBytes())) + .totalFileSizeBytes(getCounterValue(metrics.totalFilesSizeInBytes())) + .totalDurationMs(getTimerValueMsOpt(metrics.totalDuration())) + .attempts(getCounterValueInt(metrics.attempts())) + .metadata(reportMetadata) + .build(); + } + } + + // === Helper Methods === + + private static long getCounterValue(CounterResult counter) { + if (counter == null) { + return 0L; + } + return counter.value(); + } + + private static int getCounterValueInt(CounterResult counter) { + if (counter == null) { + return 0; + } + return (int) counter.value(); + } + + private static long getTimerValueMs(TimerResult timer) { + if (timer == null || timer.totalDuration() == null) { + return 0L; + } + return timer.totalDuration().toMillis(); + } + + private static Optional getTimerValueMsOpt(TimerResult timer) { + if (timer == null || timer.totalDuration() == null) { + return Optional.empty(); + } + return Optional.of(timer.totalDuration().toMillis()); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java index 5cfc20a889..8798a66f93 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/bootstrap/SchemaOptions.java @@ -21,8 +21,15 @@ import java.util.Optional; import org.apache.polaris.immutables.PolarisImmutable; +import org.immutables.value.Value; @PolarisImmutable public interface SchemaOptions { Optional schemaVersion(); + + /** Whether to include the metrics schema during bootstrap. Defaults to false. */ + @Value.Default + default boolean includeMetrics() { + return false; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java new file mode 100644 index 0000000000..6d67408cba --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/CommitMetricsRecord.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import java.util.Optional; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Backend-agnostic representation of an Iceberg commit metrics report. + * + *

This record captures all relevant metrics from an Iceberg {@code CommitReport} along with + * contextual information such as catalog identification and table location. + * + *

Common identification fields are inherited from {@link MetricsRecordIdentity}. + * + *

Note: Realm ID is not included in this record. Multi-tenancy realm context should be obtained + * from the CDI-injected {@code RealmContext} at persistence time. + * + *

Note: This type is part of the experimental Metrics Persistence SPI and may change in + * future releases. + */ +@Beta +@PolarisImmutable +public interface CommitMetricsRecord extends MetricsRecordIdentity { + + // === Commit Context === + + /** Snapshot ID created by this commit. */ + long snapshotId(); + + /** Sequence number of the snapshot. */ + Optional sequenceNumber(); + + /** Operation type (e.g., "append", "overwrite", "delete"). */ + String operation(); + + // === File Metrics - Data Files === + + /** Number of data files added. */ + long addedDataFiles(); + + /** Number of data files removed. */ + long removedDataFiles(); + + /** Total number of data files after commit. */ + long totalDataFiles(); + + // === File Metrics - Delete Files === + + /** Number of delete files added. */ + long addedDeleteFiles(); + + /** Number of delete files removed. */ + long removedDeleteFiles(); + + /** Total number of delete files after commit. */ + long totalDeleteFiles(); + + /** Number of equality delete files added. */ + long addedEqualityDeleteFiles(); + + /** Number of equality delete files removed. */ + long removedEqualityDeleteFiles(); + + /** Number of positional delete files added. */ + long addedPositionalDeleteFiles(); + + /** Number of positional delete files removed. */ + long removedPositionalDeleteFiles(); + + // === Record Metrics === + + /** Number of records added. */ + long addedRecords(); + + /** Number of records removed. */ + long removedRecords(); + + /** Total number of records after commit. */ + long totalRecords(); + + // === Size Metrics === + + /** Size of added files in bytes. */ + long addedFileSizeBytes(); + + /** Size of removed files in bytes. */ + long removedFileSizeBytes(); + + /** Total file size in bytes after commit. */ + long totalFileSizeBytes(); + + // === Timing === + + /** Total duration of the commit in milliseconds. */ + Optional totalDurationMs(); + + /** Number of commit attempts. */ + int attempts(); + + /** + * Creates a new builder for CommitMetricsRecord. + * + * @return a new builder instance + */ + static ImmutableCommitMetricsRecord.Builder builder() { + return ImmutableCommitMetricsRecord.builder(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsPersistence.java new file mode 100644 index 0000000000..1e9865701e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsPersistence.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; + +/** + * Service Provider Interface (SPI) for persisting Iceberg metrics reports. + * + *

This interface enables different persistence backends (JDBC, NoSQL, custom) to implement + * metrics storage in a way appropriate for their storage model, while allowing service code to + * remain backend-agnostic. + * + *

Implementations should be idempotent - writing the same reportId twice should have no effect. + * Implementations that don't support metrics persistence can use {@link #NOOP} which silently + * ignores write operations and returns empty pages for queries. + * + *

Dependency Injection

+ * + *

This interface is designed to be injected via CDI (Contexts and Dependency Injection). The + * deployment module (e.g., {@code polaris-quarkus-service}) should provide a {@code @Produces} + * method that creates the appropriate implementation based on the configured persistence backend. + * + *

Example producer: + * + * 

{@code
+ * @Produces
+ * @RequestScoped
+ * MetricsPersistence metricsPersistence(RealmContext realmContext, PersistenceBackend backend) {
+ *   if (backend.supportsMetrics()) {
+ *     return backend.createMetricsPersistence(realmContext);
+ *   }
+ *   return MetricsPersistence.NOOP;
+ * }
+ * }
+ * + *

Multi-Tenancy

+ * + *

Realm context is not passed in the record objects. Implementations should obtain the realm + * from the CDI-injected {@code RealmContext} at write/query time. This keeps catalog-specific code + * from needing to manage realm concerns directly. + * + *

Pagination

+ * + *

Query methods use the standard Polaris pagination pattern with {@link PageToken} for requests + * and {@link Page} for responses. This enables: + * + *

    + *
  • Backend-specific cursor implementations (RDBMS offset, NoSQL continuation tokens, etc.) + *
  • Consistent pagination interface across all Polaris persistence APIs + *
  • Efficient cursor-based pagination that works with large result sets + *
+ * + *

The {@link ReportIdToken} provides a reference cursor implementation based on report ID + * (UUID), but backends may use other cursor strategies internally. + * + *

Note: This SPI is currently experimental and not yet implemented in all persistence + * backends. The API may change in future releases. + * + * @see PageToken + * @see Page + * @see ReportIdToken + */ +@Beta +public interface MetricsPersistence { + + /** A no-op implementation for backends that don't support metrics persistence. */ + MetricsPersistence NOOP = new NoOpMetricsPersistence(); + + // ============================================================================ + // Write Operations + // ============================================================================ + + /** + * Persists a scan metrics record. + * + *

This operation is idempotent - writing the same reportId twice has no effect. + * + * @param record the scan metrics record to persist + */ + void writeScanReport(@Nonnull ScanMetricsRecord record); + + /** + * Persists a commit metrics record. + * + *

This operation is idempotent - writing the same reportId twice has no effect. + * + * @param record the commit metrics record to persist + */ + void writeCommitReport(@Nonnull CommitMetricsRecord record); + + // ============================================================================ + // Query Operations + // ============================================================================ + + /** + * Queries scan metrics reports based on the specified criteria. + * + *

Example usage: + * + * 

{@code
+   * // First page
+   * PageToken pageToken = PageToken.fromLimit(100);
+   * Page page = persistence.queryScanReports(criteria, pageToken);
+   *
+   * // Next page (if available)
+   * String nextPageToken = page.encodedResponseToken();
+   * if (nextPageToken != null) {
+   *   pageToken = PageToken.build(nextPageToken, null, () -> true);
+   *   Page nextPage = persistence.queryScanReports(criteria, pageToken);
+   * }
+   * }
+ * + * @param criteria the query criteria (filters) + * @param pageToken pagination parameters (page size and optional cursor) + * @return page of matching scan metrics records with continuation token if more results exist + */ + @Nonnull + Page queryScanReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken); + + /** + * Queries commit metrics reports based on the specified criteria. + * + * @param criteria the query criteria (filters) + * @param pageToken pagination parameters (page size and optional cursor) + * @return page of matching commit metrics records with continuation token if more results exist + * @see #queryScanReports(MetricsQueryCriteria, PageToken) for pagination example + */ + @Nonnull + Page queryCommitReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQueryCriteria.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQueryCriteria.java new file mode 100644 index 0000000000..210fa39096 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsQueryCriteria.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Query criteria for retrieving metrics reports. + * + *

This class defines the filter parameters for metrics queries. Pagination is handled separately + * via {@link org.apache.polaris.core.persistence.pagination.PageToken}, which is passed as a + * separate parameter to query methods. This separation of concerns allows: + * + *

    + *
  • Different backends to implement pagination in their optimal way + *
  • Cursor-based pagination that works with both RDBMS and NoSQL backends + *
  • Reuse of the existing Polaris pagination infrastructure + *
+ * + *

Supported Query Patterns

+ * + * + * + * + * + *
PatternFields UsedIndex Required
By Table + TimecatalogId, tableId, startTime, endTimeYes (OSS)
By Time OnlystartTime, endTimePartial (timestamp index)
+ * + *

Additional query patterns (e.g., by trace ID) can be implemented by persistence backends using + * the {@link #metadata()} filter map. Client-provided correlation data should be stored in the + * metrics record's metadata map and can be filtered using the metadata criteria. + * + *

Note: This type is part of the experimental Metrics Persistence SPI and may change in + * future releases. + * + *

Pagination

+ * + *

Pagination is handled via the {@link org.apache.polaris.core.persistence.pagination.PageToken} + * passed to query methods. The token contains: + * + *

    + *
  • {@code pageSize()} - Maximum number of results to return + *
  • {@code value()} - Optional cursor token (e.g., {@link ReportIdToken}) for continuation + *
+ * + *

Query results are returned as {@link org.apache.polaris.core.persistence.pagination.Page} + * which includes an encoded token for fetching the next page. + * + * @see org.apache.polaris.core.persistence.pagination.PageToken + * @see org.apache.polaris.core.persistence.pagination.Page + * @see ReportIdToken + */ +@Beta +@PolarisImmutable +public interface MetricsQueryCriteria { + + // === Table Identification (optional) === + + /** + * Catalog ID to filter by. + * + *

This is the internal catalog entity ID. Callers should resolve catalog names to IDs before + * querying, as catalog names can change over time. + */ + OptionalLong catalogId(); + + /** + * Table entity ID to filter by. + * + *

This is the internal table entity ID. Callers should resolve table names to IDs before + * querying, as table names can change over time. + * + *

Note: Namespace is intentionally not included as a query filter. Since we query by table ID, + * the namespace is implicit. If users want to query by namespace, the service layer should + * resolve namespace to table IDs using the current catalog state, then query by those IDs. This + * avoids confusion with table moves over time. + */ + OptionalLong tableId(); + + // === Time Range === + + /** Start time for the query (inclusive). */ + Optional startTime(); + + /** End time for the query (exclusive). */ + Optional endTime(); + + // === Metadata Filtering === + + /** + * Metadata key-value pairs to filter by. + * + *

This enables filtering metrics by client-provided correlation data stored in the record's + * metadata map. For example, clients may include a trace ID in the metadata that can be queried + * later. + * + *

Note: Metadata filtering may require custom indexes depending on the persistence backend. + * The OSS codebase provides basic support, but performance optimizations may be needed for + * high-volume deployments. + */ + Map metadata(); + + // === Factory Methods === + + /** + * Creates a new builder for MetricsQueryCriteria. + * + * @return a new builder instance + */ + static ImmutableMetricsQueryCriteria.Builder builder() { + return ImmutableMetricsQueryCriteria.builder(); + } + + /** + * Creates a builder pre-populated with table identification info. + * + *

This allows the caller to add time ranges and other filters at the call site. This pattern + * is useful when table info is resolved in one place and time ranges are added elsewhere. + * + *

Example usage: + * + * 

{@code
+   * MetricsQueryCriteria criteria = MetricsQueryCriteria.forTable(catalogId, tableId)
+   *     .startTime(startTime)
+   *     .endTime(endTime)
+   *     .build();
+   * }
+ * + * @param catalogId the catalog entity ID + * @param tableId the table entity ID + * @return a builder pre-populated with table info, ready for adding time ranges + */ + static ImmutableMetricsQueryCriteria.Builder forTable(long catalogId, long tableId) { + return builder().catalogId(catalogId).tableId(tableId); + } + + /** + * Creates empty criteria (no filters). Useful for pagination-only queries. + * + * @return empty query criteria + */ + static MetricsQueryCriteria empty() { + return builder().build(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java new file mode 100644 index 0000000000..077df90b97 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsRecordIdentity.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +/** + * Base interface containing common identification fields shared by all metrics records. + * + *

This interface defines the common fields that identify the source of a metrics report, + * including the report ID, catalog ID, table location, timestamp, and metadata. + * + *

Both {@link ScanMetricsRecord} and {@link CommitMetricsRecord} extend this interface to + * inherit these common fields while adding their own specific metrics. + * + *

Design Decisions

+ * + *

Entity IDs only (no names): We store only catalog ID and table ID, not their names. + * Names can change over time (via rename operations), which would make querying historical metrics + * by name challenging and lead to correctness issues. Queries should resolve names to IDs using the + * current catalog state. + * + *

Namespace as List<String>: Namespaces are stored as a list of levels rather than + * a dot-separated string to avoid ambiguity when namespace segments contain dots. The persistence + * implementation handles the serialization format. + * + *

Realm ID: Realm ID is intentionally not included in this interface. Multi-tenancy realm + * context should be obtained from the CDI-injected {@code RealmContext} at persistence time. This + * keeps catalog-specific code from needing to manage realm concerns. + * + *

Note: This type is part of the experimental Metrics Persistence SPI and may change in + * future releases. + */ +@Beta +public interface MetricsRecordIdentity { + + /** + * Unique identifier for this report (UUID). + * + *

This ID is generated when the record is created and serves as the primary key for the + * metrics record in persistence storage. + */ + String reportId(); + + /** + * Internal catalog ID. + * + *

This matches the catalog entity ID in Polaris persistence, as defined by {@code + * PolarisEntityCore#getId()}. The catalog name is not stored since it can change over time; + * queries should resolve names to IDs using the current catalog state. + */ + long catalogId(); + + /** + * Namespace path as a list of levels (e.g., ["db", "schema"]). + * + *

This is the namespace portion of the table identifier. Using a list avoids ambiguity when + * namespace segments contain dots. The persistence implementation handles the serialization + * format. + */ + List namespace(); + + /** + * Internal table entity ID. + * + *

This matches the table entity ID in Polaris persistence, as defined by {@code + * PolarisEntityCore#getId()}. The table name is not stored since it can change over time; queries + * should resolve names to IDs using the current catalog state. + */ + long tableId(); + + /** + * Timestamp when the report was received. + * + *

This is the server-side timestamp when the metrics report was processed, not the client-side + * timestamp when the operation occurred. + */ + Instant timestamp(); + + /** + * Additional metadata as key-value pairs. + * + *

This map can contain additional contextual information from the original Iceberg report, + * including client-provided trace IDs or other correlation data. Persistence implementations can + * store and index specific metadata fields as needed. + */ + Map metadata(); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsSchemaBootstrap.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsSchemaBootstrap.java new file mode 100644 index 0000000000..eca6ea282a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/MetricsSchemaBootstrap.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; + +/** + * Service Provider Interface (SPI) for bootstrapping the metrics schema. + * + *

This interface enables different persistence backends (JDBC, NoSQL, custom) to implement + * metrics schema initialization in a way appropriate for their storage model. The metrics schema is + * separate from the entity schema and can be bootstrapped independently. + * + *

Implementations should be idempotent - calling {@link #bootstrap(String)} multiple times on + * the same realm should have no effect after the first successful call. + * + *

Dependency Injection

+ * + *

This interface is designed to be injected via CDI (Contexts and Dependency Injection). The + * deployment module should provide a {@code @Produces} method that creates the appropriate + * implementation based on the configured persistence backend. + * + *

Usage

+ * + *

The metrics schema can be bootstrapped: + * + *

    + *
  • During initial realm bootstrap with the {@code --include-metrics} flag + *
  • Independently via the {@code bootstrap-metrics} CLI command + *
  • Programmatically by injecting this interface and calling {@link #bootstrap(String)} + *
+ * + *

Note: This SPI is currently experimental. The API may change in future releases. + * + * @see MetricsPersistence + */ +@Beta +public interface MetricsSchemaBootstrap { + + /** + * A no-op implementation for backends that don't support metrics schema bootstrap. + * + *

This implementation always reports the schema as bootstrapped and does nothing when {@link + * #bootstrap(String)} is called. + */ + MetricsSchemaBootstrap NOOP = + new MetricsSchemaBootstrap() { + @Override + public void bootstrap(String realmId) { + // No-op: metrics schema bootstrap not supported + } + + @Override + public boolean isBootstrapped(String realmId) { + // Always report as bootstrapped to avoid errors + return true; + } + + @Override + public String toString() { + return "MetricsSchemaBootstrap.NOOP"; + } + }; + + /** + * Bootstraps the metrics schema for the specified realm. + * + *

This operation is idempotent - calling it multiple times on the same realm should have no + * effect after the first successful call. + * + *

Implementations should: + * + *

    + *
  • Create the necessary tables/collections for storing metrics data + *
  • Create any required indexes for efficient querying + *
  • Record the metrics schema version for future migrations + *
+ * + * @param realmId the realm identifier to bootstrap the metrics schema for + * @throws RuntimeException if the bootstrap operation fails + */ + void bootstrap(String realmId); + + /** + * Checks if the metrics schema has been bootstrapped for the specified realm. + * + * @param realmId the realm identifier to check + * @return {@code true} if the metrics schema is already bootstrapped, {@code false} otherwise + */ + boolean isBootstrapped(String realmId); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/NoOpMetricsPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/NoOpMetricsPersistence.java new file mode 100644 index 0000000000..b33c095dc8 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/NoOpMetricsPersistence.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import jakarta.annotation.Nonnull; +import java.util.Collections; +import org.apache.polaris.core.persistence.pagination.Page; +import org.apache.polaris.core.persistence.pagination.PageToken; + +/** + * A no-op implementation of {@link MetricsPersistence} for backends that don't support metrics + * persistence. + * + *

This implementation is used as the default when a persistence backend does not support metrics + * storage. All write operations are silently ignored, and all query operations return empty pages. + */ +final class NoOpMetricsPersistence implements MetricsPersistence { + + NoOpMetricsPersistence() {} + + @Override + public void writeScanReport(@Nonnull ScanMetricsRecord record) { + // No-op + } + + @Override + public void writeCommitReport(@Nonnull CommitMetricsRecord record) { + // No-op + } + + @Nonnull + @Override + public Page queryScanReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken) { + return Page.fromItems(Collections.emptyList()); + } + + @Nonnull + @Override + public Page queryCommitReports( + @Nonnull MetricsQueryCriteria criteria, @Nonnull PageToken pageToken) { + return Page.fromItems(Collections.emptyList()); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ReportIdToken.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ReportIdToken.java new file mode 100644 index 0000000000..f3e3846953 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ReportIdToken.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.annotations.Beta; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.persistence.pagination.Token; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Pagination {@linkplain Token token} for metrics queries, backed by the report ID (UUID). + * + *

Note: This is a reference implementation provided for convenience. It is + * not required by the {@link MetricsPersistence} SPI contract. Persistence backends are + * free to implement their own {@link Token} subclass optimized for their storage model (e.g., + * timestamp-based cursors, composite keys, continuation tokens). + * + *

Only {@link org.apache.polaris.core.persistence.pagination.PageToken} (for requests) and + * {@link org.apache.polaris.core.persistence.pagination.Page} (for responses) are required by the + * SPI contract. + * + *

This token enables cursor-based pagination for metrics queries across different storage + * backends. The report ID is used as the cursor because it is: + * + *

    + *
  • Guaranteed unique across all reports + *
  • Present in both scan and commit metrics records + *
  • Stable (doesn't change over time) + *
+ * + *

Each backend implementation can use this cursor value to implement efficient pagination in + * whatever way is optimal for that storage system: + * + *

    + *
  • RDBMS: {@code WHERE report_id > :lastReportId ORDER BY report_id} + *
  • NoSQL: Use report ID as partition/sort key cursor + *
  • Time-series: Combine with timestamp for efficient range scans + *
+ * + *

Note: This type is part of the experimental Metrics Persistence SPI and may change in + * future releases. + */ +@Beta +@PolarisImmutable +@JsonSerialize(as = ImmutableReportIdToken.class) +@JsonDeserialize(as = ImmutableReportIdToken.class) +public interface ReportIdToken extends Token { + + /** Token type identifier. Short to minimize serialized token size. */ + String ID = "r"; + + /** + * The report ID to use as the cursor. + * + *

Results should start after this report ID. This is typically the {@code reportId} of the + * last item from the previous page. + */ + @JsonProperty("r") + String reportId(); + + @Override + default String getT() { + return ID; + } + + /** + * Creates a token from a report ID. + * + * @param reportId the report ID to use as cursor + * @return the token, or null if reportId is null + */ + static @Nullable ReportIdToken fromReportId(@Nullable String reportId) { + if (reportId == null) { + return null; + } + return ImmutableReportIdToken.builder().reportId(reportId).build(); + } + + /** + * Creates a token from a metrics record. + * + * @param record the record whose report ID should be used as cursor + * @return the token, or null if record is null + */ + static @Nullable ReportIdToken fromRecord(@Nullable ScanMetricsRecord record) { + if (record == null) { + return null; + } + return fromReportId(record.reportId()); + } + + /** + * Creates a token from a commit metrics record. + * + * @param record the record whose report ID should be used as cursor + * @return the token, or null if record is null + */ + static @Nullable ReportIdToken fromRecord(@Nullable CommitMetricsRecord record) { + if (record == null) { + return null; + } + return fromReportId(record.reportId()); + } + + /** Token type registration for service loader. */ + final class ReportIdTokenType implements TokenType { + @Override + public String id() { + return ID; + } + + @Override + public Class javaType() { + return ReportIdToken.class; + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java new file mode 100644 index 0000000000..44947d8f75 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/metrics/ScanMetricsRecord.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.metrics; + +import com.google.common.annotations.Beta; +import java.util.List; +import java.util.Optional; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Backend-agnostic representation of an Iceberg scan metrics report. + * + *

This record captures all relevant metrics from an Iceberg {@code ScanReport} along with + * contextual information such as catalog identification and table location. + * + *

Common identification fields are inherited from {@link MetricsRecordIdentity}. + * + *

Note: Realm ID is not included in this record. Multi-tenancy realm context should be obtained + * from the CDI-injected {@code RealmContext} at persistence time. + * + *

Note: This type is part of the experimental Metrics Persistence SPI and may change in + * future releases. + */ +@Beta +@PolarisImmutable +public interface ScanMetricsRecord extends MetricsRecordIdentity { + + // === Scan Context === + + /** Snapshot ID that was scanned. */ + Optional snapshotId(); + + /** Schema ID used for the scan. */ + Optional schemaId(); + + /** Filter expression applied to the scan (as string). */ + Optional filterExpression(); + + /** List of projected field IDs. */ + List projectedFieldIds(); + + /** List of projected field names. */ + List projectedFieldNames(); + + // === Scan Metrics - File Counts === + + /** Number of data files in the result. */ + long resultDataFiles(); + + /** Number of delete files in the result. */ + long resultDeleteFiles(); + + /** Total size of files in bytes. */ + long totalFileSizeBytes(); + + // === Scan Metrics - Manifest Counts === + + /** Total number of data manifests. */ + long totalDataManifests(); + + /** Total number of delete manifests. */ + long totalDeleteManifests(); + + /** Number of data manifests that were scanned. */ + long scannedDataManifests(); + + /** Number of delete manifests that were scanned. */ + long scannedDeleteManifests(); + + /** Number of data manifests that were skipped. */ + long skippedDataManifests(); + + /** Number of delete manifests that were skipped. */ + long skippedDeleteManifests(); + + /** Number of data files that were skipped. */ + long skippedDataFiles(); + + /** Number of delete files that were skipped. */ + long skippedDeleteFiles(); + + // === Scan Metrics - Timing === + + /** Total planning duration in milliseconds. */ + long totalPlanningDurationMs(); + + // === Scan Metrics - Delete Files === + + /** Number of equality delete files. */ + long equalityDeleteFiles(); + + /** Number of positional delete files. */ + long positionalDeleteFiles(); + + /** Number of indexed delete files. */ + long indexedDeleteFiles(); + + /** Total size of delete files in bytes. */ + long totalDeleteFileSizeBytes(); + + /** + * Creates a new builder for ScanMetricsRecord. + * + * @return a new builder instance + */ + static ImmutableScanMetricsRecord.Builder builder() { + return ImmutableScanMetricsRecord.builder(); + } +} diff --git a/polaris-core/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType b/polaris-core/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType index 3579dd29b3..d496ebeddf 100644 --- a/polaris-core/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType +++ b/polaris-core/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.pagination.Token$TokenType @@ -18,3 +18,4 @@ # org.apache.polaris.core.persistence.pagination.EntityIdToken$EntityIdTokenType +org.apache.polaris.core.persistence.metrics.ReportIdToken$ReportIdTokenType diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java index 82d92f4e18..53c89ddd5c 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapCommand.java @@ -93,6 +93,11 @@ static class SchemaInputOptions { paramLabel = "", description = "The version of the schema to load in [1, 2, 3, LATEST].") Integer schemaVersion; + + @CommandLine.Option( + names = {"--include-metrics"}, + description = "Include metrics schema tables during bootstrap.") + boolean includeMetrics; } } @@ -136,6 +141,10 @@ public Integer call() { builder.schemaVersion(inputOptions.schemaInputOptions.schemaVersion); } + if (inputOptions.schemaInputOptions.includeMetrics) { + builder.includeMetrics(true); + } + schemaOptions = builder.build(); } else { schemaOptions = ImmutableSchemaOptions.builder().build(); diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapMetricsCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapMetricsCommand.java new file mode 100644 index 0000000000..97019b63a0 --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/BootstrapMetricsCommand.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import java.util.List; +import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap; +import picocli.CommandLine; + +/** + * CLI command to bootstrap the metrics schema independently from the entity schema. + * + *

This command allows operators to add metrics persistence support to an existing Polaris + * deployment without re-bootstrapping the entity schema. It is idempotent - running it multiple + * times on the same realm has no effect after the first successful run. + * + *

Example usage: + * + * 

{@code
+ * polaris-admin bootstrap-metrics -r my-realm
+ * polaris-admin bootstrap-metrics -r realm1 -r realm2
+ * }
+ */ +@CommandLine.Command( + name = "bootstrap-metrics", + mixinStandardHelpOptions = true, + description = "Bootstraps the metrics schema for existing realms.") +public class BootstrapMetricsCommand extends BaseCommand { + + @Inject + @Identifier("relational-jdbc") + MetricsSchemaBootstrap metricsSchemaBootstrap; + + @CommandLine.Option( + names = {"-r", "--realm"}, + paramLabel = "", + required = true, + description = "The name of a realm to bootstrap metrics for.") + List realms; + + @Override + public Integer call() { + boolean success = true; + + for (String realm : realms) { + try { + if (metricsSchemaBootstrap.isBootstrapped(realm)) { + spec.commandLine() + .getOut() + .printf("Metrics schema already bootstrapped for realm '%s'. Skipping.%n", realm); + } else { + spec.commandLine() + .getOut() + .printf("Bootstrapping metrics schema for realm '%s'...%n", realm); + metricsSchemaBootstrap.bootstrap(realm); + spec.commandLine() + .getOut() + .printf("Metrics schema successfully bootstrapped for realm '%s'.%n", realm); + } + } catch (Exception e) { + spec.commandLine() + .getErr() + .printf( + "Failed to bootstrap metrics schema for realm '%s': %s%n", realm, e.getMessage()); + e.printStackTrace(spec.commandLine().getErr()); + success = false; + } + } + + if (success) { + spec.commandLine().getOut().println("Metrics bootstrap completed successfully."); + return 0; + } else { + spec.commandLine().getErr().println("Metrics bootstrap encountered errors during operation."); + return EXIT_CODE_BOOTSTRAP_ERROR; + } + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java index 66ddaf0547..aa02797f3b 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java @@ -30,6 +30,7 @@ subcommands = { HelpCommand.class, BootstrapCommand.class, + BootstrapMetricsCommand.class, PurgeCommand.class, }) public class PolarisAdminTool extends BaseMetaStoreCommand { diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/config/AdminToolProducers.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/config/AdminToolProducers.java index 236325a588..df64236eec 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/config/AdminToolProducers.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/config/AdminToolProducers.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.config.RealmConfigImpl; +import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -80,9 +81,17 @@ public PolarisConfigurationStore configurationStore() { } @Produces - public RealmConfig dummyRealmConfig(PolarisConfigurationStore configurationStore) { - // Use a random realm ID for RealmConfig since the PolarisConfigurationStore is empty anyway + public RealmContext dummyRealmContext() { + // Use UUID to protect against accidental realm ID collisions. + // This is a dummy RealmContext for the admin tool - required by JdbcMetricsPersistenceProducer + // but not actually used since the admin tool doesn't persist metrics. String absentId = UUID.randomUUID().toString(); - return new RealmConfigImpl(configurationStore, () -> absentId); + return () -> absentId; + } + + @Produces + public RealmConfig dummyRealmConfig( + PolarisConfigurationStore configurationStore, RealmContext realmContext) { + return new RealmConfigImpl(configurationStore, realmContext); } } diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapMetricsCommandTestBase.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapMetricsCommandTestBase.java new file mode 100644 index 0000000000..7e52bf27c9 --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/BootstrapMetricsCommandTestBase.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import static org.apache.polaris.admintool.BaseCommand.EXIT_CODE_USAGE; +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +/** + * Base test class for {@link BootstrapMetricsCommand}. + * + *

Subclasses should provide the appropriate test profile for their persistence backend. + */ +@QuarkusMainTest +public abstract class BootstrapMetricsCommandTestBase { + + @Test + @Launch(value = {"bootstrap-metrics", "--help"}) + public void testBootstrapMetricsHelp(LaunchResult result) { + assertThat(result.getOutput()) + .contains("bootstrap-metrics") + .contains("Bootstraps the metrics schema for existing realms") + .contains("-r, --realm"); + } + + @Test + @Launch( + value = {"bootstrap-metrics"}, + exitCode = EXIT_CODE_USAGE) + public void testBootstrapMetricsMissingRealm(LaunchResult result) { + assertThat(result.getErrorOutput()).contains("Missing required option: '--realm='"); + } + + @Test + @Launch( + value = {"bootstrap-metrics", "-r", "realm1", "--not-real-arg"}, + exitCode = EXIT_CODE_USAGE) + public void testBootstrapMetricsInvalidArg(LaunchResult result) { + assertThat(result.getErrorOutput()) + .contains("Unknown option: '--not-real-arg'") + .contains("Usage:"); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java index 31f3a9eea0..73abd2cbdf 100644 --- a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapCommandTest.java @@ -44,4 +44,27 @@ public void testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion( // assertThat(result2.exitCode()).isEqualTo(EXIT_CODE_BOOTSTRAP_ERROR); // assertThat(result2.getOutput()).contains("Cannot bootstrap due to schema version mismatch."); } + + @Test + public void testBootstrapWithIncludeMetrics(QuarkusMainLauncher launcher) { + // Test that --include-metrics option is accepted and bootstrap completes successfully. + // The metrics tables are created during bootstrap when this flag is set. + LaunchResult result = + launcher.launch( + "bootstrap", "-r", "realm1", "-c", "realm1,root,s3cr3t", "--include-metrics"); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.getOutput()) + .contains("Realm 'realm1' successfully bootstrapped.") + .contains("Bootstrap completed successfully."); + } + + @Test + public void testBootstrapWithoutIncludeMetrics(QuarkusMainLauncher launcher) { + // Test that bootstrap works without --include-metrics (default behavior) + LaunchResult result = launcher.launch("bootstrap", "-r", "realm1", "-c", "realm1,root,s3cr3t"); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.getOutput()) + .contains("Realm 'realm1' successfully bootstrapped.") + .contains("Bootstrap completed successfully."); + } } diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapMetricsCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapMetricsCommandTest.java new file mode 100644 index 0000000000..5c5836dc13 --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/relational/jdbc/RelationalJdbcBootstrapMetricsCommandTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.TestProfile; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainLauncher; +import org.apache.polaris.admintool.BootstrapMetricsCommandTestBase; +import org.junit.jupiter.api.Test; + +/** + * JDBC-specific tests for {@link org.apache.polaris.admintool.BootstrapMetricsCommand}. + * + *

These tests verify the bootstrap-metrics command works correctly with the JDBC persistence + * backend. + * + *

Note: Tests that require state persistence across multiple {@code launcher.launch()} + * calls are not possible with the current test framework because each launch gets a fresh + * PostgreSQL database. See the TODO comment in {@link + * RelationalJdbcBootstrapCommandTest#testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion} + * for details. + */ +@TestProfile(RelationalJdbcAdminProfile.class) +public class RelationalJdbcBootstrapMetricsCommandTest extends BootstrapMetricsCommandTestBase { + + @Test + public void testBootstrapMetricsForSingleRealm(QuarkusMainLauncher launcher) { + // Bootstrap entity schema and metrics schema in one launch using --include-metrics. + // Note: Each launcher.launch() gets a fresh database, so we use --include-metrics + // to bootstrap both entity and metrics schema in a single launch. + LaunchResult bootstrapResult = + launcher.launch( + "bootstrap", + "-r", + "metrics-realm1", + "-c", + "metrics-realm1,root,s3cr3t", + "--include-metrics"); + assertThat(bootstrapResult.exitCode()).isEqualTo(0); + assertThat(bootstrapResult.getOutput()) + .contains("Realm 'metrics-realm1' successfully bootstrapped."); + } + + @Test + public void testBootstrapMetricsMultipleRealmsInSingleLaunch(QuarkusMainLauncher launcher) { + // Bootstrap entity schema and metrics schema for multiple realms in one launch + LaunchResult result = + launcher.launch( + "bootstrap", + "-r", + "metrics-realm3", + "-r", + "metrics-realm4", + "-c", + "metrics-realm3,root,s3cr3t", + "-c", + "metrics-realm4,root,s3cr3t", + "--include-metrics"); + assertThat(result.exitCode()).isEqualTo(0); + assertThat(result.getOutput()) + .contains("Realm 'metrics-realm3' successfully bootstrapped.") + .contains("Realm 'metrics-realm4' successfully bootstrapped.") + .contains("Bootstrap completed successfully."); + } + + // TODO: Enable these tests once we enable postgres container reuse across launches. + // See + // RelationalJdbcBootstrapCommandTest#testBootstrapFailsWhenAddingRealmWithDifferentSchemaVersion + // + // @Test + // public void testBootstrapMetricsIdempotent(QuarkusMainLauncher launcher) { + // // First launch: bootstrap entity schema and metrics schema + // LaunchResult result1 = launcher.launch( + // "bootstrap", "-r", "realm1", "-c", "realm1,root,s3cr3t", "--include-metrics"); + // assertThat(result1.exitCode()).isEqualTo(0); + // + // // Second launch: bootstrap-metrics should detect it's already bootstrapped + // LaunchResult result2 = launcher.launch("bootstrap-metrics", "-r", "realm1"); + // assertThat(result2.exitCode()).isEqualTo(0); + // assertThat(result2.getOutput()) + // .contains("Metrics schema already bootstrapped for realm 'realm1'. Skipping."); + // } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java index 121eb382c1..121250fb0f 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java @@ -49,6 +49,8 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactoryImpl; import org.apache.polaris.core.persistence.resolver.Resolver; @@ -74,6 +76,7 @@ import org.apache.polaris.service.credentials.PolarisCredentialManagerConfiguration; import org.apache.polaris.service.events.PolarisEventListenerConfiguration; import org.apache.polaris.service.events.listeners.PolarisEventListener; +import org.apache.polaris.service.persistence.MetricsPersistenceConfiguration; import org.apache.polaris.service.persistence.PersistenceConfiguration; import org.apache.polaris.service.ratelimiter.RateLimiter; import org.apache.polaris.service.ratelimiter.RateLimiterFilterConfiguration; @@ -224,6 +227,55 @@ public PolarisMetaStoreManager polarisMetaStoreManager( return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); } + /** + * Produces a no-op {@link MetricsPersistence} bean. + * + *

This bean is selected when {@code polaris.persistence.metrics.type} is set to {@code "noop"} + * (the default). All write operations are silently ignored, and all query operations return empty + * pages. + * + * @return the no-op MetricsPersistence singleton + */ + @Produces + @Identifier("noop") + public MetricsPersistence noopMetricsPersistence() { + return MetricsPersistence.NOOP; + } + + /** + * Produces a no-op {@link MetricsSchemaBootstrap} bean. + * + *

This bean is selected for backends that don't support metrics schema bootstrap. The {@link + * MetricsSchemaBootstrap#bootstrap(String)} method does nothing, and {@link + * MetricsSchemaBootstrap#isBootstrapped(String)} always returns {@code true}. + * + * @return the no-op MetricsSchemaBootstrap singleton + */ + @Produces + @Identifier("noop") + public MetricsSchemaBootstrap noopMetricsSchemaBootstrap() { + return MetricsSchemaBootstrap.NOOP; + } + + /** + * Produces a {@link MetricsPersistence} bean for the current request. + * + *

This method selects a MetricsPersistence implementation based on the configured metrics + * persistence type. The type is configured independently from the entity metastore via {@code + * polaris.persistence.metrics.type}. + * + * @param config the metrics persistence configuration + * @param metricsPersistenceImpls all available MetricsPersistence implementations + * @return a MetricsPersistence implementation for the current realm + */ + @Produces + @RequestScoped + public MetricsPersistence metricsPersistence( + MetricsPersistenceConfiguration config, + @Any Instance metricsPersistenceImpls) { + return metricsPersistenceImpls.select(Identifier.Literal.of(config.type())).get(); + } + @Produces @RequestScoped public StorageCredentialsVendor storageCredentialsVendor( diff --git a/runtime/service/src/main/java/org/apache/polaris/service/persistence/MetricsPersistenceConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/persistence/MetricsPersistenceConfiguration.java new file mode 100644 index 0000000000..9e990553bd --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/persistence/MetricsPersistenceConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.persistence; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +/** + * Configuration for selecting the {@link + * org.apache.polaris.core.persistence.metrics.MetricsPersistence} implementation. + * + *

This configuration allows selecting the metrics persistence backend independently from the + * entity metastore. Available types include: + * + *

    + *
  • {@code noop} (default) - No persistence, metrics are discarded + *
  • {@code relational-jdbc} - Persists metrics to the JDBC database (requires metrics schema) + *
+ */ +@ConfigMapping(prefix = "polaris.persistence.metrics") +public interface MetricsPersistenceConfiguration { + + /** + * The type of the metrics persistence to use. Must be a registered {@link + * org.apache.polaris.core.persistence.metrics.MetricsPersistence} identifier. + * + *

Defaults to {@code noop} which discards all metrics. + */ + @WithDefault("noop") + String type(); +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java b/runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java new file mode 100644 index 0000000000..11cebacc28 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/reporting/PersistingMetricsReporter.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.reporting; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.metrics.iceberg.MetricsRecordConverter; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of {@link PolarisMetricsReporter} that persists metrics to the configured {@link + * MetricsPersistence} backend. + * + *

This reporter is selected when {@code polaris.iceberg-metrics.reporting.type} is set to {@code + * "persisting"}. + * + *

The reporter looks up the catalog entity by name to obtain the catalog ID, then uses {@link + * MetricsRecordConverter} to convert Iceberg metrics reports to SPI records before persisting them. + * + * @see PolarisMetricsReporter + * @see MetricsPersistence + * @see MetricsRecordConverter + */ +@RequestScoped +@Identifier("persisting") +public class PersistingMetricsReporter implements PolarisMetricsReporter { + private static final Logger LOGGER = LoggerFactory.getLogger(PersistingMetricsReporter.class); + + private final CallContext callContext; + private final PolarisMetaStoreManager metaStoreManager; + private final MetricsPersistence metricsPersistence; + + @Inject + public PersistingMetricsReporter( + CallContext callContext, + PolarisMetaStoreManager metaStoreManager, + MetricsPersistence metricsPersistence) { + this.callContext = callContext; + this.metaStoreManager = metaStoreManager; + this.metricsPersistence = metricsPersistence; + } + + @Override + public void reportMetric( + String catalogName, + TableIdentifier table, + MetricsReport metricsReport, + Instant receivedTimestamp) { + + // Look up the catalog entity to get the catalog ID + EntityResult catalogResult = + metaStoreManager.readEntityByName( + callContext.getPolarisCallContext(), + null, // catalogPath is null for top-level entities + PolarisEntityType.CATALOG, + PolarisEntitySubType.ANY_SUBTYPE, + catalogName); + + if (!catalogResult.isSuccess()) { + LOGGER.warn( + "Failed to find catalog '{}' for metrics persistence. Metrics will not be stored.", + catalogName); + return; + } + + PolarisBaseEntity catalogEntity = catalogResult.getEntity(); + long catalogId = catalogEntity.getId(); + + // Build the full path from catalog through namespace to resolve the table. + // The path contains the catalog, then each namespace level. + // The last element in the path becomes the parent for the lookup. + List entityPath = new ArrayList<>(); + entityPath.add(PolarisEntity.toCore(catalogEntity)); + + // Resolve each namespace level + String[] namespaceLevels = table.namespace().levels(); + for (String nsLevel : namespaceLevels) { + EntityResult nsResult = + metaStoreManager.readEntityByName( + callContext.getPolarisCallContext(), + entityPath, + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.ANY_SUBTYPE, + nsLevel); + + if (!nsResult.isSuccess()) { + LOGGER.warn( + "Failed to find namespace '{}' in catalog '{}' for metrics persistence. Metrics will not be stored.", + nsLevel, + catalogName); + return; + } + entityPath.add(PolarisEntity.toCore(nsResult.getEntity())); + } + + // Now look up the table with the full namespace path + EntityResult tableResult = + metaStoreManager.readEntityByName( + callContext.getPolarisCallContext(), + entityPath, + PolarisEntityType.TABLE_LIKE, + PolarisEntitySubType.ANY_SUBTYPE, + table.name()); + + if (!tableResult.isSuccess()) { + LOGGER.warn( + "Failed to find table '{}' in catalog '{}' for metrics persistence. Metrics will not be stored.", + table, + catalogName); + return; + } + + long tableId = tableResult.getEntity().getId(); + + if (metricsReport instanceof ScanReport scanReport) { + ScanMetricsRecord record = + MetricsRecordConverter.forScanReport(scanReport) + .catalogId(catalogId) + .tableId(tableId) + .timestamp(receivedTimestamp) + .build(); + metricsPersistence.writeScanReport(record); + LOGGER.debug( + "Persisted scan metrics for {}.{} (reportId={})", catalogName, table, record.reportId()); + } else if (metricsReport instanceof CommitReport commitReport) { + CommitMetricsRecord record = + MetricsRecordConverter.forCommitReport(commitReport) + .catalogId(catalogId) + .tableId(tableId) + .timestamp(receivedTimestamp) + .build(); + metricsPersistence.writeCommitReport(record); + LOGGER.debug( + "Persisted commit metrics for {}.{} (reportId={})", + catalogName, + table, + record.reportId()); + } else { + LOGGER.warn( + "Unknown metrics report type: {}. Metrics will not be stored.", + metricsReport.getClass().getName()); + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java index f8383f1c43..bf2b5e00fc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java @@ -65,6 +65,7 @@ import org.apache.polaris.service.it.env.RestApi; import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; @@ -92,6 +93,7 @@ public Map getConfigOverrides() { .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") + .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") .put("polaris.readiness.ignore-severe-issues", "true") .build(); } @@ -117,10 +119,32 @@ public void setup( baseLocation = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve(realm + "/"); } + /** + * Reset the database state before each test to ensure test isolation. The H2 in-memory database + * with DB_CLOSE_DELAY=-1 persists state across tests, so we need to clean up catalog-related + * entities while preserving the realm and principal entities set up in @BeforeAll. + */ + @BeforeEach + public void resetDatabaseState() { + if (dataSource.isResolvable()) { + try (Connection conn = dataSource.get().getConnection(); + Statement stmt = conn.createStatement()) { + // Set the schema first + stmt.execute("SET SCHEMA POLARIS_SCHEMA"); + // Only delete events - catalogs use unique names and locations so they don't conflict + stmt.execute("DELETE FROM EVENTS"); + } catch (Exception e) { + // Ignore errors - tables may not exist yet on first run + } + } + } + @Test void testCreateCatalogAndTable() throws IOException { String catalogName = client.newEntityName("testCreateCatalogAndTable"); + // Use a unique base location for this catalog to avoid overlap with other catalogs + URI catalogBaseLocation = baseLocation.resolve(catalogName + "/"); Catalog catalog = PolarisCatalog.builder() @@ -130,7 +154,7 @@ void testCreateCatalogAndTable() throws IOException { .setStorageConfigInfo( FileStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of(baseLocation.toString())) + .setAllowedLocations(List.of(catalogBaseLocation.toString())) .build()) .build(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java b/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java new file mode 100644 index 0000000000..e33500c4bd --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/reporting/PersistingMetricsReporterTest.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.reporting; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.metrics.CommitMetrics; +import org.apache.iceberg.metrics.CommitMetricsResult; +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.ImmutableCommitReport; +import org.apache.iceberg.metrics.ImmutableScanReport; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.ScanMetrics; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; +import org.apache.polaris.core.persistence.metrics.MetricsPersistence; +import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +public class PersistingMetricsReporterTest { + + private static final String CATALOG_NAME = "test-catalog"; + private static final long CATALOG_ID = 12345L; + private static final long TABLE_ID = 67890L; + private static final String TABLE_NAME = "test_table"; + private static final List NAMESPACE = Arrays.asList("db", "schema"); + private static final TableIdentifier TABLE_IDENTIFIER = + TableIdentifier.of(Namespace.of("db", "schema"), TABLE_NAME); + + private CallContext callContext; + private PolarisCallContext polarisCallContext; + private PolarisMetaStoreManager metaStoreManager; + private MetricsPersistence metricsPersistence; + private PersistingMetricsReporter reporter; + + @BeforeEach + void setUp() { + polarisCallContext = mock(PolarisCallContext.class); + callContext = mock(CallContext.class); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + + metaStoreManager = mock(PolarisMetaStoreManager.class); + metricsPersistence = mock(MetricsPersistence.class); + + reporter = new PersistingMetricsReporter(callContext, metaStoreManager, metricsPersistence); + } + + @Test + void testReportScanMetrics() { + // Setup catalog lookup + PolarisBaseEntity catalogEntity = createCatalogEntity(CATALOG_ID, CATALOG_NAME); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + eq(null), + eq(PolarisEntityType.CATALOG), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(CATALOG_NAME))) + .thenReturn(new EntityResult(catalogEntity)); + + // Setup namespace lookups - "db" and "schema" + PolarisBaseEntity dbNamespaceEntity = createNamespaceEntity(11111L, "db", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("db"))) + .thenReturn(new EntityResult(dbNamespaceEntity)); + + PolarisBaseEntity schemaNamespaceEntity = createNamespaceEntity(22222L, "schema", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("schema"))) + .thenReturn(new EntityResult(schemaNamespaceEntity)); + + // Setup table lookup + PolarisBaseEntity tableEntity = createTableEntity(TABLE_ID, TABLE_NAME, CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.TABLE_LIKE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(TABLE_NAME))) + .thenReturn(new EntityResult(tableEntity)); + + // Create a scan report + ScanReport scanReport = createScanReport(); + + // Call the reporter + reporter.reportMetric(CATALOG_NAME, TABLE_IDENTIFIER, scanReport, Instant.now()); + + // Verify persistence was called with correct record + ArgumentCaptor captor = ArgumentCaptor.forClass(ScanMetricsRecord.class); + verify(metricsPersistence).writeScanReport(captor.capture()); + + ScanMetricsRecord record = captor.getValue(); + assertThat(record.catalogId()).isEqualTo(CATALOG_ID); + assertThat(record.tableId()).isEqualTo(TABLE_ID); + assertThat(record.reportId()).isNotNull(); + } + + @Test + void testReportCommitMetrics() { + // Setup catalog lookup + PolarisBaseEntity catalogEntity = createCatalogEntity(CATALOG_ID, CATALOG_NAME); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + eq(null), + eq(PolarisEntityType.CATALOG), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(CATALOG_NAME))) + .thenReturn(new EntityResult(catalogEntity)); + + // Setup namespace lookups - "db" and "schema" + PolarisBaseEntity dbNamespaceEntity = createNamespaceEntity(11111L, "db", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("db"))) + .thenReturn(new EntityResult(dbNamespaceEntity)); + + PolarisBaseEntity schemaNamespaceEntity = createNamespaceEntity(22222L, "schema", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("schema"))) + .thenReturn(new EntityResult(schemaNamespaceEntity)); + + // Setup table lookup + PolarisBaseEntity tableEntity = createTableEntity(TABLE_ID, TABLE_NAME, CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.TABLE_LIKE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(TABLE_NAME))) + .thenReturn(new EntityResult(tableEntity)); + + // Create a commit report + CommitReport commitReport = createCommitReport(); + + // Call the reporter + reporter.reportMetric(CATALOG_NAME, TABLE_IDENTIFIER, commitReport, Instant.now()); + + // Verify persistence was called with correct record + ArgumentCaptor captor = ArgumentCaptor.forClass(CommitMetricsRecord.class); + verify(metricsPersistence).writeCommitReport(captor.capture()); + + CommitMetricsRecord record = captor.getValue(); + assertThat(record.catalogId()).isEqualTo(CATALOG_ID); + assertThat(record.tableId()).isEqualTo(TABLE_ID); + assertThat(record.reportId()).isNotNull(); + } + + @Test + void testCatalogNotFound() { + // Setup catalog lookup to return entity not found + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + eq(null), + eq(PolarisEntityType.CATALOG), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(CATALOG_NAME))) + .thenReturn( + new EntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, "Catalog not found")); + + ScanReport scanReport = createScanReport(); + + // Call the reporter - should not throw + reporter.reportMetric(CATALOG_NAME, TABLE_IDENTIFIER, scanReport, Instant.now()); + + // Verify persistence was NOT called since catalog was not found + verify(metricsPersistence, never()).writeScanReport(any()); + verify(metricsPersistence, never()).writeCommitReport(any()); + } + + @Test + void testUnknownReportType() { + // Setup catalog lookup + PolarisBaseEntity catalogEntity = createCatalogEntity(CATALOG_ID, CATALOG_NAME); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + eq(null), + eq(PolarisEntityType.CATALOG), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(CATALOG_NAME))) + .thenReturn(new EntityResult(catalogEntity)); + + // Setup namespace lookups - "db" and "schema" + PolarisBaseEntity dbNamespaceEntity = createNamespaceEntity(11111L, "db", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("db"))) + .thenReturn(new EntityResult(dbNamespaceEntity)); + + PolarisBaseEntity schemaNamespaceEntity = createNamespaceEntity(22222L, "schema", CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.NAMESPACE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq("schema"))) + .thenReturn(new EntityResult(schemaNamespaceEntity)); + + // Setup table lookup + PolarisBaseEntity tableEntity = createTableEntity(TABLE_ID, TABLE_NAME, CATALOG_ID); + when(metaStoreManager.readEntityByName( + eq(polarisCallContext), + any(), + eq(PolarisEntityType.TABLE_LIKE), + eq(PolarisEntitySubType.ANY_SUBTYPE), + eq(TABLE_NAME))) + .thenReturn(new EntityResult(tableEntity)); + + // Create an unknown report type (using a mock) + MetricsReport unknownReport = mock(MetricsReport.class); + + // Call the reporter - should not throw + reporter.reportMetric(CATALOG_NAME, TABLE_IDENTIFIER, unknownReport, Instant.now()); + + // Verify persistence was NOT called since report type is unknown + verify(metricsPersistence, never()).writeScanReport(any()); + verify(metricsPersistence, never()).writeCommitReport(any()); + } + + private PolarisBaseEntity createCatalogEntity(long id, String name) { + return new PolarisBaseEntity.Builder() + .catalogId(0L) + .id(id) + .parentId(0L) + .typeCode(PolarisEntityType.CATALOG.getCode()) + .subTypeCode(PolarisEntitySubType.NULL_SUBTYPE.getCode()) + .name(name) + .entityVersion(1) + .build(); + } + + private PolarisBaseEntity createNamespaceEntity(long id, String name, long catalogId) { + return new PolarisBaseEntity.Builder() + .catalogId(catalogId) + .id(id) + .parentId(catalogId) // Parent is the catalog for simplicity + .typeCode(PolarisEntityType.NAMESPACE.getCode()) + .subTypeCode(PolarisEntitySubType.NULL_SUBTYPE.getCode()) + .name(name) + .entityVersion(1) + .build(); + } + + private PolarisBaseEntity createTableEntity(long id, String name, long catalogId) { + return new PolarisBaseEntity.Builder() + .catalogId(catalogId) + .id(id) + .parentId(catalogId) // Parent is the catalog for simplicity + .typeCode(PolarisEntityType.TABLE_LIKE.getCode()) + .subTypeCode(PolarisEntitySubType.ICEBERG_TABLE.getCode()) + .name(name) + .entityVersion(1) + .build(); + } + + private ScanReport createScanReport() { + return ImmutableScanReport.builder() + .tableName("db.schema.test_table") + .snapshotId(123456789L) + .schemaId(1) + .filter(Expressions.alwaysTrue()) + .scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop())) + .build(); + } + + private CommitReport createCommitReport() { + CommitMetrics commitMetrics = + CommitMetrics.of(new org.apache.iceberg.metrics.DefaultMetricsContext()); + CommitMetricsResult metricsResult = CommitMetricsResult.from(commitMetrics, Map.of()); + + return ImmutableCommitReport.builder() + .tableName("db.schema.test_table") + .snapshotId(987654321L) + .sequenceNumber(5L) + .operation("append") + .commitMetrics(metricsResult) + .build(); + } +}