feat(persistence): Add JDBC persistence layer for Iceberg metrics reporting (#3337)

.gitignore

@@ -112,3 +112,4 @@ venv
 # to override default properties for local development.
 # And then use `./gradlew run -Dquarkus.profile=local` to run Polaris with dev profile.
 application-local.properties
+.polaris-work-notes.md

CHANGELOG.md

@@ -35,7 +35,6 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 
 ### Breaking changes
 
-- The (Before/After)CommitViewEvent has been removed.
 - The (Before/After)CommitTableEvent has been removed.
 - The `PolarisMetricsReporter.reportMetric()` method signature has been extended to include a `receivedTimestamp` parameter of type `java.time.Instant`.
 

persistence/relational-jdbc/build.gradle.kts

@@ -29,10 +29,16 @@ dependencies {
 
   compileOnly(platform(libs.jackson.bom))
   compileOnly("com.fasterxml.jackson.core:jackson-annotations")
+  compileOnly("com.fasterxml.jackson.core:jackson-databind")
   compileOnly(libs.jakarta.annotation.api)
   compileOnly(libs.jakarta.enterprise.cdi.api)
   compileOnly(libs.jakarta.inject.api)
 
+  // Iceberg API for metrics report conversion
+  compileOnly(platform(libs.iceberg.bom))
+  compileOnly("org.apache.iceberg:iceberg-api")
+  compileOnly("org.apache.iceberg:iceberg-core")
+
   implementation(libs.smallrye.common.annotation) // @Identifier
   implementation(libs.postgresql)
 
@@ -43,6 +49,10 @@ dependencies {
   testImplementation(libs.h2)
   testImplementation(testFixtures(project(":polaris-core")))
 
+  // Iceberg API for SpiModelConverter tests
+  testImplementation(platform(libs.iceberg.bom))
+  testImplementation("org.apache.iceberg:iceberg-api")
+
   testImplementation(platform(libs.testcontainers.bom))
 
   testImplementation("org.testcontainers:testcontainers-junit-jupiter")

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java

@@ -60,4 +60,24 @@ public InputStream openInitScriptResource(int schemaVersion) {
     ClassLoader classLoader = DatasourceOperations.class.getClassLoader();
     return classLoader.getResourceAsStream(resourceName);
   }
+
+  /**
+   * Open an InputStream that contains data from the metrics schema init script. This stream should
+   * be closed by the caller.
+   *
+   * @param metricsSchemaVersion the metrics schema version (currently only 1 is supported)
+   * @return an InputStream for the metrics schema SQL file
+   */
+  public InputStream openMetricsSchemaResource(int metricsSchemaVersion) {
+    if (metricsSchemaVersion != 1) {
+      throw new IllegalArgumentException(
+          "Unknown or invalid metrics schema version " + metricsSchemaVersion);
+    }
+
+    final String resourceName =
+        String.format("%s/schema-metrics-v%d.sql", this.getDisplayName(), metricsSchemaVersion);
+
+    ClassLoader classLoader = DatasourceOperations.class.getClassLoader();
+    return classLoader.getResourceAsStream(resourceName);
+  }
 }

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBootstrapUtils.java

@@ -88,4 +88,18 @@ public static int getRequestedSchemaVersion(BootstrapOptions bootstrapOptions) {
     }
     return -1;
   }
+
+  /**
+   * Determines whether the metrics schema should be included during bootstrap.
+   *
+   * @param bootstrapOptions The bootstrap options containing schema information.
+   * @return true if the metrics schema should be included, false otherwise.
+   */
+  public static boolean shouldIncludeMetrics(BootstrapOptions bootstrapOptions) {
+    SchemaOptions schemaOptions = bootstrapOptions.schemaOptions();
+    if (schemaOptions != null) {
+      return schemaOptions.includeMetrics();
+    }
+    return false;
+  }
 }

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java

@@ -52,6 +52,7 @@
 import org.apache.polaris.core.persistence.cache.InMemoryEntityCache;
 import org.apache.polaris.core.persistence.dao.entity.BaseResult;
 import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult;
+import org.apache.polaris.core.persistence.metrics.MetricsSchemaBootstrap;
 import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -78,6 +79,10 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory {
   @Inject RelationalJdbcConfiguration relationalJdbcConfiguration;
   @Inject RealmConfig realmConfig;
 
+  @Inject
+  @Identifier("relational-jdbc")
+  MetricsSchemaBootstrap metricsSchemaBootstrap;
+
   protected JdbcMetaStoreManagerFactory() {}
 
   protected PrincipalSecretsGenerator secretsGenerator(
@@ -172,6 +177,11 @@ public synchronized Map<String, PrincipalSecretsResult> bootstrapRealms(
               datasourceOperations
                   .getDatabaseType()
                   .openInitScriptResource(effectiveSchemaVersion));
+
+          // Run the metrics schema bootstrap if requested
+          if (JdbcBootstrapUtils.shouldIncludeMetrics(bootstrapOptions)) {
+            metricsSchemaBootstrap.bootstrap(realm);
+          }
         } catch (SQLException e) {
           throw new RuntimeException(
               String.format("Error executing sql script: %s", e.getMessage()), e);