[AURON #1745] Introduce auron-spark-tests submodule for correctness testing

.github/workflows/tpcds-reusable.yml

@@ -24,6 +24,11 @@ on:
         description: 'Maven profile id to resolve sparkVersion (e.g., spark-3.5)'
         required: true
         type: string
+      sparktests:
+        description: 'Whether to enable spark correctness tests'
+        required: false
+        type: string
+        default: 'false'
       hadoop-profile:
         description: 'Hadoop profile (e.g., hadoop2.7, hadoop3)'
         required: true
@@ -183,6 +188,11 @@ jobs:
             CMD="$CMD --uniffle $UNIFFLE_NUMBER"
           fi
 
+          SPARK_TESTS="${{ inputs.sparktests }}"
+          if [[ "$SPARK_TESTS" == "true" ]]; then
+            CMD="$CMD --sparktests true"
+          fi
+
           echo "Running: $CMD"
           exec $CMD
 

.github/workflows/tpcds.yml

@@ -60,6 +60,7 @@ jobs:
     with:
       sparkver: spark-3.3
       hadoop-profile: 'hadoop3'
+      sparktests: true
 
   test-spark-34-jdk11:
     name: Test spark-3.4

auron-build.sh

@@ -53,6 +53,7 @@ print_help() {
     echo "  --release                Activate release profile"
     echo "  --clean <true|false>     Clean before build (default: true)"
     echo "  --skiptests <true|false> Skip unit tests (default: true)"
+    echo "  --sparktests <true|false> Run spark tests (default: false)"
     echo "  --docker <true|false>    Build in Docker environment (default: false)"
     IFS=','; echo "  --image <NAME>           Docker image to use (e.g. ${SUPPORTED_OS_IMAGES[*]}, default: ${SUPPORTED_OS_IMAGES[*]:0:1})"; unset IFS
     IFS=','; echo "  --sparkver <VERSION>     Specify Spark version (e.g. ${SUPPORTED_SPARK_VERSIONS[*]})"; unset IFS
@@ -124,6 +125,7 @@ PRE_PROFILE=false
 RELEASE_PROFILE=false
 CLEAN=true
 SKIP_TESTS=true
+SPARK_TESTS=false
 SPARK_VER=""
 FLINK_VER=""
 SCALA_VER=""
@@ -187,6 +189,15 @@ while [[ $# -gt 0 ]]; do
                 exit 1
             fi
             ;;
+        --sparktests)
+            if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then
+                SPARK_TESTS="$2"
+                shift 2
+            else
+                echo "ERROR: --sparktests requires true/false" >&2
+                exit 1
+            fi
+            ;;
         --sparkver)
             if [[ -n "$2" && "$2" != -* ]]; then
                 SPARK_VER="$2"
@@ -353,6 +364,10 @@ else
     BUILD_ARGS+=("package")
 fi
 
+if [[ "$SPARK_TESTS" == true ]]; then
+    BUILD_ARGS+=("-Pspark-tests")
+fi
+
 if [[ "$PRE_PROFILE" == true ]]; then
     BUILD_ARGS+=("-Ppre")
 fi

auron-spark-tests/common/pom.xml

@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.auron</groupId>
+    <artifactId>auron-spark-tests</artifactId>
+    <version>${project.version}</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>auron-spark-tests-common</artifactId>
+  <packaging>jar</packaging>
+  <name>Auron Spark Test Common</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.auron</groupId>
+      <artifactId>spark-extension_${scalaVersion}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scalaVersion}</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-catalyst_${scalaVersion}</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scalaVersion}</artifactId>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scalaVersion}</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-catalyst_${scalaVersion}</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-hive_${scalaVersion}</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatestplus</groupId>
+      <artifactId>scalatestplus-scalacheck_${scalaVersion}</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.scalastyle</groupId>
+        <artifactId>scalastyle-maven-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.scalatest</groupId>
+        <artifactId>scalatest-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>test</id>
+            <goals>
+              <goal>test</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>prepare-test-jar</id>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+            <phase>test-compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+    <outputDirectory>target/scala-${scalaVersion}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scalaVersion}/test-classes</testOutputDirectory>
+  </build>
+</project>

auron-spark-tests/common/src/test/resources/log4j.properties

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file core/target/unit-tests.log
+log4j.rootLogger=WARN, CA
+
+#Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold=DEBUG
+log4j.appender.CA.follow=true
+
+# Some packages are noisy for no good reason.
+log4j.additivity.org.apache.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetOutputCommitter=false
+log4j.logger.org.apache.parquet.hadoop.ParquetOutputCommitter=OFF
+
+log4j.additivity.org.apache.hadoop.hive.serde2.lazy.LazyStruct=false
+log4j.logger.org.apache.hadoop.hive.serde2.lazy.LazyStruct=OFF
+
+log4j.additivity.org.apache.hadoop.hive.metastore.RetryingHMSHandler=false
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=OFF
+
+log4j.additivity.hive.ql.metadata.Hive=false
+log4j.logger.hive.ql.metadata.Hive=OFF
+
+# Parquet related logging
+log4j.logger.org.apache.parquet.CorruptStatistics=ERROR
+log4j.logger.parquet.CorruptStatistics=ERROR

auron-spark-tests/common/src/test/resources/log4j2.properties

@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file core/target/unit-tests.log
+rootLogger.level = warn
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+#Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n%ex
+appender.console.filter.threshold.type = ThresholdFilter
+appender.console.filter.threshold.level = debug
+
+#File Appender
+#appender.file.type = File
+#appender.file.name = File
+#appender.file.fileName = target/unit-tests.log
+#appender.file.layout.type = PatternLayout
+#appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
+
+# Set the logger level of File Appender to WARN
+# appender.file.filter.threshold.type = ThresholdFilter
+# appender.file.filter.threshold.level = info
+
+# Some packages are noisy for no good reason.
+logger.parquet_recordreader.name = org.apache.parquet.hadoop.ParquetRecordReader
+logger.parquet_recordreader.additivity = false
+logger.parquet_recordreader.level = off
+
+logger.parquet_outputcommitter.name = org.apache.parquet.hadoop.ParquetOutputCommitter
+logger.parquet_outputcommitter.additivity = false
+logger.parquet_outputcommitter.level = off
+
+logger.hadoop_lazystruct.name = org.apache.hadoop.hive.serde2.lazy.LazyStruct
+logger.hadoop_lazystruct.additivity = false
+logger.hadoop_lazystruct.level = off
+
+logger.hadoop_retryinghmshandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
+logger.hadoop_retryinghmshandler.additivity = false
+logger.hadoop_retryinghmshandler.level = off
+
+logger.hive_metadata.name = hive.ql.metadata.Hive
+logger.hive_metadata.additivity = false
+logger.hive_metadata.level = off
+
+# Parquet related logging
+logger.parquet1.name = org.apache.parquet.CorruptStatistics
+logger.parquet1.level = error
+
+logger.parquet2.name = parquet.CorruptStatistics
+logger.parquet2.level = error

auron-spark-tests/common/src/test/scala/org/apache/auron/utils/SQLQueryTestSettings.scala

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.utils
+
+trait SQLQueryTestSettings {
+  def getResourceFilePath: String
+
+  def getSupportedSQLQueryTests: Set[String]
+
+  def getOverwriteSQLQueryTests: Set[String]
+
+  def getIgnoredSQLQueryTests: List[String] = List.empty
+}