From 52a79299633f19e2130e7c08b6b2bf3819302342 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Mon, 7 Jul 2025 16:11:59 -0500 Subject: [PATCH 01/13] Added graal vm plugin to pom.xml and classes to reflection-config.json --- ice/pom.xml | 37 +++++ ice/src/main/resources/reflection-config.json | 134 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 ice/src/main/resources/reflection-config.json diff --git a/ice/pom.xml b/ice/pom.xml index 49aa73f8..e183ae9b 100644 --- a/ice/pom.xml +++ b/ice/pom.xml @@ -563,5 +563,42 @@ + + native + + + + org.graalvm.buildtools + native-maven-plugin + ${native.maven.plugin.version} + true + + + build-native + + compile-no-fork + + package + + + + ice + com.altinity.ice.cli.Main + + --initialize-at-build-time=ch.qos.logback + -H:ReflectionConfigurationFiles=src/main/resources/reflection-config.json + --trace-class-initialization=ch.qos.logback.classic.Logger + --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker + --trace-object-instantiation=ch.qos.logback.classic.Logger + --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-build-time=ch.qos.logback.classic.Logger + --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout + + + + + + + diff --git a/ice/src/main/resources/reflection-config.json b/ice/src/main/resources/reflection-config.json new file mode 100644 index 00000000..1d9866a5 --- /dev/null +++ b/ice/src/main/resources/reflection-config.json @@ -0,0 +1,134 @@ +[ + { + "name": "ch.qos.logback.classic.AsyncAppender", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.Logger.effectiveLevelInt", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.Logger.loggerContext", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.encoder.PatternLayoutEncoder", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.DateConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.LevelConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.LineSeparatorConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.LoggerConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.MessageConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.classic.pattern.ThreadConverter", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.core.ConsoleAppender", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "ch.qos.logback.core.FileAppender", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true, + "allDeclaredClasses": true, + "allPublicClasses": true + } +] From c530e82e4d84cc3ca722d9c615a9580e889ad67d Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 19 Nov 2025 19:20:07 -0600 Subject: [PATCH 02/13] Fixed steps to build ice native image --- .github/workflows/verify.yaml | 18 ++++++- README.md | 32 +++++++++++- build-native.sh | 95 +++++++++++++++++++++++++++++++++++ ice/pom.xml | 85 ++++++++++++++++++++++++++++++- 4 files changed, 226 insertions(+), 4 deletions(-) create mode 100755 build-native.sh diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 8e1a3be0..e9cd1b26 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -15,4 +15,20 @@ jobs: distribution: 'graalvm' cache: maven - run: ./mvnw clean verify - # TODO: check native-image can build ice + + native-build: + name: Build and test native image + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'graalvm' + cache: maven + - name: Install musl for static linking + run: sudo apt-get update && sudo apt-get install -y musl-tools + - name: Build native image (amd64 static with musl) + run: ./build-native.sh + env: + ARCH: amd64 \ No newline at end of file diff --git a/README.md b/README.md index 91e9c0dc..e83f3fdf 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,14 @@ Create/delete tables, insert data with `ice insert -p ns1.table1 file://example. ## Installation -Pre-built binaries\* (+ links to Docker images for [ice](https://hub.docker.com/r/altinity/ice) and [ice-rest-catalog](https://hub.docker.com/r/altinity/ice-rest-catalog)) are available form [GitHub Releases](https://github.com/Altinity/ice/releases) page. -> \* currently require `java` 21+ to run (available [here](https://adoptium.net/installation/)). +Pre-built binaries (+ links to Docker images for [ice](https://hub.docker.com/r/altinity/ice) and [ice-rest-catalog](https://hub.docker.com/r/altinity/ice-rest-catalog)) are available from [GitHub Releases](https://github.com/Altinity/ice/releases) page. + +**Two types of binaries are available:** + +1. **Java-based binaries** - Require Java 21+ to run (available [here](https://adoptium.net/installation/)) +2. **Native binaries** - Standalone executables with no Java dependency + - `ice-native-amd64` - Static binary (musl) for x86_64 Linux (no dependencies) + - `ice-native-arm64` - Dynamic binary for ARM64 Linux (requires glibc) ## Usage @@ -19,6 +25,8 @@ See [examples/](examples/). ## Development +### Standard Build (Java-based) + Install [sdkman](https://sdkman.io/install), then ```shell @@ -35,6 +43,26 @@ sdk env ./mvnw ``` +### Native Image Build (Standalone) + +Build standalone native binaries with no Java dependency: + +```shell +# Install prerequisites +sdk env # or ensure Java 21+ and GraalVM are available + +# For amd64 (static with musl - no dependencies) +./build-native.sh +# Produces: ice/target/ice (static binary) + +# Or use Maven directly +mvn -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true + +# For ARM64 (dynamic with glibc) +ARCH=arm64 ./build-native.sh +# Or: mvn -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true + + ## License Copyright (c) 2025, Altinity Inc and/or its affiliates. All rights reserved. diff --git a/build-native.sh b/build-native.sh new file mode 100755 index 00000000..891d534f --- /dev/null +++ b/build-native.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Build native images for ice CLI +# Supports: amd64 (static with musl), arm64 (dynamic) + +set -e + +VERSION="${VERSION:-0.0.0-SNAPSHOT}" +ARCH="${ARCH:-$(uname -m)}" + +# Convert architecture names +case "$ARCH" in + x86_64) + ARCH="amd64" + PROFILE="native-amd64-static" + ;; + aarch64|arm64) + ARCH="arm64" + PROFILE="native-arm64" + ;; + *) + echo "Unsupported architecture: $ARCH" + echo "Supported: x86_64/amd64, aarch64/arm64" + exit 1 + ;; +esac + +echo "============================================" +echo "Building native ice binary" +echo "Architecture: $ARCH" +echo "Profile: $PROFILE" +echo "Version: $VERSION" +echo "============================================" +echo "" + +# Check for musl-gcc on amd64 builds +if [ "$ARCH" = "amd64" ]; then + if ! command -v x86_64-linux-musl-gcc &> /dev/null && ! command -v musl-gcc &> /dev/null; then + echo " WARNING: musl compiler not found!" + echo "" + echo "For static builds on amd64, you need musl-tools installed." + echo "" + echo "Install it with:" + echo " Ubuntu/Debian: sudo apt-get install musl-tools" + echo " Fedora/RHEL: sudo dnf install musl-gcc musl-libc-static" + echo " Alpine: apk add musl-dev gcc" + echo "" + echo "Or build without static linking using the 'native' profile:" + echo " mvn -Pnative -pl ice clean package -Dmaven.test.skip=true" + echo "" + read -p "Continue anyway? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + else + echo "✓ musl compiler found" + fi +fi +echo "" + +# Set version +./mvnw -am -pl ice versions:set -DnewVersion=${VERSION} + +# Build native image +echo "Building native image..." +./mvnw -Pno-check -P${PROFILE} -pl ice clean package -Dmaven.test.skip=true + +echo "" +echo "============================================" +echo "✓ Native binary built successfully!" +echo "============================================" +echo "" +echo "Binary location: ice/target/ice" +echo "" +echo "Test the binary:" +echo " ./ice/target/ice --version" +echo " ./ice/target/ice check" +echo "" + +# Show binary info +if [ -f "ice/target/ice" ]; then + ls -lh ice/target/ice + file ice/target/ice || true + + if [ "$ARCH" = "amd64" ]; then + echo "" + echo "Static binary (no dependencies):" + ldd ice/target/ice 2>&1 || echo " ✓ Statically linked (expected for amd64)" + else + echo "" + echo "Dynamic binary dependencies:" + ldd ice/target/ice || true + fi +fi + diff --git a/ice/pom.xml b/ice/pom.xml index 639dc1d2..54cff814 100644 --- a/ice/pom.xml +++ b/ice/pom.xml @@ -641,7 +641,7 @@ com.altinity.ice.cli.Main --initialize-at-build-time=ch.qos.logback - -H:ReflectionConfigurationFiles=src/main/resources/reflection-config.json + -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json --trace-class-initialization=ch.qos.logback.classic.Logger --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker --trace-object-instantiation=ch.qos.logback.classic.Logger @@ -655,5 +655,88 @@ + + native-amd64-static + + + + org.graalvm.buildtools + native-maven-plugin + ${native.maven.plugin.version} + true + + + build-native + + compile-no-fork + + package + + + + ice + com.altinity.ice.cli.Main + + + --initialize-at-build-time=ch.qos.logback + -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json + --trace-class-initialization=ch.qos.logback.classic.Logger + --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker + --trace-object-instantiation=ch.qos.logback.classic.Logger + --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-build-time=ch.qos.logback.classic.Logger + --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout + + --static + --libc=musl + + -H:+RemoveUnusedSymbols + -H:+ReportExceptionStackTraces + + + + + + + + native-arm64 + + + + org.graalvm.buildtools + native-maven-plugin + ${native.maven.plugin.version} + true + + + build-native + + compile-no-fork + + package + + + + ice + com.altinity.ice.cli.Main + + + --initialize-at-build-time=ch.qos.logback + -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json + --trace-class-initialization=ch.qos.logback.classic.Logger + --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker + --trace-object-instantiation=ch.qos.logback.classic.Logger + --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-build-time=ch.qos.logback.classic.Logger + --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout + + -H:+RemoveUnusedSymbols + -H:+ReportExceptionStackTraces + + + + + + From 86fa1cb324ffdafd369590e163e3a7def6e5b1ec Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 19 Nov 2025 19:23:18 -0600 Subject: [PATCH 03/13] include amd64 in supported architecture --- build-native.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-native.sh b/build-native.sh index 891d534f..1b6c26c0 100755 --- a/build-native.sh +++ b/build-native.sh @@ -9,7 +9,7 @@ ARCH="${ARCH:-$(uname -m)}" # Convert architecture names case "$ARCH" in - x86_64) + x86_64|amd64) ARCH="amd64" PROFILE="native-amd64-static" ;; From 8bd32d291446635d25d56bba494479d58acc78aa Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 19 Nov 2025 19:28:49 -0600 Subject: [PATCH 04/13] Added runtime depedencies for native image. --- ice/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ice/pom.xml b/ice/pom.xml index 54cff814..7a74f735 100644 --- a/ice/pom.xml +++ b/ice/pom.xml @@ -646,6 +646,9 @@ --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=org.slf4j.LoggerFactory + --initialize-at-run-time=org.slf4j.helpers.Reporter + --initialize-at-run-time=picocli.CommandLine$Help$Ansi --initialize-at-build-time=ch.qos.logback.classic.Logger --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout @@ -684,6 +687,9 @@ --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=org.slf4j.LoggerFactory + --initialize-at-run-time=org.slf4j.helpers.Reporter + --initialize-at-run-time=picocli.CommandLine$Help$Ansi --initialize-at-build-time=ch.qos.logback.classic.Logger --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout @@ -727,6 +733,9 @@ --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=org.slf4j.LoggerFactory + --initialize-at-run-time=org.slf4j.helpers.Reporter + --initialize-at-run-time=picocli.CommandLine$Help$Ansi --initialize-at-build-time=ch.qos.logback.classic.Logger --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout From e2b93b4a61f3f1e57eb4e0fe5ffee3bcccf84877 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 19 Nov 2025 19:35:12 -0600 Subject: [PATCH 05/13] Added runtime depedencies for native image. --- ice/pom.xml | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/ice/pom.xml b/ice/pom.xml index 7a74f735..36422a6c 100644 --- a/ice/pom.xml +++ b/ice/pom.xml @@ -640,17 +640,9 @@ ice com.altinity.ice.cli.Main - --initialize-at-build-time=ch.qos.logback -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --trace-class-initialization=ch.qos.logback.classic.Logger - --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker - --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask - --initialize-at-run-time=org.slf4j.LoggerFactory - --initialize-at-run-time=org.slf4j.helpers.Reporter - --initialize-at-run-time=picocli.CommandLine$Help$Ansi - --initialize-at-build-time=ch.qos.logback.classic.Logger - --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout + -H:+ReportExceptionStackTraces @@ -681,17 +673,8 @@ com.altinity.ice.cli.Main - --initialize-at-build-time=ch.qos.logback -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --trace-class-initialization=ch.qos.logback.classic.Logger - --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker - --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask - --initialize-at-run-time=org.slf4j.LoggerFactory - --initialize-at-run-time=org.slf4j.helpers.Reporter - --initialize-at-run-time=picocli.CommandLine$Help$Ansi - --initialize-at-build-time=ch.qos.logback.classic.Logger - --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout --static --libc=musl @@ -727,17 +710,8 @@ com.altinity.ice.cli.Main - --initialize-at-build-time=ch.qos.logback -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --trace-class-initialization=ch.qos.logback.classic.Logger - --trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase$Worker - --trace-object-instantiation=ch.qos.logback.classic.Logger --initialize-at-run-time=io.netty.channel.ChannelHandlerMask - --initialize-at-run-time=org.slf4j.LoggerFactory - --initialize-at-run-time=org.slf4j.helpers.Reporter - --initialize-at-run-time=picocli.CommandLine$Help$Ansi - --initialize-at-build-time=ch.qos.logback.classic.Logger - --initialize-at-build-time=com.altinity.ice.internal.logback.ColorAwarePatternLayout -H:+RemoveUnusedSymbols -H:+ReportExceptionStackTraces From 52ba3c740127116098d42712e54e5a27651a39cd Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Fri, 21 Nov 2025 07:50:21 -0600 Subject: [PATCH 06/13] Fixed reflection-config.json --- .github/workflows/verify.yaml | 16 +++++++++----- ice/pom.xml | 9 +++++--- ice/src/main/resources/reflection-config.json | 22 ------------------- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index e9cd1b26..b74c5e81 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -26,9 +26,13 @@ jobs: java-version: '21' distribution: 'graalvm' cache: maven - - name: Install musl for static linking - run: sudo apt-get update && sudo apt-get install -y musl-tools - - name: Build native image (amd64 static with musl) - run: ./build-native.sh - env: - ARCH: amd64 \ No newline at end of file + - name: Build native image (dynamic linking - no musl required) + run: ./mvnw -Pnative -pl ice clean package -Dmaven.test.skip=true + - name: Test native binary + run: ./test-native-binary.sh + - name: Upload native binary as artifact + uses: actions/upload-artifact@v4 + with: + name: ice-native-amd64-dynamic + path: ice/target/ice + if-no-files-found: error \ No newline at end of file diff --git a/ice/pom.xml b/ice/pom.xml index 36422a6c..81bd81e3 100644 --- a/ice/pom.xml +++ b/ice/pom.xml @@ -641,7 +641,8 @@ com.altinity.ice.cli.Main -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=io.netty + --initialize-at-run-time=ch.qos.logback -H:+ReportExceptionStackTraces @@ -674,7 +675,8 @@ -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=io.netty + --initialize-at-run-time=ch.qos.logback --static --libc=musl @@ -711,7 +713,8 @@ -H:ReflectionConfigurationFiles=${project.basedir}/src/main/resources/reflection-config.json - --initialize-at-run-time=io.netty.channel.ChannelHandlerMask + --initialize-at-run-time=io.netty + --initialize-at-run-time=ch.qos.logback -H:+RemoveUnusedSymbols -H:+ReportExceptionStackTraces diff --git a/ice/src/main/resources/reflection-config.json b/ice/src/main/resources/reflection-config.json index 1d9866a5..15a1a7b5 100644 --- a/ice/src/main/resources/reflection-config.json +++ b/ice/src/main/resources/reflection-config.json @@ -10,28 +10,6 @@ "allDeclaredClasses": true, "allPublicClasses": true }, - { - "name": "ch.qos.logback.classic.Logger.effectiveLevelInt", - "allDeclaredConstructors": true, - "allPublicConstructors": true, - "allDeclaredMethods": true, - "allPublicMethods": true, - "allDeclaredFields": true, - "allPublicFields": true, - "allDeclaredClasses": true, - "allPublicClasses": true - }, - { - "name": "ch.qos.logback.classic.Logger.loggerContext", - "allDeclaredConstructors": true, - "allPublicConstructors": true, - "allDeclaredMethods": true, - "allPublicMethods": true, - "allDeclaredFields": true, - "allPublicFields": true, - "allDeclaredClasses": true, - "allPublicClasses": true - }, { "name": "ch.qos.logback.classic.encoder.PatternLayoutEncoder", "allDeclaredConstructors": true, From b6fcc24fe224e2e32c776dbf9ff9aa32cacd4bea Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Fri, 21 Nov 2025 08:57:32 -0600 Subject: [PATCH 07/13] remove verify binary step. --- .github/workflows/verify.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b74c5e81..c32eea1c 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -28,8 +28,6 @@ jobs: cache: maven - name: Build native image (dynamic linking - no musl required) run: ./mvnw -Pnative -pl ice clean package -Dmaven.test.skip=true - - name: Test native binary - run: ./test-native-binary.sh - name: Upload native binary as artifact uses: actions/upload-artifact@v4 with: From 2d4dbe29f67f14ee4f4fbc1fc737304124cbeb9d Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Fri, 21 Nov 2025 09:23:46 -0600 Subject: [PATCH 08/13] Add support for ARM. --- .github/workflows/verify.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index c32eea1c..2cb15e8c 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -17,8 +17,19 @@ jobs: - run: ./mvnw clean verify native-build: - name: Build and test native image - runs-on: ubuntu-24.04 + name: Build and test native image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + profile: native + artifact: ice-native-amd64-dynamic + - arch: arm64 + runner: ubuntu-24.04-arm + profile: native-arm64 + artifact: ice-native-arm64-dynamic steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -27,10 +38,10 @@ jobs: distribution: 'graalvm' cache: maven - name: Build native image (dynamic linking - no musl required) - run: ./mvnw -Pnative -pl ice clean package -Dmaven.test.skip=true + run: ./mvnw -P${{ matrix.profile }} -pl ice clean package -Dmaven.test.skip=true - name: Upload native binary as artifact uses: actions/upload-artifact@v4 with: - name: ice-native-amd64-dynamic + name: ${{ matrix.artifact }} path: ice/target/ice if-no-files-found: error \ No newline at end of file From 0c6bca669d1edabc3060fbac9689c9e01643d234 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 22 Nov 2025 08:55:09 -0600 Subject: [PATCH 09/13] Added Dockerfile. --- docker-build-native.sh | 55 ++++++++++++++++++ ice/Dockerfile.native | 52 +++++++++++++++++ ice/Dockerfile.native-amd64-static | 33 +++++++++++ ice/Dockerfile.native-arm64 | 33 +++++++++++ ice/Dockerfile.native-builder | 91 ++++++++++++++++++++++++++++++ 5 files changed, 264 insertions(+) create mode 100755 docker-build-native.sh create mode 100644 ice/Dockerfile.native create mode 100644 ice/Dockerfile.native-amd64-static create mode 100644 ice/Dockerfile.native-arm64 create mode 100644 ice/Dockerfile.native-builder diff --git a/docker-build-native.sh b/docker-build-native.sh new file mode 100755 index 00000000..f300e81c --- /dev/null +++ b/docker-build-native.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Build ICE native images using Docker +# Usage: ./docker-build-native.sh [amd64|arm64|both] + +set -euo pipefail + +ARCH="${1:-$(uname -m)}" + +# Normalize architecture names +case "$ARCH" in + x86_64|amd64) + BUILD_ARCH="amd64" + PLATFORM="linux/amd64" + ;; + aarch64|arm64) + BUILD_ARCH="arm64" + PLATFORM="linux/arm64" + ;; + both) + echo "Building for both architectures..." + $0 amd64 + $0 arm64 + exit 0 + ;; + *) + echo "Unsupported architecture: $ARCH" + echo "Usage: $0 [amd64|arm64|both]" + exit 1 + ;; +esac + +echo "========================================" +echo "Building ICE native image for $BUILD_ARCH" +echo "Platform: $PLATFORM" +echo "========================================" + +# Build the Docker image +docker build \ + --platform="$PLATFORM" \ + --build-arg ARCH="$BUILD_ARCH" \ + -f ice/Dockerfile.native-builder \ + -t ice-native-builder:"$BUILD_ARCH" \ + . + +# Extract the binary +CONTAINER_ID=$(docker create ice-native-builder:"$BUILD_ARCH") +docker cp "$CONTAINER_ID":/ice "./ice-$BUILD_ARCH" +docker rm "$CONTAINER_ID" + +echo "" +echo "✓ Native binary created: ./ice-$BUILD_ARCH" +echo "" +echo "Test it with:" +echo " ./ice-$BUILD_ARCH --help" + diff --git a/ice/Dockerfile.native b/ice/Dockerfile.native new file mode 100644 index 00000000..52caa1e3 --- /dev/null +++ b/ice/Dockerfile.native @@ -0,0 +1,52 @@ +# Multi-arch native image build for ice +# Supports: amd64 (with static musl), arm64 (dynamic) +ARG BASE_IMAGE_TAG="nonroot" +ARG VERSION="0.0.0-SNAPSHOT" +ARG TARGETARCH + +FROM --platform=$BUILDPLATFORM ghcr.io/graalvm/native-image-community:21.0.2 AS build + +WORKDIR /app + +# Install musl for static linking (amd64) +RUN microdnf install -y musl-devel || true + +# Cache maven dependencies for faster re-builds +COPY .mvn/ .mvn/ +COPY mvnw . +COPY pom.xml . +COPY ice/pom.xml ice/pom.xml +COPY ice-rest-catalog/pom.xml ice-rest-catalog/pom.xml +RUN ./mvnw -am -pl ice dependency:go-offline + +ARG VERSION +ARG TARGETARCH + +COPY . . + +RUN ./mvnw -am -pl ice versions:set -DnewVersion=${VERSION} + +# Build with appropriate profile based on architecture +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + echo "Building static native image for amd64 with musl..."; \ + ./mvnw -Pno-check -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + echo "Building native image for arm64..."; \ + ./mvnw -Pno-check -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true; \ + else \ + echo "Building native image for $TARGETARCH..."; \ + ./mvnw -Pno-check -Pnative -pl ice clean package -Dmaven.test.skip=true; \ + fi + +# Use scratch for amd64 (static), distroless for arm64 (dynamic) +FROM scratch AS final-amd64 +COPY --from=build /app/ice/target/ice /usr/local/bin/ice +ENTRYPOINT ["/usr/local/bin/ice"] + +FROM gcr.io/distroless/base-debian12:${BASE_IMAGE_TAG} AS final-arm64 +COPY --from=build /app/ice/target/ice /usr/local/bin/ice +ENTRYPOINT ["/usr/local/bin/ice"] + +# Select appropriate final stage based on architecture +FROM final-${TARGETARCH} AS final + diff --git a/ice/Dockerfile.native-amd64-static b/ice/Dockerfile.native-amd64-static new file mode 100644 index 00000000..dd0c2587 --- /dev/null +++ b/ice/Dockerfile.native-amd64-static @@ -0,0 +1,33 @@ +# Dockerfile for building ICE native image (amd64, static with musl) +# Usage: docker build -f ice/Dockerfile.native-amd64-static -t ice-native:amd64-static . + +FROM ghcr.io/graalvm/native-image-community:21-muslib AS builder + +WORKDIR /workspace + +# Copy license header file (required by license-maven-plugin) +COPY apache-header.txt . + +# Copy Maven files for dependency resolution +COPY mvnw . +COPY .mvn .mvn +COPY pom.xml . +COPY ice/pom.xml ice/ +COPY ice-rest-catalog/pom.xml ice-rest-catalog/ + +# Download dependencies (cached layer) +RUN ./mvnw dependency:go-offline -pl ice || true + +# Copy source code +COPY ice/src ice/src + +# Build static native image with musl +RUN ./mvnw -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true + +# ============================================================================ +# Final stage: minimal scratch image (truly standalone binary) +# ============================================================================ +FROM scratch +COPY --from=builder /workspace/ice/target/ice /ice +ENTRYPOINT ["/ice"] + diff --git a/ice/Dockerfile.native-arm64 b/ice/Dockerfile.native-arm64 new file mode 100644 index 00000000..4d997ec1 --- /dev/null +++ b/ice/Dockerfile.native-arm64 @@ -0,0 +1,33 @@ +# Dockerfile for building ICE native image (arm64, dynamic linking) +# Usage: docker build -f ice/Dockerfile.native-arm64 -t ice-native:arm64 . + +FROM ghcr.io/graalvm/native-image-community:21 AS builder + +WORKDIR /workspace + +# Copy license header file (required by license-maven-plugin) +COPY apache-header.txt . + +# Copy Maven files for dependency resolution +COPY mvnw . +COPY .mvn .mvn +COPY pom.xml . +COPY ice/pom.xml ice/ +COPY ice-rest-catalog/pom.xml ice-rest-catalog/ + +# Download dependencies (cached layer) +RUN ./mvnw dependency:go-offline -pl ice || true + +# Copy source code +COPY ice/src ice/src + +# Build dynamic native image +RUN ./mvnw -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true + +# ============================================================================ +# Final stage: distroless base (includes minimal glibc runtime) +# ============================================================================ +FROM gcr.io/distroless/base-debian12:nonroot +COPY --from=builder /workspace/ice/target/ice /ice +ENTRYPOINT ["/ice"] + diff --git a/ice/Dockerfile.native-builder b/ice/Dockerfile.native-builder new file mode 100644 index 00000000..0a044e6d --- /dev/null +++ b/ice/Dockerfile.native-builder @@ -0,0 +1,91 @@ +# Multi-stage Dockerfile for building ICE native images +# Supports both amd64 (with static musl) and arm64 (dynamic) + +ARG ARCH=amd64 + +# ============================================================================ +# Stage 1: Build stage with all dependencies +# ============================================================================ +FROM ghcr.io/graalvm/native-image-community:21 AS builder + +# Install build dependencies +RUN microdnf install -y \ + findutils \ + tar \ + gzip \ + wget \ + && microdnf clean all + +# Detect architecture and install appropriate dependencies +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + echo "Installing musl toolchain and zlib for amd64 static builds..."; \ + # Download and install musl-gcc + wget -q https://musl.libc.org/releases/musl-1.2.5.tar.gz && \ + tar -xzf musl-1.2.5.tar.gz && \ + cd musl-1.2.5 && \ + ./configure --prefix=/usr/local/musl --disable-shared && \ + make -j$(nproc) && \ + make install && \ + cd .. && \ + rm -rf musl-1.2.5 musl-1.2.5.tar.gz && \ + # Download and install zlib for musl + wget -q https://zlib.net/zlib-1.3.1.tar.gz && \ + tar -xzf zlib-1.3.1.tar.gz && \ + cd zlib-1.3.1 && \ + CC=/usr/local/musl/bin/musl-gcc ./configure --prefix=/usr/local/musl --static && \ + make -j$(nproc) && \ + make install && \ + cd .. && \ + rm -rf zlib-1.3.1 zlib-1.3.1.tar.gz && \ + # Create symlink for native-image to find musl-gcc + ln -sf /usr/local/musl/bin/musl-gcc /usr/local/bin/x86_64-linux-musl-gcc; \ + else \ + echo "ARM64 detected - using dynamic linking (no musl required)"; \ + fi + +WORKDIR /workspace + +# Copy license header file (required by license-maven-plugin) +COPY apache-header.txt . + +# Copy Maven wrapper and pom files first (for better caching) +COPY mvnw . +COPY .mvn .mvn +COPY pom.xml . +COPY ice/pom.xml ice/ +COPY ice-rest-catalog/pom.xml ice-rest-catalog/ + +# Download dependencies (cached layer) +RUN ./mvnw dependency:go-offline -pl ice || true + +# Copy source code +COPY ice/src ice/src + +# Build native image based on architecture +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + echo "Building static native image for amd64 with musl..."; \ + export PATH="/usr/local/musl/bin:$PATH" && \ + ./mvnw -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true; \ + else \ + echo "Building dynamic native image for arm64..."; \ + ./mvnw -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true; \ + fi + +# ============================================================================ +# Stage 2: Runtime image (minimal) +# ============================================================================ +FROM scratch AS runtime-amd64 +COPY --from=builder /workspace/ice/target/ice /ice +ENTRYPOINT ["/ice"] + +FROM gcr.io/distroless/base-debian12:nonroot AS runtime-arm64 +COPY --from=builder /workspace/ice/target/ice /ice +ENTRYPOINT ["/ice"] + +# ============================================================================ +# Stage 3: Final stage (architecture-dependent) +# ============================================================================ +FROM runtime-${ARCH} AS final + From 1e06b25eb1a6702ae2906469d0ebf66983ef7f6a Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 23 Nov 2025 10:22:59 -0600 Subject: [PATCH 10/13] Remove autogenerated files. --- .gitignore | 1 + GITHUB_RELEASE_TEMPLATE.md | 34 -- build-native.sh | 95 ----- docker-build-native.sh | 55 --- .../docker-compose/test-ssl-verification.md | 186 ++++++++++ examples/scratch/output.json | 0 ice-rest-catalog/SCENARIO_TESTING.md | 344 ++++++++++++++++++ ice/Dockerfile.native | 52 --- ice/Dockerfile.native-amd64-static | 2 +- ice/Dockerfile.native-builder | 91 ----- 10 files changed, 532 insertions(+), 328 deletions(-) delete mode 100644 GITHUB_RELEASE_TEMPLATE.md delete mode 100755 build-native.sh delete mode 100755 docker-build-native.sh create mode 100644 examples/docker-compose/test-ssl-verification.md create mode 100644 examples/scratch/output.json create mode 100644 ice-rest-catalog/SCENARIO_TESTING.md delete mode 100644 ice/Dockerfile.native delete mode 100644 ice/Dockerfile.native-builder diff --git a/.gitignore b/.gitignore index 52ecf198..4ae9503a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ target /demo/data /demo/clickhouse-udfs.xml diff --git a/GITHUB_RELEASE_TEMPLATE.md b/GITHUB_RELEASE_TEMPLATE.md deleted file mode 100644 index fe7ff576..00000000 --- a/GITHUB_RELEASE_TEMPLATE.md +++ /dev/null @@ -1,34 +0,0 @@ -## Installation - -### ice - -```sh -curl -sSL https://github.com/altinity/ice/releases/download/REPLACE_WITH_TAG/ice-REPLACE_WITH_VER \ - -o ice && chmod a+x ice && sudo mv ice /usr/local/bin/ -``` - -#### Docker - - - -- `altinity/ice:REPLACE_WITH_VER` -- `altinity/ice:debug-REPLACE_WITH_VER` - -> `debug-*` images contain busybox shell. - -### ice-rest-catalog - -```sh -curl -sSL https://github.com/altinity/ice/releases/download/REPLACE_WITH_TAG/ice-rest-catalog-REPLACE_WITH_VER \ - -o ice-rest-catalog && chmod a+x ice-rest-catalog && sudo mv ice-rest-catalog /usr/local/bin/ -``` - -#### Docker - - - -- `altinity/ice-rest-catalog:REPLACE_WITH_VER` -- `altinity/ice-rest-catalog:debug-REPLACE_WITH_VER` -- `altinity/ice-rest-catalog:debug-with-ice-REPLACE_WITH_VER` - -> `debug-*` images contain busybox shell. diff --git a/build-native.sh b/build-native.sh deleted file mode 100755 index 1b6c26c0..00000000 --- a/build-native.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# Build native images for ice CLI -# Supports: amd64 (static with musl), arm64 (dynamic) - -set -e - -VERSION="${VERSION:-0.0.0-SNAPSHOT}" -ARCH="${ARCH:-$(uname -m)}" - -# Convert architecture names -case "$ARCH" in - x86_64|amd64) - ARCH="amd64" - PROFILE="native-amd64-static" - ;; - aarch64|arm64) - ARCH="arm64" - PROFILE="native-arm64" - ;; - *) - echo "Unsupported architecture: $ARCH" - echo "Supported: x86_64/amd64, aarch64/arm64" - exit 1 - ;; -esac - -echo "============================================" -echo "Building native ice binary" -echo "Architecture: $ARCH" -echo "Profile: $PROFILE" -echo "Version: $VERSION" -echo "============================================" -echo "" - -# Check for musl-gcc on amd64 builds -if [ "$ARCH" = "amd64" ]; then - if ! command -v x86_64-linux-musl-gcc &> /dev/null && ! command -v musl-gcc &> /dev/null; then - echo " WARNING: musl compiler not found!" - echo "" - echo "For static builds on amd64, you need musl-tools installed." - echo "" - echo "Install it with:" - echo " Ubuntu/Debian: sudo apt-get install musl-tools" - echo " Fedora/RHEL: sudo dnf install musl-gcc musl-libc-static" - echo " Alpine: apk add musl-dev gcc" - echo "" - echo "Or build without static linking using the 'native' profile:" - echo " mvn -Pnative -pl ice clean package -Dmaven.test.skip=true" - echo "" - read -p "Continue anyway? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - else - echo "✓ musl compiler found" - fi -fi -echo "" - -# Set version -./mvnw -am -pl ice versions:set -DnewVersion=${VERSION} - -# Build native image -echo "Building native image..." -./mvnw -Pno-check -P${PROFILE} -pl ice clean package -Dmaven.test.skip=true - -echo "" -echo "============================================" -echo "✓ Native binary built successfully!" -echo "============================================" -echo "" -echo "Binary location: ice/target/ice" -echo "" -echo "Test the binary:" -echo " ./ice/target/ice --version" -echo " ./ice/target/ice check" -echo "" - -# Show binary info -if [ -f "ice/target/ice" ]; then - ls -lh ice/target/ice - file ice/target/ice || true - - if [ "$ARCH" = "amd64" ]; then - echo "" - echo "Static binary (no dependencies):" - ldd ice/target/ice 2>&1 || echo " ✓ Statically linked (expected for amd64)" - else - echo "" - echo "Dynamic binary dependencies:" - ldd ice/target/ice || true - fi -fi - diff --git a/docker-build-native.sh b/docker-build-native.sh deleted file mode 100755 index f300e81c..00000000 --- a/docker-build-native.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Build ICE native images using Docker -# Usage: ./docker-build-native.sh [amd64|arm64|both] - -set -euo pipefail - -ARCH="${1:-$(uname -m)}" - -# Normalize architecture names -case "$ARCH" in - x86_64|amd64) - BUILD_ARCH="amd64" - PLATFORM="linux/amd64" - ;; - aarch64|arm64) - BUILD_ARCH="arm64" - PLATFORM="linux/arm64" - ;; - both) - echo "Building for both architectures..." - $0 amd64 - $0 arm64 - exit 0 - ;; - *) - echo "Unsupported architecture: $ARCH" - echo "Usage: $0 [amd64|arm64|both]" - exit 1 - ;; -esac - -echo "========================================" -echo "Building ICE native image for $BUILD_ARCH" -echo "Platform: $PLATFORM" -echo "========================================" - -# Build the Docker image -docker build \ - --platform="$PLATFORM" \ - --build-arg ARCH="$BUILD_ARCH" \ - -f ice/Dockerfile.native-builder \ - -t ice-native-builder:"$BUILD_ARCH" \ - . - -# Extract the binary -CONTAINER_ID=$(docker create ice-native-builder:"$BUILD_ARCH") -docker cp "$CONTAINER_ID":/ice "./ice-$BUILD_ARCH" -docker rm "$CONTAINER_ID" - -echo "" -echo "✓ Native binary created: ./ice-$BUILD_ARCH" -echo "" -echo "Test it with:" -echo " ./ice-$BUILD_ARCH --help" - diff --git a/examples/docker-compose/test-ssl-verification.md b/examples/docker-compose/test-ssl-verification.md new file mode 100644 index 00000000..72ca27ea --- /dev/null +++ b/examples/docker-compose/test-ssl-verification.md @@ -0,0 +1,186 @@ +# Testing SSL Certificate Verification Skip + +This guide shows how to test the `--insecure` / `--ssl-no-verify` flag with self-signed certificates. + +## Option 1: Quick Test with a Self-Signed HTTPS Server + +### 1. Generate a self-signed certificate + +```bash +# Generate a self-signed certificate for testing +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" +``` + +### 2. Set up an HTTPS proxy to ice-rest-catalog + +Create a simple nginx config to add SSL in front of the catalog: + +**nginx-ssl.conf**: +```nginx +events { + worker_connections 1024; +} + +http { + server { + listen 5443 ssl; + server_name localhost; + + ssl_certificate /etc/nginx/cert.pem; + ssl_certificate_key /etc/nginx/key.pem; + + location / { + proxy_pass http://ice-rest-catalog:5000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} +``` + +**docker-compose.ssl-test.yaml**: +```yaml +services: + nginx-ssl: + image: nginx:alpine + ports: + - "5443:5443" + volumes: + - ./cert.pem:/etc/nginx/cert.pem:ro + - ./key.pem:/etc/nginx/key.pem:ro + - ./nginx-ssl.conf:/etc/nginx/nginx.conf:ro + depends_on: + - ice-rest-catalog +``` + +### 3. Start the services + +```bash +# Start the regular docker-compose setup +docker-compose up -d + +# Start the SSL proxy +docker-compose -f docker-compose.ssl-test.yaml up -d +``` + +### 4. Test the ice CLI + +First, format and build the code: +```bash +cd /home/kanthi/Documents/GITHUB/ALTINITY/ice +mvn com.spotify.fmt:fmt-maven-plugin:format -pl ice +mvn clean install -pl ice -am -DskipTests +``` + +Create a test config file `.ice-ssl-test.yaml`: +```yaml +uri: https://localhost:5443 +bearerToken: foo +warehouse: s3://bucket1 +sslVerify: false # Config file option +s3: + endpoint: http://localhost:8999 + pathStyleAccess: true + accessKeyID: miniouser + secretAccessKey: miniopassword + region: minio +``` + +**Test 1: Without --insecure flag (should fail with SSL error)**: +```bash +./ice/target/ice-jar -c .ice-ssl-test.yaml check +# Expected: SSL certificate verification error +``` + +**Test 2: With --insecure flag (should succeed)**: +```bash +./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure check +# Expected: OK +``` + +**Test 3: Using config file sslVerify option**: +With `sslVerify: false` in the config file: +```bash +./ice/target/ice-jar -c .ice-ssl-test.yaml check +# Expected: OK (with warning about disabled SSL verification) +``` + +**Test 4: Test other commands**: +```bash +# List namespaces +./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure describe + +# Create a namespace +./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure create-namespace test_ns + +# Create a table +./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure create-table test_ns.test_table \ + --schema '[{"name":"id","type":"int"},{"name":"name","type":"string"}]' +``` + +## Option 2: Test with Public Self-Signed Certificate Server + +You can also test against a public service with a self-signed certificate: + +```bash +# This will fail due to self-signed certificate +./ice/target/ice-jar -c config.yaml check + +# This will work with --insecure +./ice/target/ice-jar -c config.yaml --insecure check +``` + +## Option 3: Test with Internal CA Certificate + +If you have an internal CA, you can test the custom CA bundle feature: + +```yaml +uri: https://internal-catalog.company.local:5443 +caCrt: | + -----BEGIN CERTIFICATE----- + MIIDXTCCAkWgAwIBAgIJAKJ... + -----END CERTIFICATE----- +bearerToken: foo +``` + +This should work without `--insecure` because the CA certificate is provided. + +## Expected Behavior + +### Without SSL verification skip: +- ❌ Fails with certificate verification errors for self-signed certificates +- ✅ Works with valid certificates from trusted CAs +- ✅ Works when custom CA certificate is provided via `caCrt` + +### With SSL verification skip (`--insecure` or `sslVerify: false`): +- ⚠️ Displays warning: "SSL certificate verification is DISABLED. This is insecure and should only be used for development." +- ✅ Works with any certificate (self-signed, expired, wrong hostname, etc.) +- ⚠️ **INSECURE**: Should only be used for development/testing + +## Command-line Flags + +Two equivalent options: +- `--insecure` - Shorter, commonly used in tools like curl +- `--ssl-no-verify` - More explicit about what it does + +Both flags: +- Override the config file `sslVerify` setting +- Apply to all subcommands (check, describe, scan, insert, etc.) +- Display a warning message when SSL verification is disabled + +## Config File Option + +```yaml +sslVerify: false # Skip SSL certificate verification +``` + +Note: The command-line flag takes precedence over the config file option. + + + + + + diff --git a/examples/scratch/output.json b/examples/scratch/output.json new file mode 100644 index 00000000..e69de29b diff --git a/ice-rest-catalog/SCENARIO_TESTING.md b/ice-rest-catalog/SCENARIO_TESTING.md new file mode 100644 index 00000000..af54e58e --- /dev/null +++ b/ice-rest-catalog/SCENARIO_TESTING.md @@ -0,0 +1,344 @@ +# Scenario-Based Testing Framework + +This document describes the new scenario-based testing framework for ICE REST Catalog integration tests. + +## Overview + +The scenario-based testing framework provides a scalable, data-driven approach to integration testing. Instead of hardcoding test scenarios in Java test methods, each scenario is defined in its own directory with: + +- Configuration files (scenario.yaml) +- Input data (parquet files) +- Executable test scripts (run.sh.tmpl, verify.sh.tmpl) + +This approach makes it much easier to: +- Add new test scenarios without writing Java code +- Share test scenarios across teams +- Test complex workflows with multiple steps +- Provision cloud resources dynamically (future feature) + +## Directory Structure + +``` +ice-rest-catalog/src/test/resources/scenarios/ +├── README.md # Documentation for scenario structure +├── basic-operations/ # Example scenario +│ ├── scenario.yaml # Scenario configuration +│ ├── run.sh.tmpl # Main test script (templated) +│ └── verify.sh.tmpl # Optional verification script +├── insert-scan/ +│ ├── scenario.yaml +│ ├── run.sh.tmpl +│ ├── verify.sh.tmpl +│ └── input.parquet # Test data +└── insert-partitioned/ + ├── scenario.yaml + ├── run.sh.tmpl + ├── verify.sh.tmpl + └── input.parquet +``` + +## Key Components + +### 1. ScenarioConfig.java +Java class that represents the scenario.yaml configuration file. Supports: +- Scenario metadata (name, description) +- Catalog configuration overrides +- Environment variables for scripts +- Cloud resource specifications (for future provisioning) +- Test phases + +### 2. ScenarioTestRunner.java +Core test execution engine that: +- Discovers scenario directories +- Loads scenario configurations +- Processes template variables in scripts +- Executes scripts and captures output +- Reports results + +### 3. ScenarioBasedIT.java +TestNG parameterized test class that: +- Automatically discovers all scenarios +- Runs each scenario as a separate test case +- Provides test infrastructure (MinIO, REST catalog) +- Injects template variables (CLI config, endpoints, etc.) + +### 4. RESTCatalogTestBase.java (Updated) +Base test class that provides: +- MinIO container setup/teardown +- REST catalog server setup/teardown +- Helper methods for scenarios (getMinioEndpoint, getCatalogUri, etc.) + +## How It Works + +### 1. Test Discovery +When you run `mvn test -Dtest=ScenarioBasedIT`, the test framework: + +1. Scans `src/test/resources/scenarios/` for directories +2. Each directory is considered a scenario +3. Creates a parameterized test for each scenario + +### 2. Test Execution +For each scenario: + +1. **Setup Phase**: + - Start MinIO container + - Start ICE REST catalog server + - Create temporary CLI config file + - Build template variables + +2. **Execution Phase**: + - Load scenario.yaml configuration + - Process run.sh.tmpl with template variables + - Execute the processed script + - Capture stdout, stderr, and exit code + +3. **Verification Phase** (optional): + - Process verify.sh.tmpl with template variables + - Execute verification script + - Capture results + +4. **Teardown Phase**: + - Stop REST catalog server + - Stop MinIO container + - Clean up temporary files + +### 3. Template Processing +Scripts can use template variables in the format `{{VARIABLE_NAME}}`: + +```bash +# Available template variables: +# {{ICE_CLI}} - Path to ice CLI executable +# {{CLI_CONFIG}} - Path to temporary CLI config file +# {{MINIO_ENDPOINT}} - MinIO endpoint URL (e.g., http://localhost:9000) +# {{CATALOG_URI}} - REST catalog URI (e.g., http://localhost:8080) +# {{SCENARIO_DIR}} - Absolute path to scenario directory + +# Environment variables from scenario.yaml are also available +{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} +``` + +## Creating a New Scenario + +1. **Create scenario directory**: + ```bash + mkdir ice-rest-catalog/src/test/resources/scenarios/my-new-test + ``` + +2. **Create scenario.yaml**: + ```yaml + name: "My New Test" + description: "Description of what this test does" + + env: + NAMESPACE_NAME: "my_test_ns" + TABLE_NAME: "my_test_ns.my_table" + + cloudResources: + s3: + buckets: + - "test-bucket" + ``` + +3. **Create run.sh.tmpl**: + ```bash + #!/bin/bash + set -e + + echo "Running my test..." + + # Create namespace + {{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} + + # ... more test steps ... + + # Cleanup + {{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME} + + echo "Test completed successfully" + ``` + +4. **Add test data** (optional): + ```bash + cp example-data.parquet ice-rest-catalog/src/test/resources/scenarios/my-new-test/input.parquet + ``` + +5. **Run the test**: + ```bash + mvn test -Dtest=ScenarioBasedIT#testScenario[my-new-test] + ``` + +## Running Tests + +### Run all scenarios: +```bash +cd ice-rest-catalog +mvn test -Dtest=ScenarioBasedIT +``` + +### Run a specific scenario: +```bash +mvn test -Dtest=ScenarioBasedIT#testScenario[basic-operations] +``` + +### Run scenarios matching a pattern: +```bash +mvn test -Dtest=ScenarioBasedIT#testScenario[insert*] +``` + +## Benefits Over Previous Approach + +### Before (Hardcoded Java Tests): +```java +@Test +public void testScenario1() throws Exception { + File config = createTempCliConfig(); + new CommandLine(Main.class).execute("--config", config.getAbsolutePath(), + "create-namespace", "test_ns"); + new CommandLine(Main.class).execute("--config", config.getAbsolutePath(), + "delete-namespace", "test_ns"); + // ... repeat for each scenario +} + +@Test +public void testScenario2() throws Exception { + // ... duplicate setup code ... +} +``` + +**Problems**: +- Lots of duplicated code +- Hard to add new scenarios +- Difficult to test complex workflows +- No separation of test data and test logic +- Hard to share tests with non-Java developers + +### After (Scenario-Based Tests): +``` +scenarios/ + scenario1/ + scenario.yaml + run.sh.tmpl + scenario2/ + scenario.yaml + run.sh.tmpl + input.parquet +``` + +**Benefits**: +- ✅ No code duplication +- ✅ Easy to add new scenarios (just add a directory) +- ✅ Can test complex workflows with multiple steps +- ✅ Clear separation of test data and logic +- ✅ Scripts can be shared and run independently +- ✅ Can be used by QA engineers without Java knowledge +- ✅ Future: automatic cloud resource provisioning + +## Example Scenarios + +### 1. basic-operations +Tests fundamental catalog operations: +- Create namespace +- List namespaces +- Delete namespace + +### 2. insert-scan +Tests data insertion and retrieval: +- Create namespace +- Insert data from parquet file +- Scan table +- Verify data contents +- Cleanup + +### 3. insert-partitioned +Tests partitioned table creation: +- Create namespace +- Insert data with partition specification +- Verify table was created +- Cleanup + +## Future Enhancements + +1. **Cloud Resource Provisioning**: + - Automatically create S3 buckets, SQS queues, etc. based on scenario.yaml + - Use AWS CloudFormation or Terraform templates + - Clean up resources after test completion + +2. **Parallel Execution**: + - Run independent scenarios in parallel + - Speed up test suite execution + +3. **Test Data Generation**: + - Generate synthetic test data based on schema specifications + - Support for various data formats (Avro, ORC, etc.) + +4. **Scenario Composition**: + - Reuse common scenario steps + - Create scenario libraries + +5. **Performance Testing**: + - Add performance benchmarks to scenarios + - Track performance over time + +## Migration Guide + +To migrate existing hardcoded tests to scenarios: + +1. Identify the test scenario (e.g., "insert with partitioning") +2. Create a scenario directory +3. Extract configuration into scenario.yaml +4. Convert Java test code to bash script in run.sh.tmpl +5. Add test data files if needed +6. Delete the old Java test method + +Example migration: + +**Before** (RESTCatalogInsertIT.java): +```java +@Test +public void testInsertCommand() throws Exception { + File config = createTempCliConfig(); + String namespace = "test_insert"; + new CommandLine(Main.class).execute("--config", config, + "create-namespace", namespace); + // ... more code ... +} +``` + +**After** (scenarios/insert-scan/): +``` +scenario.yaml # Configuration +run.sh.tmpl # Test logic +input.parquet # Test data +``` + +## Troubleshooting + +### Scenario not discovered +- Ensure scenario directory is in `src/test/resources/scenarios/` +- Check that `scenario.yaml` exists in the directory +- Directory name should not start with `.` + +### Script execution fails +- Check script has correct shebang (`#!/bin/bash`) +- Verify template variables are correctly specified +- Check file paths are relative to {{SCENARIO_DIR}} +- Review script output in test logs + +### ICE CLI not found +- Ensure `ice/target/ice-jar` is built: `mvn -am -pl ice package` +- Or ensure `.bin/local-ice` script exists +- Check that ICE_CLI template variable is correctly set + +## Additional Resources + +- [Scenarios README](src/test/resources/scenarios/README.md) - Detailed scenario structure documentation +- [ICE CLI Documentation](../ice/README.md) - CLI command reference +- [TestNG Documentation](https://testng.org/doc/) - TestNG testing framework + + + + + + + + diff --git a/ice/Dockerfile.native b/ice/Dockerfile.native deleted file mode 100644 index 52caa1e3..00000000 --- a/ice/Dockerfile.native +++ /dev/null @@ -1,52 +0,0 @@ -# Multi-arch native image build for ice -# Supports: amd64 (with static musl), arm64 (dynamic) -ARG BASE_IMAGE_TAG="nonroot" -ARG VERSION="0.0.0-SNAPSHOT" -ARG TARGETARCH - -FROM --platform=$BUILDPLATFORM ghcr.io/graalvm/native-image-community:21.0.2 AS build - -WORKDIR /app - -# Install musl for static linking (amd64) -RUN microdnf install -y musl-devel || true - -# Cache maven dependencies for faster re-builds -COPY .mvn/ .mvn/ -COPY mvnw . -COPY pom.xml . -COPY ice/pom.xml ice/pom.xml -COPY ice-rest-catalog/pom.xml ice-rest-catalog/pom.xml -RUN ./mvnw -am -pl ice dependency:go-offline - -ARG VERSION -ARG TARGETARCH - -COPY . . - -RUN ./mvnw -am -pl ice versions:set -DnewVersion=${VERSION} - -# Build with appropriate profile based on architecture -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - echo "Building static native image for amd64 with musl..."; \ - ./mvnw -Pno-check -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true; \ - elif [ "$TARGETARCH" = "arm64" ]; then \ - echo "Building native image for arm64..."; \ - ./mvnw -Pno-check -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true; \ - else \ - echo "Building native image for $TARGETARCH..."; \ - ./mvnw -Pno-check -Pnative -pl ice clean package -Dmaven.test.skip=true; \ - fi - -# Use scratch for amd64 (static), distroless for arm64 (dynamic) -FROM scratch AS final-amd64 -COPY --from=build /app/ice/target/ice /usr/local/bin/ice -ENTRYPOINT ["/usr/local/bin/ice"] - -FROM gcr.io/distroless/base-debian12:${BASE_IMAGE_TAG} AS final-arm64 -COPY --from=build /app/ice/target/ice /usr/local/bin/ice -ENTRYPOINT ["/usr/local/bin/ice"] - -# Select appropriate final stage based on architecture -FROM final-${TARGETARCH} AS final - diff --git a/ice/Dockerfile.native-amd64-static b/ice/Dockerfile.native-amd64-static index dd0c2587..e8b4f271 100644 --- a/ice/Dockerfile.native-amd64-static +++ b/ice/Dockerfile.native-amd64-static @@ -1,5 +1,5 @@ # Dockerfile for building ICE native image (amd64, static with musl) -# Usage: docker build -f ice/Dockerfile.native-amd64-static -t ice-native:amd64-static . +# Usage: docker build -f ice/Dockerfile.native-amd64-static -t ice-native:amd64 . FROM ghcr.io/graalvm/native-image-community:21-muslib AS builder diff --git a/ice/Dockerfile.native-builder b/ice/Dockerfile.native-builder deleted file mode 100644 index 0a044e6d..00000000 --- a/ice/Dockerfile.native-builder +++ /dev/null @@ -1,91 +0,0 @@ -# Multi-stage Dockerfile for building ICE native images -# Supports both amd64 (with static musl) and arm64 (dynamic) - -ARG ARCH=amd64 - -# ============================================================================ -# Stage 1: Build stage with all dependencies -# ============================================================================ -FROM ghcr.io/graalvm/native-image-community:21 AS builder - -# Install build dependencies -RUN microdnf install -y \ - findutils \ - tar \ - gzip \ - wget \ - && microdnf clean all - -# Detect architecture and install appropriate dependencies -RUN ARCH=$(uname -m) && \ - if [ "$ARCH" = "x86_64" ]; then \ - echo "Installing musl toolchain and zlib for amd64 static builds..."; \ - # Download and install musl-gcc - wget -q https://musl.libc.org/releases/musl-1.2.5.tar.gz && \ - tar -xzf musl-1.2.5.tar.gz && \ - cd musl-1.2.5 && \ - ./configure --prefix=/usr/local/musl --disable-shared && \ - make -j$(nproc) && \ - make install && \ - cd .. && \ - rm -rf musl-1.2.5 musl-1.2.5.tar.gz && \ - # Download and install zlib for musl - wget -q https://zlib.net/zlib-1.3.1.tar.gz && \ - tar -xzf zlib-1.3.1.tar.gz && \ - cd zlib-1.3.1 && \ - CC=/usr/local/musl/bin/musl-gcc ./configure --prefix=/usr/local/musl --static && \ - make -j$(nproc) && \ - make install && \ - cd .. && \ - rm -rf zlib-1.3.1 zlib-1.3.1.tar.gz && \ - # Create symlink for native-image to find musl-gcc - ln -sf /usr/local/musl/bin/musl-gcc /usr/local/bin/x86_64-linux-musl-gcc; \ - else \ - echo "ARM64 detected - using dynamic linking (no musl required)"; \ - fi - -WORKDIR /workspace - -# Copy license header file (required by license-maven-plugin) -COPY apache-header.txt . - -# Copy Maven wrapper and pom files first (for better caching) -COPY mvnw . -COPY .mvn .mvn -COPY pom.xml . -COPY ice/pom.xml ice/ -COPY ice-rest-catalog/pom.xml ice-rest-catalog/ - -# Download dependencies (cached layer) -RUN ./mvnw dependency:go-offline -pl ice || true - -# Copy source code -COPY ice/src ice/src - -# Build native image based on architecture -RUN ARCH=$(uname -m) && \ - if [ "$ARCH" = "x86_64" ]; then \ - echo "Building static native image for amd64 with musl..."; \ - export PATH="/usr/local/musl/bin:$PATH" && \ - ./mvnw -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true; \ - else \ - echo "Building dynamic native image for arm64..."; \ - ./mvnw -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true; \ - fi - -# ============================================================================ -# Stage 2: Runtime image (minimal) -# ============================================================================ -FROM scratch AS runtime-amd64 -COPY --from=builder /workspace/ice/target/ice /ice -ENTRYPOINT ["/ice"] - -FROM gcr.io/distroless/base-debian12:nonroot AS runtime-arm64 -COPY --from=builder /workspace/ice/target/ice /ice -ENTRYPOINT ["/ice"] - -# ============================================================================ -# Stage 3: Final stage (architecture-dependent) -# ============================================================================ -FROM runtime-${ARCH} AS final - From 39319c43ef72f22c53bb2fa07d7e2a85610ed681 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 23 Nov 2025 10:27:48 -0600 Subject: [PATCH 11/13] Removed accidentally committed files. --- .../docker-compose/test-ssl-verification.md | 186 ---------- examples/scratch/output.json | 0 ice-rest-catalog/SCENARIO_TESTING.md | 344 ------------------ 3 files changed, 530 deletions(-) delete mode 100644 examples/docker-compose/test-ssl-verification.md delete mode 100644 examples/scratch/output.json delete mode 100644 ice-rest-catalog/SCENARIO_TESTING.md diff --git a/examples/docker-compose/test-ssl-verification.md b/examples/docker-compose/test-ssl-verification.md deleted file mode 100644 index 72ca27ea..00000000 --- a/examples/docker-compose/test-ssl-verification.md +++ /dev/null @@ -1,186 +0,0 @@ -# Testing SSL Certificate Verification Skip - -This guide shows how to test the `--insecure` / `--ssl-no-verify` flag with self-signed certificates. - -## Option 1: Quick Test with a Self-Signed HTTPS Server - -### 1. Generate a self-signed certificate - -```bash -# Generate a self-signed certificate for testing -openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes \ - -subj "/CN=localhost" \ - -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" -``` - -### 2. Set up an HTTPS proxy to ice-rest-catalog - -Create a simple nginx config to add SSL in front of the catalog: - -**nginx-ssl.conf**: -```nginx -events { - worker_connections 1024; -} - -http { - server { - listen 5443 ssl; - server_name localhost; - - ssl_certificate /etc/nginx/cert.pem; - ssl_certificate_key /etc/nginx/key.pem; - - location / { - proxy_pass http://ice-rest-catalog:5000; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - } -} -``` - -**docker-compose.ssl-test.yaml**: -```yaml -services: - nginx-ssl: - image: nginx:alpine - ports: - - "5443:5443" - volumes: - - ./cert.pem:/etc/nginx/cert.pem:ro - - ./key.pem:/etc/nginx/key.pem:ro - - ./nginx-ssl.conf:/etc/nginx/nginx.conf:ro - depends_on: - - ice-rest-catalog -``` - -### 3. Start the services - -```bash -# Start the regular docker-compose setup -docker-compose up -d - -# Start the SSL proxy -docker-compose -f docker-compose.ssl-test.yaml up -d -``` - -### 4. Test the ice CLI - -First, format and build the code: -```bash -cd /home/kanthi/Documents/GITHUB/ALTINITY/ice -mvn com.spotify.fmt:fmt-maven-plugin:format -pl ice -mvn clean install -pl ice -am -DskipTests -``` - -Create a test config file `.ice-ssl-test.yaml`: -```yaml -uri: https://localhost:5443 -bearerToken: foo -warehouse: s3://bucket1 -sslVerify: false # Config file option -s3: - endpoint: http://localhost:8999 - pathStyleAccess: true - accessKeyID: miniouser - secretAccessKey: miniopassword - region: minio -``` - -**Test 1: Without --insecure flag (should fail with SSL error)**: -```bash -./ice/target/ice-jar -c .ice-ssl-test.yaml check -# Expected: SSL certificate verification error -``` - -**Test 2: With --insecure flag (should succeed)**: -```bash -./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure check -# Expected: OK -``` - -**Test 3: Using config file sslVerify option**: -With `sslVerify: false` in the config file: -```bash -./ice/target/ice-jar -c .ice-ssl-test.yaml check -# Expected: OK (with warning about disabled SSL verification) -``` - -**Test 4: Test other commands**: -```bash -# List namespaces -./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure describe - -# Create a namespace -./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure create-namespace test_ns - -# Create a table -./ice/target/ice-jar -c .ice-ssl-test.yaml --insecure create-table test_ns.test_table \ - --schema '[{"name":"id","type":"int"},{"name":"name","type":"string"}]' -``` - -## Option 2: Test with Public Self-Signed Certificate Server - -You can also test against a public service with a self-signed certificate: - -```bash -# This will fail due to self-signed certificate -./ice/target/ice-jar -c config.yaml check - -# This will work with --insecure -./ice/target/ice-jar -c config.yaml --insecure check -``` - -## Option 3: Test with Internal CA Certificate - -If you have an internal CA, you can test the custom CA bundle feature: - -```yaml -uri: https://internal-catalog.company.local:5443 -caCrt: | - -----BEGIN CERTIFICATE----- - MIIDXTCCAkWgAwIBAgIJAKJ... - -----END CERTIFICATE----- -bearerToken: foo -``` - -This should work without `--insecure` because the CA certificate is provided. - -## Expected Behavior - -### Without SSL verification skip: -- ❌ Fails with certificate verification errors for self-signed certificates -- ✅ Works with valid certificates from trusted CAs -- ✅ Works when custom CA certificate is provided via `caCrt` - -### With SSL verification skip (`--insecure` or `sslVerify: false`): -- ⚠️ Displays warning: "SSL certificate verification is DISABLED. This is insecure and should only be used for development." -- ✅ Works with any certificate (self-signed, expired, wrong hostname, etc.) -- ⚠️ **INSECURE**: Should only be used for development/testing - -## Command-line Flags - -Two equivalent options: -- `--insecure` - Shorter, commonly used in tools like curl -- `--ssl-no-verify` - More explicit about what it does - -Both flags: -- Override the config file `sslVerify` setting -- Apply to all subcommands (check, describe, scan, insert, etc.) -- Display a warning message when SSL verification is disabled - -## Config File Option - -```yaml -sslVerify: false # Skip SSL certificate verification -``` - -Note: The command-line flag takes precedence over the config file option. - - - - - - diff --git a/examples/scratch/output.json b/examples/scratch/output.json deleted file mode 100644 index e69de29b..00000000 diff --git a/ice-rest-catalog/SCENARIO_TESTING.md b/ice-rest-catalog/SCENARIO_TESTING.md deleted file mode 100644 index af54e58e..00000000 --- a/ice-rest-catalog/SCENARIO_TESTING.md +++ /dev/null @@ -1,344 +0,0 @@ -# Scenario-Based Testing Framework - -This document describes the new scenario-based testing framework for ICE REST Catalog integration tests. - -## Overview - -The scenario-based testing framework provides a scalable, data-driven approach to integration testing. Instead of hardcoding test scenarios in Java test methods, each scenario is defined in its own directory with: - -- Configuration files (scenario.yaml) -- Input data (parquet files) -- Executable test scripts (run.sh.tmpl, verify.sh.tmpl) - -This approach makes it much easier to: -- Add new test scenarios without writing Java code -- Share test scenarios across teams -- Test complex workflows with multiple steps -- Provision cloud resources dynamically (future feature) - -## Directory Structure - -``` -ice-rest-catalog/src/test/resources/scenarios/ -├── README.md # Documentation for scenario structure -├── basic-operations/ # Example scenario -│ ├── scenario.yaml # Scenario configuration -│ ├── run.sh.tmpl # Main test script (templated) -│ └── verify.sh.tmpl # Optional verification script -├── insert-scan/ -│ ├── scenario.yaml -│ ├── run.sh.tmpl -│ ├── verify.sh.tmpl -│ └── input.parquet # Test data -└── insert-partitioned/ - ├── scenario.yaml - ├── run.sh.tmpl - ├── verify.sh.tmpl - └── input.parquet -``` - -## Key Components - -### 1. ScenarioConfig.java -Java class that represents the scenario.yaml configuration file. Supports: -- Scenario metadata (name, description) -- Catalog configuration overrides -- Environment variables for scripts -- Cloud resource specifications (for future provisioning) -- Test phases - -### 2. ScenarioTestRunner.java -Core test execution engine that: -- Discovers scenario directories -- Loads scenario configurations -- Processes template variables in scripts -- Executes scripts and captures output -- Reports results - -### 3. ScenarioBasedIT.java -TestNG parameterized test class that: -- Automatically discovers all scenarios -- Runs each scenario as a separate test case -- Provides test infrastructure (MinIO, REST catalog) -- Injects template variables (CLI config, endpoints, etc.) - -### 4. RESTCatalogTestBase.java (Updated) -Base test class that provides: -- MinIO container setup/teardown -- REST catalog server setup/teardown -- Helper methods for scenarios (getMinioEndpoint, getCatalogUri, etc.) - -## How It Works - -### 1. Test Discovery -When you run `mvn test -Dtest=ScenarioBasedIT`, the test framework: - -1. Scans `src/test/resources/scenarios/` for directories -2. Each directory is considered a scenario -3. Creates a parameterized test for each scenario - -### 2. Test Execution -For each scenario: - -1. **Setup Phase**: - - Start MinIO container - - Start ICE REST catalog server - - Create temporary CLI config file - - Build template variables - -2. **Execution Phase**: - - Load scenario.yaml configuration - - Process run.sh.tmpl with template variables - - Execute the processed script - - Capture stdout, stderr, and exit code - -3. **Verification Phase** (optional): - - Process verify.sh.tmpl with template variables - - Execute verification script - - Capture results - -4. **Teardown Phase**: - - Stop REST catalog server - - Stop MinIO container - - Clean up temporary files - -### 3. Template Processing -Scripts can use template variables in the format `{{VARIABLE_NAME}}`: - -```bash -# Available template variables: -# {{ICE_CLI}} - Path to ice CLI executable -# {{CLI_CONFIG}} - Path to temporary CLI config file -# {{MINIO_ENDPOINT}} - MinIO endpoint URL (e.g., http://localhost:9000) -# {{CATALOG_URI}} - REST catalog URI (e.g., http://localhost:8080) -# {{SCENARIO_DIR}} - Absolute path to scenario directory - -# Environment variables from scenario.yaml are also available -{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} -``` - -## Creating a New Scenario - -1. **Create scenario directory**: - ```bash - mkdir ice-rest-catalog/src/test/resources/scenarios/my-new-test - ``` - -2. **Create scenario.yaml**: - ```yaml - name: "My New Test" - description: "Description of what this test does" - - env: - NAMESPACE_NAME: "my_test_ns" - TABLE_NAME: "my_test_ns.my_table" - - cloudResources: - s3: - buckets: - - "test-bucket" - ``` - -3. **Create run.sh.tmpl**: - ```bash - #!/bin/bash - set -e - - echo "Running my test..." - - # Create namespace - {{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} - - # ... more test steps ... - - # Cleanup - {{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME} - - echo "Test completed successfully" - ``` - -4. **Add test data** (optional): - ```bash - cp example-data.parquet ice-rest-catalog/src/test/resources/scenarios/my-new-test/input.parquet - ``` - -5. **Run the test**: - ```bash - mvn test -Dtest=ScenarioBasedIT#testScenario[my-new-test] - ``` - -## Running Tests - -### Run all scenarios: -```bash -cd ice-rest-catalog -mvn test -Dtest=ScenarioBasedIT -``` - -### Run a specific scenario: -```bash -mvn test -Dtest=ScenarioBasedIT#testScenario[basic-operations] -``` - -### Run scenarios matching a pattern: -```bash -mvn test -Dtest=ScenarioBasedIT#testScenario[insert*] -``` - -## Benefits Over Previous Approach - -### Before (Hardcoded Java Tests): -```java -@Test -public void testScenario1() throws Exception { - File config = createTempCliConfig(); - new CommandLine(Main.class).execute("--config", config.getAbsolutePath(), - "create-namespace", "test_ns"); - new CommandLine(Main.class).execute("--config", config.getAbsolutePath(), - "delete-namespace", "test_ns"); - // ... repeat for each scenario -} - -@Test -public void testScenario2() throws Exception { - // ... duplicate setup code ... -} -``` - -**Problems**: -- Lots of duplicated code -- Hard to add new scenarios -- Difficult to test complex workflows -- No separation of test data and test logic -- Hard to share tests with non-Java developers - -### After (Scenario-Based Tests): -``` -scenarios/ - scenario1/ - scenario.yaml - run.sh.tmpl - scenario2/ - scenario.yaml - run.sh.tmpl - input.parquet -``` - -**Benefits**: -- ✅ No code duplication -- ✅ Easy to add new scenarios (just add a directory) -- ✅ Can test complex workflows with multiple steps -- ✅ Clear separation of test data and logic -- ✅ Scripts can be shared and run independently -- ✅ Can be used by QA engineers without Java knowledge -- ✅ Future: automatic cloud resource provisioning - -## Example Scenarios - -### 1. basic-operations -Tests fundamental catalog operations: -- Create namespace -- List namespaces -- Delete namespace - -### 2. insert-scan -Tests data insertion and retrieval: -- Create namespace -- Insert data from parquet file -- Scan table -- Verify data contents -- Cleanup - -### 3. insert-partitioned -Tests partitioned table creation: -- Create namespace -- Insert data with partition specification -- Verify table was created -- Cleanup - -## Future Enhancements - -1. **Cloud Resource Provisioning**: - - Automatically create S3 buckets, SQS queues, etc. based on scenario.yaml - - Use AWS CloudFormation or Terraform templates - - Clean up resources after test completion - -2. **Parallel Execution**: - - Run independent scenarios in parallel - - Speed up test suite execution - -3. **Test Data Generation**: - - Generate synthetic test data based on schema specifications - - Support for various data formats (Avro, ORC, etc.) - -4. **Scenario Composition**: - - Reuse common scenario steps - - Create scenario libraries - -5. **Performance Testing**: - - Add performance benchmarks to scenarios - - Track performance over time - -## Migration Guide - -To migrate existing hardcoded tests to scenarios: - -1. Identify the test scenario (e.g., "insert with partitioning") -2. Create a scenario directory -3. Extract configuration into scenario.yaml -4. Convert Java test code to bash script in run.sh.tmpl -5. Add test data files if needed -6. Delete the old Java test method - -Example migration: - -**Before** (RESTCatalogInsertIT.java): -```java -@Test -public void testInsertCommand() throws Exception { - File config = createTempCliConfig(); - String namespace = "test_insert"; - new CommandLine(Main.class).execute("--config", config, - "create-namespace", namespace); - // ... more code ... -} -``` - -**After** (scenarios/insert-scan/): -``` -scenario.yaml # Configuration -run.sh.tmpl # Test logic -input.parquet # Test data -``` - -## Troubleshooting - -### Scenario not discovered -- Ensure scenario directory is in `src/test/resources/scenarios/` -- Check that `scenario.yaml` exists in the directory -- Directory name should not start with `.` - -### Script execution fails -- Check script has correct shebang (`#!/bin/bash`) -- Verify template variables are correctly specified -- Check file paths are relative to {{SCENARIO_DIR}} -- Review script output in test logs - -### ICE CLI not found -- Ensure `ice/target/ice-jar` is built: `mvn -am -pl ice package` -- Or ensure `.bin/local-ice` script exists -- Check that ICE_CLI template variable is correctly set - -## Additional Resources - -- [Scenarios README](src/test/resources/scenarios/README.md) - Detailed scenario structure documentation -- [ICE CLI Documentation](../ice/README.md) - CLI command reference -- [TestNG Documentation](https://testng.org/doc/) - TestNG testing framework - - - - - - - - From 6c12bed21075f06ba6209775d7adec184bdbb3a1 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 23 Nov 2025 10:29:25 -0600 Subject: [PATCH 12/13] Added GITHUB_RELEASE_TEMPLATE.md --- GITHUB_RELEASE_TEMPLATE.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 GITHUB_RELEASE_TEMPLATE.md diff --git a/GITHUB_RELEASE_TEMPLATE.md b/GITHUB_RELEASE_TEMPLATE.md new file mode 100644 index 00000000..fe7ff576 --- /dev/null +++ b/GITHUB_RELEASE_TEMPLATE.md @@ -0,0 +1,34 @@ +## Installation + +### ice + +```sh +curl -sSL https://github.com/altinity/ice/releases/download/REPLACE_WITH_TAG/ice-REPLACE_WITH_VER \ + -o ice && chmod a+x ice && sudo mv ice /usr/local/bin/ +``` + +#### Docker + + + +- `altinity/ice:REPLACE_WITH_VER` +- `altinity/ice:debug-REPLACE_WITH_VER` + +> `debug-*` images contain busybox shell. + +### ice-rest-catalog + +```sh +curl -sSL https://github.com/altinity/ice/releases/download/REPLACE_WITH_TAG/ice-rest-catalog-REPLACE_WITH_VER \ + -o ice-rest-catalog && chmod a+x ice-rest-catalog && sudo mv ice-rest-catalog /usr/local/bin/ +``` + +#### Docker + + + +- `altinity/ice-rest-catalog:REPLACE_WITH_VER` +- `altinity/ice-rest-catalog:debug-REPLACE_WITH_VER` +- `altinity/ice-rest-catalog:debug-with-ice-REPLACE_WITH_VER` + +> `debug-*` images contain busybox shell. From cc7f9db69b329d38a36a9ad89963a100caa94556 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 23 Nov 2025 10:39:06 -0600 Subject: [PATCH 13/13] Updated README.md with instructions to build native image. --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e83f3fdf..d9f7f459 100644 --- a/README.md +++ b/README.md @@ -52,16 +52,14 @@ Build standalone native binaries with no Java dependency: sdk env # or ensure Java 21+ and GraalVM are available # For amd64 (static with musl - no dependencies) -./build-native.sh -# Produces: ice/target/ice (static binary) - -# Or use Maven directly mvn -Pnative-amd64-static -pl ice clean package -Dmaven.test.skip=true +For arm64 +mvn -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true -# For ARM64 (dynamic with glibc) -ARCH=arm64 ./build-native.sh -# Or: mvn -Pnative-arm64 -pl ice clean package -Dmaven.test.skip=true +# Docker builds +docker build -f ice/Dockerfile.native-amd64-static -t ice-native:amd64 . +docker build -f ice/Dockerfile.native-arm64 -t ice-native:arm64 . ## License