Skip to content

Commit 967b1c5

Browse files
Aryamanz29Aryamanz29
authored
Merge pull request #778 from atlanhq/APP-9140
 APP-10279 | Publish pyatlan with chainguard golden image for each release
2 parents 544ecbf + bd40ba6 commit 967b1c5

File tree

2 files changed

+194
-83
lines changed

2 files changed

+194
-83
lines changed

‎.github/workflows/pyatlan-chainguard.yaml‎

Lines changed: 40 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,7 @@ jobs:
7373
BUILD_TYPE="release"
7474
PYTHON_VERSION="3.13"
7575
PYATLAN_VERSION=$(cat pyatlan/version.txt)
76-
PYATLAN_BRANCH=""
77-
INSTALL_FROM_GIT="false"
78-
echo "Release trigger detected - using latest versions"
76+
echo "Release trigger detected - building from git tag v${PYATLAN_VERSION}"
7977
else
8078
# Manual dispatch: use provided inputs with defaults
8179
BUILD_TYPE="${{ github.event.inputs.build_type }}"
@@ -86,58 +84,60 @@ jobs:
8684
else
8785
PYTHON_VERSION="${{ github.event.inputs.python_version }}"
8886
fi
89-
90-
# Check if branch is provided (overrides version)
91-
if [ -n "${{ github.event.inputs.pyatlan_branch }}" ]; then
92-
PYATLAN_BRANCH="${{ github.event.inputs.pyatlan_branch }}"
93-
PYATLAN_VERSION="branch-${PYATLAN_BRANCH}"
94-
INSTALL_FROM_GIT="true"
95-
echo "Using pyatlan branch: $PYATLAN_BRANCH"
87+
88+
# Set Pyatlan version (reads from pyatlan/version.txt by default)
89+
# This matches the version in the repository
90+
if [ -z "${{ github.event.inputs.pyatlan_version }}" ]; then
91+
PYATLAN_VERSION=$(cat pyatlan/version.txt)
92+
echo "Using pyatlan version from version.txt: $PYATLAN_VERSION"
9693
else
97-
# Set Pyatlan version (default to version.txt if empty)
98-
if [ -z "${{ github.event.inputs.pyatlan_version }}" ]; then
99-
PYATLAN_VERSION=$(cat pyatlan/version.txt)
100-
echo "Using pyatlan version from version.txt: $PYATLAN_VERSION"
101-
else
102-
PYATLAN_VERSION="${{ github.event.inputs.pyatlan_version }}"
103-
echo "Using specified pyatlan version: $PYATLAN_VERSION"
104-
fi
105-
PYATLAN_BRANCH=""
106-
INSTALL_FROM_GIT="false"
94+
PYATLAN_VERSION="${{ github.event.inputs.pyatlan_version }}"
95+
echo "Using specified pyatlan version: $PYATLAN_VERSION"
96+
fi
97+
98+
# Handle branch input (if provided, will be used as commit SHA or branch name)
99+
if [ -n "${{ github.event.inputs.pyatlan_branch }}" ]; then
100+
echo "Note: Building from branch/commit: ${{ github.event.inputs.pyatlan_branch }}"
101+
echo "This will use PYATLAN_COMMIT_SHA build arg"
107102
fi
108103
fi
109104
110105
echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_ENV
111106
echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
112107
echo "PYATLAN_VERSION=$PYATLAN_VERSION" >> $GITHUB_ENV
113-
echo "PYATLAN_BRANCH=$PYATLAN_BRANCH" >> $GITHUB_ENV
114-
echo "INSTALL_FROM_GIT=$INSTALL_FROM_GIT" >> $GITHUB_ENV
115-
108+
116109
# Set platforms based on build type
117110
if [ "$BUILD_TYPE" = "dev" ]; then
118111
PLATFORMS="linux/amd64"
119112
else
120113
PLATFORMS="linux/amd64,linux/arm64"
121114
fi
122115
echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
123-
124-
# Generate commit hash for dev builds
116+
117+
# Generate commit hash for dev builds (for image tagging only)
125118
COMMIT_HASH=$(git rev-parse --short HEAD)
126119
echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
127-
120+
121+
# Set PYATLAN_COMMIT_SHA only if pyatlan_branch input is provided
122+
# Otherwise, leave empty to let Dockerfile use version tag
123+
if [ -n "${{ github.event.inputs.pyatlan_branch }}" ]; then
124+
echo "PYATLAN_COMMIT_SHA=${{ github.event.inputs.pyatlan_branch }}" >> $GITHUB_ENV
125+
else
126+
echo "PYATLAN_COMMIT_SHA=" >> $GITHUB_ENV
127+
fi
128+
128129
echo "Build parameters:"
129130
echo " - Trigger: ${{ github.event_name }}"
130131
echo " - Build Type: $BUILD_TYPE"
131132
echo " - Python Version: $PYTHON_VERSION"
132-
echo " - Pyatlan Version: $PYATLAN_VERSION"
133-
if [ "$INSTALL_FROM_GIT" = "true" ]; then
134-
echo " - Pyatlan Branch: $PYATLAN_BRANCH"
135-
echo " - Install Method: Git (development build)"
133+
if [ -n "${{ github.event.inputs.pyatlan_branch }}" ]; then
134+
echo " - Pyatlan Source: git commit/branch ${{ github.event.inputs.pyatlan_branch }}"
136135
else
137-
echo " - Install Method: PyPI (stable release)"
136+
echo " - Pyatlan Source: git tag v${PYATLAN_VERSION}"
138137
fi
138+
echo " - Build Method: Multi-stage build from GitHub source"
139139
echo " - Platforms: $PLATFORMS"
140-
echo " - Commit Hash: $COMMIT_HASH"
140+
echo " - Image Tag Suffix: $COMMIT_HASH"
141141
142142
- name: Generate image tags
143143
id: generate-tags
@@ -159,22 +159,6 @@ jobs:
159159
160160
echo "Generated image tag: ghcr.io/atlanhq/pyatlan-chainguard-base:$IMAGE_TAG"
161161
162-
- name: Wait for PyPI availability
163-
if: env.INSTALL_FROM_GIT == 'false' && (github.event.inputs.pyatlan_version != '' || github.event_name == 'release')
164-
uses: nick-fields/retry@v3
165-
with:
166-
max_attempts: 5
167-
timeout_minutes: 3
168-
command: |
169-
echo "Checking if pyatlan==${{ env.PYATLAN_VERSION }} is available on PyPI..."
170-
if pip index versions pyatlan | grep -q "${{ env.PYATLAN_VERSION }}"; then
171-
echo "Package is available on PyPI!"
172-
exit 0
173-
else
174-
echo "Package not available yet. Retrying..."
175-
exit 1
176-
fi
177-
178162
- name: Build and push Docker image
179163
uses: docker/build-push-action@v6
180164
with:
@@ -187,8 +171,7 @@ jobs:
187171
build-args: |
188172
PYTHON_VERSION=${{ env.PYTHON_VERSION }}
189173
PYATLAN_VERSION=${{ env.PYATLAN_VERSION }}
190-
PYATLAN_BRANCH=${{ env.PYATLAN_BRANCH }}
191-
INSTALL_FROM_GIT=${{ env.INSTALL_FROM_GIT }}
174+
PYATLAN_COMMIT_SHA=${{ env.PYATLAN_COMMIT_SHA }}
192175
cache-from: type=gha
193176
cache-to: type=gha,mode=max
194177

@@ -197,20 +180,20 @@ jobs:
197180
echo "## 🐳 Chainguard Docker Image Built Successfully!" >> $GITHUB_STEP_SUMMARY
198181
echo "" >> $GITHUB_STEP_SUMMARY
199182
echo "### Image Details:" >> $GITHUB_STEP_SUMMARY
200-
echo "- **Base Image:** Chainguard" >> $GITHUB_STEP_SUMMARY
183+
echo "- **Base Image:** Chainguard pyatlan-golden:3.11" >> $GITHUB_STEP_SUMMARY
201184
echo "- **Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
202185
echo "- **Build Type:** ${{ env.BUILD_TYPE }}" >> $GITHUB_STEP_SUMMARY
186+
echo "- **Build Method:** Multi-stage from GitHub source (no PyPI)" >> $GITHUB_STEP_SUMMARY
203187
echo "- **Python Version:** ${{ env.PYTHON_VERSION }}" >> $GITHUB_STEP_SUMMARY
204-
echo "- **Pyatlan Version:** ${{ env.PYATLAN_VERSION }}" >> $GITHUB_STEP_SUMMARY
205-
if [ "${{ env.INSTALL_FROM_GIT }}" = "true" ]; then
206-
echo "- **Pyatlan Branch:** ${{ env.PYATLAN_BRANCH }}" >> $GITHUB_STEP_SUMMARY
207-
echo "- **Install Method:** Git (development build)" >> $GITHUB_STEP_SUMMARY
188+
if [ -n "${{ env.PYATLAN_COMMIT_SHA }}" ]; then
189+
echo "- **Pyatlan Source:** Git commit/branch ${{ env.PYATLAN_COMMIT_SHA }}" >> $GITHUB_STEP_SUMMARY
208190
else
209-
echo "- **Install Method:** PyPI (stable release)" >> $GITHUB_STEP_SUMMARY
191+
echo "- **Pyatlan Source:** Git tag v${{ env.PYATLAN_VERSION }}" >> $GITHUB_STEP_SUMMARY
210192
fi
193+
echo "- **Dependencies:** Hardened from Chainguard APK + minimal pip" >> $GITHUB_STEP_SUMMARY
211194
echo "- **Platforms:** ${{ env.PLATFORMS }}" >> $GITHUB_STEP_SUMMARY
212195
if [ "${{ env.BUILD_TYPE }}" = "dev" ]; then
213-
echo "- **Commit Hash:** ${{ env.COMMIT_HASH }}" >> $GITHUB_STEP_SUMMARY
196+
echo "- **Image Tag Suffix:** ${{ env.COMMIT_HASH }}" >> $GITHUB_STEP_SUMMARY
214197
fi
215198
echo "" >> $GITHUB_STEP_SUMMARY
216199
echo "### Available Tag:" >> $GITHUB_STEP_SUMMARY

‎Dockerfile.chainguard‎

Lines changed: 154 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,166 @@
1-
FROM cgr.dev/atlan.com/python:3.11
1+
# ============================================
2+
# Stage 1: Builder - Clone pyatlan from GitHub
3+
# ============================================
4+
FROM cgr.dev/atlan.com/pyatlan-golden:3.11 AS builder
25

3-
# Build arguments for configurable versions
4-
ARG PYTHON_VERSION=3.11
6+
# Build arguments
7+
# PYATLAN_VERSION should be passed via --build-arg
8+
# Default to "latest" if not specified (workflow reads from version.txt)
59
ARG PYATLAN_VERSION=latest
6-
ARG PYATLAN_BRANCH=""
7-
ARG INSTALL_FROM_GIT=false
10+
ARG PYATLAN_COMMIT_SHA=""
811

9-
WORKDIR /app
12+
USER root
13+
WORKDIR /tmp/build
14+
15+
# Install git and build tools
16+
RUN apk add --no-cache git py3.11-build
1017

11-
# Install pyatlan - either from PyPI or git branch
12-
RUN --mount=type=cache,target=/root/.cache/uv \
13-
if [ "$INSTALL_FROM_GIT" = "true" ]; then \
14-
echo "Installing pyatlan from git branch: $PYATLAN_BRANCH"; \
15-
uv pip install --system "git+https://github.com/atlanhq/atlan-python.git@$PYATLAN_BRANCH"; \
18+
# Clone pyatlan from GitHub
19+
RUN echo "=== Cloning pyatlan from GitHub ===" && \
20+
git clone https://github.com/atlanhq/atlan-python.git /tmp/build/atlan-python
21+
22+
# Checkout specific version or commit
23+
RUN cd /tmp/build/atlan-python && \
24+
if [ -n "$PYATLAN_COMMIT_SHA" ]; then \
25+
echo "Checking out commit: $PYATLAN_COMMIT_SHA" && \
26+
git checkout "$PYATLAN_COMMIT_SHA"; \
1627
elif [ "$PYATLAN_VERSION" = "latest" ]; then \
17-
echo "Installing latest pyatlan from PyPI"; \
18-
uv pip install --system pyatlan; \
28+
echo "Using latest from main branch"; \
1929
else \
20-
echo "Installing pyatlan==$PYATLAN_VERSION from PyPI"; \
21-
uv pip install --system pyatlan==$PYATLAN_VERSION; \
22-
fi
23-
24-
# Add build information as labels
25-
RUN if [ "$INSTALL_FROM_GIT" = "true" ]; then \
26-
echo "LABEL pyatlan.source=git" >> /tmp/labels && \
27-
echo "LABEL pyatlan.branch=$PYATLAN_BRANCH" >> /tmp/labels; \
30+
echo "Checking out version: v$PYATLAN_VERSION" && \
31+
(git checkout "v$PYATLAN_VERSION" || git checkout "$PYATLAN_VERSION"); \
32+
fi && \
33+
echo "Commit: $(git rev-parse HEAD)" && \
34+
echo "Version: $(cat pyatlan/version.txt)" && \
35+
# Build wheel in builder stage (smaller than copying full source)
36+
python3 -m build --wheel --outdir /tmp/wheels . && \
37+
# Preserve version.txt before removing source
38+
cp pyatlan/version.txt /tmp/version.txt && \
39+
# Remove everything except the wheel and version.txt
40+
cd /tmp && \
41+
rm -rf /tmp/build/atlan-python && \
42+
echo "Built wheel: $(ls -lh /tmp/wheels/*.whl)" && \
43+
echo "Preserved version: $(cat /tmp/version.txt)"
44+
45+
# ============================================
46+
# Stage 2: Final - Runtime image
47+
# ============================================
48+
FROM cgr.dev/atlan.com/pyatlan-golden:3.11
49+
50+
# Build arguments for labels
51+
ARG PYTHON_VERSION=3.11
52+
ARG PYATLAN_COMMIT_SHA=""
53+
54+
# Copy version.txt from builder and set PYATLAN_VERSION if not provided
55+
COPY --from=builder /tmp/version.txt /tmp/version.txt
56+
ARG PYATLAN_VERSION
57+
RUN echo "=== Debug: Version setup ===" && \
58+
echo "Build arg PYATLAN_VERSION: '$PYATLAN_VERSION'" && \
59+
echo "Contents of version.txt: '$(cat /tmp/version.txt)'" && \
60+
if [ -z "$PYATLAN_VERSION" ]; then \
61+
PYATLAN_VERSION=$(cat /tmp/version.txt | tr -d '\n\r'); \
62+
echo "Using version from file: $PYATLAN_VERSION"; \
2863
else \
29-
echo "LABEL pyatlan.source=pypi" >> /tmp/labels && \
30-
echo "LABEL pyatlan.version=$PYATLAN_VERSION" >> /tmp/labels; \
64+
echo "Using build arg version: $PYATLAN_VERSION"; \
3165
fi && \
32-
echo "LABEL python.version=$PYTHON_VERSION" >> /tmp/labels
66+
echo "Final PYATLAN_VERSION: $PYATLAN_VERSION" && \
67+
echo "$PYATLAN_VERSION" > /tmp/final_version.txt && \
68+
echo "Contents of final_version.txt: '$(cat /tmp/final_version.txt)'"
69+
70+
USER root
71+
WORKDIR /app
72+
73+
# Set UV environment variables
74+
ENV UV_NO_MANAGED_PYTHON=true \
75+
UV_SYSTEM_PYTHON=true
76+
77+
# Install pyatlan dependencies from Chainguard APK (hardened packages)
78+
RUN apk add --no-cache \
79+
py3.11-pydantic=2.12.5-r0 \
80+
py3.11-jinja2=3.1.6-r1 \
81+
py3.11-tenacity=9.1.2-r2 \
82+
py3.11-pytz=2025.2-r2 \
83+
py3.11-python-dateutil=2.9.0-r10 \
84+
py3.11-pyyaml=6.0.3-r0 \
85+
py3.11-httpx=0.28.1-r2
86+
87+
# Copy only the wheel from builder (much smaller than full source)
88+
COPY --from=builder /tmp/wheels/*.whl /tmp/
89+
90+
# Install dependencies + pyatlan wheel + cleanup in ONE layer
91+
RUN uv pip install --system --no-cache \
92+
lazy-loader~=0.4 \
93+
nanoid~=2.0.0 \
94+
authlib~=1.6.0 \
95+
httpx-retries~=0.4.0 && \
96+
# Install pyatlan wheel with --no-deps
97+
uv pip install --system --no-cache --no-deps /tmp/*.whl && \
98+
# Cleanup everything EXCEPT version files (needed for verification)
99+
cd / && rm -rf /tmp/*.whl /root/.cache ~/.cache && \
100+
find /usr/lib/python3.11 -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
101+
find /usr/lib/python3.11 -type f -name '*.pyc' -delete 2>/dev/null || true
102+
103+
# Verify installation and version
104+
RUN echo "=== Debug: Checking version files ===" && \
105+
ls -la /tmp/ && \
106+
echo "Contents of final_version.txt:" && \
107+
cat /tmp/final_version.txt && \
108+
echo "=== Starting verification ===" && \
109+
EXPECTED_VERSION="$(cat /tmp/final_version.txt)" python3 <<'EOF'
110+
import sys
111+
import os
112+
113+
expected_version = os.environ.get("EXPECTED_VERSION", "unknown")
114+
print("=== Pyatlan Installation Verification ===")
115+
print(f"Expected version from env: {expected_version}")
116+
117+
try:
118+
import pyatlan
119+
print(f"✅ PyAtlan imported successfully")
120+
print(f"Installed version: {pyatlan.__version__}")
121+
122+
# Try importing AtlanClient
123+
from pyatlan.client.atlan import AtlanClient
124+
print(f"✅ AtlanClient imported successfully")
125+
126+
# Version validation (skip if 'latest' was requested)
127+
if expected_version != "latest" and expected_version != "unknown":
128+
if pyatlan.__version__ != expected_version:
129+
print(f"❌ ERROR: Version mismatch!")
130+
print(f" Expected: {expected_version}")
131+
print(f" Got: {pyatlan.__version__}")
132+
sys.exit(1)
133+
else:
134+
print("✅ Version verified")
135+
else:
136+
print("✅ Version validation skipped (latest or unknown)")
137+
138+
print("=== Build verification passed ===")
139+
140+
except ImportError as e:
141+
print(f"❌ Import error: {e}")
142+
sys.exit(1)
143+
except Exception as e:
144+
print(f"❌ Unexpected error: {e}")
145+
sys.exit(1)
146+
EOF
147+
148+
# Final cleanup of temporary version files
149+
RUN rm -rf /tmp/version.txt /tmp/final_version.txt
150+
151+
# Add build metadata as labels
152+
LABEL python.version="${PYTHON_VERSION}"
153+
LABEL pyatlan.commit_sha="${PYATLAN_COMMIT_SHA}"
154+
# Note: pyatlan.version label should be set by passing --build-arg PYATLAN_VERSION=$(cat pyatlan/version.txt) during build
155+
LABEL build.method="git-multistage"
156+
LABEL build.source="github"
157+
LABEL security.approach="apk-first-no-pypi"
158+
159+
# Switch to nonroot user for runtime security
160+
USER nonroot
33161

34-
# Default working directory for applications
162+
# Default working directory
35163
WORKDIR /app
36164

37-
# Default command (can be overridden by extending images)
165+
# Default command
38166
CMD ["python"]

0 commit comments

Comments
 (0)