Merge pull request #234 from BurnySc2/add-python-314-support

Add python 3.14

Author: BurnySc2

.github/workflows/ci.yml

@@ -24,8 +24,8 @@ env:
   # Docker image version, see https://hub.docker.com/r/burnysc2/python-sc2-docker/tags
   # This version should always lack behind one version behind the docker-ci.yml because it is possible that it doesn't exist
   VERSION_NUMBER: "1.0.4"
-  # TODO Change to '3.13' when a new image has been pushed
-  LATEST_PYTHON_VERSION: "3.12"
+  # TODO Change to '3.14' when a new image has been pushed
+  LATEST_PYTHON_VERSION: "3.13"
   LATEST_SC2_VERSION: "4.10"
 
 jobs:
@@ -109,7 +109,7 @@ jobs:
         # Python 3.8 support has been dropped because numpy >=1.26.0 requires Python >=3.9 (this numpy version is required to run python 3.12)
         # Python 3.9 support has been dropped since numpy >=2.1.0 (this numpy version is required to run python 3.13)
         os: [macos-latest, windows-latest, ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/docker-ci.yml

@@ -14,14 +14,41 @@ on:
       - develop
 
 env:
-  VERSION_NUMBER: "1.0.6"
-  LATEST_PYTHON_VERSION: "3.13"
+  VERSION_NUMBER: "1.0.7"
+  LATEST_PYTHON_VERSION: "3.14"
   LATEST_SC2_VERSION: "4.10"
-  EXPERIMENTAL_PYTHON_VERSION: "3.14"
+  EXPERIMENTAL_PYTHON_VERSION: "3.15"
 
 jobs:
+  download_sc2_maps:
+    name: Download and cache sc2 maps
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cache sc2 maps
+        uses: actions/cache@v4
+        id: cache-sc2-maps
+        with:
+          path: |
+            dockerfiles/maps
+          key: ${{ runner.os }}-maps-${{ hashFiles('dockerfiles/maps/**') }}
+          restore-keys: |
+            ${{ runner.os }}-maps-
+
+      - name: Download sc2 maps
+        run: sh dockerfiles/download_maps.sh
+        if: steps.cache-sc2-maps.outputs.cache-hit != 'true'
+
   run_test_docker_image:
     name: Run test_docker_image.sh
+    needs: [download_sc2_maps]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     strategy:
@@ -32,10 +59,19 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Enable experimental docker features
-        run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker.service
+      - name: Cache sc2 maps
+        uses: actions/cache@v4
+        id: cache-sc2-maps
+        with:
+          path: |
+            dockerfiles/maps
+          key: ${{ runner.os }}-maps-${{ hashFiles('dockerfiles/maps/**') }}
+          restore-keys: |
+            ${{ runner.os }}-maps-
+
+      - name: Download sc2 maps
+        run: sh dockerfiles/download_maps.sh
+        if: steps.cache-sc2-maps.outputs.cache-hit != 'true'
 
       - name: Run shell script
         env:
@@ -46,6 +82,7 @@ jobs:
 
   run_test_new_python_version:
     name: Run test_new_python_candidate.sh
+    needs: [download_sc2_maps]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     strategy:
@@ -56,10 +93,19 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Enable experimental docker features
-        run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker.service
+      - name: Cache sc2 maps
+        uses: actions/cache@v4
+        id: cache-sc2-maps
+        with:
+          path: |
+            dockerfiles/maps
+          key: ${{ runner.os }}-maps-${{ hashFiles('dockerfiles/maps/**') }}
+          restore-keys: |
+            ${{ runner.os }}-maps-
+
+      - name: Download sc2 maps
+        run: sh dockerfiles/download_maps.sh
+        if: steps.cache-sc2-maps.outputs.cache-hit != 'true'
 
       - name: Run shell script
         continue-on-error: true
@@ -71,13 +117,14 @@ jobs:
 
   docker_build:
     name: Build docker image
+    needs: [download_sc2_maps]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
         sc2-version: ["4.10"]
     env:
       IMAGE_NAME: burnysc2/python-sc2-docker:py_${{ matrix.python-version }}-sc2_${{ matrix.sc2-version }}
@@ -86,26 +133,31 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Build docker image
-        run: docker build -t $IMAGE_NAME-v$VERSION_NUMBER $BUILD_ARGS - < dockerfiles/Dockerfile
+      - name: Cache sc2 maps
+        uses: actions/cache@v4
+        id: cache-sc2-maps
+        with:
+          path: |
+            dockerfiles/maps
+          key: ${{ runner.os }}-maps-${{ hashFiles('dockerfiles/maps/**') }}
+          restore-keys: |
+            ${{ runner.os }}-maps-
 
-      - name: Enable experimental docker features
-        run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker.service
+      - name: Download sc2 maps
+        run: sh dockerfiles/download_maps.sh
+        if: steps.cache-sc2-maps.outputs.cache-hit != 'true'
 
-      - name: Build squashed image
-        run: docker build -t $IMAGE_NAME-v$VERSION_NUMBER-squashed --squash $BUILD_ARGS - < dockerfiles/Dockerfile
+      - name: Build docker image
+        run: docker build -f dockerfiles/Dockerfile -t $IMAGE_NAME-v$VERSION_NUMBER $BUILD_ARGS .
 
-      - name: Run test bots on squashed image
-        if: matrix.python-version != '3.7'
+      - name: Run test bots on image
         run: |
           echo "Start container, override the default entrypoint"
           docker run -i -d \
             --name test_container \
             --env 'PYTHONPATH=/root/python-sc2/' \
             --entrypoint /bin/bash \
-            $IMAGE_NAME-v$VERSION_NUMBER-squashed
+            $IMAGE_NAME-v$VERSION_NUMBER
           echo "Install python-sc2"
           docker exec -i test_container mkdir -p /root/python-sc2
           docker cp pyproject.toml test_container:/root/python-sc2/
@@ -131,12 +183,8 @@ jobs:
         if: github.ref == 'refs/heads/develop' && github.event_name == 'push'
         run: docker push $IMAGE_NAME-v$VERSION_NUMBER
 
-      - name: Upload squashed docker image
-        if: github.ref == 'refs/heads/develop' && github.event_name == 'push'
-        run: docker push $IMAGE_NAME-v$VERSION_NUMBER-squashed
-
-      - name: Upload squashed docker image as latest tag
+      - name: Upload docker image as latest tag
         if: github.ref == 'refs/heads/develop' && github.event_name == 'push' && matrix.python-version == env.LATEST_PYTHON_VERSION && matrix.sc2-version == env.LATEST_SC2_VERSION
         run: |
-          docker tag $IMAGE_NAME-v$VERSION_NUMBER-squashed burnysc2/python-sc2-docker:latest
+          docker tag $IMAGE_NAME-v$VERSION_NUMBER burnysc2/python-sc2-docker:latest
           docker push burnysc2/python-sc2-docker:latest

dockerfiles/Dockerfile

@@ -2,7 +2,7 @@
 ARG PYTHON_VERSION=3.10
 
 # https://docs.astral.sh/uv/guides/integration/docker/#available-images
-FROM ghcr.io/astral-sh/uv:python$PYTHON_VERSION-bookworm-slim
+FROM ghcr.io/astral-sh/uv:python$PYTHON_VERSION-bookworm-slim AS base
 
 ARG SC2_VERSION=4.10
 
@@ -37,7 +37,7 @@ WORKDIR /root/
 # Download and uncompress StarCraftII from https://github.com/Blizzard/s2client-proto#linux-packages and remove zip file
 # If file is locally available, use this instead:
 #COPY SC2.4.10.zip /root/
-RUN curl --retry 3 --retry-delay 5 -C - http://blzdistsc2-a.akamaihd.net/Linux/SC2.$SC2_VERSION.zip -o "SC2.$SC2_VERSION.zip" \
+RUN curl --retry 5 --retry-delay 5 -C - http://blzdistsc2-a.akamaihd.net/Linux/SC2.$SC2_VERSION.zip -o "SC2.$SC2_VERSION.zip" \
     && unzip -q -P iagreetotheeula "SC2.$SC2_VERSION.zip" \
     && rm SC2.$SC2_VERSION.zip
 
@@ -51,51 +51,13 @@ RUN ln -s /root/StarCraftII/Maps /root/StarCraftII/maps \
     # Remove the Maps that come with the SC2 client
     && rm -rf /root/StarCraftII/maps/*
 
-# Change to maps folder
-WORKDIR /root/StarCraftII/maps/
-
-# Maps are available here https://github.com/Blizzard/s2client-proto#map-packs and here https://sc2ai.net/wiki/maps/
-# Download and uncompress StarCraftII Maps, remove zip file - using "maps" instead of "Maps" as target folder
-
-# Get sc2ai.net ladder maps
-# -L param follows links https://stackoverflow.com/a/2663023/10882657
-RUN curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/9/ -o 1.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/14/ -o 2.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/21/ -o 3.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/35/ -o 4.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/36/ -o 5.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/38/ -o 6.zip \
-    && curl -L https://sc2ai.net/wiki/184/plugin/attachments/download/39/ -o 7.zip \
-    && unzip -q -o '*.zip' \
-    && rm *.zip
-
-# Get official blizzard maps
-RUN curl --retry 3 --retry-delay 5 -C - http://blzdistsc2-a.akamaihd.net/MapPacks/Ladder2019Season3.zip -o 'Ladder2019Season3.zip' \
-    && unzip -q -P iagreetotheeula -o 'Ladder2019Season3.zip' \
-    && mv Ladder2019Season3/* . \
-    && rm Ladder2019Season3.zip \
-    && rm -r Ladder2019Season3
-
-# Get v5.0.6 maps
-RUN curl -L https://github.com/shostyn/sc2patch/raw/4987d4915b47c801adbc05e297abaa9ca2988838/Maps/506.zip -o 506.zip \
-    && unzip -q -o '506.zip' \
-    && rm 506.zip
-
-# Get flat and empty maps
-RUN curl --retry 3 --retry-delay 5 -C - http://blzdistsc2-a.akamaihd.net/MapPacks/Melee.zip -o Melee.zip \
-    && unzip -q -P iagreetotheeula -o 'Melee.zip' \
-    && mv Melee/* . \
-    && rm Melee.zip \
-    && rm -r Melee
-
-# Remove LE suffix from file names
-RUN rename -v 's/LE.SC2Map/.SC2Map/' *.SC2Map
-
-# List all map files
-RUN tree
+# See download_maps.sh
+COPY dockerfiles/maps/* /root/StarCraftII/maps/
 
+# Squash image with trick https://stackoverflow.com/a/56118557/10882657
+FROM scratch
+COPY --from=base / /
 WORKDIR /root/
-
 ENTRYPOINT [ "/bin/bash" ]
 
 # To run a python-sc2 bot:

dockerfiles/download_maps.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Run via
+# sh dockerfiles/download_maps.sh
+set -e
+
+# Create maps directory if it doesn't exist
+mkdir -p dockerfiles/maps
+cd dockerfiles/maps
+
+# Download function with retries
+download_with_retry() {
+    local url=$1
+    local output=$2
+    curl --retry 5 --retry-delay 5 -C - -L "$url" -o "$output"
+}
+
+# Download and process sc2ai.net ladder maps
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/9/" "1.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/14/" "2.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/21/" "3.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/35/" "4.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/36/" "5.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/38/" "6.zip"
+download_with_retry "https://sc2ai.net/wiki/184/plugin/attachments/download/39/" "7.zip"
+unzip -q -o '*.zip'
+rm *.zip
+
+# Download and process official blizzard maps
+download_with_retry "http://blzdistsc2-a.akamaihd.net/MapPacks/Ladder2019Season3.zip" "Ladder2019Season3.zip"
+unzip -q -P iagreetotheeula -o "Ladder2019Season3.zip"
+mv Ladder2019Season3/* .
+rm "Ladder2019Season3.zip"
+rmdir Ladder2019Season3
+
+# Download and process v5.0.6 maps
+download_with_retry "https://github.com/shostyn/sc2patch/raw/4987d4915b47c801adbc05e297abaa9ca2988838/Maps/506.zip" "506.zip"
+unzip -q -o "506.zip"
+rm "506.zip"
+
+# Download and process flat/empty maps
+download_with_retry "http://blzdistsc2-a.akamaihd.net/MapPacks/Melee.zip" "Melee.zip"
+unzip -q -P iagreetotheeula -o "Melee.zip"
+mv Melee/* .
+rm "Melee.zip"
+rmdir Melee
+
+# Remove LE suffix from file names
+for f in *LE.SC2Map; do 
+    mv -- "$f" "${f%LE.SC2Map}.SC2Map"; 
+done

dockerfiles/test_docker_image.sh

@@ -7,27 +7,16 @@ set -e
 
 # Set which versions to use
 export VERSION_NUMBER=${VERSION_NUMBER:-0.9.9}
-export PYTHON_VERSION=${PYTHON_VERSION:-3.12}
+export PYTHON_VERSION=${PYTHON_VERSION:-3.13}
 export SC2_VERSION=${SC2_VERSION:-4.10}
 
 # For better readability, set local variables
 IMAGE_NAME=burnysc2/python-sc2-docker:py_$PYTHON_VERSION-sc2_$SC2_VERSION-v$VERSION_NUMBER
 BUILD_ARGS="--build-arg PYTHON_VERSION=$PYTHON_VERSION --build-arg SC2_VERSION=$SC2_VERSION"
 
-# Allow image squashing by enabling experimental docker features
-# https://stackoverflow.com/a/21164441/10882657
-# https://github.com/actions/virtual-environments/issues/368#issuecomment-582387669
-# file=/etc/docker/daemon.json
-# if [ ! -e "$file" ]; then
-#   echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-#   sudo systemctl restart docker.service
-# fi
-
 # Build image without context
 # https://stackoverflow.com/a/54666214/10882657
-docker build -t $IMAGE_NAME $BUILD_ARGS - < dockerfiles/Dockerfile
-# Build squashed image where the layers are combined to one
-#docker build -t $IMAGE_NAME-squashed --squash $BUILD_ARGS - < dockerfiles/Dockerfile
+docker build -f dockerfiles/Dockerfile -t $IMAGE_NAME $BUILD_ARGS .
 
 # Delete previous container if it exists
 docker rm -f test_container

dockerfiles/test_new_python_candidate.sh

@@ -8,15 +8,15 @@ set -e
 
 # Set which versions to use
 export VERSION_NUMBER=${VERSION_NUMBER:-0.9.9}
-export PYTHON_VERSION=${PYTHON_VERSION:-3.13}
+export PYTHON_VERSION=${PYTHON_VERSION:-3.14}
 export SC2_VERSION=${SC2_VERSION:-4.10}
 
 # For better readability, set local variables
 IMAGE_NAME=burnysc2/python-sc2-docker:py_$PYTHON_VERSION-sc2_$SC2_VERSION-v$VERSION_NUMBER
 BUILD_ARGS="--build-arg PYTHON_VERSION=$PYTHON_VERSION --build-arg SC2_VERSION=$SC2_VERSION"
 
 # Build image
-docker build -t $IMAGE_NAME $BUILD_ARGS - < dockerfiles/Dockerfile
+docker build -f dockerfiles/Dockerfile -t $IMAGE_NAME $BUILD_ARGS .
 
 # Delete previous container if it exists
 docker rm -f test_container
@@ -25,14 +25,22 @@ docker rm -f test_container
 # https://docs.docker.com/storage/bind-mounts/#use-a-read-only-bind-mount
 docker run -i -d \
   --name test_container \
-  --volume ./:/root/python-sc2:ro \
+  --env 'PYTHONPATH=/root/python-sc2' \
+  --entrypoint /bin/bash \
   $IMAGE_NAME
 
+# Install requirements
+docker exec -i test_container mkdir -p /root/python-sc2
+docker cp pyproject.toml test_container:/root/python-sc2/
+docker cp uv.lock test_container:/root/python-sc2/
+docker exec -i test_container bash -c "pip install uv && cd python-sc2 && uv sync --no-cache --no-install-project"
 
-# Install python-sc2, via mount the python-sc2 folder will be available
-docker exec -i test_container bash -c "pip install uv \
-    && cd python-sc2 && uv sync --frozen --no-cache --no-install-project"
+docker cp sc2 test_container:/root/python-sc2/sc2
+docker cp s2clientprotocol test_container:/root/python-sc2/s2clientprotocol
+docker cp test test_container:/root/python-sc2/test
 
 # Run various test bots
 docker exec -i test_container bash -c "cd python-sc2 && uv run python test/travis_test_script.py test/autotest_bot.py"
+
+docker cp examples test_container:/root/python-sc2/examples
 docker exec -i test_container bash -c "cd python-sc2 && uv run python test/run_example_bots_vs_computer.py"