cromper

.env

@@ -1,5 +1,4 @@
 DEBUG=on
-DUMMY_COMPILER=on
 SECRET_KEY=django-insecure-nm#!8%zhc0wwi#m_*l9l)=m*6gs4&o_^-e5b5vj*k05&yaqc1
 DATABASE_URL=sqlite:///dev.db
 API_BASE=http://127.0.0.1:8000/api

.github/workflows/ci.yml

@@ -16,40 +16,15 @@ jobs:
           docker build backend -t decompme_backend
       - name: Run tests
         run: |-
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
           # NOTE: CI does not run as 'ubuntu' user
           export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          for dir in backend/.venv sandbox local_files compilers libraries; do
-            mkdir -p "$dir"
-            sudo chown "${USER_1000}:${USER_1000}" "$dir"
-          done
+          sudo chown -R "${USER_1000}:${USER_1000}" .
           docker run \
             -v $(pwd):/decomp.me \
-            -v $(pwd)/local_files:/local_files \
-            -v $(pwd)/compilers:/compilers \
-            -v $(pwd)/libraries:/libraries \
-            --security-opt apparmor=unconfined \
-            --security-opt seccomp=unconfined \
-            --cap-drop all \
-            --cap-add setuid \
-            --cap-add setgid \
-            --cap-add setfcap \
-            --tmpfs /sandbox/tmp:exec,uid=1000,gid=1000,size=64M,mode=0700 \
             --entrypoint /bin/bash \
-            -e COMPILER_BASE_PATH=/compilers \
-            -e LIBRARY_BASE_PATH=/libraries \
-            -e WINEPREFIX=/tmp/wine \
-            -e LOCAL_FILE_DIR=/local_files \
-            -e USE_SANDBOX_JAIL=on \
-            -e SANDBOX_DISABLE_PROC=true \
-            -e TIMEOUT_SCALE_FACTOR=10 \
             decompme_backend \
               -c 'cd /decomp.me/backend && \
               uv sync && \
-              uv run compilers/download.py --compilers-dir ${COMPILER_BASE_PATH} && \
-              uv run libraries/download.py --libraries-dir ${LIBRARY_BASE_PATH} && \
-              for r in wine/*.reg; do regedit $r; done && \
               uv run python manage.py test'
 
   backend_test_docker_prod:
@@ -61,45 +36,19 @@ jobs:
       - name: Build decompme_backend image
         run: |-
           docker build backend --target prod -t decompme_backend
-      - name: Fetch compilers and libraries
-        run: |-
-          python3 -m pip install requests
-          python3 backend/compilers/download.py
-          python3 backend/libraries/download.py
-          export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          sudo chown -R ${USER_1000}:${USER_1000} backend/compilers
-          sudo chown -R ${USER_1000}:${USER_1000} backend/libraries
       - name: Run tests
         run: |-
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
-          export USER_1000=$(getent passwd 1000 | cut -d: -f1)
           mkdir -p local_files && sudo chown ${USER_1000}:${USER_1000} local_files
           docker run \
             -v $(pwd)/local_files:/local_files \
-            -v $(pwd)/backend/compilers:/compilers \
-            -v $(pwd)/backend/libraries:/libraries \
-            --security-opt apparmor=unconfined \
-            --security-opt seccomp=unconfined \
-            --cap-drop all \
-            --cap-add setuid \
-            --cap-add setgid \
-            --cap-add setfcap \
-            --tmpfs /sandbox/tmp:exec,uid=1000,gid=1000,size=64M,mode=0700 \
             --entrypoint /bin/bash \
             -e LOCAL_FILE_DIR=/local_files \
-            -e COMPILER_BASE_PATH=/compilers \
-            -e LIBRARY_BASE_PATH=/libraries \
-            -e USE_SANDBOX_JAIL=on \
-            -e SANDBOX_DISABLE_PROC=true \
-            -e TIMEOUT_SCALE_FACTOR=10 \
             -e DATABASE_URL=sqlite:///:memory: \
-            -e DUMMY_COMPILER=1 \
             -e SECRET_KEY=secret-key-secret-key-secret-key-secret-key-secret-key-secret-key \
             decompme_backend \
-              -c '\
-              for r in wine/*.reg; do regedit $r; done && \
-              uv run python manage.py test'
+              -c 'uv run python manage.py test'
+
+  # TODO: run cromper unit tests
 
   docker_compose_test:
     name: test docker compose
@@ -112,11 +61,11 @@ jobs:
       - name: Fetch compilers and libraries
         run: |
           python3 -m pip install requests
-          python3 backend/compilers/download.py
-          python3 backend/libraries/download.py
+          python3 cromper/compilers/download.py
+          python3 cromper/libraries/download.py
           export USER_1000=$(getent passwd 1000 | cut -d: -f1)
-          sudo chown -R ${USER_1000}:${USER_1000} backend/compilers
-          sudo chown -R ${USER_1000}:${USER_1000} backend/libraries
+          sudo chown -R ${USER_1000}:${USER_1000} cromper/compilers
+          sudo chown -R ${USER_1000}:${USER_1000} cromper/libraries
 
       - name: Setup dummy docker.prod.dev
         run: |
@@ -129,6 +78,7 @@ jobs:
           echo 'SANDBOX_DISABLE_PROC="true"' >> docker.prod.env
           echo 'ALLOWED_HOSTS="backend,localhost,127.0.0.1"' >> docker.prod.env
           echo 'USE_SANDBOX_JAIL="on"' >> docker.prod.env
+          echo 'CROMPER_URL=http://cromper:8888' >> docker.prod.env
           echo 'CI=true' >> docker.prod.env
 
       - name: Comment out SSL server configuration from nginx
@@ -142,6 +92,12 @@ jobs:
           docker compose logs nginx | grep "ready for start up"
           ! docker compose logs nginx | grep -q "nginx-1 exited with code"
 
+      - name: Build and bring up cromper container
+        run: |
+          docker compose -f docker-compose.prod.yaml build cromper
+          docker compose -f docker-compose.prod.yaml up -d cromper
+          timeout 15s docker compose -f docker-compose.prod.yaml logs -f || true
+
       - name: Build and bring up up backend container
         run: |
           docker compose -f docker-compose.prod.yaml build backend
@@ -158,6 +114,7 @@ jobs:
         run: |
           curl --silent http://localhost:8080/ | head -c 256
           curl --silent http://localhost:8000/api/ | jq
+          curl --silent http://localhost:8000/api/platform | jq
 
       - name: Shut everything down
         run: |
@@ -170,6 +127,7 @@ jobs:
   frontend_lint:
     name: biome
     runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js 24
@@ -187,9 +145,10 @@ jobs:
           cd frontend
           yarn lint
 
-  mypy:
-    name: mypy
+  backend_lint:
+    name: backend lint and type check
     runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - name: Install uv
@@ -202,12 +161,15 @@ jobs:
           python-version: "3.12"
       - run: |-
           cd backend
-          uv sync
-          uv run mypy
+          uv sync --all-extras
+          uv run mypy .
+          uv run ruff check
+          uv run ruff format --check
 
-  ruff:
-    name: ruff
+  cromper_lint:
+    name: cromper lint and type check
     runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - name: Install uv
@@ -219,7 +181,8 @@ jobs:
         with:
           python-version: "3.12"
       - run: |-
-          cd backend
-          uv sync
-          uv run ruff check .
-          uv run ruff format --check .
+          cd cromper
+          uv sync --all-extras
+          uv run mypy .
+          uv run ruff check
+          uv run ruff format --check

backend/.dockerignore

@@ -8,10 +8,7 @@ __pycache__/
 Dockerfile*
 docker.*.env
 
-# compilers
-compilers/*/**
-!compilers/download.py
 
-# libraries
-libraries/*/**
-!libraries/download.py
+# TEMPORARY: because i'm switching between main/cromper branches
+compilers/**
+libraries/**

backend/.gitignore

@@ -0,0 +1,3 @@
+# TEMPORARY: because i'm switching between main/cromper branches
+compilers/**
+libraries/**

backend/Dockerfile

@@ -2,11 +2,15 @@ FROM --platform=linux/amd64 ubuntu:24.04 AS base
 
 ENV DEBIAN_FRONTEND=noninteractive
 
+
+FROM base AS dependencies
+
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
     git \
+    netcat-traditional \
     python-is-python3 \
     python3 \
     python3-pip \
@@ -17,150 +21,17 @@ RUN apt-get update \
 
 COPY --from=ghcr.io/astral-sh/uv:0.9.11 /uv /bin/uv
 
-FROM base AS nsjail
-
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    autoconf \
-    bison \
-    flex \
-    g++ \
-    gcc \
-    libnl-route-3-dev \
-    libprotobuf-dev \
-    libtool \
-    make \
-    pkg-config \
-    protobuf-compiler \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN git clone "https://github.com/google/nsjail" \
-    --recursive --branch 3.4 /nsjail \
-    && cd /nsjail \
-    && make
-
-
-FROM base AS dependencies
-
-RUN dpkg --add-architecture i386 \
-    && add-apt-repository -y ppa:dosemu2/ppa \
-    && add-apt-repository -y ppa:stsp-0/dj64 \
-    && apt-get update \
-    && apt-get install -y -o APT::Immediate-Configure=false --no-install-recommends \
-    binutils-aarch64-linux-gnu \
-    binutils-arm-none-eabi \
-    binutils-djgpp \
-    binutils-mingw-w64-i686 \
-    binutils-mips-linux-gnu \
-    binutils-mipsel-linux-gnu \
-    binutils-powerpc-linux-gnu \
-    binutils-sh-elf \
-    cpp \
-    dj64 \
-    dos2unix \
-    dosemu2 \
-    gcc-mips-linux-gnu \
-    iptables \
-    libarchive-tools \
-    libc6-dev-i386 \
-    libdevmapper1.02.1 \
-    libgpgme11 \
-    libnl-route-3-200 \
-    libprotobuf-dev \
-    libtinfo6 \
-    netcat-traditional \
-    unzip \
-    wget \
-    wine \
-    wine32:i386 \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN wget http://security.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb \
-    && apt install ./libtinfo5_6.3-2ubuntu0.1_amd64.deb \
-    && rm libtinfo5_6.3-2ubuntu0.1_amd64.deb
-
-COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
-COPY --from=ghcr.io/decompals/wibo:1.0.0-beta.5 /usr/local/bin/wibo /usr/bin/
-
-# Patched mips binutils
-RUN wget "https://github.com/decompals/binutils-mips-ps2-decompals/releases/download/v0.4/binutils-mips-ps2-decompals-linux-x86-64.tar.gz" \
-    && tar xvzf binutils-mips-ps2-decompals-linux-x86-64.tar.gz -C /usr/bin mips-ps2-decompals-as mips-ps2-decompals-nm mips-ps2-decompals-objdump \
-    && rm binutils-mips-ps2-decompals-linux-x86-64.tar.gz \
-    && chmod +x /usr/bin/mips-ps2-decompals-*
-
-# Patched PowerPC binutils
-RUN curl -sSL "https://github.com/encounter/gc-wii-binutils/releases/download/2.42-1/linux-x86_64.zip" | \
-    bsdtar -xvf- -C /usr/bin \
-    && chmod +x /usr/bin/powerpc-eabi-*
-
-# MSDOS specific
-RUN wget "https://github.com/OmniBlade/binutils-gdb/releases/download/omf-build/omftools.tar.gz" \
-    && tar xvzf omftools.tar.gz -C /usr/bin jwasm \
-    && rm omftools.tar.gz \
-    && wget "https://github.com/decompals/binutils-omf/releases/download/v0.4/omftools-linux-x86_64.tar.gz" \
-    && tar xvzf omftools-linux-x86_64.tar.gz -C /usr/bin omf-nm omf-objdump \
-    && rm omftools-linux-x86_64.tar.gz
-
-RUN mkdir -p /etc/fonts
-
-ENV WINEPREFIX=/tmp/wine
-
-# Ensure /sandbox and wine dirs have correct ownership
-RUN mkdir -p /sandbox \
-    && chown -R ubuntu:ubuntu /sandbox \
-    && mkdir -p "${WINEPREFIX}" \
-    && chown ubuntu:ubuntu "${WINEPREFIX}"
-
-# Switch to non-root user
 USER ubuntu
 
-# Initialize wine files to /home/ubuntu/.wine
-RUN wineboot --init
-
 WORKDIR /backend
 
 
 FROM dependencies AS dev
 
-ARG ENABLE_DREAMCAST_SUPPORT
-ARG ENABLE_GBA_SUPPORT
-ARG ENABLE_GC_WII_SUPPORT
-ARG ENABLE_MACOSX_SUPPORT
-ARG ENABLE_MSDOS_SUPPORT
-ARG ENABLE_N3DS_SUPPORT
-ARG ENABLE_N64_SUPPORT
-ARG ENABLE_NDS_ARM9_SUPPORT
-ARG ENABLE_PS1_SUPPORT
-ARG ENABLE_PS2_SUPPORT
-ARG ENABLE_PSP_SUPPORT
-ARG ENABLE_SATURN_SUPPORT
-ARG ENABLE_SWITCH_SUPPORT
-ARG ENABLE_WIN32_SUPPORT
-
-ENV ENABLE_DREAMCAST_SUPPORT=${ENABLE_DREAMCAST_SUPPORT} \
-    ENABLE_GBA_SUPPORT=${ENABLE_GBA_SUPPORT} \
-    ENABLE_GC_WII_SUPPORT=${ENABLE_GC_WII_SUPPORT} \
-    ENABLE_MACOSX_SUPPORT=${ENABLE_MACOSX_SUPPORT} \
-    ENABLE_MSDOS_SUPPORT=${ENABLE_MSDOS_SUPPORT} \
-    ENABLE_N3DS_SUPPORT=${ENABLE_N3DS_SUPPORT} \
-    ENABLE_N64_SUPPORT=${ENABLE_N64_SUPPORT} \
-    ENABLE_NDS_ARM9_SUPPORT=${ENABLE_NDS_ARM9_SUPPORT} \
-    ENABLE_PS1_SUPPORT=${ENABLE_PS1_SUPPORT} \
-    ENABLE_PS2_SUPPORT=${ENABLE_PS2_SUPPORT} \
-    ENABLE_PSP_SUPPORT=${ENABLE_PSP_SUPPORT} \
-    ENABLE_SATURN_SUPPORT=${ENABLE_SATURN_SUPPORT} \
-    ENABLE_SWITCH_SUPPORT=${ENABLE_SWITCH_SUPPORT} \
-    ENABLE_WIN32_SUPPORT=${ENABLE_WIN32_SUPPORT}
-
 ENTRYPOINT ["/backend/docker_entrypoint.sh"]
 
 
-FROM base AS uv-install
-
-USER ubuntu
-
-WORKDIR /backend
+FROM dependencies AS uv-install
 
 COPY pyproject.toml uv.lock /backend/
 
@@ -172,12 +43,9 @@ FROM dependencies AS prod
 COPY --from=uv-install /backend/.venv /backend/.venv
 
 COPY manage.py /backend
-COPY housekeeping.py /backend
+COPY db_housekeeping.py /backend
 
-COPY wine /backend/wine
 COPY decompme /backend/decompme
-COPY libraries /backend/libraries
-COPY compilers /backend/compilers
 
 COPY coreapp /backend/coreapp