diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..28b4cf6f --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,801 @@ +name: PR Quality Gate +on: + push: + branches: + - "neoStella" + pull_request: + branches: + [ + main, + development, + remotes/origin/version_*, + ] + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NODE_VERSION: "20" + PYTHON_VERSION: "3.11" + REDIS_VERSION: "7" + +jobs: + changes: + name: Detect Changes + if: github.event_name == 'push' || github.head_ref == 'neoStella' + runs-on: ubuntu-latest + permissions: + pull-requests: read + contents: write + outputs: + frontend: ${{ steps.changes.outputs.frontend }} + backend: ${{ steps.changes.outputs.backend }} + tests: ${{ steps.changes.outputs.tests }} + workflows: ${{ steps.changes.outputs.workflows }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + frontend: + - 'frontend/**' + - 'package.json' + - 'package-lock.json' + - '.github/workflows/**' + backend: + - 'backend/**' + - 'requirements*.txt' + - 'pyproject.toml' + - '.github/workflows/**' + tests: + - 'tests/**' + workflows: + - '.github/workflows/**' + + frontend-install: + name: Frontend Dependencies + needs: changes + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache node_modules + uses: actions/cache@v4 + id: cache-node-modules + with: + path: frontend/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} + restore-keys: | + node-modules-${{ runner.os }}- + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm install --prefer-offline --no-audit + + frontend-typecheck: + name: TypeScript Check + needs: [changes, frontend-install] + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Restore node_modules + uses: actions/cache@v4 + with: + path: frontend/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} + + - name: Run TypeScript type checking + run: npx tsc --noEmit + + frontend-lint: + name: Frontend Lint & Autofix + needs: [changes, frontend-install] + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Restore node_modules + uses: actions/cache@v4 + with: + path: frontend/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} + + - name: Install ESLint if not present + run: | + if ! npm list eslint > /dev/null 2>&1; then + npm install --save-dev eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-react eslint-plugin-react-hooks + fi + + - name: Run ESLint with autofix + run: | + npx eslint . --ext .ts,.tsx,.js,.jsx \ + --ignore-pattern 'node_modules/' \ + --ignore-pattern 'build/' \ + --fix || true + + - name: Run Prettier with autofix + run: | + if npm list prettier > /dev/null 2>&1; then + npx prettier --write "**/*.{ts,tsx,js,jsx,json,css,md}" --ignore-path .gitignore || true + fi + + - name: Check for changes and create autofix artifact + id: check-changes + run: | + cd .. + if [[ -n $(git status --porcelain frontend/) ]]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + mkdir -p /tmp/autofix + git diff frontend/ > /tmp/autofix/frontend-lint.patch || true + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Upload autofix patch + if: steps.check-changes.outputs.has_changes == 'true' + uses: actions/upload-artifact@v4 + with: + name: frontend-autofix-patch + path: /tmp/autofix/frontend-lint.patch + retention-days: 5 + + - name: Check for unused dependencies + run: npx depcheck --ignores="@types/*,autoprefixer,postcss,tailwindcss,tw-animate-css" || true + + frontend-build: + name: Frontend Build + needs: [changes, frontend-install] + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Restore node_modules + uses: actions/cache@v4 + with: + path: frontend/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} + + - name: Build React Router SSR app + run: npm run build + env: + NODE_ENV: production + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/build + retention-days: 1 + + frontend-test: + name: Frontend Tests + needs: [changes, frontend-install] + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Restore node_modules + uses: actions/cache@v4 + with: + path: frontend/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} + + - name: Install Vitest if not present + run: | + if ! npm list vitest > /dev/null 2>&1; then + npm install --save-dev vitest @testing-library/react @testing-library/jest-dom jsdom @vitejs/plugin-react @vitest/coverage-v8 + fi + + - name: Run tests with coverage + run: npx vitest run --passWithNoTests --coverage + env: + CI: true + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: frontend-coverage + path: frontend/coverage + retention-days: 5 + + frontend-security: + name: Frontend Security Audit + needs: [changes, frontend-install] + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Run npm audit + run: npm audit --audit-level=high || true + + - name: Check for known vulnerabilities + run: npx audit-ci --high || true + + backend-install: + name: Backend Dependencies + needs: changes + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache pip dependencies + uses: actions/cache@v4 + id: cache-pip + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + restore-keys: | + pip-${{ runner.os }}-${{ env.PYTHON_VERSION }}- + + - name: Cache virtual environment + uses: actions/cache@v4 + id: cache-venv + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Create virtual environment + if: steps.cache-venv.outputs.cache-hit != 'true' + run: python -m venv .venv + + - name: Install dependencies + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + source .venv/bin/activate + pip install --upgrade pip wheel setuptools + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + # Linting + pip install ruff + # Testing: pytest + async + coverage + parallel + timeouts + property-based + mocking + pip install pytest pytest-asyncio pytest-cov pytest-xdist pytest-timeout hypothesis fakeredis + # Security + pip install bandit safety + + backend-lint: + name: Backend Lint & Autofix (Ruff) + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Run Ruff linter with autofix + run: | + source .venv/bin/activate + ruff check . --fix --output-format=github + + - name: Run Ruff formatter with autofix + run: | + source .venv/bin/activate + ruff format . + + - name: Check for changes and create autofix artifact + id: check-changes + run: | + cd .. + if [[ -n $(git status --porcelain backend/) ]]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + mkdir -p /tmp/autofix + git diff backend/ > /tmp/autofix/backend-ruff.patch || true + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Upload autofix patch + if: steps.check-changes.outputs.has_changes == 'true' + uses: actions/upload-artifact@v4 + with: + name: backend-autofix-patch + path: /tmp/autofix/backend-ruff.patch + retention-days: 5 + + - name: Verify formatting (fail if unfixed issues remain) + run: | + source .venv/bin/activate + ruff check . --output-format=github + ruff format --check . + + backend-test: + name: Backend Tests + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Run pytest with coverage + run: | + source .venv/bin/activate + pytest ../tests -v \ + --cov=. \ + --cov-report=xml \ + --cov-report=term-missing \ + --cov-report=html \ + --cov-fail-under=0 \ + -n auto \ + --timeout=60 \ + || true + env: + REDIS_HOST: localhost + REDIS_PORT: 6379 + DATABASE_URL: sqlite:///test.db + + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: backend-coverage + path: | + backend/coverage.xml + backend/htmlcov + retention-days: 5 + + backend-test-redis-mock: + name: Backend Tests (Mocked Redis) + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Run unit tests with fakeredis + run: | + source .venv/bin/activate + pytest ../tests/unit -v \ + --timeout=30 \ + -x \ + || true + env: + USE_FAKE_REDIS: "true" + DATABASE_URL: sqlite:///test.db + + backend-security: + name: Backend Security Scan + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Run Bandit security linter + run: | + source .venv/bin/activate + bandit -r . -ll -ii --exclude ./.venv -f json -o bandit-report.json || true + bandit -r . -ll -ii --exclude ./.venv || true + + - name: Check dependencies for vulnerabilities + run: | + source .venv/bin/activate + pip freeze > requirements-freeze.txt + safety check -r requirements-freeze.txt --full-report || true + + - name: Upload security reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: backend-security-reports + path: | + backend/bandit-report.json + retention-days: 5 + + backend-quart-check: + name: Quart App Validation + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Validate Quart app imports + run: | + source .venv/bin/activate + python -c " + import sys + try: + from api import app + print('Quart app imported successfully') + except ImportError as e: + print(f'Import warning (may be expected): {e}') + except Exception as e: + print(f'Warning: {e}') + " + env: + REDIS_HOST: localhost + REDIS_PORT: 6379 + + database-check: + name: Database Migration Check + needs: [changes, backend-install] + if: needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Restore virtual environment + uses: actions/cache@v4 + with: + path: backend/.venv + key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('backend/requirements*.txt', 'backend/pyproject.toml') }} + + - name: Validate Peewee models + run: | + source .venv/bin/activate + python -c " + try: + from models import * + print('Peewee models validated successfully') + except Exception as e: + print(f'Model validation warning: {e}') + " + + - name: Test SQLite migrations + run: | + source .venv/bin/activate + python -c " + try: + from migrate import * + print('Migration module loaded successfully') + except Exception as e: + print(f'Migration warning: {e}') + " + + integration-test: + name: Integration Tests + needs: [changes, frontend-build, backend-test] + if: | + always() && + (needs.changes.outputs.frontend == 'true' || needs.changes.outputs.backend == 'true') && + (needs.frontend-build.result == 'success' || needs.frontend-build.result == 'skipped') && + (needs.backend-test.result == 'success' || needs.backend-test.result == 'skipped') + runs-on: ubuntu-latest + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download frontend build + uses: actions/download-artifact@v4 + with: + name: frontend-build + path: frontend/build + continue-on-error: true + + - name: WebSocket connectivity test + run: | + echo "WebSocket integration test placeholder" + + - name: API endpoint validation + run: | + echo "API endpoint validation placeholder" + + code-quality: + name: Code Quality Analysis + needs: changes + if: needs.changes.outputs.frontend == 'true' || needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check for large files + run: | + find . -type f -size +5M -not -path "./.git/*" | head -20 || echo "No large files found" + + - name: Check for merge conflict markers + run: | + if grep -rn "<<<<<<< \|======= \|>>>>>>> " --include="*.ts" --include="*.tsx" --include="*.py" --include="*.js" .; then + echo "Merge conflict markers found!" + exit 1 + else + echo "No merge conflict markers found" + fi + + - name: Check for debug statements + run: | + echo "Checking for debug statements..." + grep -rn "import pdb\|pdb.set_trace()\|breakpoint()" backend/ --include="*.py" | grep -v "# noqa" | head -20 || true + grep -rn "console.log\|debugger" frontend/ --include="*.ts" --include="*.tsx" --include="*.js" | grep -v "// eslint-disable" | head -20 || true + echo "Review any debug statements above before merging" + + - name: Check TODO and FIXME comments + run: | + echo "TODO and FIXME items found:" + grep -rn "TODO\|FIXME\|XXX\|HACK" --include="*.ts" --include="*.tsx" --include="*.py" --include="*.js" . | grep -v node_modules | head -30 || echo "None found" + + docker-build: + name: Docker Build Check + needs: changes + if: needs.changes.outputs.frontend == 'true' || needs.changes.outputs.backend == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build frontend Docker image + if: needs.changes.outputs.frontend == 'true' + uses: docker/build-push-action@v6 + with: + context: ./frontend + file: ./frontend/Dockerfile + push: false + tags: benchr-frontend:pr-${{ github.event.pull_request.number }} + cache-from: type=gha + cache-to: type=gha,mode=max + + shell-lint: + name: Shell Script Lint + needs: changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install ShellCheck + run: sudo apt-get install -y shellcheck + + - name: Run ShellCheck + run: | + find . -name "*.sh" -type f -not -path "./node_modules/*" -not -path "./.git/*" | while read -r script; do + echo "Checking: $script" + shellcheck "$script" || true + done + + pr-status: + name: PR Status + needs: + - changes + - frontend-typecheck + - frontend-lint + - frontend-build + - frontend-test + - frontend-security + - backend-lint + - backend-test + - backend-test-redis-mock + - backend-security + - backend-quart-check + - database-check + - integration-test + - code-quality + - docker-build + - shell-lint + if: always() + runs-on: ubuntu-latest + steps: + - name: Check job results + run: | + echo "## PR Quality Gate Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Frontend" >> $GITHUB_STEP_SUMMARY + echo "- TypeScript: ${{ needs.frontend-typecheck.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Lint (ESLint + Prettier): ${{ needs.frontend-lint.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Build: ${{ needs.frontend-build.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Tests (Vitest): ${{ needs.frontend-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Security: ${{ needs.frontend-security.result }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Backend" >> $GITHUB_STEP_SUMMARY + echo "- Lint (Ruff): ${{ needs.backend-lint.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Tests (pytest + Redis): ${{ needs.backend-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Tests (pytest + fakeredis): ${{ needs.backend-test-redis-mock.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Security: ${{ needs.backend-security.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Quart App: ${{ needs.backend-quart-check.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Database: ${{ needs.database-check.result }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Other Checks" >> $GITHUB_STEP_SUMMARY + echo "- Integration: ${{ needs.integration-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Code Quality: ${{ needs.code-quality.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Docker: ${{ needs.docker-build.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Shell Scripts: ${{ needs.shell-lint.result }}" >> $GITHUB_STEP_SUMMARY + + - name: Determine overall status + run: | + if [[ "${{ needs.frontend-build.result }}" == "failure" ]] || \ + [[ "${{ needs.frontend-typecheck.result }}" == "failure" ]] || \ + [[ "${{ needs.backend-lint.result }}" == "failure" ]] || \ + [[ "${{ needs.backend-test.result }}" == "failure" ]]; then + echo "Critical checks failed!" + exit 1 + fi + echo "All critical checks passed!" diff --git a/.gitignore b/.gitignore index 58037854..7c23fc88 100644 --- a/.gitignore +++ b/.gitignore @@ -10,13 +10,17 @@ backend_things/other_caddys_save_for_later/other caddy /backend_things/data/ /backend_things/logs/ /backend_things/venv/ - +*migrate_problems.py +models.py +*problem_models.py __pycache__/ /backend_things/notes.txt - - - -frontend_things/notes.txt +*.ruff_cache +*settings.json +*.pytest_cache +*frontend_things/notes.txt +/frontend_things/ +/backend_things/ node_modules/ package-lock.json @@ -36,3 +40,4 @@ venv data logs +*note.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..37ba67c4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/README.md b/README.md index 7ea9fb0a..ed04d5a6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Benchr -### Benchmark your code. Compete with the world. - - +### Benchmark your code. Compare two languages. See the difference. --- @@ -10,7 +8,7 @@ Benchr is a real-time code benchmarking platform that lets you write, execute, and analyze code performance across multiple languages. Submit your code and get detailed metrics including CPU cycles, instructions per cycle (IPC), cache behavior, branch predictions, memory usage, and execution time—all running in isolated Firecracker microVMs for security and consistency. Whether -you're optimizing algorithms, comparing implementations, or competing on leaderboards, Benchr gives you the low-level insights +you're optimizing algorithms or comparing implementations, Benchr gives you the low-level insights you need. ### Project Structure diff --git a/backend/IPubSub.py b/backend/IPubSub.py index df625f01..7884bbb0 100644 --- a/backend/IPubSub.py +++ b/backend/IPubSub.py @@ -1,9 +1,10 @@ -from abc import ABC, abstractmethod -from typing import Callable, Optional, Dict, Set -import redis.asyncio as aioredis -import json import asyncio +import json import logging +from abc import ABC, abstractmethod +from typing import Dict, Optional, Set # noqa: UP035 + +import redis.asyncio as aioredis logger = logging.getLogger(__name__) @@ -68,7 +69,7 @@ class RedisPubSub(IPubSub): await pubsub.publish('job_results', {'job_id': 123}) """ - _instance: Optional['RedisPubSub'] = None + _instance: Optional["RedisPubSub"] = None _lock: asyncio.Lock = None def __init__(self): @@ -80,7 +81,7 @@ def __init__(self): self._connected = False @classmethod - async def get_instance(cls, redis_url: str = None) -> 'RedisPubSub': + async def get_instance(cls, redis_url: str = None) -> "RedisPubSub": """ Get singleton instance with lazy initialization @@ -107,7 +108,7 @@ async def connect(self, redis_url: str) -> None: self._redis = await aioredis.from_url(redis_url, decode_responses=True) self._pubsub = self._redis.pubsub() - await self._pubsub.subscribe('job_results') + await self._pubsub.subscribe("job_results") self._running = True self._connected = True self._task = asyncio.create_task(self._listen()) @@ -117,14 +118,14 @@ async def _listen(self): """Background listener task""" try: async for message in self._pubsub.listen(): - if message['type'] != 'message': + if message["type"] != "message": continue - channel = message['channel'] + channel = message["channel"] print(f"[RedisPubSub] Received on {channel}: {message['data']}") try: - data = json.loads(message['data']) + data = json.loads(message["data"]) if channel in self._subscribers: for queue in list(self._subscribers[channel]): diff --git a/backend/IQueue.py b/backend/IQueue.py index 896deca7..1246d008 100644 --- a/backend/IQueue.py +++ b/backend/IQueue.py @@ -1,7 +1,9 @@ -from abc import ABC, abstractmethod import queue +from abc import abstractmethod + import redis -import os +from dotenv import load_dotenv + class IQueue: def __init__(self, maxsize, env): @@ -32,11 +34,10 @@ def hasFront(self): def size(self): pass + class GlobalQueue(IQueue): - def __init__(self, - maxsize = 1024 - ): - self._queue = IQueue(maxsize) + def __init__(self, maxsize=1024): + self._queue = queue.Queue(maxsize=maxsize) def full(self): return self._queue.full() @@ -45,12 +46,12 @@ def empty(self): return self._queue.empty() def push(self, program_id: str): - if (self._queue.full()): + if self._queue.full(): return False self._queue.put(program_id) return True - + def pop(self): if self.empty(): return None @@ -59,20 +60,16 @@ def pop(self): def hasFront(self): return self._queue.qsize() > 0 - + def size(self): return self._queue.qsize() class RedisQueue(IQueue): - def __init__(self, - name, - redis_url, - maxsize = 1024 - ): + def __init__(self, name, redis_url, maxsize=1024): self.name = name self.redis_url = redis_url - self.redis = None # multiclients connect to one redis instance + self.redis = None # multiclients connect to one redis instance self.queued_key = f"{name}:queued" self.processing_key = f"{name}:processing" self.notify_channel = f"{name}:notify" @@ -86,13 +83,13 @@ def init(self): self.redis_url, decode_responses=True, socket_connect_timeout=5, - socket_keepalive=True + socket_keepalive=True, ) # test connection self.redis.ping() print(f"REDIS: connected to redis: {self.redis_url}") - except redis.ConnectionError as e: - print(f"REDIS: ERROR: cannot connect to redis") + except redis.ConnectionError: + print("REDIS: ERROR: cannot connect to redis") raise except Exception as e: print(f"REDIS: ERROR: {e}") @@ -112,34 +109,34 @@ def empty(self): def push(self, job_id: int): """ Add job to queued list - + Returns: True if added, False if queue is full """ if self.full(): print(f"[RedisQueue] Queue full, cannot push job {job_id}") return False - + try: # Add to right of queued list (FIFO) self.redis.rpush(self.queued_key, job_id) - + # Optional: notify subscribers self.redis.publish(self.notify_channel, job_id) - + return True except redis.ConnectionError as e: print(f"[RedisQueue] Connection error in push(): {e}") return False - + def pend(self, timeout=5): """ Move job from queued to processing (atomic operation) This marks the job as being worked on. - + Args: timeout: Seconds to wait for a job (blocking) - + Returns: job_id (int) or None if timeout """ @@ -147,56 +144,54 @@ def pend(self, timeout=5): # Atomically move from queued to processing # BRPOPLPUSH: blocking right pop from queued, left push to processing result = self.redis.brpoplpush( - self.queued_key, - self.processing_key, - timeout=timeout + self.queued_key, self.processing_key, timeout=timeout ) - + if result: return int(result) return None - + except redis.ConnectionError as e: print(f"[RedisQueue] Connection error in pend(): {e}") return None except ValueError as e: print(f"[RedisQueue] Invalid job_id in pend(): {e}") return None - + def pop(self, timeout=5): """ Remove job from processing queue (job is complete) - + Args: timeout: Seconds to wait for a job (blocking) - + Returns: job_id (int) or None if timeout/empty """ if self.redis.llen(self.processing_key) == 0: return None - + try: # Remove from right of processing list result = self.redis.brpop(self.processing_key, timeout=timeout) - + if result: _, job_id = result return int(job_id) return None - + except redis.ConnectionError as e: print(f"[RedisQueue] Connection error in pop(): {e}") return None except ValueError as e: print(f"[RedisQueue] Invalid job_id in pop(): {e}") return None - + def hasFront(self): """ Check if there are jobs waiting in queued list Note: processing jobs don't count as "queued" - + Returns: True if queued list has items """ @@ -204,11 +199,11 @@ def hasFront(self): return self.redis.llen(self.queued_key) > 0 except redis.ConnectionError: return False - + def size(self): """ Total size = queued + processing - + Returns: Total number of jobs in both lists """ @@ -219,21 +214,21 @@ def size(self): except redis.ConnectionError: print("[RedisQueue] Connection error in size()") return 0 - + def queued_size(self): """Get size of queued list only""" try: return self.redis.llen(self.queued_key) except redis.ConnectionError: return 0 - + def processing_size(self): """Get size of processing list only""" try: return self.redis.llen(self.processing_key) except redis.ConnectionError: return 0 - + def clear(self): """Clear both queued and processing lists (use with caution!)""" try: @@ -242,7 +237,7 @@ def clear(self): print("[RedisQueue] Queue cleared") except redis.ConnectionError: print("[RedisQueue] Connection error in clear()") - + def peek_queued(self): """Look at first queued job without removing it""" try: @@ -252,7 +247,7 @@ def peek_queued(self): return None except (redis.ConnectionError, ValueError): return None - + def peek_processing(self): """Look at first processing job without removing it""" try: @@ -262,12 +257,12 @@ def peek_processing(self): return None except (redis.ConnectionError, ValueError): return None - + def requeue_processing(self): """ Move all processing jobs back to queued (recovery operation) Use this if JobManager crashes and you want to retry jobs - + Returns: Number of jobs moved """ @@ -278,11 +273,11 @@ def requeue_processing(self): if not result: break count += 1 - + if count > 0: print(f"[RedisQueue] Requeued {count} processing jobs") return count - + except redis.ConnectionError: print("[RedisQueue] Connection error in requeue_processing()") return 0 diff --git a/backend/agent.py b/backend/agent.py index 48452009..38fcaaae 100644 --- a/backend/agent.py +++ b/backend/agent.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 -import socket -import struct +# import struct +import contextlib import json -import subprocess -import tempfile import os import shutil -import env -from util import Container, FirecrackerCfg, send_sock, rec_sock, run_cmd, \ -ISerializer, JsonSerializer +import socket +import subprocess +import tempfile + +# import env +from util import JsonSerializer, rec_sock, send_sock EXECUTE_SCRIPT = "/mnt/deploy/execute.sh" CFG = "vm_config.json" @@ -16,161 +17,139 @@ VSOCK_PORT = 5000 SER = JsonSerializer() + def execute_job(job_data: dict) -> dict: """Execute the job using execute.sh script""" - code = job_data.get('code', '') - lang = job_data.get('lang', 'cpp') - compiler = job_data.get('compiler', 'g++') - opts = job_data.get('opts', '-O2 -Wall') - - print(f"[Agent] Executing job, language: {lang}, compiler: {compiler}, opts: {opts}") - + code = job_data.get("code", "") + lang = job_data.get("lang", "cpp") + compiler = job_data.get("compiler", "g++") + opts = job_data.get("opts", "-O2 -Wall") + + print( + f"[Agent] Executing job, language: {lang}, compiler: {compiler}, opts: {opts}" + ) + tmpdir = tempfile.mkdtemp() - + try: - ext_map = { - 'c': '.c', - 'cpp': '.cpp', - 'py': '.py', - 'python': '.py' - } - ext = ext_map.get(lang, '.cpp') + ext_map = {"c": ".c", "cpp": ".cpp", "py": ".py", "python": ".py"} + ext = ext_map.get(lang, ".cpp") src_file = os.path.join(tmpdir, f"source{ext}") - - with open(src_file, 'w') as f: + + with open(src_file, "w") as f: f.write(code) - + result_json_path = os.path.join(tmpdir, "result.json") - - cmd = [ - EXECUTE_SCRIPT, - tmpdir, - src_file, - lang, - compiler, - opts - ] - + + cmd = [EXECUTE_SCRIPT, tmpdir, src_file, lang, compiler, opts] + print(f"[Agent] Running: {' '.join(cmd)}") - - proc = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=30 - ) - + + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if os.path.exists(result_json_path): - with open(result_json_path, 'r') as f: + with open(result_json_path) as f: result = json.load(f) - print(f"[Agent] Execution complete, success: {result.get('success', False)}") + print( + f"[Agent] Execution complete, success: {result.get('success', False)}" + ) else: result = { - 'success': False, - 'error': 'No result file generated', - 'stdout': proc.stdout, - 'stderr': proc.stderr, - 'exit_code': proc.returncode + "success": False, + "error": "No result file generated", + "stdout": proc.stdout, + "stderr": proc.stderr, + "exit_code": proc.returncode, } - print(f"[Agent] Execution failed: no result file") - + print("[Agent] Execution failed: no result file") + except subprocess.TimeoutExpired: - result = { - 'success': False, - 'error': 'Execution timeout (30s)' - } - print(f"[Agent] Execution timeout") + result = {"success": False, "error": "Execution timeout (30s)"} + print("[Agent] Execution timeout") except Exception as e: - result = { - 'success': False, - 'error': str(e) - } + result = {"success": False, "error": str(e)} print(f"[Agent] Execution error: {e}") finally: - try: + with contextlib.suppress(Exception): shutil.rmtree(tmpdir) - except: - pass - + return result + def main(): """Main agent loop - listen on vsock and process jobs""" # Read VM configuration try: - with open(CFG, 'r') as f: + with open(CFG) as f: config = json.load(f) cid = config.get("vsock", {}).get("cid", socket.VMADDR_CID_ANY) port = config.get("vsock", {}).get("port", VSOCK_PORT) - print(f"[Agent] Loaded config: CID={cid}, PORT={port}") - except Exception as e: - print(f"[Agent] Error reading config: {e}, using defaults") + print("[Agent] Loaded config: CID={cid}, PORT={port}") + except Exception: + print("[Agent] Error reading config: {e}, using defaults") cid = socket.VMADDR_CID_ANY port = VSOCK_PORT - print(f"[Agent] Starting on vsock port {port}") - + print("[Agent] Starting on vsock port {port}") + # Create vsock socket sock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) - #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((cid, port)) sock.listen(1) - - print(f"[Agent] Listening on {cid}:{port}") - + + print("[Agent] Listening on {cid}:{port}") + while True: conn = None try: print("[Agent] Waiting for connection...") conn, addr = sock.accept() - print(f"[Agent] Connected: {addr}") + print("[Agent] Connected: {addr}") print("[Agent] Ready to receive jobs (Firecracker handled handshake)") - + # Process jobs on this connection # NO handshake needed - Firecracker already sent OK to host while True: try: # Receive job data: {code, lang, compiler, opts} job_bytes = rec_sock(conn) - job_data = SER.deserialize(job_bytes) - print(f"[Agent] Received job") - + job_data = SER.deserialize(job_bytes) + print("[Agent] Received job") + # Execute job result = execute_job(job_data) - + # Send result back result_bytes = SER.serialize(result) send_sock(conn, result_bytes) - - print(f"[Agent] Sent result") - + + print("[Agent] Sent result") + except Exception as e: - print(f"[Agent] Error processing job: {e}") - error_result = { - 'success': False, - 'error': str(e) - } + print("[Agent] Error processing job: {e}") + error_result = {"success": False, "error": str(e)} try: send_sock(conn, SER.serialize(error_result)) - except: + except Exception: break - + except KeyboardInterrupt: print("\n[Agent] Shutting down...") break - except Exception as e: - print(f"[Agent] Connection error: {e}") + except Exception: + print("[Agent] Connection error: {e}") import traceback + traceback.print_exc() continue finally: if conn: - try: + with contextlib.suppress(Exception): conn.close() - except: - pass - + sock.close() + if __name__ == "__main__": main() diff --git a/backend/api.py b/backend/api.py index 58fdde28..87ef3703 100644 --- a/backend/api.py +++ b/backend/api.py @@ -1,33 +1,32 @@ -from quart import Quart, request, jsonify, websocket -from quart_cors import cors -from models import db, Job, init_db -from job_cache import JobCache -from rate_limiter import RateLimitedQueue +import asyncio +import contextlib import json -import os import logging -from config import Config import sys -import asyncio +from asyncio.events import os + import redis.asyncio as aioredis +from config import Config +from job_cache import JobCache +from models import Job, db, init_db +from quart import Quart, jsonify, request, websocket +from quart_cors import cors +from rate_limiter import RateLimitedQueue DEBUG = True REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") # Logging setup -os.makedirs('logs', exist_ok=True) +os.makedirs("logs", exist_ok=True) logging.basicConfig( level=getattr(logging, Config.LOG_LEVEL), - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(Config.LOG_FILE), - logging.StreamHandler(sys.stdout) - ] + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[logging.FileHandler(Config.LOG_FILE), logging.StreamHandler(sys.stdout)], ) logger = logging.getLogger(__name__) app = Quart(__name__) -app = cors(app, allow_origin=Config.ALLOWED_ORIGINS.split(',')) +app = cors(app, allow_origin=Config.ALLOWED_ORIGINS.split(",")) # Globals - initialized in startup queue: RateLimitedQueue = None @@ -49,7 +48,7 @@ async def startup(): queue_name=os.getenv("RATE_QUEUE_NAME", "benchr"), max_requests=int(os.getenv("RATE_MAX_REQUESTS", "100")), window_seconds=int(os.getenv("RATE_WINDOW_SEC", "60")), - max_queue_size=int(os.getenv("RATE_MAX_QUEUE_SIZE", "1000")) + max_queue_size=int(os.getenv("RATE_MAX_QUEUE_SIZE", "1000")), ) await queue.connect() @@ -72,10 +71,8 @@ async def shutdown(): if pubsub_task: pubsub_task.cancel() - try: + with contextlib.suppress(asyncio.CancelledError): await pubsub_task - except asyncio.CancelledError: - pass if queue: await queue.disconnect() @@ -105,21 +102,27 @@ async def message_handler(message): print(f"[Quart] Broadcasting job result for job_id={job_id}", flush=True) if job_id in ws_clients: - print(f"[Quart] Found {len(ws_clients[job_id])} subscribers for job {job_id}", flush=True) + print( + f"[Quart] Found {len(ws_clients[job_id])} subscribers for job {job_id}", + flush=True, + ) # Fetch job result from cache/database job_result = await cache.get(int(job_id)) print(f"[Quart] Fetched job result: {job_result}", flush=True) if job_result is None: - print(f"[Quart] WARNING: No job result found for job_id={job_id}", flush=True) - job_result = {'error': 'Job result not found', 'job_id': job_id} + print( + f"[Quart] WARNING: No job result found for job_id={job_id}", + flush=True, + ) + job_result = {"error": "Job result not found", "job_id": job_id} # Wrap in a proper message format ws_message = { - 'type': 'job_complete', - 'job_id': job_id, - 'data': job_result + "type": "job_complete", + "job_id": job_id, + "data": job_result, } # Broadcast to all subscribers @@ -128,13 +131,17 @@ async def message_handler(message): # Remove subscribers after sending result del ws_clients[job_id] - print(f"[Quart] Broadcast complete, removed subscribers for job {job_id}", flush=True) + print( + f"[Quart] Broadcast complete, removed subscribers for job {job_id}", + flush=True, + ) else: print(f"[Quart] No subscribers for job {job_id}", flush=True) except Exception as e: print(f"[Quart] Error processing message: {e}", flush=True) import traceback + traceback.print_exc() try: @@ -143,7 +150,7 @@ async def message_handler(message): print("[Quart] Pubsub connection created") async with pubsub_conn.pubsub() as pubsub: - await pubsub.subscribe(**{'job_results': message_handler}) + await pubsub.subscribe(**{"job_results": message_handler}) print("[Quart] Subscribed to job_results channel") # Run forever, processing messages as they arrive @@ -155,13 +162,16 @@ async def message_handler(message): except Exception as e: print(f"[Quart] Pubsub listener FATAL ERROR: {e}") import traceback + traceback.print_exc() finally: print("[Quart] Pubsub listener cleanup complete") -# =========== WebSocket =========== -@app.websocket('/ws') +# WebSocket + + +@app.websocket("/ws") async def ws(): """WebSocket endpoint for real-time job updates""" print("[WS] New connection attempt", flush=True) @@ -185,30 +195,42 @@ async def receive(): data = await websocket.receive_json() print(f"[WS] Received message: {data}", flush=True) - if data.get('type') == 'subscribe': - job_id = str(data.get('job_id')) + if data.get("type") == "subscribe": + job_id = str(data.get("job_id")) print(f"[WS] Subscribe request for job_id={job_id}", flush=True) if job_id: if job_id not in ws_clients: ws_clients[job_id] = set() ws_clients[job_id].add(client_queue) subscribed_jobs.add(job_id) - await websocket.send_json({'type': 'subscribed', 'job_id': job_id}) - print(f"[WS] Client subscribed to job {job_id}, total subscribers: {len(ws_clients[job_id])}", flush=True) + await websocket.send_json( + {"type": "subscribed", "job_id": job_id} + ) + print( + f"[WS] Client subscribed to job {job_id}, total subscribers: {len(ws_clients[job_id])}", # noqa: E501 + flush=True, + ) else: print("[WS] Subscribe request missing job_id", flush=True) - elif data.get('type') == 'unsubscribe': - job_id = str(data.get('job_id')) - print(f"[WS] Unsubscribe request for job_id={job_id}", flush=True) + elif data.get("type") == "unsubscribe": + job_id = str(data.get("job_id")) + print( + f"[WS] Unsubscribe request for job_id={job_id}", flush=True + ) if job_id in subscribed_jobs: subscribed_jobs.discard(job_id) if job_id in ws_clients: ws_clients[job_id].discard(client_queue) - print(f"[WS] Client unsubscribed from job {job_id}", flush=True) + print( + f"[WS] Client unsubscribed from job {job_id}", + flush=True, + ) else: - print(f"[WS] Unknown message type: {data.get('type')}", flush=True) + print( + f"[WS] Unknown message type: {data.get('type')}", flush=True + ) except json.JSONDecodeError as e: print(f"[WS] Invalid JSON received: {e}", flush=True) @@ -234,28 +256,28 @@ async def send(): send_task = asyncio.create_task(send()) done, pending = await asyncio.wait( - [receive_task, send_task], - return_when=asyncio.FIRST_EXCEPTION + [receive_task, send_task], return_when=asyncio.FIRST_EXCEPTION ) # Cancel pending tasks for task in pending: task.cancel() - try: + with contextlib.suppress(asyncio.CancelledError): await task - except asyncio.CancelledError: - pass # Check for exceptions for task in done: if task.exception(): - print(f"[WS] Task failed with exception: {task.exception()}", flush=True) + print( + f"[WS] Task failed with exception: {task.exception()}", flush=True + ) except asyncio.CancelledError: print("[WS] Connection cancelled", flush=True) except Exception as e: print(f"[WS] Connection error: {e}", flush=True) import traceback + traceback.print_exc() finally: print(f"[WS] Cleaning up, subscribed_jobs={subscribed_jobs}", flush=True) @@ -268,9 +290,10 @@ async def send(): print("[WS] Connection closed", flush=True) -# =========== REST API =========== +# REST API -@app.route('/api/submit', methods=['POST']) + +@app.route("/api/submit", methods=["POST"]) async def submit_job(): """Submit a new benchmark job""" try: @@ -279,21 +302,24 @@ async def submit_job(): if DEBUG: print(f"[SUBMIT] Received: {data}") - if not data.get('code'): - return jsonify({'error': 'Code is required'}), 400 + if not data.get("code"): + return jsonify({"error": "Code is required"}), 400 - if not data.get('lang'): - return jsonify({'error': 'Language is required'}), 400 + if not data.get("lang"): + return jsonify({"error": "Language is required"}), 400 # Create job in database (sync peewee, run in executor) loop = asyncio.get_event_loop() - job = await loop.run_in_executor(None, lambda: Job.create( - code=data['code'], - lang=data['lang'], - compiler=data.get('compiler', 'gcc'), - opts=data.get('opts', '-O2'), - status='queued' - )) + job = await loop.run_in_executor( + None, + lambda: Job.create( + code=data["code"], + lang=data["lang"], + compiler=data.get("compiler", "gcc"), + opts=data.get("opts", "-O2"), + status="queued", + ), + ) print(f"[SUBMIT] Job created: ID={job.id}") @@ -303,24 +329,22 @@ async def submit_job(): if not success: # Rate limited or queue full await loop.run_in_executor(None, lambda: Job.delete_by_id(job.id)) - return jsonify({'error': 'Rate limit exceeded or queue full'}), 429 + return jsonify({"error": "Rate limit exceeded or queue full"}), 429 queue_size = await queue.size() print(f"[SUBMIT] Job {job.id} queued. Queue size: {queue_size}") - return jsonify({ - 'job_id': job.id, - 'status': 'queued' - }), 201 + return jsonify({"job_id": job.id, "status": "queued"}), 201 except Exception as e: print(f"[SUBMIT] ERROR: {e}") import traceback + traceback.print_exc() - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/jobs/', methods=['GET']) +@app.route("/api/jobs/", methods=["GET"]) async def get_job(id): """Get job by ID""" try: @@ -331,8 +355,8 @@ async def get_job(id): job_data = dict(job.__data__) - if job_data.get('result'): - job_data['result'] = json.loads(job_data['result']) + if job_data.get("result"): + job_data["result"] = json.loads(job_data["result"]) print(f"[GET_JOB] Returning job {id}: status={job_data.get('status')}") return jsonify(job_data) @@ -340,95 +364,102 @@ async def get_job(id): except Exception as e: print(f"[GET_JOB] ERROR for job {id}: {e}") import traceback + traceback.print_exc() - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/current', methods=['GET']) +@app.route("/api/current", methods=["GET"]) async def get_current_job(): """Get most recent job""" try: loop = asyncio.get_event_loop() job = await loop.run_in_executor( - None, - lambda: Job.select().order_by(Job.created_at.desc()).first() + None, lambda: Job.select().order_by(Job.created_at.desc()).first() ) if not job: - return jsonify({'job': None}) + return jsonify({"job": None}) job_data = await cache.get(job.id) - return jsonify({'job': job_data}) + return jsonify({"job": job_data}) except Exception as e: print(f"[CURRENT] ERROR: {e}") - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/jobs', methods=['GET']) +@app.route("/api/jobs", methods=["GET"]) async def list_jobs(): """List jobs (newest first)""" try: - limit = int(request.args.get('limit', 10)) + limit = int(request.args.get("limit", 10)) loop = asyncio.get_event_loop() jobs = await loop.run_in_executor( None, - lambda: list(Job.select().order_by(Job.created_at.desc()).limit(limit)) + lambda: list(Job.select().order_by(Job.created_at.desc()).limit(limit)), ) - job_list = [{ - 'job_id': job.id, - 'lang': job.lang, - 'compiler': job.compiler, - 'status': job.status, - 'created_at': job.created_at.isoformat(), - 'completed_at': job.completed_at.isoformat() if job.completed_at else -None - } for job in jobs] - - return jsonify({'jobs': job_list}) + job_list = [ + { + "job_id": job.id, + "lang": job.lang, + "compiler": job.compiler, + "status": job.status, + "created_at": job.created_at.isoformat(), + "completed_at": job.completed_at.isoformat() + if job.completed_at + else None, + } + for job in jobs + ] + + return jsonify({"jobs": job_list}) except Exception as e: print(f"[LIST_JOBS] ERROR: {e}") - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/health', methods=['GET']) +@app.route("/api/health", methods=["GET"]) async def health(): """Health check""" - return jsonify({ - 'status': 'ok', - 'database': 'connected' if not db.is_closed() else 'disconnected' - }) + return jsonify( + { + "status": "ok", + "database": "connected" if not db.is_closed() else "disconnected", + } + ) + +# Saved Benchmarks -# =========== Saved Benchmarks =========== -@app.route('/api/saved', methods=['POST']) +@app.route("/api/saved", methods=["POST"]) async def save_benchmark(): """Save a job as a benchmark""" try: data = await request.json - job_id = data.get('job_id') - name = data.get('name') + job_id = data.get("job_id") + name = data.get("name") if not job_id: - return jsonify({'error': 'job_id is required'}), 400 + return jsonify({"error": "job_id is required"}), 400 benchmark_id = await cache.save_benchmark(job_id, name) if benchmark_id: - return jsonify({'benchmark_id': benchmark_id}), 201 + return jsonify({"benchmark_id": benchmark_id}), 201 else: - return jsonify({'error': 'Failed to save benchmark'}), 500 + return jsonify({"error": "Failed to save benchmark"}), 500 except Exception as e: print(f"[SAVE] ERROR: {e}") - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/saved/', methods=['GET']) +@app.route("/api/saved/", methods=["GET"]) async def get_saved_benchmark(id): """Get saved benchmark by ID (cached)""" try: @@ -437,51 +468,53 @@ async def get_saved_benchmark(id): if data: return jsonify(data) else: - return jsonify({'error': 'Benchmark not found'}), 404 + return jsonify({"error": "Benchmark not found"}), 404 except Exception as e: print(f"[GET_SAVED] ERROR: {e}") - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/saved/', methods=['DELETE']) +@app.route("/api/saved/", methods=["DELETE"]) async def delete_saved_benchmark(id): """Delete saved benchmark""" try: success = await cache.delete_saved(id) if success: - return jsonify({'status': 'deleted'}), 200 + return jsonify({"status": "deleted"}), 200 else: - return jsonify({'error': 'Failed to delete'}), 500 + return jsonify({"error": "Failed to delete"}), 500 except Exception as e: print(f"[DELETE_SAVED] ERROR: {e}") - return jsonify({'error': str(e)}), 500 + return jsonify({"error": str(e)}), 500 -@app.route('/api/chat', methods=['POST']) +@app.route("/api/chat", methods=["POST"]) async def chat(): """Claude AI chat endpoint (placeholder)""" try: data = await request.json - if not data.get('message'): - return jsonify({'error': 'Message is required'}), 400 + if not data.get("message"): + return jsonify({"error": "Message is required"}), 400 - message = data['message'] - result = data.get('result') + message = data["message"] # TODO: Implement Claude AI integration response_text = f"Received message: {message}" print(f"Chat request received: {message[:50]}") - return jsonify({'response': response_text}) + return jsonify({"response": response_text}) except Exception as e: - print(f"Error in chat endpoint: {e}", exc_info=True) - return jsonify({'error': str(e)}), 500 + print(f"Error in chat endpoint: {e}") + import traceback + + traceback.print_exc() + return jsonify({"error": str(e)}), 500 if __name__ == "__main__": diff --git a/backend/config.py b/backend/config.py index 7b116bdd..49b2c9b0 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,29 +1,25 @@ import os + class Config: - - DB_PATH = os.getenv('DB_PATH', 'data/benchr.db') - - - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379') - REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') - REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) - - - QUEUE_NAME = os.getenv('RATE_QUEUE_NAME', 'benchmark_jobs') - RATE_MAX_REQUESTS = int(os.getenv('RATE_MAX_REQUESTS', '100')) - RATE_WINDOW_SEC = int(os.getenv('RATE_WINDOW_SEC', '3600')) - RATE_MAX_QUEUE_SIZE = int(os.getenv('RATE_MAX_QUEUE_SIZE', '1000')) - - - FLASK_HOST = os.getenv('FLASK_HOST', '127.0.0.1') - FLASK_PORT = int(os.getenv('FLASK_PORT', '5000')) - FLASK_DEBUG = os.getenv('FLASK_DEBUG', 'False').lower() == 'true' - - - LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO') - LOG_FILE = os.getenv('LOG_FILE', 'logs/api.log') - + DB_PATH = os.getenv("DB_PATH", "data/benchr.db") + + REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379") + REDIS_HOST = os.getenv("REDIS_HOST", "localhost") + REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) + + QUEUE_NAME = os.getenv("RATE_QUEUE_NAME", "benchmark_jobs") + RATE_MAX_REQUESTS = int(os.getenv("RATE_MAX_REQUESTS", "100")) + RATE_WINDOW_SEC = int(os.getenv("RATE_WINDOW_SEC", "3600")) + RATE_MAX_QUEUE_SIZE = int(os.getenv("RATE_MAX_QUEUE_SIZE", "1000")) + + FLASK_HOST = os.getenv("FLASK_HOST", "127.0.0.1") + FLASK_PORT = int(os.getenv("FLASK_PORT", "5000")) + FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() == "true" + + LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") + LOG_FILE = os.getenv("LOG_FILE", "logs/api.log") + # Security - API_KEY_HEADER = 'X-API-Key' - ALLOWED_ORIGINS = os.getenv('ALLOWED_ORIGINS', '*') + API_KEY_HEADER = "X-API-Key" + ALLOWED_ORIGINS = os.getenv("ALLOWED_ORIGINS", "*") diff --git a/backend/env.py b/backend/env.py index b1bf9719..eeee1852 100644 --- a/backend/env.py +++ b/backend/env.py @@ -1,2 +1,2 @@ -PORT_START=5000 -CONFIG_MNT_INDEX=1 +PORT_START = 5000 +CONFIG_MNT_INDEX = 1 diff --git a/backend/job_cache.py b/backend/job_cache.py index 0c234090..5c15c284 100644 --- a/backend/job_cache.py +++ b/backend/job_cache.py @@ -6,10 +6,13 @@ All operations fall through to the database directly via models. """ -from models import db, Job, JobMetrics -from typing import Optional import datetime +import json import logging +from typing import Optional + +import redis.asyncio as aioredis +from models import Job, JobMetrics, db logger = logging.getLogger(__name__) @@ -31,11 +34,22 @@ def __init__(self, redis_url: str = None): """ self.redis_url = redis_url self._connected = False + self.redis: Optional[aioredis.Redis] = None async def connect(self) -> None: - """Connect to cache backend (no-op for now).""" + """Connect to cache backend.""" if not self._connected: db.connect(reuse_if_open=True) + + if self.redis_url: + try: + self.redis = await aioredis.from_url( + self.redis_url, decode_responses=True + ) + logger.info(f"[JobCache] Connected to Redis: {self.redis_url}") + except Exception as e: + logger.error(f"[JobCache] Failed to connect to Redis: {e}") + self._connected = True logger.info("[JobCache] Connected (database passthrough mode)") @@ -43,6 +57,10 @@ async def disconnect(self) -> None: """Disconnect from cache backend.""" if self._connected and not db.is_closed(): db.close() + + if self.redis: + await self.redis.close() + self._connected = False logger.info("[JobCache] Disconnected") @@ -59,13 +77,13 @@ async def get(self, job_id: int) -> Optional[dict]: try: job = Job.get_by_id(job_id) return { - 'id': job.id, - 'code': job.code, - 'lang': job.lang, - 'compiler': job.compiler, - 'opts': job.opts, - 'status': job.status, - 'result': job.get_result() + "id": job.id, + "code": job.code, + "lang": job.lang, + "compiler": job.compiler, + "opts": job.opts, + "status": job.status, + "result": job.get_result(), } except Exception as e: logger.warning(f"[JobCache] Failed to get job {job_id}: {e}") @@ -84,14 +102,14 @@ async def update(self, job_id: int, result: dict) -> bool: """ try: job = Job.get_by_id(job_id) - job_result = result.get('result', {}) + job_result = result.get("result", {}) job.set_result(job_result) - job.status = result.get('status', 'completed') + job.status = result.get("status", "completed") job.completed_at = datetime.datetime.now() job.save() # Save metrics if successful - if job_result.get('success'): + if job_result.get("success"): self._save_metrics(job, job_result) return True @@ -102,21 +120,21 @@ async def update(self, job_id: int, result: dict) -> bool: def _save_metrics(self, job: Job, result: dict) -> None: """Save job metrics to database.""" try: - perf = result.get('perf', {}) - time_data = result.get('time', {}) + perf = result.get("perf", {}) + time_data = result.get("time", {}) - cycles = perf.get('cycles') - instructions = perf.get('instructions') + cycles = perf.get("cycles") + instructions = perf.get("instructions") ipc = instructions / cycles if cycles and cycles > 0 else None - exec_time = time_data.get('elapsed_time_seconds') + exec_time = time_data.get("elapsed_time_seconds") JobMetrics.create( job=job, cycles=cycles, instructions=instructions, ipc=ipc, - execution_time_ms=exec_time * 1000 if exec_time else None + execution_time_ms=exec_time * 1000 if exec_time else None, ) except Exception as e: logger.warning(f"[JobCache] Failed to save metrics for job {job.id}: {e}") @@ -135,9 +153,36 @@ async def save_benchmark(self, job_id: int, name: str) -> Optional[int]: Note: This is a no-op until Redis caching is set up. Returns None to indicate the feature is not yet available. """ - # TODO: Implement Redis-based saved benchmarks - logger.info(f"[JobCache] save_benchmark called (no-op) - job_id={job_id}, name={name}") - return None + if not self.redis: + logger.warning("[JobCache] Redis not configured, cannot save benchmark") + return None + + try: + # Get job data + job_data = await self.get(job_id) + if not job_data: + logger.warning( + f"[JobCache] Job {job_id} not found, cannot save benchmark" + ) + return None + + # Add name and timestamp + job_data["name"] = name + job_data["saved_at"] = datetime.datetime.now().isoformat() + + # Generate ID + benchmark_id = await self.redis.incr("benchmarks:count") + + # Save to Redis + key = f"benchmark:{benchmark_id}" + await self.redis.set(key, json.dumps(job_data)) + + logger.info(f"[JobCache] Saved benchmark {benchmark_id} for job {job_id}") + return benchmark_id + + except Exception as e: + logger.error(f"[JobCache] Failed to save benchmark: {e}") + return None async def get_saved(self, benchmark_id: int) -> Optional[dict]: """ @@ -148,12 +193,19 @@ async def get_saved(self, benchmark_id: int) -> Optional[dict]: Returns: Benchmark data dict or None if not found - - Note: This is a no-op until Redis caching is set up. """ - # TODO: Implement Redis-based saved benchmarks - logger.info(f"[JobCache] get_saved called (no-op) - benchmark_id={benchmark_id}") - return None + if not self.redis: + return None + + try: + key = f"benchmark:{benchmark_id}" + data = await self.redis.get(key) + if data: + return json.loads(data) + return None + except Exception as e: + logger.error(f"[JobCache] Failed to get saved benchmark: {e}") + return None async def delete_saved(self, benchmark_id: int) -> bool: """ @@ -164,9 +216,14 @@ async def delete_saved(self, benchmark_id: int) -> bool: Returns: True if deleted successfully - - Note: This is a no-op until Redis caching is set up. """ - # TODO: Implement Redis-based saved benchmarks - logger.info(f"[JobCache] delete_saved called (no-op) - benchmark_id={benchmark_id}") - return False + if not self.redis: + return False + + try: + key = f"benchmark:{benchmark_id}" + result = await self.redis.delete(key) + return result > 0 + except Exception as e: + logger.error(f"[JobCache] Failed to delete saved benchmark: {e}") + return False diff --git a/backend/job_manager.py b/backend/job_manager.py index cee22b1e..318c121d 100644 --- a/backend/job_manager.py +++ b/backend/job_manager.py @@ -1,17 +1,19 @@ -from dataclasses import dataclass -import os import asyncio -from typing import Dict, Optional, List -import time +import contextlib +import json # noqa: F401 +import os +import shutil import struct -from util import ISerializer, JsonSerializer -from rate_limiter import RateLimitedQueue -from job_cache import JobCache -from IPubSub import get_pubsub +import time +from dataclasses import dataclass +from typing import Optional, dict, list # noqa: UP035 + import env from dotenv import load_dotenv -import json -import shutil +from IPubSub import get_pubsub +from job_cache import JobCache +from rate_limiter import RateLimitedQueue +from util import ISerializer, JsonSerializer load_dotenv() @@ -44,7 +46,7 @@ class Container: log: str vsock: str port: int - handle: Optional[asyncio.subprocess.Process] = None + handle: Optional[asyncio.subprocess.Process] = None # noqa: UP045 ready: bool = False @@ -74,8 +76,8 @@ def __init__(self, num_vms: int, env=".env", ser: ISerializer = None): self._fc = Firecracker() self._ctr_run = "/var/run/firecracker" self._num_vms = num_vms - self._ctrs_q: List[Container] = [] # queue - self._pending: Dict[int, asyncio.Future] = {} + self._ctrs_q: list[Container] = [] # queue + self._pending: dict[int, asyncio.Future] = {} self.event = asyncio.Event() self._serializer = ser or JsonSerializer() load_dotenv(env) @@ -144,7 +146,7 @@ async def start_pool(self): *[self._check_ready(ctr) for ctr in self._ctrs_q] ) - for ctr, success in zip(self._ctrs_q, results): + for ctr, success in zip(self._ctrs_q, results): # noqa: B905 if not success: raise RuntimeError(f"Failed to start container pool: container" f"{ctr.cid} failed to start") @@ -169,13 +171,13 @@ def create_configs(self): try: os.makedirs(self._fc.reports_path, exist_ok=True) except Exception as e: - raise RuntimeError(f"makedirs failed: {e}") + raise RuntimeError(f"makedirs failed: {e}") # noqa: B904 if not os.path.exists(self._ctr_run): try: os.makedirs(self._ctr_run, exist_ok=True) except Exception as e: - raise RuntimeError(f"makedirs failed: {e}") + raise RuntimeError(f"makedirs failed: {e}") # noqa: B904 for i in range(self._num_vms): cid = 3 + i # NOTE: cid needs to start at 3! @@ -190,7 +192,7 @@ def create_configs(self): os.makedirs(reports_path, exist_ok=True) os.makedirs(run_path, exist_ok=True) except Exception as e: - raise RuntimeError(f"makedirs failed: {e}") + raise RuntimeError(f"makedirs failed: {e}") # noqa: B904 # Shared VM deployment - copy mount files for f in self._mnt: @@ -228,7 +230,7 @@ def create_configs(self): f.write(ser) vmf.write(vm_ser) except Exception as e: - raise RuntimeError(f"json dump failed: {e}") + raise RuntimeError(f"json dump failed: {e}") # noqa: B904 # NOTE: NOT ready until started! ctr = Container( @@ -241,7 +243,7 @@ def create_configs(self): ) self._ctrs_q.append(ctr) - def get_ready(self) -> Optional[Container]: + def get_ready(self) -> Optional[Container]: # noqa: UP045 """Get first available container from pool""" for ctr in self._ctrs_q: if ctr.ready: @@ -332,7 +334,7 @@ async def _reset_ctr(self, ctr: Container): await self._start_ctr(ctr) await self._check_ready(ctr) except Exception as e: - raise RuntimeError(f"Failed to reset container {ctr.cid}: {e}") + raise RuntimeError(f"Failed to reset container {ctr.cid}: {e}") # noqa: B904 async def submit_job(self, ctr: Container, data: dict) -> asyncio.Future: """Submit job for async execution""" @@ -352,7 +354,7 @@ async def submit_job(self, ctr: Container, data: dict) -> asyncio.Future: return future - def get_finished(self) -> List[tuple]: + def get_finished(self) -> list[tuple]: """Collect completed job results""" finished = [] to_remove = [] @@ -439,10 +441,8 @@ async def main(): # Cancel pending tasks for task in pending: task.cancel() - try: + with contextlib.suppress(asyncio.CancelledError): await task - except asyncio.CancelledError: - pass # Clear internal event manager.event.clear() diff --git a/backend/migrate.py b/backend/migrate.py index ab0ef4e6..24a0109f 100644 --- a/backend/migrate.py +++ b/backend/migrate.py @@ -1,8 +1,9 @@ -from models import db, User -from playhouse.migrate import migrate, SqliteMigrator -from peewee import CharField import secrets +from models import User, db +from peewee import CharField +from playhouse.migrate import SqliteMigrator, migrate + # Create migrator migrator = SqliteMigrator(db) @@ -16,8 +17,8 @@ with db.atomic(): # Add columns migrate( - migrator.add_column('users', 'password_hash', password_hash_field), - migrator.add_column('users', 'api_key', api_key_field) + migrator.add_column("users", "password_hash", password_hash_field), + migrator.add_column("users", "api_key", api_key_field), ) print("✓ Added password_hash and api_key columns to users table") @@ -26,17 +27,17 @@ existing_users = User.select() for user in existing_users: updated = False - + if not user.password_hash: user.password_hash = "changeme123" # They can change this on first login updated = True print(f"✓ Set default password for user: {user.username}") - + if not user.api_key: user.api_key = secrets.token_urlsafe(32) updated = True print(f"✓ Generated API key for user: {user.username}") - + if updated: user.save() diff --git a/backend/migrate_problems.py b/backend/migrate_problems.py new file mode 100644 index 00000000..e44814ae --- /dev/null +++ b/backend/migrate_problems.py @@ -0,0 +1,161 @@ +""" +Migration script to create problems and test_cases tables. + +Run this script to add the new tables to an existing database. + +Usage: + python migrate_problems.py + python migrate_problems.py --seed # Also add sample data +""" + +import os +import sys + +# Ensure we can import from the backend directory +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from models import db +from problem_models import Problem, TestCase, init_problem_tables + + +def migrate(): + """Create the problems and test_cases tables if they don't exist.""" + print("Starting migration for problems and test_cases tables...") + + init_problem_tables() + + # Verify tables were created + tables = db.get_tables() + + if "problems" in tables: + print("✓ problems table created/verified") + else: + print("✗ Failed to create problems table") + return False + + if "test_cases" in tables: + print("✓ test_cases table created/verified") + else: + print("✗ Failed to create test_cases table") + return False + + print("\n✓ Migration complete!") + print("\nTable structure:") + print(" problems:") + print(" - id: Primary key (auto)") + print(" - title: VARCHAR(255)") + print(" - description: TEXT") + print(" - difficulty: VARCHAR(20)") + print(" - created_at: DATETIME") + print("\n test_cases:") + print(" - id: Primary key (auto)") + print(" - problem_id: Foreign key -> problems(id) CASCADE") + print(" - input: TEXT") + print(" - expected_output: TEXT") + print(" - is_hidden: BOOLEAN (default: false)") + print(" - test_order: INTEGER (positive, unique per problem)") + print(" - created_at: DATETIME") + print("\n Indexes:") + print(" - UNIQUE (problem_id, test_order)") + print(" - INDEX (problem_id, is_hidden)") + print("\n Constraints:") + print(" - test_order > 0") + print(" - problem_id CASCADE on delete") + + return True + + +def seed_sample_data(): + """Optionally seed some sample problems and test cases for development.""" + print("\nSeeding sample data...") + + with db.atomic(): + # Create a sample problem + problem, created = Problem.get_or_create( + title="Two Sum", + defaults={ + "description": """Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. + +You may assume that each input would have exactly one solution, and you may not use the same element twice. + +You can return the answer in any order. + +Example: +Input: nums = [2,7,11,15], target = 9 +Output: [0,1] +Explanation: Because nums[0] + nums[1] == 9, we return [0, 1].""", + "difficulty": "easy", + }, + ) + + if created: + print(f"✓ Created problem: {problem.title}") + + # Sample test cases (visible to users) + sample_tests = [ + { + "input": "[2,7,11,15]\n9", + "expected_output": "[0,1]", + "test_order": 1, + }, + {"input": "[3,2,4]\n6", "expected_output": "[1,2]", "test_order": 2}, + ] + + # Hidden test cases (for validation) + hidden_tests = [ + {"input": "[3,3]\n6", "expected_output": "[0,1]", "test_order": 3}, + { + "input": "[1,2,3,4,5]\n9", + "expected_output": "[3,4]", + "test_order": 4, + }, + { + "input": "[-1,-2,-3,-4,-5]\n-8", + "expected_output": "[2,4]", + "test_order": 5, + }, + ] + + for test in sample_tests: + TestCase.create( + problem=problem, + input=test["input"], + expected_output=test["expected_output"], + is_hidden=False, + test_order=test["test_order"], + ) + print(f" ✓ Created {len(sample_tests)} sample test cases") + + for test in hidden_tests: + TestCase.create( + problem=problem, + input=test["input"], + expected_output=test["expected_output"], + is_hidden=True, + test_order=test["test_order"], + ) + print(f" ✓ Created {len(hidden_tests)} hidden test cases") + + else: + print(f" Problem '{problem.title}' already exists, skipping seed data") + + print("\n✓ Sample data seeding complete!") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Migrate problems and test_cases tables" + ) + parser.add_argument( + "--seed", + action="store_true", + help="Also seed sample problem data for development", + ) + args = parser.parse_args() + + success = migrate() + + if success and args.seed: + seed_sample_data() diff --git a/backend/mnt/agent.py b/backend/mnt/agent.py index 48452009..25ec96e0 100644 --- a/backend/mnt/agent.py +++ b/backend/mnt/agent.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 import socket -import struct import json import subprocess import tempfile import os import shutil -import env -from util import Container, FirecrackerCfg, send_sock, rec_sock, run_cmd, \ -ISerializer, JsonSerializer +from util import send_sock, rec_sock, JsonSerializer EXECUTE_SCRIPT = "/mnt/deploy/execute.sh" CFG = "vm_config.json" @@ -16,89 +13,72 @@ VSOCK_PORT = 5000 SER = JsonSerializer() + def execute_job(job_data: dict) -> dict: """Execute the job using execute.sh script""" - code = job_data.get('code', '') - lang = job_data.get('lang', 'cpp') - compiler = job_data.get('compiler', 'g++') - opts = job_data.get('opts', '-O2 -Wall') - - print(f"[Agent] Executing job, language: {lang}, compiler: {compiler}, opts: {opts}") - + code = job_data.get("code", "") + lang = job_data.get("lang", "cpp") + compiler = job_data.get("compiler", "g++") + opts = job_data.get("opts", "-O2 -Wall") + + print( + f"[Agent] Executing job, language: {lang}, compiler: {compiler}, opts: {opts}" + ) + tmpdir = tempfile.mkdtemp() - + try: - ext_map = { - 'c': '.c', - 'cpp': '.cpp', - 'py': '.py', - 'python': '.py' - } - ext = ext_map.get(lang, '.cpp') + ext_map = {"c": ".c", "cpp": ".cpp", "py": ".py", "python": ".py"} + ext = ext_map.get(lang, ".cpp") src_file = os.path.join(tmpdir, f"source{ext}") - - with open(src_file, 'w') as f: + + with open(src_file, "w") as f: f.write(code) - + result_json_path = os.path.join(tmpdir, "result.json") - - cmd = [ - EXECUTE_SCRIPT, - tmpdir, - src_file, - lang, - compiler, - opts - ] - + + cmd = [EXECUTE_SCRIPT, tmpdir, src_file, lang, compiler, opts] + print(f"[Agent] Running: {' '.join(cmd)}") - - proc = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=30 - ) - + + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if os.path.exists(result_json_path): - with open(result_json_path, 'r') as f: + with open(result_json_path, "r") as f: result = json.load(f) - print(f"[Agent] Execution complete, success: {result.get('success', False)}") + print( + f"[Agent] Execution complete, success: {result.get('success', False)}" + ) else: result = { - 'success': False, - 'error': 'No result file generated', - 'stdout': proc.stdout, - 'stderr': proc.stderr, - 'exit_code': proc.returncode + "success": False, + "error": "No result file generated", + "stdout": proc.stdout, + "stderr": proc.stderr, + "exit_code": proc.returncode, } - print(f"[Agent] Execution failed: no result file") - + print("[Agent] Execution failed: no result file") + except subprocess.TimeoutExpired: - result = { - 'success': False, - 'error': 'Execution timeout (30s)' - } - print(f"[Agent] Execution timeout") + result = {"success": False, "error": "Execution timeout (30s)"} + print("[Agent] Execution timeout") except Exception as e: - result = { - 'success': False, - 'error': str(e) - } + result = {"success": False, "error": str(e)} print(f"[Agent] Execution error: {e}") finally: try: shutil.rmtree(tmpdir) - except: + except Exception: pass - + return result + def main(): """Main agent loop - listen on vsock and process jobs""" # Read VM configuration try: - with open(CFG, 'r') as f: + with open(CFG, "r") as f: config = json.load(f) cid = config.get("vsock", {}).get("cid", socket.VMADDR_CID_ANY) port = config.get("vsock", {}).get("port", VSOCK_PORT) @@ -109,15 +89,15 @@ def main(): port = VSOCK_PORT print(f"[Agent] Starting on vsock port {port}") - + # Create vsock socket sock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) - #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((cid, port)) sock.listen(1) - + print(f"[Agent] Listening on {cid}:{port}") - + while True: conn = None try: @@ -125,52 +105,51 @@ def main(): conn, addr = sock.accept() print(f"[Agent] Connected: {addr}") print("[Agent] Ready to receive jobs (Firecracker handled handshake)") - + # Process jobs on this connection # NO handshake needed - Firecracker already sent OK to host while True: try: # Receive job data: {code, lang, compiler, opts} job_bytes = rec_sock(conn) - job_data = SER.deserialize(job_bytes) - print(f"[Agent] Received job") - + job_data = SER.deserialize(job_bytes) + print("[Agent] Received job") + # Execute job result = execute_job(job_data) - + # Send result back result_bytes = SER.serialize(result) send_sock(conn, result_bytes) - - print(f"[Agent] Sent result") - + + print("[Agent] Sent result") + except Exception as e: print(f"[Agent] Error processing job: {e}") - error_result = { - 'success': False, - 'error': str(e) - } + error_result = {"success": False, "error": str(e)} try: send_sock(conn, SER.serialize(error_result)) - except: + except Exception: break - + except KeyboardInterrupt: print("\n[Agent] Shutting down...") break except Exception as e: print(f"[Agent] Connection error: {e}") import traceback + traceback.print_exc() continue finally: if conn: try: conn.close() - except: + except Exception: pass - + sock.close() + if __name__ == "__main__": main() diff --git a/backend/mnt/env.py b/backend/mnt/env.py index af03dcea..d8d907c9 100644 --- a/backend/mnt/env.py +++ b/backend/mnt/env.py @@ -1 +1 @@ -PORT_START=5000 +PORT_START = 5000 diff --git a/backend/mnt/util.py b/backend/mnt/util.py index 5710233b..d28c3b38 100644 --- a/backend/mnt/util.py +++ b/backend/mnt/util.py @@ -1,16 +1,15 @@ from dataclasses import dataclass -import os -import time import struct -#from dotenv import load_dotenv -import shutil + +# from dotenv import load_dotenv from typing import Optional import subprocess import socket from abc import ABC, abstractmethod import json -@dataclass + +@dataclass class Container: cid: int cfg: str @@ -20,21 +19,24 @@ class Container: sock: Optional[socket.socket] = None ready: bool = False + @dataclass class FirecrackerCfg: path: str = "." - + @property def bin(self): # and clean up return f"{self.path}/run-firecracker.sh" + # PRE: socket connection is valid, data is serialized def send_sock(sock, data: bytes): """Send length-prefixed message""" sock.sendall(struct.pack(">I", len(data))) sock.sendall(data) + # PRE: socket connection is valid # POST: need to deserialize def rec_sock(sock) -> bytes: @@ -43,9 +45,9 @@ def rec_sock(sock) -> bytes: raw_len = sock.recv(4) if not raw_len or len(raw_len) < 4: raise RuntimeError("Failed to receive length header") - + msg_len = struct.unpack(">I", raw_len)[0] - + # Read the actual message chunks = [] bytes_received = 0 @@ -55,8 +57,9 @@ def rec_sock(sock) -> bytes: raise RuntimeError("Socket connection broken") chunks.append(chunk) bytes_received += len(chunk) - - return b''.join(chunks) + + return b"".join(chunks) + def run_cmd(cmd): try: @@ -67,15 +70,20 @@ def run_cmd(cmd): print(f"Error: {e.stderr}") raise + class ISerializer(ABC): @abstractmethod def serialize(self, data: dict) -> bytes: pass + @abstractmethod def deserialize(self, data: bytes) -> dict: pass + + class JsonSerializer(ISerializer): def serialize(self, data): - return json.dumps(data).encode('utf-8') + return json.dumps(data).encode("utf-8") + def deserialize(self, data): return json.loads(data) diff --git a/backend/models.py b/backend/models.py index acd06940..693fb910 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,9 +1,22 @@ -from peewee import * -import datetime import json -from typing import Optional import os +#from datetime import datetime +from typing import Optional + +from peewee import ( + AutoField, + BigIntegerField, + CharField, + DateTimeField, + FloatField, + ForeignKeyField, + IntegerField, + Model, + SqliteDatabase, + TextField, +) + DB_PATH = os.getenv("DB_PATH", "data/benchr.db") db_dir = os.path.dirname(DB_PATH) @@ -13,54 +26,59 @@ db = SqliteDatabase( DB_PATH, pragmas={ - 'journal_mode': 'wal', - 'cache_size': -1 * 64000, - 'foreign_keys': 1, - 'synchronous': 2 - } + "journal_mode": "wal", + "cache_size": -1 * 64000, + "foreign_keys": 1, + "synchronous": 2, + }, ) + class BaseModel(Model): class Meta: database = db + class Job(BaseModel): id = AutoField(primary_key=True) # user later # xxx - + # Job input code = TextField() lang = CharField(max_length=50) compiler = CharField(max_length=50) - opts = CharField(max_length=255, default='') - + opts = CharField(max_length=255, default="") + # Job status - status = CharField(max_length=20, default='queued') # queued, running, completed, failed - + status = CharField( + max_length=20, default="queued" + ) # queued, running, completed, failed + # Job results (full JSON from execute.sh) result = TextField(null=True) - + # Timestamps started_at = DateTimeField(null=True) completed_at = DateTimeField(null=True) - + class Meta: - table_name = 'jobs' - + table_name = "jobs" + def get_result(self) -> Optional[dict]: if self.result: return json.loads(self.result) return None - + def set_result(self, result_dict: dict): self.result = json.dumps(result_dict) + class JobMetrics(BaseModel): metric_id = AutoField(primary_key=True) - job = ForeignKeyField(Job, backref='metrics', unique=True) - + job = ForeignKeyField(Job, backref="metrics", unique=True) + # Performance counters cycles = BigIntegerField(null=True) instructions = BigIntegerField(null=True) @@ -68,19 +86,20 @@ class JobMetrics(BaseModel): cache_references = BigIntegerField(null=True) branch_misses = BigIntegerField(null=True) branch_instructions = BigIntegerField(null=True) - + # Calculated metrics ipc = FloatField(null=True) cache_miss_rate = FloatField(null=True) branch_miss_rate = FloatField(null=True) - + # Time metrics execution_time_ms = FloatField(null=True) max_rss_kb = IntegerField(null=True) page_faults = IntegerField(null=True) - - class Meta: - table_name = 'job_metrics' + + class Meta: # pyright: ignore[reportIncompatibleVariableOverride] + table_name = "job_metrics" + def init_db(): """Initialize database""" @@ -88,5 +107,6 @@ def init_db(): db.create_tables([Job, JobMetrics]) print("Database initialized") + def get_db(): return db diff --git a/backend/problem_models.py b/backend/problem_models.py new file mode 100644 index 00000000..65850366 --- /dev/null +++ b/backend/problem_models.py @@ -0,0 +1,70 @@ +from datetime import datetime + +from models import BaseModel, db +from peewee import ( + AutoField, + BooleanField, + CharField, + Check, + DateTimeField, + ForeignKeyField, + IntegerField, + TextField, +) + + +class Problem(BaseModel): + id = AutoField(primary_key=True) + title = CharField(max_length=255) + description = TextField() + difficulty = CharField(max_length=20) # easy, medium, hard + created_at = DateTimeField(default=datetime.now) + + class Meta: + table_name = "problems" + + +class TestCase(BaseModel): +id = AutoField(primary_key=True) + problem = ForeignKeyField(Problem, backref="test_cases", on_delete="CASCADE") + input = TextField() # Test input data (text/JSON) + expected_output = TextField() # Expected output for this input + is_hidden = BooleanField(default=False) # false = sample test, true = hidden test + test_order = IntegerField() # Order in which tests should run (must be positive) + created_at = DateTimeField(default=datetime.now) + + class Meta: + table_name = "test_cases" + indexes = ( + # Index on (problem_id, test_order) for ordered retrieval - unique constraint + (("problem", "test_order"), True), + # Index on (problem_id, is_hidden) for filtering sample vs hidden tests + (("problem", "is_hidden"), False), + ) + constraints = [ + Check("test_order > 0"), # test_order must be positive integer + ] + + +def init_problem_tables(): + with db: + db.create_tables([Problem, TestCase], safe=True) + print("Problem tables initialized") + + +def get_test_cases_for_problem(problem_id: int, include_hidden: bool = False): + + query = TestCase.select().where(TestCase.problem == problem_id) + + if not include_hidden: + query = query.where(TestCase.is_hidden == False) + + return query.order_by(TestCase.test_order) + + +def get_sample_tests(problem_id: int): + return get_test_cases_for_problem(problem_id, include_hidden=False) + + +def get_all_tests(problem_id: int): + return get_test_cases_for_problem(problem_id, include_hidden=True) diff --git a/backend/rate_limiter.py b/backend/rate_limiter.py index 194df12c..a16e996a 100644 --- a/backend/rate_limiter.py +++ b/backend/rate_limiter.py @@ -1,8 +1,10 @@ -from IQueue import IQueue import asyncio -import redis.asyncio as aioredis -from typing import Optional import time +from typing import Optional + +import redis.asyncio as aioredis +from IQueue import IQueue + class RateLimitedQueue(IQueue): """ @@ -10,8 +12,14 @@ class RateLimitedQueue(IQueue): Extends IQueue interface. """ - def __init__(self, redis_url: str, queue_name: str, - max_requests: int, window_seconds: int, max_queue_size: int): + def __init__( + self, + redis_url: str, + queue_name: str, + max_requests: int, + window_seconds: int, + max_queue_size: int, + ): # Note: intentionally not calling super().__init__() # IQueue's init has different signature and uses sync redis self.redis_url = redis_url @@ -29,10 +37,7 @@ def __init__(self, redis_url: str, queue_name: str, self._pubsub = None async def connect(self): - self.redis = await aioredis.from_url( - self.redis_url, - decode_responses=True - ) + self.redis = await aioredis.from_url(self.redis_url, decode_responses=True) self._pubsub = self.redis.pubsub() await self._pubsub.subscribe(self.notify_channel) print(f"[RateLimitedQueue] Connected to {self.redis_url}") @@ -63,7 +68,7 @@ async def _record_request(self, job_id: int): pipe.expire(self.rate_key, self.window_seconds + 60) await pipe.execute() - # =========== IQueue interface implementation =========== + # IQueue interface implementation async def full(self): return await self.size() >= self.max_queue_size @@ -93,9 +98,7 @@ async def pend(self, timeout: float = 5.0) -> Optional[int]: """Atomically move job from queued to processing""" try: result = await self.redis.brpoplpush( - self.queued_key, - self.processing_key, - timeout=int(timeout) + self.queued_key, self.processing_key, timeout=int(timeout) ) if result: return int(result) @@ -120,15 +123,16 @@ async def size(self) -> int: processing = await self.redis.llen(self.processing_key) return queued + processing - # =========== Additional async methods =========== + # Additional async methods async def poll(self, timeout: float = 5.0): """Wait for queue notification (event-driven)""" try: msg = await asyncio.wait_for( - self._pubsub.get_message(ignore_subscribe_messages=True, -timeout=timeout), - timeout=timeout + 1 + self._pubsub.get_message( + ignore_subscribe_messages=True, timeout=timeout + ), + timeout=timeout + 1, ) return msg except asyncio.TimeoutError: diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 00000000..ab1b6894 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,12 @@ +quart +quart-cors +peewee +redis +python-dotenv +pytest +pytest-asyncio +pytest-cov +pytest-xdist +pytest-timeout +fakeredis +hypothesis diff --git a/backend/util.py b/backend/util.py index 5710233b..744d7452 100644 --- a/backend/util.py +++ b/backend/util.py @@ -1,40 +1,42 @@ -from dataclasses import dataclass -import os -import time +import json +import socket import struct -#from dotenv import load_dotenv -import shutil -from typing import Optional import subprocess -import socket from abc import ABC, abstractmethod -import json +from dataclasses import dataclass + +# from dotenv import load_dotenv +from typing import Optional -@dataclass + +@dataclass class Container: cid: int cfg: str vm_cfg: str vsock: str port: int - sock: Optional[socket.socket] = None + sock: Optional[socket.socket] = None # noqa: UP045 ready: bool = False + @dataclass class FirecrackerCfg: path: str = "." - + @property def bin(self): # and clean up return f"{self.path}/run-firecracker.sh" + # PRE: socket connection is valid, data is serialized def send_sock(sock, data: bytes): """Send length-prefixed message""" sock.sendall(struct.pack(">I", len(data))) sock.sendall(data) + # PRE: socket connection is valid # POST: need to deserialize def rec_sock(sock) -> bytes: @@ -43,9 +45,9 @@ def rec_sock(sock) -> bytes: raw_len = sock.recv(4) if not raw_len or len(raw_len) < 4: raise RuntimeError("Failed to receive length header") - + msg_len = struct.unpack(">I", raw_len)[0] - + # Read the actual message chunks = [] bytes_received = 0 @@ -55,8 +57,9 @@ def rec_sock(sock) -> bytes: raise RuntimeError("Socket connection broken") chunks.append(chunk) bytes_received += len(chunk) - - return b''.join(chunks) + + return b"".join(chunks) + def run_cmd(cmd): try: @@ -67,15 +70,20 @@ def run_cmd(cmd): print(f"Error: {e.stderr}") raise + class ISerializer(ABC): @abstractmethod def serialize(self, data: dict) -> bytes: pass + @abstractmethod def deserialize(self, data: bytes) -> dict: pass + + class JsonSerializer(ISerializer): def serialize(self, data): - return json.dumps(data).encode('utf-8') + return json.dumps(data).encode("utf-8") + def deserialize(self, data): return json.loads(data) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 207bf937..b7db154d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,12 +1,12 @@ FROM node:20-alpine AS development-dependencies-env COPY . /app WORKDIR /app -RUN npm ci +RUN npm install FROM node:20-alpine AS production-dependencies-env -COPY ./package.json package-lock.json /app/ +COPY ./package.json /app/ WORKDIR /app -RUN npm ci --omit=dev +RUN npm install --omit=dev FROM node:20-alpine AS build-env COPY . /app/ @@ -15,8 +15,8 @@ WORKDIR /app RUN npm run build FROM node:20-alpine -COPY ./package.json package-lock.json /app/ +COPY ./package.json /app/ COPY --from=production-dependencies-env /app/node_modules /app/node_modules COPY --from=build-env /app/build /app/build WORKDIR /app -CMD ["npm", "run", "start"] \ No newline at end of file +CMD ["npm", "run", "start"] diff --git a/frontend/app/components/benchmark/results/Overview.tsx b/frontend/app/components/benchmark/results/Overview.tsx index 67b6ec11..ae541d6a 100644 --- a/frontend/app/components/benchmark/results/Overview.tsx +++ b/frontend/app/components/benchmark/results/Overview.tsx @@ -35,12 +35,12 @@ export function OverviewView({ jobData }: OverviewViewProps) { const executionTime = result.time?.elapsed_time_total_seconds !== undefined ? `${(result.time.elapsed_time_total_seconds * 1000).toFixed(2)} ms` : 'N/A'; - + // Memory is in KB, convert to MB const memoryUsage = result.time?.maximum_resident_set_size ? `${(result.time.maximum_resident_set_size / 1024).toFixed(2)} MB` : 'N/A'; - + const cacheHitRatio = 'Coming Soon'; // Placeholder for future implementation return ( @@ -70,7 +70,7 @@ export function OverviewView({ jobData }: OverviewViewProps) { - +

Error Details

               {hasCompilationError
diff --git a/frontend/app/components/editor/CodeEditor.tsx b/frontend/app/components/editor/CodeEditor.tsx
index 162669b9..c4884aa7 100644
--- a/frontend/app/components/editor/CodeEditor.tsx
+++ b/frontend/app/components/editor/CodeEditor.tsx
@@ -4,89 +4,90 @@ import { ClientOnly } from '~/components/ui/ClientOnly';
 import { ErrorBoundary } from '~/components/ui/ErrorBoundary';
 import { defineAndSetBenchrTheme, BENCHR_THEME_NAME } from '~/constants/monacoTheme';
 import type { editor } from 'monaco-editor';
+import type { Language } from '~/types/benchmark';
 
 interface CodeEditorProps {
-  value: string;
-  onChange: (value: string) => void;
-  language: 'cpp' | 'c' | 'python' | 'asm';
-  theme?: string;
-  readOnly?: boolean;
-  onSave?: () => void;
+	value: string;
+	onChange: (value: string) => void;
+	language: Language | 'asm';
+	theme?: string;
+	readOnly?: boolean;
+	onSave?: () => void;
 }
 
 export const CodeEditor: React.FC = ({
-  value,
-  onChange,
-  language,
-  theme = BENCHR_THEME_NAME,
-  readOnly = false,
-  onSave
+	value,
+	onChange,
+	language,
+	theme = BENCHR_THEME_NAME,
+	readOnly = false,
+	onSave
 }) => {
-  const editorRef = useRef(null);
+	const editorRef = useRef(null);
 
-  const handleEditorMount = (editor: editor.IStandaloneCodeEditor, monaco: any) => {
-    editorRef.current = editor;
+	const handleEditorMount = (editor: editor.IStandaloneCodeEditor, monaco: any) => {
+		editorRef.current = editor;
 
-    defineAndSetBenchrTheme(monaco);
+		defineAndSetBenchrTheme(monaco);
 
-    // Add save command
-    editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => {
-      onSave?.();
-    });
-    
-    // Add find/replace
-    editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyF, () => {
-      editor.trigger('', 'actions.find', null);
-    });
-  };
+		// Add save command
+		editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => {
+			onSave?.();
+		});
 
-  return (
-    
-          

- Editor failed to load -

-

- Monaco editor encountered an error -

- - - } - > -
- - Loading editor... -
- } - > - {() => ( - onChange(val || '')} - onMount={handleEditorMount} - options={{ - readOnly, - minimap: { enabled: false }, - fontSize: 14, - lineNumbers: 'on', - scrollBeyondLastLine: false, - automaticLayout: true, - }} - /> - )} - - -
- ); + // Add find/replace + editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyF, () => { + editor.trigger('', 'actions.find', null); + }); + }; + + return ( + +

+ Editor failed to load +

+

+ Monaco editor encountered an error +

+ + + } + > +
+ + Loading editor... +
+ } + > + {() => ( + onChange(val || '')} + onMount={handleEditorMount} + options={{ + readOnly, + minimap: { enabled: false }, + fontSize: 14, + lineNumbers: 'on', + scrollBeyondLastLine: false, + automaticLayout: true, + }} + /> + )} + + +
+ ); }; diff --git a/frontend/app/components/editor/CompilerSettings.tsx b/frontend/app/components/editor/CompilerSettings.tsx index 810b4ab4..2324cd9d 100644 --- a/frontend/app/components/editor/CompilerSettings.tsx +++ b/frontend/app/components/editor/CompilerSettings.tsx @@ -1,113 +1,124 @@ import React from 'react'; +export interface CompilerInfo { + id: string; + name: string; +} + +export interface BenchmarkConfig { + optimizationLevel: string; + standardVersion?: string; + additionalFlags?: string; +} + interface CompilerSettingsProps { - compilers: CompilerInfo[]; - selectedCompiler: string; - onCompilerChange: (compiler: string) => void; - config: BenchmarkConfig; - onConfigChange: (config: BenchmarkConfig) => void; - language: string; + compilers: CompilerInfo[]; + selectedCompiler: string; + onCompilerChange: (compiler: string) => void; + config: BenchmarkConfig; + onConfigChange: (config: BenchmarkConfig) => void; + language: string; } export const CompilerSettings: React.FC = ({ - compilers, - selectedCompiler, - onCompilerChange, - config, - onConfigChange, - language + compilers, + selectedCompiler, + onCompilerChange, + config, + onConfigChange, + language }) => { - const isAssembly = language === 'assembly'; - - return ( -
-
- - -
- - {!isAssembly && ( - <> -
- - -
- - {(language === 'cpp' || language === 'c') && ( -
- - -
- )} - -
- - - onConfigChange({ ...config, additionalFlags: e.target.value }) - } - placeholder="-Wall -Wextra" - className="min-w-[150px] px-2 py-1 bg-benchr-bg-elevated border border-benchr-border text-benchr-text-light rounded text-xs placeholder:text-benchr-text-muted" - /> -
- - )} -
- ); + const isAssembly = language === 'assembly'; + + return ( +
+
+ + +
+ + {!isAssembly && ( + <> +
+ + +
+ + {(language === 'cpp' || language === 'c') && ( +
+ + +
+ )} + +
+ + + onConfigChange({ ...config, additionalFlags: e.target.value }) + } + placeholder="-Wall -Wextra" + className="min-w-[150px] px-2 py-1 bg-benchr-bg-elevated border border-benchr-border text-benchr-text-light rounded text-xs placeholder:text-benchr-text-muted" + /> +
+ + )} +
+ ); }; diff --git a/frontend/app/components/editor/LanguageSelector.tsx b/frontend/app/components/editor/LanguageSelector.tsx index d968e917..b1e8ab7b 100644 --- a/frontend/app/components/editor/LanguageSelector.tsx +++ b/frontend/app/components/editor/LanguageSelector.tsx @@ -12,7 +12,7 @@ export function LanguageSelector({ languages, currentLanguage, onLanguageChange value && onLanguageChange(value as Language)} + onValueChange={(value: string) => value && onLanguageChange(value as Language)} > {languages.map(({ id, label }) => ( void; - onFileCreate: (parentId: string | null, type: 'file' | 'folder') => void; - onFileDelete: (fileId: string) => void; - selectedFileId?: string; + files: FileNode[]; + onFileSelect: (file: FileNode) => void; + onFileCreate: (parentId: string | null, type: 'file' | 'folder') => void; + onFileDelete: (fileId: string) => void; + selectedFileId?: string; } -export function FileExplorer({ - files, - onFileSelect, - onFileCreate, - onFileDelete, - selectedFileId +export function FileExplorer({ + files, + onFileSelect, + onFileCreate, + onFileDelete, + selectedFileId }: FileExplorerProps) { - const [expandedFolders, setExpandedFolders] = useState>(new Set(['root'])); + const [expandedFolders, setExpandedFolders] = useState>(new Set(['root'])); - const toggleFolder = (folderId: string) => { - setExpandedFolders(prev => { - const next = new Set(prev); - if (next.has(folderId)) { - next.delete(folderId); - } else { - next.add(folderId); - } - return next; - }); - }; + const toggleFolder = (folderId: string) => { + setExpandedFolders(prev => { + const next = new Set(prev); + if (next.has(folderId)) { + next.delete(folderId); + } else { + next.add(folderId); + } + return next; + }); + }; - const renderNode = (node: FileNode, level: number = 0) => { - const isExpanded = expandedFolders.has(node.id); - const isSelected = selectedFileId === node.id; + const renderNode = (node: FileNode, level: number = 0) => { + const isExpanded = expandedFolders.has(node.id); + const isSelected = selectedFileId === node.id; - if (node.type === 'folder') { - return ( -
-
- -
onFileSelect(node)} - className="flex items-center gap-1 flex-1" - > - {isExpanded ? ( - - ) : ( - - )} - {node.name} -
- -
- {isExpanded && node.children && ( -
- {node.children.map(child => renderNode(child, level + 1))} -
- )} -
- ); - } + if (node.type === 'folder') { + return ( +
+
+ +
onFileSelect(node)} + className="flex items-center gap-1 flex-1" + > + {isExpanded ? ( + + ) : ( + + )} + {node.name} +
+ +
+ {isExpanded && node.children && ( +
+ {node.children.map(child => renderNode(child, level + 1))} +
+ )} +
+ ); + } - return ( -
onFileSelect(node)} - > - - {node.name} - -
- ); - }; + return ( +
onFileSelect(node)} + > + + {node.name} + +
+ ); + }; - return ( -
-
- Explorer -
- - -
-
-
- {files.map(node => renderNode(node, 0))} -
-
- ); -} \ No newline at end of file + return ( +
+
+ Explorer +
+ + +
+
+
+ {files.map(node => renderNode(node, 0))} +
+
+ ); +} diff --git a/frontend/app/components/future/TabBar.tsx b/frontend/app/components/future/TabBar.tsx index d0d4652b..c13eae95 100644 --- a/frontend/app/components/future/TabBar.tsx +++ b/frontend/app/components/future/TabBar.tsx @@ -1,41 +1,40 @@ import { X } from 'lucide-react'; -import { Button } from './components/ui/button'; -import { FileNode } from './FileExplorer'; +import { Button } from '~/components/ui/button'; +import type { FileNode } from './FileExplorer'; interface TabBarProps { - openFiles: FileNode[]; - activeFileId: string | null; - onTabClick: (file: FileNode) => void; - onTabClose: (fileId: string) => void; + openFiles: FileNode[]; + activeFileId: string | null; + onTabClick: (file: FileNode) => void; + onTabClose: (fileId: string) => void; } export function TabBar({ openFiles, activeFileId, onTabClick, onTabClose }: TabBarProps) { - return ( -
- {openFiles.map(file => ( -
onTabClick(file)} - > - {file.name} - -
- ))} -
- ); -} \ No newline at end of file + return ( +
+ {openFiles.map(file => ( +
onTabClick(file)} + > + {file.name} + +
+ ))} +
+ ); +} diff --git a/frontend/app/components/pages/BenchmarkWorkspace.tsx b/frontend/app/components/pages/BenchmarkWorkspace.tsx new file mode 100644 index 00000000..e69de29b diff --git a/frontend/app/components/pages/SandboxPage.tsx b/frontend/app/components/pages/SandboxPage.tsx index e524a833..523d95ad 100644 --- a/frontend/app/components/pages/SandboxPage.tsx +++ b/frontend/app/components/pages/SandboxPage.tsx @@ -1,12 +1,12 @@ -import { useState, useEffect, useRef } from 'react'; +import { useState, useEffect, useRef, useCallback } from 'react'; import { WorkspaceRow } from '~/components/WorkspaceRow'; import { WorkspaceProvider, useWorkspace } from '~/contexts/WorkspaceContext'; import { SavedRunsProvider } from '~/contexts/SavedRunsContext'; import { ErrorBoundary } from '~/components/ui/ErrorBoundary'; import { - ResizablePanelGroup, - ResizablePanel, - ResizableHandle, + ResizablePanelGroup, + ResizablePanel, + ResizableHandle, } from "~/components/ui/resizable"; import { SandboxSidebarLayout, @@ -16,7 +16,7 @@ import type { ImperativePanelHandle } from 'react-resizable-panels'; // Helper component to access workspace context in single view function SingleViewLayout({ onToggleCompare }: { onToggleCompare: () => void }) { - const workspace = useWorkspace(); + const workspace = useWorkspace(); return (
@@ -32,6 +32,9 @@ function SingleViewLayout({ onToggleCompare }: { onToggleCompare: () => void }) ); } +// Type for the workspace context value +type WorkspaceContextType = ReturnType; + // Component to wrap each workspace and expose its context function WorkspaceItem({ index, @@ -52,14 +55,11 @@ function WorkspaceItem({ onRunBoth?: () => void; loadingBoth?: boolean; }) { - const workspace = useWorkspace(); - - const workspaceRef = useRef(workspace); - workspaceRef.current = workspace; // Always current, no re-render + const workspace = useWorkspace(); - useEffect(() => { - onRegister(workspaceRef); // Pass the ref object once - }, [onRegister]); + useEffect(() => { + onRegister(workspace); + }, [onRegister, workspace]); return ( void; + workspaceCount: number; + onToggleCompare: () => void; }) { - const [loadingBoth, setLoadingBoth] = useState(false); - const rowRefs = useRef<(ImperativePanelHandle | null)[]>([]); - const workspaceRefs = useRef[]>([]); + const [loadingBoth, setLoadingBoth] = useState(false); + const rowRefs = useRef<(ImperativePanelHandle | null)[]>([]); + const workspaceRefs = useRef<(WorkspaceContextType | null)[]>([]); - const resetVerticalPanels = () => { - rowRefs.current.forEach(ref => ref?.resize(50)); - }; + const resetVerticalPanels = () => { + rowRefs.current.forEach(ref => ref?.resize(50)); + }; - const handleRunBoth = async () => { - setLoadingBoth(true); - try { - await Promise.all( - workspaceRefs.current.map(wsRef => wsRef.current.benchmark.handleRunBenchmark()) - ); - } finally { - setLoadingBoth(false); - } - }; + const handleRunBoth = async () => { + setLoadingBoth(true); + try { + await Promise.all( + workspaceRefs.current + .filter((wsRef): wsRef is WorkspaceContextType => wsRef !== null) + .map(ws => ws.benchmark.handleRunBenchmark()) + ); + } finally { + setLoadingBoth(false); + } + }; - const registerWorkspace = (index: number) => (ws: ReturnType) => { - workspaceRefs.current[index] = ws; - }; + const registerWorkspace = useCallback((index: number) => (ws: WorkspaceContextType) => { + workspaceRefs.current[index] = ws; + }, []); return (
@@ -129,19 +131,19 @@ function CompareViewLayout({ - {index < workspaceCount - 1 && ( - - )} - - ))} - -
- ); + {index < workspaceCount - 1 && ( + + )} +
+ ))} + + + ); } export default function SandboxPage() { - const [compareMode, setCompareMode] = useState(false); - const [workspaceCount] = useState(2); + const [compareMode, setCompareMode] = useState(false); + const [workspaceCount] = useState(2); return ( diff --git a/frontend/app/components/pages/index.ts b/frontend/app/components/pages/index.ts new file mode 100644 index 00000000..03ba4588 --- /dev/null +++ b/frontend/app/components/pages/index.ts @@ -0,0 +1,4 @@ +// This file is intentionally empty. +// Page components are imported directly from their files. +// See the routes/ directory for page routing. +export { }; diff --git a/frontend/app/constants/benchmark.tsx b/frontend/app/constants/benchmark.tsx index c2bfff54..c53deabf 100644 --- a/frontend/app/constants/benchmark.tsx +++ b/frontend/app/constants/benchmark.tsx @@ -1,30 +1,32 @@ import type { Language, LanguageConfig, LanguageOption } from '~/types/benchmark'; export const LANGUAGE_CONFIGS: Record = { - python: { - defaultCode: '# Write your Python code here\nprint("Hello, Benchr!")', - compiler: 'python3', - opts: '' - }, - cpp: { - defaultCode: '// Write your C++ code here\n#include \n\nint main() {\n std::cout << "Hello, Benchr!" << std::endl;\n return 0;\n}', - compiler: 'g++', - opts: '-O2 -std=c++17' - }, - java: { - defaultCode: '// Write your Java code here\npublic class Main {\n public static void main(String[] args) {\n System.out.println("Hello, Benchr!");\n }\n}', - compiler: 'javac', - opts: '' - }, - c: { - defaultCode: '// Write your C code here\n#include \n\nint main() {\n printf("Hello, Benchr!\\n");\n return 0;\n}', - compiler: 'gcc', - opts: '-O2 -std=c11' - } + python: { + defaultCode: '# Write your Python code here\nprint("Hello, Benchr!")', + compiler: 'python3', + opts: '' + }, + cpp: { + defaultCode: '// Write your C++ code here\n#include \n\nint main() {\n std::cout << "Hello, Benchr!" << std::endl;\n return 0;\n}', + compiler: 'g++', + opts: '-O2 -std=c++17' + }, + c: { + defaultCode: '// Write your C code here\n#include \n\nint main() {\n printf("Hello, Benchr!\\n");\n return 0;\n}', + compiler: 'gcc', + opts: '-O2 -std=c11' + }, + + java: { + defaultCode: '// Write your Java code here\npublic class Main {\n public static void main(String[] args) {\n System.out.println("Hello, Benchr!");\n }\n}', + compiler: 'javac', + opts: '' + } }; export const LANGUAGE_OPTIONS: LanguageOption[] = [ - { id: 'python', label: 'Python' }, - { id: 'cpp', label: 'C++' }, - { id: 'c', label: 'C' } + { id: 'python', label: 'Python' }, + { id: 'cpp', label: 'C++' }, + { id: 'c', label: 'C' }, + { id: 'java', label: 'Java' }, ]; diff --git a/frontend/app/contexts/WorkspaceContext.tsx b/frontend/app/contexts/WorkspaceContext.tsx index 6c1de7b0..cf784579 100644 --- a/frontend/app/contexts/WorkspaceContext.tsx +++ b/frontend/app/contexts/WorkspaceContext.tsx @@ -7,53 +7,53 @@ import type { WebSocketHook } from '~/hooks/useWebSocket'; import type { ImperativePanelHandle } from 'react-resizable-panels'; interface WorkspaceContextValue { - id: string; - editor: ReturnType; - benchmark: ReturnType; - refs: { - editor: React.RefObject; - results: React.RefObject; - }; - ws: WebSocketHook; + id: string; + editor: ReturnType; + benchmark: ReturnType; + refs: { + editor: React.RefObject; + results: React.RefObject; + }; + ws: WebSocketHook; } const WorkspaceContext = createContext(null); interface WorkspaceProviderProps { - id: string; - children: ReactNode; + id: string; + children: ReactNode; } export function WorkspaceProvider({ id, children }: WorkspaceProviderProps) { - const editor = useEditor(); - const ws = useWebSocketContext(); - const benchmark = useBenchmark(editor.editor, ws); - - const editorRef = useRef(null); - const resultsRef = useRef(null); - - const value: WorkspaceContextValue = { - id, - editor, - benchmark, - refs: { - editor: editorRef, - results: resultsRef, - }, - ws, - }; - - return ( - - {children} - - ); + const editor = useEditor(); + const ws = useWebSocketContext(); + const benchmark = useBenchmark(editor.editor, ws); + + const editorRef = useRef(null); + const resultsRef = useRef(null); + + const value: WorkspaceContextValue = { + id, + editor, + benchmark, + refs: { + editor: editorRef, + results: resultsRef, + }, + ws, + }; + + return ( + + {children} + + ); } export function useWorkspace() { - const context = useContext(WorkspaceContext); - if (!context) { - throw new Error('useWorkspace must be used within WorkspaceProvider'); - } - return context; + const context = useContext(WorkspaceContext); + if (!context) { + throw new Error('useWorkspace must be used within WorkspaceProvider'); + } + return context; } diff --git a/frontend/app/monacoConfig.ts b/frontend/app/monacoConfig.ts index 1264825a..f816920f 100644 --- a/frontend/app/monacoConfig.ts +++ b/frontend/app/monacoConfig.ts @@ -1,71 +1,152 @@ -import * as monaco from 'monaco-editor'; +// Monaco configuration for NASM assembly language support +// Using 'any' types to avoid issues when monaco-editor types aren't installed -export function configureMonacoForAssembly() { - // Register NASM syntax highlighting - monaco.languages.register({ id: 'nasm' }); +export function configureMonacoForAssembly(monaco: any) { + // Register NASM syntax highlighting + monaco.languages.register({ id: 'nasm' }); - monaco.languages.setMonarchTokensProvider('nasm', { - tokenizer: { - root: [ - // Comments - [/;.*$/, 'comment'], - - // Sections - [/section\s+\.(data|bss|text|rodata)/, 'keyword'], - - // Directives - [/\.(global|extern|equ|times|db|dw|dd|dq)/, 'keyword'], - - // Registers - [/\b(rax|rbx|rcx|rdx|rsi|rdi|rbp|rsp|r8|r9|r10|r11|r12|r13|r14|r15)\b/, 'variable.predefined'], - [/\b(eax|ebx|ecx|edx|esi|edi|ebp|esp)\b/, 'variable.predefined'], - [/\b(ax|bx|cx|dx|si|di|bp|sp)\b/, 'variable.predefined'], - [/\b(al|bl|cl|dl|ah|bh|ch|dh)\b/, 'variable.predefined'], - - // Instructions - [/\b(mov|add|sub|mul|div|inc|dec|push|pop|call|ret|jmp|je|jne|jg|jl|cmp|test|lea|syscall|int)\b/, 'keyword'], - - // Numbers - [/\b0x[0-9a-fA-F]+\b/, 'number.hex'], - [/\b\d+\b/, 'number'], - - // Labels - [/^[a-zA-Z_][a-zA-Z0-9_]*:/, 'type.identifier'], - - // Strings - [/"([^"\\]|\\.)*$/, 'string.invalid'], - [/"/, 'string', '@string'], - ], - - string: [ - [/[^\\"]+/, 'string'], - [/\\./, 'string.escape'], - [/"/, 'string', '@pop'] - ] - } - }); + monaco.languages.setMonarchTokensProvider('nasm', { + tokenizer: { + root: [ + // Comments + [/;.*$/, 'comment'], - // Configure completion provider - monaco.languages.registerCompletionItemProvider('nasm', { - provideCompletionItems: (model, position) => { - const suggestions = [ - { - label: 'mov', - kind: monaco.languages.CompletionItemKind.Keyword, - insertText: 'mov ${1:dest}, ${2:src}', - insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, - documentation: 'Move data from source to destination' - }, - { - label: 'syscall', - kind: monaco.languages.CompletionItemKind.Keyword, - insertText: 'syscall', - documentation: 'Invoke system call' - }, - // Add more... - ]; + // Sections + [/section\s+\.(data|bss|text|rodata)/, 'keyword'], - return { suggestions }; - } - }); + // Directives + [/\.(global|extern|equ|times|db|dw|dd|dq)/, 'keyword'], + + // Registers + [/\b(rax|rbx|rcx|rdx|rsi|rdi|rbp|rsp|r8|r9|r10|r11|r12|r13|r14|r15)\b/, 'variable.predefined'], + [/\b(eax|ebx|ecx|edx|esi|edi|ebp|esp)\b/, 'variable.predefined'], + [/\b(ax|bx|cx|dx|si|di|bp|sp)\b/, 'variable.predefined'], + [/\b(al|bl|cl|dl|ah|bh|ch|dh)\b/, 'variable.predefined'], + + // Instructions + [/\b(mov|add|sub|mul|div|inc|dec|push|pop|call|ret|jmp|je|jne|jg|jl|cmp|test|lea|syscall|int)\b/, 'keyword'], + + // Numbers + [/\b0x[0-9a-fA-F]+\b/, 'number.hex'], + [/\b\d+\b/, 'number'], + + // Labels + [/^[a-zA-Z_][a-zA-Z0-9_]*:/, 'type.identifier'], + + // Strings + [/"([^"\\]|\\.)*$/, 'string.invalid'], + [/"/, 'string', '@string'], + ], + + string: [ + [/[^\\"]+/, 'string'], + [/\\./, 'string.escape'], + [/"/, 'string', '@pop'] + ] + } + }); + + // Configure completion provider + monaco.languages.registerCompletionItemProvider('nasm', { + provideCompletionItems: (model: any, position: any) => { + const word = model.getWordUntilPosition(position); + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn + }; + + const suggestions = [ + { + label: 'mov', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'mov ${1:dest}, ${2:src}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Move data from source to destination', + range: range + }, + { + label: 'syscall', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'syscall', + documentation: 'Invoke system call', + range: range + }, + { + label: 'push', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'push ${1:reg}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Push value onto stack', + range: range + }, + { + label: 'pop', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'pop ${1:reg}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Pop value from stack', + range: range + }, + { + label: 'call', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'call ${1:label}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Call a procedure', + range: range + }, + { + label: 'ret', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'ret', + documentation: 'Return from procedure', + range: range + }, + { + label: 'jmp', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'jmp ${1:label}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Unconditional jump', + range: range + }, + { + label: 'cmp', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'cmp ${1:op1}, ${2:op2}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Compare two operands', + range: range + }, + { + label: 'add', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'add ${1:dest}, ${2:src}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Add source to destination', + range: range + }, + { + label: 'sub', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'sub ${1:dest}, ${2:src}', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Subtract source from destination', + range: range + }, + { + label: 'lea', + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: 'lea ${1:dest}, [${2:src}]', + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + documentation: 'Load effective address', + range: range + }, + ]; + + return { suggestions }; + } + }); } diff --git a/frontend/app/root.tsx b/frontend/app/root.tsx index 1cf9e342..5f86e589 100644 --- a/frontend/app/root.tsx +++ b/frontend/app/root.tsx @@ -1,10 +1,9 @@ import { - isRouteErrorResponse, - Links, - Meta, - Outlet, - Scripts, - ScrollRestoration, + isRouteErrorResponse, + Links, + Outlet, + Scripts, + ScrollRestoration, } from "react-router"; import type { Route } from "./+types/root"; @@ -13,70 +12,71 @@ import './styles/index.css'; import { WebSocketProvider } from '~/contexts/WebSocketContext'; export const links: Route.LinksFunction = () => [ - { rel: "preconnect", href: "https://fonts.googleapis.com" }, - { - rel: "preconnect", - href: "https://fonts.gstatic.com", - crossOrigin: "anonymous", - }, - { rel: "preconnect", href: "https://www.benchr.cc" }, - { - rel: "stylesheet", - href: "https://fonts.googleapis.com/css2?family=Inter:ital,opsz,wght@0,14..32,100..900;1,14..32,100..900&display=swap", - }, + { rel: "preconnect", href: "https://fonts.googleapis.com" }, + { + rel: "preconnect", + href: "https://fonts.gstatic.com", + crossOrigin: "anonymous", + }, + { rel: "preconnect", href: "https://www.benchr.cc" }, + { + rel: "stylesheet", + href: "https://fonts.googleapis.com/css2?family=Inter:ital,opsz,wght@0,14..32,100..900;1,14..32,100..900&display=swap", + }, ]; export function Layout({ children }: { children: React.ReactNode }) { - return ( - - - - - - - - - {children} - - - - - ); + return ( + + + + + + Benchr + + + + {children} + + + + + ); } export default function App() { - return ( - - - - ); + return ( + + + + ); } export function ErrorBoundary({ error }: Route.ErrorBoundaryProps) { - let message = "Oops!"; - let details = "An unexpected error occurred."; - let stack: string | undefined; + let message = "Oops!"; + let details = "An unexpected error occurred."; + let stack: string | undefined; - if (isRouteErrorResponse(error)) { - message = error.status === 404 ? "404" : "Error"; - details = - error.status === 404 - ? "The requested page could not be found." - : error.statusText || details; - } else if (import.meta.env.DEV && error && error instanceof Error) { - details = error.message; - stack = error.stack; - } + if (isRouteErrorResponse(error)) { + message = error.status === 404 ? "404" : "Error"; + details = + error.status === 404 + ? "The requested page could not be found." + : error.statusText || details; + } else if (import.meta.env.DEV && error && error instanceof Error) { + details = error.message; + stack = error.stack; + } - return ( -
-

{message}

-

{details}

- {stack && ( -
-          {stack}
-        
- )} -
- ); + return ( +
+

{message}

+

{details}

+ {stack && ( +
+					{stack}
+				
+ )} +
+ ); } diff --git a/frontend/app/services/api.tsx b/frontend/app/services/api.tsx index 1636b292..d8a746e5 100644 --- a/frontend/app/services/api.tsx +++ b/frontend/app/services/api.tsx @@ -3,200 +3,203 @@ import axios from 'axios'; const API_BASE_URL = 'https://www.benchr.cc/api'; const api = axios.create({ - baseURL: API_BASE_URL, - timeout: 60000, - headers: { - 'Content-Type': 'application/json', - }, + baseURL: API_BASE_URL, + timeout: 60000, + headers: { + 'Content-Type': 'application/json', + }, }); api.interceptors.response.use( - (response) => response, - (error) => { - // Request timed out - if (error.code === 'ECONNABORTED') { - return Promise.reject(new Error('Request timed out. Please try again.')); - } - - // Network error (offline, DNS failure, etc.) - if (!error.response) { - return Promise.reject(new Error('Network error. Check your connection.')); - } - - // Server errors - if (error.response.status >= 500) { - return Promise.reject(new Error('Server error. Please try again later.')); - } - - // Rate limiting - if (error.response.status === 429) { - return Promise.reject(new Error('Too many requests. Please slow down.')); - } - - // Pass through other errors with their message if available - const message = error.response?.data?.message || error.message; - return Promise.reject(new Error(message)); - } + (response) => response, + (error) => { + // Request timed out + if (error.code === 'ECONNABORTED') { + return Promise.reject(new Error('Request timed out. Please try again.')); + } + + // Network error (offline, DNS failure, etc.) + if (!error.response) { + return Promise.reject(new Error('Network error. Check your connection.')); + } + + // Server errors + if (error.response.status >= 500) { + return Promise.reject(new Error('Server error. Please try again later.')); + } + + // Rate limiting + if (error.response.status === 429) { + return Promise.reject(new Error('Too many requests. Please slow down.')); + } + + // Pass through other errors with their message if available + const message = error.response?.data?.message || error.message; + return Promise.reject(new Error(message)); + } ); // Payload structure matching the database schema interface BenchmarkPayload { - code: string; - lang: string; - compiler: string; - opts: string; + code: string; + lang: string; + compiler: string; + opts: string; } interface SubmitJobResponse { - job_id: string; - status: string; + job_id: string; + status: string; } // Performance metrics from perf tool (empty in Python, populated in C/C++) interface PerfMetrics { - cpu_cycles?: string | null; - instructions?: string | null; - cache_references?: string | null; - cache_misses?: string | null; - branch_misses?: string | null; + cpu_cycles?: string | null; + instructions?: string | null; + cache_references?: string | null; + cache_misses?: string | null; + branch_misses?: string | null; } // Time statistics from GNU time interface TimeStats { - command_being_timed: string; - user_time: number; - system_time: number; - cpu_percent: number; - elapsed_time: string; - maximum_resident_set_size: number; - major_pagefaults: number; - minor_pagefaults: number; - voluntary_context_switches: number; - involuntary_context_switches: number; - block_input_operations: number; - block_output_operations: number; - elapsed_time_total_seconds: number; - [key: string]: any; + command_being_timed: string; + user_time: number; + system_time: number; + cpu_percent: number; + elapsed_time: string; + maximum_resident_set_size: number; + major_pagefaults: number; + minor_pagefaults: number; + voluntary_context_switches: number; + involuntary_context_switches: number; + block_input_operations: number; + block_output_operations: number; + elapsed_time_total_seconds: number; + [key: string]: any; } // Virtual memory statistics interface VmStat { - runnable_procs: number; - uninterruptible_sleeping_procs: number; - virtual_mem_used: number; - free_mem: number; - buffer_mem: number; - cache_mem: number; - inactive_mem: number | null; - active_mem: number | null; - swap_in: number; - swap_out: number; - blocks_in: number; - blocks_out: number; - interrupts: number; - context_switches: number; - user_time: number; - system_time: number; - idle_time: number; - io_wait_time: number; - stolen_time: number; - [key: string]: any; + runnable_procs: number; + uninterruptible_sleeping_procs: number; + virtual_mem_used: number; + free_mem: number; + buffer_mem: number; + cache_mem: number; + inactive_mem: number | null; + active_mem: number | null; + swap_in: number; + swap_out: number; + blocks_in: number; + blocks_out: number; + interrupts: number; + context_switches: number; + user_time: number; + system_time: number; + idle_time: number; + io_wait_time: number; + stolen_time: number; + [key: string]: any; } // Compilation info interface CompilationInfo { - success: boolean; - error: string | null; - details: string; + success: boolean; + error: string | null; + details: string; } // Result metadata interface ResultMetadata { - language: string; - interpreter: string; - opts: string | null; - source_size_bytes: number; + language: string; + interpreter: string; + compiler?: string; + opts: string | null; + source_size_bytes: number; } // Main result object interface JobResult { - success: boolean; - timestamp: string; - exit_code: number; - output: string; - asm: string; - perf: PerfMetrics; - time: TimeStats; - vmstat: VmStat[]; - compilation: CompilationInfo; - metadata: ResultMetadata; + success: boolean; + timestamp: string; + exit_code: number; + output: string; + asm: string; + perf: PerfMetrics; + time: TimeStats; + vmstat: VmStat[]; + compilation: CompilationInfo; + metadata: ResultMetadata; + result?: JobResult; } // Job data structure interface JobData { - id: number; - code: string; - lang: string; - compiler: string; - opts: string; - status: string; - result: JobResult | null; - started_at: string; - completed_at: string | null; + id: number; + code: string; + lang: string; + compiler: string; + opts: string; + status: string; + result: JobResult | null; + started_at: string; + completed_at: string | null; + perf?: any; } const benchmarkService = { - /** - * Submit a new benchmark job - * POST /api/submit - * Each submit creates a new job_id for the user - */ - async submitJob(payload: BenchmarkPayload): Promise { - const response = await api.post('/submit', payload); - return response.data; - }, - - /** - * Get a specific job by ID (checks Redis queue) - * GET /api/jobs/ - * Returns job data with result metrics if completed, null result if still processing - */ - async getJobById(jobId: string): Promise { - const response = await api.get(`/jobs/${jobId}`); - return response.data; - }, - - /** - * Health check - * GET /api/health - */ - async healthCheck(): Promise<{ status: string; database: string }> { - const response = await api.get('/health'); - return response.data; - }, - - /** - * Send a chat message to Claude AI - * POST /api/chat - */ - async sendChatMessage(message: string, result: JobResult | null): Promise<{ response: string }> { - const response = await api.post('/chat', { - message, - result - }); - return response.data; - } + /** + * Submit a new benchmark job + * POST /api/submit + * Each submit creates a new job_id for the user + */ + async submitJob(payload: BenchmarkPayload): Promise { + const response = await api.post('/submit', payload); + return response.data; + }, + + /** + * Get a specific job by ID (checks Redis queue) + * GET /api/jobs/ + * Returns job data with result metrics if completed, null result if still processing + */ + async getJobById(jobId: string): Promise { + const response = await api.get(`/jobs/${jobId}`); + return response.data; + }, + + /** + * Health check + * GET /api/health + */ + async healthCheck(): Promise<{ status: string; database: string }> { + const response = await api.get('/health'); + return response.data; + }, + + /** + * Send a chat message to Claude AI + * POST /api/chat + */ + async sendChatMessage(message: string, result: JobResult | null): Promise<{ response: string }> { + const response = await api.post('/chat', { + message, + result + }); + return response.data; + } }; export default benchmarkService; export type { - BenchmarkPayload, - SubmitJobResponse, - JobData, - JobResult, - PerfMetrics, - TimeStats, - VmStat, - CompilationInfo, - ResultMetadata + BenchmarkPayload, + SubmitJobResponse, + JobData, + JobResult, + PerfMetrics, + TimeStats, + VmStat, + CompilationInfo, + ResultMetadata }; diff --git a/frontend/app/types/benchmark.tsx b/frontend/app/types/benchmark.tsx index a7750474..71cd39db 100644 --- a/frontend/app/types/benchmark.tsx +++ b/frontend/app/types/benchmark.tsx @@ -1,64 +1,64 @@ export type Language = 'cpp' | 'python' | 'java' | 'c'; export interface ExecutionResult { - success: boolean; - exit_code: number; - output: string; - asm: string; - timestamp: string; - compilation: { - success: boolean; - error: string | null; - details: string | null; - }; - time: { - elapsed_time_total_seconds: number; - cpu_percent: number; - maximum_resident_set_size: number; - user_time: number; - system_time: number; - minor_pagefaults: number; - major_pagefaults: number; - voluntary_context_switches: number; - involuntary_context_switches: number; - [key: string]: any; - }; - perf: Record; - vmstat: Array; - metadata: { - language: string; - interpreter?: string; - compiler?: string; - opts?: string | null; - source_size_bytes: number; - }; + success: boolean; + exit_code: number; + output: string; + asm: string; + timestamp: string; + compilation: { + success: boolean; + error: string | null; + details: string | null; + }; + time: { + elapsed_time_total_seconds: number; + cpu_percent: number; + maximum_resident_set_size: number; + user_time: number; + system_time: number; + minor_pagefaults: number; + major_pagefaults: number; + voluntary_context_switches: number; + involuntary_context_switches: number; + [key: string]: any; + }; + perf: Record; + vmstat: Array; + metadata: { + language: string; + interpreter?: string; + compiler?: string; + opts?: string | null; + source_size_bytes: number; + }; } export interface EditorConfig { - code: string; - language: Language; - compiler: string; - opts: string; + code: string; + language: Language; + compiler: string; + opts: string; } export interface LanguageConfig { - defaultCode: string; - compiler: string; - opts: string; + defaultCode: string; + compiler: string; + opts: string; } export interface LanguageOption { - id: Language; - label: string; + id: Language; + label: string; } export interface ChatMessage { - role: 'user' | 'assistant'; - content: string; + role: 'user' | 'assistant'; + content: string; } -export type ResultView = - | 'overview' - | 'performance' - | 'bytecode' - | 'system'; +export type ResultView = + | 'overview' + | 'performance' + | 'bytecode' + | 'system'; diff --git a/frontend/app/types/global.d.ts b/frontend/app/types/global.d.ts new file mode 100644 index 00000000..f5f2ac58 --- /dev/null +++ b/frontend/app/types/global.d.ts @@ -0,0 +1,35 @@ +export type CompilerInfo = { + id?: string; + name?: string; + version?: string; + [k: string]: any; +}; + +export type BenchmarkConfig = { + compiler?: string; + flags?: string[]; + [k: string]: any; +}; + +export type JobPerf = { [k: string]: any }; + +export type JobData = { + perf?: JobPerf; + [k: string]: any; +}; + +export type ResultMetadata = { + compiler?: string; // Overview.tsx + benchmarkName?: string; + timestamp?: string; + [k: string]: any; +}; + +export type JobResult = { + result?: any; + data?: JobData; + metadata?: ResultMetadata; + [k: string]: any; +}; + +export type AllowedLanguage = 'cpp' | 'c' | 'python' | 'asm' | 'java'; diff --git a/frontend/package.json b/frontend/package.json index a0fdc564..48be055e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,84 +1,82 @@ { - "name": "benchr", - "private": true, - "type": "module", - "scripts": { - "build": "react-router build", - "dev": "react-router dev", - "start": "react-router-serve ./build/server/index.js", - "typecheck": "react-router typegen && tsc" - }, - "dependencies": { - "@monaco-editor/react": "^4.7.0", - "@radix-ui/react-accordion": "^1.2.12", - "@radix-ui/react-alert-dialog": "^1.1.15", - "@radix-ui/react-aspect-ratio": "^1.1.7", - "@radix-ui/react-avatar": "^1.1.10", - "@radix-ui/react-checkbox": "^1.3.3", - "@radix-ui/react-collapsible": "^1.1.12", - "@radix-ui/react-context-menu": "^2.2.16", - "@radix-ui/react-dialog": "^1.1.15", - "@radix-ui/react-dropdown-menu": "^2.1.16", - "@radix-ui/react-hover-card": "^1.1.15", - "@radix-ui/react-label": "^2.1.7", - "@radix-ui/react-menubar": "^1.1.16", - "@radix-ui/react-navigation-menu": "^1.2.14", - "@radix-ui/react-popover": "^1.1.15", - "@radix-ui/react-progress": "^1.1.7", - "@radix-ui/react-radio-group": "^1.3.8", - "@radix-ui/react-scroll-area": "^1.2.10", - "@radix-ui/react-select": "^2.2.6", - "@radix-ui/react-separator": "^1.1.7", - "@radix-ui/react-slider": "^1.3.6", - "@radix-ui/react-slot": "^1.2.3", - "@radix-ui/react-toggle": "^1.1.10", - "@radix-ui/react-toggle-group": "^1.1.11", - "@radix-ui/react-tooltip": "^1.2.8", - "@react-router/node": "^7.9.2", - "@react-router/serve": "^7.9.2", - "@tanstack/react-table": "^8.21.3", - "axios": "^1.12.2", - "build": "^0.1.4", - "class-variance-authority": "^0.7.1", - "clsx": "^2.1.1", - "cmdk": "^1.1.1", - "date-fns": "^4.1.0", - "dompurify": "^3.3.0", - "embla-carousel-react": "^8.6.0", - "isbot": "^5.1.31", - "json": "^11.0.0", - "json2": "^0.4.0", - "lucide-react": "^0.548.0", - "motion": "^12.23.24", - "react": "^19.1.1", - "react-day-picker": "^9.11.1", - "react-dom": "^19.1.1", - "react-resizable-panels": "^3.0.6", - "react-router": "^7.9.4", - "recharts": "^2.15.4", - "redis": "^5.9.0", - "socket.io-client": "^4.8.1", - "tailwind-merge": "^3.3.1", - "vaul": "^1.1.2" - }, - "devDependencies": { - "@react-router/dev": "^7.9.2", - "@tailwindcss/postcss": "^4.1.17", - "@tailwindcss/vite": "^4.1.13", - "@types/node": "^22", - "@types/react": "^19.1.13", - "@types/react-dom": "^19.1.9", - "autoprefixer": "^10.4.22", - "postcss": "^8.5.6", - "tailwindcss": "^4.1.17", - "tw-animate-css": "^1.4.0", - "typescript": "^5.9.2", - "vite": "^7.1.7", - "vite-tsconfig-paths": "^5.1.4" - }, - "overrides": { - "monaco-editor": { - "dompurify": "^3.2.4" - } - } + "name": "benchr", + "private": true, + "type": "module", + "scripts": { + "build": "react-router build", + "dev": "react-router dev", + "start": "react-router-serve ./build/server/index.js", + "typecheck": "react-router typegen && tsc" + }, + "dependencies": { + "@monaco-editor/react": "^4.7.0", + "@radix-ui/react-accordion": "^1.2.12", + "@radix-ui/react-alert-dialog": "^1.1.15", + "@radix-ui/react-aspect-ratio": "^1.1.7", + "@radix-ui/react-avatar": "^1.1.10", + "@radix-ui/react-checkbox": "^1.3.3", + "@radix-ui/react-collapsible": "^1.1.12", + "@radix-ui/react-context-menu": "^2.2.16", + "@radix-ui/react-dialog": "^1.1.15", + "@radix-ui/react-dropdown-menu": "^2.1.16", + "@radix-ui/react-hover-card": "^1.1.15", + "@radix-ui/react-label": "^2.1.7", + "@radix-ui/react-menubar": "^1.1.16", + "@radix-ui/react-navigation-menu": "^1.2.14", + "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-progress": "^1.1.7", + "@radix-ui/react-radio-group": "^1.3.8", + "@radix-ui/react-scroll-area": "^1.2.10", + "@radix-ui/react-select": "^2.2.6", + "@radix-ui/react-separator": "^1.1.7", + "@radix-ui/react-slider": "^1.3.6", + "@radix-ui/react-slot": "^1.2.3", + "@radix-ui/react-toggle": "^1.1.10", + "@radix-ui/react-toggle-group": "^1.1.11", + "@radix-ui/react-tooltip": "^1.2.8", + "@react-router/node": "^7.9.2", + "@react-router/serve": "^7.9.2", + "@tanstack/react-table": "^8.21.3", + "axios": "^1.12.2", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "cmdk": "^1.1.1", + "date-fns": "^4.1.0", + "dompurify": "^3.3.0", + "embla-carousel-react": "^8.6.0", + "isbot": "^5.1.31", + "lucide-react": "^0.548.0", + "motion": "^12.23.24", + "react": "^19.1.1", + "react-day-picker": "^9.11.1", + "react-dom": "^19.1.1", + "react-resizable-panels": "^3.0.6", + "react-router": "^7.9.4", + "react-router-dom": "^7.10.1", + "recharts": "^2.15.4", + "redis": "^5.9.0", + "socket.io-client": "^4.8.1", + "tailwind-merge": "^3.3.1", + "vaul": "^1.1.2" + }, + "devDependencies": { + "@react-router/dev": "^7.9.2", + "@tailwindcss/postcss": "^4.1.17", + "@tailwindcss/vite": "^4.1.13", + "@types/node": "^22", + "@types/react": "^19.1.13", + "@types/react-dom": "^19.1.9", + "autoprefixer": "^10.4.22", + "postcss": "^8.5.6", + "tailwindcss": "^4.1.17", + "tw-animate-css": "^1.4.0", + "typescript": "^5.9.2", + "vite": "^7.1.7", + "vite-tsconfig-paths": "^5.1.4" + }, + "overrides": { + "monaco-editor": { + "dompurify": "^3.2.4" + } + } } diff --git a/frontend/pyright-report.json b/frontend/pyright-report.json new file mode 100644 index 00000000..e69de29b diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index dc391a45..87e2ef57 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -4,6 +4,7 @@ "**/.server/**/*", "**/.client/**/*", ".react-router/types/**/*" + ], "compilerOptions": { "lib": ["DOM", "DOM.Iterable", "ES2022"], @@ -15,8 +16,8 @@ "rootDirs": [".", "./.react-router/types"], "baseUrl": ".", "paths": { - "~/*": ["./app/*"] - }, + "~/*": ["./app/*", "app/~/*", "./app/types/*"] + }, "esModuleInterop": true, "verbatimModuleSyntax": true, "noEmit": true, diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..231c00e5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,14 @@ +[pytest] +testpaths = tests +asyncio_mode = auto +asyncio_default_fixture_loop_scope = function +addopts = -v --tb=short +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning +markers = + unit: Fast tests with mocked dependencies + integration: Tests requiring real Redis/DB + slow: Tests that take longer than 1s +# Python path for imports +pythonpath = backend diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..5d6fbf87 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,168 @@ +""" +Shared pytest fixtures for Benchr backend tests. + +Usage: + - Unit tests use `fake_redis` fixture (no real Redis needed) + - Integration tests use `redis_client` fixture (requires Redis service) + - API tests use `app_client` fixture (Quart test client) +""" + +import asyncio # noqa: F401 +import os +import sys + +import pytest + +backend_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backend") +sys.path.insert(0, backend_path) + +@pytest.fixture +def fake_redis(): + """ + Fake Redis client for unit tests. + No real Redis required - all operations are in-memory. + + Usage: + async def test_something(fake_redis): + await fake_redis.set("key", "value") + assert await fake_redis.get("key") == "value" + """ + fakeredis = pytest.importorskip("fakeredis") + import fakeredis.aioredis # noqa: F811 + + return fakeredis.aioredis.FakeRedis(decode_responses=True) + + +@pytest.fixture +async def redis_client(): + """ + Real Redis client for integration tests. + Requires Redis to be running (e.g., via docker-compose or CI service). + + Cleans up test keys after each test. + """ + import redis.asyncio as aioredis + + redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") + client = await aioredis.from_url(redis_url, decode_responses=True) + + try: + await client.ping() + except Exception as e: + pytest.skip(f"Redis not available: {e}") + + yield client + + async for key in client.scan_iter("test:*"): + await client.delete(key) + + await client.aclose() + +@pytest.fixture +def test_db(): + """ + In-memory SQLite database for testing Peewee models. + Creates tables before test, drops after. + """ + from models import Job, db + from peewee import SqliteDatabase + + test_database = SqliteDatabase(":memory:") + + original_db = db # noqa: F841 + Job._meta.database = test_database + with test_database: + test_database.create_tables([Job], safe=True) + + yield test_database + + with test_database: + test_database.drop_tables([Job], safe=True) + test_database.close() + + +@pytest.fixture +def sample_job(test_db): + """Create a sample job for testing.""" + from models import Job + + job = Job.create( + code='print("hello")', + lang="python", + compiler="python3", + opts="", + status="queued", + ) + return job +@pytest.fixture +def app(): + """ + Quart application instance for testing. + Configures test mode and returns app without running startup hooks. + """ + os.environ["TESTING"] = "true" + os.environ["DATABASE_URL"] = "sqlite:///test.db" + + from api import app as quart_app + + quart_app.config["TESTING"] = True + + return quart_app + + +@pytest.fixture +async def app_client(app): + """ + Quart test client for making HTTP requests. + + Usage: + async def test_health(app_client): + response = await app_client.get("/api/health") + assert response.status_code == 200 + """ + async with app.test_client() as client: + yield client + +@pytest.fixture +async def rate_limited_queue(fake_redis): + """ + RateLimitedQueue with fake Redis for unit testing. + """ + from rate_limiter import RateLimitedQueue + + queue = RateLimitedQueue( + redis_url="redis://fake", + queue_name="test:benchr", + max_requests=10, + window_seconds=60, + max_queue_size=100, + ) + + queue.redis = fake_redis + queue._pubsub = await fake_redis.pubsub() + + yield queue + + # Cleanup + await queue.clear() + +@pytest.fixture +def job_data(): + """Sample job submission data.""" + return { + "code": 'int main() { return 0; }', + "lang": "c", + "compiler": "gcc", + "opts": "-O2", + } + + +@pytest.fixture +def python_job_data(): + """Sample Python job submission data.""" + return { + "code": 'print("Hello, World!")', + "lang": "python", + "compiler": "python3", + "opts": "", + } diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py new file mode 100644 index 00000000..64749110 --- /dev/null +++ b/tests/integration/test_api.py @@ -0,0 +1,155 @@ +""" +Integration tests for Quart API endpoints. + +These tests require Redis to be running. +Run with: pytest tests/integration/test_api.py -v + +To skip if Redis unavailable, tests auto-skip via fixture. +""" + + +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio] + + +class TestHealthEndpoint: + """Tests for /api/health endpoint.""" + + async def test_health_returns_ok(self, app_client): + """Health endpoint should return 200 OK.""" + response = await app_client.get("/api/health") + assert response.status_code == 200 + data = await response.get_json() + assert data["status"] == "ok" + + async def test_health_includes_db_status(self, app_client): + """Health endpoint should include database status.""" + response = await app_client.get("/api/health") + data = await response.get_json() + + assert "database" in data + + +class TestSubmitEndpoint: + """Tests for /api/submit endpoint.""" + + async def test_submit_requires_code(self, app_client): + """Submit should require code field.""" + response = await app_client.post( + "/api/submit", + json={"lang": "python"}, + ) + + assert response.status_code == 400 + data = await response.get_json() + assert "code" in data["error"].lower() + + async def test_submit_requires_lang(self, app_client): + """Submit should require lang field.""" + response = await app_client.post( + "/api/submit", + json={"code": "print('hi')"}, + ) + + assert response.status_code == 400 + data = await response.get_json() + assert "language" in data["error"].lower() + + async def test_submit_valid_job(self, app_client, job_data): + """Should accept valid job submission.""" + response = await app_client.post("/api/submit", json=job_data) + + # Could be 201 (created) or 429 (rate limited) or 500 (redis not configured) + # In test environment without full setup, we just check it doesn't crash + assert response.status_code in [201, 429, 500] + + if response.status_code == 201: + data = await response.get_json() + assert "job_id" in data + assert data["status"] == "queued" + + +class TestJobsEndpoint: + """Tests for /api/jobs endpoints.""" + + async def test_list_jobs_returns_array(self, app_client): + """List jobs should return jobs array.""" + response = await app_client.get("/api/jobs") + + assert response.status_code == 200 + data = await response.get_json() + assert "jobs" in data + assert isinstance(data["jobs"], list) + + async def test_list_jobs_respects_limit(self, app_client): + """List jobs should respect limit parameter.""" + response = await app_client.get("/api/jobs?limit=5") + + assert response.status_code == 200 + data = await response.get_json() + assert len(data["jobs"]) <= 5 + + async def test_get_nonexistent_job(self, app_client): + """Getting nonexistent job should return error.""" + response = await app_client.get("/api/jobs/999999") + + # Should be 404 or 500 depending on error handling + assert response.status_code in [404, 500] + + +class TestCurrentJobEndpoint: + """Tests for /api/current endpoint.""" + + async def test_current_returns_job_or_null(self, app_client): + """Current endpoint should return job or null.""" + response = await app_client.get("/api/current") + + assert response.status_code == 200 + data = await response.get_json() + assert "job" in data + # job can be None if no jobs exist + + +class TestChatEndpoint: + """Tests for /api/chat endpoint.""" + + async def test_chat_requires_message(self, app_client): + """Chat should require message field.""" + response = await app_client.post("/api/chat", json={}) + + assert response.status_code == 400 + data = await response.get_json() + assert "message" in data["error"].lower() + + async def test_chat_returns_response(self, app_client): + """Chat should return a response.""" + response = await app_client.post( + "/api/chat", + json={"message": "Hello!"}, + ) + + assert response.status_code == 200 + data = await response.get_json() + assert "response" in data + + +class TestSavedBenchmarksEndpoint: + """Tests for /api/saved endpoints.""" + + async def test_save_requires_job_id(self, app_client): + """Save benchmark should require job_id.""" + response = await app_client.post( + "/api/saved", + json={"name": "my benchmark"}, + ) + + assert response.status_code == 400 + data = await response.get_json() + assert "job_id" in data["error"].lower() + + async def test_get_nonexistent_saved(self, app_client): + """Getting nonexistent saved benchmark should return 404.""" + response = await app_client.get("/api/saved/999999") + + assert response.status_code in [404, 500] diff --git a/tests/test_agent.py b/tests/test_agent.py index 5465e8e2..b3ca8cd9 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -3,9 +3,10 @@ Simple socket communication test Tests send_sock and rec_sock functions """ +import json import socket import struct -import json + def send_sock(sock, data: bytes): """Send length-prefixed message""" @@ -17,9 +18,9 @@ def rec_sock(sock) -> bytes: raw_len = sock.recv(4) if not raw_len or len(raw_len) < 4: raise RuntimeError("Failed to receive length header") - + msg_len = struct.unpack(">I", raw_len)[0] - + chunks = [] bytes_received = 0 while bytes_received < msg_len: @@ -28,7 +29,7 @@ def rec_sock(sock) -> bytes: raise RuntimeError("Socket connection broken") chunks.append(chunk) bytes_received += len(chunk) - + return b''.join(chunks) def test_vsock(): @@ -40,91 +41,91 @@ def test_vsock(): print("1. Firecracker VM must be running") print("2. Agent must be running inside VM on port 5000") print("=" * 60) - + VSOCK_PATH = "fc.vsock" - VSOCK_CID = 3 + VSOCK_CID = 3 # noqa: F841 VSOCK_PORT = 5000 - + try: # Connect to Firecracker vsock print(f"\n[Test] Connecting to vsock at {VSOCK_PATH}...") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.connect(VSOCK_PATH) print("[Test] ✓ Connected") - + # Send CONNECT command to Firecracker print(f"\n[Test] Sending CONNECT {VSOCK_PORT}...") sock.sendall(f"CONNECT {VSOCK_PORT}\n".encode('ascii')) - + # Wait for Firecracker's OK response ack = sock.recv(64).decode('ascii').strip() print(f"[Test] ✓ Received from Firecracker: {ack}") - + if not ack.startswith("OK"): raise RuntimeError(f"Firecracker connection failed: {ack}") - + print("[Test] ✓ Connected to agent via Firecracker") - + # Test job 1: Hello World print("\n" + "-" * 60) print("Test 1: Hello World") print("-" * 60) - + job1 = { "code": '#include \nint main() { printf("Hello, World!\\n"); return 0; }', "lang": "c", "compiler": "gcc", "opts": "-O2" } - + print("[Test] Sending job...") job_bytes = json.dumps(job1).encode('utf-8') send_sock(sock, job_bytes) - + print("[Test] Waiting for result...") result_bytes = rec_sock(sock) result = json.loads(result_bytes.decode('utf-8')) - + print(f"[Test] Success: {result.get('success')}") print(f"[Test] Exit Code: {result.get('exit_code')}") print(f"[Test] Output: {result.get('output', '').strip()}") - - assert result['success'] == True + + assert result['success'] == True # noqa: E712 assert result['exit_code'] == 0 print("[Test] ✓ Test 1 PASSED") - + # Test job 2: Compilation Error print("\n" + "-" * 60) print("Test 2: Compilation Error") print("-" * 60) - + job2 = { "code": '#include \nint main() { undefined_function(); return 0; }', "lang": "c", "compiler": "gcc", "opts": "-O2" } - + print("[Test] Sending job...") job_bytes = json.dumps(job2).encode('utf-8') send_sock(sock, job_bytes) - + print("[Test] Waiting for result...") result_bytes = rec_sock(sock) result = json.loads(result_bytes.decode('utf-8')) - + print(f"[Test] Success: {result.get('success')}") print(f"[Test] Error: {result.get('error')}") - - assert result['success'] == False + + assert result['success'] == False # noqa: E712 assert result['error'] == "compilation failed" print("[Test] ✓ Test 2 PASSED") - + # Test job 3: With Performance Metrics print("\n" + "-" * 60) print("Test 3: Performance Metrics") print("-" * 60) - + job3 = { "code": '''#include int main() { @@ -139,48 +140,48 @@ def test_vsock(): "compiler": "g++", "opts": "-O2 -Wall" } - + print("[Test] Sending job...") job_bytes = json.dumps(job3).encode('utf-8') send_sock(sock, job_bytes) - + print("[Test] Waiting for result...") result_bytes = rec_sock(sock) result = json.loads(result_bytes.decode('utf-8')) - + print(f"[Test] Success: {result.get('success')}") print(f"[Test] Exit Code: {result.get('exit_code')}") print(f"[Test] Has perf data: {result.get('perf') is not None}") print(f"[Test] Has time data: {result.get('time') is not None}") - + if result.get('perf'): print(f"[Test] Perf keys: {list(result['perf'].keys())}") - - assert result['success'] == True + + assert result['success'] == True # noqa: E712 print("[Test] ✓ Test 3 PASSED") - + # Close connection sock.close() print("\n" + "=" * 60) print("ALL TESTS PASSED ✓") print("=" * 60) - + except Exception as e: print(f"\n✗ TEST FAILED: {e}") import traceback traceback.print_exc() return 1 - + return 0 if __name__ == "__main__": import sys - + print("\nSimple Socket Communication Test") print("Make sure:") print(" 1. Firecracker VM is running with guest_cid=3") print(" 2. Agent is running inside the VM") print() input("Press Enter when ready...") - + sys.exit(test_vsock()) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py new file mode 100644 index 00000000..c58434d9 --- /dev/null +++ b/tests/unit/test_models.py @@ -0,0 +1,134 @@ +""" +Unit tests for Peewee models. + +Run with: pytest tests/unit/test_models.py -v +""" + +import pytest # noqa: I001 +from datetime import datetime + + +pytestmark = pytest.mark.unit + + +class TestJobModel: + """Tests for Job model.""" + + def test_create_job(self, test_db): + """Should create a job with required fields.""" + from models import Job + + job = Job.create( + code='print("test")', + lang="python", + compiler="python3", + opts="", + status="queued", + ) + + assert job.id is not None + assert job.code == 'print("test")' + assert job.lang == "python" + assert job.status == "queued" + + def test_job_has_created_at(self, test_db): + """Job should have auto-generated created_at timestamp.""" + from models import Job + + job = Job.create( + code="test", + lang="c", + compiler="gcc", + opts="-O2", + status="queued", + ) + + assert job.created_at is not None + assert isinstance(job.created_at, datetime) + + def test_job_status_transitions(self, test_db): + """Should be able to update job status.""" + from models import Job + + job = Job.create( + code="test", + lang="c", + compiler="gcc", + opts="", + status="queued", + ) + + job.status = "processing" + job.save() + + reloaded = Job.get_by_id(job.id) + assert reloaded.status == "processing" + + job.status = "completed" + job.save() + + reloaded = Job.get_by_id(job.id) + assert reloaded.status == "completed" + + def test_job_with_result(self, test_db): + """Job should store result as JSON string.""" + import json + + from models import Job + + job = Job.create( + code="test", + lang="c", + compiler="gcc", + opts="", + status="completed", + result=json.dumps({"time": 0.123, "output": "success"}), + ) + + reloaded = Job.get_by_id(job.id) + result = json.loads(reloaded.result) + + assert result["time"] == 0.123 + assert result["output"] == "success" + + def test_get_by_id(self, sample_job): + """Should retrieve job by ID.""" + from models import Job + + retrieved = Job.get_by_id(sample_job.id) + assert retrieved.id == sample_job.id + assert retrieved.code == sample_job.code + + def test_delete_job(self, test_db): + """Should be able to delete a job.""" + from models import Job + + job = Job.create( + code="test", + lang="python", + compiler="python3", + opts="", + status="queued", + ) + job_id = job.id + + Job.delete_by_id(job_id) + + with pytest.raises(Job.DoesNotExist): + Job.get_by_id(job_id) + + def test_list_jobs_ordered(self, test_db): + """Jobs should be retrievable in order.""" + import time # noqa: F401 + + from models import Job + + job1 = Job.create(code="1", lang="c", compiler="gcc", opts="", status="queued") + job2 = Job.create(code="2", lang="c", compiler="gcc", opts="", status="queued") # noqa: F841 + job3 = Job.create(code="3", lang="c", compiler="gcc", opts="", status="queued") + + jobs = list(Job.select().order_by(Job.id.desc()).limit(3)) + + assert len(jobs) == 3 + assert jobs[0].id == job3.id + assert jobs[2].id == job1.id diff --git a/tests/unit/test_queue.py b/tests/unit/test_queue.py new file mode 100644 index 00000000..3900cad6 --- /dev/null +++ b/tests/unit/test_queue.py @@ -0,0 +1,169 @@ +""" +Unit tests for IQueue implementations (GlobalQueue, RedisQueue). + +Run with: pytest tests/unit/test_queue.py -v +""" + +import pytest + +pytestmark = pytest.mark.unit + + +class TestGlobalQueue: + """Tests for in-memory GlobalQueue.""" + + def test_create_queue(self): + """Should create queue with specified maxsize.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=10) + assert queue.size() == 0 + assert queue.empty() is True + + def test_push_and_pop(self): + """Should push and pop items correctly.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=10) + + queue.push(42) + assert queue.size() == 1 + assert queue.empty() is False + + item = queue.pop() + assert item == 42 + assert queue.empty() is True + + def test_fifo_order(self): + """Queue should maintain FIFO order.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=10) + + queue.push(1) + queue.push(2) + queue.push(3) + + assert queue.pop() == 1 + assert queue.pop() == 2 + assert queue.pop() == 3 + + def test_full_queue(self): + """Should reject push when full.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=2) + + assert queue.push(1) is True + assert queue.push(2) is True + assert queue.full() is True + assert queue.push(3) is False # Rejected + + def test_pop_empty_returns_none(self): + """Pop on empty queue should return None.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=10) + assert queue.pop() is None + + def test_has_front(self): + """hasFront should reflect queue state.""" + from IQueue import GlobalQueue + + queue = GlobalQueue(maxsize=10) + + assert queue.hasFront() is False + queue.push(1) + assert queue.hasFront() is True + queue.pop() + assert queue.hasFront() is False + + +class TestRedisQueueUnit: + """ + Unit tests for RedisQueue using fakeredis. + These test the queue logic without a real Redis connection. + """ + + @pytest.fixture + def redis_queue(self, fake_redis): + """Create a RedisQueue with injected fake Redis.""" + from IQueue import RedisQueue + + # Create queue but skip real connection + queue = RedisQueue.__new__(RedisQueue) + queue.name = "test:queue" + queue.redis_url = "redis://fake" + queue.redis = fake_redis + queue.queued_key = "test:queue:queued" + queue.processing_key = "test:queue:processing" + queue.notify_channel = "test:queue:notify" + queue.maxsize = 100 + + return queue + + def test_push_increments_size(self, redis_queue): + """Push should increment queue size.""" + redis_queue.push(1) + assert redis_queue.size() == 1 + + def test_empty_on_new_queue(self, redis_queue): + """New queue should be empty.""" + assert redis_queue.empty() is True + + def test_not_empty_after_push(self, redis_queue): + """Queue should not be empty after push.""" + redis_queue.push(1) + assert redis_queue.empty() is False + + def test_has_front_reflects_state(self, redis_queue): + """hasFront should return True when items exist.""" + assert redis_queue.hasFront() is False + redis_queue.push(1) + assert redis_queue.hasFront() is True + + def test_full_at_maxsize(self, redis_queue): + """Queue should be full at maxsize.""" + redis_queue.maxsize = 3 + + redis_queue.push(1) + redis_queue.push(2) + redis_queue.push(3) + + assert redis_queue.full() is True + + def test_push_rejected_when_full(self, redis_queue): + """Push should return False when queue is full.""" + redis_queue.maxsize = 2 + + assert redis_queue.push(1) is True + assert redis_queue.push(2) is True + assert redis_queue.push(3) is False + + def test_queued_vs_processing_size(self, redis_queue): + """Should track queued and processing separately.""" + redis_queue.push(1) + redis_queue.push(2) + + assert redis_queue.queued_size() == 2 + assert redis_queue.processing_size() == 0 + + # Move one to processing + redis_queue.pend(timeout=1) + + assert redis_queue.queued_size() == 1 + assert redis_queue.processing_size() == 1 + # Total size unchanged + assert redis_queue.size() == 2 + + def test_clear_removes_all(self, redis_queue): + """Clear should remove all items.""" + redis_queue.push(1) + redis_queue.push(2) + redis_queue.pend(timeout=1) + + redis_queue.clear() + + assert redis_queue.size() == 0 + assert redis_queue.queued_size() == 0 + assert redis_queue.processing_size() == 0 diff --git a/tests/unit/test_rate_limiter.py b/tests/unit/test_rate_limiter.py new file mode 100644 index 00000000..022c83ee --- /dev/null +++ b/tests/unit/test_rate_limiter.py @@ -0,0 +1,181 @@ +""" +Unit tests for RateLimitedQueue. + +These tests use fakeredis - no real Redis required. +Run with: pytest tests/unit/test_rate_limiter.py -v +""" + +import pytest + + +pytestmark = pytest.mark.unit + + +class TestRateLimitedQueueBasics: + """Basic queue operations.""" + + async def test_push_to_empty_queue(self, rate_limited_queue): + """Should successfully push to an empty queue.""" + result = await rate_limited_queue.push(1) + assert result is True + + async def test_push_returns_job_in_queue(self, rate_limited_queue): + """Pushed job should be retrievable.""" + await rate_limited_queue.push(42) + + assert await rate_limited_queue.hasFront() is True + assert await rate_limited_queue.size() == 1 + + async def test_push_multiple_jobs(self, rate_limited_queue): + """Should handle multiple jobs.""" + for i in range(5): + await rate_limited_queue.push(i) + + assert await rate_limited_queue.size() == 5 + + async def test_empty_queue_is_empty(self, rate_limited_queue): + """Empty queue should report empty.""" + assert await rate_limited_queue.empty() is True + assert await rate_limited_queue.hasFront() is False + + async def test_queue_not_empty_after_push(self, rate_limited_queue): + """Queue should not be empty after push.""" + await rate_limited_queue.push(1) + assert await rate_limited_queue.empty() is False + + +class TestRateLimitedQueuePendPop: + """Test pend (move to processing) and pop (complete) operations.""" + + async def test_pend_moves_to_processing(self, rate_limited_queue): + """Pend should move job from queued to processing.""" + await rate_limited_queue.push(123) + + job_id = await rate_limited_queue.pend(timeout=1) + + assert job_id == 123 + # Job moved to processing, queued should be empty + assert await rate_limited_queue.queued_size() == 0 + assert await rate_limited_queue.processing_size() == 1 + + async def test_pend_empty_queue_returns_none(self, rate_limited_queue): + """Pend on empty queue should return None after timeout.""" + job_id = await rate_limited_queue.pend(timeout=0.1) + assert job_id is None + + async def test_pop_removes_from_processing(self, rate_limited_queue): + """Pop should remove job from processing queue.""" + await rate_limited_queue.push(456) + await rate_limited_queue.pend(timeout=1) + + await rate_limited_queue.pop(456) + + assert await rate_limited_queue.processing_size() == 0 + + async def test_full_lifecycle(self, rate_limited_queue): + """Test complete job lifecycle: push -> pend -> pop.""" + # Submit job + await rate_limited_queue.push(999) + assert await rate_limited_queue.size() == 1 + + # Worker picks up job + job_id = await rate_limited_queue.pend(timeout=1) + assert job_id == 999 + assert await rate_limited_queue.queued_size() == 0 + assert await rate_limited_queue.processing_size() == 1 + + # Job completes + await rate_limited_queue.pop(999) + assert await rate_limited_queue.size() == 0 + + +class TestRateLimiting: + """Test rate limiting functionality.""" + + async def test_rate_limit_allows_under_limit(self, fake_redis): + """Should allow requests under the rate limit.""" + from rate_limiter import RateLimitedQueue + + queue = RateLimitedQueue( + redis_url="redis://fake", + queue_name="test:rate", + max_requests=5, + window_seconds=60, + max_queue_size=100, + ) + queue.redis = fake_redis + + # Should allow 5 requests + for i in range(5): + result = await queue.push(i) + assert result is True, f"Request {i} should be allowed" + + async def test_rate_limit_rejects_over_limit(self, fake_redis): + """Should reject requests over the rate limit.""" + from rate_limiter import RateLimitedQueue + + queue = RateLimitedQueue( + redis_url="redis://fake", + queue_name="test:rate2", + max_requests=3, + window_seconds=60, + max_queue_size=100, + ) + queue.redis = fake_redis + + # First 3 should succeed + for i in range(3): + await queue.push(i) + + # 4th should be rejected + result = await queue.push(999) + assert result is False + + +class TestQueueCapacity: + """Test queue size limits.""" + + async def test_full_queue_rejects_push(self, fake_redis): + """Should reject push when queue is full.""" + from rate_limiter import RateLimitedQueue + + queue = RateLimitedQueue( + redis_url="redis://fake", + queue_name="test:full", + max_requests=100, # High rate limit + window_seconds=60, + max_queue_size=3, # Low capacity + ) + queue.redis = fake_redis + + # Fill the queue + for i in range(3): + await queue.push(i) + + assert await queue.full() is True + + # Should reject + result = await queue.push(999) + assert result is False + + async def test_queue_not_full_under_capacity(self, rate_limited_queue): + """Queue should not be full under capacity.""" + await rate_limited_queue.push(1) + assert await rate_limited_queue.full() is False + + +class TestClear: + """Test queue clearing.""" + + async def test_clear_empties_all_queues(self, rate_limited_queue): + """Clear should empty both queued and processing lists.""" + # Add some jobs + await rate_limited_queue.push(1) + await rate_limited_queue.push(2) + await rate_limited_queue.pend(timeout=1) # Move one to processing + + await rate_limited_queue.clear() + + assert await rate_limited_queue.size() == 0 + assert await rate_limited_queue.queued_size() == 0 + assert await rate_limited_queue.processing_size() == 0