diff --git a/README.md b/README.md index 25471ac..e91ff27 100644 --- a/README.md +++ b/README.md @@ -105,24 +105,28 @@ To run the ArduPilot Custom Firmware Builder locally without Docker, ensure you ``` 5. **Execute the Application:** - - For a development environment, run: + - For a development environment with auto-reload, run: ```bash - ./web/app.py + python3 web/main.py + ``` + To change the port, use the `--port` argument: + ```bash + python3 web/main.py --port 9000 ``` - For a production environment, use: ```bash - gunicorn web.wsgi:application + uvicorn web.main:app --host 0.0.0.0 --port 8080 ``` - During the coding and testing phases, use the development environment to easily debug and make changes. When deploying the app for end users, use the production environment to ensure better performance, scalability, and security. + During the coding and testing phases, use the development environment to easily debug and make changes with auto-reload enabled. When deploying the app for end users, use the production environment to ensure better performance, scalability, and security. - The application will automatically set up the required base directory at `./base` upon first execution. You may customize this path by using the `--basedir` option with the above commands or by setting the `CBS_BASEDIR` environment variable. + The application will automatically set up the required base directory at `./base` upon first execution. You may customize this path by setting the `CBS_BASEDIR` environment variable. 6. **Access the Web Interface:** - Once the application is running, you can access the interface in your web browser at http://localhost:5000 if running directly using app.py (development environment), or at http://localhost:8000 if using Gunicorn (production environment). + Once the application is running, you can access the interface in your web browser at http://localhost:8080. - To change the default port when running with app.py, modify the `app.run()` call in web/app.py file by passing `port=` as an argument. For Gunicorn, refer to the [commonly used arguments](https://docs.gunicorn.org/en/latest/run.html#commonly-used-arguments) section of the Gunicorn documentation to specify a different port. + The default port is 8080, or the value of the `WEB_PORT` environment variable if set. You can override this by passing the `--port` argument when running the application directly (e.g., `python3 web/main.py --port 9000`) or when using uvicorn (e.g., `uvicorn web.main:app --port 5000`). Refer to the [uvicorn documentation](https://www.uvicorn.org/) for additional configuration options. ## Directory Structure The default directory structure is established as follows: diff --git a/docker-compose.yml b/docker-compose.yml index cb55812..641029e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,14 +19,14 @@ services: CBS_LOG_LEVEL: ${CBS_LOG_LEVEL:-INFO} CBS_ENABLE_INBUILT_BUILDER: 0 CBS_GITHUB_ACCESS_TOKEN: ${CBS_GITHUB_ACCESS_TOKEN} + CBS_REMOTES_RELOAD_TOKEN: ${CBS_REMOTES_RELOAD_TOKEN} PYTHONPATH: /app - GUNICORN_CMD_ARGS: --bind=0.0.0.0:80 --timeout=300 volumes: - ./base:/base:rw depends_on: - redis ports: - - "127.0.0.1:${WEB_PORT:-8080}:80" + - "127.0.0.1:${WEB_PORT:-8080}:8080" builder: build: diff --git a/metadata_manager/ap_src_meta_fetcher.py b/metadata_manager/ap_src_meta_fetcher.py index ea0a604..33bfff0 100644 --- a/metadata_manager/ap_src_meta_fetcher.py +++ b/metadata_manager/ap_src_meta_fetcher.py @@ -478,6 +478,79 @@ def get_build_options_at_commit(self, remote: str, ) return build_options + def get_board_defaults_from_fw_server( + self, + artifacts_url: str, + board_id: str, + vehicle_id: str = None, + ) -> dict: + """ + Fetch board defaults from firmware.ardupilot.org features.txt. + + The features.txt file contains lines like: + - FEATURE_NAME (enabled features) + - !FEATURE_NAME (disabled features) + + Parameters: + artifacts_url (str): Base URL for build artifacts for a version. + board_id (str): Board identifier + vehicle_id (str): Vehicle identifier + (for special handling like Heli) + + Returns: + dict: Dictionary mapping feature define to state + (1 for enabled, 0 for disabled), or None if fetch fails + """ + import requests + + # Heli builds are stored under a separate folder + artifacts_subdir = board_id + if vehicle_id == "Heli": + artifacts_subdir += "-heli" + + features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt" + + try: + response = requests.get(features_txt_url, timeout=30) + response.raise_for_status() + + feature_states = {} + enabled_count = 0 + disabled_count = 0 + + for line in response.text.splitlines(): + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Check if feature is disabled (prefixed with !) + if line.startswith('!'): + feature_name = line[1:].strip() + if feature_name: + feature_states[feature_name] = 0 + disabled_count += 1 + else: + # Enabled feature + if line: + feature_states[line] = 1 + enabled_count += 1 + + self.logger.info( + f"Fetched board defaults from firmware server: " + f"{enabled_count} enabled, " + f"{disabled_count} disabled" + ) + + return feature_states + + except requests.RequestException as e: + self.logger.warning( + f"Failed to fetch board defaults from {features_txt_url}: {e}" + ) + return None + @staticmethod def get_singleton(): return APSourceMetadataFetcher.__singleton diff --git a/web/Dockerfile b/web/Dockerfile index aacb445..fa3c68c 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,7 +1,8 @@ FROM python:3.10.16-slim-bookworm RUN apt-get update \ - && apt-get install -y --no-install-recommends git gosu + && apt-get install -y --no-install-recommends git gosu \ + && rm -rf /var/lib/apt/lists/* RUN groupadd -g 999 ardupilot && \ useradd -u 999 -g 999 -m ardupilot --shell /bin/false && \ @@ -12,5 +13,9 @@ COPY --chown=ardupilot:ardupilot . /app WORKDIR /app/web RUN pip install --no-cache-dir -r requirements.txt +ENV PYTHONPATH=/app + +EXPOSE 8080 + ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["gunicorn", "wsgi:application"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/web/api/v1/__init__.py b/web/api/v1/__init__.py new file mode 100644 index 0000000..6cbe7be --- /dev/null +++ b/web/api/v1/__init__.py @@ -0,0 +1,4 @@ +"""API v1 module.""" +from .router import router + +__all__ = ["router"] diff --git a/web/api/v1/admin.py b/web/api/v1/admin.py new file mode 100644 index 0000000..5ce7e89 --- /dev/null +++ b/web/api/v1/admin.py @@ -0,0 +1,81 @@ +from fastapi import APIRouter, HTTPException, Depends, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials + +from schemas import RefreshRemotesResponse +from services.admin import get_admin_service, AdminService + + +router = APIRouter(prefix="/admin", tags=["admin"]) +security = HTTPBearer() + + +async def verify_admin_token( + credentials: HTTPAuthorizationCredentials = Depends(security), + admin_service: AdminService = Depends(get_admin_service) +) -> None: + """ + Verify the bearer token for admin authentication. + + Args: + credentials: HTTP authorization credentials from request header + admin_service: Admin service instance + + Raises: + 401: Invalid or missing token + 500: Server configuration error (token not configured) + """ + token = credentials.credentials + try: + if not await admin_service.verify_token(token): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication token" + ) + except RuntimeError as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + +@router.post( + "/refresh_remotes", + response_model=RefreshRemotesResponse, + responses={ + 401: {"description": "Invalid or missing authentication token"}, + 500: { + "description": ( + "Server configuration error (token not configured) " + "or refresh operation failed" + ) + } + } +) +async def refresh_remotes( + _: None = Depends(verify_admin_token), + admin_service: AdminService = Depends(get_admin_service) +): + """ + Trigger a hot reset/refresh of remote metadata. + + This endpoint requires bearer token authentication in the Authorization + header: + ``` + Authorization: Bearer + ``` + + Returns: + RefreshRemotesResponse: List of remotes that were refreshed + + Raises: + 401: Invalid or missing authentication token + 500: Refresh operation failed + """ + try: + remotes = await admin_service.refresh_remotes() + return RefreshRemotesResponse(remotes=remotes) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to refresh remotes: {str(e)}" + ) diff --git a/web/api/v1/builds.py b/web/api/v1/builds.py new file mode 100644 index 0000000..d0f0ab9 --- /dev/null +++ b/web/api/v1/builds.py @@ -0,0 +1,225 @@ +from typing import List, Optional +from fastapi import ( + APIRouter, + HTTPException, + Query, + Path, + status, + Depends, + Request +) +from fastapi.responses import FileResponse, PlainTextResponse + +from schemas import ( + BuildRequest, + BuildSubmitResponse, + BuildOut, +) +from services.builds import get_builds_service, BuildsService +from utils import RateLimitExceededException + +router = APIRouter(prefix="/builds", tags=["builds"]) + + +@router.post( + "", + response_model=BuildSubmitResponse, + status_code=status.HTTP_201_CREATED, + responses={ + 400: {"description": "Invalid build configuration"}, + 404: {"description": "Vehicle, board, or version not found"}, + 429: {"description": "Rate limit exceeded"} + } +) +async def create_build( + build_request: BuildRequest, + request: Request, + service: BuildsService = Depends(get_builds_service) +): + """ + Create a new build request. + + Args: + build_request: Build configuration including vehicle, board, version, + and selected features + + Returns: + Simple response with build_id, URL, and status + + Raises: + 400: Invalid build configuration + 404: Vehicle, board, or version not found + 429: Rate limit exceeded + """ + try: + # Get client IP for rate limiting + forwarded_for = request.headers.get('X-Forwarded-For', None) + if forwarded_for: + client_ip = forwarded_for.split(',')[0].strip() + else: + client_ip = request.client.host if request.client else "unknown" + + return service.create_build(build_request, client_ip) + except RateLimitExceededException as e: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail=str(e) + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("", response_model=List[BuildOut]) +async def list_builds( + vehicle_id: Optional[str] = Query( + None, description="Filter by vehicle ID" + ), + board_id: Optional[str] = Query( + None, description="Filter by board ID" + ), + state: Optional[str] = Query( + None, + description="Filter by build state (PENDING, RUNNING, SUCCESS, " + "FAILURE, CANCELLED)" + ), + limit: int = Query( + 20, ge=1, le=100, description="Maximum number of builds to return" + ), + offset: int = Query( + 0, ge=0, description="Number of builds to skip" + ), + service: BuildsService = Depends(get_builds_service) +): + """ + Get list of builds with optional filters. + + Args: + vehicle_id: Filter builds by vehicle + board_id: Filter builds by board + state: Filter builds by current state + limit: Maximum number of results + offset: Number of results to skip (for pagination) + + Returns: + List of builds matching the filters + """ + return service.list_builds( + vehicle_id=vehicle_id, + board_id=board_id, + state=state, + limit=limit, + offset=offset + ) + + +@router.get( + "/{build_id}", + response_model=BuildOut, + responses={ + 404: {"description": "Build not found"} + } +) +async def get_build( + build_id: str = Path(..., description="Unique build identifier"), + service: BuildsService = Depends(get_builds_service) +): + """ + Get details of a specific build. + + Args: + build_id: The unique build identifier + + Returns: + Complete build details including progress and status + + Raises: + 404: Build not found + """ + build = service.get_build(build_id) + if not build: + raise HTTPException( + status_code=404, + detail=f"Build with id '{build_id}' not found" + ) + return build + + +@router.get( + "/{build_id}/logs", + responses={ + 404: {"description": "Build not found or logs not available yet"} + } +) +async def get_build_logs( + build_id: str = Path(..., description="Unique build identifier"), + tail: Optional[int] = Query( + None, ge=1, description="Return only the last N lines" + ), + service: BuildsService = Depends(get_builds_service) +): + """ + Get build logs for a specific build. + + Args: + build_id: The unique build identifier + tail: Optional number of last lines to return + + Returns: + Build logs as text + + Raises: + 404: Build not found + 404: Logs not available yet + """ + logs = service.get_build_logs(build_id, tail) + if logs is None: + raise HTTPException( + status_code=404, + detail=f"Logs not available for build '{build_id}'" + ) + return PlainTextResponse(content=logs) + + +@router.get( + "/{build_id}/artifact", + responses={ + 404: { + "description": ( + "Build not found or artifact not available " + ) + } + } +) +async def download_artifact( + build_id: str = Path(..., description="Unique build identifier"), + service: BuildsService = Depends(get_builds_service) +): + """ + Download the build artifact (firmware binary). + + Args: + build_id: The unique build identifier + + Returns: + Binary file download + + Raises: + 404: Build not found + 404: Artifact not available (build not completed successfully) + """ + artifact_path = service.get_artifact_path(build_id) + if not artifact_path: + raise HTTPException( + status_code=404, + detail=( + f"Artifact not available for build '{build_id}'. " + "Build may not be completed or successful." + ) + ) + return FileResponse( + path=artifact_path, + media_type='application/gzip', + filename=f"{build_id}.tar.gz" + ) diff --git a/web/api/v1/router.py b/web/api/v1/router.py new file mode 100644 index 0000000..9597591 --- /dev/null +++ b/web/api/v1/router.py @@ -0,0 +1,17 @@ +""" +Main API v1 router. + +This module aggregates all v1 API endpoints and provides a single router +to be included in the main FastAPI application. +""" +from fastapi import APIRouter + +from . import vehicles, builds, admin + +# Create the main v1 router +router = APIRouter(prefix="/v1") + +# Include all sub-routers +router.include_router(vehicles.router) +router.include_router(builds.router) +router.include_router(admin.router) diff --git a/web/api/v1/vehicles.py b/web/api/v1/vehicles.py new file mode 100644 index 0000000..b5168d2 --- /dev/null +++ b/web/api/v1/vehicles.py @@ -0,0 +1,253 @@ +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, Query, Path + +from schemas import ( + VehicleBase, + VersionOut, + BoardOut, + FeatureOut, +) +from services.vehicles import get_vehicles_service, VehiclesService + +router = APIRouter(prefix="/vehicles", tags=["vehicles"]) + + +@router.get("", response_model=List[VehicleBase]) +async def list_vehicles( + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get list of all available vehicles. + + Returns: + List of vehicles with their IDs and names. + """ + return service.get_all_vehicles() + + +@router.get( + "/{vehicle_id}", + response_model=VehicleBase, + responses={ + 404: {"description": "Vehicle not found"} + } +) +async def get_vehicle( + vehicle_id: str = Path(..., description="Unique vehicle identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get a specific vehicle by ID. + + Args: + vehicle_id: The vehicle identifier (e.g., 'copter', 'plane') + + Returns: + Vehicle details + """ + vehicle = service.get_vehicle(vehicle_id) + if not vehicle: + raise HTTPException( + status_code=404, + detail=f"Vehicle with id '{vehicle_id}' not found" + ) + return vehicle + + +# --- Version Endpoints --- +@router.get("/{vehicle_id}/versions", response_model=List[VersionOut]) +async def list_versions( + vehicle_id: str = Path(..., description="Vehicle identifier"), + type: Optional[str] = Query( + None, + description=( + "Filter by version type " + "(beta, stable, latest, tag)" + ) + ), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get all versions available for a specific vehicle. + + Args: + vehicle_id: The vehicle identifier + type: Optional filter by version type + + Returns: + List of versions for the vehicle + """ + return service.get_versions(vehicle_id, type_filter=type) + + +@router.get( + "/{vehicle_id}/versions/{version_id}", + response_model=VersionOut, + responses={ + 404: {"description": "Version not found for the vehicle"} + } +) +async def get_version( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get details of a specific version for a vehicle. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + + Returns: + Version details + """ + version = service.get_version(vehicle_id, version_id) + if not version: + raise HTTPException( + status_code=404, + detail=( + f"Version '{version_id}' not found for " + f"vehicle '{vehicle_id}'" + ) + ) + return version + + +# --- Board Endpoints --- +@router.get( + "/{vehicle_id}/versions/{version_id}/boards", + response_model=List[BoardOut], + responses={ + 404: {"description": "No boards found for the vehicle version"} + } +) +async def list_boards( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get all boards available for a specific vehicle version. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + + Returns: + List of boards for the vehicle version + """ + boards = service.get_boards(vehicle_id, version_id) + if not boards: + raise HTTPException( + status_code=404, + detail=( + f"No boards found for vehicle '{vehicle_id}' and " + f"version '{version_id}'" + ) + ) + + return boards + + +@router.get( + "/{vehicle_id}/versions/{version_id}/boards/{board_id}", + response_model=BoardOut, + responses={ + 404: {"description": "Board not found"} + } +) +async def get_board( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + board_id: str = Path(..., description="Board identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get details of a specific board for a vehicle version. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + board_id: The board identifier + + Returns: + Board details + """ + board = service.get_board(vehicle_id, version_id, board_id) + if not board: + raise HTTPException( + status_code=404, + detail=f"Board '{board_id}' not found" + ) + return board + + +# --- Feature Endpoints --- +@router.get( + "/{vehicle_id}/versions/{version_id}/boards/{board_id}/features", + response_model=List[FeatureOut] +) +async def list_features( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + board_id: str = Path(..., description="Board identifier"), + category_id: Optional[str] = Query( + None, description="Filter by category ID" + ), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get all features with defaults for a specific vehicle/version/board. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + board_id: The board identifier + category_id: Optional filter by category + + Returns: + List of features with default settings for the board + """ + features = service.get_features( + vehicle_id, version_id, board_id, category_id + ) + return features + + +@router.get( + "/{vehicle_id}/versions/{version_id}/boards/{board_id}/" + "features/{feature_id}", + response_model=FeatureOut, + responses={ + 404: {"description": "Feature not found"} + } +) +async def get_feature( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + board_id: str = Path(..., description="Board identifier"), + feature_id: str = Path(..., description="Feature identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get details of a specific feature for a vehicle/version/board. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + board_id: The board identifier + feature_id: The feature identifier + + Returns: + Feature details with default settings + """ + feature = service.get_feature( + vehicle_id, version_id, board_id, feature_id + ) + if not feature: + raise HTTPException( + status_code=404, + detail=f"Feature '{feature_id}' not found" + ) + return feature diff --git a/web/app.py b/web/app.py deleted file mode 100755 index 8a4aa43..0000000 --- a/web/app.py +++ /dev/null @@ -1,404 +0,0 @@ -#!/usr/bin/env python3 - -import os -from flask import Flask, render_template, request, send_from_directory, jsonify, redirect -from threading import Thread -import sys -import requests -import signal - -from logging.config import dictConfig - -dictConfig({ - 'version': 1, - 'formatters': {'default': { - 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', - }}, - 'handlers': {'wsgi': { - 'class': 'logging.StreamHandler', - 'stream': 'ext://flask.logging.wsgi_errors_stream', - 'formatter': 'default' - }}, - 'root': { - 'level': os.getenv('CBS_LOG_LEVEL', default='INFO'), - 'handlers': ['wsgi'] - } -}) - -# let app.py know about the modules in the parent directory -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import ap_git -import metadata_manager -import build_manager -from builder import Builder - -# run at lower priority -os.nice(20) - -import optparse -parser = optparse.OptionParser("app.py") - -parser.add_option("", "--basedir", type="string", - default=os.getenv( - key="CBS_BASEDIR", - default=os.path.abspath(os.path.join(os.path.dirname(__file__),"..","base")) - ), - help="base directory") - -cmd_opts, cmd_args = parser.parse_args() - -# define directories -basedir = os.path.abspath(cmd_opts.basedir) -sourcedir = os.path.join(basedir, 'ardupilot') -outdir_parent = os.path.join(basedir, 'artifacts') -workdir_parent = os.path.join(basedir, 'workdir') - -appdir = os.path.dirname(__file__) - -builds_dict = {} -REMOTES = None - -repo = ap_git.GitRepo.clone_if_needed( - source="https://github.com/ardupilot/ardupilot.git", - dest=sourcedir, - recurse_submodules=True, -) - -vehicles_manager = metadata_manager.VehiclesManager() -ap_src_metadata_fetcher = metadata_manager.APSourceMetadataFetcher( - ap_repo=repo, - caching_enabled=True, - redis_host=os.getenv('CBS_REDIS_HOST', default='localhost'), - redis_port=os.getenv('CBS_REDIS_PORT', default='6379'), -) -versions_fetcher = metadata_manager.VersionsFetcher( - remotes_json_path=os.path.join(basedir, 'configs', 'remotes.json'), - ap_repo=repo -) - -manager = build_manager.BuildManager( - outdir=outdir_parent, - redis_host=os.getenv('CBS_REDIS_HOST', default='localhost'), - redis_port=os.getenv('CBS_REDIS_PORT', default='6379') -) -cleaner = build_manager.BuildArtifactsCleaner() -progress_updater = build_manager.BuildProgressUpdater() - -versions_fetcher.start() -cleaner.start() -progress_updater.start() - -# Initialize builder if enabled -builder = None -builder_thread = None -if os.getenv('CBS_ENABLE_INBUILT_BUILDER', default='1') == '1': - builder = Builder( - workdir=workdir_parent, - source_repo=repo - ) - builder_thread = Thread( - target=builder.run, - daemon=True - ) - builder_thread.start() - -app = Flask(__name__, template_folder='templates') - -# Setup graceful shutdown handler -def shutdown_handler(signum=None, frame=None): - """ - Gracefully shutdown all background services. - """ - app.logger.info("Shutting down application gracefully...") - - # Stop all TaskRunner instances - versions_fetcher.stop() - cleaner.stop() - progress_updater.stop() - - # Request builder shutdown if it's running - if builder is not None: - builder.shutdown() - - app.logger.info("All background services stopped successfully.") - sys.exit(0) - -# Register signal handlers for graceful shutdown -signal.signal(signal.SIGINT, shutdown_handler) -signal.signal(signal.SIGTERM, shutdown_handler) - -versions_fetcher.reload_remotes_json() -app.logger.info('Python version is: %s' % sys.version) - -def get_auth_token(): - try: - # try to read the secret token from the file - with open(os.path.join(basedir, 'secrets', 'reload_token'), 'r') as file: - token = file.read().strip() - return token - except (FileNotFoundError, PermissionError): - app.logger.error("Couldn't open token file. Checking environment for token.") - # if the file does not exist, check the environment variable - return os.getenv('CBS_REMOTES_RELOAD_TOKEN') - -@app.route('/refresh_remotes', methods=['POST']) -def refresh_remotes(): - auth_token = get_auth_token() - - if auth_token is None: - app.logger.error("Couldn't retrieve authorization token") - return "Internal Server Error", 500 - - token = request.get_json().get('token') - if not token or token != auth_token: - return "Unauthorized", 401 - - versions_fetcher.reload_remotes_json() - return "Successfully refreshed remotes", 200 - -@app.route('/generate', methods=['GET', 'POST']) -def generate(): - try: - version = request.form['version'] - vehicle = request.form['vehicle'] - - version_info = versions_fetcher.get_version_info( - vehicle_id=vehicle, - version_id=version - ) - - if version_info is None: - raise Exception("Version invalid or not listed to be built for given vehicle") - - remote_name = version_info.remote_info.name - commit_ref = version_info.commit_ref - - board = request.form['board'] - boards_at_commit = ap_src_metadata_fetcher.get_boards( - remote=remote_name, - commit_ref=commit_ref, - vehicle_id=vehicle, - ) - if board not in boards_at_commit: - raise Exception("bad board") - - all_features = ap_src_metadata_fetcher.get_build_options_at_commit( - remote=remote_name, - commit_ref=commit_ref - ) - - chosen_defines = { - feature.define - for feature in all_features - if request.form.get(feature.label) == "1" - } - - git_hash = repo.commit_id_for_remote_ref( - remote=remote_name, - commit_ref=commit_ref - ) - - build_info = build_manager.BuildInfo( - vehicle_id=vehicle, - remote_info=version_info.remote_info, - git_hash=git_hash, - board=board, - selected_features=chosen_defines - ) - - forwarded_for = request.headers.get('X-Forwarded-For', None) - if forwarded_for: - client_ip = forwarded_for.split(',')[0].strip() - else: - client_ip = request.remote_addr - - build_id = manager.submit_build( - build_info=build_info, - client_ip=client_ip, - ) - - app.logger.info('Redirecting to /viewlog') - return redirect('/viewlog/'+build_id) - - except Exception as ex: - app.logger.error(ex) - return render_template('error.html', ex=ex) - -@app.route('/add_build') -def add_build(): - app.logger.info('Rendering add_build.html') - return render_template('add_build.html') - - -def filter_build_options_by_category(build_options, category): - return sorted([f for f in build_options if f.category == category], key=lambda x: x.description.lower()) - -def parse_build_categories(build_options): - return sorted(list(set([f.category for f in build_options]))) - -@app.route('/', defaults={'token': None}, methods=['GET']) -@app.route('/viewlog/', methods=['GET']) -def home(token): - if token: - app.logger.info("Showing log for build id " + token) - app.logger.info('Rendering index.html') - return render_template('index.html', token=token) - -@app.route("/builds//artifacts/") -def download_file(build_id, name): - path = os.path.join( - basedir, - 'artifacts', - build_id, - ) - app.logger.info('Downloading %s/%s' % (path, name)) - return send_from_directory(path, name, as_attachment=False) - -@app.route("/boards_and_features//", methods=['GET']) -def boards_and_features(vehicle_id, version_id): - version_info = versions_fetcher.get_version_info( - vehicle_id=vehicle_id, - version_id=version_id - ) - - if version_info is None: - return "Bad request. Version not allowed to build for the vehicle.", 400 - - remote_name = version_info.remote_info.name - commit_reference = version_info.commit_ref - - app.logger.info('Board list and build options requested for %s %s' % (vehicle_id, version_id)) - # getting board list for the branch - with repo.get_checkout_lock(): - boards = ap_src_metadata_fetcher.get_boards( - remote=remote_name, - commit_ref=commit_reference, - vehicle_id=vehicle_id, - ) - - options = ap_src_metadata_fetcher.get_build_options_at_commit( - remote=remote_name, - commit_ref=commit_reference - ) # this is a list of Feature() objects defined in build_options.py - - # parse the set of categories from these objects - categories = parse_build_categories(options) - features = [] - for category in categories: - filtered_options = filter_build_options_by_category(options, category) - category_options = [] # options belonging to a given category - for option in filtered_options: - category_options.append({ - 'label' : option.label, - 'description' : option.description, - 'default' : option.default, - 'define' : option.define, - 'dependency' : option.dependency, - }) - features.append({ - 'name' : category, - 'options' : category_options, - }) - # creating result dictionary - result = { - 'boards' : boards, - 'default_board' : boards[0], - 'features' : features, - } - # return jsonified result dict - return jsonify(result) - -@app.route("/get_versions/", methods=['GET']) -def get_versions(vehicle_id): - versions = list() - for version_info in versions_fetcher.get_versions_for_vehicle(vehicle_id=vehicle_id): - if version_info.release_type == "latest": - title = f"Latest ({version_info.remote_info.name})" - else: - title = f"{version_info.release_type} {version_info.version_number} ({version_info.remote_info.name})" - versions.append({ - "title": title, - "id": version_info.version_id, - }) - - return jsonify(sorted(versions, key=lambda x: x['title'])) - -@app.route("/get_vehicles") -def get_vehicles(): - vehicles = [ - {"id": vehicle.id, "name": vehicle.name} - for vehicle in vehicles_manager.get_all_vehicles() - ] - return jsonify(sorted(vehicles, key=lambda x: x['id'])) - -@app.route("/get_defaults///", methods = ['GET']) -def get_deafults(vehicle_id, version_id, board_name): - vehicle = vehicles_manager.get_vehicle_by_id(vehicle_id) - if vehicle is None: - return "Invalid vehicle ID", 400 - # Heli is built on copter boards with -heli suffix - if vehicle_id == "heli": - board_name += "-heli" - - version_info = versions_fetcher.get_version_info( - vehicle_id=vehicle_id, - version_id=version_id - ) - - if version_info is None: - return "Bad request. Version is not allowed for builds for the %s." % vehicle.name, 400 - - artifacts_dir = version_info.ap_build_artifacts_url - - if artifacts_dir is None: - return "Couldn't find artifacts for requested release/branch/commit on ardupilot server", 404 - - url_to_features_txt = artifacts_dir + '/' + board_name + '/features.txt' - response = requests.get(url_to_features_txt, timeout=30) - - if not response.status_code == 200: - return ("Could not retrieve features.txt for given vehicle, version and board combination (Status Code: %d, url: %s)" % (response.status_code, url_to_features_txt), response.status_code) - # split response by new line character to get a list of defines - result = response.text.split('\n') - # omit the last two elements as they are always blank - return jsonify(result[:-2]) - -@app.route('/builds', methods=['GET']) -def get_all_builds(): - all_build_ids = manager.get_all_build_ids() - all_build_info = [ - { - **manager.get_build_info(build_id).to_dict(), - 'build_id': build_id - } - for build_id in all_build_ids - ] - - all_build_info_sorted = sorted( - all_build_info, - key=lambda x: x['time_created'], - reverse=True, - ) - - return ( - jsonify(all_build_info_sorted), - 200 - ) - -@app.route('/builds/', methods=['GET']) -def get_build_by_id(build_id): - if not manager.build_exists(build_id): - response = { - 'error': f'build with id {build_id} does not exist.', - } - return jsonify(response), 200 - - response = { - **manager.get_build_info(build_id).to_dict(), - 'build_id': build_id - } - - return jsonify(response), 200 - -if __name__ == '__main__': - app.run() diff --git a/web/core/__init__.py b/web/core/__init__.py new file mode 100644 index 0000000..1028ac4 --- /dev/null +++ b/web/core/__init__.py @@ -0,0 +1,10 @@ +""" +Core application components. +""" +from .config import get_settings +from .startup import initialize_application + +__all__ = [ + "get_settings", + "initialize_application", +] diff --git a/web/core/config.py b/web/core/config.py new file mode 100644 index 0000000..f322b0e --- /dev/null +++ b/web/core/config.py @@ -0,0 +1,85 @@ +""" +Application configuration and settings. +""" +import os +from pathlib import Path +from functools import lru_cache + + +class Settings: + """Application settings.""" + + def __init__(self): + # Application + self.app_name: str = "CustomBuild API" + self.app_version: str = "1.0.0" + self.debug: bool = False + + # Paths + self.base_dir: str = os.getenv( + "CBS_BASEDIR", + default=str(Path(__file__).parent.parent.parent.parent / "base") + ) + + # Redis + self.redis_host: str = os.getenv( + 'CBS_REDIS_HOST', + default='localhost' + ) + self.redis_port: str = os.getenv( + 'CBS_REDIS_PORT', + default='6379' + ) + + # Logging + self.log_level: str = os.getenv('CBS_LOG_LEVEL', default='INFO') + + # ArduPilot Git Repository + self.ap_git_url: str = "https://github.com/ardupilot/ardupilot.git" + + @property + def source_dir(self) -> str: + """ArduPilot source directory.""" + return os.path.join(self.base_dir, 'ardupilot') + + @property + def artifacts_dir(self) -> str: + """Build artifacts directory.""" + return os.path.join(self.base_dir, 'artifacts') + + @property + def outdir_parent(self) -> str: + """Build output directory (same as artifacts_dir).""" + return self.artifacts_dir + + @property + def workdir_parent(self) -> str: + """Work directory parent.""" + return os.path.join(self.base_dir, 'workdir') + + @property + def remotes_json_path(self) -> str: + """Path to remotes.json configuration.""" + return os.path.join(self.base_dir, 'configs', 'remotes.json') + + @property + def admin_token_file_path(self) -> str: + """Path to admin token secret file.""" + return os.path.join(self.base_dir, 'secrets', 'reload_token') + + @property + def enable_inbuilt_builder(self) -> bool: + """Whether to enable the inbuilt builder.""" + return os.getenv('CBS_ENABLE_INBUILT_BUILDER', '1') == '1' + + @property + def admin_token_env(self) -> str: + """Token required to reload remotes.json via API.""" + env = os.getenv('CBS_REMOTES_RELOAD_TOKEN', '') + return env if env != '' else None + + +@lru_cache() +def get_settings() -> Settings: + """Get cached settings instance.""" + return Settings() diff --git a/web/core/logging_config.py b/web/core/logging_config.py new file mode 100644 index 0000000..24a7bb0 --- /dev/null +++ b/web/core/logging_config.py @@ -0,0 +1,85 @@ +""" +Logging configuration for the application. +""" +import logging +import logging.config +import os +import sys + + +def setup_logging(log_level: str = None): + """ + Configure logging for the application and all imported modules. + + This must be called BEFORE importing any modules that use logging, + to ensure they all use the same logging configuration. + + Args: + log_level: The logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL). + If None, reads from CBS_LOG_LEVEL environment variable. + """ + if log_level is None: + log_level = os.getenv('CBS_LOG_LEVEL', default='INFO') + + # Configure logging with dictConfig for consistency with Flask app + logging_config = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': ( + '[%(asctime)s] %(levelname)s in %(module)s: ' + '%(message)s' + ), + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + 'detailed': { + 'format': ( + '[%(asctime)s] %(levelname)s ' + '[%(name)s.%(funcName)s:%(lineno)d] %(message)s' + ), + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'default', + 'level': log_level.upper(), + }, + }, + 'root': { + 'level': log_level.upper(), + 'handlers': ['console'], + }, + 'loggers': { + 'uvicorn': { + 'level': 'INFO', + 'handlers': ['console'], + 'propagate': False, + }, + 'uvicorn.access': { + 'level': 'INFO', + 'handlers': ['console'], + 'propagate': False, + }, + 'uvicorn.error': { + 'level': 'INFO', + 'handlers': ['console'], + 'propagate': False, + }, + 'fastapi': { + 'level': log_level.upper(), + 'handlers': ['console'], + 'propagate': False, + }, + }, + } + + logging.config.dictConfig(logging_config) + + # Log that logging has been configured + logger = logging.getLogger(__name__) + logger.info(f"Logging configured with level: {log_level.upper()}") + logger.info(f"Python version: {sys.version}") diff --git a/web/core/startup.py b/web/core/startup.py new file mode 100644 index 0000000..08c7f7d --- /dev/null +++ b/web/core/startup.py @@ -0,0 +1,104 @@ +""" +Application startup utilities. + +Handles initial setup of required directories and configuration files. +This module ensures the application environment is properly configured +before the main application starts. +""" +import os +import logging + +logger = logging.getLogger(__name__) + + +def ensure_base_structure(base_dir: str) -> None: + """ + Ensure required base directory structure exists. + + Creates necessary subdirectories for artifacts, configs, workdir, + and secrets if they don't already exist. + + Args: + base_dir: The base directory path (typically from CBS_BASEDIR) + """ + if not base_dir: + logger.warning("Base directory not specified, skipping initialization") + return + + # Define required subdirectories + subdirs = [ + 'artifacts', + 'configs', + 'workdir', + 'secrets', + ] + + for subdir in subdirs: + path = os.path.join(base_dir, subdir) + os.makedirs(path, exist_ok=True) + logger.debug(f"Ensured directory exists: {path}") + + +def ensure_remotes_json(base_dir: str, remote_name: str = "ardupilot") -> None: + """ + Ensure remotes.json configuration file exists. + + If the remotes.json file doesn't exist, creates it by fetching release + information from the specified remote. + + Args: + base_dir: The base directory path (typically from CBS_BASEDIR) + remote_name: The remote repository name to fetch releases from + """ + if not base_dir: + logger.warning( + "Base directory not specified, " + "skipping remotes.json initialization" + ) + return + + remotes_json_path = os.path.join(base_dir, 'configs', 'remotes.json') + + if not os.path.isfile(remotes_json_path): + logger.info( + f"remotes.json not found at {remotes_json_path}, " + f"creating it..." + ) + try: + from scripts import fetch_releases + fetch_releases.run( + base_dir=base_dir, + remote_name=remote_name, + ) + logger.info("Successfully created remotes.json") + except Exception as e: + logger.error(f"Failed to create remotes.json: {e}") + raise + else: + logger.debug(f"remotes.json already exists at {remotes_json_path}") + + +def initialize_application(base_dir: str) -> None: + """ + Initialize the application environment. + + Performs all necessary setup operations including: + - Creating required directory structure + - Ensuring remotes.json configuration exists + + Args: + base_dir: The base directory path (typically from CBS_BASEDIR) + """ + if not base_dir: + logger.warning("CBS_BASEDIR not set, skipping initialization") + return + + logger.info(f"Initializing application with base directory: {base_dir}") + + # Ensure directory structure + ensure_base_structure(base_dir) + + # Ensure remotes.json exists + ensure_remotes_json(base_dir) + + logger.info("Application initialization complete") diff --git a/web/main.py b/web/main.py new file mode 100755 index 0000000..4f7adbf --- /dev/null +++ b/web/main.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +""" +Main FastAPI application entry point. +""" +from contextlib import asynccontextmanager +from pathlib import Path +import threading +import os +import argparse + +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles + +from api.v1 import router as v1_router +from ui import router as ui_router +from core.config import get_settings +from core.startup import initialize_application +from core.logging_config import setup_logging + +import ap_git +import metadata_manager +import build_manager + +setup_logging() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """ + Lifespan context manager for startup and shutdown events. + """ + # Startup + settings = get_settings() + + initialize_application(settings.base_dir) + + repo = ap_git.GitRepo.clone_if_needed( + source=settings.ap_git_url, + dest=settings.source_dir, + recurse_submodules=True, + ) + + vehicles_manager = metadata_manager.VehiclesManager() + + ap_src_metadata_fetcher = metadata_manager.APSourceMetadataFetcher( + ap_repo=repo, + caching_enabled=True, + redis_host=settings.redis_host, + redis_port=settings.redis_port, + ) + + versions_fetcher = metadata_manager.VersionsFetcher( + remotes_json_path=settings.remotes_json_path, + ap_repo=repo + ) + versions_fetcher.reload_remotes_json() + + build_mgr = build_manager.BuildManager( + outdir=settings.outdir_parent, + redis_host=settings.redis_host, + redis_port=settings.redis_port + ) + + cleaner = build_manager.BuildArtifactsCleaner() + progress_updater = build_manager.BuildProgressUpdater() + + inbuilt_builder = None + inbuilt_builder_thread = None + if settings.enable_inbuilt_builder: + from builder.builder import Builder # noqa: E402 + inbuilt_builder = Builder( + workdir=settings.workdir_parent, + source_repo=repo + ) + inbuilt_builder_thread = threading.Thread( + target=inbuilt_builder.run, + daemon=True + ) + inbuilt_builder_thread.start() + + versions_fetcher.start() + cleaner.start() + progress_updater.start() + + app.state.repo = repo + app.state.ap_src_metadata_fetcher = ap_src_metadata_fetcher + app.state.versions_fetcher = versions_fetcher + app.state.vehicles_manager = vehicles_manager + app.state.build_manager = build_mgr + app.state.inbuilt_builder = inbuilt_builder + app.state.inbuilt_builder_thread = inbuilt_builder_thread + + yield + + # Shutdown + versions_fetcher.stop() + cleaner.stop() + progress_updater.stop() + if inbuilt_builder is not None: + inbuilt_builder.shutdown() + if (inbuilt_builder_thread is not None and + inbuilt_builder_thread.is_alive()): + inbuilt_builder_thread.join() + + +# Create FastAPI application +app = FastAPI( + title="CustomBuild API", + description="API for ArduPilot Custom Firmware Builder", + version="1.0.0", + docs_url="/api/docs", + redoc_url="/api/redoc", + lifespan=lifespan, +) + +# Mount static files +WEB_ROOT = Path(__file__).resolve().parent +app.mount( + "/static", + StaticFiles(directory=str(WEB_ROOT / "static")), + name="static" +) + +# Include API v1 router +app.include_router(v1_router, prefix="/api") + +# Include Web UI router +app.include_router(ui_router) + + +@app.get("/health") +async def health_check(): + """Health check endpoint.""" + return {"status": "healthy"} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="CustomBuild API Server") + parser.add_argument( + "--port", + type=int, + default=int(os.getenv("WEB_PORT", 8080)), + help="Port to run the server on (default: 8080 or WEB_PORT env var)" + ) + args = parser.parse_args() + + import uvicorn + uvicorn.run( + "main:app", + host="0.0.0.0", + port=args.port, + reload=True + ) diff --git a/web/requirements.txt b/web/requirements.txt index cd57da3..c513415 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,6 +1,10 @@ -flask -requests -jsonschema -dill==0.3.8 +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +pydantic==2.5.0 redis==5.2.1 -gunicorn==21.1 +requests==2.31.0 +jsonschema==4.20.0 +dill==0.3.8 +packaging==25.0 +jinja2==3.1.2 +python-multipart==0.0.6 diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py new file mode 100644 index 0000000..3f9340a --- /dev/null +++ b/web/schemas/__init__.py @@ -0,0 +1,54 @@ +""" +API schemas for the CustomBuild application. + +This module exports all Pydantic models used for request/response validation +across the API endpoints. +""" + +# Admin schemas +from .admin import ( + RefreshRemotesResponse, +) + +# Build schemas +from .builds import ( + RemoteInfo, + BuildProgress, + BuildRequest, + BuildSubmitResponse, + BuildOut, +) + +# Vehicle schemas +from .vehicles import ( + VehicleBase, + VersionBase, + VersionOut, + BoardBase, + BoardOut, + CategoryBase, + FeatureDefault, + FeatureBase, + FeatureOut, +) + +__all__ = [ + # Admin + "RefreshRemotesResponse", + # Builds + "RemoteInfo", + "BuildProgress", + "BuildRequest", + "BuildSubmitResponse", + "BuildOut", + # Vehicles + "VehicleBase", + "VersionBase", + "VersionOut", + "BoardBase", + "BoardOut", + "CategoryBase", + "FeatureDefault", + "FeatureBase", + "FeatureOut", +] diff --git a/web/schemas/admin.py b/web/schemas/admin.py new file mode 100644 index 0000000..ab8068e --- /dev/null +++ b/web/schemas/admin.py @@ -0,0 +1,12 @@ +from typing import List + +from pydantic import BaseModel, Field + + +# --- Refresh Remotes Response --- +class RefreshRemotesResponse(BaseModel): + """Response schema for remote refresh operation.""" + remotes: List[str] = Field( + ..., + description="List of remotes discovered in remotes.json file" + ) diff --git a/web/schemas/builds.py b/web/schemas/builds.py new file mode 100644 index 0000000..57b59bb --- /dev/null +++ b/web/schemas/builds.py @@ -0,0 +1,65 @@ +from typing import List, Literal + +from pydantic import BaseModel, Field +from schemas.vehicles import VehicleBase, BoardBase, RemoteInfo + + +# --- Build Progress --- +class BuildProgress(BaseModel): + """Build progress and status information.""" + percent: int = Field( + ..., ge=0, le=100, description="Build completion percentage" + ) + state: Literal[ + "PENDING", "RUNNING", "SUCCESS", "FAILURE", "ERROR" + ] = Field(..., description="Current build state") + + +# --- Build Request --- +class BuildRequest(BaseModel): + """Schema for creating a new build request.""" + vehicle_id: str = Field( + ..., description="Vehicle ID to build for" + ) + board_id: str = Field( + ..., description="Board ID to build for" + ) + version_id: str = Field( + ..., description="Version ID for build source code" + ) + selected_features: List[str] = Field( + default_factory=list, + description="Feature IDs to enable for this build" + ) + + +# --- Build Submit Response --- +class BuildSubmitResponse(BaseModel): + """Response schema for build submission.""" + build_id: str = Field(..., description="Unique build identifier") + url: str = Field(..., description="URL to get build details") + status: Literal["submitted"] = Field( + ..., description="Build submission status" + ) + + +# --- Build Output --- +class BuildOut(BaseModel): + """Complete build information output schema.""" + build_id: str = Field(..., description="Unique build identifier") + vehicle: VehicleBase = Field(..., description="Target vehicle information") + board: BoardBase = Field(..., description="Target board information") + git_hash: str = Field(..., description="Git commit hash used for build") + remote_info: RemoteInfo = Field( + ..., description="Source repository information" + ) + selected_features: List[str] = Field( + default_factory=list, + description="Enabled feature flags for this build" + ) + progress: BuildProgress = Field( + ..., description="Current build status and progress" + ) + time_created: float = Field( + ..., description="Unix timestamp when build was created" + ) diff --git a/web/schemas/vehicles.py b/web/schemas/vehicles.py new file mode 100644 index 0000000..64ac43c --- /dev/null +++ b/web/schemas/vehicles.py @@ -0,0 +1,93 @@ +# app/schemas/vehicles.py +from typing import Literal, Optional + +from pydantic import BaseModel, Field + + +# --- Vehicles --- +class VehicleBase(BaseModel): + id: str = Field(..., description="Unique vehicle identifier") + name: str = Field(..., description="Vehicle display name") + + +# --- Remote Information --- +class RemoteInfo(BaseModel): + """Git remote repository information.""" + name: str = Field(..., description="Remote name (e.g., 'ardupilot')") + url: str = Field(..., description="Git repository URL") + + +# --- Versions --- +class VersionBase(BaseModel): + id: str = Field(..., description="Unique version identifier") + name: str = Field(..., description="Version display name") + type: Literal["beta", "stable", "latest", "tag"] = Field( + ..., description="Version type classification" + ) + remote: RemoteInfo = Field( + ..., description="Git remote repository information for the version" + ) + commit_ref: Optional[str] = Field( + None, description="Git reference (tag, branch name, or commit SHA)" + ) + + +class VersionOut(VersionBase): + vehicle_id: str = Field( + ..., description="Vehicle identifier associated with this version" + ) + + +# --- Boards --- +class BoardBase(BaseModel): + id: str = Field(..., description="Unique board identifier") + name: str = Field(..., description="Board display name") + + +class BoardOut(BoardBase): + vehicle_id: str = Field(..., description="Associated vehicle identifier") + version_id: str = Field(..., description="Associated version identifier") + + +# --- Features --- +class CategoryBase(BaseModel): + id: str = Field(..., description="Unique category identifier") + name: str = Field(..., description="Category display name") + description: Optional[str] = Field( + None, description="Category description" + ) + + +class FeatureDefault(BaseModel): + enabled: bool = Field( + ..., description="Whether feature is enabled by default" + ) + source: Literal["firmware-server", "build-options-py"] = Field( + ..., + description=( + "Source of the default value: 'firmware-server' from " + "firmware.ardupilot.org, 'build-options-py' from git repository" + ) + ) + + +class FeatureBase(BaseModel): + id: str = Field(..., description="Unique feature identifier/flag name") + name: str = Field(..., description="Feature display name") + category: CategoryBase = Field(..., description="Feature category") + description: Optional[str] = Field( + None, description="Feature description" + ) + + +class FeatureOut(FeatureBase): + vehicle_id: str = Field(..., description="Associated vehicle identifier") + version_id: str = Field(..., description="Associated version identifier") + board_id: str = Field(..., description="Associated board identifier") + default: FeatureDefault = Field( + ..., description="Default state for this feature on this board" + ) + dependencies: list[str] = Field( + default_factory=list, + description="List of feature IDs that this feature depends on" + ) diff --git a/web/services/__init__.py b/web/services/__init__.py new file mode 100644 index 0000000..d801d2e --- /dev/null +++ b/web/services/__init__.py @@ -0,0 +1,15 @@ +""" +Business logic services for the application. +""" +from .vehicles import get_vehicles_service, VehiclesService +from .builds import get_builds_service, BuildsService +from .admin import get_admin_service, AdminService + +__all__ = [ + "get_vehicles_service", + "VehiclesService", + "get_builds_service", + "BuildsService", + "get_admin_service", + "AdminService", +] diff --git a/web/services/admin.py b/web/services/admin.py new file mode 100644 index 0000000..83aa79f --- /dev/null +++ b/web/services/admin.py @@ -0,0 +1,115 @@ +""" +Admin service for handling administrative operations. +""" +import logging +from typing import Optional, List + +from fastapi import Request + +from core.config import get_settings + +logger = logging.getLogger(__name__) + + +class AdminService: + """Service for managing administrative operations.""" + + def __init__(self, versions_fetcher=None): + """ + Initialize the admin service. + + Args: + versions_fetcher: VersionsFetcher instance for managing remotes + """ + self.versions_fetcher = versions_fetcher + self.settings = get_settings() + + def get_auth_token(self) -> Optional[str]: + """ + Retrieve the authorization token from file or environment. + + Returns: + The authorization token if found, None otherwise + """ + try: + # Try to read the secret token from the file + token_file_path = self.settings.admin_token_file_path + with open(token_file_path, 'r') as file: + token = file.read().strip() + return token + except (FileNotFoundError, PermissionError) as e: + logger.error( + f"Couldn't open token file at " + f"{self.settings.admin_token_file_path}: {e}. " + "Checking environment for token." + ) + # If the file does not exist or no permission, check environment + return self.settings.admin_token_env + except Exception as e: + logger.error( + f"Unexpected error reading token file at " + f"{self.settings.admin_token_file_path}: {e}. " + "Checking environment for token." + ) + # For any other error, fall back to environment variable + return self.settings.admin_token_env + + async def verify_token(self, token: str) -> bool: + """ + Verify that the provided token matches the expected admin token. + + Args: + token: The token to verify + + Returns: + True if token is valid, False otherwise + + Raises: + RuntimeError: If admin token is not configured on server + """ + expected_token = self.get_auth_token() + + if expected_token is None: + logger.error("No admin token configured") + raise RuntimeError("Admin token not configured on server") + + return token == expected_token + + async def refresh_remotes(self) -> List[str]: + """ + Trigger a refresh of remote metadata. + + Returns: + List of remote names that were refreshed + + Raises: + Exception: If refresh operation fails + """ + logger.info("Triggering remote metadata refresh") + + # Reload remotes.json + self.versions_fetcher.reload_remotes_json() + + # Get list of remotes that are now available + remotes_info = self.versions_fetcher.get_all_remotes_info() + remotes_refreshed = [remote.name for remote in remotes_info] + + logger.info( + f"Successfully refreshed {len(remotes_refreshed)} remotes: " + f"{remotes_refreshed}" + ) + + return remotes_refreshed + + +def get_admin_service(request: Request) -> AdminService: + """ + Get AdminService instance with dependencies from app state. + + Args: + request: FastAPI Request object + + Returns: + AdminService instance initialized with app state dependencies + """ + return AdminService(versions_fetcher=request.app.state.versions_fetcher) diff --git a/web/services/builds.py b/web/services/builds.py new file mode 100644 index 0000000..691577b --- /dev/null +++ b/web/services/builds.py @@ -0,0 +1,397 @@ +""" +Builds service for handling build-related business logic. +""" +import logging +import os +from fastapi import Request +from typing import List, Optional + +from schemas import ( + BuildRequest, + BuildSubmitResponse, + BuildOut, + BuildProgress, + RemoteInfo, +) +from schemas.vehicles import VehicleBase, BoardBase + +# Import external modules +# pylint: disable=wrong-import-position +import build_manager # noqa: E402 + +logger = logging.getLogger(__name__) + + +class BuildsService: + """Service for managing firmware builds.""" + + def __init__( + self, + build_manager=None, + versions_fetcher=None, + ap_src_metadata_fetcher=None, + repo=None, + vehicles_manager=None + ): + self.manager = build_manager + self.versions_fetcher = versions_fetcher + self.ap_src_metadata_fetcher = ap_src_metadata_fetcher + self.repo = repo + self.vehicles_manager = vehicles_manager + + def create_build( + self, + build_request: BuildRequest, + client_ip: str + ) -> BuildSubmitResponse: + """ + Create a new build request. + + Args: + build_request: Build configuration + client_ip: Client IP address for rate limiting + + Returns: + Simple response with build_id and URL + + Raises: + ValueError: If validation fails + """ + # Validate version_id + if not build_request.version_id: + raise ValueError("version_id is required") + + # Validate vehicle + vehicle_id = build_request.vehicle_id + if not vehicle_id: + raise ValueError("vehicle_id is required") + + # Get version info using version_id + version_info = self.versions_fetcher.get_version_info( + vehicle_id=vehicle_id, + version_id=build_request.version_id + ) + if version_info is None: + raise ValueError("Invalid version_id for vehicle") + + remote_name = version_info.remote_info.name + commit_ref = version_info.commit_ref + + # Validate remote + remote_info = self.versions_fetcher.get_remote_info(remote_name) + if remote_info is None: + raise ValueError(f"Remote {remote_name} is not whitelisted") + + # Validate board + board_name = build_request.board_id + if not board_name: + raise ValueError("board_id is required") + + # Check board exists at this version + with self.repo.get_checkout_lock(): + boards_at_commit = self.ap_src_metadata_fetcher.get_boards( + remote=remote_name, + commit_ref=commit_ref, + vehicle_id=vehicle_id, + ) + + if board_name not in boards_at_commit: + raise ValueError("Invalid board for this version") + + # Get git hash + git_hash = self.repo.commit_id_for_remote_ref( + remote=remote_name, + commit_ref=commit_ref + ) + + # Map feature labels (IDs from API) to defines + # (required by build manager) + selected_feature_defines = set() + if build_request.selected_features: + # Get build options to map labels to defines + with self.repo.get_checkout_lock(): + options = ( + self.ap_src_metadata_fetcher + .get_build_options_at_commit( + remote=remote_name, + commit_ref=commit_ref + ) + ) + + # Create label to define mapping + label_to_define = { + option.label: option.define for option in options + } + + # Map each selected feature label to its define + for feature_label in build_request.selected_features: + if feature_label in label_to_define: + selected_feature_defines.add( + label_to_define[feature_label] + ) + else: + logger.warning( + f"Feature label '{feature_label}' not found in " + f"build options for {vehicle_id} {remote_name} " + f"{commit_ref}" + ) + + # Create build info + build_info = build_manager.BuildInfo( + vehicle_id=vehicle_id, + remote_info=remote_info, + git_hash=git_hash, + board=board_name, + selected_features=selected_feature_defines + ) + + # Submit build + build_id = self.manager.submit_build( + build_info=build_info, + client_ip=client_ip, + ) + + # Return simple submission response + return BuildSubmitResponse( + build_id=build_id, + url=f"/api/v1/builds/{build_id}", + status="submitted" + ) + + def list_builds( + self, + vehicle_id: Optional[str] = None, + board_id: Optional[str] = None, + state: Optional[str] = None, + limit: int = 20, + offset: int = 0 + ) -> List[BuildOut]: + """ + Get list of builds with optional filters. + + Args: + vehicle_id: Filter by vehicle + board_id: Filter by board + state: Filter by build state + limit: Maximum results + offset: Results to skip + + Returns: + List of builds + """ + all_build_ids = self.manager.get_all_build_ids() + all_builds = [] + + for build_id in all_build_ids: + build_info = self.manager.get_build_info(build_id) + if build_info is None: + continue + + # Apply filters + if (vehicle_id and + build_info.vehicle_id.lower() != vehicle_id.lower()): + continue + if board_id and build_info.board != board_id: + continue + if state and build_info.progress.state.name != state: + continue + + all_builds.append( + self._build_info_to_output(build_id, build_info) + ) + + # Sort by creation time (newest first) + all_builds.sort(key=lambda x: x.time_created, reverse=True) + + # Apply pagination + return all_builds[offset:offset + limit] + + def get_build(self, build_id: str) -> Optional[BuildOut]: + """ + Get details of a specific build. + + Args: + build_id: The unique build identifier + + Returns: + Build details or None if not found + """ + if not self.manager.build_exists(build_id): + return None + + build_info = self.manager.get_build_info(build_id) + if build_info is None: + return None + + return self._build_info_to_output(build_id, build_info) + + def get_build_logs( + self, + build_id: str, + tail: Optional[int] = None + ) -> Optional[str]: + """ + Get build logs for a specific build. + + Args: + build_id: The unique build identifier + tail: Optional number of last lines to return + + Returns: + Build logs as text or None if not found/available + """ + if not self.manager.build_exists(build_id): + return None + + log_path = self.manager.get_build_log_path(build_id) + if not os.path.exists(log_path): + return None + + try: + with open(log_path, 'r') as f: + if tail: + # Read last N lines + lines = f.readlines() + return ''.join(lines[-tail:]) + else: + return f.read() + except Exception as e: + logger.error(f"Error reading log file for build {build_id}: {e}") + return None + + def get_artifact_path(self, build_id: str) -> Optional[str]: + """ + Get the path to the build artifact. + + Args: + build_id: The unique build identifier + + Returns: + Path to artifact or None if not available + """ + if not self.manager.build_exists(build_id): + return None + + build_info = self.manager.get_build_info(build_id) + if build_info is None: + return None + + # Only return artifact if build was successful + if build_info.progress.state.name != "SUCCESS": + return None + + artifact_path = self.manager.get_build_archive_path(build_id) + if os.path.exists(artifact_path): + return artifact_path + + return None + + def _build_info_to_output( + self, + build_id: str, + build_info + ) -> BuildOut: + """ + Convert BuildInfo object to BuildOut schema. + + Args: + build_id: The build identifier + build_info: BuildInfo object from build_manager + + Returns: + BuildOut schema object + """ + # Convert build_manager.BuildProgress to schema BuildProgress + progress = BuildProgress( + percent=build_info.progress.percent, + state=build_info.progress.state.name + ) + + # Convert RemoteInfo + remote_info = RemoteInfo( + name=build_info.remote_info.name, + url=build_info.remote_info.url + ) + + # Map feature defines back to labels for API response + selected_feature_labels = [] + if build_info.selected_features: + try: + # Get build options to map defines back to labels + with self.repo.get_checkout_lock(): + options = ( + self.ap_src_metadata_fetcher + .get_build_options_at_commit( + remote=build_info.remote_info.name, + commit_ref=build_info.git_hash + ) + ) + + # Create define to label mapping + define_to_label = { + option.define: option.label for option in options + } + + # Map each selected feature define to its label + for feature_define in build_info.selected_features: + if feature_define in define_to_label: + selected_feature_labels.append( + define_to_label[feature_define] + ) + else: + # Fallback: use define if label not found + logger.warning( + f"Feature define '{feature_define}' not " + f"found in build options for build " + f"{build_id}" + ) + selected_feature_labels.append(feature_define) + except Exception as e: + logger.error( + f"Error mapping feature defines to labels for " + f"build {build_id}: {e}" + ) + # Fallback: use defines as-is + selected_feature_labels = list( + build_info.selected_features + ) + + vehicle = self.vehicles_manager.get_vehicle_by_id( + build_info.vehicle_id + ) + + return BuildOut( + build_id=build_id, + vehicle=VehicleBase( + id=build_info.vehicle_id, + name=vehicle.name + ), + board=BoardBase( + id=build_info.board, + name=build_info.board # Board name is same as board ID for now + ), + git_hash=build_info.git_hash, + remote_info=remote_info, + selected_features=selected_feature_labels, + progress=progress, + time_created=build_info.time_created, + ) + + +def get_builds_service(request: Request) -> BuildsService: + """ + Get BuildsService instance with dependencies from app state. + + Args: + request: FastAPI Request object + + Returns: + BuildsService instance initialized with app state dependencies + """ + return BuildsService( + build_manager=request.app.state.build_manager, + versions_fetcher=request.app.state.versions_fetcher, + ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher, + repo=request.app.state.repo, + vehicles_manager=request.app.state.vehicles_manager, + ) diff --git a/web/services/vehicles.py b/web/services/vehicles.py new file mode 100644 index 0000000..9332293 --- /dev/null +++ b/web/services/vehicles.py @@ -0,0 +1,273 @@ +""" +Vehicles service for handling vehicle-related business logic. +""" +import logging +from typing import List, Optional +from fastapi import Request + +from schemas import ( + VehicleBase, + RemoteInfo, + VersionOut, + BoardOut, + FeatureOut, + CategoryBase, + FeatureDefault, +) + + +logger = logging.getLogger(__name__) + + +class VehiclesService: + """Service for managing vehicles, versions, boards, and features.""" + + def __init__(self, vehicle_manager=None, + versions_fetcher=None, + ap_src_metadata_fetcher=None, + repo=None): + self.vehicles_manager = vehicle_manager + self.versions_fetcher = versions_fetcher + self.ap_src_metadata_fetcher = ap_src_metadata_fetcher + self.repo = repo + + def get_all_vehicles(self) -> List[VehicleBase]: + """Get list of all available vehicles.""" + logger.info('Fetching all vehicles') + vehicles = self.vehicles_manager.get_all_vehicles() + # Sort by name for consistent ordering + sorted_vehicles = sorted(vehicles, key=lambda v: v.name) + logger.info(f'Found vehicles: {[v.name for v in sorted_vehicles]}') + return [ + VehicleBase(id=vehicle.id, name=vehicle.name) + for vehicle in sorted_vehicles + ] + + def get_vehicle(self, vehicle_id: str) -> Optional[VehicleBase]: + """Get a specific vehicle by ID.""" + vehicle = self.vehicles_manager.get_vehicle_by_id(vehicle_id) + if vehicle: + return VehicleBase(id=vehicle.id, name=vehicle.name) + return None + + def get_versions( + self, + vehicle_id: str, + type_filter: Optional[str] = None + ) -> List[VersionOut]: + """Get all versions available for a specific vehicle.""" + versions = [] + + for version_info in self.versions_fetcher.get_versions_for_vehicle( + vehicle_id=vehicle_id + ): + # Apply type filter if provided + if type_filter and version_info.release_type != type_filter: + continue + + if version_info.release_type == "latest": + title = f"Latest ({version_info.remote_info.name})" + else: + rel_type = version_info.release_type + ver_num = version_info.version_number + remote = version_info.remote_info.name + title = f"{rel_type} {ver_num} ({remote})" + + versions.append(VersionOut( + id=version_info.version_id, + name=title, + type=version_info.release_type, + remote=RemoteInfo( + name=version_info.remote_info.name, + url=version_info.remote_info.url, + ), + commit_ref=version_info.commit_ref, + vehicle_id=vehicle_id, + )) + + # Sort by name + return sorted(versions, key=lambda x: x.name) + + def get_version( + self, + vehicle_id: str, + version_id: str + ) -> Optional[VersionOut]: + """Get details of a specific version for a vehicle.""" + versions = self.get_versions(vehicle_id) + for version in versions: + if version.id == version_id: + return version + return None + + def get_boards( + self, + vehicle_id: str, + version_id: str + ) -> List[BoardOut]: + """Get all boards available for a specific vehicle version.""" + # Get version info + version_info = self.versions_fetcher.get_version_info( + vehicle_id=vehicle_id, + version_id=version_id + ) + if not version_info: + return [] + + logger.info( + f'Board list requested for {vehicle_id} ' + f'{version_info.remote_info.name} {version_info.commit_ref}' + ) + + # Get boards list + with self.repo.get_checkout_lock(): + boards = self.ap_src_metadata_fetcher.get_boards( + remote=version_info.remote_info.name, + commit_ref=version_info.commit_ref, + vehicle_id=vehicle_id, + ) + + return [ + BoardOut( + id=board, + name=board, + vehicle_id=vehicle_id, + version_id=version_id + ) + for board in boards + ] + + def get_board( + self, + vehicle_id: str, + version_id: str, + board_id: str + ) -> Optional[BoardOut]: + """Get details of a specific board for a vehicle version.""" + boards = self.get_boards(vehicle_id, version_id) + for board in boards: + if board.id == board_id: + return board + return None + + def get_features( + self, + vehicle_id: str, + version_id: str, + board_id: str, + category_id: Optional[str] = None + ) -> List[FeatureOut]: + """ + Get all features with defaults for a specific + vehicle version/board. + """ + # Get version info + version_info = self.versions_fetcher.get_version_info( + vehicle_id=vehicle_id, + version_id=version_id + ) + if not version_info: + return [] + + logger.info( + f'Features requested for {vehicle_id} ' + f'{version_info.remote_info.name} {version_info.commit_ref}' + ) + + # Get build options from source + with self.repo.get_checkout_lock(): + options = self.ap_src_metadata_fetcher.get_build_options_at_commit( + remote=version_info.remote_info.name, + commit_ref=version_info.commit_ref + ) + + # Try to fetch board-specific defaults from firmware-server + board_defaults = None + artifacts_dir = version_info.ap_build_artifacts_url + if artifacts_dir is not None: + board_defaults = ( + self.ap_src_metadata_fetcher.get_board_defaults_from_fw_server( + artifacts_url=artifacts_dir, + board_id=board_id, + vehicle_id=vehicle_id, + ) + ) + + # Build feature list + features = [] + for option in options: + # Apply category filter if provided + if category_id and option.category != category_id: + continue + + # Determine default state and source + if board_defaults and option.define in board_defaults: + # Override with firmware server data + default_enabled = (board_defaults[option.define] != 0) + default_source = 'firmware-server' + else: + # Use build-options-py fallback + default_enabled = (option.default != 0) + default_source = 'build-options-py' + + # Parse dependencies (comma-separated labels) + dependencies = [] + if option.dependency: + dependencies = [ + label.strip() + for label in option.dependency.split(',') + ] + + features.append(FeatureOut( + id=option.label, + name=option.label, + category=CategoryBase( + id=option.category, + name=option.category, + description=None + ), + description=option.description, + vehicle_id=vehicle_id, + version_id=version_id, + board_id=board_id, + default=FeatureDefault( + enabled=default_enabled, + source=default_source + ), + dependencies=dependencies + )) + + # Sort by name + return sorted(features, key=lambda x: x.category.name) + + def get_feature( + self, + vehicle_id: str, + version_id: str, + board_id: str, + feature_id: str + ) -> Optional[FeatureOut]: + """Get details of a specific feature for a vehicle version/board.""" + features = self.get_features(vehicle_id, version_id, board_id) + for feature in features: + if feature.id == feature_id: + return feature + return None + + +def get_vehicles_service(request: Request) -> VehiclesService: + """ + Get VehiclesService instance with dependencies from app state. + + Args: + request: FastAPI Request object + + Returns: + VehiclesService instance initialized with app state dependencies + """ + return VehiclesService( + vehicle_manager=request.app.state.vehicles_manager, + versions_fetcher=request.app.state.versions_fetcher, + ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher, + repo=request.app.state.repo, + ) diff --git a/web/static/js/add_build.js b/web/static/js/add_build.js index 22d6b59..b3caba8 100644 --- a/web/static/js/add_build.js +++ b/web/static/js/add_build.js @@ -1,152 +1,156 @@ const Features = (() => { - let features = {}; - let defines_dictionary = {}; - let labels_dictionary = {}; - let category_dictionary = {}; + let features = []; // Flat array of feature objects from API + let features_by_id = {}; // Map feature IDs to feature objects + let categories_grouped = {}; // Features grouped by category name let selected_options = 0; function resetDictionaries() { // clear old dictionaries - defines_dictionary = {}; - labels_dictionary = {}; - category_dictionary = {}; - - features.forEach((category) => { - category_dictionary[category.name] = category; - category['options'].forEach((option) => { - defines_dictionary[option.define] = labels_dictionary[option.label] = option; - }); - }); - } - - function store_category_in_options() { - features.forEach((category) => { - category['options'].forEach((option) => { - option.category_name = category.name; - }); + features_by_id = {}; + categories_grouped = {}; + + // Build lookup maps from flat feature array + features.forEach((feature) => { + features_by_id[feature.id] = feature; + + // Group by category + const cat_name = feature.category.name; + if (!categories_grouped[cat_name]) { + categories_grouped[cat_name] = { + name: cat_name, + description: feature.category.description, + features: [] + }; + } + categories_grouped[cat_name].features.push(feature); }); } function updateRequiredFor() { - features.forEach((category) => { - category['options'].forEach((option) => { - if (option.dependency != null) { - option.dependency.split(',').forEach((dependency) => { - let dep = getOptionByLabel(dependency); - if (dep.requiredFor == undefined) { - dep.requiredFor = []; - } - dep.requiredFor.push(option.label); - }); - } - }); + features.forEach((feature) => { + if (feature.dependencies && feature.dependencies.length > 0) { + feature.dependencies.forEach((dependency_id) => { + let dep = getOptionById(dependency_id); + if (dep && dep.requiredFor == undefined) { + dep.requiredFor = []; + } + if (dep) { + dep.requiredFor.push(feature.id); + } + }); + } }); } function reset(new_features) { features = new_features; + selected_options = 0; resetDictionaries(); updateRequiredFor(); - store_category_in_options(); } - function getOptionByDefine(define) { - return defines_dictionary[define]; + function getOptionById(id) { + return features_by_id[id]; } - function getOptionByLabel(label) { - return labels_dictionary[label]; + function getCategoryByName(category_name) { + return categories_grouped[category_name]; } - function getCategoryByName(category_name) { - return category_dictionary[category_name]; + function getAllCategories() { + return Object.values(categories_grouped); } function getCategoryIdByName(category_name) { return 'category_'+category_name.split(" ").join("_"); } - function featureIsDisabledByDefault(feature_label) { - return getOptionByLabel(feature_label).default == 0; + function featureIsDisabledByDefault(feature_id) { + let feature = getOptionById(feature_id); + return feature && !feature.default.enabled; } - function featureisEnabledByDefault(feature_label) { - return !featureIsDisabledByDefault(feature_label); + function featureisEnabledByDefault(feature_id) { + return !featureIsDisabledByDefault(feature_id); } - function updateDefaults(defines_array) { - // updates default on the basis of define array passed - // the define array consists define in format, EXAMPLE_DEFINE or !EXAMPLE_DEFINE - // we update the defaults in features object by processing those defines - for (let i=0; i { + feature.dependencies.forEach((dependency_id) => { const check = true; - checkUncheckOptionByLabel(child, check); + checkUncheckOptionById(dependency_id, check); }); } - function handleOptionStateChange(feature_label, triggered_by_ui) { - if (document.getElementById(feature_label).checked) { + function handleOptionStateChange(feature_id, triggered_by_ui) { + // feature_id is the feature ID from the API + let element = document.getElementById(feature_id); + if (!element) return; + + let feature = getOptionById(feature_id); + if (!feature) return; + + if (element.checked) { selected_options += 1; - enableDependenciesForFeature(feature_label); + enableDependenciesForFeature(feature.id); } else { selected_options -= 1; if (triggered_by_ui) { - askToDisableDependentsForFeature(feature_label); + askToDisableDependentsForFeature(feature.id); } else { - disabledDependentsForFeature(feature_label); + disabledDependentsForFeature(feature.id); } } - updateCategoryCheckboxState(getOptionByLabel(feature_label).category_name); + updateCategoryCheckboxState(feature.category.name); updateGlobalCheckboxState(); } - function askToDisableDependentsForFeature(feature_label) { - let enabled_dependent_features = getEnabledDependentFeaturesFor(feature_label); + function askToDisableDependentsForFeature(feature_id) { + let enabled_dependent_features = getEnabledDependentFeaturesFor(feature_id); if (enabled_dependent_features.length <= 0) { return; } - document.getElementById('modalBody').innerHTML = "The feature(s) "+enabled_dependent_features.join(", ")+" is/are dependant on "+feature_label+"" + + let feature = getOptionById(feature_id); + let feature_display_name = feature ? feature.name : feature_id; + + // Get display names for dependent features + let dependent_names = enabled_dependent_features.map(dep_id => { + let dep_feature = getOptionById(dep_id); + return dep_feature ? dep_feature.name : dep_id; + }); + + document.getElementById('modalBody').innerHTML = "The feature(s) "+dependent_names.join(", ")+" is/are dependant on "+feature_display_name+"" + " and hence will be disabled too.
Do you want to continue?"; - document.getElementById('modalDisableButton').onclick = () => { disabledDependentsForFeature(feature_label); }; + document.getElementById('modalDisableButton').onclick = () => { disabledDependentsForFeature(feature_id); }; document.getElementById('modalCancelButton').onclick = document.getElementById('modalCloseButton').onclick = () => { - const check = true; - checkUncheckOptionByLabel(feature_label, check); + const check = true; + if (feature) { + checkUncheckOptionById(feature.id, check); + } }; var confirmationModal = bootstrap.Modal.getOrCreateInstance(document.getElementById('dependencyCheckModal')); confirmationModal.show(); } - function disabledDependentsForFeature(feature_label) { - let feature = getOptionByLabel(feature_label); + function disabledDependentsForFeature(feature_id) { + let feature = getOptionById(feature_id); - if (feature.requiredFor == undefined) { + if (!feature || feature.requiredFor == undefined) { return; } let dependents = feature.requiredFor; - dependents.forEach((dependent) => { + dependents.forEach((dependent_id) => { const check = false; - checkUncheckOptionByLabel(dependent, false); + checkUncheckOptionById(dependent_id, check); }); } @@ -155,12 +159,14 @@ const Features = (() => { if (category == undefined) { console.log("Could not find category by given name"); + return; } let checked_options_count = 0; - category.options.forEach((option) => { - let element = document.getElementById(option.label); + category.features.forEach((feature) => { + // Use ID to find the element + let element = document.getElementById(feature.id); if (element && element.checked) { checked_options_count += 1; @@ -170,6 +176,7 @@ const Features = (() => { let category_checkbox_element = document.getElementById(getCategoryIdByName(category_name)); if (category_checkbox_element == undefined) { console.log("Could not find element for given category"); + return; } let indeterminate_state = false; @@ -177,7 +184,7 @@ const Features = (() => { case 0: category_checkbox_element.checked = false; break; - case category.options.length: + case category.features.length: category_checkbox_element.checked = true; break; default: @@ -189,7 +196,7 @@ const Features = (() => { } function updateGlobalCheckboxState() { - const total_options = Object.keys(defines_dictionary).length; + const total_options = Object.keys(features_by_id).length; let global_checkbox = document.getElementById("check-uncheck-all"); let indeterminate_state = false; @@ -208,31 +215,40 @@ const Features = (() => { global_checkbox.indeterminate = indeterminate_state; } - function getEnabledDependentFeaturesHelper(feature_label, visited, dependent_features) { - if (visited[feature_label] != undefined || document.getElementById(feature_label).checked == false) { + function getEnabledDependentFeaturesHelper(feature_id, visited, dependent_features) { + if (visited[feature_id] != undefined) { + return; + } + + let feature = getOptionById(feature_id); + if (!feature) return; + + // Use ID to check the checkbox + let element = document.getElementById(feature.id); + if (!element || element.checked == false) { return; } - visited[feature_label] = true; - dependent_features.push(feature_label); + visited[feature_id] = true; + dependent_features.push(feature_id); - let feature = getOptionByLabel(feature_label); if (feature.requiredFor == null) { return; } - feature.requiredFor.forEach((dependent_feature) => { - getEnabledDependentFeaturesHelper(dependent_feature, visited, dependent_features); + feature.requiredFor.forEach((dependent_feature_id) => { + getEnabledDependentFeaturesHelper(dependent_feature_id, visited, dependent_features); }); } - function getEnabledDependentFeaturesFor(feature_label) { + function getEnabledDependentFeaturesFor(feature_id) { let dependent_features = []; let visited = {}; - if (getOptionByLabel(feature_label).requiredFor) { - getOptionByLabel(feature_label).requiredFor.forEach((dependent_feature) => { - getEnabledDependentFeaturesHelper(dependent_feature, visited, dependent_features); + let feature = getOptionById(feature_id); + if (feature && feature.requiredFor) { + feature.requiredFor.forEach((dependent_feature_id) => { + getEnabledDependentFeaturesHelper(dependent_feature_id, visited, dependent_features); }); } @@ -240,43 +256,43 @@ const Features = (() => { } function applyDefaults() { - features.forEach(category => { - category['options'].forEach(option => { - const check = featureisEnabledByDefault(option.label); - checkUncheckOptionByLabel(option.label, check); - }); + features.forEach(feature => { + const check = featureisEnabledByDefault(feature.id); + checkUncheckOptionById(feature.id, check); }); } - function checkUncheckOptionByLabel(label, check) { - let element = document.getElementById(label); + function checkUncheckOptionById(id, check) { + let feature = getOptionById(id); + if (!feature) return; + + // Use ID to find the element + let element = document.getElementById(feature.id); if (element == undefined || element.checked == check) { return; } element.checked = check; const triggered_by_ui = false; - handleOptionStateChange(label, triggered_by_ui); + handleOptionStateChange(feature.id, triggered_by_ui); } function checkUncheckAll(check) { - features.forEach(category => { + getAllCategories().forEach(category => { checkUncheckCategory(category.name, check); }); } function checkUncheckCategory(category_name, check) { - getCategoryByName(category_name).options.forEach(option => { - checkUncheckOptionByLabel(option.label, check); + getCategoryByName(category_name).features.forEach(feature => { + checkUncheckOptionById(feature.id, check); }); } - return {reset, handleOptionStateChange, getCategoryIdByName, updateDefaults, applyDefaults, checkUncheckAll, checkUncheckCategory}; + return {reset, handleOptionStateChange, getCategoryIdByName, applyDefaults, checkUncheckAll, checkUncheckCategory, getOptionById}; })(); var init_categories_expanded = false; -var pending_update_calls = 0; // to keep track of unresolved Promises - function init() { fetchVehicles(); } @@ -309,9 +325,8 @@ function fetchVehicles() { // following elemets will be blocked (disabled) when we make the request let elements_to_block = ['vehicle', 'version', 'board', 'submit', 'reset_def', 'exp_col_button']; enableDisableElementsById(elements_to_block, false); - let request_url = '/get_vehicles'; + let request_url = '/api/v1/vehicles'; setSpinnerToDiv('vehicle_list', 'Fetching vehicles...'); - pending_update_calls += 1; sendAjaxRequestForJsonResponse(request_url) .then((json_response) => { let all_vehicles = json_response; @@ -323,8 +338,6 @@ function fetchVehicles() { }) .finally(() => { enableDisableElementsById(elements_to_block, true); - pending_update_calls -= 1; - fetchAndUpdateDefaults(); }); } @@ -341,9 +354,8 @@ function onVehicleChange(new_vehicle_id) { // following elemets will be blocked (disabled) when we make the request let elements_to_block = ['vehicle', 'version', 'board', 'submit', 'reset_def', 'exp_col_button']; enableDisableElementsById(elements_to_block, false); - let request_url = '/get_versions/'+new_vehicle_id; + let request_url = '/api/v1/vehicles/'+new_vehicle_id+'/versions'; setSpinnerToDiv('version_list', 'Fetching versions...'); - pending_update_calls += 1; sendAjaxRequestForJsonResponse(request_url) .then((json_response) => { let all_versions = json_response; @@ -356,8 +368,6 @@ function onVehicleChange(new_vehicle_id) { }) .finally(() => { enableDisableElementsById(elements_to_block, true); - pending_update_calls -= 1; - fetchAndUpdateDefaults(); }); } @@ -376,40 +386,39 @@ function onVersionChange(new_version) { enableDisableElementsById(elements_to_block, false); let vehicle_id = document.getElementById("vehicle").value; let version_id = new_version; - let request_url = `/boards_and_features/${vehicle_id}/${version_id}`; - - // create a temporary container to set spinner inside it + + // Fetch boards first + let boards_url = `/api/v1/vehicles/${vehicle_id}/versions/${version_id}/boards`; + setSpinnerToDiv('board_list', 'Fetching boards...'); + + // Clear build options and show loading state let temp_container = document.createElement('div'); temp_container.id = "temp_container"; temp_container.setAttribute('class', 'container-fluid w-25 mt-3'); - let features_list_element = document.getElementById('build_options'); // append the temp container to the main features_list container + let features_list_element = document.getElementById('build_options'); features_list_element.innerHTML = ""; features_list_element.appendChild(temp_container); setSpinnerToDiv('temp_container', 'Fetching features...'); - setSpinnerToDiv('board_list', 'Fetching boards...'); - pending_update_calls += 1; - sendAjaxRequestForJsonResponse(request_url) - .then((json_response) => { - let boards = json_response.boards; - let new_board = json_response.default_board; - let new_features = json_response.features; - Features.reset(new_features); + + // Fetch boards + sendAjaxRequestForJsonResponse(boards_url) + .then((boards_response) => { + // Keep full board objects with id and name + let boards = boards_response; + let new_board = boards.length > 0 ? boards[0].id : null; updateBoards(boards, new_board); - fillBuildOptions(new_features); }) .catch((message) => { - console.log("Boards and features update failed. "+message); + console.log("Boards update failed. "+message); }) .finally(() => { enableDisableElementsById(elements_to_block, true); - pending_update_calls -= 1; - fetchAndUpdateDefaults(); }); } function updateBoards(all_boards, new_board) { let board_element = document.getElementById('board'); - let old_board = board_element ? board.value : ''; + let old_board = board_element ? board_element.value : ''; fillBoards(all_boards, new_board); if (old_board != new_board) { onBoardChange(new_board); @@ -417,48 +426,40 @@ function updateBoards(all_boards, new_board) { } function onBoardChange(new_board) { - fetchAndUpdateDefaults(); -} - -function fetchAndUpdateDefaults() { - // return early if there is an unresolved promise (i.e., there is an ongoing ajax call) - if (pending_update_calls > 0) { - return; - } - elements_to_block = ['reset_def']; - document.getElementById('reset_def').innerHTML = 'Fetching defaults'; - enableDisableElementsById(elements_to_block, false); + // When board changes, fetch features for the new board + let vehicle_id = document.getElementById('vehicle').value; let version_id = document.getElementById('version').value; - let vehicle = document.getElementById('vehicle').value; - let board = document.getElementById('board').value; - - let request_url = '/get_defaults/'+vehicle+'/'+version_id+'/'+board; - sendAjaxRequestForJsonResponse(request_url) - .then((json_response) => { - Features.updateDefaults(json_response); + + let temp_container = document.createElement('div'); + temp_container.id = "temp_container"; + temp_container.setAttribute('class', 'container-fluid w-25 mt-3'); + let features_list_element = document.getElementById('build_options'); + features_list_element.innerHTML = ""; + features_list_element.appendChild(temp_container); + setSpinnerToDiv('temp_container', 'Fetching features...'); + + let features_url = `/api/v1/vehicles/${vehicle_id}/versions/${version_id}/boards/${new_board}/features`; + sendAjaxRequestForJsonResponse(features_url) + .then((features_response) => { + Features.reset(features_response); + fillBuildOptions(features_response); + Features.applyDefaults(); }) .catch((message) => { - console.log("Default reset failed. "+message); - }) - .finally(() => { - if (document.getElementById('auto_apply_def').checked) { - Features.applyDefaults(); - } - enableDisableElementsById(elements_to_block, true); - document.getElementById('reset_def').innerHTML = 'Reset feature defaults'; + console.log("Features update failed. "+message); }); } -function fillBoards(boards, default_board) { +function fillBoards(boards, default_board_id) { let output = document.getElementById('board_list'); output.innerHTML = '' + ''; let boardList = document.getElementById("board") boards.forEach(board => { let opt = document.createElement('option'); - opt.value = board; - opt.innerHTML = board; - opt.selected = (board === default_board); + opt.value = board.id; + opt.innerHTML = board.name; + opt.selected = (board.id === default_board_id); boardList.appendChild(opt); }); } @@ -487,13 +488,13 @@ var toggle_all_categories = (() => { return toggle_method; })(); -function createCategoryCard(category_name, options, expanded) { +function createCategoryCard(category_name, features_in_category, expanded) { options_html = ""; - options.forEach(option => { + features_in_category.forEach(feature => { options_html += '
' + - '' + - '
'; }); @@ -534,7 +535,7 @@ function createCategoryCard(category_name, options, expanded) { return card_element; } -function fillBuildOptions(buildOptions) { +function fillBuildOptions(features) { let output = document.getElementById('build_options'); output.innerHTML = `
@@ -543,7 +544,20 @@ function fillBuildOptions(buildOptions) {
`; - buildOptions.forEach((category, cat_idx) => { + // Group features by category + let categories_map = {}; + features.forEach(feature => { + const cat_name = feature.category.name; + if (!categories_map[cat_name]) { + categories_map[cat_name] = []; + } + categories_map[cat_name].push(feature); + }); + + // Convert to array and display + let categories = Object.entries(categories_map).map(([name, feats]) => ({name, features: feats})); + + categories.forEach((category, cat_idx) => { if (cat_idx % 4 == 0) { let new_row = document.createElement('div'); new_row.setAttribute('class', 'row'); @@ -552,7 +566,7 @@ function fillBuildOptions(buildOptions) { } let col_element = document.createElement('div'); col_element.setAttribute('class', 'col-md-3 col-sm-6 mb-2'); - col_element.appendChild(createCategoryCard(category['name'], category['options'], init_categories_expanded)); + col_element.appendChild(createCategoryCard(category.name, category.features, init_categories_expanded)); document.getElementById('category_'+parseInt(cat_idx/4)+'_row').appendChild(col_element); }); } @@ -617,27 +631,21 @@ function sortVersions(versions) { } versions.sort((a, b) => { - const version_a_type = a.title.split(" ")[0].toLowerCase(); - const version_b_type = b.title.split(" ")[0].toLowerCase(); - // sort the version types in order mentioned above - if (version_a_type != version_b_type) { - return order[version_a_type] - order[version_b_type]; + if (a.type != b.type) { + return order[a.type] - order[b.type]; } // for numbered versions, do reverse sorting to make sure recent versions come first - if (version_a_type == "stable" || version_b_type == "beta") { - const version_a_num = a.title.split(" ")[1]; - const version_b_num = b.title.split(" ")[1]; - - return compareVersionNums(version_a_num, version_b_num); + if (a.type == "stable" || b.type == "beta") { + return compareVersionNums(a.name.split(" ")[1], b.name.split(" ")[1]); } - return a.title.localeCompare(b.title); + return a.name.localeCompare(b.name); }); // Push the first stable version in the list to the top - const firstStableIndex = versions.findIndex(v => v.title.split(" ")[0].toLowerCase() === "stable"); + const firstStableIndex = versions.findIndex(v => v.name.split(" ")[0].toLowerCase() === "stable"); if (firstStableIndex !== -1) { const stableVersion = versions.splice(firstStableIndex, 1)[0]; versions.unshift(stableVersion); @@ -655,8 +663,78 @@ function fillVersions(versions, version_to_select) { versions.forEach(version => { opt = document.createElement('option'); opt.value = version.id; - opt.innerHTML = version.title; + opt.innerHTML = version.name; opt.selected = (version.id === version_to_select); versionList.appendChild(opt); }); } + +// Handle form submission +async function handleFormSubmit(event) { + event.preventDefault(); + + const submitButton = document.getElementById('submit'); + const originalButtonText = submitButton.innerHTML; + + try { + // Disable submit button and show loading state + submitButton.disabled = true; + submitButton.innerHTML = 'Submitting...'; + + // Collect form data + const vehicle_id = document.getElementById('vehicle').value; + const version_id = document.getElementById('version').value; + const board_id = document.getElementById('board').value; + + // Collect selected features - checkboxes now have feature IDs directly + const selected_features = []; + const checkboxes = document.querySelectorAll('.feature-checkbox:checked'); + checkboxes.forEach(checkbox => { + // The checkbox ID is already the feature define (ID) + selected_features.push(checkbox.id); + }); + + // Create build request payload + const buildRequest = { + vehicle_id: vehicle_id, + version_id: version_id, + board_id: board_id, + selected_features: selected_features + }; + + // Send POST request to API + const response = await fetch('/api/v1/builds', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(buildRequest) + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.detail || 'Failed to submit build'); + } + + const result = await response.json(); + + // Redirect to viewlog page + window.location.href = `/viewlog/${result.build_id}`; + + } catch (error) { + console.error('Error submitting build:', error); + alert('Failed to submit build: ' + error.message); + + // Re-enable submit button + submitButton.disabled = false; + submitButton.innerHTML = originalButtonText; + } +} + +// Initialize form submission handler +document.addEventListener('DOMContentLoaded', () => { + const buildForm = document.getElementById('build-form'); + if (buildForm) { + buildForm.addEventListener('submit', handleFormSubmit); + } +}); diff --git a/web/static/js/index.js b/web/static/js/index.js index aaba266..a72126e 100644 --- a/web/static/js/index.js +++ b/web/static/js/index.js @@ -8,7 +8,7 @@ function init() { function refresh_builds() { var xhr = new XMLHttpRequest(); - xhr.open('GET', "/builds"); + xhr.open('GET', "/api/v1/builds"); // disable cache, thanks to: https://stackoverflow.com/questions/22356025/force-cache-control-no-cache-in-chrome-via-xmlhttprequest-on-f5-reload xhr.setRequestHeader("Cache-Control", "no-cache, no-store, max-age=0"); @@ -82,8 +82,8 @@ function updateBuildsTable(builds) { ${build_info['progress']['state']} ${build_age} ${build_info['git_hash'].substring(0,8)} - ${build_info['board']} - ${build_info['vehicle_id']} + ${build_info['board']['name']} + ${build_info['vehicle']['name']} ${features_string.substring(0, 100)}... @@ -98,7 +98,7 @@ function updateBuildsTable(builds) { - @@ -151,7 +151,7 @@ const LogFetch = (() => { } var xhr = new XMLHttpRequest(); - xhr.open('GET', `/builds/${build_id}/artifacts/build.log`); + xhr.open('GET', `/api/v1/builds/${build_id}/logs`); // disable cache, thanks to: https://stackoverflow.com/questions/22356025/force-cache-control-no-cache-in-chrome-via-xmlhttprequest-on-f5-reload xhr.setRequestHeader("Cache-Control", "no-cache, no-store, max-age=0"); @@ -204,7 +204,7 @@ async function tryAutoDownload(buildId) { } try { - const apiUrl = `/builds/${buildId}` + const apiUrl = `/api/v1/builds/${buildId}` const response = await fetch(apiUrl); const data = await response.json(); @@ -212,7 +212,7 @@ async function tryAutoDownload(buildId) { if (previousState === "RUNNING" && currentState === "SUCCESS") { console.log("Build completed successfully. Starting download..."); - document.getElementById(`${buildId}-download-btn`).click(); + window.location.href = `/api/v1/builds/${buildId}/artifact`; } // Stop running if the build is in a terminal state diff --git a/web/templates/add_build.html b/web/templates/add_build.html index 1255975..6c97488 100644 --- a/web/templates/add_build.html +++ b/web/templates/add_build.html @@ -20,7 +20,7 @@ - + @@ -33,7 +33,7 @@
@@ -50,7 +50,7 @@ ADD NEW BUILD
-
+
@@ -91,17 +91,13 @@
-
- - -
-
+
- +
@@ -127,6 +123,6 @@
- + diff --git a/web/templates/error.html b/web/templates/error.html deleted file mode 100644 index a1f5075..0000000 --- a/web/templates/error.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - ArduPilot Custom Firmware Builder - - - -

ArduPilot Custom Firmware Builder

-

Error Occured: {{ex}}

- - diff --git a/web/templates/index.html b/web/templates/index.html index 962bca6..d9436b7 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -20,7 +20,7 @@ - + @@ -34,7 +34,7 @@
- ArduPilot Custom Firmware Builder @@ -125,7 +125,7 @@ - + {% if token != None %}