From f3cb93a75aae2ec6e42c3a54a9a6fa77f0e2f877 Mon Sep 17 00:00:00 2001 From: bogay Date: Fri, 10 Oct 2025 22:48:13 +0800 Subject: [PATCH 1/4] chore: don't log whole task result --- dispatcher/dispatcher.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dispatcher/dispatcher.py b/dispatcher/dispatcher.py index 1be1a89..9459bee 100644 --- a/dispatcher/dispatcher.py +++ b/dispatcher/dispatcher.py @@ -5,7 +5,6 @@ import requests import pathlib import queue -import textwrap import shutil from datetime import datetime @@ -293,11 +292,6 @@ def create_container( finally: self.dec_container() logger().info(f'finish task {submission_id}/{case_no}') - # truncate long stdout/stderr - _res = res.copy() - for k in ('Stdout', 'Stderr'): - _res[k] = textwrap.shorten(_res.get(k, ''), 37, placeholder='...') - logger().debug(f'runner result: {_res}') with self.locks[submission_id]: self.on_case_complete( submission_id=submission_id, From d08dd3ba5d18108b148d6af0251fb6dd1d153b6a Mon Sep 17 00:00:00 2001 From: bogay Date: Sat, 11 Oct 2025 01:26:49 +0800 Subject: [PATCH 2/4] chore(sandbox): extract all results at once --- runner/sandbox.py | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/runner/sandbox.py b/runner/sandbox.py index 05f52ec..f4fbe93 100644 --- a/runner/sandbox.py +++ b/runner/sandbox.py @@ -3,8 +3,8 @@ import tarfile import tempfile from dataclasses import dataclass -from io import BytesIO from typing import Optional +from pathlib import Path import docker @@ -115,21 +115,11 @@ def run(self): raise JudgeError # retrive result try: - result = self.get( - container=container, - path='/result/', - filename='result', - ).split('\n') - stdout = self.get( - container=container, - path='/result/', - filename='stdout', - ) - stderr = self.get( - container=container, - path='/result/', - filename='stderr', + result, stdout, stderr = self.get_result( + container, + ['result', 'stdout', 'stderr'], ) + result = result.split('\n') except Exception as e: self.client.remove_container(container, v=True, force=True) logging.error(e) @@ -147,16 +137,21 @@ def run(self): DockerExitCode=exit_status['StatusCode'], ) - def get(self, container, path, filename): - bits, _ = self.client.get_archive(container, f'{path}{filename}') - tarbits = b''.join(bits) - tar = tarfile.open(fileobj=BytesIO(tarbits)) - with tempfile.TemporaryDirectory() as extract_path: - tar.extract(filename, extract_path) - with open( - f'{extract_path}/{filename}', + def get_result(self, container, filenames: list[str]) -> list[str]: + result_dir = '/result' + bits, _ = self.client.get_archive(container, result_dir) + with (tempfile.NamedTemporaryFile() as + tarball, tempfile.TemporaryDirectory() as extract_path): + for chunk in bits: + tarball.write(chunk) + tarball.flush() + tarball.seek(0) + with tarfile.open(fileobj=tarball) as tar: + tar.extractall(extract_path) + return [ + open( + Path(extract_path) / result_dir.lstrip('/') / filename, 'r', errors='ignore', - ) as f: - contents = f.read() - return contents + ).read() for filename in filenames + ] From bd3b5d657ff3dda74bdbbae9d3afe9668fc4222d Mon Sep 17 00:00:00 2001 From: bogay Date: Sat, 11 Oct 2025 01:27:57 +0800 Subject: [PATCH 3/4] feat(dispatcher): write payload to file to reduce memory usage dumping large json payloads was consuming up to 1GB in some cases. write the payload to file before send it to lower memory consumption. in the future we might need to change the API about reporting submission result. --- dispatcher/dispatcher.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/dispatcher/dispatcher.py b/dispatcher/dispatcher.py index 9459bee..3590b86 100644 --- a/dispatcher/dispatcher.py +++ b/dispatcher/dispatcher.py @@ -6,6 +6,7 @@ import pathlib import queue import shutil +import tempfile from datetime import datetime from runner.submission import SubmissionRunner @@ -372,16 +373,26 @@ def on_submission_complete(self, submission_id: str): assert [*submission_result.keys()] == [*range(len(submission_result))] submission_result = [*submission_result.values()] # post data - submission_data = { - 'tasks': submission_result, - 'token': config.SANDBOX_TOKEN - } - self.release(submission_id) - logger().info(f'send to BE [submission_id={submission_id}]') - resp = requests.put( - f'{config.BACKEND_API}/submission/{submission_id}/complete', - json=submission_data, - ) + with tempfile.NamedTemporaryFile("w") as tmpf: + submission_data = { + 'tasks': submission_result, + 'token': config.SANDBOX_TOKEN + } + # write payload to file + json.dump(submission_data, tmpf) + tmpf.flush() + # release resources + del submission_data + self.release(submission_id) + + logger().info(f'send to BE [submission_id={submission_id}]') + # open in binary mode as requests needs a binary stream + with open(tmpf.name, "rb") as payload: + resp = requests.put( + f'{config.BACKEND_API}/submission/{submission_id}/complete', + data=payload, + headers={'Content-Type': 'application/json'}, + ) logger().debug(f'get BE response: [{resp.status_code}] {resp.text}', ) # clear if resp.ok: From 5cbd9da01c74da7362f41ece1b6511b714f7ea4f Mon Sep 17 00:00:00 2001 From: bogay Date: Mon, 13 Oct 2025 22:04:51 +0800 Subject: [PATCH 4/4] build(docker): pin python version in base image to prevent breaking changes in python 3.14 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b20d507..3377783 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:alpine +FROM python:3.13-alpine WORKDIR /app