Skip to content

Optimise download many #2005

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ObadaS merged 5 commits into from
Sep 22, 2025
+243 −70
Merged

Optimise download many #2005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ec29a37
gestion of faulty submission
nicomy Sep 17, 2025
6154fc7
changed to post instead of get and added failed.txt to bundle
nicomy Sep 18, 2025
27357e2
added limit on number of file to download simultaneously
nicomy Sep 19, 2025
1ada62d
it should work
nicomy Sep 19, 2025
102ec84
cleaned comments
nicomy Sep 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 25 additions & 18 deletions src/apps/api/views/submissions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from profiles.models import Organization, Membership
from tasks.models import Task
from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer
from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer
from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant
from leaderboards.strategies import put_on_leaderboard_by_submission_rule
from leaderboards.models import SubmissionScore, Column, Leaderboard
Expand Down Expand Up @@ -350,29 +350,34 @@ def re_run_many_submissions(self, request):
return Response({})


@action(detail=False, methods=['get'])

@action(detail=False, methods=('POST',))
def download_many(self, request):
"""
Download a ZIP containing several submissions.
"""
pks = request.query_params.get('pks')
if pks:
pks = json.loads(pks) # Convert JSON string to list
else:
return Response({"error": "`pks` query parameter is required"}, status=400)
pks = request.data.get('pks')
if not pks:
return Response({"error": "`pks` field is required"}, status=400)

# pks is already parsed as a list if JSON was sent properly
if not isinstance(pks, list):
return Response({"error": "`pks` must be a list"}, status=400)



# Get submissions
submissions = Submission.objects.filter(pk__in=pks).select_related(
"owner",
"phase",
"data"
)
# .only("id","owner", "data__data_file")
)#.only("id","owner", "data__data_file")

# .prefetch_related("phase__competition__collaborators")
if submissions.count() != len(pks):

if len(list(submissions)) != len(pks):
# if submissions.count() != len(pks):
return Response({"error": "One or more submission IDs are invalid"}, status=404)

# NH : should should create a function for this ?

# NH : should should create a function for this ?
# Check permissions
if not request.user.is_authenticated:
raise PermissionDenied("You must be logged in to download submissions")
Expand All @@ -393,14 +398,16 @@ def download_many(self, request):
"You do not have permission to download one or more of the requested submissions"
)

files = []
files = []

for sub in submissions:
file_path = sub.data.data_file.name.split('/')[-1]
short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}"

url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file']
# url = sub.data.data_file.url
url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file']
# url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file']
files.append({"name": short_name, "url": url})

return Response(files)

@action(detail=True, methods=('GET',))
Expand Down
10 changes: 6 additions & 4 deletions src/static/js/ours/client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,13 @@ CODALAB.api = {
return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`)
},
download_many_submissions: function (pks) {
const params = new URLSearchParams({ pks: JSON.stringify(pks) });
const url = `${URLS.API}submissions/download_many/?${params}`;
return CODALAB.api.request('GET', url)
return CODALAB.api.request(
'POST',
URLS.API + "submissions/download_many/",
{ pks: pks } // body is JSON by convention
);
},

/*---------------------------------------------------------------------
Leaderboards
---------------------------------------------------------------------*/
Expand Down
260 changes: 212 additions & 48 deletions src/static/riot/competitions/detail/submission_manager.tag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,65 +574,229 @@
.modal('show')
CODALAB.events.trigger('submission_clicked')
}


// self.bulk_download = function () {
// const statusBox = document.getElementById('downloadStatus');
// const progressEl = document.getElementById('downloadProgress');
// const textEl = document.getElementById('progressText');

// statusBox.style.display = "flex";
// // statusBox.style.display = "inline";


// progressEl.style.display="flex";
// progressEl.value = 0;
// textEl.textContent = "Preparing download...";

// console.log("Files returned by server:", files);


// CODALAB.api.download_many_submissions(self.checked_submissions)
// .done( async function(files) {
// // .done( function(files) {
// console.log("Files returned by server:", files);

// const zip = new JSZip();
// const total = files.length;
// let completed = 0;
// const failed = [];

// const fetchFiles = files.map(async file => {
// try {
// const response = await fetch(file.url);

// if (!response.ok) {
// throw new Error(`HTTP ${response.status}`);
// }

// const blob = await response.blob();

// zip.file(file.name.replace(/[:/\\]/g, '_'), blob);
// } catch (err) {
// console.error(`Failed to fetch ${file.name}:`, err);
// failed.push(file.name);
// } finally {
// // Update progress regardless of success/failure
// completed++;
// const percent = Math.floor((completed / total) * 100);
// progressEl.value = percent;
// textEl.textContent = `${completed} / ${total} files (${percent}%)`;
// }
// });

// Promise.allSettled(fetchFiles).then(() => {
// // If some files failed, include them as failed.txt inside the zip
// if (failed.length > 0) {
// const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`;
// zip.file("failed.txt", failedContent);
// }


// textEl.textContent = "Generating bundle";
// progressEl.style.display = "none";

// zip.generateAsync({ type: "blob" }).then(blob => {
// const link = document.createElement("a");
// link.href = URL.createObjectURL(blob);
// link.download = "bulk_submissions.zip";
// document.body.appendChild(link);
// link.click();
// document.body.removeChild(link);

// if (failed.length > 0) {
// textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`;
// } else {
// textEl.textContent = "Download ready!";
// }

// setTimeout(() => {
// statusBox.style.display = "none";
// }, 5000);
// });
// });
// })
// .fail(function(err) {
// console.error("Error downloading submissions:", err);
// textEl.textContent = "Error downloading!";
// });
// };

self.bulk_download = function () {
const statusBox = document.getElementById('downloadStatus');
const progressEl = document.getElementById('downloadProgress');
const textEl = document.getElementById('progressText');

statusBox.style.display = "flex";
// statusBox.style.display = "inline";


progressEl.style.display="flex";
progressEl.style.display = "flex";
progressEl.value = 0;
textEl.textContent = "Preparing download...";

CODALAB.api.download_many_submissions(self.checked_submissions)
.done(function(files) {
console.log("Files returned by server:", files);

const zip = new JSZip();
const total = files.length;
let completed = 0;

const fetchFiles = files.map(async file => {
const response = await fetch(file.url);
const blob = await response.blob();

zip.file(file.name.replace(/[:/\\]/g, '_'), blob);

// Update progress
completed++;
const percent = Math.round((completed / total) * 100);
progressEl.value = percent;
// progressBar.progress({ percent: percent });
textEl.textContent = `${completed} / ${total} files (${percent}%)`;
});

Promise.all(fetchFiles).then(() => {
textEl.textContent = "Generating bundle";
progressEl.style.display="none";

zip.generateAsync({ type: 'blob' }).then(blob => {
const link = document.createElement('a');
link.href = URL.createObjectURL(blob);
link.download = 'bulk_submissions.zip';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);

textEl.textContent = "Download ready!";
setTimeout(() => {
statusBox.style.display = "none";
}, 3000);
// Kick the API request
const req = CODALAB.api.download_many_submissions(self.checked_submissions);

// Common error handler
const handleError = (err) => {
console.error("Error downloading submissions:", err);
textEl.textContent = "Error downloading!";
setTimeout(() => { statusBox.style.display = "none"; }, 5000);
};

// Success handler (async because we await inside)
const handleSuccess = async (resp) => {
// Normalize response -> files array
let files = resp;
if (resp && typeof resp === 'object' && !Array.isArray(resp)) {
// common shapes: { files: [...] } or { data: [...] } or direct array
if (Array.isArray(resp.files)) files = resp.files;
else if (Array.isArray(resp.data)) files = resp.data;
else if (Array.isArray(resp.results)) files = resp.results;
else if (Array.isArray(resp)) files = resp;
else {
// if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg
if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0];
else {
console.warn("Unexpected response shape from download_many_submissions:", resp);
files = [];
}
}
}

if (!Array.isArray(files) || files.length === 0) {
textEl.textContent = "No files to download";
setTimeout(() => { statusBox.style.display = "none"; }, 3000);
return;
}

console.log("Files returned by server:", files);

const zip = new JSZip();
const total = files.length;
let completed = 0;
const failed = [];

const limit = 5;
const queue = files.slice(); // clone
const running = [];

// worker to fetch one file
async function worker(file) {
try {
const response = await fetch(file.url);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const blob = await response.blob();
zip.file(file.name.replace(/[:/\\]/g, "_"), blob);
} catch (err) {
console.error(`Failed to fetch ${file.name}:`, err);
failed.push(`${file.name} (${err.message || 'fetch error'})`);
} finally {
completed++;
const percent = Math.floor((completed / total) * 100);
progressEl.value = percent;
textEl.textContent = `${completed} / ${total} files (${percent}%)`;
}
}

// run queue with limited concurrency
while (queue.length > 0) {
while (running.length < limit && queue.length > 0) {
const file = queue.shift();
const p = worker(file).then(() => {
// remove finished promise from running
const idx = running.indexOf(p);
if (idx !== -1) running.splice(idx, 1);
});
});
})
.fail(function(err) {
console.error("Error downloading submissions:", err);
textEl.textContent = "Error downloading!";
});
running.push(p);
}
// Wait for at least one running promise to finish
if (running.length > 0) {
await Promise.race(running);
}
}
await Promise.all(running);

// Add failed.txt if necessary
if (failed.length > 0) {
const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`;
zip.file("failed.txt", failedContent);
}

textEl.textContent = "Generating bundle";
progressEl.style.display = "none";

const blob = await zip.generateAsync({ type: "blob" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "bulk_submissions.zip";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);

if (failed.length > 0) {
textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`;
} else {
textEl.textContent = "Download ready!";
}

setTimeout(() => { statusBox.style.display = "none"; }, 5000);
};

// Support both jQuery jqXHR (.done/.fail) and native Promise (.then/.catch)
if (req && typeof req.done === "function") {
// jQuery-style
req.done(function() {
// jQuery passes (data, textStatus, jqXHR)
// forward only the first argument to our handler
const args = Array.from(arguments);
handleSuccess(args[0]);
}).fail(handleError);
} else if (req && typeof req.then === "function") {
// native Promise
req.then(handleSuccess).catch(handleError);
} else {
console.error("download_many_submissions returned non-promise/non-jqXHR:", req);
handleError(new Error("Invalid request return type"));
}
};

self.submission_handling = function () {
Expand Down