diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 68b01b7ae..edea2f372 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -19,7 +19,7 @@ from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -350,29 +350,34 @@ def re_run_many_submissions(self, request): return Response({}) - @action(detail=False, methods=['get']) + + @action(detail=False, methods=('POST',)) def download_many(self, request): - """ - Download a ZIP containing several submissions. - """ - pks = request.query_params.get('pks') - if pks: - pks = json.loads(pks) # Convert JSON string to list - else: - return Response({"error": "`pks` query parameter is required"}, status=400) + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + # pks is already parsed as a list if JSON was sent properly + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) + + # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - ) - # .only("id","owner", "data__data_file") + )#.only("id","owner", "data__data_file") + # .prefetch_related("phase__competition__collaborators") - if submissions.count() != len(pks): + + if len(list(submissions)) != len(pks): + # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - # NH : should should create a function for this ? + + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -393,14 +398,16 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - files = [] + files = [] + for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" - - url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + # url = sub.data.data_file.url + url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] + # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] files.append({"name": short_name, "url": url}) - + return Response(files) @action(detail=True, methods=('GET',)) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 03746945b..e4ffaa3fc 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,11 +128,13 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - const url = `${URLS.API}submissions/download_many/?${params}`; - return CODALAB.api.request('GET', url) + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many/", + { pks: pks } // body is JSON by convention + ); }, - + /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 79d894bc3..39bcacb7c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -574,6 +574,93 @@ .modal('show') CODALAB.events.trigger('submission_clicked') } + + + // self.bulk_download = function () { + // const statusBox = document.getElementById('downloadStatus'); + // const progressEl = document.getElementById('downloadProgress'); + // const textEl = document.getElementById('progressText'); + + // statusBox.style.display = "flex"; + // // statusBox.style.display = "inline"; + + + // progressEl.style.display="flex"; + // progressEl.value = 0; + // textEl.textContent = "Preparing download..."; + + // console.log("Files returned by server:", files); + + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done( async function(files) { + // // .done( function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + // const failed = []; + + // const fetchFiles = files.map(async file => { + // try { + // const response = await fetch(file.url); + + // if (!response.ok) { + // throw new Error(`HTTP ${response.status}`); + // } + + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // } catch (err) { + // console.error(`Failed to fetch ${file.name}:`, err); + // failed.push(file.name); + // } finally { + // // Update progress regardless of success/failure + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // } + // }); + + // Promise.allSettled(fetchFiles).then(() => { + // // If some files failed, include them as failed.txt inside the zip + // if (failed.length > 0) { + // const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + // zip.file("failed.txt", failedContent); + // } + + + // textEl.textContent = "Generating bundle"; + // progressEl.style.display = "none"; + + // zip.generateAsync({ type: "blob" }).then(blob => { + // const link = document.createElement("a"); + // link.href = URL.createObjectURL(blob); + // link.download = "bulk_submissions.zip"; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // if (failed.length > 0) { + // textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + // } else { + // textEl.textContent = "Download ready!"; + // } + + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 5000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); + // }; self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); @@ -581,58 +668,135 @@ const textEl = document.getElementById('progressText'); statusBox.style.display = "flex"; - // statusBox.style.display = "inline"; - - - progressEl.style.display="flex"; + progressEl.style.display = "flex"; progressEl.value = 0; textEl.textContent = "Preparing download..."; - CODALAB.api.download_many_submissions(self.checked_submissions) - .done(function(files) { - console.log("Files returned by server:", files); - - const zip = new JSZip(); - const total = files.length; - let completed = 0; - - const fetchFiles = files.map(async file => { - const response = await fetch(file.url); - const blob = await response.blob(); - - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - - // Update progress - completed++; - const percent = Math.round((completed / total) * 100); - progressEl.value = percent; - // progressBar.progress({ percent: percent }); - textEl.textContent = `${completed} / ${total} files (${percent}%)`; - }); - - Promise.all(fetchFiles).then(() => { - textEl.textContent = "Generating bundle"; - progressEl.style.display="none"; - - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - textEl.textContent = "Download ready!"; - setTimeout(() => { - statusBox.style.display = "none"; - }, 3000); + // Kick the API request + const req = CODALAB.api.download_many_submissions(self.checked_submissions); + + // Common error handler + const handleError = (err) => { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Success handler (async because we await inside) + const handleSuccess = async (resp) => { + // Normalize response -> files array + let files = resp; + if (resp && typeof resp === 'object' && !Array.isArray(resp)) { + // common shapes: { files: [...] } or { data: [...] } or direct array + if (Array.isArray(resp.files)) files = resp.files; + else if (Array.isArray(resp.data)) files = resp.data; + else if (Array.isArray(resp.results)) files = resp.results; + else if (Array.isArray(resp)) files = resp; + else { + // if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg + if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0]; + else { + console.warn("Unexpected response shape from download_many_submissions:", resp); + files = []; + } + } + } + + if (!Array.isArray(files) || files.length === 0) { + textEl.textContent = "No files to download"; + setTimeout(() => { statusBox.style.display = "none"; }, 3000); + return; + } + + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + const failed = []; + + const limit = 5; + const queue = files.slice(); // clone + const running = []; + + // worker to fetch one file + async function worker(file) { + try { + const response = await fetch(file.url); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const blob = await response.blob(); + zip.file(file.name.replace(/[:/\\]/g, "_"), blob); + } catch (err) { + console.error(`Failed to fetch ${file.name}:`, err); + failed.push(`${file.name} (${err.message || 'fetch error'})`); + } finally { + completed++; + const percent = Math.floor((completed / total) * 100); + progressEl.value = percent; + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + } + } + + // run queue with limited concurrency + while (queue.length > 0) { + while (running.length < limit && queue.length > 0) { + const file = queue.shift(); + const p = worker(file).then(() => { + // remove finished promise from running + const idx = running.indexOf(p); + if (idx !== -1) running.splice(idx, 1); }); - }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - textEl.textContent = "Error downloading!"; - }); + running.push(p); + } + // Wait for at least one running promise to finish + if (running.length > 0) { + await Promise.race(running); + } + } + await Promise.all(running); + + // Add failed.txt if necessary + if (failed.length > 0) { + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); + } + + textEl.textContent = "Generating bundle"; + progressEl.style.display = "none"; + + const blob = await zip.generateAsync({ type: "blob" }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = "bulk_submissions.zip"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + } else { + textEl.textContent = "Download ready!"; + } + + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Support both jQuery jqXHR (.done/.fail) and native Promise (.then/.catch) + if (req && typeof req.done === "function") { + // jQuery-style + req.done(function() { + // jQuery passes (data, textStatus, jqXHR) + // forward only the first argument to our handler + const args = Array.from(arguments); + handleSuccess(args[0]); + }).fail(handleError); + } else if (req && typeof req.then === "function") { + // native Promise + req.then(handleSuccess).catch(handleError); + } else { + console.error("download_many_submissions returned non-promise/non-jqXHR:", req); + handleError(new Error("Invalid request return type")); + } }; self.submission_handling = function () {